\documentclass{article}
\usepackage[legalpaper, portrait, margin=0.99in]{geometry}

% Language setting
% Replace `english' with e.g. `spanish' to change the document language
\usepackage[english]{babel}

% Set page size and margins
% Replace `letterpaper' with`a4paper' for UK/EU standard size
\usepackage[letterpaper,top=2cm,bottom=2cm,left=3cm,right=3cm,marginparwidth=1.75cm]{geometry}
% Useful packages
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage[colorlinks=true, allcolors=blue]{hyperref}
\usepackage{authblk}
\usepackage{amsfonts}
\usepackage{comment}

\title{Vectorized Adjoint Sensitivity Method for Graph Convolutional Neural Ordinary Differential Equations}
\author[1]{Jack Cai}
\affil[1]{University of Toronto, Division of Engineering Science}
\date{August 9, 2021}

\begin{document}
\maketitle

\begin{abstract}
This document, as the title stated, is meant to provide a vectorized implementation of adjoint dynamics calculation for Graph Convolutional Neural Ordinary Differential Equations (GCDE). The adjoint sensitivity method is the gradient approximation method for neural ODEs that replaces the back propagation. When implemented on libraries such as PyTorch or Tensorflow, the adjoint can be calculated by autograd functions without the need for a hand-derived formula. In applications such as edge computing and in memristor crossbars, however, autograds are not available, and therefore we need a vectorized derivation of adjoint dynamics to efficiently map the system on hardware. This document will go over the basics, then move on to derive the vectorized adjoint dynamics for GCDE.
\end{abstract}

\section{Introduction and Preliminaries}

Neural Ordinary Differential Equations (ODE) is a class of residual neural networks that frames the layer-wise propagation of hidden states into an initial value problem using differential equation solvers.
\begin{equation}
h_{t+1} = h_{t} + f(h_{t}, \theta_{t})
\end{equation}
A typical residual network may have layer wise transformation of hidden states looks like Equation (1). If we add more layers and take smaller time step, then the entire problem can be framed as an ODE defined by Equation (2), and a solution at time $t_{1}$ defined by Equation (3).
\begin{equation}
\frac{dh(t)}{dt} = f(h(t), t, \theta)
\end{equation}
\begin{equation}
h(t_{1}) = h(t_{0}) + \int_{t_{0}}^{t_{1}} f(h(t), t, \theta)dt
\end{equation}
Compared to conventional deep neural networks (DNNs), neural ODE uses less parameters applied more times to the hidden states, thereby achieves the depth of DNN while having higher memory efficiency.

\subsection{Adjoint sensitivity method}
The adjoint sensitivity method is used for automatic differentiation for neural ODE. Unlike traditional backpropagation, a quantity named adjoint is calculated for each $t$ and another ODE solver is used to integrate the overall gradient. Let $L()$ be the scaler loss function for the neural ODE, the adjoint is defined as $a(t) = \frac{\partial L}{\partial h(t)}$, and its dynamic is given by Equation (4):
\begin{equation}
\frac{da(t)}{dt} = -a(t)^{T}\frac{\partial f(h(t), t, \theta)}{\partial h}
\end{equation}
The adjoint at each instant is calculated by a backward ODE solver, and the overall gradient of the parameters $\frac{\partial L}{\partial \theta}$ is given by Equation (5), which is integrated by another ODE solver:
\begin{equation}
\frac{\partial L}{\partial \theta} = -\int_{t_{1}}^{t_{0}} a(t)^{T}\frac{\partial f(h(t), t, \theta)}{\partial \theta}dt
\end{equation}
The proof and intuition behind adjoint sensitivity method is presented in the original paper. For the scope of this paper, we are only focused on how Equation (4) and (5) can be vectorized for GCDE.

\subsection{GCDE}
GCDE, in short, is the neural ODE version of the state of the art graph learning method Graph Convolutional Neural Network (GCN). For GCN and GCDE, hidden state is represented by a matrix H of dimension $\mathbb{R}^{N\times C}$, representing $N$ nodes with $C$ features. Each layer of GCN or each step of GCDE involves node-wise exchange of information and convolution on nodes. The dynamic of GCDE (which is the same as the layer-wise propagation of GCN) is given by Equation (6), where $A\in\mathbb{R}^{N\times N}$ is symmetric that represents the graph topology and $W\in\mathbb{R}^{C\times C}$ represents the convolution filters:
\begin{equation}
\frac{dH(t)}{dt} = f(H(t), A, W) = ReLU(AH(t)W)
\end{equation}
To calculate the adjoint, we need to calculate calculate the Jacobian of the function $f(H(t), A, W)$, namely $\frac{\partial f(H(t), A, W)}{\partial H(t)}$ and $\frac{\partial f(H(t), A, W)}{\partial W}$. This poses us a challenge as we will have to unroll the matrix into vectors. While this can be done easily with built-in autograd functions in PyTorch and Tensorflow, it becomes messy when we write it out to obtain an analytical vectorized solution.

\subsection{Matrix conventions}
Before we delve into deriving the adjoint dynamics, I would like to discuss some convention used in this document. All matrices are represented by capital letters, all vectors are represented by bold lower case letter, all entries within a matrix are represented by the $i$,$j$ subscripts which denotes the $i_{th}$ and $j_{th}$ entry of the matrix, and all entries within a vector are represented by a single subscript denoting the index. For example, $A$ is a matrix, $\mathbf{b}$ is a vector, $a_{1, 2}$ denotes the $1, 2$ entry of $A$, and $b_{2}$ denotes the second entry of $\mathbf{b}$. We use $``:"$ to denote the entire indices along a row or column, so that we represent the $i_{th}$ row of $A$ as $\mathbf{a}_{i,:}$, and we represent $j_{th}$ column of A as $\mathbf{a}_{:,j}$.

\subsection{Multivariable calculus conventions}

Let $f: \mathbb{R}^{n} \rightarrow \mathbb{R}^{m}$ where it takes in a vector $\mathbf{x} \in\mathbb{R}^{n}$ and output a vector $\mathbf{y} \in\mathbb{R}^{m}$, i.e.:

\begin{equation}
\begin{bmatrix}
    y_{1}       \\
    y_{2}      \\
    \vdots \\
    y_{m}
\end{bmatrix}
= f(
\begin{bmatrix}
    x_{1} \\
    x_{2} \\
    \vdots \\
    x_{n}
\end{bmatrix}
)
\end{equation}
Then the Jacobian matrix $J_{f} \in \mathbb{R}^{m\times n}$ is defined as:

\begin{equation}
J_{f} = f(
\begin{bmatrix}
    \frac{\partial y_{1}}{\partial x_{1}} & \frac{\partial y_{1}}{\partial x_{2}} &\hdots & \frac{\partial y_{1}}{\partial x_{n}} \\
    \frac{\partial y_{2}}{\partial x_{1}}  & \ddots &\hdots & \vdots \\
    \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial y_{m}}{\partial x_{1}}  & \hdots & \hdots & \frac{\partial y_{m}}{\partial x_{n}}
\end{bmatrix}
)
\end{equation}
Following this convention, let $f: \mathbb{R}^{n} \rightarrow \mathbb{R}^{m}$ and $g: \mathbb{R}^{m} \rightarrow \mathbb{R}^{p}$, then $g \circ f: \mathbb{R}^{n} \rightarrow \mathbb{R}^{p}$. The chain rule is defined as:
\begin{equation}
    J_{g \circ f} = J_{g}\cdot J_{f}
\end{equation}

\subsection{Partial derivatives of a matrix}
Finding the Jacobian for GCDE is challenging not only because it is a composite function, but also because the input is a matrix, we need to unroll the matrix into vectors to match our matrix Jacobian convention. Let $roll()$ and $unroll()$ to be such operations:
\begin{equation}
    \mathbb{R}^{m\times n}\ni A = roll(\mathbf{a}), \mathbf{a}\in\mathbb{R}^{mn}
\end{equation}
\begin{equation}
    \mathbb{R}^{mn}\ni\mathbf{a} = unroll(A),  A\in\mathbb{R}^{m\times n}
\end{equation}
The example below illustrate how they can be used. Suppose $f: \mathbb{R}^{m\times n} \rightarrow \mathbb{R}^{p\times n}$ defined by $f(A) = BA$, where $B \in \mathbb{R}^{p\times m}$. Using Equation (10) and (11), we can convert $f$ into $g: \mathbb{R}^{mn}\rightarrow \mathbb{R}^{pn}$, such that:
\begin{equation}
    g(\mathbf{a}) = unroll(B(roll(\mathbf{a})))
\end{equation}
As a result, we can obtained a Jacobian matrix $J_{g} \in \mathbb{R}^{pn\times mn}$, which follows the convention defined in section 1.4. The way we unroll a matrix could be by rows or by columns -- it really depends on the situation.
\section{Adjoint derivation for GCDE}
The key of finding the Jacobian matrix for $f(H(t), A, W)$ is to find the Jacobian matrix for the three-matrix multiplication step, as the derivative for $ReLU$ is just an elementwise binary step function. So before we moving on solving the Jacobian matrix for the whole thing, let's consider the general case function $f(A) = XAY: \mathbb{R}^{n\times p}  \rightarrow \mathbb{R}^{m\times q}$ where $X\in\mathbb{R}^{m\times n}$, $A\in\mathbb{R}^{n\times p}$, and $Y\in\mathbb{R}^{p\times q}$. We can think of it as composite function $f(A) = h\circ g(A)$, where:
\begin{equation}
    g(A) = XA = B, B\in\mathbb{R}^{m\times p}
\end{equation}
\begin{equation}
h(B) = BY = C, C\in\mathbb{R}^{m\times q}
\end{equation}
According to our convention, we need to unroll the matrix into vectors in order to calculate the Jacobian matrix. Let $\widehat{g}$ and $\widehat{h}$ to denote them and $\mathbf{a} = unroll(A)$ and $\mathbf{b} = unroll(B)$:
\begin{equation}
\widehat{g}(\mathbf{a}) = unroll(X(roll(\mathbf{a})))
\end{equation}
\begin{equation}
\widehat{h}(\mathbf{b}) = unroll((roll(\mathbf{b}))Y)\end{equation}
\begin{center}
    $\widehat{f}(\mathbf{a}) = \widehat{h}\circ\widehat{g}(\mathbf{a})$
\end{center}
Hence, according to chain rule, the Jacobian matrix $J_{\widehat{h}\circ\widehat{g}} = J_{\widehat{h}}\cdot J_{\widehat{g}}$ and we will calculate it step by step below.

\subsection{Jacobian matrix $J_{\widehat{g}}$ for $g(A)=XA$}
We will start by breaking $X$ into rows and $A$ into columns. According to our convention outlined in section 1.3, Equation (13) becomes:

\begin{center}$g(A) = XA = \begin{bmatrix} \mathbf{x}_{1,:} \\ \vdots \\ \mathbf{x}_{1,:} \end{bmatrix} \begin{bmatrix} \mathbf{a}_{:,1} && \hdots && \mathbf{a}_{:,p} \end{bmatrix} \\$
\end{center}

\begin{center}$
= \begin{bmatrix}
    \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1} & \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,2} &\hdots & \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,p} \\
    \mathbf{x}_{2,:} \cdot \mathbf{a}_{:,1}  & \ddots &\hdots & \vdots \\
    \vdots & \vdots & \ddots & \vdots\\
    \mathbf{x}_{m,:} \cdot \mathbf{a}_{:,1}  & \hdots & \hdots & \mathbf{x}_{m,:} \cdot \mathbf{a}_{:,p}
\end{bmatrix}
= \begin{bmatrix}
    b_{1, 1} & b_{1,2} &\hdots & b_{1,p} \\
    b_{2,1}  & \ddots &\hdots & \vdots \\
    \vdots & \vdots & \ddots & \vdots\\
    b_{m,1}  & \hdots & \hdots & b_{m,p}
\end{bmatrix}
$
\end{center}
Since we used rows of $X$ and columns of $A$, for our convenience, we will unroll $B$ by rows and $A$ by columns. Therefore our Jacobian matrix $J_{\widehat{g}}$ becomes:
\begin{center}$
J_{\widehat{g}} =
\begin{bmatrix}
    \frac{\partial b_{1,1}}{\partial a_{1,1}} & \frac{\partial b_{1,1}}{\partial a_{2,1}} &\hdots &  \frac{\partial b_{1,1}}{\partial a_{1,2}} &\hdots & \frac{\partial b_{1,1}}{\partial a_{n,p}} \\
    \frac{\partial b_{1, 2}}{\partial a_{1,1}}  & \ddots &\hdots & \frac{\partial b_{1,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial b_{2,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial b_{m,p}}{\partial a_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial b_{m,p}}{\partial a_{n,p}}
\end{bmatrix}
=
\begin{bmatrix}
    \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1}}{\partial a_{1,1}} & \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1}}{\partial a_{2,1}} &\hdots &  \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1}}{\partial a_{1,2}} &\hdots & \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1}}{\partial a_{n,p}} \\
    \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,2}}{\partial a_{1,1}}  & \ddots &\hdots & \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial \mathbf{x}_{2,:} \cdot \mathbf{a}_{:,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial \mathbf{x}_{m,:} \cdot \mathbf{a}_{:,m}}{\partial a_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial \mathbf{x}_{m,:} \cdot \mathbf{a}_{:,p}}{\partial a_{n,p}}
\end{bmatrix}
$
\end{center}
Notice that for $\frac{\partial \mathbf{x}_{1, :} \cdot \mathbf{a}_{:,1}}{\partial a_{2,1}}$, the partial derivative is non-zero because $\mathbf{x}_{1, :} \cdot \mathbf{a}_{:,1}$ depends on the entry $a_{2,1}$; whereas for $\frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,2}}{\partial a_{1,1}}$, the partial derivative is zero because $a_{1,1}$ is not an entry inside the column vector $\mathbf{a}_{:, 2}$, therefore change in $a_{1,1}$ do not impact $\mathbf{x}_{1,:} \cdot \mathbf{a}_{:,2}$. \textit{The entry must be within the column vector to have a non-zero partial derivative}. Following this pattern, our Jacobian matrix $J_{\widehat{g}}$ becomes:
\begin{center}$
J_{\widehat{g}} = \begin{bmatrix}
    \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1}}{\partial a_{1,1}} & \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,1}}{\partial a_{2,1}} &\hdots &  0 &\hdots & 0 \\
    0  & \ddots &\hdots & \frac{\partial \mathbf{x}_{1,:} \cdot \mathbf{a}_{:,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial \mathbf{x}_{2,:} \cdot \mathbf{a}_{:,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    0  & \hdots & \hdots & \hdots & \hdots & \frac{\partial \mathbf{x}_{m,:} \cdot \mathbf{a}_{:,p}}{\partial a_{n,p}}
\end{bmatrix}
$
\end{center}
Taking the derivative of the dot product, vectorize it and we get the Jacobian matrix $J_{\widehat{g}}$:
\begin{equation}
J_{\widehat{g}} = \begin{bmatrix}
    x_{1,1} &\hdots & x_{1,n}&  0 &\hdots & 0 \\
    0   &\ddots & 0 &x_{1,1}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    x_{2,1}  & \vdots & x_{2,n} & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    0  & \hdots & \hdots & \hdots & \hdots & x_{m,n}
\end{bmatrix} = \begin{bmatrix}
     \mathbf{x}_{1,:}  & &   &  \\
      & \mathbf{x}_{1,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{1,:}\\
    \mathbf{x}_{2,:}  & &   &  \\
      & \mathbf{x}_{2,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{2,:}\\
     & \hdots& \hdots &\\
     \mathbf{x}_{m,:}  & &   &  \\
      & \mathbf{x}_{m,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{m,:}
\end{bmatrix}
\in \mathbb{R}^{mp \times np}
\end{equation}

\subsection{Jacobian matrix $J_{\widehat{h}}$ for $h(B)=BY$}
Similarly to section 2.1, we will start by breaking $B$ and $Y$ into rows and columns. So Equation (14) becomes:

\begin{center}$
h(B) = BY = \begin{bmatrix} \mathbf{b}_{1,:} \\ \vdots \\ \mathbf{b}_{m,:} \end{bmatrix}
\begin{bmatrix} \mathbf{y}_{:,1} && \hdots && \mathbf{y}_{:,q} \end{bmatrix} \\$
\end{center}

\begin{center}$
= \begin{bmatrix}
    \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} & \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,2} &\hdots & \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,p} \\
    \mathbf{b}_{2,:} \cdot \mathbf{y}_{:,1}  & \ddots &\hdots & \vdots \\
    \vdots & \vdots & \ddots & \vdots\\
    \mathbf{b}_{m,:} \cdot \mathbf{y}_{:,1}  & \hdots & \hdots & \mathbf{b}_{m,:} \cdot \mathbf{y}_{:,p}
\end{bmatrix}
= \begin{bmatrix}
    c_{1,1} & c_{1,2} &\hdots & c_{1,p} \\
    c_{2,1}  & \ddots &\hdots & \vdots \\
    \vdots & \vdots & \ddots & \vdots\\
    c_{m,1}  & \hdots & \hdots & c_{m,p}
\end{bmatrix}
$
\end{center}
We will unroll B by rows and C by columns to construct our Jacobian matrix:
\begin{center}$
J_{\widehat{h}} =
\begin{bmatrix}
    \frac{\partial c_{1,1}}{\partial b_{1,1}} & \frac{\partial c_{1,1}}{\partial b_{1,2}} &\hdots &  \frac{\partial c_{1,1}}{\partial b_{2,1}} &\hdots & \frac{\partial c_{1,1}}{\partial b_{m,p}} \\
    \frac{\partial c_{1,2}}{\partial b_{1,1}}  & \ddots &\hdots & \frac{\partial c_{1,2}}{\partial b_{2,1}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial c_{2,1}}{\partial b_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial c_{m,q}}{\partial b_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial c_{m,q}}{\partial b_{m,p}}
\end{bmatrix}
=
\begin{bmatrix}
    \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} }{\partial b_{1,1}} & \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} }{\partial b_{1,2}} &\hdots &  \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} }{\partial b_{2,1}} &\hdots & \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} }{\partial b_{m,p}} \\
    \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,2} }{\partial b_{1,1}}  & \ddots &\hdots & \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,2} }{\partial b_{2,1}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial \mathbf{b}_{2,:} \cdot \mathbf{y}_{:,1} }{\partial b_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial \mathbf{b}_{m,:} \cdot \mathbf{y}_{:,q} }{\partial b_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial \mathbf{b}_{m,:} \cdot \mathbf{y}_{:,q} }{\partial b_{m,p}}
\end{bmatrix}
$
\end{center}
By the pattern that non-pairing entry and row vectors have zero partial derivatives, the Jacobian matrix can be reduced to:
\begin{center}$
J_{\widehat{h}}
=
\begin{bmatrix}
    \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} }{\partial b_{1,1}} & \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,1} }{\partial b_{1,2}} &\hdots &  0 &\hdots & 0 \\
    \frac{\partial \mathbf{b}_{1,:} \cdot \mathbf{y}_{:,2} }{\partial b_{1,1}}  & \ddots &\hdots & 0 & \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    0 & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    0  & \hdots & \hdots & \hdots & \hdots & \frac{\partial \mathbf{b}_{m,:} \cdot \mathbf{y}_{:,q} }{\partial b_{m,p}}
\end{bmatrix}
$
\end{center}
Taking the partial derivatives of the dot product:
\begin{center}$
J_{\widehat{h}}
=
\begin{bmatrix}
    y_{1,1} & y_{2,1} &\hdots & y_{p,1}&  0 &\hdots & 0 \\
    y_{1,2}  & \vdots &\ddots  &y_{p,2}&0& \hdots& \vdots \\
    \vdots &\vdots& \vdots & \ddots & \hdots & \hdots & \vdots\\
    y_{1,q} & y_{2,q} & \vdots & y_{p,q}& 0 & \hdots & \vdots\\
    0& 0 & \vdots & 0& y_{1,1} & \hdots & \vdots\\
    0 & 0 & \vdots & 0& y_{1,2} & \hdots & \vdots\\
    \vdots&\vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    0 & 0 & \vdots & 0& y_{1,q} & \hdots & \vdots\\
    \vdots&\vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    0  &\hdots& \hdots & \hdots & \hdots & \hdots & y_{p,q}
\end{bmatrix}
$
\end{center}
Notice that the diagonal $q\times p$ blocks are $Y^T$, therefore:
\begin{equation}
J_{\widehat{h}}
=
\begin{bmatrix}
    Y^T \\ & Y^T \\ & & Y^T \\ & & & \ddots \\ & & & & Y^T
\end{bmatrix} \in \mathbb{R}^{mq \times mp}
\end{equation}

\subsection{Calculating $J_{\widehat{h}\circ\widehat{g}} $ from the chain rule}
By the chain rule, we calculate $J_{\widehat{h}\circ\widehat{g}} \in \mathbb{R}^{mq \times np}$ by $J_{\widehat{h}}\cdot J_{\widehat{g}}$:
\begin{center}$
J_{\widehat{h}}\cdot J_{\widehat{g}}
=
\begin{bmatrix}
    Y^T \\ & Y^T \\ & & Y^T \\ & & & \ddots \\ & & & & Y^T
\end{bmatrix} \cdot
\begin{bmatrix}
     \mathbf{x}_{1,:}  & &   &  \\
      & \mathbf{x}_{1,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{1,:}\\
     & \hdots& \hdots &\\
     \mathbf{x}_{m,:}  & &   &  \\
      & \mathbf{x}_{m,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{m,:}
\end{bmatrix}
$
\end{center}
To simplify the matrix multiplication, we break done $Y^T$ to (note that $\mathbf{y}_{i,:}^T \in \mathbb{R}^{q\times1}$ is the transpose of the first row of $Y$):
\begin{center}$
Y^T=\begin{bmatrix}
    \mathbf{y}_{1,:}^T & \mathbf{y}_{2,:}^T & \hdots & \mathbf{y}_{p,:}^T
\end{bmatrix}
$\end{center}
And we break the matrix multiplication blocks by blocks:
\begin{center}
    $J_{\widehat{h}}\cdot J_{\widehat{g}}=
    \begin{bmatrix}
    \begin{bmatrix}Y^T & \textbf{0}& \hdots& \textbf{0}\end{bmatrix} \cdot J_{\widehat{g}}\\
    \begin{bmatrix}\textbf{0}& Y^T& \hdots& \textbf{0}\end{bmatrix} \cdot J_{\widehat{g}}\\
    \hdots     \\
     \begin{bmatrix}\textbf{0} & \textbf{0}& \hdots& Y^T\end{bmatrix} \cdot J_{\widehat{g}}
    \end{bmatrix}
    $
\end{center}
Let's look at the first block:
\begin{equation}
\begin{bmatrix}Y^T & \textbf{0}& \hdots& \textbf{0}\end{bmatrix} \cdot J_{\widehat{g}} =
    \begin{bmatrix}
    Y^T & \textbf{0}& \hdots& \textbf{0}
\end{bmatrix} \cdot
\begin{bmatrix}
     \mathbf{x}_{1,:}  & &   &  \\
      & \mathbf{x}_{1,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{1,:}\\
     & \hdots& \hdots &\\
     \mathbf{x}_{m,:}  & &   &  \\
      & \mathbf{x}_{m,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{m,:}
\end{bmatrix}
\end{equation}
Note that since $Y^T$  $\in q\times p$, only the first $p$ rows of $J_{\widehat{g}}$ matters. Hence Equation (19) becomes:
\begin{center}
    $Y^T \cdot
    \begin{bmatrix}
     \mathbf{x}_{1,:}  & &   &  \\
      & \mathbf{x}_{1,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{1,:}\\
\end{bmatrix}$
\end{center}

\begin{center}
    $=\begin{bmatrix}
    \mathbf{y}_{1,:}^T & \mathbf{y}_{2,:}^T & \hdots & \mathbf{y}_{p,:}^T
\end{bmatrix} \cdot
    \begin{bmatrix}
     \mathbf{x}_{1,:}  & &   &  \\
      & \mathbf{x}_{1,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{1,:}\\
\end{bmatrix}$
\end{center}
\begin{center}
    $= \begin{bmatrix}
     \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{1,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{1,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{1,:}\end{bmatrix} \in \mathbb{R}^{q\times pn}$
\end{center}
If we repeat for other blocks, we would get the Jacobian matrix for $J_{\widehat{h}\circ\widehat{g}} $ to be:

\begin{equation}
    J_{\widehat{h}\circ\widehat{g}} = \begin{bmatrix}
    \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{1,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{1,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{1,:} \\
    \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{2,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{2,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{2,:} \\
    \vdots & \vdots & \ddots & \vdots \\
    \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{m,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{m,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{m,:} \\
     \end{bmatrix} \in \mathbb{R}^{qm \times pn}
\end{equation}
It is also important to keep in mind what each entry within $J_{\widehat{h}\circ\widehat{g}}$ mean:
\begin{center}
    $J_{\widehat{h}\circ\widehat{g}}=J_{\widehat{h}}\cdot J_{\widehat{g}} =
    \begin{bmatrix}
    \frac{\partial c_{1,1}}{\partial b_{1,1}} & \frac{\partial c_{1,1}}{\partial b_{1,2}} &\hdots &  \frac{\partial c_{1,1}}{\partial b_{2,1}} &\hdots & \frac{\partial c_{1,1}}{\partial b_{m,p}} \\
    \frac{\partial c_{1,2}}{\partial b_{1,1}}  & \ddots &\hdots & \frac{\partial c_{1,2}}{\partial b_{2,1}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial c_{2,1}}{\partial b_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial c_{m,q}}{\partial b_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial c_{m,q}}{\partial b_{m,p}}
\end{bmatrix} \cdot
    \begin{bmatrix}
    \frac{\partial b_{1,1}}{\partial a_{1,1}} & \frac{\partial b_{1,1}}{\partial a_{2,1}} &\hdots &  \frac{\partial b_{1,1}}{\partial a_{1,2}} &\hdots & \frac{\partial b_{1,1}}{\partial a_{n,p}} \\
    \frac{\partial b_{1, 2}}{\partial a_{1,1}}  & \ddots &\hdots & \frac{\partial b_{1,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial b_{2,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial b_{m,p}}{\partial a_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial b_{m,p}}{\partial a_{n,p}}
\end{bmatrix}
$
\end{center}
\begin{equation}
    = \begin{bmatrix}
    \frac{\partial c_{1,1}}{\partial a_{1,1}} & \frac{\partial c_{1,1}}{\partial a_{2,1}} &\hdots &  \frac{\partial c_{1,1}}{\partial a_{1,2}} &\hdots & \frac{\partial c_{1,1}}{\partial a_{n,p}} \\
    \frac{\partial c_{1,2}}{\partial a_{1,1}}  & \ddots &\hdots & \frac{\partial c_{1,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial c_{2,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial c_{m,q}}{\partial a_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial c_{m,q}}{\partial a_{n,p}}
\end{bmatrix}
\end{equation}

\subsection{Adjoint calculation}
Now we have successfully found the general solution of Jacobian matrix for $f(A)=XAY$ and $g(A)=XA$ (or more precisely, the unrolled version of them, namely $f(\mathbf{a})$ and $\widehat{g}(\mathbf{a})$, and it is time to apply them in the adjoint calculation for GCDE, as they are the special cases of the general solutions outlined in section 2.1-2.3. Back in section 1.2 I stated that we need to find the Jacobian matrix $\frac{\partial f(H(t), A, W)}{\partial H(t)}$ and $\frac{\partial f(H(t), A, W)}{\partial W}$; well that does not actually make sense in our convention since (1) $H(t)$ is not a vector and (2) the adjoint must be a vector, which means $\frac{\partial L}{\partial H(t)}$ should also be unrolled. Therefore, a GCDE version of adjoint dynamics for Equation (4) and (5) is:
\begin{equation}
    -a(t)^{T}\frac{\partial f(h(t), t, \theta)}{\partial h} \rightarrow
    -\frac{\partial L}{\partial \mathbf{h}(t)}^{T}  \frac{\partial \widehat{f}(\mathbf{h}(t), A, W)}{\partial \mathbf{h}(t)}
\end{equation}
\begin{equation}
    -a(t)^{T}\frac{\partial f(h(t), t, \theta)}{\partial \theta} \rightarrow
        -\frac{\partial L}{\partial \mathbf{h}(t)}^{T}  \frac{\partial \widehat{f}(\mathbf{h}(t), A, W)}{\partial \mathbf{w}}
\end{equation}
where $\mathbf{h}(t) = unroll(H(t))$, $\mathbf{w} = unroll(W)$, and $\widehat{f}(\mathbf{h}(t), A, W) = unroll(f(roll(\mathbf{h}(t)), A, W))$.
However, this does not stop us to find a vectorized equivalence for Equation (4) and (5) for GCDE implementation on hardware, as keeping the matrices unrolled greatly increases the dimensions and could not take the parallel in-memory computing advantages brought by memristor crossbars.

%The first adjoint, $\frac{\partial L}{\partial H(t)}^T\cdot\frac{\partial f(H(t), A, W)}{\partial H(t)}$

%$\frac{\partial f(H(t), A, W)}{\partial H(t)}$ and $\frac{\partial f(H(t), A, W)}{\partial W}$

\subsection{Vectorized Equation (22)}
We start the vectorization from the general case $f(A)=XAY$ we discussed in the earlier sections. Let $ \mathbb{R}^{m\times q}\ni C = f(A)$, and $\mathbb{R}^{mq}\ni\mathbf{c} = unroll(C)$, we unroll $C$ row by row so that its partial derivatives with respect to a scalar loss function $L$ has the form:
\begin{center}
    $\frac{\partial L}{\partial \mathbf{c}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial c_{1,1}} \\ \vdots \\\frac{\partial L}{\partial c_{1,q}}\\ \frac{\partial L}{\partial c_{2,1}} \\ \vdots \\ \frac{\partial L}{\partial c_{m,q}}
    \end{bmatrix} =
    \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}}^T \\ \frac{\partial L}{\partial \mathbf{c}_{2,:}}^T\\ \vdots \\ \frac{\partial L}{\partial \mathbf{c}_{m,:}}^T
    \end{bmatrix} \in \mathbb{R}^{mq}, \frac{\partial L}{\partial \mathbf{c}_{i,:}} \in \mathbb{R}^{1\times q}
    $
\end{center}
We want to find (a reminder that $\widehat{f}(\mathbf{a})$ is a unrolled version of $f(A)$):
\begin{center}
$
-\frac{\partial L}{\partial \mathbf{c}}^{T} \frac{\partial \widehat{f}(\mathbf{a})}{\partial \mathbf{a}}=
-\frac{\partial L}{\partial \mathbf{c}}^{T}  J_{\widehat{f}} = -\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}$
\end{center}
Expand and plug in Equation (20):
\begin{center}
    $ -\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}} =
    -\begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} & \frac{\partial L}{\partial \mathbf{c}_{2,:}} & \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}}
    \end{bmatrix} \cdot
    \begin{bmatrix}
    \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{1,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{1,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{1,:} \\
    \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{2,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{2,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{2,:} \\
    \vdots & \vdots & \ddots & \vdots \\
    \mathbf{y}_{1,:}^T\cdot\mathbf{x}_{m,:}  &  \mathbf{y}_{2,:}^T\cdot\mathbf{x}_{m,:} & \hdots & \mathbf{y}_{p,:}^T\cdot\mathbf{x}_{m,:} \\
     \end{bmatrix}
    $
\end{center}
Since $\frac{\partial L}{\partial \mathbf{c}_{i,:}} \in \mathbb{R}^{1\times q}$ and $\mathbf{y}^T_{i,:}\cdot \mathbf{x}_{j,:} \in \mathbb{R}^{q\times n}$, the blocks defined above have matching dimensions. Hence we can take their dot product directly:
\begin{center}
    $ -\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}} = -
    \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{1,:}\cdot \mathbf{x}_{1,:}  + \hdots + \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{1,:}\cdot \mathbf{x}_{m,:}  &
        \hdots &
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{p,:}\cdot \mathbf{x}_{1,:}   + \hdots + \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{p,:}\cdot \mathbf{x}_{m,:}
    \end{bmatrix}
    $
\end{center}
Note that this dot product is a long $1\times np$ vector, and we can simplify it by rolling it into a $p\times n$ matrix:

\begin{equation}
    -roll(\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}) = -
    \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{1,:}\cdot \mathbf{x}_{1,:}   + \hdots + \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{1,:}\cdot \mathbf{x}_{m,:} \\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{2,:}\cdot \mathbf{x}_{1,:}   + \hdots + \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{2,:}\cdot \mathbf{x}_{m,:} \\
        \vdots\\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{p,:}\cdot \mathbf{x}_{1,:}   + \hdots + \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{p,:}\cdot \mathbf{x}_{m,:}
    \end{bmatrix}
\end{equation}
Equation (24) can be further simplified:
\begin{center}
    $-roll(\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}) = \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{1,:} & \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{1,:} \\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{2,:}  &\hdots & \frac{\partial L}{\partial \mathbf{c}_{m, :}} \cdot \mathbf{y}^T_{2,:}\\
        \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{p,:} &  \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{p,:}
    \end{bmatrix} \cdot \begin{bmatrix}
        \mathbf{x}_{1,:} \\ \mathbf{x}_{2,:} \\ \vdots \\ \mathbf{x}_{m,:}
    \end{bmatrix} = \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{1,:} & \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{1,:} \\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{2,:}  &\hdots & \frac{\partial L}{\partial \mathbf{c}_{m, :}} \cdot \mathbf{y}^T_{2,:}\\
        \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{p,:} &  \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{p,:}
    \end{bmatrix} \cdot X$
\end{center}
Let:
\begin{center}
    $E = \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{1,:} & \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{1,:} \\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{2,:}  &\hdots & \frac{\partial L}{\partial \mathbf{c}_{m, :}} \cdot \mathbf{y}^T_{2,:}\\
        \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{p,:} &  \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{p,:}
    \end{bmatrix}$
\end{center}
We can simplify E further by looking at it row by row. The first row of E is:
\begin{equation}
    \mathbf{e}_{1,:} = \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} \cdot \mathbf{y}^T_{1,:} & \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}} \cdot \mathbf{y}^T_{1,:}
    \end{bmatrix}
\end{equation}
Notice that each entry within Equation (25) is a scalar because $\frac{\partial L}{\partial \mathbf{c}_{i,:}} \in \mathbb{R}^{1\times q}$ and $\mathbf{y}^T_{1,:} \in \mathbb{R}^{q\times 1}$. Therefore, we can manipulate Equation (25) such that:
\begin{center}
    $\mathbf{e}_{1,:} = (\mathbf{y}^T_{1,:})^{T} \cdot \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}}^{T} & \frac{\partial L}{\partial \mathbf{c}_{2,:}}^{T} & \hdots &\frac{\partial L}{\partial \mathbf{c}_{m,:}}^{T}
    \end{bmatrix} = \mathbf{y}_{1,:} \cdot \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}}^{T} & \frac{\partial L}{\partial \mathbf{c}_{2,:}}^{T} & \hdots &\frac{\partial L}{\partial \mathbf{c}_{m,:}}^{T}
    \end{bmatrix}$
\end{center}
We define $\frac{\partial L}{\partial C}$ having matching entries to $C$:
\begin{center}
    $\frac{\partial L}{\partial C} = \begin{bmatrix}
        \frac{\partial L}{\partial c_{1,1}} & \frac{\partial L}{\partial c_{1,2}} & \hdots &\frac{\partial L}{\partial c_{1,q} } \\
        \frac{\partial L}{\partial c_{2,1}} & \ddots & \hdots &\vdots \\
        \vdots & \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial c_{m,1}} & \hdots & \hdots &\frac{\partial L}{\partial c_{m,q}}
    \end{bmatrix}$
\end{center}
Therefore:
\begin{center}
        $\mathbf{e}_{1,:} = \mathbf{y}_{1,:} \cdot \frac{\partial L}{\partial C}^T$
\end{center}
If we repeat for all rows of $E$, then:
\begin{center}
   $E = \begin{bmatrix}
        \mathbf{y}_{1,:} \cdot \frac{\partial L}{\partial C}^T \\
        \mathbf{y}_{2,:} \cdot \frac{\partial L}{\partial C}^T\\
        \vdots \\
        \mathbf{y}_{p,:} \cdot \frac{\partial L}{\partial C}^T
    \end{bmatrix}
    = \begin{bmatrix}
        \mathbf{y}_{1,:}  \\
        \mathbf{y}_{2,:} \\
        \vdots \\
        \mathbf{y}_{p,:}
    \end{bmatrix} \cdot \frac{\partial L}{\partial C}^T = Y \cdot \frac{\partial L}{\partial C}^T$
\end{center}
As a result:
\begin{equation}
    -roll(\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}) = -E \cdot X = -Y \cdot \frac{\partial L}{\partial C}^T \cdot X
\end{equation}
We need to keep in mind what this matrix actually represents. So we go back to Equation (21):
\begin{center}
    $ -roll(\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}) =
    -roll(\begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{c}_{1,:}} & \frac{\partial L}{\partial \mathbf{c}_{2,:}} & \hdots & \frac{\partial L}{\partial \mathbf{c}_{m,:}}
    \end{bmatrix} \cdot= \begin{bmatrix}
    \frac{\partial c_{1,1}}{\partial a_{1,1}} & \frac{\partial c_{1,1}}{\partial a_{2,1}} &\hdots &  \frac{\partial c_{1,1}}{\partial a_{1,2}} &\hdots & \frac{\partial c_{1,1}}{\partial a_{n,p}} \\
    \frac{\partial c_{1,2}}{\partial a_{1,1}}  & \ddots &\hdots & \frac{\partial c_{1,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial c_{2,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial c_{m,q}}{\partial a_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial c_{m,q}}{\partial a_{n,p}}
\end{bmatrix})
    $
\end{center}

\begin{center}
$=-roll(\begin{bmatrix}
        \frac{\partial L}{\partial a_{1,1}} & \frac{\partial L}{\partial a_{2,1}} & \hdots & \frac{\partial L}{\partial a_{n,1}} & \frac{\partial L}{\partial a_{1,2}} & \hdots& \frac{\partial L}{\partial a_{n,p}}
    \end{bmatrix})$
\end{center}
\begin{equation}
=-\begin{bmatrix}
        \frac{\partial L}{\partial a_{1,1}} & \frac{\partial L}{\partial a_{2,1}} & \hdots & \frac{\partial L}{\partial a_{n,1}} \\
        \frac{\partial L}{\partial a_{1,2}} & \frac{\partial L}{\partial a_{2,2}} & \hdots & \frac{\partial L}{\partial a_{n,2}} \\
        \vdots & \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial a_{1,p}} & \frac{\partial L}{\partial a_{2,p}} & \hdots & \frac{\partial L}{\partial a_{n,p}}
    \end{bmatrix}=
    -\begin{bmatrix}
        -\frac{\partial L}{\partial \mathbf{a}_{:,1}}^T- \\
        -\frac{\partial L}{\partial \mathbf{a}_{:,2}}^T- \\
        \vdots\\
        -\frac{\partial L}{\partial \mathbf{a}_{:,p}}^T- \\
    \end{bmatrix}
\end{equation}
Notice that the entries of this matrix matches the transpose of $A$. Hence, in order to match the entries of $A$, we will take Equation (27)'s transpose -- and here we found the general vectorized adjoint solution for $f(A)$:
\begin{center}
    $vectorized(-\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}) = -\begin{bmatrix}
        -\frac{\partial L}{\partial \mathbf{a}_{:,1}}^T- \\
        -\frac{\partial L}{\partial \mathbf{a}_{:,2}}^T- \\
        \vdots\\
        -\frac{\partial L}{\partial \mathbf{a}_{:,p}}^T- \\
    \end{bmatrix}^T = -(Y \cdot \frac{\partial L}{\partial C}^T \cdot X)^T $
\end{center}
\begin{equation}
    vectorized(-\frac{\partial L}{\partial \mathbf{c}}^{T} J_{\widehat{h}\circ\widehat{g}}) = -X^T \cdot \frac{\partial L}{\partial C} \cdot Y^T
\end{equation}
The adjoint dynamics shown in Equation (22) for GCDE is just a special of Equation (28). The derivative of the $ReLU()$ activation function is an element-wise binary step function:
\begin{equation}
    step(x)=
    \begin{cases}
      1 & x>0 \\
      0 & x\leq 0
   \end{cases}
\end{equation}
Hence, let $\odot$ denote the Hadamard product between matrices, by chain rule and Equation (28), the vectorized Equation (22) is:
\begin{center}
    $vectorized(-\frac{\partial L}{\partial \mathbf{h}(t)}^{T}  \frac{\partial \widehat{f}(\mathbf{h}(t), A, W)}{\partial \mathbf{h}(t)}) =
    -A^T \cdot (\frac{\partial L}{\partial H(t)} \odot step(H(t)))\cdot W^T$
\end{center}
where:
\begin{center}
    $\frac{\partial L}{\partial H(t)} = roll(\frac{\partial L}{\partial \mathbf{h}(t)})=\begin{bmatrix}
        \frac{\partial L}{\partial h_{1,1}} & \frac{\partial L}{\partial h_{1,2}} & \hdots &\frac{\partial L}{\partial h_{1,q} } \\
        \frac{\partial L}{\partial h_{2,1}} & \ddots & \hdots &\vdots \\
        \vdots & \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial h_{m,1}} & \hdots & \hdots &\frac{\partial L}{\partial h_{m,q}}
    \end{bmatrix}$
\end{center}
Since by definition of GCDE, the graph topology matrix A is always symmetric, the final vectorized Equation (22) is:
\begin{equation}
 vectorized(-\frac{\partial L}{\partial \mathbf{h}(t)}^{T}  \frac{\partial \widehat{f}(\mathbf{h}(t), A, W)}{\partial \mathbf{h}(t)}) =
    -A \cdot (\frac{\partial L}{\partial H(t)} \odot step(H(t)))\cdot W^T
\end{equation}
\subsection{Vectorized Equation (23)}
We will again vectorize Equation (23) from a general case, $g(A)=XA$. This is because we can treat $AH(t)$ in Equation (6) as a single matrix that linearly transforms $W$. Let $ \mathbb{R}^{m\times p}\ni B = g(A)$, and $\mathbb{R}^{mp}\ni\mathbf{b} = unroll(B)$, we unroll $B$ row by row so that its partial derivatives with respect to a scalar loss function $L$ has the form:
\begin{center}
    $\frac{\partial L}{\partial \mathbf{b}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial b_{1,1}} \\ \vdots \\\frac{\partial L}{\partial b_{1,p}}\\ \frac{\partial L}{\partial b_{2,1}} \\ \vdots \\ \frac{\partial L}{\partial b_{m,p}}
    \end{bmatrix} =
    \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{b}_{1,:}}^T \\ \frac{\partial L}{\partial \mathbf{b}_{2,:}}^T\\ \vdots \\ \frac{\partial L}{\partial \mathbf{b}_{m,:}}^T
    \end{bmatrix} \in \mathbb{R}^{mp}, \frac{\partial L}{\partial \mathbf{b}_{i,:}} \in \mathbb{R}^{1\times p}
    $
\end{center}
In addition, we define
\begin{center}
    $\frac{\partial L}{\partial B} = \begin{bmatrix}
        \frac{\partial L}{\partial b_{1,1}} & \frac{\partial L}{\partial b_{1,2}} & \hdots &\frac{\partial L}{\partial b_{1,q} } \\
        \frac{\partial L}{\partial b_{2,1}} & \ddots & \hdots &\vdots \\
        \vdots & \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial b_{m,1}} & \hdots & \hdots &\frac{\partial L}{\partial b_{m,q}}
    \end{bmatrix}$
\end{center}
Since we are considering the general case $g(A)=XA$ and we are given an arbitury adjoint $\mathbf{b}$, we can change Equation (17) to:
\begin{center}
$-\frac{\partial L}{\partial \mathbf{h}(t)}^{T}  \frac{\partial \widehat{f}(\mathbf{h}(t), A, W)}{\partial \mathbf{w}} \rightarrow
-\frac{\partial L}{\partial \mathbf{b}}^{T} \frac{\partial \widehat{g}(\mathbf{a})}{\partial \mathbf{a}} = -\frac{\partial L}{\partial \mathbf{b}}^{T}  J_{\widehat{g}}$
\end{center}
Expand and plug in Equation (17):
\begin{center}
    $ -\frac{\partial L}{\partial \mathbf{b}}^{T} J_{\widehat{g}} =
    -\begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{b}_{1,:}} & \frac{\partial L}{\partial \mathbf{b}_{2,:}} & \hdots & \frac{\partial L}{\partial \mathbf{b}_{m,:}}
    \end{bmatrix} \cdot
    \begin{bmatrix}
     \mathbf{x}_{1,:}  & &   &  \\
      & \mathbf{x}_{1,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{1,:}\\
     & \hdots& \hdots &\\
     \mathbf{x}_{m,:}  & &   &  \\
      & \mathbf{x}_{m,:} & &  \\
     &  & \ddots &\\
     &  &  & \mathbf{x}_{m,:}
\end{bmatrix}$
\end{center}
We again break it into blocks:
\begin{center}
    $ -\frac{\partial L}{\partial \mathbf{b}}^{T} J_{\widehat{g}} =
    -\begin{bmatrix} \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{b}_{1,:}} & \frac{\partial L}{\partial \mathbf{b}_{2,:}} & \hdots & \frac{\partial L}{\partial \mathbf{b}_{m,:}}
    \end{bmatrix} \cdot
    \begin{bmatrix}
     \mathbf{x}_{1,:}  \\
       \vdots\\
    \textbf{0} \\
     \mathbf{x}_{2,:} \\
      \vdots\\
     \mathbf{x}_{m,:} \\
    \vdots\\
    \textbf{0} \\
\end{bmatrix} \hdots
    \begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{b}_{1,:}} & \frac{\partial L}{\partial \mathbf{b}_{2,:}} & \hdots & \frac{\partial L}{\partial \mathbf{b}_{m,:}}
    \end{bmatrix} \cdot
    \begin{bmatrix}
      \textbf{0} \\
       \vdots\\
    \mathbf{x}_{1,:} \\
      \textbf{0} \\
      \vdots\\
      \textbf{0}\\
    \vdots\\
    \mathbf{x}_{m,:} \\
\end{bmatrix}
\end{bmatrix}$
\end{center}
If we expand the first block:
\begin{center}
    $\begin{bmatrix}
        \frac{\partial L}{\partial \mathbf{b}_{1,:}} & \frac{\partial L}{\partial \mathbf{b}_{2,:}} & \hdots & \frac{\partial L}{\partial \mathbf{b}_{m,:}}
    \end{bmatrix} \cdot
    \begin{bmatrix}
     \mathbf{x}_{1,:}  \\
       \vdots\\
    \textbf{0} \\
     \mathbf{x}_{2,:} \\
      \vdots\\
     \mathbf{x}_{m,:} \\
    \vdots\\
    \textbf{0} \\
\end{bmatrix}  = \begin{bmatrix}
 \frac{\partial L}{\partial b_{1,1}} \cdot x_{1,1} + \frac{\partial L}{\partial b_{2,1}} \cdot x_{2,1} + \hdots + \frac{\partial L}{\partial b_{m,1}} \cdot x_{m,1} \\
 \frac{\partial L}{\partial b_{1,1}} \cdot x_{1,2} + \frac{\partial L}{\partial b_{2,1}} \cdot x_{2,2} + \hdots + \frac{\partial L}{\partial b_{m,1}} \cdot x_{m,2} \\
 \vdots\\
 \frac{\partial L}{\partial b_{1,1}} \cdot x_{1,p} + \frac{\partial L}{\partial b_{2,1}} \cdot x_{2,p} + \hdots + \frac{\partial L}{\partial b_{m,1}} \cdot x_{m,p}
\end{bmatrix}^T$
\end{center}
\begin{center}
    $= (\begin{bmatrix}
  x_{1,1} &  x_{2,1} & \hdots &  x_{m,1} \\
  x_{1,2} &  x_{2,2} & \hdots &  x_{m,2} \\
 \vdots &\vdots& \ddots &\vdots\\
  x_{1,p} &  x_{2,p} & \hdots &  x_{m,p}
\end{bmatrix} \cdot \begin{bmatrix}
 \frac{\partial L}{\partial b_{1,1}} \\ \frac{\partial L}{\partial b_{2,1}} \\ \vdots \\ \frac{\partial L}{\partial b_{m,1}}
\end{bmatrix})^T = (X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,1}})^T$
\end{center}
If we repeat for other blocks, we would get:
\begin{center}
    $ -\frac{\partial L}{\partial \mathbf{b}}^{T} J_{\widehat{g}} = \begin{bmatrix}
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,1}} \\
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,2}} \\
    \vdots\\
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,p}}
    \end{bmatrix}^T$ or $ (-\frac{\partial L}{\partial \mathbf{b}}^{T} J_{\widehat{g}})^T = \begin{bmatrix}
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,1}} \\
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,2}} \\
    \vdots\\
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,p}}
    \end{bmatrix}$
\end{center}
We will make it vectorized by rolling it into a $n\times p$ matrix:
\begin{center}
    $roll((-\frac{\partial L}{\partial \mathbf{b}}^{T} J_{\widehat{g}})^T) = -\begin{bmatrix}
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,1}} &
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,2}} &
    \hdots&
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,p}}
    \end{bmatrix}$
\end{center}
Further vectorization could be done:
\begin{center}
    $-\begin{bmatrix}
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,1}} &
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,2}} &
    \hdots&
    X^T \cdot \frac{\partial L}{\partial \mathbf{b}_{:,p}}
    \end{bmatrix} = -X^T \cdot \begin{bmatrix}
     \frac{\partial L}{\partial \mathbf{b}_{:,1}} &
     \frac{\partial L}{\partial \mathbf{b}_{:,2}} &
    \hdots&
     \frac{\partial L}{\partial \mathbf{b}_{:,p}}
    \end{bmatrix}$
\end{center}
\begin{equation}
    = -X^T \cdot \frac{\partial L}{\partial B}
\end{equation}
Of course, we need to keep in mind what $-X^T \cdot \frac{\partial L}{\partial B}$ actually represents:
\begin{center}
    $roll((-\frac{\partial L}{\partial \mathbf{b}}^{T} J_{\widehat{g}})^T) =  -(\begin{bmatrix}
        \frac{\partial L}{\partial b_{1,1}} & \hdots &\frac{\partial L}{\partial b_{1,p}}& \frac{\partial L}{\partial b_{2,1}} & \hdots & \frac{\partial L}{\partial b_{m,p}}
    \end{bmatrix} \cdot \begin{bmatrix}
    \frac{\partial b_{1,1}}{\partial a_{1,1}} & \frac{\partial b_{1,1}}{\partial a_{2,1}} &\hdots &  \frac{\partial b_{1,1}}{\partial a_{1,2}} &\hdots & \frac{\partial b_{1,1}}{\partial a_{n,p}} \\
    \frac{\partial b_{1, 2}}{\partial a_{1,1}}  & \ddots &\hdots & \frac{\partial b_{1,2}}{\partial a_{1,2}}& \hdots& \vdots \\
    \vdots & \vdots & \ddots & \vdots & \hdots & \vdots\\
    \frac{\partial b_{2,1}}{\partial a_{1,1}} & \vdots & \vdots & \ddots & \hdots & \vdots\\
    \vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
    \frac{\partial b_{m,p}}{\partial a_{1,1}}  & \hdots & \hdots & \hdots & \hdots & \frac{\partial b_{m,p}}{\partial a_{n,p}}
\end{bmatrix})^T$
\end{center}
\begin{center}
$=-roll(\begin{bmatrix}
        \frac{\partial L}{\partial a_{1,1}} & \frac{\partial L}{\partial a_{2,1}} & \hdots & \frac{\partial L}{\partial a_{n,1}} & \frac{\partial L}{\partial a_{1,2}} & \hdots& \frac{\partial L}{\partial a_{n,p}}
    \end{bmatrix}^T)$
\end{center}
\begin{equation}
=-\begin{bmatrix}
        \frac{\partial L}{\partial a_{1,1}} & \frac{\partial L}{\partial a_{1,2}} & \hdots & \frac{\partial L}{\partial a_{1,p}} \\
        \frac{\partial L}{\partial a_{2,1}} & \frac{\partial L}{\partial a_{2,2}} & \hdots & \frac{\partial L}{\partial a_{2,p}} \\
        \vdots & \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial a_{n,1}} & \frac{\partial L}{\partial a_{n,2}} & \hdots & \frac{\partial L}{\partial a_{n,p}}
    \end{bmatrix}
\end{equation}
\begin{center}
    $ = -X^T \cdot \frac{\partial L}{\partial B}$
\end{center}
Since the entries of $-X^T\cdot \frac{\partial L}{\partial B}$ matches $A$, Equation (31) is a good vectorized solution. Therefore:
\begin{center}
    $vectorized(-\frac{\partial L}{\partial \mathbf{b}}^{T}  J_{\widehat{g}}) = -X^T\cdot \frac{\partial L}{\partial B}$
\end{center}
Now if we plug Equation (31) into our GCDE special case, and if we define:
\begin{center}
    $\frac{\partial L}{\partial W} = roll(\frac{\partial L}{\partial \mathbf{w}})=\begin{bmatrix}
        \frac{\partial L}{\partial w_{1,1}} & \frac{\partial L}{\partial w_{1,2}} & \hdots &\frac{\partial L}{\partial w_{1,q} } \\
        \frac{\partial L}{\partial w_{2,1}} & \ddots & \hdots &\vdots \\
        \vdots & \vdots & \ddots & \vdots\\
        \frac{\partial L}{\partial w_{m,1}} & \hdots & \hdots &\frac{\partial L}{\partial w_{m,q}}
    \end{bmatrix}$
\end{center}
Then the vectorized Equation (23) is:
\begin{equation}
 vectorized(-\frac{\partial L}{\partial \mathbf{w}}^{T}  \frac{\partial \widehat{f}(\mathbf{h}(t), A, W)}{\partial \mathbf{w}}) =
    -(AH(t))^T \cdot (\frac{\partial L}{\partial W} \odot step(H(t)))
\end{equation}
\subsection{Summary}
In conclusion, we have found the vectorized adjoint dynamics for GCDE. Using the vectorized adjoint dynamics, we do not need to unroll the hidden states and parameters, and we could take the advantages of the in-memory matrices programmed on memristor crossbar.
\section*{Acknowledgement}
I thank Louis Primeau for his thorough tutorial on multivariable calculus and matrix unrolling for Jacobian calculation. I thank Nafiseh Ghoroghchian for her guide on matrix differentiation and proofread of the work.
\bibliographystyle{alpha}
\bibliography{sample}
Neural ODE: https://arxiv.org/pdf/1806.07366.pdf \\
GCDE: https://arxiv.org/pdf/1911.07532.pdf \\
GCN: https://arxiv.org/pdf/1609.02907.pdf \\
Multivariable calculus crash course, Louis Primeau \\
Neural ODE for Memristor Crossbar, Louis Primeau
\end{document}


\section{Method}
The method is summarized in figure~\ref{fig:workflow}. It consists of three main phases:
\begin{enumerate*}[(i)]
	\item feature extraction,
	\item classification, and
	\item delineation of linear elements.
\end{enumerate*}
%\bigskip
\begin{figure}[!b]
	\centering
	\begin{tikzpicture}[node distance=0.70cm, scale=0.7, every node/.style={transform shape}]

	\node (in1) [io] {Raw Point Cloud};

	\node (param) [title, below=of in1] {Feature extraction};
	\node (pro1) [process, below left=2.5cm and -1.6cm=of param, fill=darkorange!20] {Compute nearest neighbours};
	\node (pro2) [process, below=of pro1, fill=darkorange!20] {Compute neighbourhood features};

	\begin{scope}[on background layer]
	\node (fit1) [fit=(param)(pro1)(pro2), inner sep=4pt, transform shape=false, draw=black!80, fill=darkorange, fill opacity=0.5] {};
	\end{scope}

	\node (out1) [io, above right=-0.55cm and 4.5cm=of fit1] {Point Cloud Features};
	\node (in2) [io, right=of out1] {Manually Classified Point Cloud};

	\node (class) [title, below right=1.5cm and 5.7cm=of out1] {Classification};
	\node (pro4) [process, below left=2.5cm and -6.6cm=of class, fill=lightblue!20] {Remove irrelevant points};
	\node (pro5) [process, below=of pro4, fill=lightblue!20] {Create random forest classifier using manually classified point cloud};
	\node (pro6) [process, below=of pro5, fill=lightblue!20] {Assess performance using cross validation};
	\node (pro7) [process, above right=-3.67cm and 7.5cm=of pro6, fill=lightblue!20] {Classify unknown points};
	\node (pro8) [process, below=of pro7, fill=lightblue!20] {Compute cylindrical neighbourhoods of vegetation points};
	\node (pro9) [process, below=of pro8, fill=lightblue!20] {Seperate trees from low vegetation using height difference};

	\begin{scope}[on background layer]
	\node (fit2) [fit=(class)(pro4)(pro5)(pro6)(pro7)(pro8)(pro9), inner sep=4pt, transform shape=false, draw=black!80, fill=lightblue, fill opacity=0.5] {};
	\end{scope}

	\node (out2) [io, above right= -0.55cm and 13.5cm=of fit2] {Classified Point Cloud};

	\node (lin) [title, below=of out2] {Linear Elements};
	\node (pro10) [process, below left=2.5cm and -15.6cm=of lin, fill=turquoise!20] {Convert to 2D and downsample};
	\node (pro11) [process, below=of pro10, fill=turquoise!20] {Grow regions based on rectangularity};
	\node (pro12) [process, below=of pro11, fill=turquoise!20] {Merge directionally aligned regions};
	\node (pro13) [process, below=of pro12, fill=turquoise!20] {Assess the elongatedness of the regions};

	\begin{scope}[on background layer]
	\node (fit3) [fit=(lin)(pro10)(pro11)(pro12)(pro13), inner sep=4pt, transform shape=false, draw=black!80, fill=turquoise, fill opacity=0.5] {};
	\end{scope}

	\node (out3) [io, below=of fit3] {Linear vegetation elements};

	\draw [arrow] (in1) -- (fit1);
	\draw [arrow] (fit1) -| +(2.6,0) |- (out1.west);
	\draw [arrow] (out1.south) |- +(0,-0.45) -| (fit2.north); %(out1.south|-fit2.north);
	\draw [arrow] (fit2) -| +(4.5,0) |- (out2.west);
	\draw [arrow] (out2) -- (fit3);
	\draw [arrow] (in2.south) |- +(0,-0.195) -| (fit2.north); %(in2.south|-fit2.north);

	\draw [arrow] (pro1) -- (pro2);

	\draw [arrow] (pro4) -- (pro5);
	\draw [arrow] (pro5) -- (pro6);
	\draw [arrow] (pro5) -| +(2,0) |- (pro7);
	\draw [arrow] (pro7) -- (pro8);
	\draw [arrow] (pro8) -- (pro9);

	\draw [arrow] (pro10) -- (pro11);
	\draw [arrow] (pro11) -- (pro12);
	\draw [arrow] (pro12) -- (pro13);

	\draw [arrow] (fit3) -- (out3);

	\end{tikzpicture}
	\caption{Overview of the method in a work flow diagram.}
	\label{fig:workflow}
\end{figure}

\subsection{Feature extraction}
A point cloud \(\mathcal{P}\) is a set of points \(\{p_{1}, p_{2}, \dots, p_{n}\}\) \(\in \mathbb{R}^3\). Each point \(p_{i}\) consists of the X, Y and Z coordinates of the point and additional information on the intensity (\(I\)) of the return signal, the return number, and the number of returns (\(R_{t}\)) at \(p_{i}\) is known (figure~\ref{fig:LiDAR}).
A normalized return number (\(R_{n}\)) was calculated by dividing the return number by the number of returns~\citep{guo2011relevance}.

For each point \(p_{i}\) a neighbourhood set \(\mathcal{N}_{i}\) of points \(\{q_{1}, q_{2}, \dots, q_{k}\}\) is defined, where \(q_{1} = p_{i}\).
%This can be done in three ways:
%\begin{enumerate*}[(i)]
%	\item a \(k\)-nearest neighbours neighbourhood, where the first \(k\ (-1)\) nearest points to \(p_{i}\) are part of the neighbourhood,
%	\item a spherical neighbourhood, where all the points within a certain radius (\(r_{s}\)) of \(p_{i}\) belong to the neighbourhood, and
%	\item a cylindrical neighbourhood, which includes all points within a cylinder with infinite height and a certain radius (\(r_{c}\)) of \(p_{i}\).
%\end{enumerate*}
This neighbourhood was determined using a \(k\)-nearest neighbours method with a \(k\) of 50. This method was used instead of spherical neighbourhood of a certain radius, because it is harder to define a suitable radius which works for fluctuating point densities within and across datasets~\citep{weinmann2014semantic}.

To determine these neighbours a k-d tree data structure was constructed, which is a multidimensional binary search tree that can be used for efficient nearest neighbour searches~\citep{bentley1975kdtree}.

For each neighbourhood some basic geometric properties are computed \citep{weinmann2015semantic}:

{\setlength{\abovedisplayskip}{0pt}
\begin{flalign}
	\label{eq:deltaz}
	&\text{Height difference:}&&& \Delta_{Z_{i}} &= \max_{j:\mathcal{N}_{i}}(q_{Z_{j}}) - \min_{j:\mathcal{N}_{i}}(q_{Z_{j}}) &&&\\
	\label{eq:stdz}
	&\text{Height standard deviation:}&&& \sigma_{Z_{i}} &= \sqrt{\frac{1}{k} \sum_{j=1}^k (q_{Z_{j}} - \overline{q_{Z}})^2} &&&\\
	\label{eq:radius}
	&\text{Local radius:}&&& r_{l_{i}} &= \max_{j: \mathcal{N}_{i}}(|p_{i} - q_{j}|) &&&\\
	\label{eq:density}
	&\text{Local point density:}&&& D_{i} &= \frac{k}{\frac{4}{3} \pi r_{l_{i}}^3} &&&
\end{flalign}

Additionally the neighbourhood of a point can be used to characterize local surface features as first described by~\citet{hoppe1992surface}, and further developed by~\citet{pauly2002efficient}, who used the local structure tensor to estimate the surface normals (the vector perpendicular to the surface (\(\vec{N} = (N_{x}, N_{y}, N_{z})\))) and to define the surface variation (equation~\ref{eq:surfacevariation}). The structure tensor describes the directions of the neighbourhood of a point by determining the covariance matrix of the X, Y and Z coordinates of the points and computing the eigenvalues (\(\lambda_{1}, \lambda_{2}, \lambda_{3}\), where \(\lambda_{1} > \lambda_{2} > \lambda_{3}\)) and eigenvectors of this matrix. The magnitude of the eigenvalues of this covariance matrix describe the spread of points in the direction of the eigenvector. By ranking the eigenvectors based on the respective eigenvalues the spread in the three principle directions of a local point cloud can be quantified. This was expanded upon by~\citet{west2004context} to the following eight structure tensor features:

\begin{flalign}
\label{eq:linearity}
&\text{Linearity: }&&& L_{\lambda} &= \frac{\lambda_{1} - \lambda_{2}}{\lambda_{1}}&&&\\
\label{eq:planarity}
&\text{Planarity: }&&& P_{\lambda} &= \frac{\lambda_{2} - \lambda_{3}}{\lambda_{1}}&&&\\
\label{eq:sphericity}
&\text{Sphericity: }&&& S_{\lambda} &= \frac{\lambda_{3}}{\lambda_{1}}&&&\\
\label{eq:omnivariance}
&\text{Omnivariance: }&&& O_{\lambda} &= \sqrt[3]{\lambda_{1} \lambda_{2} \lambda_{3}}&&&\\
\label{eq:anisotropy}
&\text{Anisotropy: }&&& A_{\lambda} &= \frac{\lambda_{1} - \lambda_{3}}{\lambda_{1}}&&&\\
\label{eq:eigenentropy}
&\text{Eigenentropy: }&&& E_{\lambda} &= -\lambda_{1}\ln(\lambda_{1}) -\lambda_{2}\ln(\lambda_{2}) -\lambda_{3}\ln(\lambda_{3})&&&\\
\label{eq:sumofeigenvalues}
&\text{Sum of eigenvalues: }&&& \sum_{\lambda} &= \lambda_{1} + \lambda_{2} + \lambda_{3}&&&\\
\label{eq:surfacevariation}
&\text{Local surface variation: }&&& C_{\lambda} &= \frac{\lambda_{3}}{\lambda_{1} + \lambda_{2} + \lambda_{3}}&&&
\end{flalign}

To avoid using an unnecessarily large feature set the correlation between all the features was checked by calculating pearson coefficients. Of the pairs of two extremely correlated features (above 0.98), one was removed. This resulted in the removal of anisotropy, which is negatively correlated with sphericity, and eigenentropy, which is correlated with omnivariance.

Consequently the resulting feature set used for the classification was \(\{I, R_{t}, R_{n}, \Delta_{Z}, \sigma_{Z}, r_{l}, D, N_{z}, L_{\lambda}, P_{\lambda}, S_{\lambda}, O_{\lambda}, \sum_{\lambda}, C_{\lambda}\}\).

\subsection{Classification}
\label{sec:class}
Since the linear vegetation elements consist of either trees or low vegetation (shrubs and small trees) the point cloud needs to be classified into \textit{trees}, \textit{low vegetation}, and \textit{irrelevant} points. The difference between trees and low vegetation is ambiguous and is dependent on interpretation. In previous research this boundary has been set at varying height values, ranging from 1.5~\citep{bork2007integrating}, to 3.5~\citep{antonarakis2008object}, to 4~\citep{koukoulas2005spatial}, to 5 meters~\citep{khosravipour2014generating}. We set this boundary at 4m. The irrelevant points are returns of which the linearity does not need to be checked. This may include very low vegetation (herbs), bare ground, water, buildings, and other man made objects.

\subsubsection{Data trimming}
To speed up the classification first points were removed which were certain not relevant. This was done based on the planarity (equation~\ref{eq:planarity}) and sphericity (equation~\ref{eq:sphericity}) features discussed earlier. Since scanning vegetation (which is larger than herbs) always results in a locally very scattered point cloud, with points spread out throughout the 3D space, points with a locally planar neighbourhood can be removed. That is to say points with a high planarity value, \(P_{\lambda} > 0.7\), and a low sphericity value, \(S_{\lambda} < 0.05\), were removed. These values were manually selected in a conservative way to make sure every point removed was certainly not relevant, while still removing a large portion of points.

\subsubsection{Supervised classification}
The remaining point cloud was classified into \textit{vegetation} and \textit{irrelevant} using a random forest classifier. The random forest algorithm creates a collection of decision trees each based on a random subset of the training data~\citep{ho1998random}. These decision trees are computed by a Classification And Regression Tree (CART) algorithm, which creates splits that minimize a gini impurity index~\citep{breiman1984classification}. This impurity index is the probability a randomly picked sample would be misclassified, given it was randomly classified conform the distribution of classes. For each point, each tree in the forest determines a class, and the class which gets selected by the majority of the trees is chosen as the final classification~\citep{breiman2001random}.

Training and testing data was created by analysing the research area and manually segmenting areas of \textit{vegetation} and \textit{irrelevant}. This was done based on the point cloud and aerial photos~\citep{PDOK2015luchtfoto}.

Because the point cloud is trimmed based on planarity/sphericity and it concerns an agricultural landscape it has become very imbalanced, having a lot more \textit{vegetation} than \textit{irrelevant} points, which was also very noticeable in the training and testing data. Imbalanced training data can lead to undesirable classification results~\citep{he2009learning}. Therefore a balanced random forest was used, where instead of the decision trees being made using bootstrap samples of the entire training dataset, a bootstrap sample was taken from only the minority class and a random sample was taken of the majority class based on the size of the minority class sample~\citep{Chen2004using}. Employing enough trees eventually all majority class data gets used, while still maintaining a balance between the two classes. The size of the majority sample compared to the minority sample can be adjusted to find the best balance.

The random forest parameters (i.e. maximum depth, maximum number of features, minimal samples per leaf, minimal samples per split) and the ratio between minority and majority samples were optimized using a cross validated grid search.

\subsubsection{Accuracy Assessment}

To assess the accuracy of the classification a confusion matrix was used. Such a matrix shows the predicted and the actual classes of the tested pixels/points. It gives an overview of the performance of the classification in that it identifies and quantifies the errors. Using this matrix a precision (user's accuracy), recall (producer's accuracy), and overall accuracy can be calculated~\citep{stehman1997selecting}, but these metrics are unable to provide a good picture of the performance of a classifier when the data is very imbalanced.

Instead, three metrics, which provide a useful indication of performance even when dealing with a very imbalanced dataset, were used~\citep{sun2009classification, lopez2013insight}. These are:
\begin{enumerate*}[(i)]
	\item the receiver operating characteristic (ROC) curve~\citep{bradley1997use},
	\item the Matthew's correlation coefficient (MCC)~\citep{matthews1975comparison}, and
	\item the geometric mean~\citep{kubat1998machine}.
\end{enumerate*}

To create a ROC-curve the true positive rate is plotted against the false positive rate at various decision thresholds. The area under a ROC-curve (AUC) is a measure for the performance of the classifier~\citep{bradley1997use}.

The MCC analyses the correlation between the observed and the predicted data and is defined as follows:

\begin{equation}
	\label{eq:MCC}
	{\text{MCC}}={\frac  {TP\times TN-FP\times FN}{{\sqrt  {(TP+FP)(TP+FN)(TN+FP)(TN+FN)}}}}
\end{equation}

where TP are the true positives, TN the true negatives, FP the false positives, and FN the false negatives. It often gives a more balanced evaluation of a classifier than the precision, recall and accuracy~\citep{baldi2000assessing} and it is an effective metric for unbalanced datasets~\citep{kohavi1995study}.

The geometric mean of the recall of both classes evaluates the balanced performance of the classifier for the two classes~\citep{kubat1998machine, sun2009classification}.

These scores were acquired by performing a 10-fold cross validation. This is done by splitting the data into 10 randomly mutually exclusive subsets and using a subset as testing data on a classifier trained on the remaining data~\citep{kohavi1995study}. This method is effective against overfitting and does not further reduce the training data (which a test and validation set would).

To assess which features are most influential when separating vegetation from non-vegetation a feature importance analysis was performed. This is done by summing the decreases in gini impurity due to a feature in all the trees divided by the total amount of trees~\citep{breiman2002manual}.

\subsubsection{Delineating trees and low vegetation}
To divide the vegetation class into \textit{tree} and \textit{low vegetation} for each point a cylindrical neighbourhood was determined, which includes all points within a cylinder with infinite height and a radius of 2 meters. Within this neighbourhood the height difference (equation~\ref{eq:deltaz}) was computed. As mentioned earlier, if this value exceeded 4 meters the point was classified as a \textit{tree} point, otherwise as a \textit{low vegetation} point.

\subsection{Delineating Linear Elements}
To delineate linear from non-linear vegetation elements the two classes were segmented into rectangular objects, since those can be assessed for linearity by their length and width. This is done by growing regions within the relevant points as long as the shape remains rectangular.

\begin{figure}
	\centering
	\includegraphics[scale=0.40]{./img/downsample}
	\caption{An example showing the tree points before and after downsampling.}
	\label{fig:downsample}
\end{figure}
Since the linearity of an object is solely determined by the 2D spatial distribution of points, the point cloud was converted to 2D by removing the Z-coordinate from all the points. To speed up the computation the 2D point cloud was spatially downsampled to 1 meter distance between all points for low vegetation and 2 meter for trees (figure~\ref{fig:downsample}). Low vegetation objects are often smaller than trees and consequently require more points for an accurate delineation. This leads to a slightly less precise delineation, but results in a substantially decreased computation time.

\subsubsection{Rectangularity}
\begin{figure}
	\centering
	\input{./img/hulls.tikz}
	\caption{An example showing the different hulls used for determining the rectangularity.}
	\label{fig:hulls}
\end{figure}
The rectangularity of an object can be described as the ratio between the area of an object and the area of its minimum bounding rectangle~\citep{rosin1999measuring}.

The minimum bounding rectangle (figure~\ref{fig:hulls}) is computed with rotating calipers as described by~\citet{toussaint1983solving}. First a convex hull is determined for the points, which is the smallest convex set of points which contains every point (figure~\ref{fig:hulls}), by using the QuickHull algorithm as defined by~\citet{preparata1985computational}. The minimum bounding rectangle has a side collinear with one of the edges of the convex hull~\citep{freeman1975determining}. Consequently while rotating by the angles of the edges rectangles can be created using the minima and maxima of the coordinates in the rotated system. Keeping track of the rectangle with the minimal area the minimum bounding rectangle can be found.

The area of the object can be found by computing an alpha shape of the set of points belonging to the object~\citep{edelsbrunner1983shape}. An alpha shape is a hull, similar to a convex hull, which describes the shape of a set of points (figure~\ref{fig:hulls}). It is created by computing a Delaunay triangulation of the points~\citep{delaunay1934sphere} and removing the triangles with a circumradius higher than \(1/\alpha\), where \(\alpha\) is a parameter which consequently influences the amount of triangles removed from the triangulation and thus the shape and area of the alpha shape. Higher alphas lead to more complex shapes, while lower ones to more smooth shapes (figure~\ref{fig:hulls}).

\subsubsection{Region Growing}
To segment the point cloud into rectangular objects a region growing segmentation technique was used. This method was introduced by \citet{besl1988segmentation} and has since been an effective approach to segment a point cloud~\citep{tovari2005segmentation, rabbani2006segmentation, nurunnabi2012robust, vosselman2013point, elberink2014user, vo2015octree}. The region growing method consists of two main steps: seed selection and region growing. The seed locations can be selected randomly or based on some properties of the points. The growing is based on a similarity criterion based on the proximity and attributes of the points.

The region growing used in this method is a bit different. Instead of using the attributes to compare the points and grow based on some similarity criteria, the regions are grown based only on proximity and a constraint of rectangularity (figure~\ref{fig:regiongrowing}). The seed selection is done based on the coordinates to minimize the chances of starting at boundary regions.

\begin{figure}
	\centering
	\input{./img/regiongrowing.tikz}
	\caption{An example showing the region growing process.}
	\label{fig:regiongrowing}
\end{figure}

Thus the point with the minimal x-coordinate and its 20 closest neighbours were used as the starting region (given this region is rectangular, otherwise the point is discarded and the process repeated) and subsequently points were added as long as the region’s rectangularity did not drop below a threshold of 0.55. This threshold was determined by experimentation and manual evaluation. Once no more new points could be added to the region the procedure was repeated for the next region until all the points are checked.

\subsubsection{Merging Objects}
The objects can still be quite fragmented after they have been grown, especially for curved regions. To fix this, objects were merged which were
\begin{enumerate*}[(i)]
	\item in proximity of each other,
	\item aligned with each other, and
	\item facing about the same direction.
\end{enumerate*}
The direction was determined by computing the angle between one of the long sides of the minimum bounding box and the x-axis. The alignment was checked by comparing the angle of the line between the two centre points of the objects with the directions. Once merged the lengths of the objects were added and the maximum of the widths taken as the new width.

\subsubsection{Elongatedness}
The resulting rectangular regions were assessed for linearity by determining the elongatedness of an object, which is defined as the ratio between its length and its width~\citep{nagao2013structural}.

The minimum elongatedness of objects was set at 2.5, based on a manual trial and error. To prevent long, but wide patches of forest to be classified as linear elements a maximum width of 60 meters was used.

\subsubsection{Accuracy Assesment}
To assess the accuracy of the delineation of linear objects a manual delineation was done. First a polygon was made both from the tree and low vegetation points using alpha shapes. Subsequently the two polygons were manually further segmented into objects. Finally the polygons belonging to a linear object according to our interpretation were manually marked as being linear. The resulting map was compared with the automated delineation. This was done by a difference map and confusion matrices where we compared the true positive, true negative, false positive and false negative in area. Since this data is, unlike before, not very imbalanced we can use traditional metrics for accuracy. Therefore a user's, producer's and overall accuracy was calculated, as well as an F1, kappa and MCC score~\citep{congalton2008assessing}.

\subsection{Software \& Hardware}
We chose to use solely free and open source software (FOSS), for two main reasons:
\begin{enumerate*}[(i)]
	\item to avoid `black boxes', where it is unclear exactly what calculations or algorithms are being used to produce the results, and
	\item to make the method more easily runnable with cloud- or supercomputing solutions, and thus able to compute for very large areas.
\end{enumerate*}

The large majority was made in Python (2.7.12) with the NumPy~(1.11.2), SciPy~(0.18.1), pandas~(0.18.1), scikit-learn~(0.17.1) and Shapely~(1.5.13) libraries. For the preprocessing of data LASTools~(version~160429) and CloudCompare~(v2.8) were used and Cloudcompare was also used for visualizing and downsampling the point cloud.

For the computation a desktop computer was used with an Intel Xeon E3-1220 3.1 GHz quad-core processor and 16GB of RAM.


\thispagestyle{empty}
%----------------------------------------------------------------------
\chapter{Control and optimization}
\label{control.chap}
%----------------------------------------------------------------------

\section{Objectives}
%----------------------------------------------------------------------
Once a system has been enough understood, there is still to do the main work: enhance it. This means most of the time optimization, and control can be seen as an online optimization (i. e. continuously making the system better). There are several control techniques as there are many optimization techniques. Since this reader aims at keeping things practical, we will not develop all possible ideas here. Instead we insist onto 2 main ideas:
\begin{enumerate}
	\item explicit a criteria to be optimized: in principle, this is rooted in the system description and what we want to do with the system. There are many ways to enhance a system, and very often the models are too complicated to have an explicit optimum (whenever they have one). But without criteria, there is even no way to know what is better (or worse).
	\item simplify the model so that the optimization (or control) problem can be solved by simple tools. As a rule, it is often better to get a simple but efficient control rather than to try to compute the optimal one. This rules also holds for optimization: the best is the enemy of the good; well enough is enough in lots of practical systems.
\end{enumerate}

Therefore some tools for PDE are applied in the next sections as well as simple tools that can be applied by smartly simplifying the problem.

\section{Tools and examples}

\subsection{Kalman filter}
\input{control-kalman}

\subsection{PID control}

\subsection{Adjoint optimization}


\chapter{Basic Concepts}
\label{Chp:Concepts}

The GraphBLAS C API is used to construct
graph algorithms expressed ``in the language of linear algebra.''
Graphs are expressed as matrices, and the operations over
these matrices are generalized through the use of a
semiring algebraic structure.

In this chapter, we will define the basic concepts used to
define the GraphBLAS C API.  We provide the following elements:
\begin{itemize}
\item Glossary of terms used in this document.

\item Notation

\item Execution model

\item Error model

\end{itemize}

\section{Glossary}

%TGM I'm leaving a few definitions in here just as examples

\subsection{Basic definitions}

\glossBegin

\glossItem{application} A program that calls methods from the GraphBLAS C API to
solve a problem.

\glossItem{GraphBLAS C API} The application programming interface that fully defines the types, objects,
literals, and other elements of the C binding to the GraphBLAS.

\glossEnd

\subsection{Objects and their structure}

\glossBegin
\glossItem{handle}  A variable that uses one of the GraphBLAS opaque data types.
The value of this variable holds a reference to a GraphBLAS object but not the contents of the object itself.
Hence, assigning a value of one handle to another variable copies the reference to the GraphBLAS object
but not the contents of the object.

\glossItem{non-opaque datatype} Any datatype that exposes its internal structure.
This is contrasted
with an \emph{opaque datatype} that hides its internal structure and can
be manipulated only through an API.

\glossEnd

\vfill

\newgeometry{left=2.5cm,top=2cm,bottom=2cm}

\section{Notation}

\begin{tabular}[H]{l|p{5in}}
Notation & Description \\
\hline
$\Dout, \Dinn, \Din1, \Din2$  & Refers to output and input domains of various GraphBLAS operators. \\
$\bDout(*), \bDinn(*),$ & Evaluates to output and input domains of GraphBLAS operators (usually \\
~~~~$\bDin1(*), \bDin2(*)$ & a unary or binary operator, or semiring). \\
$\mathbf{D}(*)$   & Evaluates to the (only) domain of a GraphBLAS object (usually a monoid, vector, or matrix). \\
$f$             & An arbitrary unary function, usually a component of a unary operator. \\
$\mathbf{f}(F_u)$ & Evaluates to the unary function contained in the unary operator given as the argument. \\
$\odot$         & An arbitrary binary function, usually a component of a binary operator. \\
$\mathbf{\bigodot}(*)$ & Evaluates to the binary function contained in the binary operator or monoid given as the argument. \\
$\otimes$       & Multiplicative binary operator of a semiring. \\
$\oplus$        & Additive binary operator of a semiring. \\
$\mathbf{\bigotimes}(S)$ & Evaluates to the multiplicative binary operator of the semiring given as the argument. \\
$\mathbf{\bigoplus}(S)$ & Evaluates to the additive binary operator of the semiring given as the argument. \\
$\mathbf{0}(*)$   & The identity of a monoid, or the additive identity of a GraphBLAS semiring. \\
$\mathbf{L}(*)$   & The contents (all stored values) of the vector or matrix GraphBLAS objects.  For a vector, it is the set of (index, value) pairs, and for a matrix it is the set of (row, col, value) triples. \\
$\mathbf{v}(i)$ or $v_i$   & The $i^{th}$ element of the vector $\vector{v}$.\\
$\mathbf{size}(\vector{v})$ & The size of the vector $\vector{v}$.\\
$\mathbf{ind}(\vector{v})$ & The set of indices corresponding to the stored values of the vector $\vector{v}$.\\
$\mathbf{nrows}(\vector{A})$ & The number of rows in the $\matrix{A}$.\\
$\mathbf{ncols}(\vector{A})$ & The number of columns in the $\matrix{A}$.\\
$\mathbf{indrow}(\vector{A})$ & The set of row indices corresponding to rows in $\matrix{A}$ that have stored values.  \\
$\mathbf{indcol}(\vector{A})$ & The set of column indices corresponding to columns in $\matrix{A}$ that have stored values. \\
$\mathbf{ind}(\vector{A})$ & The set of $(i,j)$ indices corresponding to the stored values of the matrix. \\
$\mathbf{A}(i,j)$ or $A_{ij}$ & The element of $\matrix{A}$ with row index $i$ and column index $j$.\\
$\matrix{A}(:,j)$ & The $j^{th}$ column of the the matrix $\matrix{A}$.\\
$\matrix{A}(i,:)$ & The $i^{th}$ row of the the matrix $\matrix{A}$.\\
$\matrix{A}^T$ &The transpose of the matrix $\matrix{A}$. \\
$\neg\matrix{M}$ & The complement of $\matrix{M}$.\\
$\vector{\widetilde{t}}$ & A temporary object created  by the GraphBLAS implementation. \\
$<type>$ & A method argument type that is {\sf void *} or one of the types from Table~\ref{Tab:PredefinedTypes}. \\
{\sf GrB\_ALL} & A method argument literal to indicate that all indices of an input array should be used.\\
{\sf GrB\_Type} & A method argument type that is either a user defined type or one of the  types from Table~\ref{Tab:PredefinedTypes}.\\
{\sf GrB\_Object} &  A method argument type referencing any of the GraphBLAS object types.\\
{\sf GrB\_NULL} & The GraphBLAS NULL.\\
\end{tabular}

\restoregeometry

\section{Error model}

\section{Execution Model}
\label{Sec:ExecutionModel}

%% TGM: I've left the GraphBLAS model here just as a point of reference.

A program using the GraphBLAS C API constructs GraphBLAS objects,
manipulates them to implement a graph algorithm, and then extracts
values from the GraphBLAS objects as the result of the algorithm.
Functions defined within the GraphBLAS C API that manipulate GraphBLAS
objects are called \emph{methods}.  If the method corresponds to one
of the operations defined in the GraphBLAS mathematical specification,
we refer to the method as an \emph{operation}.

Graph algorithms are expressed as an ordered collection of GraphBLAS
method calls defined by the order they are encountered in a program.
This is called the \emph{program order}.  Each method in the collection
uniquely and unambiguously defines the output GraphBLAS objects based
on the GraphBLAS operation and the input GraphBLAS objects. This is the
case as long as there are no execution errors, which can put objects in
an invalid state (see Section~\ref{Sec:ErrorModel}).

The GraphBLAS method calls in program order are organized into contiguous
and nonoverlapping \emph{sequences}.  A sequence is an ordered collection
of method calls as encountered by an executing thread. (For more on
threads and GraphBLAS, see Section~\ref{Sec:ThreadSafety}.)  A sequence
begins with either (1) the first GraphBLAS method called by a thread,
or (2) the first method called by a thread after the end of the
previous sequence.  A sequence can end (terminate) in a variety of ways.
A call to the GraphBLAS {\sf GrB\_wait()} method (Section~\ref{Sec:GrB_wait})
always ends a sequence.  The GraphBLAS {\sf GrB\_finalize()} method
(Section~\ref{Sec:GrB_finalize}) also implicitly ends a sequence. Finally,
in blocking mode (see below), each GraphBLAS method starts and ends its
own sequence.

The GraphBLAS objects are fully defined at any point in a sequence by
the methods in the sequence as long as there are no execution errors.
In particular, as soon as a GraphBLAS method call returns, its output
can be used in the next GraphBLAS method call.  However, individual
operations in a sequence may not be \emph{complete}. We say that an
operation is complete when all the computations in the operation have
finished and all the values of its output object have been produced and
committed to the address space of the program. Furthermore, no additional
execution time can be charged to a completed operation and no additional
errors can be attributed to a completed operation.

The opaqueness of GraphBLAS objects allows execution to proceed
from one method to the next even when operations are not complete.
Processing of nonopaque objects is never deferred in GraphBLAS. That is,
methods that consume nonopaque objects (\eg, {\sf GrB\_Matrix\_build()},
Section~\ref{Sec:Matrix_build}) and methods that produce nonopaque objects (\eg,
{\sf GrB\_Matrix\_extractTuples()}, Section~\ref{Sec:Matrix_extractTuples})
always finish consuming or producing those nonopaque objects before
returning.

\comment{Furthermore, methods that extract values from opaque GraphBLAS objects
into nonopaque user objects (see Table~\ref{Tab:ExtractMethods})
always force completion of all pending computations on the
corresponding GraphBLAS source object.

\begin{table}[htb]
    \hrule
    \begin{center}
        \caption{Methods that extract values from a GraphBLAS object, thereby
        forcing completion of the operations contributing to that particular object.}
        \label{Tab:ExtractMethods}

        \begin{tabular}{l|l}
            Method    & Section \\ \hline

            {\sf GrB\_Vector\_nvals}        & \ref{Sec:Vector_nvals}        \\
            {\sf GrB\_Vector\_extractElement}     & \ref{Sec:extract_single_element_vec}    \\
            {\sf GrB\_Vector\_extractTuples}    & \ref{Sec:Vector_extractTuples}    \\
            {\sf GrB\_Matrix\_nvals}        & \ref{Sec:Matrix_nvals}        \\
            {\sf GrB\_Matrix\_extractElement}     & \ref{Sec:extract_single_element_mat}    \\
            {\sf GrB\_Matrix\_extractTuples}    & \ref{Sec:Matrix_extractTuples}    \\
            {\sf GrB\_reduce} (vector-scalar variant)        & \ref{Sec:Reduce_vector_scalar}        \\
            {\sf GrB\_reduce} (matrix-scalar variant)        & \ref{Sec:Reduce_matrix_scalar}        \\
        \end{tabular}
    \end{center}
    \hrule
\end{table}
}

\section{Error Model}
\label{Sec:ErrorModel}

%%TGM  I've left the GraphBLAS model here as a point of reference

All GraphBLAS methods return a value of type {\sf GrB\_Info} to provide
information available to the system at the time the method returns. The
returned value can be either {\sf GrB\_SUCCESS} or one of the defined
error values shown in Table~\ref{Tab:ErrorValues}. The errors fall into
two groups: API errors (Table~\ref{Tab:ErrorValues}(a)) and execution
errors (Table~\ref{Tab:ErrorValues}(b)).

\begin{table}[bh]
\hrule
\begin{center}
\caption{Error values returned by GraphBLAS methods.}
\label{Tab:ErrorValues}

\vspace{1\baselineskip}
(a) API errors
\vspace{1\baselineskip}

\begin{tabular}{l|p{3in}}
Error code    & Description \\ \hline
{\sf GrB\_UNINITIALIZED\_OBJECT} & A GraphBLAS object is passed to a method before {\sf new} was called on it.\\
{\sf GrB\_NULL\_POINTER} & A NULL is passed for a pointer parameter. \\
{\sf GrB\_INVALID\_VALUE} & Miscellaneous incorrect values. \\
{\sf GrB\_INVALID\_INDEX} & Indices passed are larger than dimensions of the matrix or vector being accessed. \\
{\sf GrB\_DOMAIN\_MISMATCH} & A mismatch between domains of collections and operations when user-defined domains are in use.\\
{\sf GrB\_DIMENSION\_MISMATCH} & Operations on matrices and vectors with incompatible dimensions. \\
{\sf GrB\_OUTPUT\_NOT\_EMPTY} & An attempt was made to build a matrix or vector using an output object that already contains valid tuples (elements).\\
%{\sf GrB\_NO\_VALUE} & An attempt was made to extract a value from a tuple within a matrix or vector for which there is no stored value.
{\sf GrB\_NO\_VALUE} & A location in a matrix or vector is being accessed that has no stored value at the specified location. \scott{It depends on whether or not the non-opaque scalar is
well-defined on return from {\sf extract}}\\
\end{tabular}

\vspace{1\baselineskip}
(b) Execution errors
\vspace{1\baselineskip}

\begin{tabular}{l|p{3in}}
Error code    & Description \\ \hline
{\sf GrB\_OUT\_OF\_MEMORY}         & Not enough memory for operations. \\
{\sf GrB\_INSUFFICIENT\_SPACE}     & The array provided is not large enough to hold output. \\
{\sf GrB\_INVALID\_OBJECT}         & One of the opaque GraphBLAS objects (input or output) is in an invalid state caused by a previous execution error. \\
{\sf GrB\_INDEX\_OUT\_OF\_BOUNDS}  & Reference to a vector or matrix element that is outside the defined dimensions of the object. \\
{\sf GrB\_PANIC}        & Unknown internal error. \\
\end{tabular}

\end{center}
\hrule
\end{table}

An API error means that a GraphBLAS method was called with parameters that
violate the rules for that method.  These errors are restricted to those
that can be determined by inspecting the types and domains of GraphBLAS
objects, GraphBLAS operators, or the values of scalar parameters fixed at
the time a method is called.  API errors are deterministic and consistent
across platforms and implementations.  API errors are never deferred,
even in nonblocking mode. That is, if a method is called in a manner
that would generate an API error, it always returns with the appropriate
API error value.  If a GraphBLAS method returns with an API error, it
is guaranteed that none of the arguments to the method (or any other
program data) have been modified.

Execution errors indicate that something went wrong during the execution
of a legal GraphBLAS method invocation.  Their occurrence may depend on
specifics of the executing environment and data values being manipulated.
This does not mean that execution errors are the fault of the GraphBLAS
implementation.  For example, a memory leak could arise from an error in
an application's source code (a ``program error''), but it may manifest
itself in different points of a program's execution (or not at all)
depending on the platform, problem size, or what else is running at
that time.  Index-out-of-bounds and insuficient space execution errors
always indicate a program error.

In blocking mode, where each method executes to completion, a returned
execution error value applies to the specific method.  If a GraphBLAS
method, executing in blocking mode, returns with any execution error
from Table~\ref{Tab:ErrorValues}(b) other than {\sf GrB\_PANIC}, it
is guaranteed that no argument used as input-only has been modified.
Output arguments may be left in an invalid state, and their use downstream
in the program flow may cause additional errors.  If a GraphBLAS method
returns with a {\sf GrB\_PANIC} execution error, no guarantees can be
made about the state of any program data.

In nonblocking mode, execution errors can be deferred.  A return value
of {\sf GrB\_SUCCESS} only guarantees that there are no API errors in
the method invocation.  If an execution error value is returned by a
method in nonblocking mode, it indicates that an error was found during
execution of the sequence, up to and including the {\sf GrB\_wait()}
method (Section~\ref{Sec:GrB_wait}) call that ends the sequence. When possible, that return value
will provide information concerning the cause of the error.

As discussed in Section~\ref{Sec:GrB_waitOne}, a {\sf GrB\_wait(obj)} on
a specific GraphBLAS object {\sf obj} does not necessarily end a
sequence. However, no additional errors on the methods of the sequence that
have {\sf obj} as an {\sf OUT} or {\sf INOUT} argument can be reported.
From a GraphBLAS perspective, those methods are {\em complete}.

If a GraphBLAS method, executing in nonblocking mode, returns with
any execution error from Table~\ref{Tab:ErrorValues}(b) other than
{\sf GrB\_PANIC}, it is guaranteed that no argument used as input-only
through the entire sequence has been modified.  Any output argument in
the sequence may be left in an invalid state and its use downstream in the
program flow may cause additional errors.  If a GraphBLAS method returns
with a {\sf GrB\_PANIC}, no guarantees can be made about the state of
any program data.

\begin{figure}[tb]
    \hrule
    \vspace{1\baselineskip}
    \begin{center}
        \begin{minipage}{3in}
            \begin{verbatim}
            const char *GrB_error();
            \end{verbatim}
        \end{minipage}
    \end{center}
    \caption{Signature of {\sf GrB\_error()} function.}
    \label{Fig:GrB_error}
    \hrule
\end{figure}

After a call to any GraphBLAS method, the program can retrieve additional
error information (beyond the error code returned by the method) though a
call to the function {\sf GrB\_error()}. The signature of that function is
shown in Figure~\ref{Fig:GrB_error}.  The function returns a pointer to a
NULL-terminated string, and the contents of that string are implementation
dependent. In particular, a null string (not a {\sf NULL} pointer) is always a valid error string.
The pointer is valid until the next call to any GraphBLAS method by the same thread.
{\sf GrB\_error()} is a thread-safe function, in the sense that multiple threads can
call it simultaneously and each will get its own error string back, referring to the
last GraphBLAS method it called.


\documentclass[a4paper]{article}

\usepackage[utf8]{inputenc}
\usepackage{a4wide}
\usepackage{relsize}
\usepackage{indentfirst}
\usepackage{minted}
\usepackage{graphicx}
\usepackage{float}
\usepackage{amsmath, amssymb, amsfonts, amsthm}
\usepackage{hyperref}
\usepackage[capitalise]{cleveref}
\usepackage[toc, page]{appendix}

\AtBeginEnvironment{minted}{\let\itshape\relax}

\begin{document}

\section*{Exercise 1}

See \cref{fig:dhparam}.

\section*{Exercise 2}

See \Cref{fig:dsa-prime,fig:dsa-generator}.

\section*{Exercise 3}

Without the \texttt{dsaparam} option, the generation of DH parameters takes much longer because it
generates a \textit{strong prime}, \textit{i.e.} a prime number $p$ such that $(p-1)/2$ is also prime.
This ensures that the multiplicative group $\mathbb{Z}_p^\ast$ does not contain small subgroups. The
existence of small subgroups within a larger group in a cryptographic protocol is undesirable in the
sense that it confines shared secrets to a smaller set of possible values than if it were to use the
whole group $\mathbb{Z}_p^\ast$.

On the other hand, using the \texttt{dsaparam} DSA rather than DH parameters are read or generate, which
are then they are converted to DH format, making the key exchange process more efficient.

\vspace{\baselineskip}

\textbf{Note:} See \cref{fig:sage-dh,fig:sage-dsa}.

\section*{Exercise 4}

Running the following function with the previously generated DH parameters, we can see that
$X^y \ (\mathrm{mod} \ p) = Y^x \ (\mathrm{mod} \ p)$ holds.

\vspace{\baselineskip}

\begin{minted}{py}
from sage.all import *

def ex4(p, q):
    x = randrange(q)
    y = randrange(q)

    X = Mod(q ** x, p)
    Y = Mod(q ** y, p)

    return Mod(X ** y, p) == Mod(Y ** x, p)
\end{minted}

\vspace{\baselineskip}

\textbf{Note:} See \cref{fig:ex4-dhparam,fig:ex4-dsa}.

\section*{Exercise 5}

We know that $p = 1373, g = 2, X = 974$ and $y = 871$. We can determine $Y$ by computing the
following:

\[
Y = g^y \ \mathrm{mod} \ p = 2^{871} \ \mathrm{mod} \ 1373 = 805
\]

Knowing the value of $X$ and $y$, we can then determine the shared secret $g^{xy}$ by computing
the following:

\[
g^{xy} = (g^x)^y = X ^ y = 974 ^ {871} \ \mathrm{mod} \ 1373 = 397
\]

\pagebreak

Finally, we can find Alice's secret exponent by solving:

\[
X = g^x \ \mathrm{mod} \ p \Leftrightarrow 974 = 2^x \ \mathrm{mod} \ 1373 \Leftrightarrow x = 587
\]

\section*{Exercise 6}

The Computational Diffie-Hellman Problem (CHD) consists in computing the shared secret $g^{ab}$ given
only the public values $g^a$ and $g^b$, and not any of the secret values $a$ or $b$. The motivation
is to ensure that even if an eavesdropper captures $g^a$ and $g^b$, they will not be able to determine
the shared secret $g^{ab}$.

The Decisional Diffie-Hellman Problem (DDH) is stronger than the CDH. To ensure that an attacker can't
learn anything about the shared secret $g^{ab}$, this value needs only to be indistinguishable from a
random group element. This being said, given $g^a, g^b$ and a value that is either $g^{ab}$ or $g^c$
for a random value $c$, each with probability $1/2$, the DDH problem consists of determining whether
$g^{ab}$ was chosen.

An algorithm that solves the Computational Diffie–Hellman problem can be used to solve the Decisional
Diffie–Hellman problem. Given $g^a, g^b$ and $g^c$, and assuming we can solve CDH, we can derive $g^{ab}$
from $g^a$ and $g^b$. Then, to solve DDH, we only have to check if the result equals $g^c$.

\pagebreak

\begin{appendices}

\section*{Exercise 1}

\begin{figure}[H]
    \centering
    \includegraphics[width=.9\textwidth]{img/dhparam.png}
    \caption{Generation of DH parameters without the \texttt{dsaparam} option}
    \label{fig:dhparam}
\end{figure}

\section*{Exercise 2}

\begin{figure}[H]
    \centering
    \includegraphics[width=.8\textwidth]{img/dsa-prime.png}
    \caption{Prime number generated using DH parameter generation with the \texttt{dsaparam} option}
    \label{fig:dsa-prime}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=.8\textwidth]{img/dsa-generator.png}
    \caption{Group generator generated using DH parameter generation with the \texttt{dsaparam} option}
    \label{fig:dsa-generator}
\end{figure}

\section*{Exercise 3}

\begin{figure}[H]
    \centering
    \includegraphics[width=\textwidth]{img/dh-sage.png}
    \caption{Prime number generated using DH parameter generation without the \texttt{dsaparam} option}
    \label{fig:sage-dh}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=\textwidth]{img/dsa-sage.png}
    \caption{Prime number generated using DH parameter generation with the \texttt{dsaparam} option}
    \label{fig:sage-dsa}
\end{figure}

\section*{Exercise 4}

\begin{figure}[H]
    \centering
    \includegraphics[width=\textwidth]{img/dh-py.png}
    \caption{Checking that $X^y \ (\mathrm{mod} \ p) = Y^x \ (\mathrm{mod} \ p)$ holds for the DH parameters generated without the \texttt{dsaparam} option}
    \label{fig:ex4-dhparam}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=\textwidth]{img/dsa-py.png}
    \caption{Checking that $X^y \ (\mathrm{mod} \ p) = Y^x \ (\mathrm{mod} \ p)$ holds for the DH parameters generated with the \texttt{dsaparam} option}
    \label{fig:ex4-dsa}
\end{figure}

\end{appendices}

\end{document}