\documentclass[../../main.tex]{subfiles}

\begin{document}

\subsection{Abstract}

A generalized multilinear regression model, termed the Higher-Order Partial Least Squares (HOPLS) \cite{HOPLS}, is introduced with the aim to predict a tensor $\tensor{Y}$ from a tensor $\tensor{X}$ through projecting the data onto the latent space and performing regression on the corresponding latent variables. This method could be applied to a wide range of datasets. To solve the problem of predicting eye movement we get time series of frames passed through Convolutional Neural Network (CNN).

\subsection{Problem statement}

We are given a dataset, that consists of several videos with eye movements and its positions. Let $\tensor{X}$ is an output of CNN, $\tensor{Y}$ --- eye coordinate. Assume $\tensor{X} \in \mathbb{R}^{I_1 \times ... \times I_N}$ and $\tensor{Y} \in \mathbb{R}^{J_1 \times ... \times J_M}$. We assume $\tensor{X}$ is decomposed as a sum of rank-$(1,L_2,...,L_N)$ Tucker blocks, while $\tensor{Y}$ is decomposed as a sum of rank-$(1, K_2,...,K_M)$ Tucker blocks, which can be expressed as

\begin{equation}
    \begin{split}
        \tensor{X} = \sum_{r=1}^R \tensor{G}_r \times_1 \textbf{t}_r \times_2 \textbf{P}_r^{(1)} \times_3 ... \times_n \textbf{P}_r^{(n-1)} + \tensor{E}_R \\
        \tensor{Y} = \sum_{r=1}^R \tensor{D}_r \times_1 \textbf{t}_r \times_2 \textbf{Q}_r^{(1)} \times_3 ... \times_n \textbf{Q}_r^{(n-1)} + \tensor{F}_R
    \end{split}
\end{equation}

\begin{figure}[h!]
\centering
\includegraphics[width=0.8\textwidth]{figures/HOPLS}
\caption{Schematic diagram of the HOPLS model}
\label{fig:eye_pred:1}
\end{figure}

\noindent where $R$ is the number of latent vectors, $t_r \in \mathbb{R}^{I_1}$ is the $r$-th latent vector, $\left\{\textbf{P}_r^{(n)}\right\}_{n=1}^{N-1} \in \mathbb{R}^{I_{n+1} \times L_{n+1}}$ and
$\left\{\textbf{Q}_r^{(m)}\right\}_{m=1}^{M-1} \in \mathbb{R}^{J_{n+1} \times K_{n+1}}$
are loading matrices on mode-$n$ and mode-$m$ respectively, and $\tensor{G}_r \in \mathbb{R}^{1 \times L_2 \times ... \times L_N}$ and $\tensor{D}_r \in \mathbb{R}^{1 \times K_2 \times ... \times K_M}$ are core tensors.

To make a prediction we should use
\begin{equation}
    \label{prediction}
    \hat{\tensor{Y}} = \tensor{X} \textbf{W} \textbf{Q}^{* \top}
\end{equation}
where $\textbf{W}$ and $\textbf{Q}^{*}$ have $R$ columns, represented by
\begin{equation}
    \begin{split}
        & \textbf{w}_r = \left( \textbf{P}_r^{(N-1)} \otimes ... \otimes \textbf{P}_r^{(1)} \right) \tensor{G}_{r}^+ \\
        & \textbf{q}^*_r = \tensor{D}_{r} \left( \textbf{Q}_r^{(M-1)} \otimes ... \otimes \textbf{Q}_r^{(1)} \right)
    \end{split}
\end{equation}

\subsection{Problem solution}

Use CNN that converts 480x640 pixels images to 24x32 resolution. Its output is $\tensor{X}$. So $N = 3, I_2 = 34, I_3 = 39$. $\tensor{Y}$ tensor is given and it is 2 dimensional tensor with $J_2 = 2$ (Fig. \ref{fig:eye_pred:2}). For these tensors we use Higher-Order Partial Least Squares algorithm described in \cite{HOPLS} and make a prediction according \eqref{prediction}.

\begin{figure}[h!]
\centering
\includegraphics[width=0.8\textwidth]{figures/trajectory}
\caption{Trajectory of the center of eye}
\label{fig:eye_pred:2}
\end{figure}

\subsection{Code}

HOPLS algorithm was made by Arthur Dehgan, co-author of \cite{HOPLS} and avalible at  https://github.com/arthurdehgan/HOPLS.
The code for computation experiment is located on https://github.com/artem062/ForecastingMethods.

\subsection{Experiment}

It was used LPW dataset, that consist of 66 videos with eye moving. This videos was compressed and converted to the tensors. To archive the best $R$ plot the value of $Q^2 = 1 - \|\tensor{Y} - \hat{\tensor{Y}}\|^2_F / \|\tensor{Y}\|^2_F$.

\begin{figure}[h!]
\centering
\includegraphics[width=1.0\textwidth]{figures/plot}
\caption{Graph of dependence of $Q^2$ on $R$}
\label{fig:eye_pred:3}
\end{figure}

This graph shows that optimal value of $R$ by train dataset is 54, but for test it is 45.

\end{document}


\section{Part 1}

The shared code for this part, as well as the random generator for all the parts is here:

\lstinputlisting{one.py}

\section{Part 1.a Normally Distributed pseudo-random numbers}

The file of the functions used for this exercise is:

\lstinputlisting{one_a.py}

My script produces the following plots, see Fig. \ref{fig:xi_xi}, Fig. \ref{fig:thousand}, and Fig. \ref{fig:million}.
These plots suggest the random number generator is sufficiently random and is not biased towards any numbers.

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/Xi_Xi_1.png}
  \caption{The results of the first 1000 numbers of the random generator. }
  \label{fig:xi_xi}
\end{figure}

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/Index_Xi_1.png}
  \caption{First 1000 random numbers vs index.}
  \label{fig:thousand}
\end{figure}

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/1000000_rand.png}
  \caption{1,000,000 random numbers plotted in 20 bins 0.05 wide.}
  \label{fig:million}
\end{figure}

\section{Part 1.b Box-Muller Method}

The file of the functions used for this exercise is:

\lstinputlisting{one_b.py}

My script produces the following result, see Fig. \ref{fig:boxmuller}. As can be seen, the Box-Muller implementation seems
to create a Gaussian distribution.

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/box_gauss.png}
  \caption{Box-Muller method.}
  \label{fig:boxmuller}
\end{figure}

\section{Part 1.c KS-test}

The file of the functions used for this exercise is:

\lstinputlisting{one_c.py}

The result of this function is given by the Figures \ref{fig:kstest}, and \ref{fig:kstestP}.

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/KStest.png}
  \caption{KS Test.}
  \label{fig:kstest}
\end{figure}

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/KStest_pvalue.png}
  \caption{KS Test P-values.}
  \label{fig:kstestP}
\end{figure}

This seems to show that the KS tests are fairly similar in their results, and their results seem to
suggest that the Box-Muller implementation is consistent with a Gaussian distribution.

\section{Part 1.d Kuiper Test}

The file of the functions used for this exercise is:

\lstinputlisting{one_d.py}

My script produces the following results, see Fig. \ref{fig:kuiperTest} and Fig. \ref{fig:kuiperP}.

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/KuiperTest.png}
  \caption{Kuiper Test implementation.}
  \label{fig:kuiperTest}
\end{figure}

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/Kuiper_pvalue.png}
  \caption{Kuiper Test implementation.}
  \label{fig:kuiperP}
\end{figure}

This seems to show that the probabilities derived from my Kuiper Test differs somewhat from the Kuiper test in Astropy, although the cause for that is unknown. At
the same time, the value for V is similar between both the Astropy version and my own.
It also shows that my the Box-Muller implementation seems to still be consistent with a Gaussian Distribution.

\section{Part 1.e 10 Sets of Numbers}

The file of the functions used for this exercise is:

\lstinputlisting{one_e.py}

My script produces the following result, see Fig. \ref{fig:10_sets} for my CDF, and Fig. \ref{fig:10_sets_sci} for results using
Scipy's norm.cdf as a comparison.

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/RandNumKS.png}
  \caption{Results of KS Test on 10 sets of numbers.}
  \label{fig:10_sets}
\end{figure}

\begin{figure}[h!]
  \centering
  \includegraphics[width=0.9\linewidth]{./plots/RandNumKS_sci.png}
  \caption{Results of KS Test on 10 sets of numbers with Scipy norm.cdf.}
  \label{fig:10_sets_sci}
\end{figure}

From these results, we can see some of the random number sets are consistent with Gaussians for small amounts of numbers,
but when using $10^5$ points, none of the sets of random numbers seems consistent. Overall, the closest seems to be
set 3 from the p-values. When lookng at the results using Scipy's norm.cdf as comparison, there seems to also be set 5 that
close to a Gaussian the most times on the graph compared to other ones, but is still not consistent at $10^5$ points. Therefore,
I conclude that sets 5 and 3 are the most consistent with a Gaussian distribution when taking in large numbers of points, sets
1, 9, and 6 are consistent with a Gaussian at specific numbers of points, but none are consistent over all the range in numbers of points.


\documentclass{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{changepage}
\usepackage[margin=0.75in]{geometry}

\setlength\parindent{0pt}

\begin{document}
Arguably the most common stable distribution parameterization has characteristic function given by
\begin{equation*}
	\varphi(t; \alpha, \beta, c, \mu) = \exp(it\mu - \lvert ct \rvert^\alpha (1 - i\beta\operatorname{sgn}(t)\Phi)), \qquad \Phi = \begin{cases}
		\tan(\frac{\pi\alpha}{2}), &\text{if } \alpha \neq 1\\
		-\frac{2}{\pi}\log\lvert t \rvert, &\text{if } \alpha = 1
	\end{cases}
\end{equation*}
For simplicity, we force the scale $c=1$ and the location $\mu = 0$, yielding
\begin{equation*}
	\varphi(t; \alpha, \beta) = \exp(-\lvert t \rvert^\alpha (1 - i\beta\operatorname{sgn}(t)\Phi))
\end{equation*}
with $\Phi$ defined as before.

\subsection*{PDF Integrand Derivation}

Probability density functions are the Fourier transforms of their respective characteristic functions, so
\begin{align*}
	f(x) &= \frac{1}{2\pi} \int_{-\infty}^\infty \varphi(t) e^{-ixt} \,dt\\
	&= \frac{1}{2\pi} \int_{-\infty}^\infty \exp(-\lvert t \rvert^\alpha (1 - i\beta\operatorname{sgn}(t)\Phi)) \exp(-ixt) \,dt\\
	\intertext{By Euler's formula and ignoring strictly complex terms (since the PDF is real-valued),}
	&= \frac{1}{2\pi} \int_{-\infty}^\infty \exp(-\lvert t \rvert^\alpha) [\cos(\lvert t \rvert^\alpha \beta \operatorname{sgn}(t)\Phi)\cos(xt)+\sin(\lvert t \rvert^\alpha \beta \operatorname{sgn}(t)\Phi)\sin(xt)] \,dt
	\intertext{Lastly, by symmetry,}
	f(x) &= \frac{1}{\pi} \int_0^\infty \exp(-t^\alpha) [\cos(t^\alpha\beta\Phi)\cos(xt)+\sin(t^\alpha\beta\Phi)\sin(xt)] \,dt
\end{align*}
Note that the oscillatory terms $\cos(xt)$ and $\sin(xt)$ should be incorporated into a weighting function in QUADPACK routines.\\

Transforming into the $S_0$ parameterization of Nolan amounts to computing $f(x+\beta\tan(\pi\alpha/2))$ instead of $f(x)$ when $\alpha \neq 1$.

\subsection*{CDF Integrand Derivation}

Due to an inversion formula by Gil-Pelaez, we have
\begin{align*}
	F(x) &= \frac{1}{2} - \frac{1}{\pi} \int_0^\infty \frac{\operatorname{Im}[\varphi(t)e^{-ixt}]}{t} \,dt
	\intertext{Rewriting to make use of $t \geq 0$ yields}
	&= \frac{1}{2} - \frac{1}{\pi} \int_0^\infty \frac{\operatorname{Im}[ \exp(-t^\alpha (1 - i\beta\Phi)) \exp(-ixt)]}{t} \,dt
	\intertext{and applying Euler's formula gives us}
	F(x) &= \frac{1}{2} - \frac{1}{\pi} \int_0^\infty \frac{\exp(-t^\alpha)[-\cos(t^\alpha \beta\Phi) \sin(xt)+\sin(t^\alpha \beta\Phi)\cos(xt)]}{t} \,dt
\end{align*}

Note that the oscillatory terms $\cos(xt)$ and $\sin(xt)$ should be incorporated into a weighting function in QUADPACK routines.\\

Transforming into the $S_0$ parameterization of Nolan amounts to computing $F(x+\beta\tan(\pi\alpha/2))$ instead of $F(x)$ when $\alpha \neq 1$.
\end{document}


% A (minimal) template for problem sets and solutions using the exam document class

% Organization:
%% Define new commands, macros, etc. in macros.tex
%% Anything that you would put before \begin{document} should go in prelude.tex

%% For multiple psets, each should get its own file to \input into main with a \section{}
\input{prelude}
\begin{document}

\title{Computational Fluid Dynamics\\Homework 5}
\author{Francisco Jose Castillo Carrasco}
\date{\today}
\maketitle

\input{macros}

%% Content goes here
\section*{Introduction}
In this assignment I will be using the Crank-Nicholson method to solve the following PDE
\begin{align*}
\frac{\partial T}{\partial t}=\alpha\frac{\partial^2 T}{\partial x^2}+q(x,t),
\end{align*}
defined on the domain $x\in[-1,1]$ with boundary conditions
\begin{align*}
T(-1,t)&=2-\sin\left(\frac{3\pi}{2}t\right),\\
\frac{\partial T}{\partial x}(1,t)&=0,
\end{align*}
and initial condition $T(x,0)=2$, with $\alpha=0.1$. The source term $q$ for $t>0$ is given by
\begin{align*}
q(x,t)=\frac{3\pi}{2\sqrt{t}}&\sin\left(\frac{\pi}{2}\sqrt{t}\right)\cos\left(\frac{\pi}{2}\sqrt{t}\right)\left(x^3-x^2-x+1\right)+\frac{3\pi}{2}\cos\left(\frac{3\pi}{2}t\right)\sin\left(\frac{\pi}{2}x\right)\\
&+\alpha\left(\sin^2\left(\frac{\pi}{2}\sqrt{t}\right)(6-18x)+\frac{\pi^2}{4}\sin\left(\frac{3\pi}{2}t\right)\sin\left(\frac{\pi}{2}x\right)\right),
\end{align*}
and $q(x,0)=0$.

This first part of the assignment (the one typed) includes the formulation of the problem using the Crank Nicholson method for a node based and a cell based mesh.
\subsection*{Node Based Mesh}
\input{NodeMesh}

\subsection*{Cell Based Mesh}
\input{CellMesh}
\section*{Maximum stable time step for the FTCS}
As we saw in class, the maximum stable time step size for the FTCS is
\begin{align*}
\Delta t=\frac{h^2}{2\alpha},
\end{align*}
and the time step used in the code is four time this,
\begin{align*}
\Delta t=2\frac{h^2}{\alpha}.
\end{align*}
There is no index form for this.
\end{document}


\color{black}
\subsection*{Stationary distribution a Markov model}

Compute the stationary distribution
\begin{equation*}
\vec{\pi} = \mat{P}_{\lambda_1} \vec{\pi}
\end{equation*}
of the Markov chain $\lambda_1$. Proceed as follows: introduce
\begin{equation*}
  \mat{A} = \begin{bmatrix} \mat{I} - \mat{P}_{\lambda_1} \\ \trn{\vec{1}} \end{bmatrix} \qquad \text{and} \qquad
  \vec{b} = \begin{bmatrix} \vec{0} \\ 1 \end{bmatrix}
\end{equation*}
where the matrix $\mat{I}$ and the vector $\vec{0}$ and $\vec{1}$ have to be of appropriate sizes. Then solve
\begin{equation*}
\mat{A} \, \vec{\pi} = \vec{b}
\end{equation*}
for $\vec{\pi}$. Round your result to two decimals.
\color{blue}
%%%%%
%%%%% enter your answer after the '=' sign
%%%%%
\begin{equation*}
\vec{\pi} = \begin{bmatrix} 0.33333333 \\ 0.33333333 \\ 0.33333333 \end{bmatrix}
\end{equation*}
%%%%%
%%%%%
%%%%%
\color{black}

If you like, paste the code you used to practically compute your result after the following \texttt{import} statements
\color{blue}
%%%%%
%%%%% paset your code after the import statements
%%%%%
\begin{PythonCode}
import numpy as np
import numpy.linalg as la

\end{PythonCode}
%%%%%
%%%%%
%%%%%
\color{black}


\lab{Index of Pseudocode Notation}{Index of Notation}
\label{notation_index}

\section*{Objects}
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{l l}
$A$ 					& A capitol letter typically denotes a matrix, or 2-D NumPy array\\
$[[a, b], [c, d]]$ 			& The explicit array $\left(\begin{array}{cc}
a & b\\
c & d\\\end{array}\right)$\\
$A \gets \allocate{m}{n}$,
$\v \gets \allocate{k}$ 	& Allocate memory for an $m \times n$ NumPy array $A$
						or for a vector $\v$ of length $k$\\
$\Id{n}$	 			& The $n\times n$ array with 1's on the diagonal and 0's elsewhere \\
$a$ 					& A lowercase letter typically denotes a scalar\\

$A[:, j], \v[i]$ 			& Slice the 2-D array $A$ or 1-D array $\v$ as in Python\\
$\v$ 					& A boldface letter typically denotes a vector, or 1-D NumPy array\\
$\zeros{m}{n}$,
$\zeros{k}$	 		& The $m\times n$ array or length-$k$ vector of zeros
\end{tabular}

\section*{Operations}
\begin{tabular}{l l}
$\makecopy{A}$		& Make a copy of the array $A$\\
$A/a$, $\v/a$, $b/a$ 		& Divide by the scalar $a$\\
$AB$, $aB$			& Multiply scalars or matrices\\
$\norm{\v}, \norm{\v}_2$ 		& Find the (Euclidean) norm of $\v$\\
$\shape{A}$			& Return the dimensions of the array $A$\\
$\sign(a)$				& Return the sign of the scalar $a$\\
$\size{A}$				& Return the number of elements in the array $A$\\
$A\trp$				& Transpose the array $A$
\end{tabular}

\section*{Programming constructs}
\begin{tabular}{p{5.5cm} p{8cm}}
$\vartriangleright$		& Comment\\
${\bf for}\; i=1 \ldots n\; {\bf do}$ & ``For'' loop. Iterate from 1 to $n$.\\
$\gets$				& Assignment operator. If $a \gets b$ then $a$ is assigned the value of $b$.\\
$\bf{if} \;\ldots\; \bf{else}$		& ``If'' construction with optional ``else''\\
$\bf{Procedure}<${\sc Name}$>$(Parameters)		& Defines the start of the algorithm called $<${\sc Name}$>$ which accepts the specified parameters as inputs\\
$\bf{while}$			& ``While'' construction\\
$\bf{return}$			& End of algorithm. Return any values specified.
\end{tabular}


%----------------------------------------------------------------------------------------
%	CHAPTER 2
%----------------------------------------------------------------------------------------
%\chapterimage{chapter_head_void.pdf} % Chapter heading image

\chapter{In-text Elements}

\section{Theorems}\index{Theorems}

This is an example of theorems. Theorema \ref{theo:A}, \ref{theo:B}, \ref{theo:C}, \ref{theo:D} e \ref{theo:E}.

\subsection{Several equations}\index{Theorems!Several Equations}
This is a theorem consisting of several equations.

\begin{theorem}[Name of the theorem]
\label{theo:A}
In $E=\mathbb{R}^n$ all norms are equivalent. It has the properties:
\begin{align}
& \big| ||\mathbf{x}|| - ||\mathbf{y}|| \big|\leq || \mathbf{x}- \mathbf{y}||\\
&  ||\sum_{i=1}^n\mathbf{x}_i||\leq \sum_{i=1}^n||\mathbf{x}_i||\quad\text{where $n$ is a finite integer}
\end{align}
\end{theorem}

\begin{proofraw}[Relativa ao Teorema \ref{theo:A}]
In $E=\mathbb{R}^n$ all norms are equivalent. It has the properties:
\begin{align}
& \big| ||\mathbf{x}|| - ||\mathbf{y}|| \big|\leq || \mathbf{x}- \mathbf{y}||\\
&  ||\sum_{i=1}^n\mathbf{x}_i||\leq \sum_{i=1}^n||\mathbf{x}_i||\quad\text{where $n$ is a finite integer}
\end{align}
\end{proofraw}

\subsection{Single Line}\index{Theorems!Single Line}
This is a theorem consisting of just one line.

\begin{theorem}
\label{theo:B}
A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$. A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$.
A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$. A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$.
\end{theorem}

\begin{exercise}[Exercise name]
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{exercise}

\begin{exercise}[Exercise name A0]
\label{ex:A0}
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{exercise}

%------------------------------------------------

\section{Definitions}\index{Definitions}

This is an example of a definition. A definition could be mathematical or it could define a concept.
\begin{theorem}
\label{theo:C}
A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$. A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$.
A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$. A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$.
\end{theorem}

\begin{theorem}
\label{theo:D}
A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$. A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$.
A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$. A set $\mathcal{D}(G)$ in dense in $L^2(G)$, $|\cdot|_0$.
\end{theorem}

\begin{definition}[Definition name]
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{definition}

\begin{theorem}[Theorem name]
\label{theo:E}
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{theorem}

\begin{example}[Example name]
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{example}

\begin{exercise}[Exercise name A]
\label{exer:A}
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{exercise}

\begin{exercise}[Exercise name]
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{exercise}

\begin{exercise}[Exercise name B]
\label{exer:B}
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{exercise}
%------------------------------------------------

Exer \ref{ex:A0}(Exercise name A0), \ref{exer:A}(Exercise name A) e \ref{exer:B}(Exercise name B).

\section{Notations}\index{Notations}

Notations \ref{not:A}, \ref{not:B}, \ref{not:C} e \ref{not:D}.

\begin{notation}
\label{not:A}
Given an open subset $G$ of $\mathbb{R}^n$, the set of functions $\varphi$ are:
\begin{enumerate}
\item Bounded support $G$;
\item Infinitely differentiable;
\end{enumerate}
a vector space is denoted by $\mathcal{D}(G)$.
\end{notation}

\begin{notation}[Notation name title very large]
\label{not:B}
Given an open subset $G$ of $\mathbb{R}^n$, the set of functions $\varphi$ are:
\begin{enumerate}
\item Bounded support $G$;
\item Infinitely differentiable;
\end{enumerate}
a vector space is denoted by $\mathcal{D}(G)$.
\end{notation}

\begin{notation}
\label{not:C}
Given an open subset $G$ of $\mathbb{R}^n$, the set of functions $\varphi$ are:
\begin{enumerate}
\item Bounded support $G$;
\item Infinitely differentiable;
\end{enumerate}
a vector space is denoted by $\mathcal{D}(G)$.
\end{notation}

\begin{notation}[Notation name]
\label{not:D}
Given an open subset $G$ of $\mathbb{R}^n$, the set of functions $\varphi$ are:
\begin{enumerate}
\item Bounded support $G$;
\item Infinitely differentiable;
\end{enumerate}
a vector space is denoted by $\mathcal{D}(G)$.
\end{notation}

%------------------------------------------------

\section{Boxequation}\index{Boxequation}

\begin{boxequation}
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{boxequation}

\section{Frasebox}\index{Frasebox}

\begin{frasebox}{Frase title}{Fernando P. R.}
Given a vector space $E$, a norm on $E$ is an application, denoted $||\cdot||$, $E$ in $\mathbb{R}^+=[0,+\infty[$ such that:
\begin{align}
& ||\mathbf{x}||=0\ \Rightarrow\ \mathbf{x}=\mathbf{0}\\
& ||\lambda \mathbf{x}||=|\lambda|\cdot ||\mathbf{x}||\\
& ||\mathbf{x}+\mathbf{y}||\leq ||\mathbf{x}||+||\mathbf{y}||
\end{align}
\end{frasebox}
%------------------------------------------------

\section{Attentionbox}\index{Attentionbox}

This is an example of a remark.

\begin{attentionbox}
The concepts presented here are now in conventional employment in mathematics.
Vector spaces are taken over the field $\mathbb{K}=\mathbb{R}$, however, established properties are easily extended to $\mathbb{K}=\mathbb{C}$.
\end{attentionbox}

%------------------------------------------------

\section{Informationbox}\index{Informationbox}

This is an example of a remark.

\begin{informationbox}{Título A}
The concepts presented here are now in conventional employment in mathematics.
Vector spaces are taken over the field $\mathbb{K}=\mathbb{R}$, however, established properties are easily extended to $\mathbb{K}=\mathbb{C}$.
\end{informationbox}

%------------------------------------------------

\section{Elaborationbox}\index{Elaborationbox}

This is an example of a remark.

\begin{elaborationbox}{título}
The concepts presented here are now in conventional employment in mathematics.
Vector spaces are taken over the field $\mathbb{K}=\mathbb{R}$, however, established properties are easily extended to $\mathbb{K}=\mathbb{C}$.
\end{elaborationbox}
%------------------------------------------------


\section{The matrix of a linear transformation}

\begin{outcome}
  \begin{enumerate}
  \item Find the matrix of a linear transformation with respect to
    general bases in vector spaces.
  \end{enumerate}
\end{outcome}

In Section~\ref{sec:matrix-of-transformation}, we saw that linear
transformations $T:\R^n\to\R^m$ are in one-to-one correspondence with
$m\times n$-matrices. Here, we will generalize this correspondence to
arbitrary finite-dimensional vector spaces. There is an important
difference, however. While $\R^n$ comes with a natural coordinate
system (i.e., every vector in $\R^n$ has a first component, second
component, and so on), there is no distinguished coordinate system on
an arbitrary vector space. To define the matrix of a linear
transformation $T:V\to W$, we must first choose a basis, or
equivalently a coordinate system, for $V$ and for $W$. Different
choices of basis will give rise to different matrices.

Let $V$ be a vector space with basis
$B=\set{\vect{v}_1,\ldots,\vect{v}_n}$. Recall from
Section~\ref{ssec:bases-and-coordinates} that the \textbf{coordinates}
of a vector $\vect{v}$ with respect to the basis $B$%
\index{coordinate!with respect to basis}%
\index{coordinate system!and basis} are the unique scalars
$a_1,\ldots,a_n$ such that
\begin{equation*}
  \vect{v} = a_1\,\vect{v}_1 + \ldots + a_n\,\vect{v}_n.
\end{equation*}
As before, we write
\begin{equation*}
  \coord{\vect{v}}_B = \begin{mymatrix}{c} a_1 \\ a_2 \\ a_3 \end{mymatrix}
\end{equation*}
to denote the coordinates of $\vect{v}$ with respect to the basis $B$.
We will now see how to use bases and coordinates to encode any linear
map between finite-dimensional vector spaces as a matrix.

\begin{proposition}{The matrix of a linear transformation}{matrix-of-linear-transformation-vector-space}
  Let $V$ and $W$ be finite-dimensional vector spaces over a field
  $K$. Assume $B$ is a basis of $V$ and $C$ is a basis of $W$.  Let
  $T:V\to W$ be a linear transformation. Then there exists a unique
  $m\times n$-matrix $A$ such that for all $\vect{v}\in V$,
  \begin{equation*}
    A\coord{\vect{v}}_B = \coord{T\vect{v}}_C.
  \end{equation*}
  Moreover, $A$ can be computed as follows: the $i\th$ column of $A$
  holds the coordinates of $T(\vect{v}_i)$, where $\vect{v}_i$ is the
  $i\th$ vector of the basis $B$, and the coordinates are computed
  with respect to the basis $C$.
\end{proposition}

\begin{proof}
  Let $B=\set{\vect{v}_1,\ldots,\vect{v}_n}$ and
  $C=\set{\vect{w}_1,\ldots,\vect{w}_m}$.  By
  Theorem~\ref{thm:transformation-basis}, the linear transformation
  $T$ is completely determined by the images of the basis vectors,
  $T(\vect{v}_1),\ldots,T(\vect{v}_n)\in W$. Since
  $\set{\vect{w}_1,\ldots,\vect{w}_m}$ is a basis of $W$, we can write
  each $T(\vect{v}_i)$ as a linear combination of
  $\vect{w}_1,\ldots,\vect{w}_m$:
  \begin{eqnarray*}
    T(\vect{v}_1)
    &=& a_{11}\vect{w}_1 + a_{21}\vect{w}_2 + \ldots + a_{m1}\vect{w}_m, \\
    T(\vect{v}_2)
    &=& a_{12}\vect{w}_1 + a_{22}\vect{w}_2 + \ldots + a_{m2}\vect{w}_m, \\
    &\cdots& \\
    T(\vect{v}_n)
    &=& a_{1n}\vect{w}_1 + a_{2n}\vect{w}_2 + \ldots + a_{mn}\vect{w}_m.
  \end{eqnarray*}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{cccc}
      a_{11} & a_{12} & \cdots & a_{1n} \\
      a_{21} & a_{22} & \cdots & a_{2n} \\
      \vdots & \vdots & \ddots & \vdots \\
      a_{m1} & a_{m2} & \cdots & a_{mn} \\
    \end{mymatrix}.
  \end{equation*}
  Then $A$ is an $m\times n$-matrix. We must prove that it has the
  desired property, i.e., that
  $A\coord{\vect{v}}_B = \coord{T\vect{v}}_C$, for all
  $\vect{v}\in V$. Since both the left-hand side and the right-hand
  side are linear functions of $\vect{v}$, it suffices to check that
  this property holds for basis vectors. Consider, therefore, one of
  the basis vectors $\vect{v}_i$.  Note that
  $\vect{v}_i = 0\vect{v}_1 + 0\vect{v}_2 + \ldots + 1\vect{v}_i +
  \ldots + 0\vect{v}_n$. Therefore, the coordinates of $\vect{v}_i$
  with respect to the basis $B$ are
  \begin{equation*}
    \coord{\vect{v}_i}_B =
    \begin{mysmallmatrix}{c}
      0 \\ \vdots \\ 1 \\ \vdots \\ 0
    \end{mysmallmatrix}
    = \vect{e}_i,
  \end{equation*}
  where $\vect{e}_i$ is the usual $i\th$ standard basis vector.  On
  the other hand, since
  $T(\vect{v}_i) = a_{1i}\vect{w}_1 + \ldots + a_{mi}\vect{w}_m$,
  we have
  \begin{equation*}
    \coord{T(\vect{v}_i)}_C =
    \begin{mysmallmatrix}{c}
      a_{1i} \\ \vdots \\ a_{mi}
    \end{mysmallmatrix}
    = A\vect{e}_i.
  \end{equation*}
  Here, in the last equation, we have used the fact that $A\vect{e}_i$
  is the same thing as the $i\th$ column of $A$. We therefore have
  $\coord{T(\vect{v}_i)}_C = A\vect{e}_i = A \coord{\vect{v}_i}_B$, as
  desired.
\end{proof}

\begin{definition}{The matrix of a linear transformation}{matrix-of-linear-transformation-vector-space}
  The matrix $A$ of
  Proposition~\ref{prop:matrix-of-linear-transformation-vector-space} is called the
  \textbf{matrix of the linear transformation\/ $T$ with respect to
    the bases $B$ and $C$}%
  \index{linear transformation!matrix of!w.r.t. basis}%
  \index{matrix!of a linear transformation!w.r.t. basis}.  We also
  write
  \begin{equation*}
    A = \coord{T}_{C,B}.
  \end{equation*}
  Therefore,
  \begin{equation*}
    \coord{T}_{C,B}\coord{\vect{v}}_B = \coord{T\vect{v}}_C
  \end{equation*}
  for all $\vect{v}\in V$.
\end{definition}

\begin{example}{Finding the matrix of a linear transformation}{matrix-of-linear-transformation-vector-space}
  Find the matrix of the derivative operator $D:\Poly_3\to\Poly_2$
  with respect to the basis $B=\set{1,x,x^2,x^3}$ of $\Poly_3$ and the
  basis $C=\set{1,x,x^2}$ of $\Poly_2$.
\end{example}

\begin{solution}
  We first find the images of each basis vector of the basis $B$, and
  we write each of them as a linear combination of basis vectors from
  the basis $C$. Let us denote the basis vectors of $B$ as
  $\vect{v}_1 = 1$, $\vect{v}_2 = x$, $\vect{v}_3 = x^2$, and
  $\vect{v}_4=x^3$, and the basis vectors of $C$ as $\vect{w}_1 = 1$,
  $\vect{w}_2 = x$, and $\vect{w}_3 = x^2$. We have
  \begin{equation*}
    \begin{array}{rclclclcl}
      D(\vect{v}_1) &=& D(1) &=& 0 &=& 0 + 0x + 0x^2 &=& 0\vect{w}_1 + 0\vect{w}_2 + 0\vect{w}_3, \\
      D(\vect{v}_2) &=& D(x) &=& 1 &=& 1 + 0x + 0x^2 &=& 1\vect{w}_1 + 0\vect{w}_2 + 0\vect{w}_3, \\
      D(\vect{v}_3) &=& D(x^2) &=& 2x &=& 0 + 2x + 0x^2 &=& 0\vect{w}_1 + 2\vect{w}_2 + 0\vect{w}_3, \\
      D(\vect{v}_4) &=& D(x^3) &=& 3x^2 &=& 0 + 0x + 3x^2 &=& 0\vect{w}_1 + 0\vect{w}_2 + 3\vect{w}_3. \\
    \end{array}
  \end{equation*}
  Therefore, we have
  \begin{equation*}
    A=\coord{D}_{C,B} =
    \begin{mymatrix}{rrrr}
      0 & 1 & 0 & 0 \\
      0 & 0 & 2 & 0 \\
      0 & 0 & 0 & 3 \\
    \end{mymatrix}.
  \end{equation*}
\end{solution}

\begin{example}{Same linear transformation, different bases}{matrix-of-linear-transformation-vector-space2}
  Find the matrix of the derivative operator $D:\Poly_3\to\Poly_2$
  with respect to the basis $B'=\set{1,x+1,x^2+x+1,x^3+x^2+x+1}$ of
  $\Poly_3$ and the basis $C'=\set{1,x-1,x^2-1}$ of $\Poly_2$.
\end{example}

\begin{solution}
  This is the same linear transformation as in the previous example,
  but we are given different bases. Let us denote the basis vectors of
  $B'$ as $\vect{v}_1 = 1$, $\vect{v}_2 = x+1$, $\vect{v}_3 = x^2+x+1$, and
  $\vect{v}_4=x^3+x^2+x+1$, and the basis vectors of $C'$ as $\vect{w}_1 = 1$,
  $\vect{w}_2 = x-1$, and $\vect{w}_3 = x^2-1$. We must write each
  $D(\vect{v}_i)$ as a linear combination of $\vect{w}_1$,
  $\vect{w}_2$, and $\vect{w}_3$, which requires solving a system of
  linear equations for each of them. We have:
  \begin{equation*}
    \begin{array}{rclclcl}
      D(\vect{v}_1) &=& D(1) &=& 0 &=& 0\vect{w}_1 + 0\vect{w}_2 + 0\vect{w}_3, \\
      D(\vect{v}_2) &=& D(x+1) &=& 1 &=& 1\vect{w}_1 + 0\vect{w}_2 + 0\vect{w}_3, \\
      D(\vect{v}_3) &=& D(x^2+x+1) &=& 2x+1 &=& 3\vect{w}_1 + 2\vect{w}_2 + 0\vect{w}_3, \\
      D(\vect{v}_4) &=& D(x^3+x^2+x+1) &=& 3x^2+2x+1 &=& 6\vect{w}_1 + 2\vect{w}_2 + 3\vect{w}_3. \\
    \end{array}
  \end{equation*}
  Therefore, the matrix is
  \begin{equation*}
    \coord{D}_{C',B'} =
    \begin{mymatrix}{rrrr}
      0 & 1 & 3 & 6 \\
      0 & 0 & 2 & 2 \\
      0 & 0 & 0 & 3 \\
    \end{mymatrix}.
  \end{equation*}
\end{solution}

The last two examples illustrate that a linear transformation can have
many different matrices, because the matrix depends not only on the
linear transformation, but also on the given bases. The art of linear
algebra often lies in choosing ``convenient'' bases for a given
application. Often, a ``convenient'' basis is one that gives rise to
simple matrices, for example, matrices containing many zeros, or
matrices that are diagonal.

\begin{example}{Finding a convenient basis}{matrix-of-linear-transformation-vector-space3}
  Let $T:\Poly_3\to\Mat_{2,2}$ be the linear transformation
  defined by
  \begin{equation*}
    T(ax^3+bx^2+cx+d)=
    \begin{mymatrix}{cc} a+d & b-c \\ b+c & a-d \end{mymatrix}
  \end{equation*}
  for all $ax^3+bx^2+cx+d\in\Poly_3$. Let
  $B=\set{x^3, x^2, x, 1}$ and
  \begin{equation*}
    C=\set{
      \begin{mymatrix}{cc} 1 & 0 \\ 0 & 0 \end{mymatrix},
      \begin{mymatrix}{cc} 0 & 1 \\ 0 & 0 \end{mymatrix},
      \begin{mymatrix}{cc} 0 & 0 \\ 1 & 0 \end{mymatrix},
      \begin{mymatrix}{cc} 0 & 0 \\ 0 & 1 \end{mymatrix}
    }
  \end{equation*}
  be bases of $\Poly_3$ and $\Mat_{2,2}$, respectively.
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item Find $\coord{T}_{C,B}$.
    \item Find a basis $C'$ of $\Mat_{2,2}$ such that
      $\coord{T}_{C',B}$ is the identity matrix.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{example}

\begin{solution}
  (a) We have
  \begin{equation*}
    \begin{array}{rcccl}
      T(x^3)
      &=& \begin{mymatrix}{rr} 1 & 0 \\ 0 & 1 \end{mymatrix}
      &=&
      1 \begin{mymatrix}{cc} 1 & 0 \\ 0 & 0 \end{mymatrix}
      + 0 \begin{mymatrix}{cc} 0 & 1 \\ 0 & 0 \end{mymatrix}
      + 0 \begin{mymatrix}{cc} 0 & 0 \\ 1 & 0 \end{mymatrix}
      + 1 \begin{mymatrix}{cc} 0 & 0 \\ 0 & 1 \end{mymatrix}, \\\\[-2ex]
      T(x^2)
      &=& \begin{mymatrix}{rr} 0 & 1 \\ 1 & 0 \end{mymatrix}
      &=&
      0 \begin{mymatrix}{cc} 1 & 0 \\ 0 & 0 \end{mymatrix}
      + 1 \begin{mymatrix}{cc} 0 & 1 \\ 0 & 0 \end{mymatrix}
      + 1 \begin{mymatrix}{cc} 0 & 0 \\ 1 & 0 \end{mymatrix}
      + 0 \begin{mymatrix}{cc} 0 & 0 \\ 0 & 1 \end{mymatrix}, \\\\[-2ex]
      T(x)
      &=& \begin{mymatrix}{rr} 0 & -1 \\ 1 & 0 \end{mymatrix}
      &=&
      0 \begin{mymatrix}{cc} 1 & 0 \\ 0 & 0 \end{mymatrix}
      - 1 \begin{mymatrix}{cc} 0 & 1 \\ 0 & 0 \end{mymatrix}
      + 1 \begin{mymatrix}{cc} 0 & 0 \\ 1 & 0 \end{mymatrix}
      + 0 \begin{mymatrix}{cc} 0 & 0 \\ 0 & 1 \end{mymatrix}, \\\\[-2ex]
      T(1)
      &=& \begin{mymatrix}{rr} 1 & 0 \\ 0 & -1 \end{mymatrix}
      &=&
      1 \begin{mymatrix}{cc} 1 & 0 \\ 0 & 0 \end{mymatrix}
      + 0 \begin{mymatrix}{cc} 0 & 1 \\ 0 & 0 \end{mymatrix}
      + 0 \begin{mymatrix}{cc} 0 & 0 \\ 1 & 0 \end{mymatrix}
      - 1 \begin{mymatrix}{cc} 0 & 0 \\ 0 & 1 \end{mymatrix}.
    \end{array}
  \end{equation*}
  Therefore,
  \begin{equation*}
    \coord{T}_{C,B}
    =
    \begin{mymatrix}{rrrr}
      1 & 0 &  0 &  1 \\
      0 & 1 & -1 &  0 \\
      0 & 1 &  1 &  0 \\
      1 & 0 &  0 & -1 \\
    \end{mymatrix}.
  \end{equation*}

  \noindent (b) Since the matrices $T(x^3)$, $T(x^2)$, $T(x)$, and $T(1)$
  form a basis of $\Mat_{2,2}$, we can take $C'$ to consist of
  these four matrices, i.e.,
  \begin{equation*}
    C' = \set{
      \begin{mymatrix}{cc} 1 &  0 \\ 0 &  1 \end{mymatrix},
      \begin{mymatrix}{cc} 0 &  1 \\ 1 &  0 \end{mymatrix},
      \begin{mymatrix}{cc} 0 & -1 \\ 1 &  0 \end{mymatrix},
      \begin{mymatrix}{cc} 1 &  0 \\ 0 & -1 \end{mymatrix}
    }.
  \end{equation*}
  Then
  \begin{equation*}
    \coord{T}_{C',B}
    =
    \begin{mymatrix}{rrrr}
      1 & 0 &  0 &  0 \\
      0 & 1 &  0 &  0 \\
      0 & 0 &  1 &  0 \\
      0 & 0 &  0 &  1 \\
    \end{mymatrix}
  \end{equation*}
  is the identity matrix.
\end{solution}

We end this section with some properties of matrices of linear
transformations.

\begin{proposition}{Properties of matrices of linear transformations.}{matrix-of-linear-transformation-properties}
  Let $V$, $W$, and $U$ be finite-dimensional vector spaces with
  respective bases $B$, $C$, and $D$. Suppose $T,T':V\to W$ and
  $S:W\to U$ are linear transformations. The following hold:
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item $\coord{S\circ T}_{D,B} = \coord{S}_{D,C}\coord{T}_{C,B}$.
    \item $\coord{1_V}_{B,B} = I$.
    \item $T$ is invertible if and only if $\coord{T}_{C,B}$ is
      invertible, and in that case,
      $\coord{T^{-1}}_{B,C} = (\coord{T}_{C,B})^{-1}$.
    \item $\coord{0}_{C,B} = 0$.
    \item $\coord{T+T'}_{C,B} = \coord{T}_{C,B} + \coord{T'}_{C,B}$.
    \item $\coord{kT}_{C,B} = k\coord{T}_{C,B}$.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{proposition}


\section{Common Families of Distributions}

\subsection{Exercise 1}
\begin{align*}
        \mathrm{E} [X] &= \sum_{i = N_0}^{N_1} \frac{i}{N_1 - N_0 + 1} = \frac{1}{N_1 - N_0 + 1} (\sum_{i = 1}^{N_1} i - \sum_{i = 1}^{N_0 - 1} i) = \frac{N_1 + N_0}{2} \\
        \mathrm{E} [X^2] &= \frac{1}{N_1 - N_0 + 1} (\sum_{i = 1}^{N_1} i^2 - \sum_{i = 1}^{N_0 - 1} i^2) = \frac{2N_0^2 + 2N_0N_1 - N_0 +2N_1^2 + N_1}{6} \\
        \mathrm{Var} [X] &= \mathrm{E} [X^2] - \mathrm{E} [X]^2 = \frac{(N_1 - N_0 + 1)^2 - 1}{12}
\end{align*}

\subsection{Exercise 2}
(a) Suppose the number of defective parts in the lot is $D$. The likelihood that we accept a sample from such
a lot is $P(X = 0)$, where $X$ is distributed according to $\text{Hypergeom}(100, D, K)$.
Given that a lot is unacceptable if $D > 5$, we need only consider the case where $D = 6$, since larger $D$
lead to lower likelihood of being incorrectly accepted. Thus, we solve
\begin{align*}
        \frac{\binom{94}{K}}{\binom{100}{K}} < 0.1
\end{align*}
by running the following Python code:
\begin{lstlisting}[language=python]
  from scipy import special

  K = 1
  while special.binom(94, K) / special.binom(100, K) > 0.1:
    K += 1
\end{lstlisting}
to get that $K = 32$.

(b) This is the same as part (a) except that we now have to consider $P(X = 0) + P(X = 1) < 0.1$. Slightly
modifying the Python code above yields $K = 51$.

\subsection{Exercise 3}
At each second, the probability that at least one car passes in the next 3 seconds is $1 - P(X = 0)$, where
$X$ is binomial with $n = 3$ and $p = p$. Thus, the probability that the pedestrian has to wait 4 seconds
before starting to cross is the probability that at least one car passes in the first three seconds multiplied
by the probabilities that a car crosses at the 4th second and that no cars cross during seconds 5-7. This
probability is $(1 - (1 - p)^3) p (1 - p)^3$.

\textbf{NOTE:} I compared my answer to the actual solutions, but I don't buy that the actual
answer is $(1 - p (1 - p)^3) (1 - p)^3$ based on the logic provided. What if cars pass on the first and second
seconds, but then no cars pass after? The pedestrian could then leave after two seconds.

\subsection{Exercise 4}
(a) If unsuccessful keys are not eliminated, then this is the same as asking for the expectation of a
geometric random variable with $p = \frac{1}{n}$, so we expect that it will take $n$ trials.

(b) If we eliminate unsuccessful keys, then our expectation looks like
\begin{align*}
        \mathrm{E} [X] &= \frac{1}{n} + 2 * \frac{n - 1}{n} * \frac{1}{n - 1} + ... \\
                       &= \frac{1}{n} \sum_{i = 1}^n i = \frac{n + 1}{2}
\end{align*}

\subsection{Exercise 5}
The standard drug's effectiveness can be modeled as a binomial distribution with $n = 100$ and $p = 0.8$, since
we are given that the drug is expected to be effective in 80 out of 100 cases. Computing $P(X \geq 85)$ yields
a value of approximately 0.129, which means that it is fairly unlikely for the new drug to work in 85
(or more) cases if it was worse than the first drug. Thus, we conclude that the new drug is superior.

\subsection{Exercise 7}
We would like to have $P(X \geq 2) = 1 - P(X < 2) > 0.99$. Thus, we can solve
$e^{-\lambda} (1 + \lambda) < 0.01$ to get that $\lambda > 6.638$.

\subsection{Exercise 12}
I tried to do this analytically but struggled to show equivalence between the summations; obtaining the result
through qualitative reasoning is much simpler. $F_X(r - 1)$ is the probability that there are at most $r - 1$
successes in $n$ Bernoulli trials, which is the same as saying that we get success number $r$ after $n$ trials.
Thus, we must have at least $n + 1 - r$ failures, which is $1 - F_Y(n - r)$ (I don't deserve credit for this,
I looked up part of the solution here - maybe I'll retry the analytic approach later).

\subsection{Exercise 15}
I spent an absurd amount of time on this question because I was working off of Wikipedia's provided negative
binomial MGF, which treats $p$ as $1 - p$. After finding the proper MGF, the result is straightforward:
\begin{align*}
        \lim_{r \to \infty} \bigg(\frac{p}{1 - (1 - p)e^t}\bigg)^r &=
        \lim_{r \to \infty}\bigg(1 + \frac{p}{1 - (1 - p)e^t} - \frac{1 - (1 - p)e^t}{1 - (1 - p)e^t}\bigg)^r \\
                                                                   &= \lim_{r \to \infty} \bigg(1 + \frac{r\frac{-(1 - p) + (1 - p)e^t}{1 - (1 - p)e^t}}{r}\bigg)^r \\
                                                                   &= \exp(-\lambda + \lambda e^t)
\end{align*}

\subsection{Exercise 19}
Since $\alpha = 1, 2, 3, ...$ we have $\Gamma(\alpha) = (\alpha - 1)!$. We can then integrate by parts with
$u = \frac{1}{(\alpha - 1)!} z^{\alpha - 1}$ and $dv = e^{-z}$ to get
\begin{align*}
        \int_{x}^{\infty} \frac{1}{(\alpha - 1)!} z^{\alpha - 1} e^{-z} dz &= \frac{x^{\alpha - 1} e^{-x}}{(\alpha - 1)!} + \int_{x}^{\infty} \frac{1}{(\alpha - 2)!} z^{\alpha - 2} e^{-z} dz \\
                                                                           &= \sum_{y = 0}^{\alpha - 1} \frac{x^y e^{-x}}{y!}
\end{align*}
where the last line follows from repeatedly integrating by parts until $z^{\alpha - k} = 0$. Probabilistically,
this equality gives that $P(X \geq x) = P(Y \leq \alpha - 1)$ where $X$ is Gamma distributed with
 $\alpha = \alpha, \beta = 1$ and $Y$ is Poisson with $\lambda = x$.

\subsection{Exercise 44}
Directly applying Chebychev to $P(\abs{X} \geq b)$ gives $P(\abs{X} \geq b) \leq \frac{\mathrm{E}[\abs{X}]}{b}$.
Furthermore, since $P(\abs{X} \geq b) = P(X^2 \geq b^2)$, we can apply Chebychev again to get
$P(\abs{X} \geq b) \leq \frac{\mathrm{E} [X^2]}{b^2}$. Both $\mathrm{E} [\abs{X}]$ and $\mathrm{E} [X^2]$
can be computed via integration by parts as 1 and 2 respectively. Thus, for $b = 3$ the $X^2$ bound is
better and for  $b = \sqrt{2}$ the $\abs{X}$ bound is better.

\subsection{Exercise 45}
(a)
\begin{align*}
        M_X(t) &= \int_{-\infty}^{\infty} e^{tx} f_X(x) dx \\
               &\geq e^{at} \int_{x \geq a} f_X(x) dx \quad \text{since } e^{xt} \geq e^{at} \text{ for } x \geq a, t \geq 0 \\
               &\geq e^{at} P(X \geq a)
\end{align*}

(b) Identical to (a), except we use  $x \leq a$ in step 2 since $t < 0$.

(c) We just need $h(t, x)$ to be nonnegative and $h(t, x) \geq 1$ for all $x, t \geq 0$, since then we can
replicate the steps of (a).


\documentclass[12pt, letterpaper]{article}
\usepackage[fleqn]{amsmath}
\begin{document}
\section{Equations of motion}
\begin{equation} m \dot{v} + bv = F(t) \end{equation}
\begin{equation} \dot{x} = v \end{equation}

\section{Define state variables}
\begin{equation} x_{1} = x \end{equation}
\begin{equation} x_{2} = \dot{x} = v \end{equation}
\begin{equation} \dot{x_{1}} = x_{2} \end{equation}
\begin{equation} \dot{x_{2}} = \dot{v} = \frac{F(t)}{m} - \frac{bx_{2}}{m} \end{equation}
\begin{equation} y = x_{1} \end{equation}
\begin{equation} u = F(t)\end{equation}

\section{State space form}
\begin{equation}
\begin{bmatrix} \dot{x_{1}} \\ \dot{x_{2}} \end{bmatrix}
=
\begin{bmatrix} 0 & 1 \\ 0 & -\frac{b}{m} \end{bmatrix}
\begin{bmatrix} x_{1}\\ x_{2} \end{bmatrix}
+
\begin{bmatrix} 0\\ \frac{1}{m} \end{bmatrix}u
\end{equation}

\begin{equation} y = \begin{bmatrix} 1 & 0 \end{bmatrix} \begin{bmatrix} x_{1} \\ x_{2} \end{bmatrix} + \begin{bmatrix} 0 \end{bmatrix}u
\end{equation}

\end{document}


\chapter{THE LEXIMIN METHOD} \label{ch:algorithm}% Must have a blank line after every section label

\epigraph{Young man, in mathematics you don't understand things. You just get used to them.}{John von Neumann}

\noindent In this chapter, I describe the leximin method for hierarchical community detection. It relates a common problem in traffic engineering to the problem of community detection and finds communities in a divisive, hierarchical way. By maximizing all-pairs concurrent flow on the graph in an iterative process, edges become saturated with flow. At each level, these edges separate the graph into two or more parts. Flat clusters can be extracted based on some objective function; we select the clustering which maximizes modularity.

I define \say{leximin} and explain its connection to this problem. Next, I explain how the method identifies sparsest cuts to form a community structure. After defining the method, I illustrate the differences from the related MCF cut algorithm~\cite{mann2008extensions}. Finally, I give the computational complexity of the method as essentially $O(N^{11})$ in terms of the number of nodes $N$ and provide remarks to guide a practical implementation.

\section{Introduction}

The leximin method works to find cuts in the graph that separate it into well-connected subgraphs. The intuition of the method is a progressive filling process on a network of pipes~\cite{le2005rate}.

\begin{method}{Progressive Filling~\cite{le2005rate}}

Begin with all rates equal to zero. Grow all rates at the same pace (or at a pace proportional to demand, when demand is not unity). Continue until one or more edges is saturated. These are the bottlenecks, or critical edges. The rates for demand pairs that use these links are now fixed: They do not increase anymore. Rates between other demand pairs continues to increase until a new bottleneck is found, and so on until the entire graph is saturated---all demand pairs have found a bottleneck. The process terminates because all edge capacities are finite and the graph is finite. The resulting allocation is max-min fair and unique~\cite{le2005rate}.
\end{method}

Matula first introduced the leximin method for community detection in 1985~\cite{matula1985divisive}. It rests on a hierarchical version of the maximum concurrent flow problem (MCFP) that was independently rediscovered nearly twenty years later by Nace and Doan in 2003~\cite{nace2003some}. In 2004, Allalouf and Shavitt made explicit the MCFP with non-unit demand which Matula identified in 1985 and Nace and Doan say is \say{easily generalized} from their model~\cite{allalouf2004maximum}. Much work has also built on Matula's MCFP, examining the simpler case that flow follows a fixed path or set of paths~\cite{pioro2003efficient, nace2003some, nace2006tutorial}.

The MCFP finds a fair allocation of flow between all pairs---that is, an allocation that is maximin maximal. When maximizing throughput between all pairs, a set of \emph{critical edges} will become fully saturated, corresponding to the idea of \emph{breakdown} in traffic theory: \say{the onset of congested traffic in an initial free traffic flow}~\cite{kerner2009traffic}. Because of these bottlenecks, there is residual capacity that can be utilized. The hierarchical MCFP (sometimes \say{lexicographically maximum concurrent flow problem}) utilizes that residual capacity to produce a max-min fair allocation---a leximin maximal satisfaction of demand.

The remainder of this chapter is structured as follows. In \autoref{sec:leximin_meaning}, we define \emph{leximin} as it pertains to flow networks. In \autoref{sec:sparsest cut}, we define sparsest cuts and show how they create a canonical hierarchical decomposition. In \autoref{sec:leximin algorithm}, we discuss the hierarchical maximum concurrent flow problem and how it is used to determine sparsest cuts and grids. \autoref{sec:differences} outlines the differences between the leximin method and the related MCF cut algorithm. Finally, in \autoref{sec:complexity and size}, we opine on the complexity, implementation, and practical limits of the algorithm.

\section{Leximin fairness} \label{sec:leximin_meaning}

The leximin method is so named because it identifies a lexicographic maximin (i.e.\ leximin) satisfaction of demand. A \emph{maximin} process is one that \emph{max}imizes the \emph{min}imum utility. For our purposes, utility and throughput are synonymous. A maximin flow allocation is one that has the highest minimum throughput. This is what the MCFP identifies. \emph{Lexicographic ordering} is a generalization of familiar alphabetic ordering: words are ordered by their first position, then their second, and so on. Lexicographic maximin extends the idea of maximin to multiple levels: we maximize the minimum utility, then for those who are not yet constrained, we maximize \emph{their} minimum utility, and so on. Economists have adopted the shorthand \say{leximin} because it is more economical in its number of syllables.

The leximin satisfaction of demand has a desirable property in traffic networks because it enforces \say{max-min fairness}. Bertsekas et al.~\cite{bertsekas1992data} discuss the idea of max-min fairness: that no pair's allocation can increase without reducing allocation to a pair with equal or lesser allocation. More casually, no-one can be made better off without taking from someone as well off or worse off. This is the same idea as leximin, and it is often an objective in bandwidth allocation for computer networks. Solving the hierarchical MCFP is a polynomial-time method for identifying such an allocation.

\section{The Sparsest Cut} \label{sec:sparsest cut}

On an unweighted graph (one with unit capacity on each edge), the sparsest cut is the minimal set of edges, with respect to the potential edges that could exist between the subgraphs it separates. In the case that an edge set~$\hat{E}$ cuts a graph into two parts~$A$ and~$\overline{A}$, the cut density is:

$$\frac{|\hat{E}|}{|A| \cdot |\overline{A}|}$$

The sparsest cut is the edge set~$\hat{E}$ that minimizes this value. Sometimes multiple cuts exist at a single level of throughput. Beyond the two-part case, the reader is referred to Mann~\cite{mann2008extensions, mann2008sparsest}. The edges that are not in $\hat{E}$ are non-critical. As there exists residual capacity on these edges after saturating the sparsest cut, the level of throughput can be raised on these edges. The pipes can keep being filled.

\subsection{The Sparsest Cut Hierarchy}

Beginning from the level of flow that saturated our sparsest cut, we keep filling the pipes, raising throughput. All the water already in the pipes is still there, eating up capacity. We approach a next-level bottleneck, and a next, until the entire network is partitioned into single nodes. Importantly, solving the hierarchical MCFP yields a unique dendrogram. The throughput continues to increase for each subproblem. Since the set of critical edges is canonical at every level, the dendrogram is canonical. The complication is in the case of ties, which cause \emph{degenerate cuts}: cuts at the same throughput value.

While a sparsest cut at a given throughput level~$z_i$ is not always unique, it is unique as long as there is no \emph{tie}, which is discussed in \autoref{ch:robustness}. When a tie occurs, the set of sparsest cuts is unique, and they will each be identified at the same throughput level~$z_i$. This guarantees a unique dendrogram, even in the face of ties.

In the following section, we discuss how the leximin method uses the hierarchical MCFP to find sparsest cuts and yield the dendrogram.

\section{The Leximin Method for Community Detection} \label{sec:leximin algorithm}

This algorithm identifies communities by identifying sparsest cuts in the graph. To find hierarchical structure, this can be repeated until the graph is decomposed into singletons. A flat structure can be extracted that maximizes some measure of community quality, such as stability of clusters~\cite{mcinnes2017hdbscan} or modularity~\cite{newman2006modularity}. Some of these measures allow early stopping, rather than generating the entire top-down structure. Identifying sparsest cuts is NP-hard~\cite{shahrokhi1990maximum, matula1990sparsest}, but the maximum concurrent flow problem (MCFP) can identify a class of sparsest cuts (\say{bottlenecks}) in polynomial time~\cite{matula1990sparsest}.

\subsection{The MCFP and hierarchical MCFP}

The MCFP is a multicommodity flow problem with demand between all pairs of nodes in the network~\cite{matula1985concurrent}. The MCFP maximizes \emph{concurrent} flow: flow is supplied between all pairs in proportion to the demand between them. With unit demand, this means that flow between all pairs must be equal. The ratio of supplied flow to demand is called \emph{throughput} or \emph{satisfaction ratio}, and it must be the same between all pairs. The MCFP maximizes this throughput under the constraints of capacity on the edges.

A problem instance is represented by the tuple $\left< G, c, d \right>$ where $G = (V, E)$ is a graph, $c$ is a function $c\colon \{i, j\}  \rightarrow c_e \  \forall \{i, j\} \in E$ which provides the capacity of each edge, and $d$ is a function $d\colon \{i, j\}  \rightarrow d_{ij} \  \forall \{i, j\} \in D$ which defines the demand between all pairs. ($D \subseteq V \times V$~is the set of node pairs with demand between them.) The standard formulation assumes unit capacity on each edge and unit demand between all pairs.

There exist multiple possible LP representations of the MCFP. While more compact, efficient methods have emerged, the edge--path formulation is the conceptually simplest:
\begin{align}
    \max z(G, c, d)  & && \text{throughput} \nonumber\\
    \mathrm{s.t.} \sum_{p \in P_{ij}} f_p &= zd_{ij} \forall \{i, j\} \in D && \text{fairness constraint} \nonumber\\
    \sum_{\{p \in P:\{i,j\} \in E_p\}} f_p &\leq c_{ij} \forall \{i, j\} \in E && \text{capacity constraint} \nonumber\\
    f_p &\geq 0,  p \in P && \text{non-negativity of flow} \label{eq:mcfp}
\end{align}
We seek to maximize the concurrent throughput~$z$. Our first constraint enforces proportional maximization: the proportion of demand satisfied is the same for all pairs. The throughput~$z$ is  the sum of flows~$f_p$ for every path~$p$ in the set of paths~$P_{i, j}$ between nodes~$i$ and~$j$ (i.e. the total satisfied demand between~$i$ and~$j$).

\subsection{Formulation of the Leximin Method by Linear Programming}

In the hierarchical form, we maximize not just concurrent flow, but lexicographic concurrent flow. Residual capacity is not wasted. Instead, we use it to raise throughput for demand pairs that have not been constrained. We do this by solving a new LP for each throughput level, depending on the previous LPs:

\begin{align}
    \max z_n(G, c, d)  & && \text{throughput} \nonumber\\
    \mathrm{s.t.} \sum_{p \in P_{ij}} f_p &= z d_{ij} \forall \{i, j\} \in D_n && \text{fairness constraint} \nonumber\\
    \mathrm{s.t.} \sum_{p \in P_{ij}} f_p &= z_l d_{ij} \forall \{i, j\} \in D_l, l = 1,\ldots,D_{n-1} && \text{lower levels constraint} \nonumber\\
%    \sum_{\{p \in P:\{i,j\} \in E_p\}} f_p &\leq c_{ij} \forall \{i, j\} \in E \setminus \mathit{crit}_n && \text{free capacity constraint} \nonumber\\
    \sum_{\{p \in P:\{i,j\} \in E_p\}} f_p &\leq c_{ij} \forall \{i, j\} \in E && \text{capacity constraint} \nonumber\\
    f_p &\geq 0,  p \in P && \text{non-negativity of flow} \label{eq:mcfp}
\end{align}

%\todo{Please check the formula before I send it to the committee.}
Here, $z_l$ is a constant: the throughput of the $l$th subproblem, which was identified previously. $D_n$ identifies the subset of demand pairs that has not yet been saturated in subproblem $n$, and $D_l$ is the set of edges saturated in subproblem $l$. Some capacity has been utilized to achieve the previous level of throughput. Discounting that from the network's capacity \say{leaves the water in the pipes}, so we find a next solution building from there. We also no longer seek to further maximize demand between pairs that have already been constrained by a bottleneck. Because there is a bottleneck separating those pairs, they could not be maximized anyway.

\subsection{Determining the Sparsest Cut from the Hierarchical MCFP}

There is a unique set of critical edges saturated to capacity by every optimal assignment of flows, and removing these critical edges partitions the graph into two or more components~\cite{matula1985concurrent}. The remaining edges are non-critical, and their residual capacity can be used to improve throughput for the remainder of the graph. This creates a new set of critical edges and a new division.

Sometimes, the entire edge set of a subgraph bounded by cuts is saturated. This is a phenomenon called \emph{gridlock}, which is particular to maximizing concurrent flow. The subgraph is sufficiently homogeneous that there is no need to further dissect it and identify further community structure. The gridlock phenomenon is explored in \autoref{ch:random}.

\subsection{Main Steps of the Leximin Method}

The following steps are the iterative procedure to identify hierarchical community structure by identifying a leximin allocation of concurrent flow.

\begin{algorithm}
	\caption{Leximin clustering method}
	\label{alg:hmcfp-cc}
		\KwData{A graph $G = (V, E)$, edge capacities $c$, demand pairs $d$}
		\KwResult{A clustering $\mathcal{C}(G)$}
		$n \longleftarrow 1$\;
		\While{$\exists d \in D$ not saturated} {
		    \textbf{Solve the hierarchical MCFP subproblem} $P_n$\;
		    Extract critical edges~$\hat{E}_n$: those with nonzero shadow prices\;
		    Reduce available capacity on edges by $z$\;
		    $\forall d \in D_n$: $\lambda_d \longleftarrow z$ \;
		    $i \longleftarrow n + 1$\;
		}
		The final assignment of flows is max-min fair, and the satisfied demand is leximin maximal. The critical edges and their corresponding throughputs $z_i$ define a hierarchical structure of cuts.
		\vspace{0.25cm}
\end{algorithm}

The final consideration in constructing the dendrogram is the height at which each community should be merged. This height is called the \emph{cophenetic distance}~\cite{sokal1962comparison}. In this method, arbitrarily broken ties are reconciled by using throughput~$z$ as the cophenetic distance, rather than simply the number of divisions that have occurred to this point. This means that the ties (explored in \autoref{ch:robustness}) are reconciled as a multiway split at the same height, rather than remaining arbitrarily broken.

\section{Differences from the MCF Cut Algorithm} \label{sec:differences}

\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{fig/Paw3}
\caption{The paw graph}
\label{fig:paw3}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{fig/level1}
\caption{The first cut of the paw graph by the leximin algorithm. Green lines indicate paths of flow to fairly maximize minimum throughput. The red line indicates the cut.}
\label{fig:level1}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{fig/MCF_level2}
\caption{The subproblems for the MCF cut algorithm after its first cut: $K_3$ and a singleton}
\label{fig:MCF_level2}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=0.4\textwidth]{fig/dendrogram_4_tie}
\caption{The dendrogram produced by the MCF cut algorithm on the paw graph}
\label{fig:MCF_dendrogram}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{fig/level2}
\caption{The second cut of the paw graph by the leximin algorithm. Green lines indicate paths of flow to fairly maximize minimum throughput. Red lines indicate cuts. Lighter colors represent fixed flow and previous cuts.}
\label{fig:level2}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{fig/level3}
\caption{The third and final cut of the paw graph by the leximin method. The decision to cut $\{2, 3\}$ again instead of $\{1, 3\}$ is arbitrary.}
\label{fig:level3}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=0.4\textwidth]{fig/dendrogram_4_cascade}
\caption{The dendrogram produced by the leximin method on the paw graph}
\label{fig:leximin_dendrogram}
\end{figure}

The MCF cut algorithm is a similar technique to the leximin method which takes a divide-and-conquer approach~\cite{mann2008extensions}. It solves the MCFP on the network, then splits the network into components to process separately. This amounts to progressive filling up to the first bottleneck, then blocking the saturated pipes, draining the sections on each side, and starting anew on each side of the bottleneck. The components separated by the cut are re-solved independently of one another. This makes each subproblem smaller, but yields a different sequence of cuts and a different dendrogram.

The difference in behavior of the leximin and MCF Cut methods is best shown on a simple example. The paw graph is a graph formed by adding a fourth node to the complete graph $K_3$ and joining it to one of the existing vertices by an edge, as shown in \autoref{fig:paw3}.

The operation of the two algorithms begins identically; they differ in their treatment of subsequent steps. First, they progressively fill the network until an edge is saturated. Because this is the first iteration, there are no prior capacity usages in the leximin's hierarchical MCFP. This makes it equivalent to the MCFP solved in the MCF cut algorithm. \autoref{fig:level1} shows that the edge $\{3, 4\}$ (the \say{tail} of the paw graph) is saturated first, when there is $1/3$~unit of flow between all pairs. This edge is identified as critical. This is where the behavior deviates.

The MCF cut algorithm separates this graph into two subproblems: a triangle ($K_3$) and a singleton, as shown in \autoref{fig:MCF_level2}. When processing the triangle, we start from an initial state of no flow. Although the edge $\{1, 2\}$ had the least flow on it at the end of the first step, it is on equal footing with the other two edges of the triangle. The dendrogram for this method is then shown in \autoref{fig:MCF_dendrogram}.

In contrast to the MCF cut method, the leximin method does not remove flow already present in the network due to previous iterations. The starting point for the second iteration is shown in \autoref{fig:level2} as the lighter-colored lines. The flow required to saturate a new set of critical edges is shown in darker green. Here, there is an unambiguous next cut. The final edge is cut in \autoref{fig:level3}, which gives the final dendrogram: \autoref{fig:leximin_dendrogram}.

Because there is no clear definition of a community---and much less, a hierarchical communty---it is unclear which of these results is preferable. The leximin method has a clearer real-world significance, though: it partitions the graph based on the load applied to the graph. Further, the leximin method has desirable theoretical properties discussed in \autoref{ch:robustness}.

\section{Complexity and Size Considerations} \label{sec:complexity and size}

%The leximin algorithm is dramatically slower than competing algorithms, which can process networks with tens and hundreds of millions of nodes. The bulk of the mathematical effort  is solving the hierarchical MCFP. Proposals to increase this step's efficiency have been proposed. Dong et al.\ present the \say{triples} formulation of the MCFP, which drastically reduces the number of variables over the edge--path formulation, while also reducing the number of constraints~\cite{dong2015compact}.  Nevertheless, a dense network of 160 nodes can take an hour to process on current hardware.

Solving a linear program is a polynomial-time technique, when inputs are algebraic numbers~\cite{adler1992polynomial}. The fastest known LP method has complexity $$O\left(\frac{n^3}{\log n}L\right)$$ where $n$ is the number of variables in the standard form of the LP and $L$ is the information length of the LP, which grows with the product of the number of variables and constraints~\cite{anstreicher1999linear}. In the efficient \say{triples} formulation of the MCFP, there are $O(mN)$ variables and $O(N^2)$ constraints.

Since real-world networks have sparse connectivity~\cite{chakrabarti2006graph}, we can restrict our focus to these. Even on sparse graphs, solving an LP will take essentially $O(N^{10})$ time relative to the number of nodes in the graph. Given that up to $(N-1)$ LPs will be solved (one for each level of the hierarchy~\cite{nace2003some}), this amounts to a complexity of near $O(N^{11})$. The next slowest options are Walktrap ($O(N^{3.2})$) and Girvan--Newman ($O(N^3)$ on sparse graphs).

\subsection{Practical Considerations for Implementation}
The edge--path form of the MCFP involves an exponential number of variables, which is untenable. Fortunately, more efficient representations have emerged. An implementer should prefer the triples formulation of the MCFP~\cite{dong2015compact}, which is easily extended to the hierarchical case. This method has a polynomial number of variables and constraints, yielding a tractable linear program in the complexity class P\@. Work to expedite handling degenerate cuts has also been done~\cite{danna2012practical}. The details of such a method, as well as its implementation, are out of the scope of this work.

Additionally, the method should only be applied to graphs of up to a few hundred nodes. On a sparse graph of 240 nodes, the runtime is measured in hours.

%\begin{algorithm}
%	\caption{MCF cut algorithm~\cite{mann2008extensions}}
%	\label{alg:mcf_cut}
%		\KwData{A graph $G = (V, E)$, edge capacities $c$, demand pairs $d$}
%		\KwResult{A clustering $\mathcal{C}(G)$}
%		$queue \longleftarrow \{G\}$
%		\While{$queue \neq \emptyset$}{
%			$H \longleftarrow queue.dequeue()$\;
%			\textbf{Solve the MCFP on $H$}\;
%		    Extract critical edges~$\hat{E}_i$: those with nonzero shadow prices \;
%		    \Unless{Something}{
%			\ForEach{component $H_j$ in $H$ cut by $\hat{E}_i$}{
%				$queue.enqueue(H_j)$\;
%			}}
%		}
%		\vspace{0.25cm}
%\end{algorithm}