\documentclass[10pt]{article}
\renewcommand{\baselinestretch}{0.5}

\usepackage{geometry}
\geometry{top=0.5in, left=0.5in, right=0.5in, bottom=0.5in}

\pagenumbering{gobble}

\usepackage{multicol, multirow}

\usepackage{titlesec}
\titleformat*{\section}{\large\bfseries}
\titlespacing{\section}{0em}{0em}{0em}

\usepackage{enumitem}
\setlist{itemsep=0em, leftmargin=!, labelindent=0em, itemindent=-1em}

\usepackage{amsmath}
\usepackage{amssymb}

\begin{document}
\small

\begin{center}
    \Large{\textbf{MATH 407 -- Hasty Reference Sheet -- 2021-12-08}}
\end{center}

\begin{multicols*}{2}

\section*{Combinatorics}
\begin{itemize}
    \item \textbf{Multiplication Principle:} $|\{A, B, C\} \times \{X, Y\}| = 3 \cdot 2 = 6$
    \item \textbf{Addition Principle:} $|\{A, B, C\} \cup \{X, Y\}| = 3 + 2 = 5$
    \item \textbf{Permutations:} $n!$ ways to order $n$ distinct objects \\
    partial permutations ($k$-permutations of $n$): $\frac{n!}{(n - k)!}$ \\
    permutations of multisets (handle repetition): $\frac{n!}{n_1! n_2! \cdots n_r!}$ \\
    ($n_1$ are alike, $n_2$ are alike, and so on)
    \item \textbf{Combinations:} from $n$ choose $k$: $\binom{n}{k} = \frac{n!}{k!(n - k)!}$
    \item \textbf{Binomial Theorem:} $(x + y)^n = \sum_{k=0}^n \binom{n}{k} x^k y^{n-k}$ \\
    {\footnotesize Pascal's triangle! (rows increment $n$, columns increment $k$)}
    \item \textbf{Multinomial Coefficient:} $\binom{n}{n_1,\dots,n_k}$ \\
    {\footnotesize sort $n$ distinct items into groups of size $n_1,\dots,n_k$}
\end{itemize}

\section*{Probability Space}
\begin{itemize}
    \item \textbf{Definition:} \\
    1. universal sample space of all possible outcomes: $\Omega$ \\
    2. event space that is $\sigma$-algebra of subsets of $\Omega$: $\mathcal{F}$ \\
    3. prob. function satisfying Kolmogorov Axioms: $P(E)$
    \item \textbf{Kolmogorov Axioms:} \\
    1. $P: \mathcal{F} \rightarrow \mathbb{R}$, $P(E) \geq 0$ \\
    2. $P(\Omega) = 1$ (``something always happens'') \\
    3. $\sigma$-additivity ($P(E_1 \cup E_2) = P(E_1) + P(E_2)$ if disjoint)
    \item \textbf{Notes:} \\
    - events are subsets of $Omega$, elements of $\mathcal{F}$ \\
    - $P(E) = 1 - P(E^\complement)$ \\
    - $P(\varnothing) = 0$ (``nothing never happens'') \\
    - $P(E_1 \cup E_2) = P(E_1) + P(E_2) - P(E_1 \cap E_2$)
\end{itemize}

\section*{Conditional Probability, Independence}
\begin{itemize}
    \item \textbf{Definition:} $P(A|B) = \frac{P(A \cap B)}{P(B)}$ when $P(B) > 0$
    \item \textbf{Notes:} \\
    - $P(A|B)$ is ``probability of $A$ given $B$'' \\
    - $\frac{|A \cap B|}{|B|}$ is for uniformly distributed and finite space
    - Bayesian approach uses conditional probability as axioms (Laplace?)
    \item \textbf{Bayes' Theorem:} $P(A|B)P(B) = (B|A)P(A)$
    {\footnotesize (symmetry!)} \\
    practical form: $P(A|B) = \frac{P(B|A)P(A)}{P(B)}$ \\
    decomposition: $P(A) = P(A|B)P(B) + P(A|B^\complement)P(B^\complement)$
\end{itemize}

\section*{Random Variables}
\begin{itemize}
    \item \textbf{Definition:} a real-valued RV on $\Omega$ is a function $X$ \\
    $X: \Omega \rightarrow \mathbb{R}$ must satisfy measurability \\
    (a RV on $\Omega$ assigns numbers to every outcome in $\Omega$) \\
    discrete when it can take on countably infinitely many different outputs,
    each with non-zero probability
    \item \textbf{Example:} 3d8 is a RV where $\Omega$ consists of all 3-tuples of values in
    $\{1, 2, 3, 4, 5, 6, 7, 8\}$
    \item \textbf{Notes:} \\
    - if you know what specific outcome in $\Omega$ had occurred, you know value of RV \\
    - although RVs are functions, outputs often seen w/o args
    - ``given a range of $X$, what are the outcomes of $\Omega$
    (or, the \textit{event} in $\Omega$)
    that caused $X$ to take the values?'' \\
    - $X^{-1}(S) \in \mathcal{F}$ is the pre-image of $S$ under $X$ \\
    - $3\text{d}8^{-1}(\{4\}) = \{(1, 1, 2), (1, 2, 1), (2, 1, 1)\}$
    \item \textbf{Discrete RVs:} $P(X = x_n) > 0$ for all $n \geq 1$ \\
    and $\sum_n P(X = x_n) = 1$
    \item \textbf{Probability Mass Function:} $p_X(x) = P(X = x)$ \\
    $p_X(x)$ is the probability that the RV $X$ is equal to $x$ \\
    - PMFs ``completely determine'' discrete RVs \\
    - RVs $X$ and $Y$ are equal iff they have the same PMFs
    \item \textbf{Expected Value of a Discrete RV:} $E[X]$ \\
    - expected value, expectation, mean, weighted average \\
    - $\mu = E[X] = \sum_{x\in\mathbb{R}:P(X=x)>0} x \cdot P(X=x)$ \\
    - sum is taken over all real numbers $x$ in the range of $X$ \\
    - {\footnotesize linear: $E[X+Y] = E[X] + E[Y]$, $E[aX] = aE[X]$, $E[a] = a$}
    \item \textbf{Cumulative Distribution Function (Discrete):} $F_X(x)$ \\
    $F_X(x) = \sum_{a\leq x:P(X=a)>0} x \cdot P(X=a)$
    \columnbreak
    \item \textbf{Joint Discrete Random Variables:} function of multiple RVs \\
    for discrete RVs, use joint probability mass functions \\
    $p_{X,Y}(x, y) = P((X=x)\cap(Y=y))$
    \item \textbf{Independence:} $X$ and $Y$ are independent if... \\
    $P((X\in S)\cap(Y\in T)) = P(X\in S) \cdot P(Y\in T)$
    \item \textbf{Bernoulli Random Variable:} $\Omega = \{0, 1\}$ \\
    $P(X=0) = 1 - p$; $P(X=1) = p$ \\
    PMF (Bernoulli distribution) of a Bernoulli RV: \\
    - $p_B(x) = P(X_B = x) = 1-p$ if $x=0$ \\
    - $p_B(x) = P(X_B = x) = p$ if $x=1$ \\
    - $p_B(x) = P(X_B = x) = 0$ if not $x\in\{0, 1\}$ \\
    $Var(X_B) = p(1 - p)$
    \item \textbf{Binomial RV:}
    $P(X=k)=\binom{N}{k}p^k(1-p)^{N-k}$ \\
    in an experiment where outcome is success of failure... \\
    a binomial RV $X$ counts $k$, the number of successes in... \\
    performing the experiment $N$ times given $N \geq 1$ and where successes have probability $p$ \\
    - $E[X] = N \cdot p$ \\
    - $Var(X) = N p (1-p)$
    \item \textbf{Geometric RV:}
    $P(X=k)=p(1-p)^{k-1}$ where $k \geq 1$ \\
    geometric RV represents trials needed for 1 success\\
    - $E[X] = \frac{1}{p}$ \\
    - $Var(X) = \frac{1-p}{p^2}$
    \item \textbf{Poisson RV:}
    $P(X=k)=e^{-\lambda}\cdot\frac{\lambda^k}{k!}$ \\
    models \# of events occurring over set interval when independent or weakly independent \\
    approximates $Binomial(n, p)$ for large $n$s and small $p$s \\
    Poisson approximate of binomial RVs: let $\lambda = np$ \\
    $P(Binomial(n, p)=k)\approx P(Poissoon(np)=k)$ \\
    Poisson paradigm: $n$ events with event $i$ having
    \item \textbf{Variance:} $Var(X) = E[(X-\mu)^2]= E[(X-E[X])^2]$ \\
    $Var(X) = E[X^2] - (E[X])^2$ \\
    - $\sqrt{Var(X)} = \sigma$ (standard deviation) \\
    - not linear \\
    - $Var(aX+b) = a^2Var(X)$ \\
    - $Var(X+Y) = Var(X) + Var(Y)$ only if independent
    \item \textbf{Continuous RV:} $P(X=a)=0$ \\
    - calculus \\
    - $P(a \leq X \leq b) = \int_a^bF'(X)dx$ \\
    - instead of PMF, use derivative of CDF: PDF
    \item \textbf{Normal Distribution:}
    $f(x) = \frac{e^{-\frac{(x-\mu)^2}{2\sigma^2}}}{\sigma\sqrt{2\pi}}$ \\
    - RV is $Normal(\mu, \sigma^2)$ \\
    - standard normal has $\mu=0$, $\sigma=1$ \\
    - CDF of standard normal RV:
    $\Phi(x) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^x e^{-\frac{y^2}{2}}$ \\
    - CDF of normal RV with $\mu$ and SD $\sigma$:
    $F_X(x) = \Phi(\frac{x-\mu}{\sigma})$
    \item \textbf{Chi-Squared Distribution:} $f(x) = \text{no}$ \\
    - $\Gamma = \int_0^\infty x^{s-1}e^{-x}dx$ \\
    - $E[Chi-Squared(N)] = N$ \\
    - $Var(Chi-Squared(N)) = 2N$
    \item \textbf{Jointly Distributed Random Variables:} \\
    - joint CDF: $F_{X,Y}(x,y) = P((X\leq x)\cap(Y\leq y))$ \\
    - joint PDF: $f_{X,Y} = \frac{\partial^2 F_{X,Y}}{\partial x \partial y}$
\end{itemize}

%\newpage

\end{multicols*}

\end{document}


\chapter{Test Appendix}

%
\begin{equation}
a=b+c
\end{equation}
%


\documentclass[10pt, a4paper]{article}

\usepackage[utf8]{inputenc}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{amsfonts}
\usepackage[margin=2.5cm]{geometry}
\usepackage{amsmath}
\usepackage{color}
\usepackage{graphicx}
\newcommand{\my}[1]{{\color{blue} #1 }}
\setlength{\parindent}{0pt}
\floatname{algorithm}{Procedure}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}

\usepackage{tikz}
\usetikzlibrary{shapes,decorations.pathreplacing,calc,shapes.callouts,decorations.pathmorphing}

\usepackage[normalem]{ulem}
\newcommand{\tikzmark}[1]{\tikz[overlay,remember picture] \node (#1) {};}
\definecolor{mygreen}{rgb}{.222,.62,.49}

\title{Comparing the Jacobi Method and LLL lattice reduction algorithms for cryptographic applications}
\date{Fall 2014}
\author{Frederic Jacobs\\ EPFL Bachelor Semester Project\\ frederic.jacobs@epfl.ch}

\renewcommand{\vec}[1]{\mathbf{#1}}

\begin{document}

\maketitle

\section{Introduction}
In 2012, Sanzheng Qiao published a paper~\cite{originalJacobiMethodLatticeBasisReduction} proposing the use of the Jacobi method for lattice reduction applications. In following papers, Qiao proposed several enhancements to the generic Jacobi method. The purpose of their research is to generate a reduction that produces good reduction on random matrices for which the columns represent almost orthogonal vectors. This is in contrast to lattices that are used in cryptography as public keys. From the cryptanalytic point of view, we are interested in random lattices where the given basis vectors are skewed and not almost orthogonal. We implemented the Jacobi method reduction algorithm as presented by Qiao et al. and compared the quality of the outputs with the LLL algorithm for almost orthogonal matrices and random lattices.

\section{Preliminaries}

In this section, we briefly cover the basics of lattices and lattice reduction required in the scope of this report.

\subsection{Euclidean norm}

The Euclidean norm in $\mathbb{R}^n$ is a mapping from $\mathbb{R}^n $ to $\mathbb{R}$ that assigns a length to any vector of the space $\mathbb{R}^n$.

The mapping is defined for any vector $\vec{b}$ of $\mathbb{R}^n$ with coordinates $b_i$ as follows:

\[
\|\mathbf{b}\| = \sqrt{b_1^2 + b_2^2 + ... + b_n^2}
\]

which results from the Pythagorean theorem.

\subsection{Lattice, also called Euclidean lattice}

Informally, a lattice can be defined as the infinite set of intersection points of an infinite regular grid as shown in Fig.~\ref{fig:latti}.
\begin{figure}
\centering

\input{figlat/latticegrid}\hspace{2cm}
\input{figlat/latticeZ}

\caption{Two lattices in two-dimensional space. Left: Parallelogrammic lattice Right: Square lattice}
%wording from wikipedia: http://en.wikipedia.org/wiki/Lattice_(group)
\label{fig:latti}
\end{figure}

This can be translated into a more formal definition~\cite{SchnorrStanfordNotes} as follows:\newline
Let $\vec{b_1}, \vec{b_2}, ... \vec{b_n} \in \mathbb{R}^m$ be linearly independent vectors for $n\leq m$. We call the discrete additive subgroup

\[
L(\vec{b_1}, \vec{b_2},..., \vec{b_n}):= \displaystyle\sum_{i=1}^{n} \vec{b_i} \mathbb{Z}
\] of $\mathbb{R}^m$ a lattice with basis $\vec{b_1,b_2,...,b_n}$.
A lattice that has $\vec{b_1},\vec{b_2},...,\vec{b_n}$ as basis vectors is fully described by this basis which we denote in matrix form as $B$ where each vector $\vec{b_i}$ is a column of the matrix.
The \emph{dimension of the lattice} is $dim(L) := rank(L) := n$.

A lattice $L \subseteq \mathbb{R}^m$ is \emph{full dimensional} when $rank(L)=m$. We will work exclusively with full dimensional lattices. This means that we will work on matrices with $m=n$ linearly independent vectors in $\mathbb{R}^n$.

\subsubsection{Equivalent bases}

As this article focuses on lattice basis reduction, it is important to define the equivalence of two bases. Two bases $B_1 , B_2$ are equivalent iff $B_2 = B_1 U$ for a unimodular matrix $U$.
A \emph{unimodular matrix} $U$ is a square integer matrix such that $det(U) = \pm 1$. $B_1$ and $B_2$ are bases to the same lattice. The right multiplication of a matrix $B_1$ with a unimodular matrix $U$ yields $B_2$ by performing a combination of the following elementary integer column operations:
\begin{itemize}
\item Swap two columns
\item Negate the elements of a column
\item Addition of an integer multiple of one column to another
\end{itemize}

A lattice can be described by infinitely many equivalent bases. The dimension, the length of a shortest vector and the volume, as described next, stay invariant under change of basis.
\subsubsection{Volume}

The volume of a lattice, also known as the determinant of the lattice, is defined as:
\[
\text{vol} \,L := \sqrt{\det(B^T B)}
\] where $B$ is a matrix of the lattice's basis vectors.

The determinant of an unimodular matrix being $\pm 1$, the determinant in the above equation is an invariant under change of basis. It follows that the volume of a lattice, vol$(L)$, is an invariant under change of basis.

Since we will be studying full dimensional lattices, the number of basis vectors will be equal to the dimension of the lattice and hence, the volume of the lattice can be computed by taking the absolute value of the determinant of the basis matrix.
For full dimensional lattices, the volume can be computed:
\[
\text{vol} \,L_{\text{full}} = |\det B|
\] where $B$ is a matrix of the linearly independent lattice basis vectors.

\subsubsection{Hermite Normal Form}

The \emph{Hermite Normal Form}~\cite{SchnorrStanfordNotes} representation of a basis matrix $B$, abbreviated \emph{HNF(B)}, is unique. It is the analogue of the reduced echelon form for integer matrices.

A full-rank matrix $[a_{ij}] \in M_{n,n} (\mathbb{R})$ is in HNF when:
\begin{enumerate}
\item $a_{ij} = 0$ for $j > i$, i.e. $A$ is lower triangular.
\item $a_{ii} > 0$ for $i=1,2,...,n$
\item $0 \leq a_{ij} < a_{ii}$ for $j < i$
\end{enumerate}

\subsection{Two types of lattices}
For our experiments, we will take into account two types of lattices: Random lattices as used in cryptology and special lattices that allow an almost orthogonal basis.

\subsubsection{Random Lattice}

We say that a lattice is a random lattice $L$ of prime volume $P$ if under HNF form its basis matrix $B$ has the following properties:

\begin{itemize}
\item the diagonal has 1 for all its entries except one position that is set to a prime number $P$. Hence, the $\det(B)$ is prime.
\item All row entries of the matrix right to the position that is set to $P$ are smaller than $P$ in absolute value.
\end{itemize}

Without loss of generality, we hence restrict tests to random lattices of volume $P$ whose basis in HNF form is as follows: $$\left[\begin{array}{ccccc}
P & a_2 & \dots & a_{m}\\
& 1&   & \\
& & \ddots & \\
 & & &1
\end{array}\right]$$
where $|a_i| < P$.

\subsubsection{Almost Orthogonal Lattice Bases}

We define an \emph{almost orthogonal lattice basis} $M$ of dimension $n$ and of bit length $k$ as an $n \times n$ square matrix whose entries are $k$-bit integers picked at random. These lattices are frequently used for signal processing applications~\cite{originalJacobiMethodLatticeBasisReduction}.

\subsection{Lattice Reduction}

Lattice reduction is the process of finding nearly orthogonal vectors from an integer lattice basis. Giving an exact definition of "nearly orthogonal vectors" is tricky and each reduction algorithm is likely to give it's own definition of what it means for a basis to be reduced.

\subsection{Quality of a reduction}

In this section, we will discuss two lattice reduction quality indicators that we will use to compare the result of reduction algorithms.

\subsubsection{Orthogonality defect of a basis}
\label{sec:orthogonalityDefect}
The \emph{orthogonality defect} of a basis $\vec{b_1},\vec{b_2},...,\vec{b_n}$ of a lattice $L$ is defined by:
\[
    \text{OrthDefect}(L) :=  \sqrt[n]{\frac{\displaystyle\prod^{n}_{i=1} \|\vec{b_i}\| }{\det(L)}}
\]

The orthogonality defect is a quality indicator that we will use to compare the reduction of \emph{the set of basis vectors}. As the basis vectors are getting more orthogonal during a reduction, the nominator will become smaller. The lower bound of the orthogonality defect is $\text{OrthDefect} \ge 1$. In the special case where the basis would be completely orthogonal, the orthogonality defect would be $\text{OrthDefect} = 1$.

\subsubsection{Hermite factor of a basis}
\label{sec:hermiteFactor}
The \emph{Hermite Factor} of basis vectors $\vec{b_1}, \vec{b_2},...,\vec{b_n}$ of a lattice $L$ is defined by

\[
    \text{HF}(L) := \frac{\|\vec{b_1}\|}{\sqrt[n]{\det(L)}}
\]

The Hermite Factor is a quality indicator that we will use to compare the reduction to a \emph{shortest vector}.

\subsection{Lattice Hard Problems}

Lattices have a few known hard problems that can be used for asymmetric cryptography. We will only cover the most famous one, the Shortest Vector Problem.

\subsubsection{Shortest Vector Problem (SVP)}

Given a basis of an $n$-rank integral lattice $L$, find $\vec{u} \in L$ such that $\|\vec{u}\| = \lambda_1 (L)$

In other words, we are looking for the shortest vector of the lattice. Finding exactly the shortest vector of a lattice is known to be NP-hard under randomized reductions~\cite{Ajtai:1998:SVP} but the complexity of the deterministic reduction remains open.

\subsubsection{Minkowski's Convex Body Theorem}
Let $L$ be a full-rank lattice of $\mathbb{R}^n$ and $C$, a measurable subset of $\mathbb{R}^n$, convex, symmetric with respect to 0, and of measure $> 2^n \text{vol}(L)$. Then $C$ contains at least a nonzero point of the lattice $L$.

A corollary of Minkowski's theorem tells us that there exists for any $n$-dimensional full-rank lattice $L$ a nonzero lattice-vector $x$ whose length is bounded by $\sqrt{n} (\det L)^{1/n}$. So we can derive a bound on the length of a shortest lattice-vector:
\[
\lambda_1(L) \leq \sqrt{n} (\det L)^{1/n}.
\]
In the 80's lattice reduction showed to be successful in finding shortest vectors whose length was significantly smaller than indicated by the above bound.

\subsection{Gram-Schmidt orthogonalization process}
The \emph{Gram-Schmidt orthogonalization (\emph GSO) process} is a method for orthonormalising a set of vectors. Two vectors are orthonormal if they are both orthogonal and unit vectors.

The GSO process  can be defined recursively:
\[
\begin{cases}
\vec{b_1}^{\star} = \vec{b_i}  \\
\vec{b_i}^{\star} = \displaystyle\sum^{i-1}_{j=1} \mu_{i,j} \vec{b_j^\star}
\end{cases}
\]
where $\mu_{ij} = \frac{\langle \vec{b}_i, \vec{b}_j^\star \rangle}{\| b_j^\star\|^2}$. The set of vectors $b_i^*$ are only constituting a basis of the lattice if their $\mu$ factors $\in \mathbb{Z}$

\section{The LLL Algorithm}

The LLL algorithm~\cite{lllpaper} was the first polynomial-time reduction algorithm to be introduced outputting a nearly orthogonal basis. LLL and BKZ 2.0 are the two reduction algorithms that are used in practice. LLL is used in a first step and BKZ, with a suitable blocksize, is then used in a second step to reduce the basis further if needed.

\subsection{$\delta$-LLL Reduced}
We say that a basis is $\delta$-LLL reduced if:
\begin{itemize}
\item $| \mu_{i,j}| \leq \frac{1}{2} $ for $1 \leq j < i \leq n$
\item Lovász's condition: $\forall (\vec{b_i, b_{i+1}})$, we have $(\delta - \mu^2_{i+1,i}) \|\vec{b}^{\star}_{i}\|^2 \leq \| \vec{b}^{\star}_{i+1} \|^2$
\end{itemize}

The first condition guarantees the length reduction of the ordered basis.
The Lovász conditions assures us that although the Gram-Schmidt vectors can get shorter and shorter, their length in a sorted basis cannot decrease too quickly.

Since the LLL-reduction algorithm returns a $\delta$-LLL reduced basis, we can observe the following properties on the basis:
\begin{itemize}
\item $\| \vec{b_1} \| \leq \alpha^{(n-1)/2} \lambda_1 $
\item $\max_i \| \vec{b_i} \| \leq \alpha^{(n-1)/2} \lambda_n $
\item $\| \vec{b}_1 \| \leq \alpha^{(n-1)/4} \det L^{1/n}$\end{itemize}
where the parameter $\alpha = 1/(\delta - \frac{1}{4})$. Since $\delta$ is bounded by $1/4 < \delta < 1$, parameter $\alpha > 4/3$.

\subsection{LLL overview}

The LLL algorithm, like many reduction algorithms, can be broken down into two parts: size-reduction and vector swapping. Size-reduction reduces the vectors from the input basis. After that, the vectors of the basis are checked against the Lovász condition. If a pair of vectors of the basis $(\vec{b_{i}}, \vec{b_{i+1}})$ don't satisfy the condition, we swap the vectors and go back to the size reduction (cfr. \textbf{Procedure \ref{alg:lllAlgorithm}}).

\begin{algorithm}[H]
\caption{Overview of naive-iterative LLL}
\label{alg:lllAlgorithm}
\begin{algorithmic}[1]
\REQUIRE a basis ($\vec{b_1, ..., b_n}$) of a lattice $L$.
\ENSURE $\delta$-LLL reduced basis $(\vec{b_1, ..., b_n})$

\STATE Size-reduce $(\vec{b_1, ..., b_n})$
\IF{there exists an index $j$ which does not satisfy Lovász' condition} \STATE{swap $\vec{b_j}$ and $\vec{b_{j+1}}$, go back to step 1} \ENDIF

\end{algorithmic}
\end{algorithm}

\section{The Jacobi Method}

In 2012, Sanzheng Qiao suggested to use a Jacobi method for lattice basis reduction~\cite{originalJacobiMethodLatticeBasisReduction}. The idea of the Jacobi method is to reduce larger dimension lattice bases by using the Lagrange algorithm iteratively.

\subsection{Lagrange-reduced basis}

Lagrange was the first to formalize the notion of reduction for rank-two lattices~\cite{lagrangeArithmetique}.

Let $L$ be a rank-two lattice of $\mathbb{R}^n$. A basis $(\vec{b_1}, \vec{b_2})$ of $L$ is said to be Lagrange-reduced if and only if $\|\vec{b_1}\| \leq \|\vec{b_2}\|$ and $| \langle \vec{b_1} \vec{b_2} \rangle | \leq \| \vec{b_1} \|^2 / 2 $.

\subsection{Lagrange algorithm}
\label{sec:lagrangeAlgorithm}

Lagrange's algorithm solves the shortest vector problem exactly for lattice bases of dimension 2. It is a variant of Euler's centered algorithm that is used to compute the greatest common divisor of two numbers. GCD can be thought to be a one dimension reduction problem. The GCD is the first minimum of the lattice $n \mathbb{Z} + m \mathbb{Z}$ spanned by $n$ and $m$.

\input{algorithms/lagrange}
\subsection{The Generic Jacobi Method}
We call \emph{generic Jacobi method} the algorithm that simply applies the Lagrange algorithm to every pair of the basis vectors for an $n$ dimensional lattice. This method was introduced by Qiao in 2012~\cite{originalJacobiMethodLatticeBasisReduction} claiming experimental results to be considerably faster than LLL for almost orthogonal bases.

From the definition of a Lagrange-reduced basis, we can define a Jacobi-reduced basis for which:
\begin{align}
\langle \vec{b_i} \vec{b_j} \rangle &\leq \frac{1}{2} \| \vec{b_i}\|^2 \text{ (for all $1 \leq i < j \leq n$)}.\\
\| \vec{b_i}\| &\leq \|\vec{b_j} \| \text{ (for all $1 \leq i < j \leq n$)}
\end{align}

\input{algorithms/genericJacobi}

\subsection{A Jacobi Variant}

In this part, we present the Jacobi-variant that was used for running our benchmarking.

\subsubsection{$\omega$-Lagrange-reduced conditions}
There are two conditions for a basis to be $\omega$-Lagrange-reduced expressed with the Gram matrix $G$ of the basis $B$. The Gram matrix $G$ of a basis $B$ is computed as $G = B^T B$. By construction, a Gram matrix is always symmetric, for every pair of indices $(i,j)$, we have $g_{ij} = g_{ji}$. A basis is $\omega$-Lagrange-reduced if it satifies:

\[
\begin{cases}
\lfloor g_{sl} / g_{ss} \rceil \leq 1, \\
\omega^2 g_{ll} \leq g_{ss} + g_{ll} - 2|g_{sl}|

\end{cases}
\]  where $1/\sqrt{3} \leq \omega < 1$ and $1\leq s,\ell \leq n $. The $s$ and $l$ indices are referring to the index of the short and long vectors in the basis Gram matrix of the basis.

\subsubsection{Size-reduction Lagrange}
The LagrangeIT algorithm is a variant of the Lagrange algorithm that we presented in section~\ref{sec:lagrangeAlgorithm}. The reduction is only performed if the $\omega$-Lagrange-reduced conditions are not satisfied. In which case, both the unimodular reduction matrix $Z$ and the Gram matrix $G$ are updated by applying the Lagrange reduction to the longest of the basis vectors at index $i$ or $j$.

\input{algorithms/lagrangeIT-full}

\subsection{Jacobi-variant algorithm}
This Jacobi-variant performs Lagrange reductions iteratively on the Gram matrix until no more reduction is done. That is when both $\omega$-reduced Lagrange conditions are satisfied.

\input{algorithms/fast-Jacobi}

\section{Experimental Results}

\subsection{Our Implementation}

We implemented the Fast-Jacobi~\cite{fastJacobi} Jacobi reduction, Proc.\ref{jacobiVariantReduction}, in C++ on top of the \emph{newNTL} library. The FPLLL implementation of LLL was used to compare the results after reduction.

\subsection{Scope of testing}

Our experimentation focused on the quality of the reduction. The two quality metrics we used for comparing the reductions are:

\begin{itemize}
\item the orthogonality defect (See section: \ref{sec:orthogonalityDefect}), gives us an indication about the size of the reduced basis vectors for a given lattice in comparison to the volume.
\item the Hermite factor (See section: \ref{sec:hermiteFactor}), gives us an indication about the size of the shortest vector of the reduced basis for a given lattice.
\end{itemize}

\subsection{Input bases}
Numerous papers were written about the Jacobi method for lattice reduction by S. Qiao , namely~\cite{originalJacobiMethodLatticeBasisReduction}, \cite{complexityAnalysisOfJacobiMethod} and ~\cite{enhancedJacobi}, but none of the papers is clarifying what bitsize was used for the randomly generated entries of the vectors. The main focus of the authors was the use of lattice reduction for signal processing, more specifically multiinput and multioutput (MIMO), the use of multiple antennas at both the transmitter and receiver to improve communication reliance. MIMO uses smaller dimensions lattices. In \cite{enhancedJacobi} and \cite{tian2014hybrid}, a reduction is performed on lattices of dimensions 20 to 100. These dimensions are inappropriate for cryptographic applications.

In our experiments, we have used a bitsize of around 10 bits entries.

\subsection{On Almost Orthogonal Bases}

We initially ran our Jacobi reduction variant on almost orthogonal bases to attempt to reproduce Qiao's results. The orthogonality defects are similar to what the authors found in \cite{tian2014hybrid} and \cite{fastJacobi}. Our results are shown in Fig.~\ref{fig1} and Fig.~\ref{fig2}. By increasing the parameter $\omega$, we are making the $\omega$-Lagrange-reduced conditions more restrictive. More specifically, for every pair of vectors of the basis, the following condition needs to be satisfied
\[
 \omega \| \vec{a}_l \| \leq \|\vec{a}_l \pm \vec{a}_s \|
\] where $\vec{a}_l$ is the longest vector and $\vec{a}_s$ is the shortest. Hence, the reduction renders the basis more orthogonal as $\omega$ grows. We see that for $\omega=0.6$ , LLL outperforms the Jacobi method while for $\omega = 0.99$, the Jacobi method outperforms LLL in terms of orthogonality defect.

\begin{figure}[H]
\begin{minipage}{.45\textwidth}
  \centering

    \includegraphics[width=0.9\textwidth]{results-graphs/random-matrix-omega-06defect.png}

      \caption{LLL with delta $\delta = 0.99$ and Jacobi with $\omega = 0.6$}
  \label{fig1}
\end{minipage}
\hspace{.05\textwidth}
\begin{minipage}{.45\textwidth}

  \centering

    \includegraphics[width=0.9\textwidth]{results-graphs/random-matrix-omega-09defect.png}

  \caption{LLL with delta $\delta = 0.99$ and Jacobi with $\omega = 0.99$}
  \label{fig2}
\end{minipage}
\end{figure}

The increase in the parameter $\omega$ comes at a significant performance cost as illustrated in Fig~\ref{fig3}.

\begin{figure}[H]
\begin{minipage}{.45\textwidth}
\includegraphics[width=0.9\textwidth]{results-graphs/runningtime.png}

\caption{Running time (microseconds) increasing substantially with $\omega$ parameter}
\label{fig3}
\end{minipage}
\hspace{.05\textwidth}
\begin{minipage}{.45\textwidth}
    \includegraphics[width=0.9\textwidth]{results-graphs/runningtime-dimension.png}
  \caption{Running time (microseconds) increasing substantially with dimension}
  \label{fig6}
\end{minipage}

\end{figure}

\subsection{On Random Lattices}

Our experiments on random lattices on researched dimensions ($> 50$) have been aborted due to excessive running times.

\subsection{Jacobi after LLL}

As we found that Jacobi could reduce a random lattice further than LLL when $\omega$ is close to 1, we tried to run the Jacobi reduction algorithm after LLL reduction of a random lattice to see if it was able to reduce it further. Experimental results (Fig.~\ref{fig4}-\ref{fig5}) are showing that the Jacobi method does indeed reduce the bases further.

\begin{figure}[H]
\begin{minipage}{.45\textwidth}
  \centering
    \includegraphics[width=0.9\textwidth]{results-graphs/LLLThenJacobi/defect.png}
      \caption{Jacobi after LLL}
  \label{fig4}
\end{minipage}
\hspace{.05\textwidth}
\begin{minipage}{.45\textwidth}

  \centering
    \includegraphics[width=0.9\textwidth]{results-graphs/LLLThenJacobi/defect-logscale-difference.png}

  \caption{Logarithmic-scale difference in orthogonality defects}
  \label{fig5}
\end{minipage}
\end{figure}

We notice that the Jacobi algorithm does reduce the bases further than LLL. Looking at the conditions to be $\delta$-LLL reduced and $\omega-$Jacobi reduced, we can find for example two bases that are LLL reduced but not Jacobi reduced and will miss the shortest vector.

\begin{enumerate}

\item \[
B = \begin{bmatrix}
  \vec{b_1} \\
  \vec{b_2} \\
  \vec{b_3}
 \end{bmatrix} = \begin{bmatrix}
  0 & 2 & 0 \\
  0 & 1 & 2 \\
  2 & 0 & 0
 \end{bmatrix}
\]

\item \[
B = \begin{bmatrix}
  \vec{b_1} \\
  \vec{b_2} \\
  \vec{b_3}
 \end{bmatrix} = \begin{bmatrix}
  4/3 & 0 & 0 \\
  2/3 & \sqrt{4/3} & 0 \\
  2/3 & \sqrt{4/3}/2 & 1
 \end{bmatrix}
\]

In the first case, LLL will stop while Jacobi does exchange $\vec{b}_2$ and $\vec{b}_3$. In the second case the Jacobi Method will find a shortest vector by subtracting $\vec{b_1}-\vec{b_2}$, while LLL won't be able to reduce further.
\end{enumerate}

\subsection{Note on running times}
Although the analysis of the running time was not a metric that was researched in this paper, we did notice that the Jacobi method running time was growing substantially with the dimensions. We were unable to reproduce reduction running times lower than those of LLL as reported by the authors.

\section{Further research}
As follow-up of this research, we believe it would be useful to:
\begin{itemize}
\item determine the algorithmic complexity of the Jacobi method
\item benchmark the Jacobi method against BKZ after LLL
\end{itemize}

\section{Conclusions}

Although the Jacobi method might have practical applications in signal processing, it is not appropriate for cryptographic applications on random lattices or higher dimensional bases.

\section{Acknowledgments}
I would like to particularly thank Anja Becker and Nicolas Gama from LACAL who have been providing significant feedback and advice during my research.

\bibliographystyle{alpha}
\bibliography{references}

\end{document}


\section{Test Description and Success Criteria}
This test is located in \texttt{simulation/dynamics/nHingedRigidBodies/UnitTest/\newline
test\_nHingedRigidBodyStateEffector.py}. In this integrated test there are two hinged rigid bodies connected to the spacecraft hub, one with 4 interconnected panels and one with 3 interconnected panels.  Depending on the scenario, there are different success criteria. These are outlined in the following subsections:
\subsection{Gravity integrated test}
In this test the simulation is placed into orbit around Earth with point gravity and has no damping in the hinged rigid bodies. The following parameters are being tested.
\begin{itemize}
	\item Conservation of orbital angular momentum
	\item Conservation of orbital energy
	\item Conservation of rotational angular momentum
	\item Conservation of rotational energy
\end{itemize}

\subsection{No gravity integrated test}
In this test, the spacecraft is placed in free space (no gravity) and has no damping in the hinged rigid bodies. The following parameters describe the success criteria.
\begin{itemize}
\item Conservation of orbital angular momentum
\item Conservation of orbital energy
\item Conservation of rotational angular momentum
\item Conservation of rotational energy
\end{itemize}

\section{Test Parameters}

Since this is an integrated test, the inputs to the test are the physical parameters of the spacecraft along with the initial conditions of the states. These parameters can be seen in the test file. Additionally, the error tolerances can be seen in Table~\ref{tab:errortol}.

\begin{table}[htbp]
	\caption{Error Tolerance - Note: Relative Tolerance is $\textnormal{abs}(\frac{\textnormal{truth} - \textnormal{value}}{\textnormal{truth}}$)}
	\label{tab:errortol}
	\centering \fontsize{10}{10}\selectfont
	\begin{tabular}{| c | c |} % Column formatting,
		\hline
		Test   & Relative Tolerance \\
		\hline
		Energy and Momentum Conservation & 1e-10 \\
		\hline
	\end{tabular}
\end{table}

\section{Test Results}

The following figures show the conservation of the quantities described in the success criteria for each scenario. The conservation plots are all relative difference plots. All conservation plots show integration error which is the desired result. In the python test these values are automatically checked therefore when the tests pass, these values have all been confirmed to be conserved.

\subsection{Gravity with no damping scenario}
\input{AutoTex/ChangeInOrbitalAngularMomentumGravity}
\input{AutoTex/ChangeInOrbitalEnergyGravity}
\input{AutoTex/ChangeInRotationalAngularMomentumGravity}
\input{AutoTex/ChangeInRotationalEnergyGravity}
\clearpage

\subsection{No Gravity with no damping scenario}
\input{AutoTex/ChangeInOrbitalAngularMomentumNoGravity}
\input{AutoTex/ChangeInOrbitalEnergyNoGravity}
\input{AutoTex/ChangeInRotationalAngularMomentumNoGravity}
\input{AutoTex/ChangeInRotationalEnergyNoGravity}
\clearpage


\documentclass[a4paper, 10pt]{article}

\usepackage{minted}
\usepackage{graphicx}
\usepackage{tabularx}
\usepackage{syntax}
\usepackage{fullpage}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{hyperref}

% Fonts
\usepackage[lf]{Baskervaldx} % lining figures
%\usepackage[bigdelims,vvarbb]{newtxmath} % math italic letters from Nimbus Roman
%\usepackage[cal=boondoxo]{mathalfa} % mathcal from STIX, unslanted a bit
\renewcommand*\oldstylenums[1]{\textosf{#1}}

\usepackage[T1]{fontenc}

\title{Defining Neural Networks using constraint programming}
\author{Noric Couderc}
\date{\today}

\begin{document}

\maketitle
\tableofcontents

\section{Background}

\subsection*{Neural networks}

A network is a graph, with nodes, and edges.

In a neural network, the nodes are neurons, and the edges are synapses. Computer
scientists use an abstraction of these neurons to perform various kinds of
computations. Neural networks are practical because they are able to compute
virtually any function, given a large enough network.
In most cases these artificial neurons are composed of several parts:
\begin{itemize}
    \item A number of entry points, on which the current is represented by a
        numerical value representing the intensity of the signal.
    \item A summation unit that computes the weighted sum of the inputs, where the
        weights are parameters.
    \item A (user defined) function that transforms the output of the summation unit
\end{itemize}

In other words, one can see a neuron as a function $f : \mathbb{R}^n \rightarrow
\mathbb{R}^m$.
The function $f$ is a composition of a dot product by a weight vector with a
threshold-like function.

The threshold function $t$ is applied to each component of the input vector:
$t(x, y, z) = (t'(x), t'(y), t'(z))$. The result of a neuron corresponds to a
function $f$ with $f(\mathbf{v}) = t(\mathbf{v} \cdot \mathbf{w})$.  The threshold function
$t'$ is either a usual threshold function or a smoother function, like $\tanh$,
a sigmoid function  like $t(x) = \frac{1}{1 + \exp(-x)}$, or a rectifier
$t(x) = \max(0, x)$.

When it comes to the structure of the network itself, we are going to consider
the most general case. In this case, the input of the network is a number of
values  -- an input vector -- and all inputs are connected to all the nodes of a
first layer of neurons. The first layer has an output connected to all the
inputs of the following layer, etc. This means each layer is considered a
transformation $\ell : \mathbb{R}^n \rightarrow \mathbb{R}^m$. In the current
setup, the input vector is multiplied by a matrix before application of the
threshold function.

The process of ``learning'' consists of deriving, from a relation
-- i.e. a set $R \subseteq \mathbb{R}^n \times \mathbb{R}^m$ -- a computational
device that can produce a new element of the relation, while keeping the
semantic structure of the relation.

%The ``learning process'' of a neural network consists of finding for each layer
%$\ell_1, \ell_2, \dots \ell_L$ the right matrices $M_1, M_2, \dots M_L$ so that,
%for each input of the dataset $(\mathbf{x}, \mathbf{y}) \in R$, $(\mathbf{x},
%f(\mathbf{x})) \in R$.

\subsection*{Motivation}

The parameters of the system are the weights that are used in each neuron.

Finding these parameters, considering the constraints of input-output pairs is a
constraint satisfaction problem. This article presents an alternative method for
training neural networks by using interval arithmetic and constraint
propagation. Indeed, constraint propagation allows dropping entire uninteresting
sections of the parameter space, while making no assumptions about its topology
or shape.

\subsection*{Use cases and API}

Before defining the model of neural networks, it might be interesting to
define first how the networks are going to be defined and how they are
going to be used.

First, the main task of the neural networks is to find a match between two data
sets, or more specifically, a number of pairs of input vectors and output
vectors.

On the other hand, one of the most important features to have is to be able to
easily \emph{change} the network structure in case the matching process did not
work.

Moreover, the last task is to be able to export the representation of the neural
network to something one could use outside of the constraint programming
paradigm.

\subsubsection*{Enumerating network configurations}
In case a matching did not succeed, one possible solution would be to change the
network structure. Is it possible to do so in an easy way? And is it possible to
make sure \emph{some} configuration will work?

In the current setup, only a specific kind of neural networks is considered:
networks which have an input layer, and output layer, and hidden layers in
between, so that each layer is fully connected to the next.

Therefore, the smallest possible network for a function $f : \mathbb{R}^m
\rightarrow \mathbb{R}^n$ has only one layer, with $m$ inputs and $n$ outputs.
This network could be represented by a pair $[(m,n)]$.

In the same fashion, a possible network for a function $f : \mathbb{R}^m \rightarrow
\mathbb{R}^n$ could be represented by any list $[(m, d_1),(d_1, d_2), (d_2,
d_3), \cdots (d_j, n)]$. Removing the duplicates, a network structure can be
represented by a list $[m, d_1, d_2, \cdots, d_j, n]$, although this
representation is less explicit about the functional aspect of the layers, it
show clearly the number of neurons each layer contains.

By fixing limits to the size of the list and the number of values -- here the
maximum number of dimensions a layer can output -- it is possible to enumerate
all possible network configurations.

\subsubsection*{Increasing flexibility}
In many cases, it is not possible to find a perfect match between all the
input-output pairs. In other words, it is often impossible to have a neural
network with 0\% error. In constraint programming, allowing error in a system is
done \emph{explicitly}. One of the elegant aspects of constraint programming is
that it allows one to deal with variables in the mathematical sense. That is,
variables are defined by the set of values they can take, and no more
assumptions are made. Hence, if one wants to find a function $f(x) = y$ where
the unknown value is the function $f$, and $x$ and $y$ are given, it is possible
to find an approximate matching by having the same equation $f(x) = y'$ where $x$
is a value but $y'$ is a variable, and inserting an additional constraint $y'
\in [y - \delta, y + \delta]$, allowing some flexibility.

% In order to be able to deal with such approximation, the expected output of the
% network is separated from the actual output of the network. For a dataset of
% size $S$, the $i$th expected output is called $\mathbf{y}_i$, the $i$th actual
% output is called $\mathbf{o}_i$, and there is an error vector
% $\boldsymbol{\delta}_i$ so that:
% \begin{equation*}
%     \forall i
%     \in [1, S] : \mathbf{y}_i = \mathbf{o}_i + \boldsymbol{\delta}_i
% \end{equation*}
%
% In this context, defining an error of 80\% means defining another constraint on
% $\boldsymbol{\delta}_i$.
%
% \begin{equation*}
%     |\boldsymbol{\delta}_i| = 0.8 \times |\mathbf{o}_i|
% \end{equation*}

\subsubsection*{Exporting networks}
TODO

\section{A model of neural networks}

\subsection*{Parameters}

\begin{description}
    \item{\textbf{The relation to be modeled:}} A set $R \subseteq
        \mathbb{R}^{d_i} \times \mathbb{R}^{d_o}$, representing the function to be modeled.
        Here $d_i$ will denote the number of dimensions in input data and $d_o$
        the number of dimensions in output data.
    \item{\textbf{The input and output data:}} The relation $R$ is represented
        by two matrices $X$ and $Y$ of respective sizes $|R| \times d_i$ and $|R| \times
        d_o$.
    \item{\textbf{Layer specification:}} To describe the structure of the network,
        several parameters are needed: $L$ denotes the number of layers of the
        network.  The rest of the structure is represented as a list $[d_i, d_1, d_2 \dots
        d_{L-1}, d_o]$. In other words, specifying the layer structure amounts to
        specify a sequence of transformations from one
        space to another, the only constraint being that the output of a
        transformation corresponds to the input of the following transformation.

    \item{\textbf{Threshold function(s):}} The threshold function, called $t$, for each neuron
        is also provided by the user.
        % What about several threshold functions at once.
\end{description}

\subsection*{Decision Variables}
Each layer $\ell$ has:
\begin{itemize}
    \item A $|R| \times n$ input matrix $I_\ell$.
    \item A $n \times m$  weight matrix $M_\ell$.
    \item A $|R| \times m$ \emph{raw} output matrix $R_\ell$, it represents the
        result of $I_\ell M_\ell$ before applying the threshold function.
    \item A $|R| \times m$ \emph{processed} output matrix $O_\ell$, which represents the
        result of mapping the threshold function on $R_\ell$.
\end{itemize}

All decision variables defined above have the same domain, namely the
largest range between the range of the input data and the output data.

% For each input-output vector couple, the vector $\boldsymbol{\delta}$
% represents the difference between the expected output and the actual output of the network.
% \begin{itemize}
%     \item For each couple $(\mathbf{v}_i, \mathbf{w}_i)$ where $i \in [1, |R|[$
%         there exists an error vector: $\boldsymbol{\delta}_i$
% \end{itemize}

\subsection*{Constraints}
The constraints of the system are the mathematical definitions of matrix
multiplication and application of a function.

\begin{description}
    \item{\textbf{Matrix multiplication:}} Given an $m \times n$ matrix $A$ and a $n
        \times p$ matrix $B$:
        \begin{equation*}
            C = AB \leftrightarrow
            \forall i \in [1, m[: \forall j \in [1, p[: C_{ij} = \sum_{k = 1}^{n}A_{ik}B_{kj}
        \end{equation*}
    \item{\textbf{Layer matrix transformation:}} For each layer $\ell$,
        \begin{equation*}
            R_\ell = I_\ell M_\ell
        \end{equation*}

    \item{\textbf{Map with the threshold function:}} For each layer $\ell$,
        \begin{equation*}
            \forall i \in [1, |R|[:
            \forall j \in [1, m[:
                    O_{\ell, ij} = t(R_{\ell, ij})
        \end{equation*}
    \item{\textbf{The input layer and the output layer match the relation
        parameters:}} The input values of the first layer match the relation
        domain, and the output values of the last layer match the relation
        range.
        \begin{equation*}
            I_i = X \land O_o = Y
        \end{equation*}
\end{description}

% The allowed error of the network is expressed as a possible difference --
% denoted $\Delta$ -- between the output matrix of the last layer $O_L$
% and the expected output matrix $W$.
% \begin{description}
%
%         \item{\textbf{Error:}}
%             \begin{equation*}
%                 O_L = Y + \Delta
%             \end{equation*}
% \end{description}

\subsection*{Process}

The data and the constraints are handled in two steps:
\begin{enumerate}
    \item{Create storage for intermediate results}:
        Create the structure that will hold the results of each layer of the
        network, in the form of matrices.

    \item{Build the layers, that connect the various matrices}:
        Given the sizes of the matrices, and considering the rules of matrix
        multiplication. The intermediate structure of the layer is built and
        constrained.
\end{enumerate}

\subsection*{Branching heuristics}

\subsubsection*{Variable selection heuristic}

\subsubsection*{Value selection heuristic}

\section{Performance comparison}

\section{Results, discussion and analysis}

\section{Conclusion}

\bibliographystyle{ieeetr}
\bibliography{literature}

\end{document}


\documentclass{article}

\title{Learning Incoherent Subspaces: Classification via Incoherent Dictionary Learning.}

\author{Daniele Barchiesi and Mark D. Plumbley}
%\affiliation{Centre for Digital Music\\
%	Queen Mary University of London\\
%	Mile End Road, London E1 4NS, UK}

%%%%%%%%%%%%%%%%%%%%%%PACKAGES%%%%%%%%%%%%%%%%%%%%%
\usepackage{amsmath}
\usepackage[english]{babel}
\usepackage[applemac]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsfonts,amssymb,amsmath,amsthm,bm}
\usepackage[boxruled,linesnumbered]{algorithm2e}
\usepackage{subfigure,graphicx}
\usepackage{setspace}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{graphicx}
\usepackage{epstopdf}

%%%%%%%%%%%%%%%%%%%%%DEFINITIONS%%%%%%%%%%%%%%%%%%
\input{definitions.tex}

\def \nComponents{L} 	%number of significant components of ICA feature transform
\def \fea{\Vector{x}} 	%vector of features
\def \Feas{\Matrix{X}} 	%matrix containing set of training features
\def \iFea{n}	 		%component index of features vector
\def \nDim{N} 			%dimensionality of features vector
\def \newFea{\Vector{y}} 	%features vector after feature transform
\def \NewFeas{\Matrix{Y}}%matrix of training features after feature transform
\def \Dic{\Matrix{\Phi}} %dictionary learned from features
\def \nAto{K} 			%number of atoms in the dictionary
\def \iAto{k} 			%atom index
\def \atom{\Vector{\phi}}%atom in a dictionary
\def \nFea{M}			%number of signals or features
\def \iFea{m}			%signal index
\def \Coeff{\Matrix{A}} 	%matrix of sparse approximation coefficients
\def \coherence{\mu} 		%mutual coherence
\def \coeff{\Vector{\alpha}}%vector of sparse approximation coefficients
\def \nActiveAtoms{S} 	%number of active atoms
\def \orthmat{\Matrix{W}} 	%orthonormal matrix
\def \cat{c} 			%category
\def \cats{\Vector{c}}	%vector of categories of training signals
\def \Cat{\mathcal{C}} 	%set of possible categories
\def \uniCat{C} 			%element of the set of possible categories
\def \iCat{p}  		%index of the elements in the set of possible categories
\def \nCat{P} 		%number of elements in the set of possible categories
\def \definition{\overset{\Label{def}}{=}}	%definitions
\def \Gram{\Matrix{G}} 						%Gram matrix
\def \gram{g} 								%element of the Gram matrix
\def \admissibleDictionary{\Function{D}} 	%set of admissible dictionaries
\def \ambient{\Set{R}} 						%ambient
\def \ipr{\Acronym{ipr}} 					%Iterative projections and rotations
\def \ip{\Acronym{ip}}						%Iterative projections
\def \nDimSub{Q}
\def \Spa{\Matrix{\Psi}}

\begin{document}
%\ninept
%

\maketitle
%
\begin{abstract}
In this article we present the supervised iterative projections and rotations (\Acronym{s-ipr}) algorithm, a method for learning discriminative incoherent subspaces from data. We derive \Acronym{s-ipr} as a supervised extension of our previously proposed iterative projections and rotations (\Acronym{ipr}) algorithm for incoherent dictionary learning, and we employ it to learn incoherent sub-spaces that model signals belonging to different classes. We test our method as a feature transform for supervised classification, first by visualising transformed features from a synthetic dataset and from the `iris' dataset, then by using the resulting features in a classification experiment. While the visualisation results are promising, we find that \Acronym{s-ipr} generally performs worse than traditional and state-of-the-art techniques for supervised dimensionality reduction in terms of the misclassification ratio.
\end{abstract}
%
%\begin{keywords}
%Feature transforms, sparse approximation, dictionary learning, supervised classification.
%\end{keywords}
%
\section{Introduction: Classification And Feature Transforms}\label{sec:intro}
Supervised classification is one of the classic problems in machine learning where a system is designed to discriminate the category of an observed signal, having previously observed representative examples from the considered classes \cite{Duda1973Pa}.

Typically, a classification algorithm consists of a training phase where  class-specific models are learned from labelled samples, followed by a testing phase where unlabelled data are classified by comparison with the learned models. Both training and testing comprise various stages. Firstly, we observe a signal that measures a process of interest, such as the recording of a sound or image, or a log of the temperatures in a particular geographic area. Then, a set of features are extracted from the raw signals using signal processing techniques. This step is performed in order to reduce the dimensionality of the data and provide a new signal that allows generalisation among examples of the same class, while retaining enough information to discriminate between different classes.

Following the features extraction step, a feature transform can be employed to further reduce the dimensionality of the data and to enhance discrimination between classes. Thus classification benefits from feature transforms especially when features are not separable, that is, when it is not possible to optimise a simple function that maps features belonging to signals of a given class to the corresponding category.
A further dimensionalty reduction may be performed when dealing with high dimensional signals (such as audio or high resolution images) by fitting the parameters of global statistical distributions with features learned on portions of the signal. Models learned on different classes are finally compared using a distance metric to the model learned form an unlabelled signal, which is typically assigned to the nearest class.

\subsection{Traditional Algorithms For Feature Transform}\label{sec:aft}
Two of the main feature transform techniques include principal component analysis (\Acronym{pca}) \cite{Pearson1901On} and Fisher's linear discriminant analysis (\Acronym{lda}) \cite{Duda1973Pa}.

\subsubsection{\Acronym{pca}}

Let $\curlyb{\fea_{\iFea} \in \real^{\nDim}}_{\iFea=1}^{\nFea}$ be a set of vectors containing features extracted from $\nFea$ training signals. The goal of \Acronym{pca} is to learn an orthonormal set of basis functions $\curlyb{\atom_{\iAto} \in \ambient^{\nDim}}_{\iAto=1}^{\nDim}$ such that $ \norm{\atom_{\iAtom}}{2}=1$ and $\inner{\atom_{i}}{\atom_{j}}=0 \; \forall i\neq j$ that are placed along the columns of a so-called \emph{dictionary} $\Dic\in\real^{\nDim\times\nDim}$. The bases are optimised from the data to identify their principal components, that is, the sub-spaces that retain the maximum variance of the features.

To compute the dictionary, the eigenvalue decomposition of the outer product
\begin{equation}
\Matrix{X}\Transpose{\Matrix{X}} = \Matrix{Q}\Matrix{\Lambda}\Transpose{\Matrix{Q}}
\end{equation}
is first calculated. Then, the $\nComponents$ eigenvectors corresponding to the $\nComponents$ largest eigenvalues are selected from the matrix $\Matrix{Q}$, and scaled to unit $\ell_{2}$ norm to form the dictionary $\Dic$. A new set of transformed features $\newFea_{\Acronym{pca}} = \Dic\Transpose{\Dic}\fea$ is computed by projecting the data onto the sub-space spanned by the columns of $\Dic$ (that is, onto the $\nComponents$-dimensional principal sub-space). This operation reduces the dimensionality of the features by projecting them onto a linear subspace embedded in $\real^{\nDim}$. It is an unsupervised technique that does not exploit knowledge about the classes associated with the training set, but implicitly relies in the assumption that the principal component directions encode relevant differences between classes.

\subsubsection{\Acronym{lda}}
In contrast, \Acronym{lda} is a supervised method for feature transform whose objective is to explicitly maximise the separability of classes in the transformed domain.

Let $\obsSet_{\iUniCat}$ be a set indexing features extracted from data belonging to the $\iUniCat$-th category, let
\begin{equation}
\average{\fea}_{\iUniCat} \definition \frac{1}{\abs{\obsSet_{\iUniCat}}}\sum_{\iObs\in\obsSet_{\iUniCat}} \fea_{\iObs}
\end{equation}
be the $\iUniCat$-th class feature centroid, and $\average{\fea}\definition\sum_{\iObs=1}^{\nObs}\fea_{\iObs}$ the centroid of the features extracted from the entire training dataset. The between-classes scatter matrix
\begin{equation}
\Matrix{S_{b}} \definition \sum_{\iUniCat=1}^{\nUniCat}\abs{\obsSet_{\iUniCat}}\roundb{\average{\fea}_{\iUniCat}-\average{\fea}}\Transpose{\roundb{\average{\fea}_{\iUniCat}-\average{\fea}}}
\end{equation}
is defined to measure the mutual distances between the centroids of different classes, while the within-classes scatter matrix
\begin{equation}
\Matrix{S}_{w}\definition \sum_{\iUniCat=1}^{\nUniCat}\sum_{\iObs\in\obsSet_{\iUniCat}}\roundb{\fea_{\iObs}-\average{\fea}_{\iUniCat}}\Transpose{\roundb{\fea_{\iObs}-\average{\fea}_{\iUniCat}}}
\end{equation}
quantifies the distances between features belonging to the same class.
To maximise an objective function $\objective(\Matrix{W})\definition\frac{\abs{\Transpose{\Matrix{W}}\Matrix{S}_{b}\Matrix{W}}}{\abs{\Transpose{\Matrix{W}}\Matrix{S}_{w}\Matrix{W}}}$ that promotes features belonging to the same class to be near each other and far away from features belonging to other classes, the eigenvalue decomposition of the matrix
\begin{equation}
\pseudoinverse{\Matrix{S}_{w}}\Matrix{S}_{b} = \Matrix{Q}\Matrix{\Lambda}\Transpose{\Matrix{Q}}
\end{equation}
is computed, and the features $\fea$ are projected onto the space spanned by its $(\nUniCat-1)$ eigenvectors corresponding to the largest $(\nUniCat-1)$ eigenvalues.

\Acronym{lda} explicitly seeks to enhance the discriminative power of features by optimising the objective $\objective$.
\subsection{Supervised \Acronym{pca}}
Related works that extend \Acronym{pca} include the supervised \Acronym{pca} (\Acronym{s-pca}) proposed by Barshan et al. \cite{Barshan2011Su}. \Acronym{s-pca} is based on the theory of reproducing kernel Hilbert spaces (\Acronym{rkhs}) (that are spaces of functions which satisfy certain properties and map elements from an arbitrary set to the set of complex numbers) \cite{Aronszajn:1950}, and on the so-called Hilbert-Schmidt independence criterion (\Acronym{hsic})\cite{gretton2005measuring}. The \Acronym{hsic} is used to estimate the statistical dependence of two random variables based on the fact that this quantity is related to the correlation of functions belonging to their respective \Acronym{rkhs}. While \Acronym{hsic} is defined in terms of the probability density function of the two random variables,  empirical estimates of \Acronym{hsic} can be obtained from finite sequences of their realisations. The empirical \Acronym{hsic} can be used in turn to construct an objective function that maximises the dependence between the two variables. Hence, this strategy is adopted within the context of classification to maximise the statistical dependence between a transformed feature $\newFea_{\Acronym{s-pca}}$ and its corresponding category $\cat$.

In practice, \Acronym{s-pca} differs from \Acronym{pca} in that it calculates the eigenvalue decomposition of a matrix $\Matrix{R}$ defined as follows:
\begin{equation}
\Matrix{R} \definition \Matrix{X}\Matrix{H}\Matrix{L}\Matrix{H}\Transpose{\Matrix{X}}
\end{equation}
were $\Matrix{H} \definition \Matrix{I} - \Vector{e}\Transpose{\Vector{e}}$ is a so-called \emph{centring} matrix\footnote{Here $\Vector{e}$ is a vector of ones.} and $\Matrix{L} \definition \Vector{\cat}\Transpose{\Vector{\cat}}$ is the kernel matrix of the class variable that is constructed by computing the outer product of the vectors resulting from assigning different numerical values to each category.

\subsection{Other related work}
The union of incoherent sub-spaces model proposed by Schnass and Vandergheynst \cite{Schnass2010A-} employes a very similar intuition to the one that inspired our proposed method, and models features belonging to different classes using incoherent subspaces. Other methods for supervised dimensionality reduction include metric learning algorithms \cite{xing2002distance}, sufficient dimensionality reduction \cite{li1991sliced} and Bair's supervised principal components \cite{Bair06predictionby}.

Manifold learning techniques are used to model nonlinear data and reviewed by Van Der Maaten et al. \cite{Van-Der-Maaten2009Di}. Finally,  the sparse sub-space clustering technique developed by Elhamifar and Vidal \cite{Elhamifar2013Sp} that applies concepts and algorithm from the field of sparse approximation to tackle unsupervised clustering problems.

\subsection{Paper organisation}
The method proposed in this paper is aimed at learning discriminative sub-spaces that allow dimensionality reduction, while at the same time enhancing the separability between classes. It is derived from our previous work on learning incoherent dictionaries for sparse approximation \cite{Barchiesi2013Le}.

The incoherent dictionary learning problem will be introduced in Section \ref{sec:idl}, while Section \ref{sec:lis} will contain the main contribution of this paper consisting in learning incoherent subspaces for classification. Numerical experiments are presented in Section \ref{sec:ne}, and conclusions are drawn in Section \ref{sec:end}.

\section{Incoherent Dictionary Learning}\label{sec:idl}
A sparse approximation of a signal $\fea\in\real^{\nDim}$ is a linear combination of $\nAto\geq\nDim$ basis functions $\curlyb{\atom_{\iAto}\in\real^{\nDim}}_{\iAto=1}^{\nAto}$ called \emph{atoms} described by:
\begin{equation}
	\fea \approx \approximant{\fea} = \sum_{\iAto=1}^{\nAto} \alpha_{\iAto}\atom_{\iAto}
\end{equation}
where the vector of coefficients $\coeff$ contains a \emph{small} number of non-zero components, corresponding to a small number of atoms actively contributing to the approximation $\approximant{\fea}$. Given a signal $\fea$ and a dictionary, various algorithms have been proposed to find a sparse approximation that minimises the residual error $\norm{\fea-\approximant{\fea}}{2}$\cite{Elad2010Sp}.

Dictionary learning aims at optimising a dictionary $\Dic$ for sparse approximation given a set of training data. It is an unsupervised technique that can be thought as being a generalisation of \Acronym{pca}, as both methods learn linear subspaces that minimise the approximation error of the signals. Dictionary learning, however, is generally more flexible than \Acronym{pca} because it can be employed to learn more general non-orthogonal over-complete dictionaries \cite{Rubinstein2010Di}.

\subsection{The incoherent dictionary learning problem}
Dictionaries for sparse approximation have important intrinsic properties that describe the relations between their atoms, like the mutual coherence $\mu(\Dic)=\underset{i\neq j}{\max}{\inner{\atom_{i}}{\atom_{j}}}$ that is defined as the maximum inner product between any two different atoms. The goal of incoherent dictionary learning is to learn atoms that are well adapted to sparsely approximate a set of training signals, and that are at the same time mutually incoherent \cite{Barchiesi2013Le}.

Given a set of $\nFea$ training signals contained in the columns of the matrix $\Feas \in \real^{\nDim\times\nFea}$ and a matrix $\Coeff\in\real^{\nAto\times\nFea}$ indicating the sparse approximation coefficients, the incoherent dictionary learning problem can be expressed as:

\begin{align}\label{eq:iprcost}
\optimal{\Dic} = \MinimiseST{\Dic}{\norm{\Feas-\Dic\Coeff}{\F}}{\coherence(\Dic) \leq \coherence_{0} \nonumber \\
			&\norm{\coeff_{\iFea}}{0}\leq \nActiveAtoms \quad \forall \iFea}
\end{align}
where $\coherence_{0}$ is a fixed mutual coherence constraint, the $\ell_{0}$ pseudo-norm $\norm{\cdot}{0}$ counts the number of non-zero components of its argument and $\nActiveAtoms$ is a fixed number of active atoms. Algorithms for (incoherent) dictionary learning generally follow an alternate optimisation heuristic, iteratively updating $\Dic$ and $\Coeff$ until a stopping criterion is met. In the case of the iterative projections and rotations algorithm (\Acronym{ipr}) algorithm \cite{Barchiesi2013Le}, a dictionary de-correlation step is added after updating the dictionary in order to satisfy the mutual coherence constraint.

Given $\Feas$, fixed $\coherence_{0}$, $\nActiveAtoms$ and a stopping criterion (such as a maximum number of iterations), the optimisation of \eqref{eq:iprcost} is tackled by iteratively performing the following steps:
\begin{itemize}
	\item\emph{Sparse coding}: fix $\Dic$ and compute the matrix $\Coeff$ using a suitable sparse approximation method.
	\item\emph{Dictionary update}: fix $\Coeff$ and update $\Dic$ using a suitable method for dictionary learning.
	\item\emph{Dictionary de-correlation}: given $\Feas$, $\Dic$ and $\Coeff$ update the dictionary $\Dic$ to reduce its mutual coherence under the level $\coherence_{0}$.
\end{itemize}

\subsection{The iterative projections and rotations algorithm}\label{sec:ipr}
The \Acronym{ipr} algorithm has been proposed in order to solve the dictionary de-correlation step, while ensuring that the updated dictionary provides a sparse approximation with low residual norm, as indicated by the objective function \eqref{eq:iprcost} \cite{Barchiesi2013Le}.

The \Acronym{ipr} algorithm requires the calculation of the Gram matrix $\Gram=\Transpose{\Dic}\Dic$ which contains the inner products between any two atoms in the dictionary. $\Gram$ is iteratively projected onto two constraint sets, namely the structural constraint set $\stcset$ and the spectral constraint set $\spcset$. The former is the set of symmetric square matrices with unit diagonal values and off-diagonal values with magnitude smaller or equal than $\coherence_{0}$:
\small
\begin{equation*}
	\stcset \definition \curlyb{\stcmat \in \ambient^{\nAtoms \times \nAtoms} : \stcmat = \Transpose{\stcmat}, \stcel_{i,i}=1,\max_{i > j}|\stcel_{i,j}|\leq \coherence_{0}}.
\end{equation*}
\normalsize
The latter is the set of symmetric positive semidefinite square matrices with rank smaller than or equal to $\nDimensions$:
\begin{equation*}
\spcset \definition \curlyb{ \spcmat \in \ambient^{\nAtoms \times \nAtoms} : \spcmat = \Transpose{\spcmat}, \operatorname{eig}(\spcmat)\geq \Vector{0}, \operatorname{rank}(\spcmat)\leq \nDimensions}
\end{equation*}
where the operator $\operatorname{eig}(\cdot)$ returns the vector of eigenvalues of its argument.

Starting from the Gram matrix of an initial dictionary $\Dictionary$, the \Acronym{ipr} method iteratively performs the following operations.
\begin{itemize}
\item \emph{Projection onto the structural constraint set}. The projection $\stcmat = \Projection_{\stcset}(\Gram)$ can be obtained by:
\begin{enumerate}
\item setting $\stcel_{i,i} = 1$,
\item limiting the off-diagonal elements so that, for $i \neq j$,
\small
\begin{equation}\label{eq:pscs}
\stcel_{i,j} = \operatorname{Limit}({\gram}_{i,j},\coherence_{0}) = \left\{ \begin{array}{rl}
	\gram_{i,j}  & \text{if} \quad |\gram_{i,j}|\leq \coherence_{0} \\
	\operatorname{sgn}(\gram_{i,j})\coherence_{0}  & \text{if} \quad |\gram_{i,j}| > \coherence_{0}
\end{array} \right.
\end{equation}
\normalsize
\end{enumerate}
\item \emph{Projection onto the spectral constraint set and factorization}. The projection $\spcmat = \Projection_{\spcset}(\Gram)$ and subsequent factorisation are obtained by:
\begin{enumerate}
\item calculating the eigenvalue decomposition (\Acronym{evd}) $\Gram = \eigvecmat \eigvalmat \Transpose{\eigvecmat}$,
\item thresholding the eigenvalues by keeping only the $\nDimensions$ largest positive ones.
\begin{equation*}
	\left[\operatorname{Thresh}(\eigvalmat,\nDimensions) \right]_{i,i} = \left\{ \begin{array}{rl}
	\lambda_{i,i}  & \text{if} \quad i \leq N \; \text{and} \; \lambda_{i,i}>0 \\
	0  & \text{if} \quad i > N \; \text{or} \; \lambda_{i,i}\leq 0
	\end{array}\right.
\end{equation*}
where the eigenvalues in $\eigvalmat$ are ordered from the largest to the smallest. Following this step, at most $\nDimensions$ eigenvalues of the Gram matrix are different from zero,
\item factorizing the projected Gram matrix into the product $\Gram=\Transpose{\Dic}\Dic$ by setting:
\begin{equation}
\Dic = \eigvalmat^{1/2}\Transpose{\eigvecmat}.
\end{equation}
\end{enumerate}
\item \emph{Dictionary rotation}. Rotate the dictionary $\Dic$ to align it to the training set by solving the problem:
\begin{equation}\label{eq:rot}
	\optimal{\orthmat} = \Minimise{\orthmat \Transpose{\orthmat} = \Matrix{I}}{\norm{\Feas - \orthmat\Dic\Coeff}{\Label{F}}}.
\end{equation}
The optimal rotation matrix can be calculated by:
\begin{enumerate}
\item computing the sample covariance between the observed signals and their approximations $\CovMat \definition (\Dic\Coeff)\Transpose{\Feas}$,
\item calculating the \Acronym{svd} of the covariance $\CovMat = \Matrix{U}\Matrix{\Sigma}\Transpose{\Matrix{V}}$,
\item setting the optimal rotation matrix to $\optimal{\orthmat}=\Matrix{V}\Transpose{\Matrix{U}}$,
\item rotating the dictionary $\Dic \leftarrow \optimal{\orthmat}\Dic$.
\end{enumerate}
\end{itemize}

More details about the \Acronym{ipr} algorithm can be found in  \cite{Barchiesi2013Le}, including details of its computational cost.
%The code of the \Acronym{ipr} method is illustrated in Algorithm \ref{algo:ipr}.
%
%\begin{algo}
%\KwIn{$\Feas, \Dictionary, \Coeff, \coherence_{0}, \nIter$}
%\KwOut{$\optimal{\Dictionary}$}
%$\iter\gets1$\;
%\While{$\iter \leq \nIter$ and $\coherence(\Dictionary)>\coherence_{0}$}{
%	\tcp{Calculate Gram matrix}
%	$\Gram \gets \Transpose{\Dictionary}\Dictionary$\;
%	\tcp{Project ont structural c.s.}
%	$\operatorname{diag(\Gram)} \gets \Vector{1}$\;
%	$\Gram \gets \operatorname{Limit}(\Gram,\coherence_{0})$\;
%	\tcp{Project Gram matrix onto spectral c.s. and factorize}
%	$[\eigvecmat, \eigvalmat] \gets \Acronym{evd}(\Gram)$\;
%	$\eigvalmat \gets \operatorname{Thresh}(\eigvalmat,\nDimensions)$\;
%	$\Dictionary \gets \eigvalmat^{1/2}\Transpose{\eigvecmat}$\;
%	\tcp{Rotate dictionary}
%	$\CovMat \gets \Feas\Transpose{(\Dictionary\Coeff)}$\;
%	$[\Matrix{U},\Matrix{\Sigma},\Matrix{V}] \gets \Acronym{svd}(\CovMat)$\label{algo:ipr:svd}\;
%	$\OrthMat \gets \Matrix{V}\Transpose{\Matrix{U}}$\;
%	$\Dictionary \gets \OrthMat\Dictionary$\;
%	$\iter\gets\iter+1$\;
%}
%\caption{\label{algo:ipr}Iterative projections and rotations (\Acronym{ipr})}
%\end{algo}

\section{Learning Incoherent Subspaces}\label{sec:lis}
The \Acronym{ipr} algorithm learns a dictionary where all the atoms are mutually incoherent. Therefore, given any two disjoint sets $\Lambda\bigcap\Gamma=\emptyset$ that identify non-overlapping collections of atoms, the sub-dictionaries $\Dic_{\Lambda}, \Dic_{\Gamma}$ are also mutually incoherent.

Starting from this observation, the main intuition driving the development of a supervised \Acronym{ipr} (\Acronym{s-ipr}) algorithm for classification is to learn mutually incoherent sub-dictionaries that approximate features from different classes of signals. The sub-dictionaries are in turn used to define incoherent sub-spaces, and features are projected onto these sub-spaces yielding discriminative dimensionality reduction.

\subsection{The supervised \Acronym{ipr} algorithm}\label{sec:iprclass}
Let $\curlyb{\cat_{\iFea}\in\Cat}_{\iFea=1}^{\nFea},\; \Cat=\curlyb{\uniCat_{1},\uniCat_{2},\dots,\uniCat_{\nCat}}$ be a set of labels that identify the category of the vectors of features $\fea_{\iFea}$, whose elements belong to a set $\Cat$ of $\nCat$ possible categories. The columns of the matrix $\Feas_{\iCat}$ contain a selection of the features extracted from signals belonging to the $\iCat$-th category.

To learn incoherent sub-dictionaries from the entire set of features, we must first cluster the atoms to different classes\footnote{Note that the term \emph{cluster} implies that a this stage the algorithm needs to make an unsupervised decision, since there is no any a-priori reason to assign a given atom to any particular class.}, and then only proceed with their de-correlation  if they are assigned to different categories (while allowing coherent atoms to approximate features from the same class). To this aim, we employ the matrix $\Coeff$ to measure the contribution of every atom to the approximation of features belonging to each class.

Let $\coeff_{\iCat}^{\iAto}$ indicate the $\iAto$-th row of the matrix $\Coeff_{\iCat}$ containing the coefficients that contribute to the approximation of $\Feas_{\iCat}$, and $\nDim_{\iCat}$ indicate the number of its elements. A coefficient $\gamma_{\iAto,\iCat}$ is defined as:
\begin{equation}
	\gamma_{\iAto,\iCat} \definition \frac{1}{\nDim_{\iCat}}\norm{\coeff_{\iCat}^{\iAto}}{1},
\end{equation}
and every atom $\atom_{\iAtom}$ is associated with the category to which it maximally contributes $\optimal{\iCat}_{\iAto} = \underset{\iCat}{\arg\max}\curlyb{\gamma_{\iAto,\iCat}}$.

Grouping together atoms that have been assigned to the same class leads to a set of sub-dictionaries whose size and rank depends on the number of atoms for each class, and to their linear dependence. As a general heuristic, if  features corresponding to different classes do not occupy the same sub-space (according to the active elements in $\Coeff$), a full-rank dictionary $\Dic$ with $\nAto \geq \nDim \gg \nCat$ ensures that $\optimal{\iCat}_{\iAto}$ identify $\nCat$ non-empty and disjoint sub-dictionaries $\curlyb{\Dic_{\iCat}}_{\iCat=1}^{\nCat}$.

Once the atoms have been clustered, the Gram matrix $\Gram$ is computed and iteratively projected as in the method described in Section \ref{sec:ipr}, with the difference that equation \eqref{eq:pscs} is modified in order to only constraint the mutual coherence between atoms assigned to different categories
\small
\begin{equation}\label{eq:pscs2}
\operatorname{Limit}({\gram}_{i,j},\coherence_{0},\optimal{\Vector{\iCat}}) = \left\{ \begin{array}{rl}
	\gram_{i,j}  & \text{if} \quad |\gram_{i,j}|\leq \coherence_{0} \, \text{or} \, \optimal{\iCat}_{i}=\optimal{\iCat}_{j} \\
	\operatorname{sgn}(\gram_{i,j}) \coherence_{0}  & \text{if} \quad |\gram_{i,j}|   >  \coherence_{0} \, \text{and} \, \optimal{\iCat}_{i}\neq\optimal{\iCat}_{j}
\end{array} \right.
\end{equation}
\normalsize

A further modification of the standard \Acronym{ipr} algorithm presented in \cite{Barchiesi2013Le} consists in the update of the Gram matrix, performed by computing its element-wise average with the projection $\stcmat = \Projection_{\stcset}(\Gram)$ (rather than by using the projection alone). This heuristic has led to improved empirical results by preventing $\Gram$ from changing too abruptly.

The complete supervised \Acronym{s-ipr} method is summarised in Algorithm \ref{algo:sipr}. Note that the mutual coherence $\coherence_{\optimal{\iCat}}(\Dic) = \underset{\optimal{\iCat}_{i}\neq\optimal{\iCat}_{j}}{\arg\max}\inner{\atom_{i}}{\atom_{j}}$ indicated in this algorithm measures the inner product between any two atoms assigned to different categories since atoms assigned to the same category are allowed to be mutually coherent.
\begin{algo}
\KwIn{$\Feas, \Dictionary, \Coeff, \coherence_{0}, \cats, \nIter$}
\KwOut{$\optimal{\Dictionary}$}
$\iter\gets1$\;
\tcp{Cluster atoms}
$\Coeff_{\iCat} \gets \squareb{\coeff_{j}} \forall j \in \uniCat_{\iCat}$\;
$\gamma_{\iAto,\iCat} \gets \norm{\coeff_{\iCat}^{\iAto}}{1}/\nDim_{\iCat}$\;
$\optimal{\iCat}_{\iAto} = \underset{\iCat}{\arg\max}\curlyb{\gamma_{\iAto,\iCat}}$\;
\While{$\iter \leq \nIter$ and $\coherence_{\optimal{\iCat}}(\Dictionary)>\coherence_{0}$}{
	\tcp{Calculate Gram matrix}
	$\Gram \gets \Transpose{\Dictionary}\Dictionary$\;
	\tcp{Project onto structural c.s.}
	$\operatorname{diag}(\stcmat) \gets \Vector{1}$\;
	$\stcmat \gets \operatorname{Limit}(\Gram,\coherence_{0},\optimal{\Vector{\iCat}})$\;
	$\Gram \gets \frac{1}{2}\Gram + \frac{1}{2}\stcmat$\;
	\tcp{Project onto spectral c.s. and factorize}
	$[\eigvecmat, \eigvalmat] \gets \Acronym{evd}(\Gram)$\;
	$\eigvalmat \gets \operatorname{Thresh}(\eigvalmat,\nDimensions)$\;
	$\Dictionary \gets \eigvalmat^{1/2}\Transpose{\eigvecmat}$\;
	\tcp{Rotate dictionary}
	$\CovMat \gets \Feas\Transpose{(\Dictionary\Coeff)}$\;
	$[\Matrix{U},\Matrix{\Sigma},\Matrix{V}] \gets \Acronym{svd}(\CovMat)$\label{algo:ipr:svd}\;
	$\OrthMat \gets \Matrix{V}\Transpose{\Matrix{U}}$\;
	$\Dictionary \gets \OrthMat\Dictionary$\;
	$\iter\gets\iter+1$\;
}
\caption{\label{algo:sipr}Supervised \Acronym{ipr}}
\end{algo}
\subsection{Classification via incoherent subspaces}\label{sec:class}
The \Acronym{s-ipr} algorithm allows to learn a set of sub-dictionaries $\curlyb{\Dic_{\iCat}}$ that contain mutually incoherent atoms. These cannot be directly used to define discriminative subspaces because, depending on $\nDim$ and on the rank of each sub-dictionary, atoms belonging to disjoint sub-dictionaries might span identical subspaces. Instead, we fix a rank $\nDimSub\leq\floor{\nDim/\nCat}$ and choose a collection of $\nDimSub$ linearly independent atoms from each sub-dictionary $\Dic_{\iCat}$, using the largest values of $\gamma_{\iAto,\iCat}$ to define a picking order. Thus, we obtain a set $\curlyb{\Spa_{\iCat}}_{\iCat=1}^{\nCat}$ of incoherent sub-spaces of rank $\nDimSub$ embedded in the space $\ambient^{\nDim}$, and use them to derive a feature transform for classification.

Each feature vector $\fea_{\iFea}$ that belongs to the class $\cat_{\iFea}$ is projected onto the relative subspace, yielding a set of transformed features $\curlyb{\newFea_{\iFea}}_{\iFea=1}^{\nFea}$.
\begin{equation}
\newFea_{\iFea} = \Spa_{\cat_{\iFea}}\pseudoinverse{\Spa_{\cat_{\iFea}}}\fea_{\iFea}
\end{equation}
where $\Spa^{\dagger}$ denotes the Moore-Penrose pseudo-inverse of the matrix $\Spa$ and needs to be used in place of the transposition operator because the columns of $\Spa$ are in general not orthogonal.

When an unlabelled signal is presented to the classifier, the corresponding vector of features $\fea$ is projected onto all the learned sub-spaces. Then, the nearest sub-space is chosen using an Euclidean distance measure, and the corresponding projection $\newFea$ used as the transformed feature.
\begin{align}
	\optimal{\iCat} &= \underset{\iCat}{\arg\min}\norm{\fea-\Spa_{\iCat}\pseudoinverse{\Spa_{\iCat}}\fea}{2}	\\
	\newFea &= \Spa_{\optimal{\iCat}}\pseudoinverse{\Spa_{\optimal{\iCat}}}\fea
\end{align}
The subspace $\optimal{\iCat}$ can be directly used as an estimator of the category of the signal $\optimal{\cat}$. Alternatively, a simple \emph{k-neaerst neighbour} classifier can be employed on the transformed features, and a class can be inferred as:
\begin{equation}
\optimal{\cat} = \texttt{knn}(\newFea,\NewFeas,\cats)
\end{equation}
where $\NewFeas$ represents the matrix of training features after the transform stage. This latter approach is especially suitable when working with a large number of classes in a space of relatively small dimension, as in this case multiple classes might be assigned to the same subspace.
\section{Numerical Experiments}\label{sec:ne}
\subsection{Feature visualisation}\label{sec:visu}
To illustrate the \Acronym{s-ipr} algorithm for feature transform, we first run visualisation experiments depicting how different feature transform methods act on training and test data.
\subsubsection{Synthetic data}
\begin{figure}
\centering
\includegraphics[width=.6\textwidth]{./Code/Datasets/GetToyExampleDataset.pdf}
\caption{\label{fig:toy}Synthetic data generated along one-dimensional subspaces of $\real^{2}$.}
\end{figure}
Figure \ref{fig:toy} displays a total of $1500$ synthetic features in $\real^{2}$ belonging to $3$ different classes that we generated for this experiment. For each class, first we draw values distributed uniformly in the interval $\squareb{-1,1}$ and assign them to the first component of the features (the \emph{x} coordinate). Then, we add Gaussian noise with variance $0.1$ to the second component (the \emph{y} coordinate), and we rotate the resulting data by the angles $\theta_{0}=0$, $\theta_{1}=\pi/4$ and $\theta_{3}=\pi/2$ for the $3$ classes respectively. This way, features belonging to different classes are clustered along different one-dimensional sub-spaces of $\real^{2}$.
\begin{figure}
\includegraphics[width=\textwidth]{./Code/toyvisu.pdf}
\caption{\label{fig:toyNewFea}Feature transform applied to the synthetic data in Figure \ref{fig:toy}. Different colours correspond to different classes, `+' and `o' markers represent samples taken from the training and test set respectively.}
\end{figure}

Figure \ref{fig:toyNewFea} displays the result of the application of feature transforms to the data depicted in Figure \ref{fig:toy} using subspaces of dimension $1$ (with the exception of \Acronym{lda} that projects the data onto a space of dimension $\nUniCat-1=2$). To generate the plots, we divided the data into a training set (displayed using the `+' marker) and a test set (displayed using the `o' marker). Samples were drawn in random order from the dataset and assigned to either the training set or the test set, with the former containing $70\%$ of the total data and the latter containing the remaining $30\%$. Then, we applied feature transforms on the training set, thereby learning the transform operators, and applied them to the test set.

Starting from the top-left plot, we can observe that \Acronym{PCA} identified the direction $x=y$ as the one-dimensional subspace that contains most of the variance of the training set. However, given the type of dataset and the dimensionality reduction caused by \Acronym{pca}, features from all classes are overlapping, making this transform a poor choice for classification. Similar observations can be drawn from analysing the result of \Acronym{s-pca}, although this transform identifies the direction $y=0$ as the one that leads to statistical dependence between the value of the transformed features in the training set and the relative class. \Acronym{lda} does not introduce any dimensionality reduction in this case, as it projects the features onto a space of dimension $\nUniCat-1=2$, leaving the original features unaltered. However, in the \Acronym{lda} plot we can appreciate the separation between training set and test set that is difficult to notice in the other plots.

Finally, the plot at the right-bottom corner of Figure \ref{fig:toyNewFea} displays the results of the \Acronym{s-ipr} algorithm. In setting the parameters of \Acronym{s-ipr}, we chose a $2$ times over-complete dictionary, a number of active atoms equal to half the dimension of the data, and minimal mutual coherence. In the case considered here, this means $\nAto = 4$, $\nActiveAtoms = 1$ and $\coherence = \sqrt{(\nAto-\nDim)/\nDim(\nAto-1)} \approx 0.33$. As discussed in Section \ref{sec:iprclass}, \Acronym{s-ipr} does not project whole sets of features onto a unique sub-space, but rather learns one sub-space for each category, and projects features onto the nearest sub-space. The result depicted here shows that three directions were identified containing data from mostly one category each. Since the incoherent dictionary learning is designed to learn atoms with minimal mutual coherence, the angles between the directions of the sub-spaces learned by \Acronym{s-ipr} are approximately equal. Prior information regarding the directions of the data would allow to relax the parameter $\coherence$, and track more closely the directions of the three data classes.

\subsubsection{Iris dataset}
\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{./Code/Datasets/GetFisherIrisDataset.pdf}
\caption{\label{fig:fisher3} First three features of the `iris' dataset depicting measurements of sepal length, sepal width and petal length of three iris species.}
\end{figure}
Figure \ref{fig:fisher3} displays a subset of the `iris' dataset, a popular database that has been used extensively to test and benchmark classification algorithms. The original dataset contains measurements of the sepal length, sepal width, petal length and petal width of three species of iris, namely `setosa', `versicolor' and `virginica'. In this visualisation experiment we selected the first $3$ features to be able to depict the data using three dimensional scatter plots. From observing the distribution of the data in the feature space, we see that `setosa' is relatively separated from the other two classes, while the features relative to `virginica' and `versicolor' substantially overlap, with only a few exemplars of `virginica' being distinguishable due to large sepal length and petal length.
\begin{figure}
\includegraphics[width=\textwidth]{./Code/fishervisu.eps}
\caption{\label{fig:fisher3NewFea}Feature transform applied to the iris data in Figure \ref{fig:fisher3}. Different colours correspond to different classes, `+' and `o' markers represent samples taken from the training and test set respectively.}
\end{figure}

The results of feature transforms are depicted in Figure \ref{fig:fisher3NewFea}. This time, we learn $2$ dimensional subspaces from the $3$ dimensional data points and plot the transformed features, along with the learned planes. We observe that \Acronym{pca} identifies a direction along a diagonal axis that follows the distribution of features displayed in Figure \ref{fig:fisher3}. \Acronym{s-pca}
, on the other hand, projects the features onto a horizontal plane that slightly enhances the separation between `versicolor' and `virginica' samples. \Acronym{lda} results in a projection where features belonging to the same category are closely clustered together, but fails to separate the classes `versicolor' and `virginica'. Finally, the output of \Acronym{s-ipr} displays three distinct sub-spaces associated with the three classes. As in the other plots, the separation between `versicolor' and `virginica' is far from perfect, however features from the  two classes are mostly projected onto the respective sub-spaces. Features belonging to the `setosa' category are mostly clustered together as a result of their projection onto the black subspace, however we can note a few test samples that have been associated by the algorithm to the blue sub-space.
\subsection{Classification}\label{sec:cla}
\begin{table}
\centering
\begin{tabular}{lccc}\textbf{Name} & \nDim & \nUniCat & \nFea \\
\hline
Iris & 4 & 3 & 150\\
Balance & 4 & 3 & 625\\
Parkinsons & 23 & 2 & 197 \\
Sonar & 60 & 2 & 208\\
USPS & 256 & 3 & 1405
\end{tabular}
\caption{\label{tab:datasets}Dataset used in the classification evaluation of feature transform algorithms. All the datasets can be downloaded from \href{http://archive.ics.uci.edu/ml/datasets.html}{http://archive.ics.uci.edu/ml/datasets.html}. Note that we only use a subset of the USPS dataset containing the digits $1$, $3$ and $8$.}
\end{table}

In the previous section, we have illustrated how the \Acronym{s-ipr} algorithm is able to learn incoherent sub-spaces that model the distribution of features belonging to different classes. Here we evaluate \Acronym{s-ipr} and the other feature transform algorithms in the context of supervised classification. To perform the classification, features are transformed using the methods already used for comparison in Section \ref{sec:visu} by learning a transform operator on the training set and applying it to the test set. We use a $5$-fold stratified cross-validation to classify all the features in a dataset during the test stage. This method produces $5$ independent classification problems with a ratio between the number of training and test samples equal to $8:2$. Once the features have been transformed, a $k$-nearest neighbour classifier with $k=5$ is used to estimate a class.

We employ the datasets detailed in Table \ref{tab:datasets}, and for each of them we evaluate the misclassification ratio, that is defined as the fraction of misclassified samples as a proportion of the total number of samples in the test set, averaged over the $5$ independent classification problems created by the stratified cross-validation protocol.
\begin{figure}
\centering
\subfigure{\includegraphics[width=.48\textwidth]{Code/Util/fisheriris.pdf}}
\subfigure{\includegraphics[width=.48\textwidth]{Code/Util/balance.pdf}}
\subfigure{\includegraphics[width=.48\textwidth]{Code/Util/parkinsons.pdf}}
\subfigure{\includegraphics[width=.48\textwidth]{Code/Util/sonar.pdf}}
%\subfigure{\includegraphics[width=.7\textwidth]{Code/Util/usps.pdf}}
\caption{\label{fig:class} Misclassification ratio as a function of the rank of the subspace employed during feature transforms for the datasets  `iris',`balance',`Parkinsons' and `sonar'}
\end{figure}

Figure \ref{fig:class} displays for each dataset the misclassification ratio as a function of the rank of the subspace learned by the algorithms. In the plots `none' indicates that no feature transform was applied (hence resulting in a sub-space rank equal to the dimension of the original features). In general we can see that \Acronym{s-ipr} does not perform as well as the other techniques, and is only comparable at high ranks that do not achieve an overall better classification ratio. Starting from the `iris' dataset, \Acronym{lda} achieves the best performance followed by one-dimensional subspaces learned using \Acronym{pca}. Both \Acronym{s-pca} and \Acronym{s-ipr} work better when learning subspaces of high rank. Note that, at rank $\nDim=4$ all the methods are equivalent because they are not performing dimensionality reduction. The results relative to the balance dataset are similar, with again \Acronym{lda} achieving the best misclassification ratio. Although the results on the `Parkinsons' and `sonar' datasets present similar trends regarding \Acronym{s-ipr}, here \Acronym{lda} does not prove to be as successful as \Acronym{pca} and \Acronym{s-pca} in separating features belonging to different classes.

\section{Conclusion}\label{sec:end}
\subsection{Summary}
We have presented the \Acronym{s-ipr} algorithm for learning incoherent subspaces from data belonging to different categories. The encouraging experimental results obtained on the visualisation of the synthetic dataset and of a subset of features taken from the 'iris` dataset motivated us to test \Acronym{s-ipr} as a general method for feature transform to be used in classification problems. Unfortunately, we found that the performance of our proposed method on a group of datasets commonly used to benchmark classification algorithms is only competitive compared to traditional and state-of-the-art methods for feature transform at high sub-space ranks.

The negative results presented in Section \ref{sec:cla} do not imply that \Acronym{s-ipr} is completely unsuitable as a tool for modelling data for classification, but they rather open a few important areas of future research that should be investigate to better understand the strengths and limitations of the proposed method.

\subsection{Future work}
The main assumption made when using incoherent dictionary learning for classification is that high dimensional features are arranged onto lower-dimensional sub-spaces, and that features belonging to different classes can be modelled using different subspaces that are mutually incoherent. This assumption might be met by some datasets, but might not generally be satisfied by others. Understanding the general distribution of the features in a  dataset might be a necessary first step to inform a subsequent choice of algorithm, so that \Acronym{s-ipr} can be used in cases where its premise about the feature distribution is valid. This same argument holds for the whole class of linear models that comprises the dictionary learning model. Indeed, many feature transform techniques have equivalent \emph{kernelized} versions to model non-linear data.

Other substantial improvements can be made on the algorithm itself. The present implementation of \Acronym{s-ipr} contains a fixed parameter $\coherence$ that promotes minimal mutual coherence between the sub-spaces used to approximate different data classes. Knowledge about the distribution of the features might lead to relaxing this parameter, learning sub-spaces that are closer to the true distribution of the features and in turn improving class separation. Moreover, different values of mutual coherence for different pairs of subspaces can be easily included in the optimisation, greatly enhancing the flexibility of \Acronym{s-ipr} as a modelling tool.

\bibliographystyle{IEEEbib}
\bibliography{bibliography.bib}
\end{document}