\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 4.6 Solution of a Positive-Definite System \hbox{with Cholesky Factorization}

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

This subroutine computes the solution vector ${\bf x}$ for a system of
equations of the form
\begin{equation}
\label{O1}P{\bf x}={\bf d},
\end{equation}
where $P$ is an N$\times $N positive-definite symmetric matrix, and ${\bf d}$ is an
N-vector. This subroutine also returns the Cholesky factor of $P$ and thus is
applicable where computing the Cholesky factor is the objective.

\subsection{Usage}

\subsubsection{Program Prototype, Double Precision}

\begin{description}
\item[DOUBLE PRECISION]  {\bf P}(LDP, $\geq $N) [LDP $\geq $ N]{\bf , D}($%
\geq $N$)${\bf , U, TOL}

\item[INTEGER]  \ {\bf LDP, N, IERR}
\end{description}

Assign values to P(,), LDP, N, D(), U, and TOL.
$$
\fbox{{\bf CALL DCHOL (P, LDP, N, D, U, TOL, IERR)}}
$$
The solution vector ${\bf x}$ will be stored in D(). Additional computed
quantities that may be of interest to the user in some situations will be
stored in P(,) and U.

\subsubsection{Argument Definitions}

\begin{description}
\item[P(,)]  \ [inout] On entry this array must contain the N$\times $N
symmetric positive-definite matrix $P$ of Eq.\,(1). It suffices to provide only
the elements on and above the diagonal. On return this array will contain the
N$\times $N upper triangular matrix F defined by Eq.\,(2) on and above the diagonal
positions of the array P(,). Locations of the array P(,) below the diagonal
will not be referenced or modified by this subroutine.

\item[LDP]  \ [in] Dimension of the first subscript of the storage array
P(,). Require LDP $\geq $ N.

\item[N]  \ [in] Order of the matrix $P$. Require N $\geq 1.$

\item[D()]  \ [inout] On entry D() must contain the vector ${\bf d}$ of Eq.
(1). On return D() contains the solution vector ${\bf x}$ for Eq.\,(1).

\item[U]  \ [inout] If U contains the number ${\bf u}$ of Eq.\,(10) or (15)
respectively on entry, then on return U will contain the number $\rho $ of
Eq.\,(11) or (16) respectively. If the user is not interested in having the
number $\rho $ computed, U should be zero on entry and will be unchanged on
return.

\item[TOL]  \ [in] A user-provided relative tolerance parameter to be used
in the conditioning test of Eq.\,(17). We suggest setting TOL to a value of $%
10^{-(k+1)}$ where $k=\min (k_A$, $k_b)$. Here $k_A$ is the user's estimate
of the number of significant decimal digits in the elements of the matrix $A$
and $k_b$ is the corresponding estimate for ${\bf b}$. See Eq.\,(7) or (12)
for the definitions of $A$ and ${\bf b}$. If the TOL input is $<\varepsilon $,
where $\varepsilon $ is the relative machine precision $(i.e$. the smallest
positive number such that $1.0+\varepsilon \neq 1.0$ in the machine's
floating point arithmetic), then $\varepsilon $ is used for TOL internally.

\item[IERR]  \ [out] On return this is set to~0 if $t_{\min }$ defined in
Eq.\,(17) is greater than~0. Otherwise results are of
questionable validity and $|$IERR$|$ will equal the index of the equation
that resulted in the value for $t_{\min }$. See Section E for more details.
\end{description}

\subsubsection{Modifications for Single Precision}

We recommend the use of double precision for this computation except on
machines such as the Cray that have $10^{-14}$ precision in single
precision. To use single precision change DCHOL to SCHOL, and the DOUBLE
PRECISION type statement to REAL.

\subsection{Examples and Remarks}

Consider the least-squares problem $A{\bf x}\simeq {\bf b}$ where $A$ is the
3~by~2 matrix and ${\bf b}$ is the 3-vector defined by the DATA statements
in the program DRDCHOL below. This program forms normal equations by
computing $P=A^TA$ and ${\bf d}=A^T{\bf b}$. It also computes u $={\bf b}^T%
{\bf b}$. It uses the subroutine DCHOL to solve the normal equations $P{\bf x%
}={\bf d}$, and to compute the quantity RNORM $=\rho =\Vert {\bf b}-A{\bf x}%
\Vert $. Output from this program is given in the file ODDCHOL.

For programming convenience one may prefer to store the matrix $A$ and the
vector ${\bf b}$ together in the same array. The code in DRDCHOL2 shows how
this example can be programmed storing $A$ and ${\bf b}$ together in the
array AB() and using the array PDU() to hold $P$, ${\bf d}$, and $u.$

\subsection{Functional Description}

Given the problem $P{\bf x}={\bf d}$, where $P$ is an N$\times $N
positive-definite symmetric matrix, there exists an upper triangular
N$\times $N matrix $F$ satisfying
\begin{equation}
\label{O2}F^TF=P
\end{equation}
Eq.\,(2) defines the Cholesky decomposition of $P$. The upper triangular
elements of $F$ will be computed from those of $P$ by the following
equations, where $i=1$, ..., N.
\begin{align}
\label{O3}g_i&=p_{i,i}-\sum_{k=1}^{i-1}f_{k,i}^2\\
\label{O4}f_{i,i}&=g_{i}^{1/2}\\
\label{O5}f_{i,j}&=\frac{\displaystyle p_{i,j}-\sum_{k=1}^{i-1}f_{k,i}f_{k,j}%
}{f_{i,i}},\ \ j=i+1,...,\text{N}\hspace{-2in}
\end{align}
In these formulas the summation is to be skipped when $i=1.$

After computing $F$ the subroutine solves the lower triangular system of
equations,%
\begin{equation*}
F^T{\bf y}={\bf d}
\end{equation*}
and then computes the vector ${\bf x}$ which satisfies $P{\bf x}={\bf d}$ by
solving the upper triangular system%
\begin{equation*}
F{\bf x}={\bf y}
\end{equation*}
Besides computing the solution vector ${\bf x}$ this subroutine uses the
input number $u$ given in the Fortran variable U to compute
\begin{equation}
\label{O6}\rho =\left[ \max (0,u-{\bf y}^T{\bf y})\right] ^{1/2}
\end{equation}
This number $\rho $ is stored in U on return. If the problem $P{\bf x}={\bf d%
}$ arose as the system of normal equations for a least-squares problem and
if $u$ was computed appropriately by the user then $\rho $ represents the
norm of the residual vector for the least-squares problem.

Specifically if the user wishes to solve the least-squares problem of
minimizing
\begin{equation}
\label{O7}\left\| {\bf b}-A{\bf x}\right\| =\left[ \left( {\bf b}-A{\bf x}%
\right) ^T\left( {\bf b}-A{\bf x}\right) \right] ^{1/2}
\end{equation}
then $P$, ${\bf d}$, and $u$ should be initialized as
\begin{align}
\label{O8}P&=A^TA\\
\label{O9}{\bf d}&=A^T{\bf b}\\
\label{O10}u&={\bf b}^T{\bf b}
\end{align}
Then theoretically, the quantity, $u-{\bf y}^T{\bf y}$ of Eq.\,(6) will be
nonnegative and the number $\rho $ of Eq.\,(6) will have the interpretation
\begin{equation}
\label{O11}\rho =\left\| {\bf b}-A{\bf x}\right\|
\end{equation}
More generally, if the user is solving the weighted least-squares problem of
minimizing
\begin{equation}
\label{O12}\left[ \left( {\bf b}-A{\bf x}\right) ^TW\left( {\bf b}-A{\bf x}%
\right) \right] ^{1/2}
\end{equation}
where $W$ is a positive definite symmetric matrix, then $P$, ${\bf d}$, and $%
u$ should be initialized as
\begin{align}
\label{O13}P&=A^TWA\\
\label{O14}{\bf d}&=A^TW{\bf b}\\
\label{O15}u&={\bf b}^TW{\bf b}
\end{align}
Then, theoretically, the quantity $u-{\bf y}^T{\bf y}$ of Eq.\,(6) will be
nonnegative and the number $\rho $ of Eq.\,(6) will have the interpretation
\begin{equation}
\label{O16}\rho =\left[ \left( {\bf b}-A{\bf x}\right) ^TW\left( {\bf b}-A%
{\bf x}\right) \right] ^{1/2}
\end{equation}
The Cholesky factor matrix $F$ will appear in the upper triangular portion
of the array P(,) on return. If IERR $\geq 0$, the user can input this
matrix $F$ to the library subroutine DCOV2 of Chapter~4.2 to compute the
unscaled covariance matrix for the associated least-squares problem. This
requires building the IP() array: IP(I) = I, for I $=1$, ..., N.

Theoretically the numbers $g_i$ of Eq.\,(3) will be strictly positive for all
$i$ if and only if the symmetric matrix $P$ is positive-definite. If all $%
g_i $ are positive but the ratio $g_i/p_{i,i}$ is very small for some $i$
this is an indication that the problem is ill-conditioned. The square of the
relative tolerance parameter TOL is used to test this ratio. Let
\begin{equation}
\label{o17}t_{\min }=\min _{1\leq i\leq \text{N}}\left\{ g_i-(TOL)^2\times
|p_{i,i}|\right\} .
\end{equation}
If $t_{\min }\geq 0$, then IERR is set to~0. Otherwise let $m$ be a value of
$i$ that gives the minimum value in Eq.\,(17). Then IERR is set to $m$ if $%
g_m>0$, and is set to $-m$ otherwise. See Section E below for more details.

If one knows or suspects that the least-squares problem is ill-conditioned
it is suggested that the Singular Value Analysis subroutine, Chapter~4.3, be
used to obtain a more complete analysis and a more reliable solution for the
problem.

A nonnegative definite symmetric matrix has a Cholesky factor even if it is
singular. In computing a Cholesky factor for such matrices this subroutine
does the following: If $g_i$ of Eq.\,(3) is nonpositive, Eqs.~(4--5) are
replaced by
\begin{equation}
\label{O18}f_{i,j}=0,\quad j=i,i+1,...,\text{N}
\end{equation}
When solving the triangular systems below Eq.\,(5), if $f_{i,i}=0$ the
solution components $y_i$ and $x_i$ are set to zero. If $P$ is a singular
nonnegative definite matrix, the matrix $F$ produced in this way is its
(nonunique) Cholesky factor, $i.e.$, it satisfies Eq.\,(2). In such a case
Eq.\,(1) may or may not have a solution and the vector ${\bf x}$ produced in
this way is the solution only if a solution exists.

\subsection{Error Procedures and Restrictions}

If $t_{\min }<0$ in Eq.\,(17), the subroutine sets IERR nonzero as
indicated above. When IERR $< 0$, at least one row of the augmented
matrix [upper triangle of $P$, $D$] will have been set to zero.

If IERR $\neq 0$ we suggest that the user apply the Singular Value Analysis
subroutine, Chapter~4.3, to the associated least-squares problem.\vfill

\subsection{Supporting Information}

The source language is ANSI Fortran~77.

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DCHOL & \hspace{.3in} AMACH, DCHOL\rule[-5pt]{0pt}{8pt}\\
SCHOL & \hspace{.3in} AMACH, SCHOL\\
\end{tabular}

Programmed by: C. L. Lawson, JPL, May~1969.

Program Revised by: F. T. Krogh, JPL, September~1991.

\begcode

\medskip\
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRDCHOL}\vspace{10pt}
\lstinputlisting{\codeloc{dchol}}

\vspace{30pt}\centerline{\bf \large ODDCHOL}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{dchol}}

\newpage\
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRDCHOL2}\vspace{10pt}
\lstinputlisting{\codeloc{dchol2}}

\vspace{30pt}\centerline{\bf \large ODDCHOL2}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{dchol2}}
\end{document}


\documentclass[a4paper, 11pt]{article}
\input{../preamble.tex}

\DeclareMathOperator{\N}{\mathbb{N}}
\DeclareMathOperator{\R}{\mathbb{R}}
\DeclareMathOperator{\Q}{\mathbb{Q}}
\DeclareMathOperator*{\Z}{\mathbb{Z}}

\title{The Exponential Function}

\begin{document}

\tableofcontents
\section{The exponential Function}
\begin{definition}[The exponential Function]
The function $\exp: \R \rightarrow \R$ is  is defined by $\sum_{n=0}^{\infty}\frac{x^{n}}{{n}!}$.
Let $\phi: \R \rightarrow \R$ defined by $\phi(x) = \lim_{n\rightarrow \infty} \left( 1 + \frac{x}{n} \right)^n$.
We wish t that $\phi = \exp$
\end{definition}

\begin{proposition}
Let us fix $x \in \R$. Then:
\begin{enumerate}
  \item For every natural number $n > |x|$, then $\left(1 + \frac{x}{n}\right)^{n} < (1 - \frac{x}{n})^{-n}$
  \begin{proof}
    Suppose $n > |x|$. Then $0 \leq \frac{x^{2}}{n^{2}}<1$. Then using this fact,
    consider the product
    \begin{align*}
      \left(1 + \frac{x}{n}\right)^n \cdot \left(1 - \frac{x}{n}\right)^n
      &= \left(\left(1 + \frac{x}{n}\right)\left(1 - \frac{x}{n}\right)\right)^n \\
      &=\left(1 - \frac{x^2}{n^2}\right) \\
      &<1
    \end{align*}
    This follows from multiplying both parts of the previous inequality by the same power. If we divide
    both sides by $\left(1 - \frac{x}{n}\right)^n$, we conclude that $\left(1 + \frac{x}{n}\right)^n  < \left(1 - \frac{x}{n}\right)^{-n}$.

  \end{proof}
  \item The sequence $\left(1 + \frac{x}{n}\right)^n$ for $n > |x|$ is strictly increasing.
  \begin{proof}
  Suppose $n > |x|$, we want to show that $(n + 1)$st element of the sequence is greather than the $n$th element of the sequence.

  That is $\left(x + \frac{x}{n}\right)^n \leq \left(x + \frac{x}{n+1}\right)^{n + 1}$. The A + M inequality:
  Assume that $n \geq2 $ and given $a_1, \dots a_n \in \R,$ $(a_1 \dots a_n)^{1/n} \leq \frac{a_1 + \dots a_n}{n}$.
  Consider $a_1 = 1, a_2 = \left(1 + \frac{x}{n}\right), \dots a_{n+1} = \left(1 + \frac{x}{n}\right)$.
  \end{proof}
  \item The sequence $\left(1 - \frac{x}{n}\right)^{-n}$ for $n > |x|$ is strictly decreasing.
  \item $\left(1 + \frac{x}{n}\right)^{n} - \left(1 - \frac{x}{n}\right)^{-n} \rightarrow 0$ as $n \rightarrow \infty$.
  These two sequences approach each other from the left and right.
\end{enumerate}
\end{proposition}

By (2), (3), and (4) of the proposition for $n > |x|$, the sequence of the closed intervals,
\begin{equation*}
  \left[ \left(1 + \frac{x}{n}\right)^n, \left(1 - \frac{x}{n}\right)^{-n}\right]
\end{equation*}
is nested. By the \textbf{Nested Interval Lemma}, there is a unique real number contained in the intersection of all of these
intervals. The common limit of the sequences of left endpoints and the sequence of right endpoints.

This enables us to make the following defintion
\begin{definition}
Define the function $\phi: \R \rightarrow \R$  for each $x \in \R$ where
\begin{equation}
  \phi(x) = \lim_{n\rightarrow \infty} \left(1 + \frac{x}{n}\right)^n = \left(1 - \frac{x}{n}\right)^{-n}
\end{equation}
Note: $\phi(0) = 1$. We also define $e = \phi(1)$ as the \textbf{Euler's constant}.
\end{definition}

\begin{remark}{}
  \begin{enumerate}
    \item For every natural number $n \in \N$ such that $n > |x|$
    \begin{equation}
      \left(1 + \frac{x}{n}\right)^n \leq \phi(x) \leq \left(1 - \frac{x}{n}\right)^{-n}
    \end{equation}
    \item
  \end{enumerate}
\end{remark}
\end{document}


\section{Existing Protocols for Local DP}

Apart from R.R., several L.D.P. protocols have been implemented during the years, with many of them being widely used by companies in order to protect users' data. One of the most famous protocols is \emph{RAPPOR}([15]), created by Google, and being currently used in the Chrome browser for the company to provide useful info to its users without compromising their privacy. Also, Apple has created ts own protocol of L.D.P., and utilize it in their products.

However, we are not going to focus on those protocols moving forward, than the ones presented in [10], a paper which introduces many algorithms for L.D.P., each one with different perturbation techniques and suitable for different circumstances.

During this chapter we are going to give a definition of each algorithm, implement it using Python, and compare the accuracy results produced by those protocols, just like during our testings of the G.D.P. models. Each protocol has two parts: the \emph{users} and the \emph{aggregator}. For the users we must each time define the following functions:

\begin{itemize}
    \item $Encode()$: Encodes the true value that the user wants to report
    \item $Perturb()$: Perturbs the encoded value, in order to produce the random value that will be reported
\end{itemize}

For the aggregator we must each time define the  $Aggregate()$ function, that collects the reported random values of the users, and produces the results according to the model.

\subsection{Basic RAPPOR}
As mentioned earlier, RAPPOR is a protocol created by Google. Its simpler form, Basic RAPPOR is used in Chrome, where it collects answers to questions such as the user's home page. The protocol's functions are the following:

\textbf{Encoding:} $Encode(v) = A_0$, where $A_0$ is a d-bit vector, such that: $A_0[v] = 1$ and $A_0[i] = 0$ for every $i \neq v$.

\textbf{Perturbation:} The perturbation consists of 2 steps: the permanent and the instantaneous. The permanent one is carried out only one time, and is the following:

\begin{equation*}
    Pr[A_1[i] = 1] =
	\begin{cases}
		1 - \frac{1}{2}f & \mbox{if } A_0[i]=1 \\
		\frac{1}{2}f & \mbox{ otherwise}
			\end{cases}
\end{equation*}

The instantaneous step is carried out every time a user reports a value, and is defined as:

\begin{equation*}
    Pr[A_2[i] = 1] =
	\begin{cases}
		p & \mbox{if } A_1[i]=1 \\
		q & \mbox{ otherwise}
			\end{cases}
\end{equation*}

We observe from the above functions, that the user must define the $f, p $ and $q$ parameters. Google suggests that we set $f = \frac{1}{2}$ or $\frac{1}{4}$, and $p = 0.75$, thus $q = 0.25$. During our testings, those exact parameters were used.

\subsection{Random Matrix Projection}
In [14], a protocol with a random matrix projection is proposed, introducing an additional setup step.

\textbf{Setup:} A random and uniform matrix is generated before any encoding, with it being public and drawn as: $\Phi \in \{-\frac{1}{m}, \frac{1}{m}\}^{m \times d}$, where $m$ and $d$ are user defined. In our testings, we opt to set $m = 5$ and $d = 10$.

\textbf{Encoding:} When it comes down to encoding, the function used is the following: $Encoding = (r,x)$, where $r$ is uniformly randomly selected from the range of m, and $x$ is the v-th element of the r-row of the random matrix.

\textbf{Perturbation}: The perturbation function is defined as following:
\begin{align*}
 Perturb(r,x) = (r, b\cdot c \cdot m \cdot x)
\end{align*}
 where
\begin{equation*}
    b =
	\begin{cases}
		1 & \mbox{with } p = \frac{e^\epsilon}{e^\epsilon + 1} \\
		-1 & \mbox{with } q = \frac{1}{e^\epsilon + 1}
	\end{cases}
\end{equation*}
and $c = \frac{e^\epsilon +1}{e^\epsilon -1}$

\textbf{Aggregation:} Given all the tuples reported by $j$ users in the form $(r, y)$, the estimation for the i-th value of the dataset, is produced by
\begin{align*}
\sum_{j} y^j \cdot \Phi[r^j,i]
\end{align*}

\subsection{Pure Protocols}

The following protocols are presented in [10], and are called "pure" protocols, because of the way they aggregate the data produced by the user. For each one of them, we should define a $Support()$ function, that indicates for each value of the possible outcomes, the reported values that are supported. Thus, with the notation $\sum_{j} Support(y^j)$, we mean the sum of all the supported values of the y-th element of the dataset.

Also, for a protocol to be pure, two probabilities must be defined, $p^*$ and $q^*$, where the first notes the probability that the true value is supported by an element $y$, and the second one the probability of another value is supported by the element $y$. The protocol is pure if and only if $p^* > q^*$.

If a protocol is pure, the estimation of the total reported values for an element of the dataset $i$, is the following:

\begin{align}
    \text{Estimation} = \frac{\sum_{j} 1_{support(y^j)}(i) - nq*}{p^* - q^*}
\end{align}
where $j$ denotes the j-th user reporting their value, and $n$ the total size of the vector of the reported values.

\subsubsection{Direct Encoding}

This protocol is the natural method of extending the Randomized Response, without the limitation of 2 possible answers.

\textbf{Encoding:} The protocol does not feature an encoding procedure, thus

\begin{align*}
    Encode(v) = v
\end{align*}

\textbf{Perturbation:} The perturbation is based on the epsilon setting given by the user, and its function is defined as following:

\begin{equation*}
    Pr[Perturb(x) = i] =
	\begin{cases}
    	p = \frac{e^\epsilon}{e^\epsilon + d - 1} & \mbox{if } i = x \\
    	q = \frac{1}{e^\epsilon + d - 1} & \mbox{if } i \neq x
	\end{cases}
\end{equation*}

where $d$ the size of the dataset of the possible answers, $x$ the true value and $i$ the value selected.

\textbf{Aggregation:} The protocol is pure with $p^* = p$, $q^* = q$ and $Support(i) = i$, thus the predicted results for each of the dataset's values can be calculated from the Equation 4.1.

We observe that this protocol strongly depends on the size of the dataset of the possible answers, thus when the dataset size increases, the protocol becomes less accurate, due to the decreased probability of selecting the truth. Moreover, for the D.E. protocol, all the false values have the same probability to get chosen, a rather disturbing detail for a query such as a person's age. We will return to these thoughts on later sections.

\subsubsection{Histogram Encoding}

An other protocol presented is Histogram Encoding, where an input when having $d$ options is encoded as a $d$-length vector.

\textbf{Encoding:} The encoding function is for the protocol is

\begin{align*}
    Encoding(v) = [0, 0, \dots, 1, \dots, 0]
\end{align*}

where only the v-th element of the vector is equal to 1.

\textbf{Perturbation:} The result of perturbing the encoded vector, is a new vector $B'$, s.t.:

\begin{align*}
    B'[i] = B[i] + Lap(\frac{2}{\epsilon})
\end{align*}
where $Lap()$ denotes the noise drawn from the Laplace distribution, where

\begin{align*}
    Pr[Lap(\beta) = x] = \frac{1}{2\beta}e^{\frac{-|x|}{\beta}}
\end{align*}

\textbf{Aggregation:} Several methods are proposed for aggregating the results created by the H.E. protocol, but as mentioned by the authors, the best one is called \emph{Thresholding with H.E.}, where a threshold value is introduced in order to decide what to keep from the reported values. The support function is altered as following:

\begin{align*}
    Support(B) = \{v | B[v] > \theta\}
\end{align*}
thus, if a noisy output is grater than theta, is set to support the corresponding value. According to the authors, the optimal value for θ is in the range of $(\frac{1}{2}, 1)$. During the testings that are going to be conducted, we are going to use a threshold of $\frac{2}{3}$.

Comparing this protocol to D.E., we observe that is solves the problem of the dependence of the noise drawn by the number of options to choose from. In H.E., no matter how large the domain size is, the noise solely depends on the epsilon value chosen by the user. Thus, when having a large domain size, it is clear that we should prefer the H.E. protocol over D.E.

\subsection{Unary Encoding}

The last protocol that is going to take part in the accuracy testings, is the Unary Encoding method, a further exploration of the Basic RAPPOR. It is a unique protocol, as the user does not set the level of privacy using epsilon, but by giving two probabilities, $p$ and $q$ and the epsilon value is computed using those two parameters.

\textbf{Encoding:} Exactly like in the H.E. method:
\begin{align*}
Encoding(v) = [0, 0, \dots, 1, \dots, 0]
\end{align*}
where only the v-th element of the vector is equal to 1.

\textbf{Perturbation:} This step is different than those that we already saw, and is carried out using the following function:

\begin{equation*}
    Pr[B'[i] = i] =
	\begin{cases}
    	p  & \mbox{if } B[i] = 1 \\
    	q & \mbox{if } B[i] = 0
	\end{cases}
\end{equation*}

The epsilon value is decided given $p$ and $q$, and is defined as following:

\begin{align*}
    \epsilon = ln(\frac{p\cdot(1-q)}{(1-p)\cdot q})
\end{align*}

\textbf{Aggregation:} The Support function is once again altered, as in the U.E. protocol is defined as following:

\begin{align*}
    Support(B) = \{i | B[i] = 1\}
\end{align*}

and of course, $p^* = p$ and $q^* = q$, in order to make the protocol pure. As for the choice of $p$ and $q$, we opt to choose $p = \frac{1}{2}$, and $q = \frac{1}{e^\epsilon + 1}$.

\section{Testings}

\subsection{Setup}

Now that all those protocols where introduced, we are going to compare them in order to decide which is better to use when wanting to apply L.D.P. in a dataset. We are going to use a dataset that was created using random values, but corresponds to the age of a group of people. The distribution of the values of the dataset is shown in the histogram in \textbf{Figure 4.2. }

\begin{figure}[!htb]\centering
    \includegraphics[width=1\textwidth]{images/true_answers_ldp.png}
    \caption{True Answers for the Dataset of LDP}
\end{figure}

Each user will report one of these 50 values, and the aggregator of each protocol will gather the data given, and try to re-create this histogram in the best manner possible.

\subsection{Goal}

We want to decide which protocols behaves better, thus a number of different metrics will be used. The main focus of our testings will be the vectors that the aggregators provide, which we will compare with each other, as well as with the vector containing the true answers. In a similar way as our G.D.P. testings, we are going to run the protocols for different values of epsilon, and different number of users used. The second one is extremely important in L.D.P., as we mentioned earlier that many protocols struggle with a small number of input, as the noise drawn is significant.

With respect to the choice of metrics, we are going to use the \emph{Manhattan Distance}, known as the $l1$-norm, as well as the \emph{Kantorovic Distance}, explained in 3.2.5.1.

\subsection{Epsilon Measurements}

The first comparison between the protocols will be with a changing epsilon value, in order to observe how they behave for lower and higher values of the privacy setting. During these testings, all of the users of the dataset were used (approximately 20 thousand), and each run of each protocol was carried out 10 times, just like in other testings, in order to eliminate the danger of drawing extreme values of noise.

First, we are going to run all the protocols and compare them using the Manhattan Distance. The results are shown in \textbf{Figure 4.3.}

\begin{figure}[!htb]\centering
    \includegraphics[width=1\textwidth]{images/epsilon_others_l1.png}
    \caption{Epsilon Measurements compared by Manhattan Distance}
\end{figure}

We gather many useful observations from the graph:

\begin{itemize}
    \item The Random Matrix protocol does not function as expected, as \emph{its accuracy does not follow the logarithmic curve we are used to when epsilon increases.} However, for small values of ε, its results are acceptable, and some times even better than the pure protocols.
    \item \emph{The pure protocols behave in a similar way}, with the error stabilizing when epsilon gets higher than 2.5.
    \item \emph{The Direct Encoding protocol was the worst behaviour among the pure ones}, with its error being extremely high for epsilon values lower than 1.  This is mainly due to the fact that when ε gets too small, the probability of telling the truth gets significantly low, thus creating a big error in accuracy.
    \item \emph{The optimized U.E. protocol has the best behaviour} in comparison to the other protocols tested.
\end{itemize}

Next up, we are going to run the same testings, but this time using the Kantorovich metric. We expect the protocols to behave even worse, because of the identity of the metric: the Kant. metric pays attention to the distance of the reported answer from the true one. The current protocols do not take into account the distance of the two answers, thus the metric will probably report a higher error. The results of the runs are shown in \textbf{Figure 4.4.}

\begin{figure}[!htb]\centering
    \includegraphics[width=1\textwidth]{images/epsilon_others_kant.png}
    \caption{Epsilon Measurements compared by Kantorovic Distance}
\end{figure}

As we expected, the protocols produce a higher error, with the D.E. being the worst among them, especially for lower values of epsilon. This is a rather alarming notice, in which we will come back in Section 4.6.

\subsection{Increasing number of users}

The second experiment that we will conduct is the accuracy error depending on the number of users used during the survey covered by the protocol. In the definition of L.D.P. the observation of the need of lots of users was made, and it is now time that we examine it. We are going to use a \emph{fixed epsilon value}, one that our protocols behave similarly for (at least the pure ones, in which we will focus our research moving forward). Our epsilon value that we are going ot used will be fixed and equal to 1.5.

We are going to run the protocols and compare them using the Manhattan Distance. Additionally, we are each time going to divide the result of the metric with the number of users participated, as the simple error is going to increase when the users increase. Hence, this division is going to give us the error depending on the size of our domain. The results are shown in \textbf{Figure 4.5.}

\begin{figure}[!htb]\centering
    \includegraphics[width=1\textwidth]{images/nusers_others_l1.png}
    \caption{Increasing n. of users compared by Manhattan Distance}
\end{figure}

The results confirm the allegations made after explaining the definition of L.D.P.: When the number of participants in a survey is low, the error produced is very high. Every protocol has similar behaviour, as we can see that for fewer than 1000 users the relative error is even 6 times larger than for more than 1000 users. Actually, as we can see from the graph, the turning point is around 2000 users: the relative error drops and stabilizes after this number of participants.

An other observation made, is that the protocol with the best behaviour during our epsilon measurements (Unary Encoding), is the worst one in the low users testings. This of course gets better as the number of participants rises, and eventually beats the other protocols when it comes down to relative accuracy error.

Next up, we are going to perform the same testings while comparing the protocols with the Kantorovich metric. The results are shown in \textbf{Figure 4.6}.

\begin{figure}[!htb]\centering
    \includegraphics[width=1\textwidth]{images/nusers_others_kant.png}
    \caption{Increasing n. of users compared by Kantorovich Distance}
\end{figure}

The results for a low number of users differ a lot from the runs with the $l1$ distance as a metric. Here, the D.E. method has obviously the worst behaviour, and the U.E. method the best. We can even claim that U.E. seems to solve our problem of high accuracy error. However, no one of those protocols takes into account the distance of the reported answer from the true one, thus it makes sense that the Kantorovich metric produces a huge amount of accuracy error.

That fact, triggered our thoughts, on what could possibly be done in order to reduce that problem. The thoughts made on this subject are analyzed in the next chapter, by creating a new L.D.P. protocol, sensitive to the distance between the true and the reported values.


\section{The derivation of the ZOGY difference image}
%
\par In this document, we study some practical issues of performing
the ZOGY subtraction and its relation to the Alard--Lupton (AL) method
\citep{AL1998} combined with the decorrelation afterburner. We assume
that the reader is somewhat familiar with the Zackay--Ofek--Gal-Yam
(ZOGY) algorithm deduction steps in the ZOGY
paper\citep{ZOGY2016} Appendix A.
%
\par We recall four key ideas behind the ZOGY subtraction method: a) Given
an image with uncorrelated, homoscedastic pixel noise (the noise variance in
each pixel is the same value all over the image), its convolution with an
arbitrary kernel leads to noise correlation between the resulting
pixels. However, in frequency space, the frequencies remain independent (as
random variables), only the amplitude (variance) of their noise content
changes. b) The independent frequency space pixels are complex Gaussian
random variables. For a signal detection purpose, similar log
probability expressions can be written as for the real-valued random
variables (image pixels). c) Log probability can be calculated as a sum over
all the independent frequencies, weighting the squared absolute difference
at each frequency with its inverse noise variance. d) The detection
statistic in frequency space can be split into two multiplicative terms. In
image space, these two terms can be interpreted as a difference image and
its PSF, which produces a per-pixel detection statistic by convolution. The
difference image is constructed in frequency space so that each frequency
bin is a (complex) random variable and has the same variance. This implies
that the difference image in image space has uncorrelated (and under the
model assumptions), homoscedastic noise in its pixels. The difference image
noise remains uncorrelated despite the PSF matching procedure.
%
We recall the following equations from the ZOGY paper.
The new \(N\) science and the reference \(R\) images are modeled as:
\begin{align}
R &= F_rT\otimes P_r + \epsilon_r\\
N &= F_nT\otimes P_n + \epsilon_n
\label{eq:N}
\end{align}
where \(F_r, F_n\) are photometric scaling constants, \(T\) is the
\emph{truth} image, \(P_r, P_n\) are the image PSFs and \(\epsilon_r,
\epsilon_n\) are per-pixel Gaussian white noise with homogenous variance in
the images.
%
\par The detection statistic of \(N\) having a different \(T\) value than \(R\)
at any pixel position can be written in frequency space as:
\begin{equation}
  \hat{S} = \frac{F_nF_r^2\overline{\hat{P}_n}\abs{\hat{P}_r}^2 \hat{N}
 - F_rF_n^2\overline{\hat{P}_r}\abs{\hat{P}_n}^2 \hat{R}
}
{ \sigma_r^2F_n^2\abs{\hat{P}_n}^2 + \sigma_n^2F_r^2\abs{\hat{P}_r}^2 }
\label{eq:S}
\end{equation}
In image space, \(S\) is called the score or significance image and
represents the significance of a source detection for each pixel.
%
The difference image is defined as:
\begin{equation}
\hat{D} = \frac{F_r \hat{P}_r\hat{N} - F_n \hat{P}_n \hat{R}}
{\sqrt{\sigma_r^2F_n^2\abs{\hat{P}_n}^2 + \sigma_n^2F_r^2\abs{\hat{P}_r}^2}}
\label{eq:Dhat}
\end{equation}
and its PSF:
\begin{equation}
\hat{P}_D = \frac{F_rF_n\hat{P}_n\hat{P}_r}
{F_D\sqrt{\sigma_r^2F_n^2\abs{\hat{P}_n}^2 +
    \sigma_n^2F_r^2\abs{\hat{P}_r}^2}}
\label{eq:Pdhat}
\end{equation}
so that:
\begin{equation}
\hat{S} = F_D\hat{D}\overline{\hat{P}_D}
\label{eq:Shat}
\end{equation}
The difference image (and similarly the score image) can be written as the
difference of two ``matched'' images as in \Cref{eq:c1N_c2R}.
\begin{equation}
\hat{D}_z = \frac{\frac{\hat{P}_r}{F_n}}
{\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_r}^2
 + \frac{\sigma_r^2}{F_r^2}\abs{\hat{P}_n}^2}}
\hat{N} -
\frac{\frac{\hat{P}_n}{F_r}}
{\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_r}^2
 + \frac{\sigma_r^2}{F_r^2}\abs{\hat{P}_n}^2}}
\hat{R}
=
\hat{c}_n \hat{N} - \hat{c}_r\hat{R}
\label{eq:c1N_c2R}
\end{equation}
%
\par Here \(\hat{c}_n\) and \(\hat{c}_r\) are the matching kernels for
the original science and template images. If the original image PSFs
are accurately described by \(P_r, P_n\), then the frequency space
multiplications transform the PSFs of the two images to be identical,
\(P_D\), \Cref{eq:Pdhat}. Note that while we followed the terminology
of the ZOGY paper here and referred to the images as science and
reference images, the entire ZOGY method is symmetrical to the swapping
of the images. In the following, we may simply denote images with 1
and 2 indices.
%
\section{Discussion points}
We list the following questions that can define the direction of future
ZOGY image differencing code development in the LSST stack.
%
\par How does an ideal Gaussian PSF point source look like theoretically in
a ZOGY difference image? Discussed in \Cref{sec:ZOGYtheo}.  In
\Cref{sec:ZOGYFFT}, we look for answers: What causes the extensive,
oscillating visual patterns in the ZOGY difference image around certain
sources and image features (\Cref{sec:FFTlimits})? What shall we do with the
numerical problems that appear in certain regions in frequency space and
appear as pattern artifacts in image space (\Cref{sec:workaround})? Shall we
implement a Gaussian PSF approximator that produces the PSF frequency space
representation directly?  Shall we implement a Gaussian PSF width estimation
to determine which input PSF is sharper so that a realistic limiting value
can be used at frequencies when both PSFs (in frequency space) are below a
threshold (\Cref{sec:workaround})?  How shall we handle
division by zero scenarios in the ZOGY difference and significance image
calculation (\Cref{sec:PSFzero})?
%
\par In the Appendix, among other smaller topics, we raise the question whether
we can use zero padding for calculating the score image, or shall we use
model white noise padding (\Cref{sec:zeropadS})?
%
\section{The theoretical solution of the ZOGY matching kernel and difference
image PSF\label{sec:ZOGYtheo}}
%
\par In this section, we derive numerical solutions for pure Gaussian
PSFs. The inverse Fourier transforms of the ZOGY matching
kernel or difference image PSF expressions are not expressible in closed
symbolic forms, even in this case. We perform numerical integration of the
functions.
%
\begin{figure}
\begin{center}
  \includegraphics[width=5.5in]{fig/zogy_theo_Gaussians_ft_Pd_c1_c2.pdf}
\end{center}
\caption{\label{fig:theo_Gaussians_ft}1D slice along the x-axis in frequency
  space of the matching kernels and the PSF of the ZOGY difference
  image. The Fourier space representation of the input PSFs is also
  shown. The two PSFs have widths of $\sigma_1 = 1$, $\sigma_2 = 2$ in image
  space, i.e.\ \(\mathrm{PSF}_1\) is originally the narrower but the Fourier
  transformation swaps this relation.}
\end{figure}
%
\par In \Cref{fig:theo_Gaussians_ft}, we show 1D slices of the 2D solutions
of \(P_d\), \(c_1\) and \(c_2\). Noise variances and photometric scaling
factors are unity for simplicity. As the input PSFs are pure Gaussians,
i.e.\ symmetric, real value functions, their Fourier transforms are also
real and symmetric.\footnote{Detailed calculation notebooks are part of
  DM-26087.} Note the different behavior of the two matching kernels towards
high frequencies. The matching kernel of the narrower PSF image \(c_1\) goes
to zero, while the other one goes to unity here. While the graphs of \(c_1\)
and \(c_2\) resemble to Gaussians, they are not anymore, and we must use
numerical integration to calculate their inverse Fourier transform. Their
image space values for points along the x-axis is shown in
\Cref{fig:theo_Gaussians_img_c1,fig:theo_Gaussians_img_c2}.
%
\begin{figure}
\begin{center}
\includegraphics[width=5.5in]{fig/zogy_theo_Gaussians_img_Pd.pdf}
\end{center}
\caption{\label{fig:theo_Gaussians_img_Pd}1D slice along the x-axis in image
  space of the PSF of the ZOGY difference image with the input image PSFs.}
\end{figure}
%
\par In \Cref{fig:theo_Gaussians_img_Pd}, the PSF of the ZOGY difference
  image is shown. It is close to the wider input PSF, but strictly it's not
  a Gaussian, it has a negative overshoot, about 1\% of its peak
  value. This means that in an ideal case, signals in a ZOGY difference
  image are expected to have small negative rings around their positive
  peaks.
%
\begin{figure}
\begin{center}
\includegraphics[width=4.5in]{fig/zogy_theo_Gaussians_img_c1.pdf}
\includegraphics[width=4.5in]{fig/zogy_theo_Gaussians_img_c1_tails.pdf}
\end{center}
\caption{\label{fig:theo_Gaussians_img_c1}1D slice along the x-axis in image
  space of the matching kernel for the narrower PSF image. The matching
  kernel is a Gaussian-like curve that has a small oscillating correction in
  the tails.}
\end{figure}
%
\par For the narrower PSF input image, the matched PSF is created by
convolution with the matching kernel \(c_1\), shown in
\Cref{fig:theo_Gaussians_img_c1}. This matching kernel is similar to usual
Gaussian blurring but slightly narrower and has a negative tail itself.
%
\begin{figure}
\begin{center}
\includegraphics[width=4.5in]{fig/zogy_theo_Gaussians_img_c2.pdf}\,
\includegraphics[width=4.5in]{fig/zogy_theo_Gaussians_img_c2_tails.pdf}
\end{center}
\caption{\label{fig:theo_Gaussians_img_c2}1D slice along the x-axis in image
  space of the matching kernel for the wider PSF image. The matching kernel
 is the sum of a Dirac delta minus a Gaussian-like curve.}
\end{figure}
%
\par For the wider image, the matching kernel \(c_2\) is an identity Dirac
delta kernel minus a Gaussian-like correction
(\Cref{fig:theo_Gaussians_img_c2}). The Dirac delta is the inverse transform
of the non-zero constant level of \(c_2\) in \Cref{fig:theo_Gaussians_ft}
that must be subtracted for the numerical integration to converge. The Dirac
delta peak is manually added to the result in \Cref{fig:theo_Gaussians_img_c2}.
%
\par Note that in case of identical PSFs, both \(c_1\) and \(c_2\) become
constant in \Cref{fig:theo_Gaussians_ft} which correspond to Dirac deltas in
image space. I.e. the matching operation is naturally reduced to the
identity operation if the two PSFs are already identical.
%
\clearpage
%
\section{The FFT calculated matching kernel of the ZOGY difference image\label{sec:ZOGYFFT}}
%
\par In practice, the ZOGY subtraction is implemented by Fast Fourier
Transforms (FFT). In this and the following sections, we study
practical numerical aspects of this approach.
%
\par In \Cref{fig:hits_zogy_artifacts}, we show typical patterns that appear
around features that do not subtract well. These sources are present in all
visits in the HiTS2015 data AL image difference processings and in some
visits they produce different artifacts in the AL subtraction as well;
however, AL artifacts are spatially more localized to the source than in the
ZOGY case. The ZOGY patterns can also appear in the vicinity of masked
regions, cosmic rays, or close to the image edges.
%
\begin{figure}
\begin{center}
\includegraphics[width=3in]{fig/zogy_artifacts_v412060.png}
\includegraphics[width=3in]{fig/AL_goodvisit_v412060.png}
\end{center}
\caption{\label{fig:hits_zogy_artifacts}High frequency artifacts in the ZOGY
  difference image (left) around bright sources that are present in all
  visits in the AL processing as well. In the same visit, the AL subtraction
  (right) has less pronounced visual imperfections.}
\end{figure}
%
\subsection{Zero values of the PSF\label{sec:PSFzero}}
%
\par In \Cref{eq:Dhat}, \(\hat{D}\) is not defined at frequencies where both
image PSFs are zero. Indeed, according to the image models (\Cref{eq:N}), at
these frequencies, the input images do not carry any information about the
true image. They consist of pure noise. In accordance with this, these
frequencies have zero contribution to \(\hat{S}\).
%
\par We cannot allow zero division in our calculations anyway, thus we
need to have a workaround for pixels where the denominator in
\Cref{eq:S,eq:Dhat} are zero. We define \(\hat{D}\) at these
frequencies as the straightforward subtraction of the two images with
the same scaling to keep the variance constant at all
frequencies. Of course, \(\hat{S}=0\) at these pixels. This is
currently implemented in the code stack.
%
\subsection{Matching kernel limit values in frequency space\label{sec:FFTlimits}}
\begin{figure}
\begin{center}
\includegraphics[width=5.5in]{fig/fft_steps_fc1.pdf}
\end{center}
\caption{\label{fig:fft_steps_fc1}FFT calculated matching kernel for two
  Gaussian PSFs, here for the wider PSF image. This is a Fourier space image
  with low frequencies at the corners. In this calculation \(\sigma_1=3.3\),
  \(\sigma_2=2.2\) PSFs were generated in a 31x31 size image, that were zero
  padded to 1024x1024 image size before FFT. Per pixel noise variance is 100
  for both images, photometric scalings are unity. All values are real due
  to symmetry in the inputs.}
\end{figure}
%
\begin{figure}
\begin{center}
\includegraphics[width=5.5in]{fig/fft_steps_fc2.pdf}
\end{center}
\caption{\label{fig:fft_steps_fc2}FFT calculated matching kernel for the
  narrower PSF image. This is a Fourier space image with low frequencies at
  the corners. All values are real due to symmetry in the inputs.}
\end{figure}
%
\par We saw in the theoretical solution section, that in case of
Gaussian PSF-s, the matching kernels in frequency space have tails
converging to different limit values. The limit values are either zero
or a non-zero constant depending on whether the matching kernel belongs
to the narrower or wider input PSF image, respectively.
%
\par In \Cref{fig:fft_steps_fc1,fig:fft_steps_fc2}, \(\hat{c}_1\),
\(\hat{c}_2\) are calculated from two 31x31 pixel size Gaussian PSFs
that were zero padded for a 1024x1024 image size, with
\(\sigma_1=3.3\), \(\sigma_2=2.2\).\footnote{Detailed calculation
notebooks are part of DM-26941.} All numbers are real in this
case. The shown frequency space images are in their natural FFT
orientation with zero frequencies at the corners and highest
frequencies in the centers. Starting from the corners, both solutions
follow our expectations, converging either down to zero or to their
expected non zero constant (\( 1/\sigma_{\mathrm{pixelnoise}} \) )
plateau. The trend breaks for both kernels in high frequency regions
however, and high value noise appears.
\begin{equation}
\hat{c}_1 \sim \frac{1}{\sqrt{1 + \left(\frac{\hat{P}_1}{\hat{P}_2}\right)^2}}
\label{eq:c1conv}
\end{equation}
\par The matching kernel limit values depend on whether
\(\hat{P}_1/\hat{P}_2\) is converging to zero or diverges as it can be seen
in \Cref{eq:c1conv}. Once we reach the point where the Gaussian tails are
dominated by noises\footnote{See \Cref{sec:floating_point}}, the convergence
properties of these fractions become lost and the calculated matching kernel
values significantly deviate from their expected limit values.
%
\subsection{Patterns in image space\label{sec:patterns}}
\par What does this mean for our calculated matching kernels back in
image space?
%
\par In \Cref{fig:twoG_c1}, we show \(c_1\) transformed and
re-centered back to image space (but still in its fully padded image
size). The purple structure indicates that there is a sign oscillation
pattern all across padded size image.
\begin{figure}
\begin{center}
\includegraphics[width=5.5in]{fig/twoG_defaults_c1.pdf}
\end{center}
\caption{\label{fig:twoG_c1}Two Gaussian PSFs with spatial widths of \(\sigma_N = 3.3\)
  \(\sigma_R = 2.2\) pixels. \(c_1\), the ZOGY matching convolution in image space
  of the new (\(N\)) image. The purple pattern is an indication of
  sign oscillation all over the image.}
\end{figure}
%
\par We can see that in the direction of the two axes, there are definite
purplish patterns. The purple color on this red-blue color scale shows a
sign oscillation that can be verified in zoomed-in versions of the
figure. These patterns do not fade away in the direction of the axes from
the center, indicating that these oscillating sign values have roughly the
same order of magnitude absolute values. The original PSF size 31x31 cannot
be clearly identified any more in the image either. We note that
the appearance of these patterns is independent of the padding
size.
%
In \Cref{fig:twoG_pd} \(P_d\) is shown, calculated by \Cref{eq:Pdhat}.
\begin{figure}
\begin{center}
\includegraphics[width=5.5in]{fig/twoG_defaults_Pd.pdf}
\end{center}
\caption{\label{fig:twoG_pd}Two Gaussian PSFs with spatial widths of
  \(\sigma_N = 3.3\) \(\sigma_R = 2.2\). \(P_d\), the PSF of the zogy
  difference image.}
\end{figure}
We also show the PSF of \(S\) in \Cref{fig:twoG_ps}. The PSF of the
score image shows how a Dirac delta signal (in the truth image)
appears in \(S\), though in source detection, only the actual pixel
values matter in \(S\), the shape of the PSF does not.
\begin{figure}
  \begin{center}
    \includegraphics[width=5.5in]{fig/twoG_defaults_Ps.pdf}
  \end{center}
\caption{\label{fig:twoG_ps}Two Gaussian PSFs with spatial widths of
  \(\sigma_N = 3.3\) \(\sigma_R = 2.2\) \(P_s\), the PSF of the score
  image.}
\end{figure}
%
\par \(P_d\) and \(P_s\) have much cleaner images, contained in size
in image space, and close to our theoretical
expectations. (\(\hat{P}_s \sim \hat{P}_d \overline{\hat{P}_d}\)).
%
\par Recall that while we expressed \(P_d\) in \Cref{eq:Pdhat} as the
function of the input PSFs, in a difference image this is the result
of the convolution of the images with the matching kernels.  The high
frequency noise in the matching kernel is not disturbing, so long the
image follows the model PSF assumption and has approximately Gaussian
PSF features that suppress high frequencies. If there are edges, or
signals with high frequency components in the input images, the noisy
high frequency features of the matching kernel becomes visible in the
difference image. Our current understanding is that the deviation of
the image PSF from the model assumption and the numerical noise
in the matching kernels together cause the visible artifacts in the
difference images produced by the code stack. This conclusion is
supported by tests on simulated images that have sources only with
perfect Gaussian PSFs. In these cases, no visual artifacts can be seen.
%
\subsection{Workaround for artifact suppression\label{sec:workaround}}
We propose the following workarounds for the difference image artifact
problem:
\begin{itemize}
  \item In a Gaussian PSF approximation, we can directly create the
    PSF in the padded, full-size frequency space, avoiding the zero
    padding of a small image then the FFT operation. However, this
    approach restricts our input kernels strictly to Gaussians.
  \item In a more generic approach, we can still use the padded, FFT-d
    detected PSFs of the input images. Using a radius approximation, we
    can determine which input PSF is the wider one in a Gaussian
    approximation. Then we can introduce a configurable threshold in
    \emph{frequency space} and pixels in the matching kernels can be
    replaced with their Gaussian limit values wherever the input PSFs
    go below the threshold (in absolute value, in frequency space).
  \item As a third option, we should recall, that the noise artifacts
    appear only in the difference image. In the score image, these are
    automatically suppressed by further convolution with \(P_d\). We
    can choose to use the score image only directly for detection
    significance.
\end{itemize}
%
\par We repeated the above exercise by generating the Gaussian PSFs directly
in frequency space and performed exactly the same matching kernel and
\(P_d\) calculations. These results can be seen in
\Cref{fig:fft_steps_direct_fc1fc2,fig:fft_steps_direct_c1c2,fig:fft_steps_direct_Pd}. These
solutions fully satisfy the theoretical limit values and their image space
counterparts are free from any noisy patterns.
%
\begin{figure}
\begin{center}
  \includegraphics[width=4.5in]{fig/fft_steps_direct_fc1.pdf}
  \includegraphics[width=4.5in]{fig/fft_steps_direct_fc2.pdf}
\end{center}
\caption{\label{fig:fft_steps_direct_fc1fc2}The matching kernels for two
  Gaussian PSFs in frequency space. In this calculation, PSFs were
  directly generated in an 1024x1024 frequency space image
  corresponding to image space \(\sigma_1=3.3\), \(\sigma_2=2.2\)
  widths. Per pixel noise variance is 100 for both images, photometric
  scalings are unity.}
\end{figure}
%
\begin{figure}
\begin{center}
  \includegraphics[width=4.5in]{fig/fft_steps_direct_c1_zoomed.pdf}
  \includegraphics[width=4.5in]{fig/fft_steps_direct_c2_zoomed.pdf}
\end{center}
\caption{\label{fig:fft_steps_direct_c1c2}The matching kernels inverse
  FFT-d into image space, re-centered and zoomed in for
  details.}
\end{figure}
%
\begin{figure}
\begin{center}
\includegraphics[width=5.5in]{fig/fft_steps_direct_Pd_zoomed.pdf}
\end{center}
\caption{\label{fig:fft_steps_direct_Pd}The difference image PSF
  inverse FFT-d into image space, re-centered and zoomed in for
  details.}
\end{figure}
%
\clearpage
%
\section{Variance plane calculation of the difference image\label{sec:varplane}}
\par While the ZOGY image model does not strictly allow for different
per-pixel noise values (its noise model assumes homogeneous variance
noise across all pixels), from the \Cref{eq:c1N_c2R} form of the
difference image, we can propagate the different pixel variance
information in the variance planes into the difference image. To do
this, we notice that if we do convolution on an image of independent
noise then, in image space, the variance plane should be convolved by
the square of the convolution kernel. This is the well-known square
addition of variances of independent random variables.\footnote{See
  also ZOGY paper eqs. 26-29.}
%
\par We'd like to emphasize that this step cannot be applied to an
image with already correlated noise; the square addition of pixel
noise in the variance plane do not account for the covariance terms
and would result in underestimation of the pixel
variance. Notably, the effect of a noise decorrelation
(whitening) kernel on an already convolved image cannot be applied to
the variance plane based on the square addition rule as it would lower
the variance further instead of reverting it to the uncorrelated
level. In accordance with this, the image space square operation is
not distributive with respect to convolution in general; the square of the
convolution of two kernels is not the same as the convolution of the
squared kernels, and we should always perform the former.
%
\par To calculate the variance plane of the difference image, we
should calculate \(c_n\), \(c_r\) in \Cref{eq:c1N_c2R}, transform them
back to image space, square them in image space, and convolve the
original images' variance planes with these squared matching kernels
(\Cref{eq:VarD}). In practice, this convolution is more
straightforward to be performed in frequency space again because these
images already share common, full image size dimensions (the
dimensions of our ZOGY frequency space calculations).
\begin{align}
  V_D &= V_N \otimes \left(c_n^2\right) + V_R \otimes \left(c_r^2\right)\label{eq:VarD}\\
  \sigma_D^2 &= \sigma_N^2 \sum c_n^2 + \sigma_R^2 \sum c_r^2\label{eq:sigmaD}
\end{align}
%
\par In the homoscedastic approximation (\Cref{eq:sigmaD}), using the
Parseval theorem (\Cref{sec:parseval}), it can be seen that the zogy
\(\hat{D}\) is scaled (\Cref{eq:Dhat}) so that \(\sigma_D^2 = 1\) for
every pixel, while the flux is scaled to \(F_D\). Note that in the AL
plus decorrelation afterburner case, the flux is preserved, and the
noise variance is scaled to \(\sigma_n^2 / F_n^2 + \sigma_r^2
/F_r^2\). See \Cref{eq:K,eq:KPre}.
%
\par \Cref{eq:Shat} can also be written similarly to \Cref{eq:c1N_c2R}
and the variance plane of S can be calculated analogously
(\Cref{eq:VarD}) this way as well.
%
\section{ZOGY and AL equivalence\label{sec:ALZOGYequiv}}
\par The classic AL algorithm matches the reference image to the new
science image by convolving it with a matching kernel. In frequency
space, the matching kernel solution ideally equals to the quotient of
the two image PSFs as shown in \Cref{eq:Dal}.
%
\begin{equation}
  \hat{D}_{AL} =
  \frac{\hat{P}_{pre}}{F_n}\hat{N} -
    \frac{\hat{P}_{mk}}{F_r}\hat{R}
  =
  \frac{\hat{P}_{pre}\hat{N}}{F_n} -
    \frac{\hat{P}_n\hat{P}_{pre}\hat{R}}{\hat{P}_r F_r}
                 \label{eq:Dal}
\end{equation}
%
\par The decorrelation afterburner was created as a post-processing
correctional step on the difference image. It is calculated in
frequency space so that it decorrelates (whitens) the noise of the
difference image back in image space. We introduce the formula in
\Cref{eq:K} and discuss more details in \Cref{sec:decorrab}.
%
\begin{align}
\hat{K}  &= \frac
  {1}
  {\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_{pre}}^2 +
      \frac{\sigma_r^2}{F_r^2}\abs{\hat{P}_{mk}}^2}}
  \sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}}\label{eq:K}
\end{align}
%
\par In \Cref{eq:DzPz,eq:DzPrPrPz,eq:DaldPald}, we write the ZOGY score image in frequency
space and expand the expression to demonstrate that the AL matching
and subtraction combined with the decorrelation afterburner noise
whitening theoretically leads to the same detection statistics.
%
\begin{align}
  \hat{S} \sim \hat{D}_Z\overline{\hat{P}_Z} &= \frac
  {\frac{\hat{P}_r\hat{N}}{F_n} - \frac{\hat{P}_n\hat{R}}{F_r}}
  {\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_r}^2 +
      \frac{\sigma_r^2}{F_r^2}\abs{\hat{P}_n}^2}}
  \cdot
  \frac{\overline{\hat{P}_n\hat{P}_r}\sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}}}
  {\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_r}^2 +
      \frac{\sigma_r^2}{F_r^2}\abs{\hat{P}_n}^2}} = \label{eq:DzPz}\\
 &= F_D \frac
  {\left(\frac{\hat{N}}{F_n} -  \frac{\hat{R}}{F_r} \cdot
    \frac{\hat{P}_n}{\hat{P}_r}\right) \sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}} }
  {\sqrt{\frac{\sigma_n^2}{F_n^2} +
      \frac{\sigma_r^2}{F_r^2}\abs{\frac{\hat{P}_n}{\hat{P}_r}}^2}}
  \cdot
  \underbrace{
  \frac{\hat{P}_r}{\abs{\hat{P}_r}}
  \cdot
  \frac{\overline{\hat{P}_r}}{\abs{\hat{P}_r}}
  }_{1} % underbrace
  \cdot
  \frac{\overline{\hat{P}_n}\sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}}}
  {\sqrt{\frac{\sigma_n^2}{F_n^2} +
      \frac{\sigma_r^2}{F_r^2}\frac{\abs{\hat{P}_n}^2}{\abs{\hat{P}_r}^2}}} = \label{eq:DzPrPrPz}\\
  &= F_D \hat{D}_{AL+d} \cdot \frac{\overline{\hat{P}_n}
  \sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}}}
  {\sqrt{\frac{\sigma_n^2}{F_n^2} +
      \frac{\sigma_r^2}{F_r^2}\abs{\frac{\hat{P}_n}{\hat{P}_r}}^2}} =
F_D \hat{D}_{AL+d}\overline{\hat{P}_{AL+d}}\label{eq:DaldPald}
\end{align}
%
%
\begin{align}
F_D &= \frac{1}{\sqrt{\frac{\sigma_n^2}{F_n^2} +
    \frac{\sigma_r^2}{F_r^2}}} \\
\hat{D}_{AL+d} &= \frac{\hat{D}_Z}{F_D}\cdot\frac{\abs{\hat{P}_r}}{\hat{P}_r}\label{eq:Dald}
\end{align}
%
\par We start with the ZOGY score image in \Cref{eq:DzPz} and
demonstrate that the expression is equivalent with the score
calculated from a perfectly matching, decorrelated AL solution in
\Cref{eq:DaldPald}.  In the AL approach the role of the two images are
not symmetric, the template PSF is matched to the science image, and
the science image is left intact. Assuming that the AL optimization
finds the perfect matching kernel, it should be
\(\hat{P}_n/\hat{P}_r\). Indeed, considering the score image, the
difference between the AL and ZOGY images are only a factor of
\(\hat{P}_r/\abs{\hat{P}_r}\). Compared to the AL, in the ZOGY case
both the difference image and its PSF carry an extra
\(\hat{P}_{r}/\abs{\hat{P}_{r}}\) factor that cancel from the overall
expression of the score image. Indeed, as
\(\overline{\hat{P}_r}\hat{P}_r/\abs{\hat{P}_r}^2 = 1\) at all
frequencies, we can reduce or expand this factor in the difference
image and its PSF terms without changing their resulting product, the
score image.
%
\par Furthermore, note that \(\hat{P}_r/\abs{\hat{P}_r}=1\) itself, if
\(\hat{P}_r\) is real and positive at all frequencies. This is the
case if \(\hat{P}_r\) is a Gaussian PSF function. In this case \(D_Z\)
and \(D_{AL+d}\) are mathematically the same as shown in
\Cref{eq:Dald}; expanding or reducing the fractions in frequency space
by arbitrary real, positive kernels have the corresponding operations
of pre-convolution and deconvolution in image space.
%
\par In the decorrelated AL approach, we can also apply an arbitrary
Gaussian pre-convolution kernel without changing the difference image
in theory. However, calculating \Cref{eq:DalPre} in two separate
steps, as a difference image that is decorrelated afterwards in a
second step is numerically problematic as discussed in
\Cref{sec:decorrab}.
%
\begin{align}
  \hat{D}_{AL+d} =  \hat{D}_{AL} \cdot \hat{K}  &= \frac
  {\frac{\hat{P}_{pre}\hat{N}}{F_n} -
    \frac{\hat{P}_n\hat{P}_{pre}\hat{R}}{\hat{P}_r F_r} }
  {\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_{pre}}^2 +
      \frac{\sigma_r^2}{F_r^2}\abs{\frac{\hat{P}_n\hat{P}_{pre}}{\hat{P}_r}}^2}}
  \sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}}
  \label{eq:DalPre}\\
%
  &= \frac
  {\frac{\hat{P}_{pre}}{F_n}\hat{N} -
    \frac{\hat{P}_{mk}}{F_r}\hat{R} }
  {\sqrt{\frac{\sigma_n^2}{F_n^2}\abs{\hat{P}_{pre}}^2 +
      \frac{\sigma_r^2}{F_r^2}\abs{\hat{P}_{mk}}^2}}
  \sqrt{\frac{\sigma_n^2}{F_n^2} + \frac{\sigma_r^2}{F_r^2}}\label{eq:DaldPreMk}
\end{align}
%
\section{The decorrelation afterburner\label{sec:decorrab}}
Let's consider the decorrelation afterburner first without the
pre-convolution kernel, when (\(\hat{P}_{pre}=1\)) at all frequencies
in \Cref{eq:K}.
%
\par Assuming a Gaussian matching kernel (\(\hat{P}_{mk}\)) that
converges to zero towards high frequencies, the overall expression of
\(\hat{K}\) in \Cref{eq:K} converges to \(F_{n}/\sigma_n\). The
decorrelation correction function in frequency space is similar to
\(\hat{c}_2\) in \Cref{fig:theo_Gaussians_ft}. In image space, its
graph follows a dirac delta plus a negative overshoot as in
\Cref{fig:theo_Gaussians_img_c2}. In the straightforward AL case, when
we convolve the template image, we don't expect any complication in
calculating such a decorrelation correction. As \(\sigma_r \ll
\sigma_n\), \Cref{eq:K} can keep its convergence properties even if
\(\hat{P}_{mk}\) values are noisy in their Gaussian tails.
%
\par Now let's consider the swapped image case, when we convolve the
science image.  The numerical stability of this case is less
certain. As \(\sigma_r \gg \sigma_n\) in this case, it can prevent
\(\sigma_{n}^2/F_n^2\) from becoming the dominant term in the
denominator of \Cref{eq:K} and the numerical noise in the tails of
\(\hat{P}_{mk}\) may remain in the high frequency values \(\hat{K}\).
%
\par Let's assume now a Gaussian pre-convolution kernel. It can be
seen that the overall expression of \(\hat{K}\) becomes divergent in
\Cref{eq:K} towards high frequencies. The denominator converges to
zero. Even if we don't directly hit numerical overflows in calculating
\(\hat{K}\), a divergent \(\hat{K}\) cannot be meaningfully
applied. The difference image converges to zero (\(\hat{D}_{AL}\),
\Cref{eq:Dal}), so correcting the already computed difference image is
similar to inverting a multiplication operation of smaller and smaller
numbers. In \Cref{eq:DaldPreMk}, we can see that calculating the
corrected difference image (\(\hat{D}_{AL+d}\)) is exactly the same
problem as calculating the ZOGY difference image in \Cref{eq:Dhat}. In
these expressions, both the numerators and the denominators converge to
zero which in practice result in numerical noise in the tail regions
of the input Gaussians (image PSFs, pre-convolution and matching
kernels). We face the very same numerical problems that was discussed
in \Cref{sec:ZOGYFFT}.
%
\subsection{Decorrelation afterburner with pre-convolution\label{sec:decorr_preconv}}
%
\par We saw that the decorrelation afterburner of \Cref{eq:K} is
problematic if we use a pre-convolution kernel. In this form the
decorrelation afterburner wants to recover the proper difference
image, inverting the pre-convolution operation completely.
%
\par Recall that pre-convolution is a practical way to ensure that the
science image PSF is wider than our template PSF. We also know that if
we choose to correlate an image with its own
PSF\footnote{Pre-convolution with the flipped PSF in image space;
  multiplication with complex conjugate in frequency space (e.g.\
  \Cref{eq:Shat}).}, under the independent Gaussian noise model
assumptions, we get a detection likelihood image.
%
\par We show here that we can apply a form of the decorrelation
afterburner in the pre-convolution case if we pre-convolve the science
image with its own PSF. This form of the decorrelation afterburner
does not invert the pre-convolution operation, in this sense it does
not do noise decorrelation any more. Rather it corrects the
pre-convoled and AL PSF matched likelhood difference image directly
and results in the equivalent of the zogy score image as an optimal
detection statistics in the case when both the science and template
images have noise.
%
We can rewrite \Cref{eq:S}:
\begin{align}
  \hat{S} &=
            \frac{\overline{\hat{P}_n} \frac{\hat{N}}{F_n} -
            \frac{\abs{\hat{P}_n}^2}{\hat{P}_r} \frac{\hat{R}}{F_r} }
            % denom
            { \frac{\sigma_n^2}{F_n^2} +
            \frac{\sigma_r^2}{F_r^2}
            \frac{\frac{\abs{\hat{P_n}}^2}{\hat{P}_r}}
            {\overline{\hat{P}_r}}
            } =
            \frac{\overline{\hat{P}_n} \frac{\hat{N}}{F_n} -
            \frac{\abs{\hat{P}_n}^2}{\hat{P}_r} \frac{\hat{R}}{F_r} }
            % denom
            { \frac{\sigma_n^2}{F_n^2} +
            \frac{\sigma_r^2}{F_r^2}
            \frac{\abs{\hat{P}_{mk}}^2}{\abs{\hat{P}_{n}}^2}
            } =
            % two terms
            \underbrace{\left(
            \overline{\hat{P}_n} \frac{\hat{N}}{F_n} -
            \frac{\abs{\hat{P}_n}^2}{\hat{P}_r} \frac{\hat{R}}{F_r}
            \right)}_{\textrm{pre-convolved AL}} \cdot
            % denom
            \frac{1}
            { \frac{\sigma_n^2}{F_n^2} +
            \frac{\sigma_r^2}{F_r^2}
            \frac{\abs{\hat{P}_{mk}}^2}{\abs{\hat{P}_{n}}^2}
            }
            \label{eq:SPre}\\
  \hat{P}_{mk} &= \frac{\abs{\hat{P_n}}^2}{\hat{P}_r} \label{eq:Smk}\\
  \hat{S} &= \hat{c}_{sn} \hat{N} - \hat{c}_{sr}\hat{R} =
            \frac{\frac{\overline{\hat{P}_n}}{ F_n} }
            % denom
            { \frac{\sigma_n^2}{F_n^2} +
            \frac{\sigma_r^2}{F_r^2}
            \frac{\abs{\hat{P}_{mk}}^2}{\abs{\hat{P}_{n}}^2}
            } \hat{N}
            -
            \frac{
            \frac{\abs{\hat{P}_n}^2}{\hat{P}_rF_r} }
            % denom
            { \frac{\sigma_n^2}{F_n^2} +
            \frac{\sigma_r^2}{F_r^2}
            \frac{\abs{\hat{P}_{mk}}^2}{\abs{\hat{P}_{n}}^2}
            } \hat{R}\label{eq:Scncr}
\end{align}
%
\par In \Cref{eq:SPre}, the first term in the numerator is the PSF
pre-convolved science image that now has a PSF of
\(\abs{\hat{P}_n}^2\), and the theoretical PSF matching kernel is
written in \Cref{eq:Smk}. We use the matching kernel to express the
correction in \Cref{eq:SPre} on the right side. Compared to the
original decorrelation afterburner expression, the correction is
squared and the correction with the matching kernel is deconvolved
with the pre-convolution kernel.
%
\par Let's consider the convergence properties towards high
frequencies in case of Gaussian PSFs. The numerator of this expression
always goes to zero while the denominator never goes to zero,
irrespectively of the widths of \(\hat{P}_{mk}\) and
\(\hat{P}_{n}\). As such, \(\hat{S}\) always goes to zero.
%
\par If we use pre-convolution to avoid the AL deconvolution case, it
is guaranteed that the matching kernel solution will be narrower in
image space than the PSF of the science image. In frequency space, the
relation is the opposite, and the denominator of \Cref{eq:SPre} should
actually diverge, making the overall zero convergence faster.
%
\par Note that the zero convergence property of \(\hat{S}\) is kept
even if the convergence properties are numerically lost in the second
term of the denominator at high frequencies. Therefore we do not
expect \emph{worse} high frequency noise to appear in our ``afterburned''
likelihood/score image result than it would be in a simple frequency
space PSF convolution operation.
%
\par We emphasize that while we can apply this correction on the
already calculated pre-convolved AL likelihood difference image, we
cannot do it on its variance plane. Because the noise in the score
image is not white, we cannot easily estimate the noise level even in
the homoscedastic noise model case. Instead, we need to follow the
considerations of \Cref{sec:varplane} for the variance plane
calculation and express \(\hat{S}\) with the overall coefficients of
\(\hat{N}\) and \(\hat{R}\) as shown in \Cref{eq:Scncr}. Recall, we
transform \(\hat{c}_{sn}\) and \(\hat{c}_{sr}\) back to image space,
square them in image space, and convolve the original images' variance
planes with these squared matching kernels (\Cref{eq:VarD}). In the
homoscedastic approximation, we multiply \(\sigma_n^2\) and
\(\sigma_r^2\) with the sum of squares (\Cref{eq:sigmaD}), but we
still need to calculate \(c_{sn}\) and \(c_{sr}\). In this
approximation, the inverse FFT of \(\hat{c}_{sn}\) and
\(\hat{c}_{sr}\) can be spared and the sum can be calculated directly
in frequency space (Parseval theorem, \Cref{sec:parseval}).
%
\subsection{Decorrelation afterburner normalization}
\par The classic AL method can be applied to images without
considering their photometric scaling factors (\(F_n, F_r\)). In this
case, the AL optimisation itself solves for the ratio of the
photometric scaling between the images. This scaling appears as the
sum of the AL matching kernel. So far, we've assumed that our AL
matching kernels are normalized like image PSFs. Let's relax this
assumption and separate the sum of the pre-convolution and matching
kernels into standalone factors \(S_{pre}\), \(S_{mk}\). To preserve
the photometric flux, the decorrelation afterburner correction should
still be overall a convolution with a normalized correction kernel in
image space.
%
\par Image space sum normalization can be easily ensured in the
corresponding frequency space expression. If we evaluate any
convolution kernel at 0 frequency, based on \Cref{eq:X0sum} they
should be equal to 1. We should include \(S_{pre}\), \(S_{mk}\) in the
numerator scaling factor to satisfy the normalization criterion. The
normalized decorrelation afterburner can be written as \Cref{eq:KPre}.
%
\begin{align}
  \hat{K} &= \frac{\sqrt{\frac{\sigma_n^2}{F_n^2}S_{pre}^2 + \frac{\sigma_r^2}{F_r^2}S_{mk}^2}}
  {\sqrt{\frac{\sigma_n^2}{F_n^2}S_{pre}^2\abs{\hat{P}_{pre}}^2 + \frac{\sigma_r^2}{F_r^2}S_{mk}^2
  \abs{\hat{P}_{mk}}^2}}
\label{eq:KPre}
\end{align}
%
\par Similarly, should the PSFs be not normalized to sum 1 in the ZOGY
image differencing, the sum of the PSFs should appear in \(F_{D}\) in
the same way as a scaling factor.
%
\subsection{A possible fusion of AL and ZOGY}
To keep the advantage of AL of not assuming a prior knowledge of the
image PSFs and of ZOGY to produce a proper difference image, we
propose the following algorithmic fusion of the two approaches.
%
\par Let's choose a Gaussian pre-convolution kernel (\(P_{pre}\)), so
that the AL algorithm can determine an accurate \(P_{mk}\) reliably in
image space; perhaps in a spatially varying manner for the whole
image. The optimisation of the pre-convolution kernel can focus solely
on the quality of the AL matching kernel solution. The expected
tradeoff here is that the wider the pre-convolution kernel, the easier
to find a Gaussian matching kernel but on the other hand, a wider
pre-convolution kernel means more noise correlation and a less accurate
minimisation in the AL algorithm. We do not need to calculate the
classic AL difference image in its original \Cref{eq:Dal} form here.
%
\par In the next step, using \Cref{eq:DaldPreMk}, we perform a ZOGY
difference image calculation using the \(\hat{P}_{pre}\) and
\(\hat{P}_{mk}\) solutions from the previous step. They behave like
the ``image PSFs'' in the original zogy calculations. We should also
apply all the precautions and numerical considerations discussed
earlier. As \(\hat{P}_{pre}\) is positive and real, it should
disappear completely from the proper difference image and score image
values. We note however, that the wider is \({P}_{pre}\) in image
space, the narrower \(\hat{P}_{pre}\) becomes in frequency
space. While multiplication with a Gaussian functions is fully
invertible theoretically, numerically such a multiplication will
suppress high frequency values and indirectly throws away high
frequency (spatial) information from the score image. This is
different from the tails (and zero points) of the image PSFs. At the
PSF zero points the images do not carry any information, while with
the pre-convolution, we throw away information from the data. Ad
absurdum, imagine that we keep only the 0th frequency in frequency
space. It'd correspond to a pre-convolution kernel of uniform values
in image space. We'd average out our whole images and would get one
uniform detection score for all pixels.
%
\section{Conclusions}
%
\par We studied the ZOGY difference image matching kernels for Gaussian
input PSFs in this document. In the theoretical calculations
(\Cref{sec:ZOGYtheo}), we showed that the matching kernels have different
convergence values in their tails depending whether they belong to the
narrower of wider PSF input image. In practice, using FFT, these convergence
properties are not well reproduced and the resulting image space matching
kernels have oscillating patterns all over the image
(\Cref{sec:ZOGYFFT}). We concluded that this noise is still acceptable if
the input images follow their PSF models and suppress high frequencies,
however noise patterns appear in the difference image if the PSFs
deviate. This noise is extended, visually unappealing, and can disrupt other
algorithms' performance on the difference image; however, it has little
impact on the source detection statistic.
%
\par We tested the direct Gaussian PSF generation in frequency space as a
possible way to avoid the convergence problems in our calculations. We
expect that it would produce difference images without large scale patterns
for all inputs. It is a strong restriction on the PSFs, so we also plan to
look for weaker constraints in suppressing the artifacts in the difference
image. We also need to consider sampling (aliasing) details before
implementation.
%
\par In \Cref{sec:varplane} we discussed how to properly calculate the variance
plane in frequency space when we have noise whitening decorrelation
operations.
%
\par In \Cref{sec:ALZOGYequiv}, we demonstrated that the AL method combined with
the decorrelation afterburner leads to the same detection statistic as the
ZOGY method. With preconvolution, they can theoretically lead to the very
same difference image. We believe though that his approach would meet
similar practical problems as the ZOGY subtraction has. This is a possible
future topic to study.
%
\par In \Cref{sec:appendix} we discuss various considerations that have
relevance in the actual and for future frequency space image differencing
code implementations.
%
\par Finally, the noisy matching kernels cause complications in
implementing solutions in frequency space for spatial variations of the PSF
in a large image. This is also a topic we plan to study in the future.


\lab{Facial Recognition Using Eigenfaces}{Facial Recognition Using Eigenfaces}
\label{lab:FacialRecognition}

\objective{Use the singular value decomposition to implement a simple facial recognition system.}

Suppose we have a large database containing images of human faces.
We would like to identify people by comparing their pictures to those in the database.
This task is called \emph{facial recognition}.

One way to automate the comparison process is know as the \emph{eigenfaces} method.
As the name suggests, this method uses eigenvectors of matrices related to the collection of face images.
The method essentially projects face images to a low-dimensional subspace, in a way that preserves their distinguishing characteristics.
Comparing the images in fewer dimensions is faster and allows us to store the images using less data.

The idea of projecting to fewer dimensions is not unique to the eigenfaces method.
This method is an example of \emph{principal component analysis}, where data is compared based on its principal components in a lower-dimensional vector space.
Principal component analysis can be applied to many computing problems besides facial recognition.
% TODO: list 2-3 other applications

\begin{comment}
Facial recognition is important in law enforcement, as well as other situations.
For example, facial recognition can be combined with video surveillance to identify when a person is not authorized to be in a location.

Humans can easily compare two face images and determine whether they belong to the same person, but automating this process is more challenging.
One technique for automated facial recognition uses \emph{eigenfaces}.
\end{comment}

\section*{Load the Data} % ====================================================
\begin{comment}
Eigenfaces are an efficient way to store and query a database of face images.
As the name suggests, this method uses eigenvectors of matrices related to the collection of face images.
Essentially, the method of eigenfaces projects face images to a lower-dimensional subspace in a way that preserves their distinguishing characteristics.
In the lower-dimensional subspace, comparing face images is much faster.
\end{comment}

The first step is to obtain a dataset of face images.
Recall that a digital image may be stored as an $m \times n$ array of pixels.
In this lab, we will store the images as $mn$-vectors by concatenating the rows of the $m \times n$ arrays.

\begin{problem}
\label{prob:getTrainingFaces}
In this lab we will use the \li{faces94} face image dataset found at \url{http://cswww.essex.ac.uk/mv/allfaces/faces94.html}
This problem will make sure that you can load and display the images from the dataset.
\begin{enumerate}
\item Download the \li{faces94} dataset from the link above and extract the files.
You should now have a directory named ``faces94" which contains photographs of 153 people, organized into folders by person.

\item The function \li{getFaces()} is given in the appendix. %or spec file?
It constructs a set of face images by selecting exactly one face image for each person in the directory.
It should return an array whose columns are flattened face images.
Feel free to modify the given code.
You may have to replace the parameter \li{\"./faces94\"} with the location of the directory \li{faces94} on your machine.

Test this function to make sure it runs without errors.
Check that the return value \li{F} is a $36000 \times 153$ array.
The columns of this array are 153 flattened face images of 153 different people.

\item Use \li{plt.imshow()} to display one of the faces.
The original image dimensions are $200 \times 180$.
You may find it useful to write a helper function that accepts a flattened image and displays it.

\end{enumerate}
\end{problem}

\section*{Shift By the Mean} % ================================================

The facial recognition algorithm is more robust if we first \emph{shift by the mean}. When we shift a set of data by the mean, the distinguishing features are exaggerated. Therefore, in the context of facial recognition, shifting by the mean accentuates the unique features of the face.
Suppose we have a collection of $k$ face images represented as vectors $\f_1, \f_2, \ldots, \f_k$ of length $mn$.
Define the \emph{mean face} ${\boldsymbol \mu}$ to be the average of the $\f_i$:
\[
{\boldsymbol \mu} = \frac{1}{k}\sum_{i=1}^k \f_i.
\]
\begin{figure}
\includegraphics[width=0.3\textwidth]{meanFace.png}
\caption{The mean face.}
\label{facialRecognition:meanFace}
\end{figure}

\begin{problem}
\label{prob:meanFace}
\leavevmode

The facial recognition method you will write in this lab will be structured as a class.
An outline of the \li{FacialRec} class is provided in the appendix. %or maybe in the spec file?
You will write the methods of the class.

When initialized, the \li{FacialRec} object first loads the face images using \li{getFaces} and stores the result.
The next step is to compute the mean face.
\begin{enumerate}
\item In your class definition, implement the method \li{FacialRec.initMeanImage()}.
Compute the mean face and store it as \li{self.mu}.
This can be done in one line of code using NumPy.

\item Display the mean face.
Your result should match Figure \ref{facialRecognition:meanFace}.
\end{enumerate}
\end{problem}

\begin{figure}
\begin{subfigure}[b]{0.3\textwidth}
\includegraphics[width=\textwidth]{differenceFace0.png}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
\includegraphics[width=\textwidth]{differenceFace1.png}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
\includegraphics[width=\textwidth]{differenceFace2.png}
\end{subfigure}
\caption{Three mean-shifted faces from the dataset.}
\label{facialRecognition:differenceFaces}
\end{figure}
For each $i = 1,\ldots, k$, define $\bar{\f}_i := \f_i - {\boldsymbol \mu}$.
The mean-shifted face vector $\bar{\f}_i$ is the deviation of the $i$-th face from the mean, and thus captures the unique features of the face.
Now form the $mn \times k$ matrix $\bar{F}$ whose columns are given by the mean-shifted face vectors, i.e.
\[
\bar{F} = \begin{bmatrix}
\bar{\f}_1 & \bar{\f}_2 & \cdots & \bar{\f}_k
\end{bmatrix}.
\]
\begin{problem}
\leavevmode
\begin{enumerate}
\item In your class definition, implement the method \li{FacialRec.initDifferences()}.
Compute $\bar{F}$ and store it as \li{self.Fbar}.
This can be done in one line using array broadcasting.
\item
Display one of the mean-shifted faces.
The output should be similar to the faces in Figure \ref{facialRecognition:differenceFaces}.
\end{enumerate}
\end{problem}

\section*{Project to a Subspace} % ============================================

Now suppose we are given a new face vector $\g$.
We first shift $\g$ by the mean of the dataset, giving us $\bar{\g} = \g - {\boldsymbol \mu}$.
The closest match to $\bar{\g}$ is the vector $\bar{\f}_i$ that minimizes $\|\bar{\g}-\bar{\f}_i\|_2$.
If there are $k$ images in the dataset, we find this match by comparing $\bar{\g}$ to every element of $\{\bar{\f}_1, \ldots \bar{\f}_k\}$.

However, comparing the original face images pixel by pixel is computationally expensive and inefficient.
The vectors $\bar{\f}_i$ and $\bar{\g}$ are length $mn$, which in our case equals 36000 and in practice may be many times larger.
Computing the difference between face vectors of this length is time consuming, especially when the dataset is very large.
It also requires us to use $mn$ values to store each face, which is an inefficient use of space.
In addition, pixel by pixel comparison is not very robust to small changes in individual pixels.

Instead, in order to store and compare our face vectors, we would like to represent each one with fewer than $mn$ values.
We can do this by projecting to a subspace.
Mathematically, we want to choose $s$, $s \ll mn$, and project the face vectors to an $s$-dimensional subspace of the original $mn$-dimensional space of images.
We can then use just $s$ values to store each face in terms of the basis vectors of the new subspace.

The ``best" subspace to project to is the one that is closest in the least squares sense (i.e., such that the sum of the squared errors between $\{\bar{\f}_1, \ldots \bar{\f}_k\}$ and their projections is minimized).
Let $U \Sigma V^T$ be an SVD of $\bar{F}$, with $\u_i$ the columns of $U$.
As we will prove below, the $s$-dimensional subspace that minimizes the squared error is the span of $\u_1, \ldots, \u_s$.
Note that $\u_1, \ldots, \u_s$ is an orthonormal basis for this subspace.

The projection matrix is $P_s = U_s U_s^T$ where $U_s = [ \u_1\; \ldots \; \u_s]$.
%The matrix for the projection is $P_s = U_s U_s^T$ where $U_s$ is the first $s$ columns of $U$.
This matrix projects the original face vectors into the optimal $s$-dimensional subspace.

\begin{comment} % OLD VERSION
\section*{Project to a Subspace} % ============================================
Now suppose we have a new face vector $\g$.
The closest face image to $\g$ should be the vector $\bar{\f}_i$ that minimizes $\|\bar{\g}-\bar{\f}_i\|_2$, where $\bar{\g} = \g - {\boldsymbol \mu}$.
Unfortunately, computing $\|\bar{\g}-\bar{\f}_i\|_2$ for each $i$ is computationally intractable when the length $mn$ of the vectors is large.
Since a low-resolution photo may easily have $100\times 100 = 10,000$ pixels, in practice $mn$ is very large indeed.

Here is the trick: instead of computing in the $mn$-dimensional space of all possible images, we will compute in a lower-dimensional subspace.
We could start by using the subspace spanned by the vectors $\bar{\f}_1, \ldots \bar{\f}_k$, which is at most $k$-dimensional.
Unfortunately, in practice $k$ is still too large for this subspace to be computationally efficient.
%We need to project our problem into some subspace of the span of $\bar{\f}_1, \ldots \bar{\f}_k$.

Therefore, we want to project to a subspace of $\bar{\f}_1, \ldots \bar{\f}_k$ in a way that retains as much information about the $\bar{\f_i}$ as possible.
Mathematically, we want to find the $s$-dimensional subspace of $\text{span}\{\bar{\f}_1, \ldots \bar{\f}_k\}$ that is closest to the $\bar{\f_i}$ in the least-squares sense.
In a minute, we will prove that the SVD of $\bar{F}$ (whose columns are $\bar{\f}_1, \ldots \bar{\f}_k$) solves this problem.
But first, let us summarize how the solution works.

Let $U \Sigma V^T$ be an SVD of $\bar{F}$ with $\u_i$ the columns of $U$.
Then the ``best'' $s$-dimensional subspace for approximating $\text{span}\{\bar{\f}_1, \ldots \bar{\f}_k\}$ is the span of $\u_1, \ldots, \u_s$.
The matrix for this projection is $P_s = U_s U_s^T$ where $U_s = [ \u_1\; \ldots \; \u_s]$.

Because the vectors $\u_i$ are eigenvectors of $\bar{F}\bar{F}^T$, we call them \emph{eigenfaces}.
Therefore, the best $s$-dimensional subspace for solving the facial recognition problem is exactly the span of the first $s$ eigenfaces.
\end{comment}

\subsection*{The Proof: SVD as a Least Squares Solution} % --------------------
\begin{theorem}
Let $\f_1, \ldots, \f_k$ be vectors on $\mathbb{R}^{mn}$, and let $\bar{F} = [\bar{f}_1 \; \ldots \; \bar{f}_k]$. Suppose $U\Sigma V^T$ is an SVD for $\bar{F}$. Then the $s$-dimensional subspace that solves the least squares problem for $\f_1, \ldots, \f_k$ is the span of the first $s$ columns of $U$. If $U_s$ is the first $s$ columns of $U$, then the matrix $U_sU_s^T$ is the projection onto this subspace.
\end{theorem}
\begin{proof}
We seek a rank-$s$ projection matrix $P_s$ so that $\sum_{i=1}^k \|P_s\bar{\f}_i - \bar{\f}_i\|_2^2$ is minimized---i.e.,
 the sum of the squares of the ``errors'' is minimal when we project $\bar{\f}_i$ via $P_s$.
 But minimizing this quantity is the same as minimizing its square, which happens to equal the Frobenius norm of $P_s\bar{F} - \bar{F}$.
Written mathematically,
 \begin{align*}
\inf_{\text{rank}(P_s)=s} \sum_{i=1}^k \|P_s\bar{\f}_i - \bar{\f}_i\|_2^2 &=  \inf_{\text{rank}(P_s)=s} \left( \sum_{i=1}^k \|P_s\bar{\f}_i - \bar{\f}_i\|_2^2 \right) ^2\\
 & =  \inf_{\text{rank}(P_s)=s} \| P_s\bar{F}-\bar{F}\|_F.
 \end{align*}

Now let $U \Sigma V^T$ be an SVD of $\bar{F}$ with $\u_i$ the columns of $U$, $\v_i$ the columns of $V$, and $\sigma_i$ the singular values of $\bar{F}$.
If $P_s = \sum_{i=1}^s \u_i \u_i^T$, then
\begin{align*}
P_s\bar{F} &=  \left( \sum_{i=1}^s \u_i \u_i^T \right)\left(  \sum_{j=1}^k \sigma_j \u_i \v_i^T \right)
= \sum_{i=1}^s\sum_{j=1}^k \sigma_j \u_i\u_i^T\u_j\v_j^T\\
&=  \sum_{i=1}^s\sum_{j=1}^k \sigma_j \u_i\delta_{ij}\v_j^T
= \sum_{i=1}^s \sigma_i \u_i\v_i^T.
\end{align*}

In fact, the Schmidt-Eckart-Young-Mirsky Theorem from Lab \ref{lab:SVD} tells us that $X = \sum_{i=1}^s \sigma_i \u_i\v_i^T$ is exactly the rank-$s$ matrix that minimizes $\|X - \bar{F}\|_F$.
Since $P_s \bar{F}$ will always have rank $s$ or less, the projection $P_s =  \sum_{i=1}^s \u_i \u_i^T$ is the one we seek.
If we let $U_s = [ \u_1\; \ldots \; \u_s]$, then we may write $P_s = U_sU_s^T$. Notice that $P_s$ is projection onto the subspace spanned by the columns of $U_s$.
\end{proof}

The $s$ basis vectors $\u_1, \ldots, \u_s$ are eigenvectors of $\bar{F}\bar{F}^T$.
They also resemble face images.
Hence, they are commonly called the ``eigenfaces."

\begin{problem}
\label{prob:svd}
\leavevmode
\begin{enumerate}
\item In your class definition, implement the method \li{FacialRec.initEigenfaces()}.

Compute the SVD (\li{scipy.linalg.svd()} is a good implementation to use) and store the array \li{U} containing the eigenfaces as its columns.
Because we will only use the first few columns of \li{U}, specify the keyword parameter \li{full_matrices=False} to compute only the compact SVD.
\item Plot the first eigenface (i.e. the first column of \li{U}).
It should resemble the first eigenface shown in Figure \ref{facialRecognition:eigenfaces}.
\end{enumerate}
\end{problem}

\section*{Change Basis} % =====================================================

The projection matrix $P_s = U_s U_s^T$ projects a face vector into the $s$-dimensional subspace spanned by the eigenfaces, but still keeps it as a vector in $\mathbb{R}^{mn}$.
The change-of-basis matrix $U_s^T$ both projects the face vector and and changes the basis.
The resulting vector has length $s$ and represents the face in terms of eigenfaces.

To represent any face vector in terms of the first $s$ eigenfaces, multiply by $U_s^T$.
To change back to a full length-$mn$ projection, multiply again by $U_s$.

\begin{comment}
It does us no good to project all our vectors into an $s$-dimensional space if we still store them as vectors in $\mathbb{R}^{nm}$.
Instead, we must store our face vectors in terms of the columns of $U_s$.
This way, each vector is a length-$s$ array in NumPy, instead of a length-$mn$ array.

The change-of-basis matrix is $U_s^T$, so $U_s^TP_s = U_s^TU_sU_s^T = U_s^T$.
Thus we can project into the subspace and change basis by multiplying by $U_s^T$.
To change back to the full $mn$-vector, multiply by $U_s$.
\end{comment}

\begin{figure}
\begin{subfigure}[b]{0.3\textwidth}
\includegraphics[width=\textwidth]{eigenface0.png}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
\includegraphics[width=\textwidth]{eigenface1.png}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
\includegraphics[width=\textwidth]{eigenface2.png}
\end{subfigure}
\caption{The top three eigenfaces.}
\label{facialRecognition:eigenfaces}
\end{figure}

\begin{problem}
\label{prob:top_n}
\leavevmode
\begin{enumerate}
\item Implement the method  \li{FacialRec.project()} in your class definition.
This should accept a flattened image or an array with flattened images as its columns.
It should also accept a value for $s$.
The function should project the image or images into the appropriate $s$-dimensional subspace and change basis, then return the result.
\item Let \li{face} be the first mean-shifted face (the first column of \li{facialRec.Fbar}). Do the following:
\begin{enumerate}
\item Project \li{face} so that it is represented in terms of the first 19 eigenfaces.
\item Change basis again back to the standard basis on $\mathbb{R}^{mn}$.
\item Add back the mean face \li{facialRec.mu}.
\item Plot the resulting image.
\end{enumerate}
Your image should match Figure \ref{fig:75_efaces}.
\end{enumerate}
\end{problem}

\begin{figure}
\begin{subfigure}[b]{0.3\textwidth}
    \includegraphics[width=\textwidth]{rebuiltThirtySecond.png}
    \caption{5 eigenfaces, about 1/32 of the total.}
    \label{fig:5_efaces}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
    \includegraphics[width=\textwidth]{rebuiltSixteenth.png}
    \caption{9 eigenfaces, or 1/16 of the total.}
    \label{fig:9_efaces}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
    \includegraphics[width=\textwidth]{rebuiltEighth.png}
    \caption{19 eigenfaces, about 1/8 of the total.}
    \label{fig:19_efaces}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
    \includegraphics[width=\textwidth]{rebuiltFourth.png}
    \caption{38 eigenfaces, about 1/4 of the total.}
    \label{fig:38_efaces}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
    \includegraphics[width=\textwidth]{rebuiltHalf.png}
    \caption{75 eigenfaces, about 1/2 of the total.}
    \label{fig:75_efaces}
\end{subfigure}
\begin{subfigure}[b]{0.3\textwidth}
    \includegraphics[width=\textwidth]{rebuiltAll.png}
    \caption{All 153 of the eigenfaces.}
    \label{fig:all_efaces}
\end{subfigure}
\caption{Image rebuilt with various numbers of eigenfaces. The image is somewhat recognizable when it is reconstructed with only 1/8 of the eigenfaces.}
\label{fig:rebuiltImage}
\end{figure}

\begin{comment}
Of course, moving to a lower-dimensional subspace by using only the top $n$ eigenfaces introduces some error, in the sense that we cannot perfectly
reconstruct the face images from their coordinate vectors.
However, this error is tolerable, provided $n$ is not \emph{too} small.
Note that we can approximately reconstruct $\bar{f}_i$ from the coordinate vector $\hat{f}_i$ by
\[
\bar{f}_i \approx U_n\hat{f}_i.
\]
See Figure \ref{facialRecognition:rebuiltImage} for examples of reconstructed face images giving different numbers of eigenfaces.
\end{comment}

\section*{Match Faces} % ======================================================

Finally, we are ready to identify which mean-shifted image $\bar{\f}_i$ is closest to an input image, $\bar{\g}$.
We begin by projecting all vectors to some $s$-dimensional subspace and writing them in terms of the basis vectors, which are the eigenfaces.
This is done by multiplying by the change-of-basis matrix:
\[
\widehat{\f}_i = U_s^T(\f_i-{\boldsymbol \mu}) \qquad \widehat{\g} = U_s^T(\g-{\boldsymbol \mu}).
\]

Next, we compute which $\widehat{\f}_i$ is closest to $\widehat{\g}$.
Since the columns of $U_s$ are an orthonormal basis, we get the same result doing the computation in this basis as we would in the standard Euclidean basis.
Define
\[
i^* = \text{argmin}_i \|\widehat{\f}_i - \widehat{\g}\|_2.
\]
Then the $i^*$-th face image is the best match for $\g$.

\begin{problem}
\label{prob:nearest}
\leavevmode
\begin{enumerate}
\item
Implement the method \li{FacialRec.findNearest()} as follows.
\begin{lstlisting}
def findNearest(self, image, s=38):
    Fhat = # Project Fbar, producing a matrix whose columns are the f-hat defined above
    ghat = # Shift 'image' by the mean and project, producing g-hat as defined above
    # for both Fhat and ghat, use your project function from the previous problem

    # Return the index that minimizes ||fhat_i - ghat||_2.
\end{lstlisting}
The functions \li{np.linalg.norm()} and \li{np.argmin()} will be useful for the last line. When using \li{np.linalg.norm}, make sure you indicate the correct axis.

\item Test your facial recognition system on faces selected randomly from the \li{faces94} dataset.
The function \li{sampleFaces(n_tests, path)} at the end of this lab will build an array of \li{n_tests} random faces from the \li{faces94} dataset.

Plot the random face beside the face returned by your facial recognition code to see if your system is accurately recognizing faces.
%The function \li{show2()} at the end of this lab will plot two face vectors side-by-side.

\end{enumerate}
\end{problem}

By this point, you have created a basic facial recognition system.
We can extend the system to detect when a face doesn't match anything currently in the dataset, and then add this new face.
We can also make the system more robust by including multiple pictures of the same face with different expressions and lighting conditions.

Although there are other approaches to facial recognition that utilize more complex techniques, the method of eigenfaces remains
a wonderfully simple and effective solution, illustrating another application of the singular value decomposition.

\section*{Appendix: Helper Code} % ============================================

This section contains some functions to help you implement the facial recognition class outlined in the problems of this lab.

\begin{lstlisting}
import numpy as np
from scipy import linalg as la
from os import walk
from scipy.ndimage import imread
from matplotlib import pyplot as plt
from random import sample

def getFaces(path="./faces94"):
    """Traverse the directory specified by 'path' and return an array containing
    one column vector per subdirectory.

    For the faces94 dataset, this gives an array with just one column for each
    face in the dataset. Each column corresponds to a flattened grayscale image.
    """

    # Traverse the directory and get one image per subdirectory.
    faces = []
    for (dirpath, dirnames, filenames) in walk(path):
        for f in filenames:
            if f[-3:]=="jpg": # only get jpg images
                # load image, convert to grayscale, flatten into vector
                face = imread(dirpath+"/"+f).mean(axis=2).ravel()
                faces.append(face)
                break

    # put all the face vectors column-wise into a matrix.
    F = np.array(faces).T
    return F

def sampleFaces(n_tests,path = "./faces94")
    """Return an array containing a sample of n_tests images contained
    in the path as flattened images in the columns of the output.
    """
    files = []
    for (dirpath, dirnames, filenames) in walk(path):
        for f in filenames:
            if f[-3:]=="jpg": # only get jpg images
            files.append(dirpath+"/"+f)

    #Get a sample of the images
    test_files = sample(files, n_tests)
    #Flatten and average the pixel values
    images = np.array([imread(f).mean(axis=2).ravel() for f in test_files]).T
    return images

\end{lstlisting}

The following is the outline of the Facial Recognition class.

\begin{lstlisting}
class FacialRec:
    ##########Members##########
    #   F, mu, Fbar, and U
    ###########################
    def __init__(self,path):
        self.initFaces(path)
        self.initMeanImage()
        self.initDifferences()
        self.initEigenfaces()

    def initFaces(self, path):
        self.F = getFaces(path)
    def initMeanImage(self):
        pass
    def initDifferences(self):
        pass
    def initEigenfaces(self):
        pass
    def project(self, A, s=38):
        pass
    def findNearest(self, image, s=38):
        pass
\end{lstlisting}