% !TEX root = hw1.tex

\section{Link Analysis [25 points]}

\subsection{Personalized PageRank I [12 points]}
Personalizing PageRank is a very important real-world problem: different users
find different pages relevant, so search engines can provide better results if
they tailor their page relevance estimates to the users they are serving. Recall
from class that PageRank can be specialized with clever modifications of the
teleport vector. In this question, we will explore how this can be applied to
personalize the PageRank algorithm.

Assume that people's interests are represented by a set of representative
pages. For example, if Zuzanna is interested in sports and food, then we could
represent her interests with the set of pages $\{$\texttt{www.espn.com},
\texttt{www.epicurious.com}$\}$. For notational convenience, we will use
integers as names for webpages.

Suppose you have already computed the personalized PageRank vectors for the following users:
\vspace{-0.25cm}
\begin{itemize}\setlength\itemsep{0pt}
\item Agatha, whose interests are represented by the teleport set $\{1,2,3\}$
\item Bertha, whose interests are represented by the teleport set $\{3,4,5\}$
\item Clementine, whose interests are represented by the teleport set $\{1,4,5\}$
\item Dolo, whose interests are represented by the teleport set $\{1\}$
\end{itemize}

Assume that the weights for each node in a teleport set are uniform.  Without
looking at the graph, can you compute the personalized PageRank vectors for the
following users? If so, how? If not, why not? Assume a fixed teleport parameter
$\beta$.

\vspace{-0.25cm}
\begin{enumerate}[i.]
\item \textbf{[4 points]} Eloise, whose interests are represented by the teleport set $\{2\}$.

\item \textbf{[4 points]} Felicity, whose interests are represented by the teleport set $\{5\}$.

\item \textbf{[4 points]} Glynnis, whose interests are represented by the teleport set $\{1,2,3,4,5\}$ with weights 0.1, 0.2, 0.3, 0.2, 0.2, respectively.
\end{enumerate}

\subsection{Personalized PageRank II [5 points]}

Suppose that you've already computed the personalized PageRank vectors of a set
of users (denote the computed vectors $V$). What is the set of all personalized
PageRank vectors that you can compute from $V$ without accessing the web graph?

\subsection{A different equation for PageRank [8 points]}
In class you that the PageRank equation can be written in different ways. You will prove this in this question.
Recall that the PageRank equation is:
\begin{align}
\label{equation:PageRank}
    \mathbf{r} = \mathbf{A} \mathbf{r},
\end{align}
where $\mathbf{r} \in \mathbb{R}^N$ is the PageRank vector, i.e. the vector of PageRank scores for each of the $N$ nodes, and (where $\mathbf{1}$ is the all-ones column vector of length $N$):
\begin{align}
    \mathbf{A} = \beta \mathbf{M} + \frac{1-\beta}{N}\mathbf{1}\mathbf{1}^\top.
\end{align}
Recall that here, $\mathbf{M}$ is the stochastic adjacency matrix, defined as
\begin{align}
    \mathbf{M}_{ij} = \left\{\begin{array}{lr}
        \frac{1}{d_j} , & \text{if (j,i) is an edge}\\
        0               & \text{otherwise}
        \end{array}\right\}
\end{align}
where $d_j$ is the degree of node $j$. Assume every node has degree at least 1.

Prove that (\ref{equation:PageRank}) is equivalent to
\begin{align}
    \mathbf{r} = \beta\mathbf{M} \mathbf{r}  + \frac{1-\beta}{N}\mathbf{1}.
\end{align}

Hint: Recall that $\mathbf{r}$ is normalized, i.e. $\sum_{i=1}^N \mathbf{r}_i = 1$.

\subsection*{What to submit}
\begin{enumerate}[2.1:]
\setcounter{enumi}{0}
\item
  \begin{itemize} \item  For each of $(i)$,$(ii)$, and $(iii)$, `yes' or `no' and a brief explanation of why or why not. \end{itemize}

\item
  \begin{itemize} \item A mathematical expression for the set in terms of $V$ and a brief explanation. \end{itemize}

\item
  \begin{itemize}
  \item Proof that the two PageRank equations are equivalent.
  \end{itemize}

\end{enumerate}


\section{Hamiltonian Monte Carlo}%
\label{sec:l2hmc_hmc}
We can improve upon the random-walk guess and check strategy of the generic
Markov Chain Monte Carlo algorithm by ``guiding'' the simulation according to
the systems natural dynamics using a method known as Hamiltonian (Hybrid)
Monte Carlo (HMC).

In HMC, model samples can be obtained by simulating a physical system
governed by a Hamiltonian comprised of kinetic and potential energy functions
that govern a particles dynamics.
%
By transforming the density function to a potential energy function and
introducing the auxiliary momentum variable $v$, HMC lifts the target
distribution onto a joint probability distribution in phase space $(x, v)$,
where $x$ is the original variable of interest (e.g.\ position in Euclidean
space).
%
A new state is then obtained by solving the equations of motion for a fixed
period of time using a volume-preserving integrator (most commonly the
\emph{leapfrog integrator}).
%
The addition of random (typically normally distributed) momenta encourages
long-distance jumps in state space with a single Metropolis-Hastings (MH)
step.

Let the `position' of the physical state be denoted by a vector $x
\in\mathbb{R}^{n}$ and the conjugate momenta of the physical state be denoted
by a vector $v \in\mathbb{R}^{n}$.
%
Then the Hamiltonian reads
%
\begin{align}
  H(x, v) &= U(x) + K(v)\\
                    & = U(x) + \frac{1}{2} v^{T} \,v,
  \label{eq:hamiltonian}
\end{align}
%
where $U(x)$ is the potential energy, and $K(v)=\frac{1}{2}v^{T}v$ the
kinetic energy.%
%
We assume without loss of generality that the position and momentum variables
are independently distributed.
%
That is, we assume the target distribution of the system can be written as
$\pi(x, v) = \pi(x) \pi(v)$.
%
Further, instead of sampling $\pi(x)$ directly, HMC operates by sampling from the
canonical distribution $\pi(x, v) = \frac{1}{Z} \exp(-H(x, v)) = \pi(x)
\pi(v)$, for some partition function $Z$ that provides a normalization factor.
%
Additionally, we assume the momentum is distributed according to an
identity-covariance Gaussian given by $\pi(v) \propto \exp{(-\frac{1}{2} v^{T} \,
v)}$ For convenience, we will denote the combined state of the system by $\xi
\equiv (x, v)$.
%
From this augmented state $\xi$, HMC produces a proposed state $\xi^{\prime} =
(x^{\prime}, v^{\prime})$ by approximately integrating Hamiltonian dynamics
jointly on $x$ and $v$.
%
This integration is performed along approximate iso-probability contours of
$\pi(x, v) = \pi(x) \pi(v)$ due to the Hamiltonians energy conservation.
%
\subsection{Hamiltonian Dynamics}%
\label{subsec:mcmc_hamiltonian_dynamics}
One of the characteristic properties of Hamilton's equations is that they
conserve the value of the Hamiltonian.
%
Because of this, every Hamiltonian trajectory is confined to an energy
\emph{level set},
%
\begin{equation}
H^{(-1)}(E) = \{x, v | H(x, v) = E\}.
\end{equation}
%
Our state $\xi = (x, v)$ then proceeds to explore this level set by integrating
Hamilton's equations, which are shown as a system of differential equations in
Eq.~\ref{eq:hamiltons_equations}.
% along wich the state $\xi = (x, v)$
% The state $\xi$ is modified in such a way that $H(\xi)$ remains constant thorughout the simulation.
%
% The differential equations governing the motion through state space are given by
%
\begin{align}
  \dot x_i &= \frac{\partial H}{\partial v_i} = v_i\\
  \dot v_i &= -\frac{\partial H}{\partial x_i} = - \frac{\partial
      U}{\partial x_i}
\label{eq:hamiltons_equations}
\end{align}
%
It can be shown~\cite{Neal_2012} that the above transformation is
volume-preserving and reversible, two necessary factors to guarantee asymptotic
convergence of the simulation to the target distribution.
%
The dynamics are simulated using the leapfrog integrator, which for a single
time step consists of:
%
\begin{align}
  v^{\frac{1}{2}} &= v - \frac{\eps}{2} \partial_x U(x)\\
  x^{\prime} &= x + \eps v^{\frac{1}{2}}\\
  v^{\prime} &= v - \frac{\eps}{2} \partial_x U(x^{\prime}).
  \label{eq:generic_leapfrog}
\end{align}
%
We write the action of the leapfrog integrator in terms of an operator
$\mathbf{L}: \mathbf{L}\xi \equiv \mathbf{L}(x, v) \equiv (x^{\prime},
v^{\prime})$, and introduce a momentum flip operator $\mathbf{F}: \mathbf{F}(x,
v) \equiv (x, -v)$.
%
The Metropolis-Hastings acceptance probability for the HMC proposal is given
by:
%
\begin{equation}
  A(\mathbf{F}\mathbf{L} \xi | \xi) = \min\left(1,
      \frac{\pi(\mathbf{F}\mathbf{L}\xi)}{\pi(\xi)}\left|
      \frac{\partial\left[\mathbf{F}\mathbf{L}\xi\right]}
      {\partial\xi^{T}}\right|\right),
\label{eq:metropolis_hastings}
\end{equation}
%
Where $\left|\frac{\partial\left[\mathbf{F}\mathbf{L}\xi\right]}
{\partial\xi^{T}}\right|$ denotes the determinant of the Jacobian describing
the transformation, and is equal to $1$ for traditional HMC.\@
%
In order to utilize these Hamiltonian trajectories to construct an efficient
Markov transition, we need a mechanism for introducing momentum to a given
point in the target parameter space.

Fortunately, this can be done by exploiting the probabilistic structure of the
system~\cite{Betancourt_2017}.
%
To lift an initial point in parameter space into one on phase space, we simply
sample from the conditional distribution over the momentum,
%
\begin{equation}
v \sim \pi(x | v).
\end{equation}
%
Sampling the momentum directly from the conditional distribution ensures that
this lift will fall into the typical set in phase space.
%
We can then proceed to explore the joint typical set by integrating Hamilton's
equations as demonstrated above to obtain a new configuration $\xi \rightarrow
\xip$.
%
We can then return to the target parameter space by simply projecting away the
momentum,
%
\begin{equation}
(x, v) \rightarrow x
\end{equation}
%
These three steps when performed in series gives a complete Hamiltonian Markov
transition composed of random trajectories that rapidly explore the target
distribution, as desired.
%
An example of this process can be seen in Fig~\ref{fig:hmc_phase_space}.
%
\begin{figure}[htpb]
\includegraphics[width=\textwidth]{new_figures/hmc_phase_space11}
\caption{\emph{Visualizing HMC for a $1D$ Gaussian} (example
  from~\cite{Betancourt_2017}, figure adapted with permission
  from~\cite{joeyl2hmc}). Each Hamiltonian Markov transition lifts the
  initial state onto a \color{gray}{random level set of the Hamiltonian,
  }\color{black} $H^{(-1)}(E)$, which can then be explored with a
  \color{blue}{Hamiltonian trajectory }\color{black} before
  \color{red}{projecting back down }\color{black} to the \color{green}{target
  parameter space}\color{black}.}%
\label{fig:hmc_phase_space}
\end{figure}
%
\vspace{-10pt}
\subsubsection{Properties of Hamiltonian Dynamics}
%
There are three fundamental properties of Hamiltonian dynamics which are
crucial to its use in constructing Markov Chain Monte Carlo updates.
%
\begin{enumerate}
  \item \textbf{Reversibility:} Hamiltonian dynamics are \textit{reversible}
      --- the mapping from $\mathbf{L}: \xi(t) \rightarrow \xi^{\prime} =
      \xi(t + s)$ is one-to-one, and consequently has an inverse
      $\mathbf{L}^{-1}$, obtained by negating the time derivatives in
      Eq.~\ref{eq:hamiltons_equations}.
  \item \textbf{Conservation of the Hamiltonian:} Moreover, the dynamics
      \textit{keeps the Hamiltonian invariant}.
  \item \textbf{Volume preservation:} The final property of Hamiltonian
      dynamics is that it \textit{preserves volume} in $(x, v)$ phase space
      (i.e. Liouville's Theorem).
\end{enumerate}
%
All in all, HMC offers noticeable improvements compared to the `random-walk'
approach of generic MCMC, but tends to perform poorly on high-dimensional
distributions.
%
This becomes immediately apparent when it is used for simulations in lattice
gauge theory and lattice QCD, where large autocorrelations and slow `burn-in'
can become prohibitively expensive.
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\subsection{Mean and variance of predictions}

\subsubsection{Bias}

\(\hat y =\theta x\)

\(E[\hat y-y]=E[\theta x-y]\)

\(y=\hat y + \epsilon \)

\(E[y-\hat y |X]\)

\(E[\epsilon |X]\)

Unbiased so long as independent of error term.

\subsubsection{Variance}

\(Var [\hat y-y]=Var [\theta x-y]\)

\(Var[y-\hat y |X]\)

\(Var[\epsilon |X]\)


\section{Logic Circuits}

\subsection*{A Half Adder}

A \emph{half adder} demonstrates the way in which computer logic gates can correctly add (binary) numbers, though here it can add only two \emph{binary digits} (``bits''). Thus, each half adder can only count to \emph{two}, but that is enough!  Long cascades of half adders and \emph{full adders} enable computers to count large numbers. Below, see two examples of half adders. The second example replaces the XOR gate with a series of simpler gates, but the results are the same. Each half adder takes two bits and adds them, passing on the bits for further work or as an answer itself.

\bigskip

\begin{figure}[!ht]
\begin{center}
\begin{tabular}{m{3.25in} m{2.0in}}

\input{./include/halfadder1.tex}

&

\begin{tabular}{ll | cc | c}
\multicolumn{5}{c}{\textbf{Truth Table}}\\
\hline\\[\negsep]
\textbf{A} & \textbf{B} & \textbf{Sum} & \textbf{Carry} & \textbf{Decimal}\\
\hline
0 & 0 & 0 & 0 & 0 \\
1 & 0 & 1  & 0 & 1 \\
0 & 1 & 1  & 0 & 1 \\
1 & 1 & 0  & 1 & 2 \\
\hline
\end{tabular}

\\

\end{tabular}

\caption{A half-adder circuit. It adds two bits together. If the result is larger than 1, the ``carry" signal is high.}
\end{center}
\end{figure}

Figure \ref{fig:threehalfadders} shows each of the three possible outcomes of a half adder. Make sure you understand how each result happens.

\begin{figure}[!hb]
\begin{center}

\input{./include/halfadder2.tex}

\caption{Another half-adder circuit. The exclusive-OR gate has been replaced by an equivalent circuit made up of five simpler gates. You can see why XOR requires so many transistors!}
\end{center}
\end{figure}

\input{./include/three-half-adder-outputs.tex}

\clearpage

\subsection*{Full Adder}

This type of adder is a little more difficult to implement than a half-adder. The main difference between a half-adder and a full-adder is that the full-adder has \emph{three} inputs and two outputs. The first two inputs are A and B, just like with every half-adder, and the third input is an input carry, designated as $C_{in}$. Enabling the circuit to handle a carry \emph{in} from another adder makes it possible to add many binary orders of magnitude, since many carry-outs and carry-ins will be needed to add numbers larger than 1. Once we implement a full adder, we can string eight of them together to create a byte-wide adder and cascade the carry bit from one adder to the next.
\bigskip

\begin{figure}[!hb]
\begin{center}
\input{./include/fulladder1.tex}
\caption{A full-adder circuit, made up of two half-adder circuits. A full-adder enables carry-in as well as carry-out. This means that the full adder can add \emph{three} inputs together, and keep track of all of them. If the sum is two or three, the carry-out signal goes high. These units can be linked together to make binary addition work even for very large numbers.}
\end{center}
\end{figure}

Since full adders can accept three inputs, the truth table is pretty big.

\begin{center}
\begin{tabular}{ccc  cc  c}
\multicolumn{6}{c}{\textbf{Full Adder Truth Table}}\\
\hline\\[\negsep]
 &  & \textbf{Carry} &  & \textbf{Carry} & \\
\textbf{A} & \textbf{B} & \textbf{In} & \textbf{Sum} & \textbf{Out} & \textbf{Decimal}\\
\hline
0 & 0 & 0 & 0  & 0 & 0 \\
\grr
1 & 0 & 0 & 1  & 0 & 1 \\
0 & 1 & 0 & 1  & 0 & 1 \\
\grr
1 & 1 & 0 & 0  & 1 & 2 \\[\sep]
\hline\\[\negsep]
0 & 0 & 1 & 0  & 0 & 1 \\
\grr
1 & 0 & 1 & 0  & 1 & 2 \\
0 & 1 & 1 & 0  & 1 & 2 \\
\grr
1 & 1 & 1 & 1  & 1 & 3 \\

\hline
\end{tabular}
\end{center}

\begin{figure}[!ht]
\begin{center}
\input{./include/fulladder1_example001.tex}
\caption{One of the possible input conditions for a full adder. Here, only the carry-in bit is 1. The inputs for A and B are 0. Since both A and B are the same, $XOR_1$ outputs 0 (no voltage). Since the carry-in bit (signal) is 1, and the output from $XOR_1$ is 0, the inputs to $XOR_2$ are different: so $XOR_2$ outputs 1. The inputs to $AND_2$ are not both 1, so $AND_2$ outputs 0. A and B are both 0, so $AND_1$ outputs 0. Since both $AND_1$ and $AND_2$ output 0, the OR gate outputs 0. Here, the value from the carry-out signal from another full adder is the only input. So at \emph{this} digit (``order of magnitude''), the sum is 1, and there is no value to carry out. }
\end{center}
\end{figure}

\begin{figure}[!ht]
\begin{center}
\includegraphics[scale=0.19]{FullAdderboard.png}
\caption{A single full-adder circuit for adding two bits plus a carry-in digit, implemented on a small circuit board using individual integrated circuit gates instead of discrete transistors. You can see the gates easily, but the board is much simpler to work with, compared to forming up a full adder from five of the single-gate boards.}
\end{center}
\end{figure}

\clearpage

\subsection*{Equality}

Comparing two one-bit numbers to see if they are identical is done with XNOR gates. These gates are the same as an XOR gate passing output into a NOT gate.

\begin{center}

\begin{tabular}{p{2.5in} p{3.5in} }
\hline\\[\negsep]

The symbol for an XNOR gate is:

\vspace{0.25in}

\begin{circuitikz}
\draw
	(0,0) node[xnor port](xnorGate) {}
	(xnorGate.in 1) node[left] {{\color{red}$INPUT~A$}}
	(xnorGate.in 2) node[left] {{\color{red}$INPUT~B$}}
	(xnorGate.out) node[right] {{\color{red}$OUT$}}
;
\end{circuitikz}

&

\centering

The ``Truth Table" (how they behave) is:
\vspace{0.15in}

\begin{tabular}{ll | c}
\multicolumn{3}{c}{\textbf{XNOR Gate }}\\
\multicolumn{3}{c}{\textbf{Truth Table}}\\
\hline\\[\negsep]
\textbf{A} & \textbf{B} & \textbf{OUT}\\
\hline
0 & 0 & 1  \\
1 & 0 & 0  \\
0 & 1 & 0  \\
1 & 1 & 1  \\
\hline
\end{tabular}
\\
\tabularnewline

\hline\\[\negsep]

\end{tabular}
\end{center}

\begin{figure}[hb!]
\begin{center}
\includegraphics[scale=0.17]{XNOR1.jpg}
\caption{A discrete-transistor XNOR gate made up of one XOR and one NOT gate. Tying the output of the XOR gate to the input of the NOT gate makes the result of the NOT gate an XNOR of the original two XOR gate inputs---answering the question ``are the two inputs equal?"}
\end{center}
\end{figure}

To compare large numbers, the circuit requires an XNOR gate for each binary digit (bit) in the numbers being compared. If any XNOR gate returns zero, the numbers are not equivalent. To build all of the XNOR gates into a single yes or no answer, each XNOR output could go into a series of AND gates -- that is, since AND returns a ``yes'' only if both inputs are one, a single ``no'' prevents the AND gate from returning ``yes''.

\begin{figure}[h!]
\begin{center}
\input{./include/andcascade.tex}
\caption{A circuit to provide a yes/no answer to the question, ``are  two four-bit numbers, A and B, equal?" Each XNOR gate compares the bits in each position of each number, and passes the answer to an AND gate input. The AND results ripple forward to a final answer.}
\end{center}
\end{figure}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=0.16, clip=TRUE, trim=0 0 500 0]{4bitxnor.jpg}
\caption{Testing for equality with four XNOR gates. Here, XNOR gates are created with XOR gates that pass the output (green wires) to a NOT gate. An XOR gate only goes logic high if the inputs are different. Thus, NOT XOR means both inputs are the same! Each XNOR output (blue wires) passes to a NAND gate. Recall that NAND gates only return zero if both inputs are 1. The output of each NAND gate (purple wires) passes to one input of a NOR gate. So if both inputs to a given NAND gate are ``1'', the NAND gate will show ``0" as its result. NOR gates only return 1 if both inputs are zero. The result: the NOR gate lights up an LED if all inputs are equal. It is no different than the AND-gate cascade shown above in its result, it just uses negative logic to get there. Here, both inputs are equal to 1, so the LED is lit.}
\end{center}
\end{figure}

\clearpage


\section{Multivariable Equations}
\subsection{Functions of Multiple Independent Variables}
Consider $f(x,y)$ and a small vector displacement $\underline{\mathrm ds}=(\mathrm dx,\mathrm dy)$, so the change along $\underline{\mathrm ds}$ is
$$\mathrm df=\frac{\partial f}{\partial x}\mathrm dx+\frac{\partial f}{\partial y}\mathrm dy=\underline{\mathrm ds}\cdot(\nabla f)$$
by the multivariate chain rule.
Here, $\nabla f=(f_x,f_y)$ is called the gradient of $f$.
If we write $\underline{\mathrm ds}=\mathrm ds\underline{\hat{s}}$ with $|\underline{\hat{s}}|=1$, so $\mathrm df=\mathrm ds(\underline{\hat{s}}\cdot\nabla f)$
\begin{definition}
    The directional derivative is defined as
    $$\frac{\mathrm df}{\mathrm ds}=\underline{\hat{s}}\cdot\nabla f$$
\end{definition}
\begin{proposition}
    1. The magnitude of $\nabla f$ is the maximum rate of change of $f$, that is
    $$|\nabla f|=\sup_{\underline{\hat{s}},|\underline{\hat{s}}|=1}\frac{\mathrm df}{\mathrm ds}$$
    And the supremum can be attained.\\
    2. The direction of $\nabla f$ is the direction where $f$ increases most rapidly.\\
    3. If $\underline{\hat{s}}$ is parallel to the contour of $f$, then $\mathrm df/\mathrm ds=0$.
\end{proposition}
\begin{proof}
    1. Cauchy-Schwarz and take $\underline{\hat{s}}=\nabla{f}/|\nabla f|$.\\
    2,3. Obvious.
\end{proof}
\begin{corollary}
    There is always one direction where $\mathrm df/\mathrm ds=0$.
\end{corollary}
\begin{proof}
    Immediate.
\end{proof}
\begin{proposition}
    Local extrema of $f$ have $\mathrm df/\mathrm ds=0$ for any direction.
    Hence $\nabla f=\underline{0}$.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
However, if $\nabla f=\underline{0}$ at some point, it does \textit{not} follow that the point is an local extremum, since it could be a saddle point.
Near local extrema, the contour of $f$ is elliptical while it is hyperboly near saddle points.
Also, contours of $f$ can only cross at saddle point.
\subsection{Taylor Series for Multivariate Functions}
Consider $f(x,y)$ near a point $\underline{s_0}\in\mathbb R^2$ and a displacement $\delta s$ along the line $\underline{\delta s}$, so
\begin{align*}
    f(s_0+\delta s)&=f(s_0)+\delta s\frac{\mathrm df}{\mathrm ds}+\frac{(\delta s)^2}{2}\frac{\mathrm d^2f}{\mathrm ds^2}+\cdots\\
    &=f(s_0)+\delta s(\underline{\hat s}\cdot\nabla f)+\frac{(\delta s)^2}{2}(\underline{\hat{s}}\cdot\nabla)(\underline{\hat s}\cdot\nabla)f+\cdots
\end{align*}
To write it out in coordinate form,
\begin{definition}
    The Hessian matrix is defined by
    $$H=
    \begin{pmatrix}
        f_{xx}&f_{xy}\\
        f_{yx}&f_{yy}
    \end{pmatrix}=\nabla(\nabla f)$$
    Note that $H$ is symmetric whenever $f$ is nice enough (e.g. $C^2$) to have $f_{yx}=f_{xy}$.
\end{definition}
We have, by this notation,
\begin{align*}
    f(\underline{x})&=f(x,y)\\
    &=f(x_0,y_0)+(x-x_0)f_x+(y-y_0)f_y\\
    &\quad+\frac{1}{2}((x-x_0)^2f_{xx}+2(x-x_0)(y-y_0)f_{xy}+(y-y_0)^2f_{yy})+\cdots\\
    &=f(\underline{x_0})+\nabla f(\underline{x_0})(\underline{x}-\underline{x_0})^\top+\frac{1}{2}(\underline{x}-\underline{x_0}) H(x_0)(\underline{x}-\underline{x_0})^\top+\cdots
\end{align*}
\subsection{Classification of Stationary Points}
When $\nabla f=\underline{0}$ at some $\underline{x_0}$, we have
$$f(\underline{x_0}+\underline{\delta x})\approx f(\underline{x_0})+\frac{1}{2}\underline{\delta x} H(x_0)\underline{\delta x}^\top$$
Note that this can extend analogously to $n$ dimensions.
\begin{definition}
    The Hessian in $n$ dimensions is defined by
    $$H=
    \begin{pmatrix}
        f_{x_1x_1}&f_{x_1x_2}&\dots&f_{x_1x_n}\\
        f_{x_2x_1}&f_{x_2x_2}&\dots&f_{x_2x_n}\\
        \vdots&\vdots&\ddots&\vdots\\
        f_{x_nx_1}&f_{x_nx_2}&\dots&f_{x_nx_n}
    \end{pmatrix}$$
\end{definition}
If $f$ is nice enough to allow change of order of partial derivatives, then $H(x_0)$ is real and symmetric, hence diagonalizable by Spectral Theorem.
So we can choose the basis to be the eigenbasis (which can be chosen to be orthonormal), therefore
$$\underline{\delta x}H(x_0)\underline{\delta x}^\top=\sum_{i=1}^n\lambda_i(\delta x_i)^2$$
With this, we can classify the stationary points in $3$ cases.\\
Case 1: $\forall\underline{\delta x}\in\mathbb R^n\setminus\{\underline{0}\},\underline{\delta x}H(x_0)\underline{\delta x}^\top>0$.
This happens iff $\lambda_i>0$ for each $i$, that is, $H$ is positive definite.\\
Case 2: $\forall\underline{\delta x}\in\mathbb R^n\setminus\{\underline{0}\},\underline{\delta x}H(x_0)\underline{\delta x}^\top<0$.
This happens iff $\lambda_i<0$ for each $i$, that is, $H$ is negative definite.\\
Case 3: Otherwise, $H$ is indefinite.
\begin{definition}
    The signature of $H$ is the pattern of signs of its subdeterminant.
\end{definition}
For example, for $f(x_1,x_2,\ldots x_n)$, the subdeterminants are
$$H_k=
\begin{pmatrix}
    f_{x_1x_1}&f_{x_1x_2}&\dots&f_{x_1x_k}\\
    f_{x_2x_1}&f_{x_2x_2}&\dots&f_{x_2x_k}\\
    \vdots&\vdots&\ddots&\vdots\\
    f_{x_kx_1}&f_{x_kx_2}&\dots&f_{x_kx_k}
\end{pmatrix}$$
Then the sign is the signs of $|H_1|,|H_2|,\ldots,|H_n|$.
\begin{theorem}
    If $H$ is positive (negative) definite, so is all of $H_i$.
\end{theorem}
Therefore a minimum (maximum) point of a real function in $\mathbb R^n$ is also a minimum (maximum) point in any subspace of $\mathbb R^n$ that includes the point.
\begin{center}
    \begin{tabular}{c|c}
        Type of S.P.&Signature\\
        \hline
        Minimum&$+,+,+,+,\ldots$\\
        Maximum&$-,+,-,+,\ldots$
    \end{tabular}
\end{center}
Sometimes $|H|=0$, in which case this stationary point is called degenerate, so we need to look at higher order terms in the Taylor series.
The helps us in sketching the contour of a two dimensional function.
Consider the coordinate system aligned with the (orthonormal) eigenbasis of $H$, so $\underline{\delta x}=\underline{x}-\underline{x_0}=(\xi_1,\xi_2)$, where $\underline{x_0}$ is a stationary point.
In a small region near $x_0$, contour of $f$ satisfies
$$\text{const}=f\approx f(\underline{x_0})+\frac{1}{2}\underline{\delta x}H\underline{\delta x}^\top\implies \lambda_1\xi_1^2+\lambda_2\xi_2^2=\text{const}$$
Near min/max, $\lambda_1,\lambda_2$ have the same sign, so the contour looks like an ellipse.
Near saddles, they have the different sign and the contour looks like a hyperbola.
\begin{example}
    We want to find and classify the stationary points of
    $$f(x,y)=4x^3-12xy+y^2+10y+6$$
    and sketch its contour.\\
    We have $f_x=12x^2-12y,f_y=-12x+2y+10$, so at the stationary point, we have $(x,y)=(1,1),(5,25)$.
    Now $f_{xx}=24x,f_{xy}=f_{yx}=-12,f_{yy}=2$.
    So at $(1,1)$,
    $$H=\begin{pmatrix}
        24&-12\\
        -12&2
    \end{pmatrix}\implies |H_1|>0,|H_2|<0$$
    So it is neither a maximum nor a minimum, hence it is a saddle point.\\
    As for $(5,25)$,
    $$H=\begin{pmatrix}
        120&-12\\
        -12&2
    \end{pmatrix}\implies |H_1|>0,|H_2|>0$$
    so it is a minimum.
    We can then sketch the contour by drawing ellipses near $(5,25)$ and hyperbolic curves near $(1,1)$.
\end{example}
\subsection{System of Linear ODEs}
Consider a few dependent variables $y_1(t),y_2(t),\ldots$ which satisfies system of coupled ODEs.
\begin{example}
    Consider
    $$
    \begin{cases}
        \dot{y_1}=ay_1+by_2+f_1(t)\\
        \dot{y_2}=cy_1+dy_2+f_2(t)
    \end{cases}
    $$
    So in vector form
    $$\underline{\dot{y}}=M\underline{y}+\underline{f}$$
\end{example}
Any $n^{th}$ order ODE can be written as a system of $n$ first-order ODEs.
\begin{example}
    Consider $\ddot{y}+a\dot{y}+by=f$, so let $y_1=y,y_2=y^\prime$, then we have
    $$\frac{\mathrm d}{\mathrm dt}\begin{pmatrix}
        y_1\\
        y_2
    \end{pmatrix}=
    \begin{pmatrix}
        0&1\\
        -b&-a
    \end{pmatrix}
    \begin{pmatrix}
        y_1\\
        y_2
    \end{pmatrix}+
    \begin{pmatrix}
        0\\
        f
    \end{pmatrix}$$
\end{example}
One way to solve it is to use matrix methods.
Consider
$$\underline{\dot{y}}=M\underline{y}+\underline{f}$$
First, we find the general solution $\underline{y_c}$ to
$$\underline{\dot{y}_c}-M\underline{y_c}=0$$
Then we find a particular solution $\underline{y_p}$ to the system, so by the superposition principle the general solution is $\underline{y_p}+\underline{y_c}$.\\
To find solution to the homogeneous equation, we try $\underline{y_c}=\underline{v}e^{\lambda t}$ which leads us to the conclusion that $\lambda$ must be an eigenvalue of $M$ with eigenvector $\underline{v}$.
\begin{example}
    Consider
    $$\underline{\dot{y}}-\begin{pmatrix}
        -4&24\\
        1&-2
    \end{pmatrix}\underline{y}=\begin{pmatrix}
        4\\
        1
    \end{pmatrix}e^t$$
    We can find the eigenvalues and eigenvectors of $M$ and find that $\lambda_1=2,\underline{v_1}=(4,1)^\top,\lambda_2=-8,\underline{v_2}=(-6,1)^\top$
    So we have
    $$\underline{y_c}(t)=A\begin{pmatrix}
        4\\
        1
    \end{pmatrix}e^{2t}+B\begin{pmatrix}
        -6\\
        1
    \end{pmatrix}e^{-8t}$$
    We can sketch the phase portrait of $y_2$ against $y_1$ to find a hyperbola-like path of $(y_1,y_2)$.\\
    To find the particular integral, we can try
    $$\underline{y_p}=\begin{pmatrix}
        u_1\\
        u_2
    \end{pmatrix}e^t$$
    and find that $(u_1,u_2)=(-4,-1)$ solves the system, therefore the general solution is
    $$
    \underline{y}=
    \begin{pmatrix}
        -4\\
        -1
    \end{pmatrix}e^t+
    A\begin{pmatrix}
        4\\
        1
    \end{pmatrix}e^{2t}+B\begin{pmatrix}
        -6\\
        1
    \end{pmatrix}e^{-8t}
    $$
    where $A,B$ are constants.
\end{example}
If the forcing matches the eigenvalue and eigenvector, we can try multiplying a polynomial in $t$ (mostly just $t^k$ will work).\\
From a linear system of $n$ first-order ODEs, we can construct $n$ uncoupled $n^{th}$ order ODEs.
\begin{example}
    Consider the same equation
    $$\underline{\dot{y}}=\begin{pmatrix}
        -4&24\\
        1&-2
    \end{pmatrix}\underline{y}+\begin{pmatrix}
        4\\
        1
    \end{pmatrix}e^t$$
    We can differentiate the first component to get $\ddot{y_1}=-4\dot{y_1}+24\dot{y_2}+4e^t=-6\dot{y_1}+16y_1+36e^t$, which we know how to solve.
    Similar way works for $y_2$.
    One can check that this essentially gives the same solution.
\end{example}
We can discuss the concept of Phase Portrait in a more general way.
For complementary function $\underline{y_c}$ satisfying $\underline{\dot{y}_c}=M\underline{y_c}$, then $y_c=\underline{v_1}e^{\lambda_1 t}+\underline{v_2}e^{\lambda_2 t}$\\
Case 1: $\lambda_1,\lambda_2$ are real and $\lambda_1\lambda_2<0$, WLOG $\lambda_1>0>\lambda_2$, then the Phase Portrait is hyperbolic and converging towards the direction of $\underline{v_1}$.\\
Case 2: $\lambda_1,\lambda_2$ are real and $\lambda_1\lambda_2>0$.
If $\lambda_1<\lambda_2<0$, the phase portrait converge to $0$.
This is called the stable node.
If $\lambda_1>\lambda_2>0$, the phase portrait diverge from $0$.
this is called the unstable node.\\
Case 3: They are both complex.\\
Case 3(a): The real parts are both negative, then the amplitude will decrease in time, so it produces a spiral-like phase portrait converging to $0$ (stable spiral).\\
Case 3(b): They are both positive, so the amplitude grows in time, so it produces a spiral-like curve spiraling out of $0$, then it is an unstable spiral.\\
Case 3(c): They are both zero, then the amplitudes does not change and it is purely oscillating (i.e. circles centering at $0$).
We can find the direction of the oscillation by evaluating the equation at a given point and find the sign of $\dot{y}_2$.
\subsection{Nonlinear Systems}
Consider a nonlinear autonomous system
$$\begin{cases}
    \dot{x}=f(x,y)\\
    \dot{y}=g(x,y)
\end{cases}$$
We want to find equilibrium (fixed) point, so for $\dot{x}=\dot{y}=0$, we have $f(x_0,y_0)=0=g(x_0,y_0)$
We can do perturbation analysis by a small displacement $(x,y)=(x_0+\xi(t),y_0+\eta(t))$ around the fixed point, so
$$\dot{\xi}=f(x_0+\xi,y_0+\eta)=f(x_0,y_0)+\xi f_x(x_0,y_0)+\eta f_y(x_0,y_0)+\cdots$$
$$\dot{\eta}=g(x_0+\xi,y_0+\eta)=g(x_0,y_0)+\xi g_x(x_0,y_0)+\eta g_y(x_0,y_0)+\cdots$$
But $f,g$ are $0$ at $(x_0,y_0)$, so
$$\begin{pmatrix}
    \dot{\xi}\\
    \dot{\eta}
\end{pmatrix}\approx
\left.\begin{pmatrix}
    f_x&f_y\\
    g_x&g_y
\end{pmatrix}\right|_{(x_0,y_0)}\begin{pmatrix}
    \xi\\
    \eta
\end{pmatrix}$$
\begin{example}
    Lotka-Volterra Model of predator and prey.
    $$\begin{cases}
        \dot{x}=\alpha x-\beta xy=f(x,y)\\
        \dot{y}=\delta xy-\gamma y=g(x,y)
    \end{cases}$$
    where $\alpha,\beta,\gamma,\delta>0$.
    The fixed point is $(0,0)$ and $(\gamma/\delta,\alpha/\beta)$.
    Note that we have
    $$\begin{pmatrix}
        f_x&f_y\\
        g_x&g_y
    \end{pmatrix}=\begin{pmatrix}
        \alpha-\beta y&-\beta x\\
        \delta y&\delta x-\gamma
    \end{pmatrix}$$
    At $(0,0)$, we have
    $$\begin{pmatrix}
        \dot{\xi}\\
        \dot{\eta}
    \end{pmatrix}\approx
    \begin{pmatrix}
        \alpha&0\\
        0&-\gamma
    \end{pmatrix}\begin{pmatrix}
        \xi\\
        \eta
    \end{pmatrix}$$
    So the eigenvalues are $\alpha,-\gamma$, thus it is a saddle point.
    For the other fixed point,
    $$\begin{pmatrix}
        \dot{\xi}\\
        \dot{\eta}
    \end{pmatrix}\approx
    \begin{pmatrix}
        0&\frac{-\beta\gamma}{\delta}\\
        \frac{\alpha\delta}{\beta}&0
    \end{pmatrix}\begin{pmatrix}
        \xi\\
        \eta
    \end{pmatrix}$$
    So it has a pair of purely imaginary eigenvalues, so it a center.
    The direction of rotation is counterclockwise.
    Indeed, we have
    $$\dot{\xi}=\frac{-\beta\gamma}{\delta}\eta<0$$
    for $\eta>0$.
    We can sketch the solutions.
\end{example}
\subsection{Partial Differential Equations}
\begin{definition}
    A PDE is a DE with partial derivatives.
\end{definition}
Here, we will only consider three examples.
\subsubsection{First Order Wave Equation}
This is the PDE
$$\frac{\partial y}{\partial t}-c\frac{\partial y}{\partial x}=0$$
where $y=y(x,t)$ and $c$ is a constant.
We can solve it with the method of characteristics.
Imagine the contour of $y$ in the $x-t$ plane, then we can start at some point and move it back and forth along a path.
So along a path $x(t)$, if we have $y(x(t),t)$, then plugging it in our equation by the multivariate chain rule,
$$\frac{\mathrm dy}{\mathrm dt}=\frac{\partial y}{\partial t}+\frac{\partial y}{\partial x}\frac{\mathrm dx}{\mathrm dt}$$
So we can take the path where $\dot{x}=-c$ thus $\dot{y}=0$ (so $x=x_0-ct$), hence $y$ will be constant along that line.
These paths are called characteristics.
If $y(x,t=0)=f(x)$, then $y=f(x_0)$ along the characteristic.
Therefore the general solution is
$$y=f(x+ct)$$ for some differentiable $f$.
\begin{example}
    With the initial condition $y(x,0)=x^2-3$, since we have $y=f(x+ct)$, $y(x,t)=(x+ct)^2-3$.
\end{example}
If we add some forcing,
\begin{example}
    Consider
    $$\frac{\partial y}{\partial t}+5\frac{\partial y}{\partial x}=e^{-t}$$
    with $y(x,0)=e^{-x^2}$.
    So $\mathrm dy/\mathrm dt$ along the path with $\mathrm dx/\mathrm dt=5$, so $y=A-e^{-t}$ along a path.
    Note that $A$ depends on $x(0)=x_0$, the initial point of the path.
    We know $y(x,0)=A-1=e^{-x_0^2}$, so $A=1+e^{-x_0^2}$.
    Hence $y=1+e^{-(x-5t)^2}-e^{-t}$.
\end{example}
\subsubsection{Second Order Wave Equation}
We want to solve
$$\frac{\partial^2 y}{\partial t^2}-c^2\frac{\partial^2 y}{\partial x^2}=0$$
So we can ``factorize'' it to have
$$\left( \frac{\partial}{\partial t}-c\frac{\partial}{\partial x} \right)\left( \frac{\partial}{\partial t}+c\frac{\partial}{\partial x} \right)y=0$$
Hence solutions can have $y_t\pm cy_x=0$, so the solution is in the form $f(x+ct)+g(x-ct)$.
\begin{example}
    Suppose we want to solve $y_{tt}-c^2y_{xx}=0$ and $y=1/(1+x^2),y_t=0$ at $t=0$ and $y\to 0$ as $x\to\infty$.
    So
    $$\begin{cases}
        f(x)+g(x)=1/(1+x^2)\\
        cf^\prime(x)-cg^\prime(x)=0\implies f=g+A
    \end{cases}$$
    for some constant $A$.
    So we can solve to get
    $$g(x)=\frac{1}{2(1+x^2)}-\frac{A}{2},f(x)=\frac{1}{2(1+x^2)}+\frac{A}{2}$$
    So
    $$y=\frac{1}{2(1+(x+ct)^2)}+\frac{1}{2(1+(x-ct)^2)}$$
    We can sketch the solution to find that this gives two waves moving to the two ends.
\end{example}
\subsubsection{Diffusion Equation}
We consider
$$\frac{\partial y}{\partial t}=\kappa\frac{\partial^2 y}{\partial x^2}$$
where $\kappa$ is a constant.
Typical cases where diffusion occurs are pollution transport, heat conduction and movement of microles.
Integrate the equation over $\mathbb R$ to get
$$\frac{\partial}{\partial t}\int_{-\infty}^\infty y\,\mathrm dx=\kappa [y_x]^\infty_{-\infty}$$
So if $y_x\to 0$ as $x\to\pm\infty$, the integeral of $y$ over $\mathbb R$ will be constant.\\
We can solve by the use of similarity variable.
\begin{example}
    Consider $y_t=\kappa y_{xx}$ where $y(x,0)=\delta (x)$ and $y\to 0$ when $x\to\pm\infty$.
    Define $\eta=x^2/(4\kappa t)$.
    \footnote{Obtained from dimensional analysis}
    We can try solutions of the form $y=t^{-\alpha}f(\eta)$ to get
    $$-\frac{\alpha}{t}+f^\prime\eta_t=\kappa f^{\prime\prime}(\eta_x)^2+\kappa f^\prime\eta_{xx}$$
    But note that $\eta_t=-\eta/t,\eta_x=\eta/(\kappa t),\eta_{xx}=2/(4\kappa t)$.
    All of the terms then have a factor of $1/t$, so we can remove the time dependence and get
    $$\alpha f+f^\prime \eta+f^{\prime\prime}\eta+f^\prime/2=0$$
    Let $\alpha=1/2$, we have $\eta F^\prime+F/2=0$ where $F=f+f^\prime$.
    If $F=0$ (which is a solution) then $f(\eta)=Ae^{-\eta}$.
    Then $y=At^{-1/2}e^{-x^2/(4\kappa t)}$, then from the delta function condition we have $A=1/\sqrt{4\pi\kappa}$, hence
    $$y(x,t)=\frac{1}{\sqrt{4\pi\kappa}}t^{-1/2}e^{-x^2/(4\kappa t)}$$
    is a solution.
\end{example}


\subsection{Commutation}

We define a function, the commuter, between two objects \(a\) and \(b\) as:

\([a,b]=ab-ba\)

For numbers, \(ab-ba=0\), however for matrices this is not generally true.

\subsection{Commutators and eigenvectors}

Consider two matrices which share an eigenvector \(v\).

\(Av=\lambda_A v\)

\(Bv=\lambda_B v\)

Now consider:

\(ABv=A\lambda_B v\)

\(ABv=\lambda_A\lambda_B v\)

\(BAv=\lambda_A\lambda_B v\)

If the matrices share all the same eigenvectors, then the matrices commute, and \(AB=BA\).


\section{Entropy from Noise Modeling}
\index{multiscale entropy}
\subsection{Information and wavelet coefficient}
In the case of signal restoration, the noise is the main problem. This
means that we should not consider the probability of appearance of
a pixel value in an image, but rather its probability of being due to the
signal (or to the noise).
If we consider a variable $x$ which follows a probability distribution
$p(x)$, we
can define the information in $x$ by $- \ln(p(x))$, and a signal $S$ can
be considered as a set of individual variables $x_k$ (pixels), each of which
follows the same probability distribution.
Then the information contained in the data
can be measured by $- \sum_{k=1}^N \ln(p(x_k))$. If $X$ follows a Gaussian
distribution with zero mean, we have
\begin{eqnarray}
H(X) = \sum_{k=1}^N \frac{x_k^2}{2 \sigma^2} + \rm{Cst}
\end{eqnarray}
The energy gives a good measurement of information. But many of the required
criteria are not fulfilled by using such an entropy (correlation between
pixels, background-independence, etc.). It seems difficult to derive
a good probability distribution from the pixel values which fulfill the
entropy requirements.

This is not so for transformed data, especially when using
the wavelet transform.
This has  already been done, in fact,
for finding threshold levels in filtering
methods by means of wavelet coefficient thresholding
\cite{starck:sta95_1,rest:moulin99,rest:donoho93_1,rest:krim92,wave:chipman97,wave:amato97}.
Thus we must introduce the concept of multiresolution into our entropy.
We will now consider that the information contained in some dataset
is the sum of the information at different resolution levels $j$.
The wavelet transform $W$ of a signal by a fast algorithm contains a set
of coefficients $w_{j,k}$ ($j$ being the scale index), and a set
of coefficients $c_{k}$ representing the signal at a very low resolution
(see \cite{ima:mallat98,starck:book98} for more information about the
wavelet transform). If the number of scales is enough large,
we can assume that the coefficients $c_{k}$ furnish information only
about the background, and not on the signal of interest. The entropy of $X$
must be measured only from the
wavelet coefficients $w_{j,k}$.

% Choosing the \`a trous wavelet transform (see \cite{starck:sta95_1}
% for a description of this wavelet transform algorithm), a signal $X$ can
% be represented by:
% \begin{eqnarray}
% X_k = \sum_{j=1}^{l} w_{j,k} + c_{l,k}
% \end{eqnarray}
% where $k$ is the pixel index, $w_j$ are the wavelet coefficients of $S$,
% $j$ the resolution
% level, and $c_l$ is smoothed version of $S$.
Due to the properties of
the wavelet transform, the set $w_{j}$ for all $j$ has a zero mean. From
noise modeling, we can derive the probability distribution in the
wavelet space of a wavelet coefficient, assuming it is due to the noise.
The entropy becomes
\begin{eqnarray}
H(X) = \sum_{j=1}^{l} \sum_{k=1}^{N_j}  h(w_{j,k})
\end{eqnarray}
with $h(w_{j,k})  = - \ln p(w_{j,k})$. We will note in the following
$h$ for the entropy (or information) relative to a wavelet coefficient,
and $H$ for the multiscale entropy (MSE) of a signal or an image.
For Gaussian noise, we get
\begin{eqnarray}
H(X) =  \sum_{j=1}^{l}  \sum_{k=1}^{N_j} \frac{w_{j,k}^2}{2 \sigma_j^2}+ \rm{Cst}
\end{eqnarray}
where $\sigma_j$ is the noise at scale $j$. We see that
the information is proportional
to the energy of the wavelet coefficients.
The higher a wavelet coefficient, then the lower will be the  probability, and the
higher will
be the information furnished by this wavelet coefficient. As the constant
has no effect in the solution calculation in restoration problems,
we take the liberty to remove it in the following.

We can see
easily that this entropy fulfills all the requirements of
section~\ref{sect_entr}.
If we consider two signals $S_1$, $S_2$, derived from a third one $S_0$ by
adding noise:
\begin{eqnarray}
S_1 & = & S_0 + N_1(\sigma_1) \nonumber \\
S_2 & = & S_0 + N_2(\sigma_2)
\end{eqnarray}
then we have:
\begin{eqnarray}
\mbox{if } \sigma_1 < \sigma_2 \mbox{ then } H(S_1) > H(S_2)
\end{eqnarray}
and a flat image has zero entropy.

Our entropy definition is completely dependent on the noise modeling.
If we consider a signal $S$, and we assume that the noise is Gaussian, with
a  standard deviation equal to $\sigma$, we won't measure the same
information compared to  the case when we consider that the noise has
another standard deviation
value, or if the noise follows another distribution.
As for the Shannon
entropy, the information increases with the entropy, and using such an
entropy leads to a Minimum Entropy Method.

\begin{figure}[htb]
\centerline{
\hbox{
\psfig{figure=fig_multi_memscale.ps,bbllx=2.5cm,bblly=13cm,bburx=19.5cm,bbury=25.5cm,width=14cm,height=7cm,clip=}
}}
\caption{Multiscale entropy of the saturn image (continuous curve),
and multiscale
entropy of the scrambled image (dashed curve).}
\label{fig_multi_memscale}
\end{figure}
Figure~\ref{fig_multi_memscale} shows the information measure at each
scale for both the saturn image and its scrambled version. The global
information is the addition of the information at each scale. We see
that for the scrambled image (dashed curve), the
information-versus-scale
curve is flat, while for the unscrambled saturn image,
it increases with the scale.

\subsection{Signal information and noise information}

In the previous section, we have seen how it was possible to measure
the information $H$ related to a wavelet coefficient.
Assuming the signal $X$ is still composed of the three elements $S$,$B$,$N$
(X=$S+F+B$, signal of interest, background, and noise),
$H$ is independent of $B$. But some of this information
is spurious and undesirable and has been introduced via the noise $N$.
To get the useful information, we must subtract out this spurious portion.
Trying to decompose our information measure
into two components, one ($H_S$) corresponding to the non-corrupted part, and
another  ($H_N$) to the corrupted part, we have
\begin{eqnarray}
H(X) = H_S(X) + H_N(X)
\end{eqnarray}
We will define in the following $H_S$ as the signal information, and $H_N$
as the noise information. It must be clear that noise does not
contain any information, and what we call noise information is a quantity
which is measured as information by the multiscale entropy, and which is
probably not informative to us.

If a wavelet coefficient is small, its value can be due to noise,
and the information $h$ relative to this single wavelet coefficient
should be assigned to $H_N$.
If the wavelet coefficient is high, compared to the noise standard
deviation, its value cannot be due to the noise, and $h$ should be assigned to $H_S$.
$h$ can be distributed as $H_N$ or $H_S$ based on  the probability $P_n(w_{j,k})$
that the wavelet coefficient is due to noise, or the probability
$P_s(w_{j,k})$ that it is due to
signal.  We have $P_s(w_{j,k}) = 1 - P_n(w_{j,k})$.
For the Gaussian noise case, we estimate $P_n(w_{j,k})$ that a wavelet
coefficient is due to the noise by
\begin{eqnarray*}
P_n(w_{j,k}) = \mathrm{Prob}(W > \mid w_{j,k} \mid)  & =  & \frac{2}{\sqrt{2 \pi}
\sigma_j} \int_{\mid w_{j,k} \mid}^{+\infty} \exp(-W^2/2\sigma^2_j) dW \nonumber \\
 & = & \mbox{erfc}(\frac{\mid w_{j,k} \mid }{\sqrt{2}\sigma_j})
\end{eqnarray*}
For each wavelet coefficient $w_{j,k}$, we have to estimate now the fractions
$h_n$ and $h_s$ of $h$ which should be assigned to $H_n$ and $H_s$.
Hence  signal information and  noise information are defined by
\begin{eqnarray}
H_s(X) & = & \sum_{j=1}^{l} \sum_{k=1}^{N_j} h_s(w_{j,k})   \nonumber  \\
H_n(X) & = & \sum_{j=1}^{l} \sum_{k=1}^{N_j} h_n(w_{j,k})
\label{eq_entrop_result_2}
\end{eqnarray}
Note that $H_s(X) + H_n(X)$ is always equal to $H(X)$.

\subsubsection{N1-MSE}
A first approach for deriving $h_s$ and $h_n$ from $P_s$ and $P_n$ is to
just consider $P_s$ and $P_n$ as weights on the information $h$. Then we
have:
\begin{eqnarray}
h_s( w_{j,k}) & = & P_s(w_{j,k})  h(w_{j,k})    \nonumber \\
h_n( w_{j,k}) & = & P_n(w_{j,k})  h(w_{j,k})
\label{eq_mse1}
\end{eqnarray}
and the noise and signal information in a signal are
\begin{eqnarray}
H_s(X) & = & \sum_{j=1}^{l} \sum_{k=1}^{N_j} h_s(w_{j,k})    \nonumber \\
H_n(X) & = & \sum_{j=1}^{l} \sum_{k=1}^{N_j} h_n(w_{j,k})
\label{eq_entrop_result_1}
\end{eqnarray}
which leads for a Gaussian noise to:
\begin{eqnarray}
H_s(X) &= & \sum_{j=1}^{l}  \sum_{k=1}^{N_j}  \frac{w_{j,k}^2}{2\sigma_j^2}
\mbox{erf}(\frac{\mid w_{j,k} \mid }{\sqrt{2}\sigma_j}) \nonumber \\
H_n(X) &= & \sum_{j=1}^{l}  \sum_{k=1}^{N_j}  \frac{w_{j,k}^2}{2\sigma_j^2}
\mbox{erfc}(\frac{\mid w_{j,k} \mid }{\sqrt{2}\sigma_j})
\label{eq_entrop_gauss_result_1}
\end{eqnarray}
We will refer to these functions by the name N1-MSE in the following.

\subsubsection{N2-MSE}

By the previous  entropy measure, information relative to high wavelet coefficients is
completely assigned to the signal. For a restoration, this allows us
also to exclude
wavelet coefficients with high signal-to-noise ratio (SNR)
from the regularization.
It leads to perfect fit of the solution with the data at scales and
 positions with high SNR. If we want to consider the information due
to noise, even for significant wavelet coefficients, the noise information
relative to a wavelet coefficient must be estimated differently.
 The idea for deriving $h_s$ and $h_n$ is the following: we imagine that  the
information $h$ relative to a wavelet coefficient
is a sum of small information components $dh$, each of them
having a probability of being noise information, or signal
information \cite{starck:sta98_2}.
For example,
for two coefficients  $u$ and  $w$ ($w > u$), with a Gaussian noise
($\sigma=1$), the information relative to $w$ is $h(w)=w^2$, and by
varying  $u$ from $0$  to $w$ with a step $du$, the information $h(u)$
increases until it is equal to $h(w)$.
 When $u$ becomes closer to $w$,
the difference $w - u$ can be due to the noise, and the added information
$dh(u) = h(u+du) - h(u)$ is contaminated by the noise. The idea is to weight
$dh(u)$ with the probability that $w - u$ is due to the noise.
Hence, $h_n$ and $h_n$ are calculated by:
\begin{eqnarray}
h_n(w_{j,k}) =  \int_{0}^{\mid w_{j,k} \mid } P_n(\mid w_{j,k} \mid - u) (\frac{\partial h(x)}{\partial x})_{x=u} du
\end{eqnarray}
is the noise information relative to a single wavelet coefficient, and
\begin{eqnarray}
h_s(w_{j,k}) =  \int_{0}^{\mid w_{j,k} \mid } P_s(\mid w_{j,k} \mid - u)
                                   (\frac{\partial h(x)}{\partial x})_{x=u} du
\end{eqnarray}
is the signal information relative to a single wavelet coefficient.
For Gaussian noise, we have
\begin{eqnarray}
h_n(w_{j,k}) & = &  \frac{1}{\sigma_j^2} \int_{0}^{\mid w_{j,k} \mid} u
         \mbox{ erfc}(\frac{\mid w_{j,k} \mid -u}{\sqrt{2} \sigma_j}) du \nonumber \\
h_s(w_{j,k}) & = &  \frac{1}{\sigma_j^2} \int_{0}^{\mid w_{j,k} \mid} u
         \mbox{ erf}(\frac{\mid w_{j,k} \mid -u}{\sqrt{2} \sigma_j})
\label{eqn_hn2}
\end{eqnarray}
and the noise and signal information in a signal are
\begin{eqnarray}
H_s(X) & = & \sum_{j=1}^{l} \sum_{k=1}^{N_j} h_s(w_{j,k})   \nonumber \\
H_n(X) & = & \sum_{j=1}^{l} \sum_{k=1}^{N_j} h_n(w_{j,k})
\label{eq_entrop_gauss_result_2}
\end{eqnarray}
We will refer to these functions by the name N2-MSE in the following.

Equations~\ref{eq_mse1} and  %\ref{eq_entrop_result_2}
\ref{eq_entrop_result_2}
lead to two
different ways to regularize a signal. The first requires that we use
all the information which is furnished in high wavelet coefficients, and
leads to an exact preservation of the flux in a structure. If the signal
presents high discontinuities, artifacts can appear in the solution
due to the fact that the wavelet coefficients located at the discontinuities
are not noisy, but have been modified like noise. The second equation
doesn't have this drawback, but a part of the flux of a structure
(compatible with noise amplitude) can be lost in the restoration process.
It is however not as effective as in the standard maximum entropy methods.

\subsubsection{LOG-MSE}
The multiscale entropy function used in \cite{starck:pan96} (we call it LOG-MSE
in the following) can be considered in our framework if $h$ is defined by:
\begin{eqnarray}
h(w_{j,k}) = {\sigma_j \over \sigma_X^2} [w_{j,k} - M_j -
       \mid w_{j,k} \mid \log( {\mid w_{j,k} \mid \over K_m \sigma_j})]
\label{eqn_logmse}
\end{eqnarray}
where $\sigma_X$ is the noise standard deviation in the data.
And $h_n$ is defined by:
\begin{eqnarray}
h_n(w_{j,k}) = A(p_n(w_{j,k})) h(w_{j,k})
\end{eqnarray}
where $A$ is a function which takes the values 0 or 1 depending on $p_n(w_{j,k})$:
\begin{eqnarray}
A(p_n(w_{j,k}))  = \left\{
  \begin{array}{ll}
  \mbox{ 1 } & \mbox{ if }  p_n(w_{j,k}) > \epsilon   \\
  \mbox{ 0 } & \mbox{ if }  p_n(w_{j,k}) \leq \epsilon
  \end{array}
  \right.
\label{eqn_mressupp}
\end{eqnarray}

Wavelet coefficients which are significant will impose $A(p_n(w_{j,k}))$ to
be equal to 0 (because    their probabilities of being due to noise is
very small), and do not contribute to $H_n$. This means that using $H_n$ in
a regularization process will have an effect only on scales and positions
where no significant wavelet coefficient is detected.

In practice we prefer N1-MSE and N2-MSE for several reasons. First the way
the model is used in equation~\ref{eqn_logmse} is a bit artificial, and
there is an undetermination when the wavelet coefficient is equal to 0.
Furthermore, LOG-MSE seems difficult to generalize to other classes of noise,
which is not the case for N1-MSE and N2-MSE. N2-MSE has the advantage of
estimating the corrupted part in the measured information $h$, even for large
wavelet coefficients.

\subsection{Conclusion}

In practice we prefer N1-MSE and N2-MSE for several reasons. First the way
the model is used in equation~\ref{eqn_logmse} is a bit artificial, and
there is an undetermination when the wavelet coefficient is equal to 0.
Furthermore, LOG-MSE seems difficult to generalize to other classes of noise,
which is not the case for N1-MSE and N2-MSE. N2-MSE has the advantage of
estimating the corrupted part in the measured information $h$, even for large
wavelet coefficients.
Concerning the five points, cited in section~\ref{sect_5pt}:
\begin{enumerate}
\item {\em The information in a flat signal is zero}: \\
 It is true for N1-MSE and N2-MSE. IN the case of LOG-MSE, it is true
if we take $\sigma=0$.
\item {\em The amount of information in a signal is independent
of the background}: \\
It is always true because the last scale of the wavelet transform is
never taken into account in the entropy calculation.
\item {\em The amount of information is dependent on the noise}: \\
Whatever the method, it is normalized coefficients which are used, so it
is always true.
\item {\em The entropy must work in the same way for a pixel which
has a value $B + \epsilon$, and
for a pixel which has a value $B - \epsilon$}: \\
For N1-MSE and N2-MSE, it is true because the entropy is calculated
from the absolute values or the square of the wavelet coefficients.
For LOG-MSE, this point is not verified because a $w_{j,k}$ appears.
But it has no effect on the solution, because it is the
derivated of the entropy which is used in the calculations,
and this term becomes constant.
\item {\em The amount of information is dependent on the correlation in the signal.
If the signal $S$  presents large features above the noise, it contains
a lot of information}: \\
As the entropy is calculated from the wavelet coefficients, this point
is always verified.
\end{enumerate}


\section{Unitary and hermitian matrices}

\begin{outcome}
  \begin{enumerate}
  \item Determine whether a complex linear transformation is an
    isometry and/or unitary.
  \item Determine whether a complex matrix is unitary.
  \item Determine whether a complex matrix is hermitian.
  \item Calculate the eigenvalues and eigenvectors of a hermitian
    matrix.
  \item Compute an orthogonal basis of eigenvectors for a hermitian
    matrix.
  \item Unitarily diagonalize a hermitian matrix.
  \end{enumerate}
\end{outcome}

In the context of real inner product spaces, we studied orthogonal
functions, orthogonal matrices, and symmetric matrices. The
corresponding concepts in the context of complex inner product spaces
are unitary functions, unitary matrices, and hermitian matrices. We
will introduce these concepts in this section. Since the proofs are
similar to those in Section~\ref{sec:orthogonal-matrices}, we omit
most of them.

\begin{definition}{Isometries and unitary maps of complex inner product spaces}{complex-isometries-unitary}
  Let $V,W$ be complex inner product spaces. A linear transformation
  $T:V\to W$ is called an \textbf{isometry}%
  \index{isometry}%
  \index{linear transformation!isometry} if for all
  $\vect{u},\vect{v}\in V$,
  \begin{equation*}
    \iprod{T(\vect{u}),T(\vect{v})} = \iprod{\vect{u},\vect{v}}.
  \end{equation*}
  An isometry that is also invertible is called a \textbf{unitary
    transformation}%
  \index{unitary transformation}%
  \index{linear transformation!unitary}, or simply \textbf{unitary}.
\end{definition}

In the case of real inner product spaces, we found in
Proposition~\ref{prop:matrix-orthogonal} that a square matrix $P$ is
the matrix of an orthogonal transformation (with respect to
orthonormal bases) if and only if $P^TP=I$. In the complex case, we
have an analogous property, except that we must use the adjoint
instead of the transpose.

\begin{proposition}{The matrix of a unitary transformation}{matrix-unitary}
  Let $T:V\to W$ be a linear transformation between finite-dimensional
  complex inner product spaces $V$ and $W$. Let $B$ and $C$ be
  orthonormal bases of $V$ and $W$, respectively, and let
  $P=\coord{T}_{C,B}$ be the matrix of $T$ with respect to the bases
  $B$ and $C$. Then
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item $T$ is an isometry if and only if $P^{\adjoint}P = I$.
    \item $T$ is unitary if and only if $P^{\adjoint}P = I$ and
      $\dim V=\dim W$.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{proposition}

This motivates the following definition.  We therefore define a
\textbf{unitary}%
\index{unitary matrix} matrix to be an $n\times n$-matrix satisfying
$P^{\adjoint}P=I$.

\begin{definition}{Unitary matrix}{unitary-matrix}
  An $n\times n$-matrix $P$ is called \textbf{unitary}%
  \index{unitary matrix}%
  \index{matrix!unitary} if $P^{\adjoint}P=I$.
\end{definition}

We have the following analogue of
Proposition~\ref{prop:conditions-orthogonal-matrix}:

\begin{proposition}{Conditions for unitary matrices}{conditions-unitary-matrix}
  The following are equivalent for a complex $n\times n$-matrix $P$:
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item $P$ is unitary.
    \item $P^{\adjoint}P=I$.
    \item $P$ is invertible and $P^{-1}=P^{\adjoint}$.
    \item $PP^{\adjoint}=I$.
    \item $P^{\adjoint}$ is unitary.
    \item The columns of $P$ form an orthonormal set of vectors.
    \item The rows of $P$ form an orthonormal set of vectors.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{proposition}

\begin{example}{Unitary matrices}{unitary-matrices}
  Determine which of the following matrices are unitary.
  \begin{equation*}
    A = \begin{mymatrix}{cc}
      1 & 0 \\
      0 & i \\
    \end{mymatrix},
    \quad
    B = \frac{1}{\sqrt{2}}
    \begin{mymatrix}{rr}
      1 & i \\
      i & 1 \\
    \end{mymatrix},
    \quad
    C = \frac{1}{\sqrt{2}}
    \begin{mymatrix}{rr}
      1 & i \\
      -i & 1 \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We have
  \begin{equation*}
    A^{\adjoint}A =
    \begin{mymatrix}{cc}
      1 & 0 \\
      0 & -i \\
    \end{mymatrix}\begin{mymatrix}{cc}
      1 & 0 \\
      0 & i \\
    \end{mymatrix}
    = \begin{mymatrix}{cc}
      1 & 0 \\
      0 & 1 \\
    \end{mymatrix}
    = I,
  \end{equation*}
  so $A$ is unitary. Similarly, we have
  \begin{equation*}
    B^{\adjoint}B =
    \frac{1}{2}
    \begin{mymatrix}{rr}
      1 & i \\
      i & 1 \\
    \end{mymatrix}\begin{mymatrix}{rr}
      1 & -i \\
      -i & 1 \\
    \end{mymatrix}
    = \frac{1}{2}
    \begin{mymatrix}{rr}
      2 & 0 \\
      0 & 2 \\
    \end{mymatrix}
    = I,
  \end{equation*}
  so $B$ is unitary as well. On the other hand,
  \begin{equation*}
    C^{\adjoint}C =
    \frac{1}{2}
    \begin{mymatrix}{rr}
      1 & i \\
      -i & 1 \\
    \end{mymatrix}\begin{mymatrix}{rr}
      1 & i \\
      -i & 1 \\
    \end{mymatrix}
    = \frac{1}{2}
    \begin{mymatrix}{rr}
      2 & 2i \\
      -2i & 2 \\
    \end{mymatrix}
    \neq I,
  \end{equation*}
  so $C$ is not unitary. Equivalently, we could have checked whether
  the columns of $A$, $B$, and $C$ form an orthonormal set of vectors
  (they do, in the case of $A$ and $B$, but don't, in the case of
  $C$. See also Example~\ref{exa:orthogonal-vectors-complex}).
\end{solution}

Of course, if $P$ happens to be a matrix with real entries, then $P$
is unitary if and only if it is orthogonal, because in that case
$P^{\adjoint}=\conjugate{P}^T = P^T$.

Recall that a matrix $A$ is called \textbf{symmetric} if $A=A^T$. In
the complex world, we are more often interested in the property
$A=A^{\adjoint}$. A matrix with this property is called
\textbf{hermitian} (after the French mathematician Charles Hermite,
1822--1901).

\begin{definition}{Hermitian matrix}{hermitian-matrix}
  A complex $n\times n$-matrix $A$ is called \textbf{hermitian}%
  \index{hermitian matrix}%
  \index{matrix!hermitian}
  if $A=A^{\adjoint}$.
\end{definition}

\begin{example}{Hermitian vs. symmetric matrices}{hermitian-matrix}
  Which of the following matrices are hermitian? Which ones are
  symmetric?
  \begin{equation*}
    A = \begin{mymatrix}{cc} 1 & 2 \\ 2 & 1 \end{mymatrix}, \quad
    B = \begin{mymatrix}{cc} 1 & 2i \\ 2i & 1 \end{mymatrix}, \quad
    C = \begin{mymatrix}{cc} 1 & 2i \\ -2i & 1 \end{mymatrix}, \quad
    D = \begin{mymatrix}{cc} i & 2 \\ 2 & i \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  The matrix $A$ is symmetric and also hermitian. The matrix $B$ is
  symmetric but not hermitian. In fact, we have
  \begin{equation*}
    B^{\adjoint} = \begin{mymatrix}{cc} 1 & -2i \\ -2i & 1 \end{mymatrix} \neq B.
  \end{equation*}
  The matrix $C$ is hermitian but not symmetric. The matrix $D$ is
  symmetric but not hermitian.
\end{solution}

A matrix $A=\mat{a_{ij}}$ is hermitian if and only if
$a_{ij} = \conjugate{a_{ji}}$, for all $i,j$. In particular, the
diagonal entries of a hermitian matrix are always real, and the
off-diagonal entries come in complex conjugate pairs. If all of the
entries in a matrix $A$ are real, then $A$ is hermitian if and only if
it is symmetric.

Hermitian matrices are of interest, among other things, because their
eigenvalues are always real. Moreover, eigenvectors for distinct
eigenvalues are orthogonal. The following proposition is analogous to
Proposition~\ref{prop:eigenvalues-symmetric}.

\begin{proposition}{Eigenvalues and eigenvectors of hermitian matrices}{eigenvalues-hermitian}
  Let $A$ be a hermitian matrix. Then
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item All eigenvalues of $A$ are real.
    \item Eigenvectors for distinct eigenvalues of $A$ are orthogonal.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{proposition}

\begin{proof}
  (a) Suppose $\eigenvar$ is an eigenvalue of $A$, with eigenvector
  $\vect{v}$. We will evaluate $\vect{v}^{\adjoint}A\vect{v}$ in two
  different ways:
  \begin{eqnarray*}
    \vect{v}^{\adjoint}A\vect{v}
    &=& \vect{v}^{\adjoint}(A\vect{v})
        = \vect{v}^{\adjoint}(\eigenvar\vect{v})
        = \eigenvar\vect{v}^{\adjoint}\vect{v}, \\
    \vect{v}^{\adjoint}A\vect{v}
    &=& (\vect{v}^{\adjoint}A)\vect{v}
        = (\vect{v}^{\adjoint}A^{\adjoint})\vect{v}
        = (A\vect{v})^{\adjoint}\vect{v}
        = (\eigenvar\vect{v})^{\adjoint}\vect{v}
        = \conjugate{\eigenvar}\vect{v}^{\adjoint}\vect{v}.
  \end{eqnarray*}
  Therefore,
  $\eigenvar\vect{v}^{\adjoint}\vect{v} =
  \conjugate{\eigenvar}\vect{v}^{\adjoint}\vect{v}$. Since
  $\vect{v}^{\adjoint}\vect{v}$ is non-zero, this implies
  $\eigenvar=\conjugate{\eigenvar}$, i.e., $\eigenvar$ is real.

  \noindent
  (b) Suppose $\vect{v}$ and $\vect{w}$ are eigenvectors for
  eigenvalues $\eigenvarA$ and $\eigenvarB$, respectively, and
  $\eigenvarA\neq\eigenvarB$. From part (a), we know that both
  $\eigenvarA$ and $\eigenvarB$ are real.  By evaluating
  $\vect{v}^{\adjoint}A\vect{w}$ in two different ways, we find that
  \begin{eqnarray*}
    \vect{v}^{\adjoint}A\vect{w}
    &=& \vect{v}^{\adjoint}(A\vect{w})
    = \vect{v}^{\adjoint}(\eigenvarB\vect{w})
    = \eigenvarB \vect{v}^{\adjoint}\vect{w}, \\
    \vect{v}^{\adjoint}A\vect{w}
    &=& (\vect{v}^{\adjoint}A)\vect{w}
    = (A^{\adjoint}\vect{v})^{\adjoint}\vect{w}
    = (A\vect{v})^{\adjoint}\vect{w}
    = (\eigenvarA\vect{v})^{\adjoint}\vect{w}
    = \eigenvarA\vect{v}^{\adjoint}\vect{w}.
  \end{eqnarray*}
  Therefore,
  $\eigenvarB\vect{v}^{\adjoint}\vect{w} = \eigenvarA\vect{v}^{\adjoint}\vect{w}$, or
  equivalently $(\eigenvarA - \eigenvarB)\vect{v}^{\adjoint}\vect{w} =
  0$. Since by assumption, $\eigenvarA - \eigenvarB\neq 0$, we must
  have $\vect{v}^{\adjoint}\vect{w} = 0$, i.e., $\vect{v}\orth\vect{w}$.
\end{proof}

We say that a matrix $A$ is \textbf{unitarily diagonalizable}%
\index{unitarily diagonalizable matrix}%
\index{diagonalizable matrix!unitarily diagonalizable}%
\index{diagonalization!unitary diagonalization}%
\index{matrix!unitarily diagonalizable}%
\index{matrix!diagonalizable!unitarily} if there exists a unitary
matrix $P$ and a diagonal matrix $D$ such that $D = P^{-1}AP$.  The
following is the main theorem about the diagonalization of hermitian
matrices. It is analogous to
Theorem~\ref{thm:diagonalization-symmetric} in the real case.

\begin{theorem}{Diagonalization of hermitian matrices}{diagonalization-hermitian}
  Every hermitian matrix $A$ is unitarily diagonalizable as
  $D = P^{-1}AP$. Moreover, the entries of $D$ are real.
\end{theorem}

\begin{example}{Diagonalization of hermitian matrices}{diagonalization-hermitian}
  The matrix
  \begin{equation*}
    A = \begin{mymatrix}{cc}  3 & -2i \\ 2i & 6 \end{mymatrix}
  \end{equation*}
  is hermitian. Unitarily diagonalize $A$, i.e., find a unitary matrix
  $P$ and a real diagonal matrix $D$ such that $D = P^{-1}AP$.
\end{example}

\begin{solution}
  The characteristic polynomial is
  \begin{equation*}
    \det(A-\eigenvar I)
    = \begin{absmatrix}{cc} 3-\eigenvar & -2i \\ 2i & 6-\eigenvar \end{absmatrix}
    = (3-\eigenvar)(6-\eigenvar) + 4i\,^2
    = \eigenvar^2 - 9\eigenvar + 14.
  \end{equation*}
  Its roots are $\eigenvar_1=2$ and $\eigenvar_2=7$. For the eigenvalue
  $\eigenvar_1=2$, we find the normalized eigenvector
  \begin{equation*}
    \vect{v}_1 = \frac{1}{\sqrt{5}}\begin{mymatrix}{c} 2i \\ 1 \end{mymatrix}.
  \end{equation*}
  For the eigenvalue
  $\eigenvar_2=7$, we find the normalized eigenvector
  \begin{equation*}
    \vect{v}_2 = \frac{1}{\sqrt{5}}\begin{mymatrix}{c} 1 \\ 2i \end{mymatrix}.
  \end{equation*}
  Note that these eigenvectors are orthogonal to each other,
  confirming Proposition~\ref{prop:eigenvalues-hermitian}. We
  therefore have $D=P^{-1}AP$, where
  \begin{equation*}
    D = \begin{mymatrix}{cc} 2 & 0 \\ 0 & 7 \end{mymatrix}
    \quad\mbox{and}\quad
    P = \frac{1}{\sqrt{5}} \begin{mymatrix}{cc} 2i & 1 \\ 1 & 2i \end{mymatrix}.
  \end{equation*}
  Note that $P$ is unitary and $D$ is real diagonal.
\end{solution}


\section{Complex Functions}

\subsection{Analytic Functions}

\subsubsection{Exercise 4}
Let $f(z) = u(z) + iv(z)$ with $\abs{f(z)} = c$. Then we have that
\begin{align*}
        &u^2(z) + v^2(z) = c^2 \\
        \implies &u(z) (\pdv{u}{x} - \pdv{u}{y}) + v(z) (\pdv{v}{x} - \pdv{v}{y}) = 0.
\end{align*}
If $u(z) = 0$ or $v(z) = 0$, then $f(z)$ is clearly a constant. Otherwise, both differences of partial
derivatives must be 0. Applying Cauchy-Riemann then gives that $\pdv{u}{x} = -\pdv{u}{x}$ (and similar for
$v(z)$), so $f(z)$ must also be constant in this case.

\subsubsection{Exercise 5}
Let $f(z) = u(z) + iv(z)$. Then $\overline{f(\bar{z})} = u(\bar{z}) - iv(\bar{z})$, which we can write as
$u(x, -y) - iv(x, -y)$. Differentiating with respect to $x$ and $y$, it is straightforward to check that
$\overline{f(\bar{z})}$ satisfies Cauchy-Riemann and is therefore analytic if $f(z)$ is analytic.

\subsubsection{Exercise 6}
Since $\pdv{u}{y} \pdv{y}{-y} = -\pdv{u}{y}$, we have that $u'(x, y) = u(x, -y)$ satisfies $\Delta u' = 0$ and
is thus harmonic as well.

\subsubsection{Exercise 7}
Can be shown by following the procedure outlined in the text; let $x = \frac{1}{2} (z + \bar{z})$ and
$y = \frac{-i}{2} (z - \bar{z})$ and differentiate.


\lab{Method of Mean Weighted Residuals}{Method of Mean Weighted Residuals}
\label{lab:pseudospectral1_revision}
\objective{We introduce the method of mean weighted residuals (MWR) and use it to derive a pseudospectral method. This method will then be used to solve several boundary value problems.}

Consider a linear differential equation
\[Lu = f, \]
defined on the interval $[-1,1]$, together with associated boundary conditions.
We will approximate the solution $u(x)$ by a linear combination of $N+1$ basis functions $\phi_i$, so that
\[
u(x) \approx u_N(x) = \sum_{i=0}^N a_i \phi_i(x).
\]
To determine appropriate constants $a_i$, we then minimize the residual function
\[
R(x,u_N) = Lu_N - f.
\]
Note that $R(x,u) = Lu - f = 0$ for the true solution $u(x)$.

This general strategy is often called the method of mean weighted residuals (MWR method). The MWR method is a general framework that describes many other, more specific methods. These more specific methods come from differing approaches to minimizing the residual $R(x,u_N)$, and the choice of basis functions $\phi_i$.

\section*{The Pseudospectral Method}
The pseudospectral or collocation method is obtained from the MWR method by forcing the residual function $R(x,u_N)$ to equal zero at $N+1$ points in $[-1,1]$, called collocation points.
When done correctly, the pseudospectral method gives high accuracy and converges rapidly.

We will let the basis functions $\phi_i$ be the Chebyshev polynomials, and the collocation points will be the Gauss-Lobatto points, $x_i = \cos(\pi i /N)$, $ i = 0, \ldots, N$.
The appropriate solution $u_N$ may be represented with two equivalent forms.
First, $u_N$ can be described with the first $N+1$ coefficients  $\{a_i\}_{i=0}^N$ of its expansion in the Chebyshev polynomials.
Since $u_N$ is a polynomial of order $N$, it may be uniquely described by its values at the collocation points, that is, the unknown values  $\{u_N(x_i)\}_{i=0}^N$.

These equivalent forms satisfy
\begin{align}
	MA = F \label{spectral1b:chebychev_expansion}
\end{align}
and
\begin{align}
	LU &= F \label{spectral1b:grid_point}
\end{align}
where
\begin{align*}
	U_i &= u(x_i),\\
	A_i &= a_i,\\
	F_i &= f(x_i),\\
	L_{ij} &= \left.(LC_j(x))\right|_{x=x_i},\\
	M_{ij} &= \left.(L\phi_j(x))\right|_{x=x_i}.
\end{align*}

The functions $C_j$ above are the cardinal functions, defined to be the polynomials of least degree satisfying
\begin{equation*}
C_j(x_i) = \begin{cases} 1 & i=j \\ 0 & i \not = j.
   \end{cases}
\end{equation*}
Thus, $u_N$ can also be expanded in the basis of the cardinal functions:
\begin{align*}
	u_N(x) &= \sum_{j=0}^N u_N(x_j)C_j(x).
\end{align*}

When $L = d/dx$, the matrix corresponding to equation \eqref{spectral1b:grid_point} is given by
\begin{align*}
L_{ij} &= \frac{dC_j}{dx}(x_i) =
\begin{cases} (1+2N^2)/6 & i=j=0, \\ -(1+2N^2)/6 & i=j=N, \\
-x_j/[2(1-x_j^2)] & i=j, \, 0<j<N, \\
(-1)^{i+j}\alpha_i/[\alpha_j(x_i-x_j)] & i \not = j.
   \end{cases}
\end{align*}
where $\alpha_0 = \alpha_N = 2,$ and $\alpha_j = 1$ otherwise.

This matrix is often called the differentiation matrix ($D$), and can be used to piece together the matrix $L$ for more complicated differential operators.
A stable, vectorized function to build the differentiation matrix is given below.

\begin{lstlisting}
import numpy as np

def cheb(N):
	x =  np.cos((np.pi/N)*np.linspace(0,N,N+1))
	x.shape = (N+1,1)
	lin = np.linspace(0,N,N+1)
	lin.shape = (N+1,1)

	c = np.ones((N+1,1))
	c[0], c[-1] = 2., 2.
	c = c*(-1.)**lin
	X = x*np.ones(N+1) # broadcast along 2nd dimension (columns)

	dX = X - X.T

	D = (c*(1./c).T)/(dX + np.eye(N+1))
	D  = D - np.diag(np.sum(D.T,axis=0))
	x.shape = (N+1,)
	# Here we return the differentiation matrix and the Chebyshev points,
	# numbered from x_0 = 1 to x_N = -1
	return D, x

\end{lstlisting}

\section*{Using the Differentiation Matrix}

\begin{problem}
Use the differentiation matrix to numerically approximate the derivative of $u(x) = e^{x}\cos(6x)$ on a grid of $N$ Chebychev points where $N=6, 8,$ and $10.$
(Use the linear system $D U \approx U'$.)
Then use barycentric interpolation to approximate $u'$ on a grid of 100 evenly spaced points.

Graphically compare your approximation to the exact derivative.
Note that this convergence would not be occurring if the collocation points were equally spaced.
\end{problem}

To approximate $u''(x)$ on the grid $\{x_i\}$, we use
\[U'' \approx D^2 U.\]
The BVP
\begin{align*}
&{ }u'' = f(x), \quad x \in [-1,1],\\
&{ }u(-1) = 0, \quad u(1) = 0,
\end{align*}
can be discretized by the linear system
\begin{align}
	D^2 U &= F, \label{spectral1b:discretization}
\end{align}
where $F = [f(x_0),\ldots, f(x_N)]^T$.
Since we have Dirichlet boundary conditions of $0$, we can satisfy the boundary condition by forcing $U[0] = U[N] = 0$.
This is done by replacing the first and last equations in \eqref{spectral1b:discretization} by the boundary conditions.
% This allows us to ignore the first and last equations in the system, giving us the new system
% \[\tilde{D}^2 \tilde{U} = \tilde{F},\]
% where $\tilde{D} = D[1:N,1:N]$, $\tilde{U} = U[1:N]$, and $\tilde{F} = F[1:N]$.

\begin{problem}
Use the pseudospectral method to solve the boundary value problem
\begin{align*}
&{ } u'' = e^{2x}, \quad x \in [-1,1], \\
&{ } u(-1) = 0, \quad u(1) = 0.
\end{align*}
Compare your numerical solution with the exact solution,
\[
u(x) = \frac{- \cosh(2) - \sinh(2)x + e^{2x}}{4}.
\]
\end{problem}

% \begin{figure}
% \centering
% \includegraphics[width=\textwidth]{nonzeroDirichlet.pdf}
% \caption{The solution of $u'' + u' = e^{3x}$, subject to the boundary conditions
% $u(-1) = 2$, $u(1) = -1$.}
% \label{fig:nonzeroDirichlet}
% \end{figure}

\begin{problem}
Use the pseudospectral method to solve the boundary value problem
\begin{align*}
&{ } u'' + u' = e^{3x}, \quad x \in [-1,1], \\
&{ } u(-1) = 2, \quad u(1) = -1.
\end{align*}
% Check that your numerical solution is converging.
% How many subintervals are required to find the solution correct to three decimal places?
% See Figure \ref{fig:nonzeroDirichlet}.

% Hint: Reduce the problem to one with zero Dirichlet conditions by letting $u = U+G$, where $G$ is a (simple) function satisfying the boundary conditions.
\end{problem}

\section*{Minimizing the Area of a Surface of Revolution}
A surface of revolution that minimizes its area is an example of a larger class of surfaces called minimal surfaces. A famous example of a minimal surface is a soap bubble. Soap bubbles minimize their surface area while containing a fixed volume of air. This behavior extends to merged bubbles, and a soap film whose boundary is a wire frame. Minimal surfaces have applications in molecular engineering and material science, and general relativity, where they describe the apparent horizon of a black hole.

Consider a function $y(x)$ defined on $[-1,1]$ satisfying $y(-1) = a $, $y(1) = b. $ The area of the surface obtained by revolving the graph of $y(x)$ about the $x$-axis is given by
\[T[y(x)] = \int_{-1}^1 2 \pi y(x) \sqrt{1 + (y'(x))^2}\, dx .\]
To find the function $y(x)$ whose surface of revolution minimizes surface area, we must minimize the functional $T[y]$.
This is a classical problem from a branch of mathematics called the calculus of variations.
Standard derivatives allow us to find the minimum values of functions defined on $\mathbb{R}^n$, and where they occur.
The calculus of variations allows us to find the minimum values of functions whose input are other functions.

From the calculus of variations we know that a necessary condition for $y(x)$ to minimize $T[y]$ is that the Euler-Lagrange equation must be satisfied:
\begin{align*}
	L_y - \frac{d}{dx}L_{y'} = 0,
\end{align*}
where $L(x,y,y') = 2 \pi y \sqrt{1 + (y')^2}$.
Simplifying the Euler-Lagrange equation for our problem results in the ODE
\[y y'' - (y')^2 -1 = 0.\]
Discretizing this ODE using the pseudospectral method results in the (nonlinear) system of equations
\[
Y \cdot (D^2 Y) - (DY) \cdot (DY) = I,
\]
where $I$ is a vector of ones.

\begin{problem}
Find the function $y(x)$ that satisfies $y(-1) = 1$, $y(1) = 7$, and whose surface of revolution (about the $x$-axis) minimizes surface area.
Compute the surface area, and plot the surface. \label{prob:pseudospectral1_revision:minimal_surface}
\end{problem}

\begin{figure}
\centering
\includegraphics[width=\textwidth]{minimal_surface.pdf}
\caption{The minimal surface corresponding to Problem  \ref{prob:pseudospectral1_revision:minimal_surface}.}
\label{fig:pseudospectral1_revision:minimal_surface}
\end{figure}

\begin{comment}
\section*{The Method of Weighted Residuals}
We may write our differential/integral equation in operator notation as
\[(Lu)(x) = f(x), \quad x \in \Omega\]
where $u$ belongs to some infinite-dimensional function space $V.$
We seek for an approximation $u_N$ to the solution $u$.
Our approximation $u_N$ will come from a finite-dimensional function space $S_N$ called the trial space.
Here we use $N$ to denote the dimension of $S_N \subset V$.
The space $S_N$ can be described by a basis $\{\phi_1(x), \ldots, \phi_N(x)\}$.
Then an approximation $u_N \in S_N$ has the form
\[u_N = \sum_{j=1}^N \gamma_j \phi_j\]

We can then define an operator $\mathcal{R}$ on the trial space $S_N$ by
\[\mathcal{R}u_N(x) = Lu_N(x) - f(x)\]
$\mathcal{R}u_N$ is called the residual or the error of the trial function $u_N$.
Note that the residual of the true solution $u$ is zero.
The method of weighted residuals is a family of methods that determine the coefficients $\gamma_j$ of the approximate solution $u_N$ by forcing the residual $\mathcal{R}u_N$ to be zero in some weighted average over $\Omega$.
In other words, given some collection of weight/test functions $\{w_i\}_{i=1}^M$, we require that
\begin{align}
\int_{\Omega}\mathcal{R}u_N(x) w_i(x)\, dx &= 0, \quad \text{ for } i = 1, \ldots M
\label{eqn:Spectral1_weightedaverage}
\end{align}

After doing the integration described by (\ref{eqn:Spectral1_weightedaverage}), we obtain a system of algebraic equations that may be used to determine the coefficients $\gamma_i$.
Different choices of the trial space $S_N$ and the weight/test functions $w_i(x)$ result in different methods.

We obtain the pseudospectral method (or collocation method) by choosing a collection of points $\{x_i\}_{i=1}^M$ in our space $\Omega$ called collocation points.
The weight functions $w_i(x)$ are then given by $w_i(x) = \delta(x-x_i); $ that is, $w_i$ is the Dirac delta function centered at $x_i$.
Then the equations \ref{eqn:Spectral1_weightedaverage} are given by
\begin{align*}
	\int_{\Omega} \mathcal{R}u_N w_i(x) \, dx &= 0,\\
	\int_{\Omega} \mathcal{R}u_N \delta(x-x_i) \, dx &= 0, \\
	\mathcal{R}u_N(x_i) &= 0 \quad \text{ for } i = 1, \ldots, M
\end{align*}
Thus the pseudospectral method requires the residual of the approximate solution to be exactly zero at the collocation points.
Alternatively, the approximate solution satisfies the differential equation exactly at the collocation points.
\end{comment}


\subsection{Rings}

Consider an abelian group \((S,+)\).

A ring takes this and adds a multiplicative function which satisfies the distributive property.

Groups have an identity element for their function. Rings must have identity elements for both their functions.

The multiplicative function does not have to be a bijection. For example the set of integers, addition and multiplication form a ring.