\section{Categories, functors, natural transformations}
%Office hours. Hood Chatham's are Mondays, 1:30 - 3:30, in 2-390A. Miller's is Tuesdays, 3:00-5:00 in 2-478. (commented out because not relevant to readers)
%Replaced \mathbf{C} with \cC as brought up in ``Stuff to fix.''
%Rearranged some stuff. Added a notational remark following the definition of a category.
From spaces and continuous maps, we constructed graded abelian groups and homomorphisms. We now cast this construction in the more general language of category theory.

Our discussion of category theory will be interspersed throughout the text, introducing new concepts as they are needed. Here we begin by introducing the basic definitions.

\begin{definition}
A \emph{category} $\cc$ is a class $\mathrm{ob}(\cc)$ of objects, such that for every two objects $X$ and $Y$ there is a set of \emph{morphisms} $\cc(X,Y)$ (thought of as a set of maps from $X$ to $Y$). We require that for all $X\in\mathrm{ob}(\cc)$, there exists an identity element $1_X\in\cc(X,X)$, and for all $X,Y,Z\in\mathrm{ob}(\cc)$, there is a composition $\cc(X,Y)\times\cc(Y,Z)\to\cc(X,Z)$ sending $(f,g)\mapsto g \circ f$. These in turn satisfy the following:
\begin{itemize}
\item $1_Y\circ f=f$, and $f\circ 1_X=f$.
\item Composition is associative.
\end{itemize}
\end{definition}
Note that, for set-theoretic reasons, we require the collection of objects to be a class. This enables us to talk about a ``category of all sets'' for example, but not a ``category of all categories'' because that is too large.

We will often write $X\in\cc$ to mean $X\in\mathrm{ob}(\cc)$, and $f\colon X\to Y$ to mean $f\in \cc(X,Y)$.
\begin{definition}
If $X,Y\in \cc$, then $f\colon X\to Y$ is an \emph{isomorphism} if there exists $g\colon Y\to X$ with $f \circ g=1_Y$ and $g\circ f=1_X$, and we write $X\cong Y$.
\end{definition}

\begin{example}
Many common mathematical structures can be arranged in categories.
\begin{itemize}
\item Sets and functions between them form a category $\set$.
\item Abelian groups and homomorphisms form a category $\mathbf{Ab}$.
\item Topological spaces and continuous maps form a category $\mathbf{Top}$.
\item Simplicial sets and their maps form a category $s\set$.
\item A monoid is the same as a category with one object, where the elements of the monoid are the morphisms in the category.
\item The sets $[n]=\{0,\ldots,n\}$ for $n\geq 0$ together with weakly order-preserving maps between them form the simplex category $\Delta$.
\item A poset forms a category in which there is a morphism from $x$ to $y$ iff $x\leq y$. However, note that $x\leq y$ and $y\leq x$ imply $x\cong y$ rather than $x=y$. The latter holds if the only isomorphisms are identities.
\end{itemize}
\end{example}
A small category is one such that $\mathrm{ob}(\cc)$ is a set, not necessarily a class. While we cannot consider the category of all categories, it is sensible to define the category $\mathbf{Cat}$ of all small categories. But first, we must first define a ``morphism of categories.''
\begin{definition}
Let $\cc,\cd$ be categories. A \emph{functor} $F\colon\cc\to\cd$ is a function $\mathrm{ob}(\cc)\to\mathrm{ob}(\cd)$, such that for all $x,y\in\mathrm{ob}(\cc)$, there is a map $\cc(x,y)\to\cc(F(x),F(y))$ that respects composition and the identity.
\end{definition}
The diagram at the beginning of the previous section shows the functors we have built thus far (although explicit verification of functoriality is left to the reader).

We go a step further. Suppose we fix categories $\cc$ and $\cd$ and consider all functors $\cc\to\cd$. What is a ``morphism of functors''?
\begin{definition}
Let $F,G\colon \cc\to\cd$. A \emph{natural transformation} $\theta\colon F\to G$ consists of maps $\theta(X)\colon F(X)\to G(X)$ for all $X\in\mathrm{ob}(\cc)$ such that the following diagram commutes for all $f\colon X\to Y$:
\begin{equation*}
\xymatrix{F(X)\ar[d]^{F(f)}\ar[r]^{\theta(X)} & G(X)\ar[d]^{G(f)}\\
F(Y)\ar[r]^{\theta(Y)} & G(Y)}
\end{equation*}
\end{definition}
\begin{definition}
If $\cc,\cd$ are categories, $\mathbf{Fun}(\cc,\cd)$ is the category whose objects are functors $\cc\to\cd$ and whose morphisms are natural transformations.
\end{definition}
Going further down the rabbit hole leads to higher category theory, which we will not delve into.

Natural transformations are central to algebraic topology. We will frequently describe certain maps as ``natural,'' which is to say that they are natural transformations. The reader should determine the functors involved if they are not explicitly stated.
\begin{example}
The boundary map $\partial\colon S_n\to S_{n-1}$ is a natural transformation.

Let $G$ be a group viewed as a one-point category. Any element $F\in\mathrm{Fun}(G,\mathbf{Ab})$ is simply a group action of $G$ on $F(\ast)=A$, i.e., a representation of $G$ in abelian groups. Given another $F^\prime\in\mathrm{Fun}(G,\mathbf{Ab})$ with $F^\prime(\ast)=A^\prime$, then a natural transformation from $F\to F^\prime$ is precisely a $G$-equivariant map $A\to A^\prime$.
\end{example}


\section{Calculation of structure factor and Ewald energy: first version}

Structure factor

\begin{juliacode}
# Calculate structure factor
# special case: only for 1 species with Z = 1
function structure_factor( Xpos::Array{Float64,2}, G::Array{Float64,2} )
  Ng = size(G)[2]
  Na = size(Xpos)[2]
  Sf = zeros(Complex128,Ng)
  for ia = 1:Na
    for ig = 1:Ng
      GX = Xpos[1,ia]*G[1,ig] +
           Xpos[2,ia]*G[2,ig] +
           Xpos[3,ia]*G[3,ig]
      Sf[ig] = Sf[ig] + cos(GX) - im*sin(GX)
    end
  end
  return Sf
end
\end{juliacode}

A simple method to calculate Ewald energy

\begin{juliacode}
function calc_ewald( pw::PWGrid, Xpos, Sf; sigma=0.25 )
  #
  const Npoints = pw.Npoints
  const Ω  = pw.Ω
  const r  = pw.r
  const Ns = pw.Ns
  const G2 = pw.G2
  #
  # Generate array of distances
  center = sum(pw.LatVecs,2)/2
  dr = gen_dr( r, center )
  #
  # Generate charge density
  rho = gen_rho( Ns, dr, sigma, Sf )
  intrho = sum(rho)*Ω/Npoints
  #
  # Solve Poisson equation and calculate Hartree energy
  ctmp = 4.0*pi*R_to_G( Ns, rho )
  ctmp[1] = 0.0
  for ip = 2:Npoints
    ctmp[ip] = ctmp[ip] / G2[ip]
  end
  phi = real( G_to_R( Ns, ctmp ) )
  Ehartree = 0.5*dot( phi, rho ) * Ω/Npoints
  #
  Eself = 1.0/(2*sqrt(pi))*(1.0/sigma)*size(Xpos,2)
  return Ehartree - Eself
end
\end{juliacode}


\newpage
\section{Boltzmann Machines and RBMs}

\subsection{Before BMs\cite{rojas1996hopfield, hopfield, coursera_nn}}

\subsubsection{General RNNs}
In early 1980-x general \textbf{recurrent neural networks} (RNNs), described by (general) \emph{asyclic} graph began to arise.
\begin{itemize}
\item     mostly binary units, $\{0, 1\}$ (or $\{-1, 1\}$);
\gooditem signal feedback $\Rightarrow$ \textbf{memory} compared to FF NNs (more specifically, \emph{content-addressable memory} (CAM) $\Leftrightarrow$ weights themselves are used to store patterns);
\baditem  in general if activations are non-linear function they are hard to train (oscillations and chaotic behavior).
\end{itemize}

\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.4]{img/general_rnn.png}
\centering
\caption{Example architecture of RNN}
\label{fig:general_rnn}
\end{mdframed}
\end{figure}

\vspace{1em}
\subsubsection{Hopfield Nets}
Later in $\approx$1982 John Hopfield has shown that if RNN has symmetric weights ($W_{ij}=W_{ji}$), and has no self-loops ($W_{ii}=0$), network's dynamics \textbf{is guaranteed} to converge to some stationary state. Fully-connected variant (complete graph) of such a network is called a \emph{Hopfield Network}.

Moreover, he also shown that each state of the Hopfield Net is associated with a scalar value referred to as the \emph{energy} of the network ($s_{i}\in\{-1, 1\}$ -- state of i-th neuron, $b_{i}$ -- bias of i-th neuron ($-b_{i}$ is activation threshold for the unit)):
\begin{align}
E = -\frac{1}{2}\sum_{i,j}W_{ij}s_is_j-\sum_{i}b_is_i=\commenttwo{$W_{ij}=W_{ji}$,}{$W_{ii}=0$} = -\sum_{i<j}W_{ij}s_is_j-\sum_{i}b_is_i
\end{align}
and local minima of $E(\mathbf{s};\mathbf{W}, \mathbf{b})\;\Leftrightarrow\;$ stable configurations of network. It is the first example of so-called \textbf{energy-based model}. Learning algorithm for such a model alter its (global) energy function to achieve desired properties. For instance, if a Hopfield Net is trained as autoassociator, the goal of learning is to shape energy function in such a way, that its local minima correspond to training examples (= patterns to "remember").
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.2]{img/energy_landscape.png}
\centering
\caption{Energy Landscape of a Hopfield Network, highlighting the current state of the network (up the hill), an attractor state to which it will eventually converge, a minimum energy level and a basin of attraction shaded in green. Note how the update of the Hopfield Network is always going down in Energy.}
\label{fig:energy_landscape}
\end{mdframed}
\end{figure}

\vspace{1em}
\subsubsection{Learning algorithms for Hopfield Nets}
There are exist couple of learning algorithms for Hopfield Nets.
First, thanks to simple quadratic shape of $E$, it is straightforward to calculate how i-th neuron influences the global energy:
\begin{gather*}
\Delta E_i = E(\mathbf{s}|s_i=\text{off}) - E(\mathbf{s}|s_i=\text{on}) = E(\mathbf{s}|s_i=0) - E(\mathbf{s}|s_i=1) =\\= -\sum_{k<j}W_{kj}s_ks_j\evalat{s_i=0} -\sum_{j}b_js_j\evalat{s_i=0}  + \sum_{k<j}W_{kj}s_ks_j\evalat{s_i=1} + \sum_{j}b_js_j\evalat{s_i=1} =\\=
\underbrace{-\l( \sum_{k<j, k \neq i, j \neq i}W_{kj}s_ks_j + \sum_{j \neq i}b_js_j \r) + \l( \sum_{k<j, k \neq i, j \neq i}W_{kj}s_ks_j + \sum_{j \neq i}b_js_j \r)}_{=0} +
\underbrace{\sum_{j<i}W_{ji}s_j}_{k=i} + \underbrace{\sum_{i<k}W_{ik}s_k}_{j=i} + b_i =\\= \comment{$W_{ij}=W_{ji}$, $W_{jj}=0$} = \sum_jW_{ij}s_is_j + b_i
\end{gather*}
This lead to the learning algorithm called \textbf{Binary Threshold Decision Rule} (BTDR):
\begin{itemize}
\item Randomly initialize states (or to desired pattern if want CAM);
\item While not converged or for fixed number of iterations:
	\subitem For each neuron:
		\subsubitem Change its state if this will decrease global energy. More specifically:
		 $$
		 s_i \leftarrow
		 \begin{cases}
		 +1,  \;\;\Delta E_i > 0,\\
		 s_i, \;\;\Delta E_i = 0,\\
		 -1,  \;\;\Delta E_i < 0.\\
		 \end{cases}
		 $$
\end{itemize}
This learning algorithm is \emph{local} and \emph{incremental}, thus biologically plausible. Also neurons could have been updated simultaneously, but it is less likely that there is exists "global clock" in biological system, and such kind of updates can also cause oscillation or chaotic behavior.

One important \underline{drawback} of such an algorithm, is that once we stuck in poor local minimum, we cannot escape it, it is also one of the reasons why Boltzmann Machines was more successful.

\vspace{2em}
Another learning algorithm is based on famous \textbf{Hebb rule}: "Cells that fire together, wire together". In the simplest case it simply says:
\begin{gather}
W_{ij} \leftarrow x_ix_j,
\end{gather}
where $\mathbf{x}=\{x_1 \ldots x_N\}$ is binary input pattern. If $x_i$ and $x_j$ are the same, then $W_{ij}$ is positive thus i-th and j-th state tend to become equal. Opposite happens when $x_i$ and $x_j$ are different. Now one can show that network's dynamics will be the same if
\begin{gather}
W_{ij} \leftarrow \frac{1}{N}x_ix_j
\end{gather}
If we need to remember not 1 but $P$ patterns, sum corresponding update for each pattern (for each pattern pretend there are no others):
\begin{gather}
W_{ij} \leftarrow \frac{1}{N}\sum_{p=1}^Px_i^{(p)}x_j^{(p)}
\end{gather}
This way network will act as CAM, see Fig. \ref{fig:hopfield}. The network will converge to a "remembered" state if it is given only part of the state $\Rightarrow$ can be used to recover from a distorted input to the trained state that is most similar to the input.
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.4]{img/hopfield.png}
\centering
\caption{A Hopfield network as an autoassociator. One enters a pattern in
blue nodes and let the network evolve. After a while one reads out the yellow
nodes. The memory of the Hopfield network associates the yellow node pattern
with the blue node pattern.}
\label{fig:hopfield}
\end{mdframed}
\end{figure}

\vspace{1em}
For instance, train Hopfield net such that $(1, -1, 1, -1, 1)$ is a local minimum of $E$ (phase 1 -- memorization). Next, if the network is properly trained, it can recover $(1, -1, 1, -1, 1)$ from $(1, -1, \mathbf{-1}, -1, 1)$ input (phase 2 -- recognition). Hopfield nets can be used for denoising of simple fonts.

\vspace{1em}
Many generalizations of this rule exist, but fundamentally Hopfield nets had quite some drawbacks (even for binary data):
\begin{itemize}
\baditem patterns are stored in the network itself, thus can be intractable for large $N$;
\baditem capacity, one can reliably store only $P << N$ patterns;
\baditem spurious minima, poor local minima;
\baditem no probabilistic interpretation.
\end{itemize}
But anyway Hopfield Nets and energy-based models played a big role in development of deep learning and now we proceed to the one of the most famous modification of Hopfield Net -- Boltzmann Machine.

\subsection{Boltzmann Machines\cite{coursera_nn, aarts1988simulated, fischer2012introduction, tutorial2014lisa}}
\subsubsection{Main ideas}
Boltzmann Machine (developed $\approx$1980-85 by G. Hinton) is a \emph{stochastic}, \emph{generative} counter-part of Hopfield Nets. From PGM point of view, it is an example of MRF (undirected graphical model). It is also an example of Ising model.
\\[1em]
Two main ideas:
\begin{enumerate}
\item Instead of storing "memories" in stable configurations of Hopfield net, use them for "interpretation" of the input data. Thus, all units are divided into 2 groups: \emph{visible} and \emph{hidden} (see Fig. \ref{fig:bm}).
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.4]{img/bm.png}
\centering
\caption{A graphical representation of an example Boltzmann machine. Each undirected edge represents dependency. In this example there are 3 hidden units and 4 visible units. This is not a restricted Boltzmann machine.}
\label{fig:bm}
\end{mdframed}
\end{figure}

\item To escape local minima, assume units are \emph{Gibbs distibuted} with the same energy function (1) and use more advanced learning algorithms (see below).
\end{enumerate}

\subsubsection{Historical development and informal explanation}
Inspired from statistical physics, assume that units' states are Gibbs distributed (= Boltzmann distributed) random variables for global energy (1) $\;\Rightarrow\;$ energy of the state is proportional to the negative log-probability of that state and also the following identity holds:
\begin{gather}
\Delta E_i = E(\mathbf{s}|s_i=0) - E(\mathbf{s}|s_i=1) \;\propto\; -k_BT\log p\{s_i=0\}+k_BT\log p\{s_i=1\},
\end{gather}
By absorbing Boltzmann's constant $k_B$ into introduced notion of (artificial) temperature $T$ ($T\leftarrow k_BT$) ($\Leftrightarrow$ by considering \emph{dimensionless} units where $k_B=1$) , we can rearrange equation (5) and solve for $ p\{s_i=1\}$. We will obtain so-called \emph{normal logistic equation}:
\begin{gather}
 p\{s_i=1\}=\frac{1}{1+\exp\l(-\frac{\Delta E_i}{T}\r)}=\text{sigm}\l(-\frac{\Delta E_i}{T}\r)
\end{gather}
We obtained a \textbf{Modified BTDR}:
\begin{itemize}
\item Randomly initialize states;
\item In cycle for each unit if its state reversal will yield energy decrease, reverse its state with probability described by (6).
\end{itemize}
After running for long enough time at certain $T$, it turns out that probability of a global state of network will \emph{depend only upon global state's energy}, and not on the initial state (in accordance with Boltzmann distribution).
\\$\Rightarrow$ Distribution of all possible configurations (global states) converges to same \emph{stationary distribution} (this state in physics is called thermal equilibrium). Note that being in thermal equilibrium, system does not necessary is in the state of the lowest energy, which still might be oscillating, \textbf{but} if we start running the network from a high temperature, and gradually decrease it over time until we reach a thermal equilibrium at low $T$, we may converge to a distribution where the energy level fluctuates around the global minimum (at least this can happen with higher probability than in Hopfield Nets).
\\This process is called \textbf{simulated annealing}. Yes, it seems like development of BMs gave fundament to one of the most famous and successful algorithms of global optimization.
\\[1em]
Now if we want to train the network so that the chance it will converge to a global state is according to an external distribution (e.g. data) that we have over these states, we need to \emph{set the weights} so that the global states with the highest probabilities will get the lowest energies. By altering parameters we can shape the distribution produced by BM using (6), it is the main idea of Boltzmann machine.
\\[1em]
BM is interpreted in the following way. Visible units provide open interface to the world and represent data. Hidden units represent hidden patterns in the data. Usually it is trained by using maximum likelihood estimation using approximate (why see below) gradient ascent. It is also equivalent \cite{goodfellow2016deep} to minimizing KL-divergence between external distribution (which is usually empirical data distribution) and model distribution produced by the network.

\subsubsection{More formal derivations}
\u{Notations}:
\\$\mathbf{v}\in\{0,1\}^D=\{0,1\}^V$ -- vector of visible units,
\\$\mathbf{h}\in\{0,1\}^H$ -- vector of hidden units,

Energy of a particular state $(\mathbf{v},\mathbf{h})$ is:
\begin{gather}
E(\mathbf{v},\mathbf{h};\bs{\psi})=-\frac{1}{2}\mb{v}^T\mb{L}\mb{v}-\frac{1}{2}\mb{h}^T\mb{J}\mb{h}-\mb{v}^T\mb{W}\mb{h}-\mb{b}^T\mb{v}-\mb{c}^T\mb{h},
\end{gather}
it is nothing else but (1) with old $\mb{W}$ split into $\mb{L}$ describing vis-vis weights, $\mb{J}$ describing hid-hid weights, and new $\mb{W}$ describing vis-hid connections, and the same for biases. The reason is to keep notations consistent with later parts where we will get rid of $\mb{L}$ and $\mb{J}$. Of course, $\mb{L}$ and $\mb{J}$ are symmetric with zeros on the main diagonal. $\bs{\psi}=\{\mb{L},\mb{J},\mb{W},\mb{b},\mb{c}\}$ are model parameters.

Probability of the configuration $(\mb{v},\mb{h})$ is (according to the Boltzmann distribution):
\begin{gather}
p(\mathbf{v},\mathbf{h};\bs{\psi})=\frac{e^{-E(\mathbf{v},\mathbf{h};\bs{\psi})}}{\sum_{\mb{\t{v}},\mb{\t{h}}} e^{-E(\mathbf{\t{v}},\mathbf{\t{h}};\bs{\psi})} }=:\frac{p^*(\mb{v},\mb{h}; \bs{\psi})}{Z(\bs{\psi})},
\end{gather}
where $p^*(\mb{v},\mb{h}; \bs{\psi})$ is unnormalized probability of the configuration and $Z(\bs{\psi})$ is a normalizer also called \emph{partition function}.
\\$\Rightarrow$
\\Probability that model assigns to a visible vector $\mb{v}$ is
\begin{gather}
p(\mb{v};\bs{\psi})=\sum_{\mb{h}}p(\mathbf{v},\mathbf{h};\bs{\psi})=:\frac{e^{-\mathcal{F}(\mb{v};\bs{\psi})}}{\sum_{\mb{\t{v}}} e^{-\mathcal{F}(\mb{\t{v}};\bs{\psi})} },
\end{gather}
where
\begin{gather}
\mc{F}(\mb{v};\bs{\psi})=-\log\l( \sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})} \r)
\end{gather}
is called \emph{free energy} (\emph{free} because all the hidden states are marginalized out; the name is inspired again from physics), useful quantity which will be used later.
\\[1em]
\u{Now lets calculate} $p(v_i=1|\mb{h},\mb{v}_{-i})$: (omit $\bs{\psi}$ for brevity)

\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
p(v_i=1|\mb{h},\mb{v}_{-i})=\frac{p(v_i=1,\mb{v}_{-i},\mb{h})}{p(\mb{v}_{-i},\mb{h})}=
\frac{p(v_i=1,\mb{v}_{-i},\mb{h})}{p(v_i=0,\mb{v}_{-i},\mb{h})+p(v_i=1,\mb{v}_{-i},\mb{h})}
\\=\frac{e^{-E(v_i=1,\mb{v}_{-i},\mb{h})}}{e^{-E(v_i=0,\mb{v}_{-i},\mb{h})} + e^{-E(v_i=1,\mb{v}_{-i},\mb{h})}}=\frac{1}{1+e^{-\l[E(v_i=0,\ldots)-E(v_i=1,\ldots)\r]}}=\text{sigm}\l[E(v_i=0,\ldots)-E(v_i=1,\ldots)\r]=
\\=\comment{$E(\mb{v},\mb{h})=-\sum_{j<k}L_{jk}v_jv_k -\sum_{l<m}J_{lm}h_lh_m -\sum_{j,l}W_{jl}v_jh_l-\sum_jb_jv_j-\sum_lc_lh_l $}=
\\=\comment{sums w/o $v_i$ (2nd, 5th) cancel out}=\text{sigm}\l[\underbrace{-\ldots+\sum_{j<k,j\neq i, k\neq i}L_{jk}v_jv_k}_{=0}+\underbrace{\sum_{i<k}L_{ik}v_k}_{j=i}+ \underbrace{\sum_{j<i}L_{ji}v_j}_{k=i} -\r.\\\l.\underbrace{-\ldots+\sum_{j\neq i, l}W_{jl}v_jh_l}_{=0}+  \underbrace{\sum_lW_{il}h_l}_{j=i}+b_i\r]=\comment{$L_{ij}=L_{ji}, L_{ii}=0$}=\text{sigm}\l[ \sum_lW_{il}h_l+\sum_kL_{ik}v_k+b_i \r]
\end{empheq}
So:
\begin{gather}
\boxed{p(v_i=1|\mb{h},\mb{v}_{-i})=\text{sigm}\l( \sum_kL_{ik}v_k+\sum_lW_{il}h_l+b_i \r)}
\end{gather}
\u{Symmetrically},
\begin{gather}
\boxed{p(h_j=1|\mb{v},\mb{h}_{-j})=\text{sigm}\l( \sum_lJ_{jl}h_l+\sum_iW_{ij}v_i+c_j \r)}
\end{gather}
\\[1em]
\u{Maximum Likelihood learning}
\\Suppose we have dataset $\mc{D}=\{\mb{x}_1,\ldots\mb{x}_N\}, \mb{x}_n\in\{0,1\}^D$. The goal is to maximize $\sum_{n=1}^N\log p(\mb{x}_n;\bs{\psi})$ for parameters $\bs{\psi}$. For a single training example $\mb{v}$ and any parameter $\theta$:
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\frac{\partial}{\partial\theta}\log p(\mb{v};\bs{\psi})=
\frac{\partial}{\partial\theta}\l(\log \sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})}-
\log\sum_{\mb{\t{v}},\mb{\t{h}}}e^{-E(\mb{\t{v}},\mb{\t{h}};\bs{\psi})} \r)=
\\=-\frac{1}{\sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})}}\sum_{\mb{h}}\l[e^{-E(\mb{v},\mb{h};\bs{\psi})} \cdot \frac{\partial}{\partial\theta}E(\mb{v},\mb{h};\bs{\psi}) \r]+
\frac{1}{\sum_{\mb{\t{v}},\mb{\t{h}}}e^{-E(\mb{\t{v}},\mb{\t{h}};\bs{\psi})}}
\sum_{\mb{\t{v}},\mb{\t{h}}}\l[e^{-E(\mb{\t{v}},\mb{\t{h}};\bs{\psi})}\cdot \frac{\partial}{\partial\theta} E(\mb{\t{v}},\mb{\t{h}};\bs{\psi}) \r]=
\\=\comment{$\frac{e^{-E(\mb{v},\mb{h};\bs{\psi})}}{\sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})}}=\frac{\frac{1}{Z(\theta)}e^{-E(\mb{v},\mb{h};\bs{\psi})}}{\frac{1}{Z(\theta)}\sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})}}=\frac{p(\mb{h},\mb{v};\bs{\psi})}{p(\mb{v};\bs{\psi})}=p(\mb{h}|\mb{v};\bs{\psi})$}=
\\=-\underbrace{\sum_{\mb{h}}p(\mb{h}|\mb{v};\bs{\psi})\cdot\frac{\partial}{\partial\theta}E(\mb{v},\mb{h};\bs{\psi})}_{\E_{\mb{h}|\mb{v};\bs{\psi}}\l[\frac{\partial}{\partial\theta}E(\mb{v},\mb{h};\bs{\psi})\r]}
+\underbrace{\sum_{\mb{\t{v}},\mb{\t{h}}}p(\mb{\t{v}},\mb{\t{h}};\bs{\psi})\cdot\frac{\partial}{\partial\theta}E(\mb{\t{v}},\mb{\t{h}};\bs{\psi})}_{\E_{\mb{\t{v}},\mb{\t{h}};\bs{\psi}}\l[\frac{\partial}{\partial\theta}E(\mb{\t{v}},\mb{\t{h}};\bs{\psi})\r]}
\end{empheq}
This we obtain
\begin{gather}
\boxed{\frac{\partial}{\partial\theta}\log p(\mb{v};\bs{\psi})=-\E_{\mb{h}|\mb{v};\bs{\psi}}\l[\frac{\partial E}{\partial\theta}(\mb{v},\mb{h};\bs{\psi})\r]+\E_{\mb{\t{v}},\mb{\t{h}};\bs{\psi}}\l[\frac{\partial E}{\partial\theta}(\mb{\t{v}},\mb{\t{h}};\bs{\psi})\r]}
\end{gather}
And avegared over all data points:
\begin{gather}
\frac{1}{N}\sum_{n=1}^N \frac{\partial}{\partial\theta} \log p(\mb{x}_n;\bs{\psi})=-\frac{1}{N}\sum_{n=1}^N \E_{\mb{h}|\mb{v};\bs{\psi}}\l[\frac{\partial E}{\partial\theta}(\mb{x}_n,\mb{h};\bs{\psi})\r] + \E_{\mb{\t{v}},\mb{\t{h}};\bs{\psi}}\l[\frac{\partial E}{\partial\theta}(\mb{\t{v}},\mb{\t{h}};\bs{\psi})\r]
\end{gather}
or solely in terms of expectations:
\begin{gather}
\boxed{\E_{\mb{v}\sim P_{\text{data}}(\mb{v})}\l[ \frac{\partial}{\partial\theta}\log p(\mb{v};\bs{\psi}) \r]=-\E_{\mb{v},\mb{h}\sim P_{\text{data}}(\mb{v},\mb{h};\bs{\psi})}\l[ \frac{\partial E}{\partial\theta}(\mb{v},\mb{h};\bs{\psi}) \r] + \E_{\mb{v},\mb{h}\sim P_{\text{model}}(\mb{v},\mb{h};\bs{\psi})}\l[ \frac{\partial E}{\partial\theta}(\mb{v},\mb{h};\bs{\psi}) \r]}
\end{gather}
where $P_{\text{data}}(\mb{v})=\frac{1}{N}\sum_{n=1}^N \delta_{\mb{v},\mb{x}_n}$ is \emph{empirical distribution}, $P_{\text{data}}(\mb{v},\mb{h};\bs{\psi})=p(\mb{h}|\mb{v};\bs{\psi})P_{\text{data}}(\mb{v})$ is \emph{complete-data distribution} and $P_{\text{model}}$ is a distribution modelled by BM.
\\[1em]
Before we move on, there is one more equivalent form of gradient of data log-likelihood:
\begin{gather}
-\frac{\partial}{\partial\theta}\log p(\mb{v};\bs{\psi})=\frac{\partial}{\partial\theta}\mc{F}(\mb{v})-\sum_{\mb{\t{v}}}p(\mb{\t{v}})\frac{\partial}{\partial\theta}\mc{F}(\mb{\t{v}})
\end{gather}
Notice that the above gradient contains two terms, which are referred to as the \textbf{positive} and \textbf{negative} phase. The terms positive and negative do not refer to the sign of each term in the equation, but rather reflect their effect on the probability density defined by the model. The first term increases the probability of training data (by reducing the corresponding free energy), while the second term decreases the probability of samples generated by the model.
\\[1em]
\u{Derivatives of the energy w.r.t. model parameters}:
$$
E(\mathbf{v},\mathbf{h};\bs{\psi})=-\frac{1}{2}\mb{v}^T\mb{L}\mb{v}-\frac{1}{2}\mb{h}^T\mb{J}\mb{h}-\mb{v}^T\mb{W}\mb{h}-\mb{b}^T\mb{v}-\mb{c}^T\mb{h},
$$
\begin{itemize}
\item $\frac{\partial E}{\partial \mb{L}}=-\mb{v}\mb{v}^T$ (remember that $\mb{L}$ is symmetric)
\item Similarly $\frac{\partial E}{\partial \mb{J}}=-\mb{h}\mb{h}^T$ and $\frac{\partial E}{\partial \mb{W}}=-\mb{v}\mb{h}^T$
\item Finally $\frac{\partial E}{\partial \mb{b}}=-\mb{v}$ and $\frac{\partial E}{\partial \mb{c}}=-\mb{h}$
\end{itemize}
So 1 update of the weights $\mb{W}$ using vanilla gradient ascent looks like:
\begin{align}
\mb{g}_{\mb{W}} &\;\leftarrow\; \E_{\mb{v},\mb{h}\sim P_{\text{data}}(\mb{v},\mb{h};\bs{\psi})}\l[\mb{v}\mb{h}^T\r]-\E_{\mb{v},\mb{h}\sim P_{\text{model}}(\mb{v},\mb{h};\bs{\psi})}\l[\mb{v}\mb{h}^T\r]
\\
\mb{W} &\;\leftarrow\; \mb{W} + \alpha\cdot\mb{g}_{\mb{W}}
\end{align}
and similarly for other parameters.
\\[1em]
\u{Problems}
\\
So far it seems like everything is ok, but there is 1 very serious problem, \textbf{both expectations in (13) $\Leftrightarrow$ (15) are simply intractable}. First sum has $O\l(2^H\r)$ terms while the second one has $O\l(2^{V+H}\r)$ terms. Moreover, even $p(\mb{h}|\mb{v};\bs{\psi})$ or $p(\mb{v},\mb{h};\bs{\psi})$ are intractable because of partition function in denominators which itself has exponentially many in $V+H$ terms.
\\[1em]To overcome this computational burden, G. Hinton and T. Sejnowski in 1983 proposed algorithm that uses Gibbs sampling to approximate both expectations. The idea is that both expectations are approximated by states of 2 Markov chains (Monte-Carlo sampling), which are updated using Gibbs sampling $\forall$ training example during training.
\\[1em]The main problem with the last algorithm is \tb{time} required to approach distribution, especially when estimating the model's expectations, since the Gibbs chain may need to explore a highly multimodal energy landscape + exponential in machine size and magnitude of weights time to collect equilibrium statistics. We will see that in RBM Gibbs sampling can be performed much more efficiently thus having much more efficient learning algorithm.
\\[1em]
To summarize, Boltzmann Machines is a Monte Carlo version of Hopfield Net with discrimination between visible and hidden units and that uses annealed Gibbs sampling. Boltzmann Machines were one of the first neural networks capable of learning internal representations, and are able to represent and (given sufficient time) solve difficult combinatoric problems + they are again are considered a biologically plausiable because during learning only local information is used, i.e. the update for a particular weight connecting two units depend only on the statistics of those two units (this can be seen by considering equation (17) element-wisely).
\subsubsection{Additional facts}
\textbullet{} from formulae (11),(12) we can see that probability of one unit being on is given by a linear model (logistic regression) from the values of the other units.
\\
\textbullet{} in the presence of hidden units, BM becomes a \emph{universal approximator} in a sense that it can learn arbitraty point mass function over discrete variables (without hidden units it could have learned only linear relationships between variables) \cite{goodfellow2016deep}
\\
\textbullet{} it is also interesting that in this model we can compute $p(\mb{h}|\mb{v};\bs{\psi})$ exactly and efficiently, unlike many other models with hidden variables like VAE etc., and still this model can learn fairly complex distributions (+ previous bullet)

\subsection{Restricted Boltzmann Machines\cite{coursera_nn, fischer2012introduction, tutorial2014lisa, hinton2010practical, fischer2014training, smolensky1986information, gibbs_wiki}}
\subsubsection{Intro and comparison with general BM}
A \emph{restricted Boltzmann machine} was invented by P. Smolensky in 1986 and is characterized by absense of hid-hid and vis-vis connections (see Fig. \ref{fig:rbm}).
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.5]{img/rbm.png}
\centering
\caption{Diagram of a restricted Boltzmann machine with four visible units and three hidden units (no bias units).}
\label{fig:rbm}
\end{mdframed}
\end{figure}
In this case we have $\mb{L}=\mb{J}=\mb{0}$ and the energy function for RBM simplifies:
\bg
E(\mathbf{v},\mathbf{h};\bs{\psi})=-\mb{v}^T\mb{W}\mb{h}-\mb{b}^T\mb{v}-\mb{c}^T\mb{h}
\eg
Because of the \emph{bipartite} structure of the graph, in this MRF \tb{all hidden units are conditionally independent given visible ones and vice versa} ($\bs{\psi}$ is omitted for brevity):
\bg
p(\mb{h}|\mb{v})=\prod_jp(h_j|\mb{v}), \;\;\; p(\mb{v}|\mb{h})=\prod_ip(v_i|\mb{h})
\eg
(Formally every path in this graphical model between different $h_j$ and $h_l$ is blocked by $\mb{v}$ and vice versa).
\\[1em]
Taking this into account, formulae (11) and (12) simplify too:
\bg
p(v_i=1|\mb{h})=\text{sigm}\l( \sum_lW_{il}h_l+b_i \r),
\\
p(h_j=1|\mb{v})=\text{sigm}\l( \sum_iW_{ij}v_i+c_j \r)
\eg
very important that each of these formulae can be computed \tb{in parallel}:
\bg
p(\mb{v}=\mb{1}|\mb{h})=\text{sigm}\l( \mb{W}\mb{h}+\mb{b} \r),
\\
p(\mb{h}=\mb{1}|\mb{v})=\text{sigm}\l( \mb{W}^T\mb{v}+\mb{c} \r)
\eg
this makes exact inference tractable (as opposed to general BM).
\\[1em]
It is also easier to calculate marginalized distribution of the visible variables (see Fig. \ref{fig:prod_experts})
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.4]{formulae/prod_experts.png}
\centering
\caption{$p(\mb{v})$}
\label{fig:prod_experts}
\end{mdframed}
\end{figure}
\\[1em]
\u{Gradient of Log-Likelihood}
\\Recall
$$
\frac{\partial}{\partial\theta}\log p(\mb{v};\bs{\psi})=
-\sum_{\mb{h}}p(\mb{h}|\mb{v};\bs{\psi})\cdot\frac{\partial}{\partial\theta}E(\mb{v},\mb{h};\bs{\psi})
+\sum_{\mb{\t{v}},\mb{\t{h}}}p(\mb{\t{v}},\mb{\t{h}};\bs{\psi})\cdot\frac{\partial}{\partial\theta}E(\mb{\t{v}},\mb{\t{h}};\bs{\psi})
$$
First term for $\theta=W_{ij}$ (omit $\bs{\psi}$):
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
[1]=-\sum_{\mb{h}} \prod_kp(h_k|\mb{v}) \cdot\frac{\partial}{\partial W_{ij}}E(\mb{v},\mb{h})=\comment{$\frac{\partial}{\partial W_{ij}}E(\mb{v},\mb{h})=-v_ih_j$}=\sum_{\mb{h}} \prod_kp(h_k|\mb{v})v_ih_j=
\\=\sum_{h_j}\sum_{\mb{h}_{-j}}p(h_j|\mb{v})p(\mb{h}_{-j}|\mb{v})h_jv_i=
\sum_{h_j\in\{0,1\}}h_jp(h_j|\mb{v})v_i \underbrace{\sum_{\mb{h}_{-j}}p(\mb{h}_{-j}|\mb{v})}_{=1}=p(h_j=1|\mb{v})\cdot v_i
\end{empheq}
The second term
$$
[2]=\sum_{\mb{\t{v}},\mb{\t{h}}}p(\mb{\t{v}},\mb{\t{h}})\cdot\frac{\partial}{\partial\theta}E(\mb{\t{v}},\mb{\t{h}})=\sum_{\mb{\t{v}}}p(\mb{\t{v}})\l(\sum_{\mb{\t{h}}}p(\mb{\t{h}}|\mb{\t{v}})\cdot\frac{\partial}{\partial\theta}E(\mb{\t{v}},\mb{\t{h}})\r)
$$
and thus can be computed in the same manner.
\\Eventually we obtain the following expressions for log-likelihood gradients:
\begin{align}
\frac{\partial}{\partial W_{ij}}\log p(\mb{v};\bs{\psi}) &= p(h_j=1|\mb{v})\cdot v_i
-\sum_{\mb{\t{v}}}p(\mb{\t{v}})\cdot p(h_j=1|\mb{\t{v}})\cdot \t{v}_i
\\
\frac{\partial}{\partial b_{i}}\log p(\mb{v};\bs{\psi}) &= v_i -\sum_{\mb{\t{v}}}p(\mb{\t{v}}) \cdot \t{v}_i
\\
\frac{\partial}{\partial c_{j}}\log p(\mb{v};\bs{\psi}) &= p(h_j=1|\mb{v}) - \sum_{\mb{\t{v}}}p(\mb{\t{v}}) \cdot p(h_j=1|\mb{\t{v}})
\end{align}
+ use formulae (21), (22). Averaged over all training examples, these formulae can also be rewritten in terms of expectations. Note that $\sum_{\mb{\t{v}}} p(\mb{\t{v}})[\cdots]=\E_{\mb{\t{v}}\sim P_{\text{\text{model}}}}[\cdots]$.
\\[1em]
Despite of the sound simplification and reducing of computational complexity because of the nice factorization, (25) - (27) are still intractable for regular RBMs because of the second terms which are exponential in $\min\{V, H\}$ (if $H<V$ we can factorize $p(\mb{v},\mb{h})=p(\mb{h})p(\mb{v}|\mb{h})$).
\\[1em]Thus, to avoid this complexity, we will approximate expectations by samples from the model distribution using MCMC-based algorithm \emph{Contrastive Divergence} (described below).
\\[1em]
\u{Quick summary}
\begin{center}
\begin{tabular}{ |c|c|c| }
\hline
& \tb{general BM} & \tb{RBM} \\
\hline
\tb{exact Maximum Likelihood learning} & \color{red}intractable & intractable$^{*}$ \\
\hline
\tb{inference} & \color{red}approximate & \color{green}exact \\
\hline
\end{tabular}
\end{center}
$^{*}$ -- learning still can be done efficiently using Contrastive Divergence.

\subsubsection{Contrastive Divergence algorithm and modifications}
All common training algorithms for RBMs approximate expectations in log-likelihood gradients (25) - (27) by only single sample from RBM and perform gradient ascent on these approximations.
\\[1em]
Samples in RBM can be obtained by running a Markov chain to convergence, using Gibbs sampling as the transition operator.
In general, Gibbs sampling of the joint of $N$ random variables $\bs{\xi}=(\xi_1\ldots\xi_N)$ is done through a sequence of $N$ sampling sub-steps of the form $\xi_i\sim p(\xi_i|\bs{\xi}_{-i})$. If done sequentially, one can show that these sub-steps define reversible Markov chain with desired invariant joint distribution.
\\[1em]
For RBMs, $\xi$ consists of the set of visible and hidden units. However, since they are conditionally independent, $p(v_i|\mb{h},\mb{v}_{-i})=p(v_i|\mb{h})$ and $p(h_j|\mb{v},\mb{h}_{-j})=p(h_j|\mb{v})$ one can perform \tb{block} Gibbs sampling (which is impossible in BM). In this setting, visible units are sampled simultaneously given fixed values of the hidden units. Similarly, hidden units are sampled simultaneously given the visibles. A step in the Markov chain is thus taken as follows:
\bg
\mb{h}^{(n+1)}\;\sim\;\text{Ber}(\text{sigm}\l( \mb{W}^T\mb{v}^{(n)}+\mb{c} \r)),
\\
\mb{v}^{(n+1)}\;\sim\;\text{Ber}(\text{sigm}\l( \mb{W}\mb{h}^{(n+1)}+\mb{b} \r))
\eg
Graphically:
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.6]{img/gibbs.png}
\centering
\caption{Gibbs sampling in RBM.}
\label{fig:gibbs}
\end{mdframed}
\end{figure}
\\
As $n \rightarrow \infty$, samples $\left(\mb{v}^{(n)},\mb{h}^{(n)}\right)$ are guaranteed to be accurate samples of the model joint.
\\[1em]
In theory, each parameter update in the learning process would require running one such chain to convergence to obtain unbiased estimates of gradients of log-likelihood, which requires many sampling steps (prohibitively expensive) and moreover it is typically unclear for how long chain has to be run. As such, several algorithms have been devised for RBMs, in order to efficiently sample from.
\\[1em]
\u{Contrastive Divergence} (CD-k)
\\
This algorithm was proposed in 2002 by G.Hinton and uses two tricks to speed up the sampling process:
\begin{itemize}
\item since we eventually want $p_{\text{model}}(\mb{v}) \approx p_{\text{data}}(\mb{v})$ (Maximum Likelihood $\Leftrightarrow$ Minimizing KL divergence between $p_{\text{model}}$ and $p_{\text{data}}$ (empirical distribution)), \emph{we initialize the Markov chain with a training example}, so that the chain will be already close to having converged to its desired distribution.
\item CD does not wait for the chain to converge. Samples are obtained after only $k$-steps of Gibbs sampling. In practice, $k=1$ has been shown to work surprisingly well.
\end{itemize}
General formula for gradient approximation:
\bg
\frac{\partial}{\partial\theta}\log p(\mb{v}^{(0)};\bs{\psi})\approx
-\sum_{\mb{h}}p(\mb{h}|\mb{v}^{(0)};\bs{\psi})\cdot\frac{\partial}{\partial\theta}E(\mb{v}^{(0)},\mb{h};\bs{\psi})
+\sum_{\mb{h}}p(\mb{h}|\mb{v}^{(k)};\bs{\psi})\cdot\frac{\partial}{\partial\theta}E(\mb{v}^{(k)},\mb{h};\bs{\psi})
\eg
Concrete formulae for gradient approximations:
\begin{align}
g_{W_{ij}} &\leftarrow p(h_j=1|\mb{v}^{(0)})\cdot v_i^{(0)}-p(h_j=1|\mb{v}^{(k)})\cdot v_i^{(k)}
\\
g_{b_i} &\leftarrow v_i^{(0)}-v_i^{(k)}
\\
g_{c_j} &\leftarrow p(h_j=1|\mb{v}^{(0)})-p(h_j=1|\mb{v}^{(k)})
\end{align}
with $\mb{v}^{(0)}:=\mb{v}$ -- training example
\\
Each gradient is a difference of the corresponding \tb{positive} and \tb{negative} gradients, see (16). It is thus described through sequence of \emph{positive phases}, where we clamp visible units to a particular state vector sampled from the training set $\mb{v}\leftarrow\mb{x}_i \sim p_{\text{data}}$, and \emph{negative phases}, where the network is run freely without fixing any states.
\\[0.5em]
\tb{Note}: originally formulae (31)-(33) should contain instead of $p(h_j=1|\mb{v})$ the samples of hidden units themselves (apart from sampling involved to estimate model's expectations), but stated variants typically provide slightly less noisy and thus faster learning, and it is not so important at all when RBM is used to pretrain hidden layer of units for DBN/DBM \cite{hinton2010practical}.
\\[1em]
Since $\mb{v}^{(k)}$ is not a sample from the stationary distribution the approximation the approximation (30) is biased. Obviously, as $k \rightarrow \infty$, bias vanishes. One can also show that CD does not maximize the likelihood of the data under the model, but minimize the difference of two KL-divergences:
\bg
D_{\text{KL}}(p_{\text{data}}(\mb{v})\;\|\;p_{\text{model}}(\mb{v}))-D_{\text{KL}}(p^{(k)}(\mb{v})\;\|\;p_{\text{model}}(\mb{v})),
\eg
where $p^{(k)}(\mb{v})$ is a distribution of visible variables after k steps of Markov chain. If chain already reached stationarity it holds that $p^{(k)}=p_{\text{model}} \;\Rightarrow\; D_{\text{KL}}(p^{(k)}(\mb{v})\;\|\;p_{\text{model}}(\mb{v}))=0$ and the approximation error of CD is vanishes.
\\[1em]
CD does not follow the gradient of any function \cite{hinton2010practical}.
\\[1em]
\u{Persistent CD}
In \cite{tieleman2008training} they rely on a single Markov chain, which has a persistent state (i.e., not restarting a chain for each observed example). For each parameter update, we extract new samples by simply running the chain for $k$-steps. The state of the chain is then preserved for subsequent updates. The general intuition is that if parameter updates are small enough compared to the mixing rate of the chain, the Markov chain should be able to "catch up" to changes in the model.
\\[1em]
Typically one uses as much markov chains as there are training examples in a minibatch. These persistent states of Markov chains can be used to generate samples after training. In \cite{hinton2010practical} they mention PCD learns significantly better models than CD-k for various k and is the recommended method if the aim is to build the best density model of the data. We will use a modification of PCD for DBM training.
\\[1em]
Also, recently the new algorithm \u{Parallel tempering} is proposed \cite{fischer2012introduction}. It introduces supplementary Gibbs chains that sample from more and more smoothed replicas of the original distribution. In the price of computational overhead it gives faster mixing Markov chain and thus less biased gradient approximation.
\subsubsection{Extensions}
\u{More general RBMs/BMs}
\\[1em]
So far we considered only case of binary (Bernoulli) both visible and hiddent units. But RBMs (and BMs) can also be extended to model $\R$-valued inputs/outputs. This can be achieved by altering the energy functio (more examples below). More generally RBM can be defined as any MRF (undirected graphical model) with conditionally independent visible units given hidden and vice versa and with energy function of the following kind:
\bg
E(\mb{v},\mb{h};\bs{\psi})=\sum_{i,j}\phi_{ij}(v_i,h_j;\bs{\psi})+\sum_i \omega_i(v_i;\bs{\psi}) + \sum_j \nu_j(h_j;\bs{\psi})
\eg
with $\R$-valued functions $\phi_{ij}, \omega_i, \nu_j$ for which partition function is finite $Z(\bs{\psi})<\infty$.
\\[1em]
\u{Conditional RBMs}
\\[1em]
some of the parameters in $E$ are replaced by parameterized functions of some conditioning random variables.
\\[1em]
\u{Classification RBMs (cRBMs)}
\\[1em]
In regular RBM we model $p(\mb{x})$. In classification RBM there are couple of choices:
\begin{itemize}
\item model $p(\mb{x},\mb{y})$ (\emph{generative mode})
\item model $p(\mb{y}|\mb{x})$ (\emph{discriminative mode}), equivalent to the above via Bayes theorem
\item $\alpha\cdot p(\mb{x},\mb{y})+(1-\alpha)\cdot p(\mb{y}|\mb{x})\rightarrow \text{max}$ (\emph{hybrid mode})
\end{itemize}
See more in \cite{hinton2010practical}.

\subsubsection{Different types of RBM units}
For a binary unit, the probability of turning on is given by the logistic sigmoid function of its total input, $x$.
\bg
p=\text{sigm}(x)=\frac{1}{1+e^{-x}}=\frac{e^x}{e^x+e^0}
\eg
The energy contributed by the unit is $-x$ if it is on and 0 if it is off. Equation (36) makes it clear that the probability of each of the two possible states is proportional to the negative exponential of its energy. This can be generalized to $K$ alternative states.
\\[1em]
\u{Softmax units}
\bg
p_j=\text{softmax}_j(\mb{x})=\frac{e^{x_j}}{\sum_{k=1}^K e^{x_k}}
\eg
It is the appropriate way to deal with a quantity that has $K$ alternative \emph{mutually exclusive} values which are not ordered in any way. When viewed in this way, the learning rule for the binary units in a softmax is identical to the rule for standard binary units.
\tb{The only difference is in the way the probabilities of the states are computed and the samples are taken} + Fig. \ref{fig:softmax_sampling}.
\\[0.5em]
Formally, for binary visible and softmax hidden:
$$\mb{v}\in\{0,1\}^D,\mb{h}\in\text{One-hot}(H)=\l\{\mb{q}\in\{0,1\}^H \bigg| \sum_kq_k=1\r\}, H=K$$
Energy function has the same functional form, as for binary-binary RBM:
\bg
E(\mb{v},\mb{h};\bs{\psi})=-\sum_{j,l}W_{jl}v_jh_l-\sum_jb_jv_j-\sum_lc_lh_l
\eg
\\[1em]
\u{Multinomial units}
\\
Afurther generalization of the softmax unit is to sample $M$ times (with replacement) from the probability distribution instead of just sampling once. The $K$ different states can then have integer values bigger than 1, but the values must add to $M$. This is called a \emph{multinomial unit} and, again, the learning rule is unchanged. It is also equivalent \cite{hinton2009replicated} to $M$ softmax units with shared weights.
\\[0.5em]
Formally, for binary visible and multinomial hidden with $M$ samples:
$$\mb{v}\in\{0,1\}^D,\t{\mb{h}}\in\l\{\mb{q}\in\{0,1,\ldots,M\}^H \bigg| \sum_kq_k=M\r\}, H=K$$
Energy function has the same functional form, as for binary-binary RBM:
\bg
E(\mb{v},\t{\mb{h}};\bs{\psi})=-\sum_{j,l}W_{jl}v_j\t{h}_l-\sum_jb_jv_j-\sum_lc_l\t{h}_l,
\eg
where $\t{h}_l=\sum_{m=1}^M h_l^{(m)}$ -- count for $l$-th discrete value of hidden units. Typically, this variant of RBM is used for topic modelling \cite{hinton2009replicated} (visible are Multinomial, hidden -- binary). Often $M$ is equal to $K$. In this case, it is also useful to scale hidden bias term by $M$, this will allow to behave sensible when deal with documents of the different lengths.
\\[1em]
\u{Gaussian visible units}
\\
To be able to model real-valued data $\mb{v}\in\R^D$, one solution is to replace the binary visible units by linear units with independent Gaussian noise. The energy function then becomes:
\bg
E(\mb{v},\mb{h};\bs{\psi})=\frac{1}{2}\sum_{i,j}\frac{(v_i-b_i)^2}{\sigma_i^2}-\sum_jc_jh_j-\sum_{i,j}W_{ij}\frac{v_i}{\sigma_i}h_j
\eg
where $\sigma_i$ is the standard deviation of the Gaussian noise for visible unit $i$. It is possible to learn the variance of the noise for each visible unit but this is difficult using CD-k. In many applications, it is much easier to first normalise each component of the data to have zero mean and unit variance and then to use noise free reconstructions, with the variance in equation (40) set to 1.
\\[1em]
From (38) one can derive formulae for activations \cite{salakhutdinov2013learning, krizhevsky2009learning}:
\bg
p(h_j=1|\mb{v})=\text{sigm}\l(\sum_iW_{ij}\frac{v_i}{\sigma_i}+c_j\r),
\\
v_i|\mb{h}\;\sim\;\mc{N}\l(\sigma_i\sum_jW_{ij}h_j+b_i;\;\sigma_i^2\r)=\sigma_i\sum_jW_{ij}h_j+b_i+\sigma_i\cdot\mc{N}(0; 1)
\eg
\\[1em]
There also exist other types of units, such as gaussian for both visible and hidden units, binomial units, rectifier linear units etc., but they more rarely used and are out of the scope of these notes. See more in \cite{hinton2010practical}.

\subsubsection{Free energies formulae}
\textbullet{} \u{Free energy for binary visible and hidden units (19)} (Similarly to Fig. \ref{fig:prod_experts}):
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\mc{F}(\mb{v};\bs{\psi})=-\log\l( \sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})} \r)=
-\log \sum_{\mb{h}} \exp\l(\sum_{i,j}W_{ij}v_ih_j+\sum_ib_iv_i+\sum_jc_jh_j\r)=\\
=-\log \l[\exp\l(\sum_ib_iv_i\r)\cdot\sum_{\mb{h}} \exp\l(\sum_{i,j}W_{ij}v_ih_j+\sum_jc_jh_j\r)\r]=\\
=-\sum_ib_iv_i-\log \sum_{\mb{h}} \underbrace{\exp\l(\sum_jh_j\l(\sum_iW_{ij}v_i+c_j\r)\r)}_{\prod_j \exp\l(h_j\l(\sum_iW_{ij}v_i+c_j\r)\r)}=\comment{$\sum_{\mb{h}}\prod_j f_j(h_j)=\prod_j\sum_{h_j}f_j(h_j)$}=\\
=-\mb{b}\cdot\mb{v}-\sum_j \log\l( \underbrace{ \sum_{h_j\in\{0,1\}}\exp \l[h_i \sum_iW_{ij}v_i+c_j\r]}_{1+\exp\l(\sum_iW_{ij}v_i+c_j\r)}\r)
=-\mb{b}\cdot\mb{v}-\sum_j \text{softplus}\l(\sum_iW_{ij}v_i+c_j\r),
\end{empheq}
where $\text{softplus}(x):=\log(1+e^x)$.
\begin{gather}
\boxed{\mc{F}(\mb{v};\bs{\psi})=-\mb{b}\cdot\mb{v}-\sum_j \text{softplus}\l(\sum_iW_{ij}v_i+c_j\r)}
\end{gather}
\\[1em]
\textbullet{} \u{Free energy for Bernoulli-Softmax RBM (38)}:
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\mc{F}(\mb{v};\bs{\psi})=-\log\l( \sum_{\mb{h}}e^{-E(\mb{v},\mb{h};\bs{\psi})} \r)=
-\sum_ib_iv_i-\log \sum_{\mb{h}} \underbrace{\exp\l(\sum_jh_j\l(\sum_iW_{ij}v_i+c_j\r)\r)}_{\prod_j \exp\l(h_j\l(\sum_iW_{ij}v_i+c_j\r)\r)}=\\
=-\mb{b}\cdot\mb{v}-\log \sum_{j=1}^K\exp\l(\sum_iW_{ij}v_i+c_j\r),
\end{empheq}
\begin{gather}
\boxed{\mc{F}(\mb{v};\bs{\psi})=-\mb{b}\cdot\mb{v}-\log \sum_{j=1}^K\exp\l(\sum_iW_{ij}v_i+c_j\r)}
\end{gather}
\\[1em]
\textbullet{} \u{Free energy for Bernoulli-Multinomial RBM (39)}:
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\mc{F}(\mb{v};\bs{\psi})=-\log\l( \sum_{\t{\mb{h}}}e^{-E(\mb{v},\t{\mb{h}};\bs{\psi})} \r)=
-\sum_ib_iv_i-\log \sum_{\t{\mb{h}}} \underbrace{\exp\l(\sum_j\t{h}_j\l(\sum_iW_{ij}v_i+c_j\r)\r)}_{\prod_j \exp\l(\t{h}_j\l(\sum_iW_{ij}v_i+c_j\r)\r)}=\\
=-\mb{b}\cdot\mb{v}-\log \sum_{\t{h}_1+\ldots+\t{h}_K=M,0\leq \t{h}_l\leq M}\prod_j\exp\l(\t{h}_j\l(\sum_iW_{ij}v_i+c_j\r)\r),
\end{empheq}
For large $M$ this equation seems to be intractable to compute. But we can approximate this by sampling $\t{\mb{h}}$ from Multinomial distribution with equiprobable states and applying appropriate scaling, note that
$$
\l | \l\{\mb{q}\in\{0,1,\ldots,M\}^H \bigg| \sum_kq_k=M\r\} \r |=\#_{M,K}=\binom{M+K-1}{K-1}=\frac{\Gamma(M+K)}{\Gamma(M+1)\Gamma(K)}
$$
So
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\t{\mb{h}}\;\sim\;\text{Multinomial}\l(\mb{p}=\l(\frac{1}{K}\ldots\frac{1}{K}\r); M\r),
\\
\mc{F}(\mb{v};\bs{\psi}) \approx -\mb{b}\cdot\mb{v}-\log \l(\frac{\Gamma(M+K)}{\Gamma(M+1)\Gamma(K)}\r)-\sum_j \t{h}_j \l(\sum_iW_{ij}v_i+c_j\r)
\end{empheq}
\begin{gather}
\boxed{ \mc{F}(\mb{v};\bs{\psi})\approx -\mb{b}\cdot\mb{v}-\texttt{lgamma}(M+K)+\texttt{lgamma}(M+1)+\texttt{lgamma}(K)-\sum_j \t{h}_j \l(\sum_iW_{ij}v_i+c_j\r) }
\end{gather}
\\[1em]
\textbullet{} \u{Free energy for Gaussian-Bernoulli RBM}:
\\Derivation is straightforward, the formula is very similar to (43), but with accordingly changed bias term for visible units, and scaled visible units by their resp. std. deviation:
\begin{gather}
\boxed{\mc{F}(\mb{v};\bs{\psi})=\frac{1}{2}\l\|\frac{\mb{v}-\mb{b}}{\bs{\sigma}}\r\|^2-\sum_j \text{softplus}\l(\sum_iW_{ij}\frac{v_i}{\sigma_i}+c_j\r)}
\end{gather}
or equivalently if $\t{\mb{v}}\leftarrow \mb{v}/\bs{\sigma}$ (element-wise division, and also in (46),(47)):
\begin{gather}
\boxed{\mc{F}(\mb{v};\bs{\psi})=\frac{1}{2}\l\|\t{\mb{v}}-\frac{\mb{b}}{\bs{\sigma}}\r\|^2-\sum_j \text{softplus}\l(\sum_iW_{ij}\t{v_i}+c_j\r)}
\end{gather}


% Created 2021-07-22 Thu 19:21
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=1610]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble, euscript}
\DeclareMathOperator{\atantwo}{atan2}
\newcommand*{\ctrb}{\EuScript{C}}
\newcommand*{\obsv}{\EuScript{O}}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{\today}
\title{State feedback}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={State feedback},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.4.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Apollo moon lander}
\label{sec:org1fe45b2}
\begin{frame}[label={sec:org431d74e}]{Example - The Apollo lunar module}
\begin{center}
\includegraphics[width=\linewidth]{fig-apollo}
\end{center}
\end{frame}

\begin{frame}[label={sec:org3448043}]{Example - The Apollo lunar module}
State variables: \(x = \begin{bmatrix} x_1 & x_2 & x_3 \end{bmatrix}^T = \begin{bmatrix} \dot{\theta} & \theta & \dot{z} \end{bmatrix}^T\). With dynamics
\[ \begin{cases} \dot{x}_1 =  \ddot{\theta} = k_1 u\\ \dot{x}_2 = \dot{\theta} = x_1\\ \dot{x}_3 = \ddot{z} = k_2\theta = k_2x_2 \end{cases} \]

\[ \dot{x} = \begin{bmatrix} \dot{x}_1\\\dot{x}_2\\\dot{x}_3\end{bmatrix} = \underbrace{\begin{bmatrix} \textcolor{red!60!black}{0} & \textcolor{red!60!black}{0} &\textcolor{red!60!black}{0} \\\textcolor{red!60!black}{1} & \textcolor{red!60!black}{0}& \textcolor{red!60!black}{0}\\ \textcolor{red!60!black}{0}& \textcolor{red!60!black}{k_2} &\textcolor{red!60!black}{0} \end{bmatrix}}_{A} \begin{bmatrix} x_1\\x_2\\x_3\end{bmatrix} + \underbrace{\begin{bmatrix} \textcolor{red!60!black}{k_1} \\ \textcolor{red!60!black}{0} \\\textcolor{red!60!black}{0}  \end{bmatrix}}_{B} u \]
\end{frame}

\begin{frame}[label={sec:org0a7ddbd}]{Example - The Apollo lunar module}
 \begin{align*}
  x(kh+h) &= \mathrm{e}^{Ah} x(kh) + \int_{0}^{h} \mathrm{e}^{As} B u(kh+h-s) ds\\
   &= \underbrace{\mathrm{e}^{Ah}}_{\Phi(h)} x(kh) + \underbrace{\left(\int_{0}^h \mathrm{e}^{As} B ds \right)}_{\Gamma(h)} u(kh)\\
   &= \begin{bmatrix} 1 & 0 & 0\\h & 1 & 0\\\frac{h^2k_2}{2} & hk_2 & 1\end{bmatrix} x(kh) + k_1 \begin{bmatrix} h\\ \frac{h^2}{2} \\ \frac{k_2 h^3}{6} \end{bmatrix} u(kh)
\end{align*}
\end{frame}

\section{Stability}
\label{sec:orgd7bcbae}
\begin{frame}[label={sec:org67bf27a}]{Stability}
\end{frame}
\begin{frame}[label={sec:orgbe68e19}]{Eigenvalues and eigenvectors}
\alert{Definition} The eigenvalues \(\lambda_i  \in \mathbb{R}\) and eigenvectors \(v_i \in \mathbb{R}^n\) of a matrix \(\Phi \in \mathbb{R}^{n\times{}n}\) are the \(n\) pairs \((\lambda_i, v_i \neq 0 ), \; i=1,2,\ldots,n\) that satisfy
\[ \Phi v_i = \lambda_i v_i \]
\end{frame}

\begin{frame}[label={sec:org28643b7}]{Stability}
The system
\begin{equation*}
x(k+1)=\Phi x(k), \ \ x(0)=x_0
\end{equation*}
is \alert{stable} if  \(\underset{t\to\infty}{\lim}x(kh)=0, \quad \forall\;  x_0\in\Bbb{R}^n\).

A necessary and sufficient requirement for stability is that \alert{all the eigenvalues of \(\Phi\) are inside the unit circle.}

The \alert{eigenvalues} of \(\Phi\) are the  \alert{poles} of the system.
\end{frame}

\begin{frame}[label={sec:org49e757d}]{Eigenvalues and eigenvectors - exercise}
\alert{Activity} Verify that the vector
\[ v = \begin{bmatrix}1\\0\end{bmatrix}\]
is an eigenvector of
\[ \Phi = \begin{bmatrix} 2 & 0\\0 & \frac{1}{2} \end{bmatrix}. \]
What is the corresponding eigenvalue?
\end{frame}

\section{Controllability and observability}
\label{sec:org9bb04e3}

\begin{frame}[label={sec:orgcb694f3}]{Controllability}
Controllability is the answer to the question \emph{Can we drive the state of the system to any location in the state space by a suitable input sequence \(u(k),\; k=0,1,2,\ldots,n-1\)?}

Consider
\[ x(k+1) = \Phi x(k) + \Gamma u(k), \quad x(0)= x_0 \]
with solution
\begin{equation}
\begin{split}
x(n) &= \Phi^nx(0) + \Phi^{n-1}\Gamma u(0) + \Phi^{n-2}\Gamma u(1) + \cdots + \Gamma u(n-1)\\
     &= \Phi^nx(0) + W_c U,
\end{split}
\end{equation}
where
\begin{align*}
W_c &= \bbm \Gamma & \Phi\Gamma & \cdots & \Phi^{n-1}\Gamma\ebm\\
U &= \bbm u(n-1) & u(n-2) & \cdots & u(0) \ebm\transp
\end{align*}
\end{frame}

\begin{frame}[label={sec:orge71ee18}]{Controllability}
To find the input sequence \(u(k)\) that takes the state from  \(x(0)=x_0\) to \(x(n) = x_d\) we may solve for \(U\) in the equation
\[ x_d = \Phi^nx_0 + W_cU.\]

\[ U = W_c\inv \left(x_d - \Phi^nx(0)\right) \]

This is possible when the matrix \(W_x\) is \alert{invertible}:

The state-space system above is controllable if and only if the \emph{Controllability matrix} \(W_c\)  has rank \(n\), i.e.
\[ \det W_c \neq 0.\]
\end{frame}

\section{Observability}
\label{sec:org7e5fdb0}
\begin{frame}[label={sec:orgd9c3ab5}]{Observability}
\footnotesize

Observability is the answer to the question "Can we determine the initial state \(x(0)\) if we only know \(y(k), \; k=0,1,2,\ldots, n-1\)?"

The first \(n\) values of the output sequence are given by
\begin{align*}
y(0) &= Cx(0)\\
y(1) &= Cx(1) = C \left( \Phi x(0) + \Gamma u(0)  \right)\\
& \vdots\\
y(n-1) &= Cx(n-1) = C \left( \Phi^{n-1} x(0) + W_c U \right).
\end{align*}
This gives the equation
\[ \bbm C\\C\Phi\\\vdots\\C\Phi^{n-1} \ebm x(0) = \bbm y(0)\\y(1) - C\Gamma u(0)\\\vdots\\ y(n-1) - CW_c U\ebm \]
which can be solved for \(x(0)\) if and only if the matrix
\[W_o = \bbm C\\C\Phi\\\vdots\\C\Phi^{n-1} \ebm\] has full rank.
\end{frame}

\begin{frame}[label={sec:org3f9df66}]{Observability, contd}
The equation
\[ \bbm C\\C\Phi\\\vdots\\C\Phi^{n-1} \ebm x(0) = \bbm y(0)\\y(1) - C\Gamma u(0)\\\vdots\\ y(n-1) - CW_c U\ebm \]
 can be solved for \(x(0)\) if and only if the matrix
\[W_o = \bbm C\\C\Phi\\\vdots\\C\Phi^{n-1} \ebm\] has full rank. If this is the case, the system is said to be \alert{observable}.
\end{frame}

\section{State feedback}
\label{sec:orgf1d8532}
\begin{frame}[label={sec:org24865e0}]{State feedback control}
\end{frame}
\begin{frame}[label={sec:org6fb357b}]{State feedback control}
Given
 \begin{equation}
 \begin{split}
  x(k+1) &= \Phi x(k) + \Gamma u(k)\\
  y(k) &= C x(k)
 \end{split}
 \label{eq:ssmodel}
\end{equation}
and measurements (or an estimate) of the state vector \(x(k)\).

\alert{Linear state feedback} is the control law
\begin{equation*}
\begin{split}
 u(k) &= f\big((x(k), u_c(k)\big) = -l_1x_1(k) - l_2x_2(k) - \cdots - l_n x_n(k) + l_0u_c(k)\\
      &= -Lx(k) + l_0u_c(k),
\end{split}
\end{equation*}
where \[ L = \bbm l_1 & l_2 & \cdots & l_n \ebm. \]
Substituting this in the state-space model \eqref{eq:ssmodel} gives
 \begin{equation}
 \begin{split}
  x(k+1) &= \left(\Phi -\Gamma L \right) x(k) + m\Gamma u_c(k)\\
  y(k) &= C x(k)
 \end{split}
 \label{eq:closedloop}
\end{equation}
\end{frame}

\begin{frame}[label={sec:orgc58c4bb}]{Pole placement by state feedback}
Given a desired placement of the closed-loop poles \(p_1, p_2, \ldots, p_n\), being roots of the desired characteristic polynomial
\begin{equation}
a_c(z) = (z-p_1)(z-p_2)\cdots(z-p_n) = z^n + \alpha_1 z^{n-1} + \cdots \alpha_n.
\label{eq:desiredpoles}
\end{equation}

Linear state feedback gives the system
 \begin{equation}
 \begin{split}
  x(k+1) &= \left(\Phi -\Gamma L \right) x(k) + l_0\Gamma u_c(k)\\
  y(k) &= C x(k)
 \end{split}
 \label{eq:closedloop}
\end{equation}
with characteristic polynomial
\begin{equation}
\det\left(zI - (\Phi - \Gamma L)\right) = z^n + \beta_1(l_1,\ldots,l_n) z^{n-1} + \cdots \beta_n(l_1, \ldots, l_n).
\label{eq:poles}
\end{equation}

Set the coefficients of the desired characteristic polynomial \eqref{eq:desiredpoles} equal to the coefficients of \eqref{eq:poles} to obtain the system of equations
\begin{equation*}
\begin{split}
\beta_1(l_1, \ldots, l_n) &= \alpha_1\\
\beta_2(l_1, \ldots, l_n) &= \alpha_2\\
&\vdots\\
\beta_n(l_1, \ldots, l_n) &= \alpha_n
\end{split}
\label{eq:coeffs}
\end{equation*}
\end{frame}

\begin{frame}[label={sec:org6a1ea81}]{Pole placement by state feedback}
The system of equations
\begin{equation*}
\begin{split}
\beta_1(l_1, \ldots, l_n) &= \alpha_1\\
\beta_2(l_1, \ldots, l_n) &= \alpha_2\\
&\vdots\\
\beta_n(l_1, \ldots, l_n) &= \alpha_n
\end{split}
\label{eq:coeffs}
\end{equation*}
is always linear in the parameters of the controller, henc
\begin{equation*}
M L\transp = \alpha,
\end{equation*}
where \(\alpha\transp = \bbm \alpha_1 & \alpha_2 & \cdots & \alpha_n \ebm.\)
\end{frame}

\begin{frame}[label={sec:org63429f3}]{Pole placement and controllability}
It can be shown that the controllability matrix
\[W_c = \bbm \Gamma & \Phi\Gamma & \cdots & \Phi^{n-1}\Gamma\ebm\]
is a factor of the matrix \(M\)
\[ M = \bar{M} W_c. \] Hence, in general, the equations
\begin{equation}
\bar{M}W_c L\transp = \alpha \qquad \Rightarrow \qquad L\transp = W_c^{-1}\bar{M}^{-1}\alpha
\label{eq:poleplace}
\end{equation}
only has a solution if \(W_c\) is invertible, that is when the system is \emph{controllable}.

 Note that the equations \eqref{eq:poleplace} may also have solutions when the system is not controllable, if  \alert{\(\alpha\) is in the column space of \(M\)}. That is
\[ \alpha = b_1 M_{:,1} + b_2M_{:,2} + \cdots + b_M_{:,m}, \; m < n \]
\end{frame}

\begin{frame}[label={sec:org58305a1},fragile]{Pole placement by state feedback}
 Given a desired placement of the closed-loop poles \(p_1, p_2, \ldots, p_n\), being roots of the desired characteristic polynomial
\begin{equation}
a_c(z) = (z-p_1)(z-p_2)\cdots(z-p_n) = z^n + \alpha_1 z^{n-1} + \cdots \alpha_n.
\label{eq:desiredpoles}
\end{equation}
and closed-loop system
 \begin{equation}
 \begin{split}
  x(k+1) &= \left(\Phi -\Gamma L \right) x(k) + l_0\Gamma u_c(k)\\
  y(k) &= C x(k)
 \end{split}
 \label{eq:closedloop}
\end{equation}

The Matlab (\emph{control systems toolbox}) has methods for computing the gain vector \(L\)

\begin{enumerate}
\item \alert{Ackerman's method}
\begin{verbatim}
L = acker(Phi, Gamma, pd)
\end{verbatim}
\item \alert{Numerically more stable method}
\begin{verbatim}
L = place(Phi, Gamma, pd)
\end{verbatim}
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:org6cebb83}]{The reference input gain \(l_0\)}
The closed-loop state space system
\begin{equation*}
\begin{split}
 x(k+1) &= \underbrace{\left(\Phi -\Gamma L \right)}_{\Phi_c} x(k) + l_0\Gamma u_c(k)\\
 y(k) &= C x(k)
\end{split}
\end{equation*}
has the steady-state solution (\(x(k+1)=x(k)\)) for constant reference signal \(u_c(k) = u_{c,f}\)
\[ y_f = l_0 C(I - \Phi_c)^{-1}\Gamma u_{c,f}.\]
We want \(y_f =  u_{c,f}\),
\[ \Rightarrow \qquad l_0 = \frac{1}{C(I-\Phi_c)^{-1}\Gamma}\]
\end{frame}

\section{Exercise}
\label{sec:org0d8e2aa}

\begin{frame}[label={sec:org31fcf52}]{Exercise - The harddisk drive arm}
\footnotesize
The model of the arm of the harddisk drive
\begin{center}
\includegraphics[width=0.2\linewidth]{../../figures/hard-drive.png}
\end{center}
can, with suitable choice of sampling period, be written

\[x(k+1) = \Phi x(k) + \Gamma u(k) = \begin{bmatrix} 1 & 0.4\\ 0 &1 \end{bmatrix} x + \begin{bmatrix}0.16\\0.8\end{bmatrix}u.\]
With linear state feedback \(u(k) = -Lx(k) + l_0u_c(k)\) the closed-loop system is
\begin{equation*}
   \begin{split}
    x(k+1) &= \left(\Phi -\Gamma L \right) x(k) + l_0\Gamma u_c(k)\\
           &= \begin{bmatrix} 1-0.16l_1 & 0.4 - 0.16l_2\\-0.8l_1 & 1-0.8l_2\end{bmatrix} x(k) + l_0\Gamma u_c(k).
   \end{split}
\end{equation*}

\alert{Determine} the characteristic polynomial of the closed-loop system \(\det \Big( zI - (\Phi - \Gamma L)\Big)\)
\end{frame}
\end{document}


\input{../header_function}

%---------- start document ---------- %
 \section{prime -- primality test , prime generation}\linkedzero{prime}
%
  \subsection{trialDivision -- trial division test}\linkedone{prime}{trialDivision}
   \func{trialDivision}
   {\hiki{n}{integer},\ \hikiopt{bound}{integer/float}{0}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Trial division primality test for an odd natural number.\\
   \spacing
   % added document
   %\spacing
   % input, output document
   \quad \param{bound} is a search bound of primes.
   If it returns \(1\) under the condition that \param{bound} is given and
   less than the square root of \param{n},
   it only means there is no prime factor less than \param{bound}.
%
  \subsection{spsp -- strong pseudo-prime test}\linkedone{prime}{spsp}
   \func{spsp}{\hiki{n}{integer},\ \hiki{base}{integer},\ \hikiopt{s}{integer}{None},\ \hikiopt{t}{integer}{None}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Strong Pseudo-Prime test on base \param{base}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   \quad \param{s} and \param{t} are the numbers such that $n-1 = 2^\param{s}\param{t}$ and \param{t} is odd.
%
 \subsection{smallSpsp -- strong pseudo-prime test for small number}\linkedone{prime}{smallSpsp}
   \func{smallSpsp}{\hiki{n}{integer},\ \hikiopt{s}{integer}{None},\ \hikiopt{t}{integer}{None}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Strong Pseudo-Prime test for integer \param{n} less than $10^{12}$.\\
   \spacing
   % added document
   \quad $4$ spsp tests are sufficient to determine whether an integer less than $10^{12}$ is prime or not.
   \spacing
   % input, output document
   \quad \param{s} and \param{t} are the numbers such that $n-1 = 2^\param{s}\param{t}$ and \param{t} is odd.
%
  \subsection{miller -- Miller's primality test}\linkedone{prime}{miller}
   \func{miller}
   {\hiki{n}{integer}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Miller's primality test.\\
   \spacing
   % added document
   \quad This test is valid under GRH. See \linkingzero{config}.
   \spacing
   % input, output document
   %\quad
%
  \subsection{millerRabin -- Miller-Rabin primality test}\linkedone{prime}{millerRabin}
   \func{millerRabin}
   {\hiki{n}{integer},\ \hikiopt{times}{integer}{20}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Miller's primality test.\\
   \spacing
   % added document
   \quad The difference from \linkingone{prime}{miller} is that
   the Miller-Rabin method uses fast but probabilistic algorithm.
   On the other hand, \linkingone{prime}{miller} employs deterministic
   algorithm valid under GRH.
   \spacing
   % input, output document
   \quad \param{times} (default to $20$) is the number of repetition.
   The error probability is at most $4^{-\param{times}}$.
%
 \subsection{lpsp -- Lucas test}\linkedone{prime}{lpsp}
   \func{lpsp}
   {\hiki{n}{integer},\ \hiki{a}{integer},\ \hiki{b}{integer}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Lucas Pseudo-Prime test.\\
   \spacing
   % added document
   \quad Return True if \param{n} is a Lucas pseudo-prime of parameters \param{a}, \param{b},
    i.e. with respect to $x^2-\param{a}x+\param{b}$.
   \spacing
   % input, output document
   %\quad
%
 \subsection{fpsp -- Frobenius test}\linkedone{prime}{fpsp}
   \func{fpsp}
   {\hiki{n}{integer},\ \hiki{a}{integer},\ \hiki{b}{integer}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Frobenius Pseudo-Prime test.\\
   \spacing
   % added document
   \quad Return True if \param{n} is a Frobenius pseudo-prime of parameters \param{a}, \param{b},
    i.e. with respect to $x^2-\param{a}x+\param{b}$.
   \spacing
   % input, output document
   %\quad
%
 \subsection{by\_primitive\_root -- Lehmer's test}\linkedone{prime}{by\_primitive\_root}
   \func{by\_primitive\_root}
   {\hiki{n}{integer},\ \hiki{divisors}{sequence}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Lehmer's primality test~\cite{Lehmer1927}.\\
   \spacing
   % added document
   \quad Return True iff \param{n} is prime.\\
    The method proves the primality of \param{n} by existence of a primitive
    root.
   \spacing
   % input, output document
   \quad \param{divisors} is a sequence (list, tuple, etc.) of prime divisors
   of $n - 1$.
   %\quad
%
 \subsection{full\_euler -- Brillhart \& Selfridge's test}\linkedone{prime}{full\_euler}
   \func{full\_euler}
   {\hiki{n}{integer},\ \hiki{divisors}{sequence}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Brillhart \& Selfridge's primality test~\cite{BS1967}.\\
   \spacing
   % added document
   \quad Return True iff \param{n} is prime.\\
    The method proves the primality of \param{n} by the equality
    $\varphi(n) = n - 1$, where $\varphi$ denotes the Euler totient
    (see \linkingone{multiplicative}{euler}).
    It requires a sequence of all prime divisors of $n - 1$.
   \spacing
   % input, output document
   \quad \param{divisors} is a sequence (list, tuple, etc.) of prime divisors
   of $n - 1$.
   %\quad
   \quad
%
 \subsection{apr -- Jacobi sum test}\linkedone{prime}{apr}
   \func{apr}
   {\hiki{n}{integer}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad APR (Adleman-Pomerance-Rumery) primality test or the Jacobi sum test.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   \quad Assuming \param{n} has no prime factors less than $32$.
    Assuming \param{n} is spsp (strong pseudo-prime) for several bases.
%
 \subsection{aks -- Cyclotomic Congruence test}\linkedone{prime}{aks}
   \func{aks}
   {\hiki{n}{integer}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad AKS (Agrawal-Kayal-Saxena) primality test or the cyclotomic congruence test.\\
   \spacing
   % added document
   \quad Return True iff \param{n} is prime.\\
   The algorithm determines whether a number \param{n} is prime or composite within polynomial time. For large number \param{n}, you can use apr and any other test in practical use.
   %\spacing
   % input, output document
   %\quad
%
 \subsection{primeq -- primality test automatically}\linkedone{prime}{primeq}
   \func{primeq}
   {\hiki{n}{integer}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad A convenient function for primality test.\\
   \spacing
   % added document
   \quad It uses one of \linkingone{prime}{trialDivision}, \linkingone{prime}{smallSpsp} or \linkingone{prime}{apr} depending on the size of \param{n}.
   \spacing
   % input, output document
   %\quad
%
%\subsection{bigprimeq -- primality test automatically}\linkedone{prime}{bigprimeq}
%   \func{bigprimeq}
%   {\hiki{z}{integer}}{\out{True/False}}\\
%   \spacing
   % document of basic document
%   \quad Giving up rigorous proof of primality, return True for a probable prime.
%   \spacing
   % added document
%   \quad
%   \spacing
   % input, output document
   %\quad
%
 \subsection{prime -- $n$-th prime number}\linkedone{prime}{prime}
   \func{prime}
   {\hiki{n}{integer}}{\out{integer}}\\
   \spacing
   % document of basic document
   \quad Return the \param{n}-th prime number.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
%
 \subsection{nextPrime -- generate next prime}\linkedone{prime}{nextPrime}
   \func{nextPrime}
   {\hiki{n}{integer}}{\out{integer}}\\
   \spacing
   % document of basic document
   \quad Return the smallest prime bigger than the given integer \param{n}.
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
%
 \subsection{randPrime -- generate random prime}\linkedone{prime}{randPrime}
   \func{randPrime}
   {\hiki{n}{integer}}{\out{integer}}\\
   \spacing
   % document of basic document
   \quad Return a random \param{n}-digits prime.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
%
 \subsection{generator -- generate primes}\linkedone{prime}{generator}
   \func{generator}
   {(None)}{\out{generator}}\\
   \spacing
   % document of basic document
   \quad Generate primes from $2$ to $\infty$ (as generator).\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
%
 \subsection{generator\_eratosthenes -- generate primes using Eratosthenes sieve}\linkedone{prime}{generator\_eratosthenes}
   \func{generator\_eratosthenes}
   {\hiki{n}{integer}}{\out{generator}}\\
   \spacing
   % document of basic document
   \quad Generate primes up to \param{n} using Eratosthenes sieve.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
%
 \subsection{primonial -- product of primes}\linkedone{prime}{primonial}
   \func{primonial}
   {\hiki{p}{integer}}{\out{integer}}\\
   \spacing
   % document of basic document
   \quad Return the product
   \begin{equation*}
   \prod_{q \in \mathbb{P}_{\le \param{p}}} q=2\cdot 3\cdot 5\cdots \param{p}\ .
   \end{equation*}
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
%
 \subsection{properDivisors -- proper divisors}\linkedone{prime}{properDivisors}
   \func{properDivisors}
   {\hiki{n}{integer}}{\out{list}}\\
   \spacing
   % document of basic document
   \quad Return proper divisors of \param{n} (all divisors of \param{n} excluding $1$ and \param{n}).\\
   \spacing
   % added document
   \quad  It is only useful for a product of small primes.
   Use \linkingtwo{factor.misc}{FactoredInteger}{proper\_divisors} in a more
   general case.
   \spacing
   % input, output document
   \quad The output is the list of all proper divisors.\\
   \paragraph{DEPRECATION:} This function will be removed in the next release.
   Please use \linkingtwo{factor.misc}{FactoredInteger}{proper\_divisors} instead.\\
%
 \subsection{primitive\_root -- primitive root}\linkedone{prime}{primitive\_root}
   \func{primitive\_root}
   {\hiki{p}{integer}}{\out{integer}}\\
   \spacing
   % document of basic document
   \quad Return a primitive root of \param{p}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   \quad \param{p} must be an odd prime.
%
 \subsection{Lucas\_chain -- Lucas sequence}\linkedone{prime}{Lucas\_chain}
   \func{Lucas\_chain}
   {\hiki{n}{integer},\ \hiki{f}{function},\ \hiki{g}{function},\ \hiki{x\_0}{integer},\ \hiki{x\_1}{integer}}{(\out{integer},\ \out{integer})}\\
   \spacing
   % document of basic document
   \quad Return the value of ($x_n$,\ $x_{n+1}$) for the sequnce $\{ x_i \}$ defined as:\\
   \begin{eqnarray*}
      x_{2i} = \param{f}(x_i)\\
      x_{2i+1} = \param{g}(x_i, x_{i+1})\ ,
   \end{eqnarray*}
   where the initial values \param{x\_0},\ \param{x\_1}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   \quad \param{f} is the function which can be input as $1$-ary integer.
   \param{g} is the function which can be input as $2$-ary integer.\\
%
\begin{ex}
>>> prime.primeq(131)
True
>>> prime.primeq(133)
False
>>> g = prime.generator()
>>> g.next()
2
>>> g.next()
3
>>> prime.prime(10)
29
>>> prime.nextPrime(100)
101
>>> prime.primitive_root(23)
5
\end{ex}%Don't indent!(indent causes an error.)
\C

%---------- end document ---------- %

\input{../footer}


% !TEX root = ../main.tex
% chktex-file 21
% chktex-file 46
\section{Spectral Graph Theory}%
\label{sec:sgt}

We start with an introduction to spectral graph theory, which will allow us to characterize and compare the structure of graphs.
Graphs are most commonly described in their vertex base, i.e.\  the connection strength $w_{i j}$ of pairs $(v_i, v_j)$ of vertices.
\begin{align}
	G :=&\, (\mathcal{V}, \mathcal{E}, W)\quad\text{with edges } \mathcal{E} \subseteq \mathcal{V} \times \mathcal{V}\text{, weights } W \in \mathbb{R}^{N \times N}\text{ and vertex count } N := |\mathcal{V}|
\end{align}
In this paper we will only consider undirected graphs with non-negative real weights and without self-loops ($\forall i: w_{i i} = 0$).

The core idea of spectral graph theory is to perform a change of basis of $W$ and describe graphs in terms of their so called \textit{spectral base} instead of their \textit{vertex base}.
To see what this means, we interpret the adjacency/weight matrix $W$ as a linear operator that operates on so called signals $x \in \mathbb{R}^N$.
A signal $x$ can be interpreted as a function $x: \mathcal{V} \to \mathbb{R}$ that assigns a signal strength to each vertex.
By applying $Wx$ the given signal strengths $x_i$ are shifted according to the connection strenghts $w_{i j}$ to neighboring vertices $v_j$.
This interpretation of graphs is very similar to that of Markov chains where signals represent probability distributions.

\subsection{Relating Graph Signals to Real-valued Functions}%
\label{sec:sgt:real}

Let us now compare discrete graph signals $x: \mathcal{V} \to \mathbb{R}$ to continuous real-valued functions $f: \mathbb{R} \to \mathbb{R}$.
Both signatures only differ in their domain.
The domain $\mathbb{R}$ of $f$ has an inherent structure, the real number line, which provides a strict ordering of its elements and a notion of distance between them.
The domain $\mathcal{V}$ of $x$ however has no such inherent structure, i.e.\  $v_1 < v_2$ for two vertices $v_1, v_2$ does not have a clear meaning.
The structure of $\mathcal{V}$ fully depends on the graph $G$ that is acting on it.
Intuitively graph signals can thus be understood as a discretized generalization of real-valued functions, where the underlying structure of the input domain is not fixed but can be freely chosen.
\begin{figure}[ht]
	\centering
	\includegraphics[width=0.8\linewidth]{gfx/sgt/real-graph.pdf}
	\caption{%
		Illustration of how the discretized real number line can be interpreted as an infinite linear graph, compared to some arbitrary finite non-linear graph.
		The bars show the signal strengths $f(t)$ and $x_i$ at the vertices $b_t$ and $b_i$ respectively.
	}\label{fig:sgt:realGraph}
\end{figure}

\Cref{fig:sgt:realGraph} shows that all real-valued functions $f$ can be seen as signals $x$ of the graph described by the real number line\footnote{%
	Technically this is not correct, since $\mathbb{R}$ is continuous whereas all vertex sets $\mathcal{V}$ have to be discrete.
	To build an intuition for graph signals, this detail can however be ignored.
}.
Both, real-valued functions and graph signals, can be described as vectors in their time and vertex base respectively:
\begin{equation*}
	\begin{split}
		f = \int_{\mathbb{R}} f(t) b_t dt
		\Rightarrow \langle b_t, f \rangle = f(t)
	\end{split}
	\quad\text{and}\quad
	\begin{split}
		x = \sum_{i = 1}^{N} x_i b_i
		\Rightarrow \langle b_i, x \rangle = x_i
	\end{split}
\end{equation*}
In this equation ${\{ b_i \}}_{v_i \in \mathcal{V}}$ denotes the standard basis of the adjacency matrix $W$, i.e.\ $b_i \in \mathbb{R}^N$ has a signal strength of $0$ for all $v_{j \neq i}$ and a signal strength of $1$ for $v_i$.
Similarly ${\{ b_t \}}_{t \in \mathbb{R}}$ denotes the infinite dimensional standard basis of the space of real functions, where $\langle b_t, \cdot \rangle := \delta_t(\cdot)$, with $\delta$ denoting the Dirac delta function.

\vspace{-1em}%
\subsection{Extending the Fourier Transform to Graphs}%
\label{sec:sgt:fourier}

As mentioned at the beginning of this section, the core idea of spectral graph theory is to express graph signal vectors $x$ in the spectral basis ${\{ u_i \}}_{i = 1}^{N}$ instead of the standard vertex basis ${\{ b_i \}}_{v_i \in \mathcal{V}}$.
This idea is analogous to the classical Fourier transform on real-valued functions.
In the first step we are going to give an intuition for the classical Fourier transform.
Afterwards we will extend this intuition to the graph domain.

The Fourier basis ${\{ u_\xi \}}_{\xi \in \mathbb{R}}$ describes a function $f$ not in terms of the values $f(t) = \langle b_t, f \rangle$ it takes at position $t$ but instead in terms of the amplitudes $\hat{f}(\xi) = \langle u_\xi, f \rangle$ of the complex exponentials $u_\xi(t) = e^{2\pi i \xi t}$.
Using this perspective, the Fourier transform can be viewed as a change of basis operator from the orthonormal standard basis ${\{ b_t \}}_{t \in \mathbb{R}}$ to the also orthonormal Fourier basis ${\{ u_\xi \}}_{\xi \in \mathbb{R}}$:
\begin{align*}
	f = \int_{\mathbb{R}} \underbrace{\langle b_t, f \rangle}_{f(t)} b_t dt\quad=\quad\int_{\mathbb{R}} \underbrace{\langle u_\xi, f \rangle}_{\hat{f}(\xi)} u_\xi d\xi
\end{align*}
This property of the Fourier transform by itself is not special; in fact there are infinitely many orthonormal bases on the space of real-valued functions.
The distinguishing property of ${\{ u_\xi \}}_{\xi \in \mathbb{R}}$, making it useful in many domains, is that it is an \textit{eigenbasis} of the \textit{Laplacian} $\Updelta$, i.e.\  $\Updelta u_\xi = \lambda_\xi u_\xi$ for the eigenvalue $\lambda_\xi \in \mathbb{R}$.
The Laplacian is a multi\-dimensional generalization of the second derivative.
For real-valued functions this boils down to $\Updelta u_\xi = \frac{\partial^2}{\partial t^2} u_\xi = -{(2 \pi \xi)}^2 e^{2 \pi i \xi t}$ with the eigenvalue $\lambda_\xi = {-(2\pi\xi)}^2$ depending solely on the frequency $\xi$.
The original motivation to express functions in terms of the Laplacian's eigenbasis was to solve the physical heat equation\footnote{%
	More generally the Fourier basis turns out to be meaningful for all \textit{linear time-invariant} (LTI) systems, of which the heat equations are only one instance.
}.
For this reason it is a useful intuition to think about signals as temperature distributions that will converge to an equilibrium state over time.
This intuition also works in the graph setting where heat only flows between neighboring vertices in proportion to their connection strengths.

Now that we have looked at the Fourier transform of real-valued functions, we will extend this notion to graphs.
Just like the classical Fourier transform, the graph Fourier transform performs a change of basis of a signal $x$ from the vertex basis ${\{ b_i \}}_{v_i \in \mathcal{V}}$ to the Fourier basis ${\{ u_k \}}_{k = 1}^{N}$.
This Fourier basis again is characterized by it being an eigenbasis of the Laplacian, more specifically the so called \textit{combinatorial graph Laplacian} $L$ in this case:
\begin{align}
	L := D - W\quad\text{with the degree matrix } D := {
		\renewcommand*{\arraystretch}{0.5}
		\begin{pmatrix}
			d_1 & & \\
			& \ddots & \\
			& & d_N
		\end{pmatrix}
	}, d_i := \sum_{j = 1}^{N} w_{i j}
\end{align}
Using this definition, the application $L x$ is a discrete generalized analogue of the second derivative $\frac{\partial^2}{\partial t^2} f$.
Putting both Laplacian variants, $\frac{\partial^2}{\partial t^2}$ and $L$, side-by-side gives an intuition for why this is the case:
\begin{equation*}
	\begin{split}
		\left(-\frac{\partial^2}{\partial t^2} f\right)\mkern-4mu(t) = \lim_{h \to 0} \frac{1}{h^2} (\underbrace{f(t) - f(t-h)}_{\Delta_{t, t-h}} + \underbrace{f(t) -  f(t+h)}_{\Delta_{t, t+h}})
	\end{split}
	\begin{split}\ \left|\ %
		{(L x)}_i = \sum_{j = 1}^{N} w_{i j} \underbrace{(x_i - x_j)}_{\Delta_{i, j}}
	\right.\end{split}
\end{equation*}
The second derivative of a function $f$ essentially averages the differences in signal strength in the neighborhood of a point $t$.
For real-valued functions this neighborhood only consists of the two infinitesimally close points to the left and to the right of $t$, i.e.\ $t - h$ and $t + h$.
The graph Laplacian represents the same operation, where each point/vertex might however have more than two neighbors that need to be averaged.

Based on the graph Laplacian $L$ we just defined, the graph Fourier basis ${\{ u_k \}}_{k = 1}^{N}$ is the set of eigenvectors of $L$.
In the rest of this paper we will assume that these eigenvectors $u_k$ are sorted in ascending order w.r.t.\  their eigenvalues $\lambda_k$, i.e.\  $\lambda_1 \leq \lambda_2 \leq \cdots \leq \lambda_N$.
\Cref{fig:sgt:graphFourier} shows how this definition generalizes the Fourier basis from the real number line to an arbitrary graph.
It also shows that the eigenvalues $\lambda_k$ of the graph Fourier basis encode some notion of frequency, just like they do for the classical Fourier basis.
Analogous to the classical Fourier transform, the eigenvalues of a graph's Laplacian are therefore also called its \textit{spectrum}.
\begin{figure}[ht]
	\centering
	\makebox[\textwidth][c]{\includegraphics[width=\linewidth]{gfx/sgt/graph-fourier.pdf}}
	\caption{%
		Comparison between the basis functions/vectors of the classical Fourier transform and the graph Fourier transform.
		For the eigenfunctions on the upper half only the real cosine components of the complex exponentials are shown.
		\source[based on]{Shuman2013}
	}\label{fig:sgt:graphFourier}
\end{figure}

\subsection{Spectral Properties of Graphs}%
\label{sec:sgt:spectrum}

Next we will give an intuition for the relation between the spectrum of a graph and its structural properties\footnote{%
	Only a general overview will be given. For a more detailed discussion we refer to \citet{Shuman2013}.
}.
For any graph the smallest eigenvalue always is $\lambda_1 = 0$ with the associated eigenvector $u_1 = \frac{1}{\sqrt{N}} {(1, \dots, 1)}^\top$ being a uniform signal over all vertices.
Using the heat analogy, this simply means that any system in which everything has the same temperature is in an equilibrium state and no heat is flowing.
More generally for graphs with $c$ separate connected components the first $c$ eigenvalues are all $0$ since each component can have its own equilibrium temperature without causing heat flow.
If the spectrum of a graph is known, this fact can be used to quickly determine whether a graph is connected or not.

Another meaningful eigenvalue is $\lambda_2$ and its eigenvector $u_2$, also called the \textit{Fiedler value} and \textit{Fiedler vector} respectively.
As we have just seen, a Fiedler value of $\lambda_2 = 0$ means that a graph is not connected.
More generally the Fielder value can be interpreted as a measure of overall graph connectivity\footnote{%
	Formally this measure is called \textit{algebraic connectivity}, as opposed to regular \textit{graph connectivity}, which is defined as $\min_{v_i \in \mathcal{V}} \deg(v_i)$.
}.
If there are two clusters of vertices $\mathcal{V}_+$ and $\mathcal{V}_-$ in a graph that are connected via relatively few edges (compared to the overall number of edges), the Fielder value is small;
for well connected graphs on the other hand the Fiedler value is large.
Using the heat analogy, the Fielder value essentially measures the ``width'' of a bottleneck between two parts of a graph through which heat will only flow very slowly or even not at all in case of $\lambda_2 = 0$.
The partitioning of vertices into $\mathcal{V}_+$ and $\mathcal{V}_-$ can be retrieved via the Fiedler vector $u_2$;
vertices with a positive signal strength in $u_2$ are in $\mathcal{V}_+$, the other vertices are in $\mathcal{V}_-$.
The so called \textit{spectral clustering} algorithm is based on this relationship.
\Cref{fig:sgt:graphFourier} shows how the Fiedler vector $u_2$ partitions a graph via the signs of the vertex signal strengths.

To summarize, expressing a graph $G$ in terms of the spectrum of its Laplacian $L$ gives access to graph characteristics like its overall connectivity.
Similarly to how the low-frequency components of the Fourier transform of a function represent the overall shape of that function, the eigenvectors associated with the small eigenvalues of $L$ represent overall characteristics of $G$.
The details of a function or graph on the other hand are encoded in the high-frequency components of its Fourier transform, i.e.\  the eigenvectors associated with large eigenvalues.
This perspective already hints at how spectral graph analysis might be useful to approximate graphs.


\documentclass[]{article}

\usepackage[utf8]{inputenc}
\usepackage{hyperref}
\usepackage{makecell}
\usepackage{url}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{array}
\usepackage{times,color}
\usepackage[rflt]{floatflt}
\usepackage{epsfig,times,graphicx}
\usepackage{amsmath,amssymb,amsopn,algorithm,algorithmic,theorem,float,bbm,bm,enumerate,color,multirow}
\usepackage{rotating}
\usepackage{array}
%\usepackage{slashbox}
\usepackage{makecell}
\usepackage{multirow}
\usepackage{hhline}
\usepackage{xspace}
\usepackage{mathtools}
\usepackage{bm}

\input{notation}

\title{Projection onto Elastic Net Ball}

\begin{document}

How can we project to an arbitrary norm ball?

\section{Major Lessons}
I think the general way of doing this is to go for proximal operators. Proximal operator of a closed proper convex function $f(\cdot)$ is defined as:
\beq
\prox_f (\v) = \argmin_\x \frac{1}{2} \norm{\v - \x}{2}^2 + f(\x)
\eeq
Note that the objective of the above optimization is strongly convex, so there is a unique minimizer for it.

\subsection{Generalizing Projection}
Proximal operator is a generalization of projection in the following sense. Remember the indicator of a set $\cS$, i.e.,
\beq
\indic_\cS(\x) =
\begin{cases}
	0& \x \in \cS \\
	\infty& \x \notin \cS \\
\end{cases}
\eeq
Then the proximal operator reduces to Euclidian projection:
\beq
\prox_f (\v) = \argmin_{\x\in \cS} \frac{1}{2} \norm{\v - \x}{2}^2 = \Pi_\cS(\v)
\eeq
\subsection{Generalizing Orthogonal Decomposition}
Let $\cS$ and $\cS^\perp$ be two orthogonal subspaces. Then for any vector $\v$ we have:
\beq
\v = \Pi_{\cS}(\v) + \Pi_{\cS^\perp}(\v)
\eeq
We have the same decomposition for closed proper convex $f$ and its convex conjugate $f^*(\y) = \argmin_\x \x^T\y - f(\x)$:
\beq
\v = \prox_f (\v) + \prox_{f^*} (\v)
\eeq
The above result is known as {\bf Moreau Decomposition}.

{\bf Proof:}
For the $\u = \prox_f (\v)$, from the optimality condition we should have:
\beq
\v - \u \in \partial f(\u) \iff \u \in \partial f^*(\v - \u ) \iff (\v - \u) = \prox_{f^*}(\u)
\eeq

Above, we used a useful property which relates subgradient of $f$ and $f^*$:
\beq
\u \in \partial f(\v) \iff \v \in \partial f^*(\u) \iff f^*(\u) + f(\v) = \u^T \v
\eeq

\subsection{Moreau Decomposotion for a Norm}
Here we first need to following for an arbitrary norm $f(\x) = \norm{\x}{}$:
\beq
f^*(\y) = \indic_{\norm{\cdot}{*} \leq 1}(\y)
\eeq
where $\norm{\cdot}{*}$ is the daul norm. In other words, the convex conjugate of a norm is the indicator function of its dual norm.

From the above equality and the Moreau decomposition we get:
\be
\v
&= \prox_{\norm{\cdot}{}} + \prox_{\indic_{\norm{\cdot}{*} \leq 1}(\cdot)}
\\
&= \prox_{\norm{\cdot}{}} + \Pi_{\indic_{\norm{\cdot}{*} \leq 1}} (\v)
\ee
Above equality gives us a way to switch between proximal operator $\prox$ and projection $\Pi$.

\end{document}


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

\usepackage{graphicx}

\title{MATH 505B Homework 4}
\author{Saket Choudhary\\skchoudh@usc.edu}

\begin{document}
\maketitle

\section*{Problem 6.14.1}

\begin{align*}
\langle\ \mathbf{x}, \mathbf{Py} \rangle &= \sum_{k\in \theta} x_k (\mathbf{Py})_k\pi_k \\
&= \sum_{k\in \theta} x_k (\sum_jp_{kj}y_j)\pi_k\\
&= \sum_{k\in \theta} x_k (\sum_jp_{kj}\pi_ky_j)\\
&= \sum_{k,j} x_k p_{kj}\pi_ky_j\\
&= \sum_{k,j} x_k (p_{jk}\pi_jy_j) \text{ using reversibility criterion  } \pi_jp_{jk} = \pi_kp_{kj}\\
&= \sum_{j} p_{jk}x_k \pi_jy_j\\
&= \sum_{j \in \theta} (\sum_k p_{jk}x_k) \pi_jy_j\\
&= \langle \mathbf{Px,y} \rangle
\end{align*}

\section*{Problem 6.14.2}
For reversibility: $\pi_ip_{ij} = \pi_jp_{ji}$:

$p'_{ij}= b_{ij}g_{ij} = \frac{\pi_jg_{ji}g_{ij}}{\pi_ig_{ij}+\pi_jg_{ji}}$ and $p'_{ji} = \frac{\pi_ig_{ij}g_{ij}}{\pi_ig_{ij}+\pi_jg_{ji}}$

Hence, $\pi_ip'_{ij} = \pi_jp'_{ji}$ and hence $b_{ij}$ satisfies reversibility criterion besides $ 0 \leq b_{ij} \leq 1$.

\section*{Problem 7.2.1}
\subsection*{7.2.1(a)}

\begin{align*}
\{E(|X+Y|^p)\}^{1/p} &= \{E|X^p|\}^{1/p} + \{E|Y^p|^{1/p}\}\\
E[|X|] &= E[|X_n+X-X_n|]\\
\{E[|X|^p]\}^{1/p} &= \{E[|X_n+X-X_n|^p]\}^{1/p}\\
&\leq \{E[|X_n|^p]\}^{1/p}+\{E[|X-X_n|^p]\}^{1/p}\\
\implies \lim_{n \longrightarrow \infty} \inf E[|X|^p] &\leq E[|X_n|^p]^{1/p}\numberthis \label{eqn1} \\
\end{align*}

Similarly,

\begin{align*}
\{E[|X_n|^p]\}^{1/p} &= \{E[|X_n-X+X|^p]\}^{1/p}\\
&\leq \{E[|X|^p]\}^{1/p} + \{E[|X_n-X|]\}^{1/p}\\
\implies  \lim_{n \longrightarrow \infty} \sup E[|X_n|^p] &\leq E[|X|^p] \numberthis \label{eqn2}
\end{align*}

Combining \ref{eqn1}, \ref{eqn2} : $E[|X_n|^p] \longrightarrow
E[|X|^p]$ $p \geq 1$

\subsection*{7.2.1(b)}
Using $p=1$ in part (a)
\subsection*{7.2.1(c)}
Using part (a) $E[X_n^2] \longrightarrow E[X^2]$
$(X_n \xrightarrow{2} X) \Rightarrow (X_n \xrightarrow{1} X)\ \implies E[X_n] \longrightarrow E[X]$ and hence $Var(X_n) \longrightarrow Var(X)$

\section*{Problem 7.2.3}

Consider $k \geq 0$, $ n\geq 1$,  $X_n=k/n \leq X < (k+1)/n$.
$X-1/n \leq X_n \leq X$

Define similarly $Y_n$. $Y_n,X_n$ are independent by definition

$E[X_n] \longrightarrow E[X]$ and $E[Y_n] \longrightarrow E[Y]$

Thus, using independence and convergence relations $E[X_nY_n] = E[X_n]E[Y_n] \longrightarrow E[X]E[Y] $

Now,
\begin{eqnarray*}
(X-1/n)(Y-1/n)\leq X_nY_n \leq XY\\
\implies E[(X-1/n)(Y-1/n)]\leq E[X_nY_n] \leq E[XY]\\
E[(X-1/n)(Y-1/n)] = E[XY-\frac{X+Y}{n}+1/n^2]
\implies E[X_nY_n] \longrightarrow E[XY]
\end{eqnarray*}

Thus combining, the above two results $E[X_nY_n] \longrightarrow E[XY]$ and $E[X_nY_n] \longrightarrow E[X][Y]$
we get $E[XY]=E[X]E[Y]$

\section*{Problem 7.2.10}

$\sum_r X_r \sim Poisson(\sum_r \lambda_r)$

Define $t=\sum_{r=1}^n \lambda_r$:
\begin{align*}
P(\sum_r X_r \leq x) &= \sum_{i=0}^x \frac{e^{-t}t^i}{i!}\\
\lim_{n \longrightarrow \infty}P(\sum_r X_r \leq x)  &= \begin{cases} 0 & t\longrightarrow \infty\\
Poisson(t) & t \text{is finite}
\end{cases}
\end{align*}

\section*{Problem 7.4.1}

\begin{align*}
E[X_1] &= 0*(1-\frac{1}{n\log{n}} + 0*\frac{1}{2n\log n}\\
&=0\\
E[X_1^2] &= \frac{2n^2}{2n\log{n}}+0*(1-\frac{1}{n\log{n}}\\
&= \frac{n}{\log{n}}\\
E[(\frac{1}{n}S_n-0)^2] &= \frac{1}{n^2}Var(S_n)\\
&= \frac{1}{n^2}\frac{n}{\log{n}}\\
&= \frac{1}{n \log{n}}
&\longrightarrow 0
\end{align*}

$\sum_i P(|X_i| \geq i)
\longrightarrow \infty$ Hence, using Borel-Cantelli Lemma(7.3.10b)
we have $P(|X_j| \geq j) =1$ for some $j$
$|X_j| = |S_j-S_{j-1}| \geq j$ and hence $S_j/j$ diverges.

\section*{Problem 7.5.1}
Define $I_{i}(j)$ as the indicator variable denoting if the $X_j$ lies in the $i^{th}$ interval,
\begin{align*}
\log{R_m} &= \sum_{i=1}^n Z_m(i) \log p_i\\
&= \sum_{i=1}^n \sum_{j=1}^m I_i(j) p_i\\
\end{align*}
Define $\sum_{i=1}^m I_i(j)=Y_j$, then $\log{R_m} = \sum_{j=1}^m Y_j$

$E[Y_j]=\sum_{i=1}^np_i\log{p_i}=-h$
Thus, by strong law of convergence $\frac{1}{m}\sum_{j=1}^m Y_j = \ \longrightarrow -h=E[Y_j]$

\section*{Problem 7.5.3}
Transient $P(X_n=i|X_0=i)< 1$

Using strong law $S_n/n \longrightarrow E[X_1]$ If $E[X_1]\neq 0$ then $P[S_n=0|S_1=0] < 1$ as $S_n=0$ happens only finitely often

\section*{Problem 7.7.1}
\begin{align*}
E[X_iX_j] &= E[E[X_iX_j|X_0,X_1,\dots, X_{j-1}]]\\
&= E[E[X_i(S_j-S_{j-1})|X_0,X_1,\dots,X_{j-1}]]\\
&= E[X_i(E[S_j-S_{j-1}|X_0,X_1,\dots,X_{j-1}])]\\
&= E[X_i(E[S_j|X_0,X_1,\dots,X_{j-1}]-S_{j-1})]\\
&= E[X_i(S_{j-1}-S_{j-1})]\\
&=0
\end{align*}

\section*{Problem 7.7.3}
\begin{align*}
E[X_{n+1}|X_0,X_1,\dots,X_n] &= aX_n+X_{n-1} \\
E[S_{n+1}|X_0,X_1,\dots, X_n] &= E[\alpha X_{n+1}+X_{n}|X_0,X_1,\dots,X_n]\\
&= \alpha E[X_{n+1}|X_0,X_1,\dots, X_n] + X_n\\
&= (\alpha a+1)X_n + \alpha bX_{n-1}\\
&= S_n = \alpha X_n+X_{n-1}\\
\implies \alpha = \frac{1}{1-a}, b=\frac{1}{\alpha}
\end{align*}

\section*{Problem 7.7.4}
$X_n$: Net profit per unit stake on $n^{th}$ play.

$S_{i} = S_{i-1}+f_{i}(X_1,X_2,\dots,X_i)$
such that $S_1=X_1Y$

Thus, $S_{n} = \sum_{i=1}^n X_if_{i-1}(X_1,X_2,\dots, X_{i-1})$

\begin{align*}
S_{n+1} &=  S_n + f_{n+1}(X_1,X_2,\dots, X_n)X_{n+1}\\
E[S_{n+1}-S_n|X_1,X_2,\dots,X_n] &= E[X_{n+1}f_{n+1}(X_1,X_2,\dots, X_n)|X_1,X_2,\dots,X_n]\\
&= f_{n+1}(X_1,X_2,\dots, X_n)E[X_{n+1}|X_1,X_2,\dots,X_n]\\
&= 0\\
\implies E[S_{n+1}|X_1,X_2,\dots,X_n] &= S_n
\end{align*}

\section*{Problem 7.8.1}
$E[X_i]=0$
By Doob-Kolmogorov inquality:

\begin{align*}
P(\max_{i\leq j \leq n} |S_j| > \epsilon) &\leq \frac{1}{\epsilon^2} \sum_{j=1}^{n}E[S_n^2]\\
E[S_n^2] &= Var(S_n)+E[S_n]^2\\
&= Var(S_n)\\
&= \sum Var(X_i)\\
\implies P(\max_{i\leq j \leq n} |S_j| > \epsilon) \leq \frac{1}{\epsilon^2} \sum_{j=1}^{n}Var(X_j)
\end{align*}

\section*{Problem 7.8.3}
By theorem 7.8.1 $S_n$ converges to $S$ almost surely
Now, using the above proved fact that $S_n \longrightarrow S \implies Var(S_n) \longrightarrow Var(S) \implies Var(S) \longrightarrow 0$

\end{document}


\section*{Appendix}
\begingroup
\def\thetheorem{\ref{lem:decrease}}
\begin{lemma}
If $f_i$ is continuously differentiable and log-concave, then the functions $l_i,d_i,v_i$ are continuous, strictly decreasing, and
\[l_i(p) < d_i(p) < v_i(p) \text{ for all }p.\]
\end{lemma}
\addtocounter{theorem}{-1}
\endgroup
\begin{proof}
Continuity of $F_i,f_i,f'_i$ implies that of $l_i,d_i,v_i$. It's known~\cite{concave} that log-concavity of $f_i$ implies log-concavity of both $F_i$ and $1-F_i$. As a result, $l_i$, $d_i$, and $v_i$ are derivatives of strictly concave functions; therefore, they are strictly decreasing. In particular, each of

\[v'_i(p) = \frac{f'_i(p)}{F_i(p)} - \frac{f_i(p)^2}{F_i(p)^2},\quad
l'_i(p) = \frac{-f'_i(p)}{1-F_i(p)} - \frac{f_i(p)^2}{(1-F_i(p))^2},\]

are negative for all $p$, so we conclude that

\begin{align*}
d_i(p) - v_i(p)
= \frac{f'_i(p)}{f_i(p)} - \frac{f_i(p)}{F_i(p)}
&= \frac{F_i(p)}{f_i(p)} v'_i(p)
< 0,
\\l_i(p) - d_i(p)
= -\frac{f'_i(p)}{f_i(p)} -\frac{f_i(p)}{1-F_i(p)}
&= \frac{1-F_i(p)}{f_i(p)} l'_i(p)
< 0.
\end{align*}

\end{proof}

\begingroup
\def\thetheorem{\ref{thm:uniq-max}}
\begin{theorem}
Suppose that for all $j$, $f_j$ is continuously differentiable and log-concave. Then the unique maximizer of $\Pr(P_i=p\mid E^L_i,E^W_i)$ is given by the unique zero of
\[Q_i(p) = \sum_{j \succ i} l_j(p) + \sum_{j \sim i} d_j(p) + \sum_{j \prec i} v_j(p).\]
\end{theorem}
\addtocounter{theorem}{-1}
\endgroup

\begin{proof}
First, we rank the players by their buckets according to $\floor{P_j/\epsilon}$, and take the limiting probabilities as $\epsilon\rightarrow 0$:
\begin{align*}
    \Pr(\floor{\frac{P_j}\epsilon} > \floor{\frac{p}\epsilon})
    &= \Pr(p_j \ge \epsilon\floor{\frac{p}\epsilon} + \epsilon)
    \\&= 1 - F_j(\epsilon\floor{\frac{p}\epsilon} + \epsilon)
    \rightarrow 1 - F_j(p),
    \\\Pr(\floor{\frac{P_j}\epsilon} < \floor{\frac{p}\epsilon})
    &= \Pr(p_j < \epsilon\floor{\frac{p}\epsilon})
    \\&= F_j(\epsilon\floor{\frac{p}\epsilon})
    \rightarrow F_j(p),
    \\\frac 1\epsilon \Pr(\floor{\frac{P_j}\epsilon} = \floor{\frac{p}\epsilon})
    &= \frac 1\epsilon \Pr(\epsilon\floor{\frac{p}\epsilon} \le P_j < \epsilon\floor{\frac{p}\epsilon} + \epsilon)
    \\&= \frac 1\epsilon\left( F_j(\epsilon\floor{\frac{p}\epsilon} + \epsilon) - F_j(\epsilon\floor{\frac{p}\epsilon}) \right)
    \rightarrow f_j(p).
\end{align*}

Let $L_{jp}^\epsilon$, $W_{jp}^\epsilon$, and $D_{jp}^\epsilon$ be shorthand for the events $\floor{\frac{P_j}\epsilon} > \floor{\frac{p}\epsilon}$, $\floor{\frac{P_j}\epsilon} < \floor{\frac{p}\epsilon}$, and $\floor{\frac{P_j}\epsilon} = \floor{\frac{p}\epsilon}$. respectively. These correspond to a player who performs at $p$ losing, winning, and drawing against $j$, respectively, when outcomes are determined by $\epsilon$-buckets. Then,
\begin{align*}
\Pr(E^W_i,E^L_i\mid P_i=p)
&= \lim_{\epsilon\rightarrow 0}
\prod_{j \succ i} \Pr(L_{jp}^\epsilon)
\prod_{j \prec i} \Pr(W_{jp}^\epsilon)
\prod_{j \sim i, j\ne i} \frac{\Pr(D_{jp}^\epsilon)}\epsilon
\\&= \prod_{j \succ i} (1 - F_j(p)) \prod_{j \prec i} F_j(p) \prod_{j \sim i, j\ne i} f_j(p),
\\\Pr(P_i=p \mid E^L_i,E^W_i)
&\propto f_i(p) \Pr(E^L_i,E^W_i\mid P_i=p)
\\&= \prod_{j \succ i} (1 - F_j(p)) \prod_{j \prec i} F_j(p) \prod_{j \sim i} f_j(p),
\\\ddp\ln \Pr(P_i=p \mid E^L_i,& E^W_i) = \sum_{j \succ i} l_j(p) + \sum_{j \prec i} v_j(p) + \sum_{j \sim i} d_j(p) = Q_i(p).
\end{align*}

Since \Cref{lem:decrease} tells us that $Q_i$ is strictly decreasing, it only remains to show that it has a zero. If the zero exists, it must be unique and it will be the unique maximum of $\Pr(P_i=p \mid E^L_i,E^W_i)$.

To start, we want to prove the existence of $p^*$ such that $Q_i(p^*) < 0$. Note that it's not possible to have $f'_j(p) \ge 0$ for all $p$, as in that case the density would integrate to either zero or infinity. Thus, for each $j$ such that $j\sim i$, we can choose $p_j$ such that $f'_j(p_j) < 0$, and so $d_j(p_j) < 0$. Let $\alpha = -\sum_{j\sim i} d_j(p_j) > 0$.

Let $n = |\{j:\,j \prec i\}|$. For each $j$ such that $j \prec i$, since $\lim_{p\rightarrow\infty}v_j(p) = 0/1 = 0$, we can choose $p_j$ such that $v_j(p_j) < \alpha/n$. Let $p^* = \max_{j\preceq i} p_j$. Then,
\[
\sum_{j \succ i} l_j(p^*) \le 0, \quad \sum_{j \sim i} d_j(p^*) \le -\alpha, \quad \sum_{j \prec i} v_j(p^*) < \alpha.
\]

Therefore,
\begin{align*}
Q_i(p^*)
&= \sum_{j \succ i} l_j(p^*) + \sum_{j \sim i} d_j(p^*) + \sum_{j \prec i} v_j(p^*)
\\&< 0 - \alpha + \alpha = 0.
\end{align*}

By a symmetric argument, there also exists some $q^*$ for which $Q_i(q^*) > 0$. By the intermediate value theorem with $Q_i$ continuous, there exists $p\in (q^*,p^*)$ such that $Q_i(p) = 0$, as desired.
\end{proof}