\documentclass[a4paper,11pt]{article}
\usepackage{graphicx}

\begin{document}

\begin{flushright}

\vspace{1.1cm}

%Put ISR Title
{\bf\Huge Problem Set 5}

\rule{0.25\linewidth}{0.5pt}

\vspace{0.5cm}
%Put Authors
Justin Ely
\linebreak
\newline
%Put Author's affiliations
\footnotesize{AST615 University of Maryland College Park, MD\\}
\vspace{0.5cm}
% Date here below
01 December, 2011
\end{flushright}

\noindent\rule{\linewidth}{1.0pt}
\section*{Problem 1}
Figures 1,2, and 3 show the integration of $\frac{dx^2}{dt^2}+x=0$ using the Euler, RK4, and Leapfrog methods for various stepsizes.  The integrations are plotted with the analytical solution, $sin(t)$, and the plots include a loglog plot of the difference between the analytical and numerical solutions aa function of stepsize.

From the plots showing the numerical and exact solutions, we can see that the Euler method can get very wrong very quickly, and that error compounds as time goes on.  The RK4 and Leapfrog methods, however, appear to have small, non-compounding errors even with larger stepsizes.

The error of these 3 methods appear to have an exponentially decreasing relationship with stepsize, where the error with RK4 method is the smallest and the Euler method being the largers.  It is also interesting that the Euler method gets a smaller decrease in error with each smaller stepsize, Leapfrog is fairly stable in loglog space, and the gain gets much larger with each stepsize for the RK4 method.

\section*{Problem 2}
Given a 2D orbit described by the potential $\Phi=\frac{-1}{\sqrt{1+2x^2+2y^2}}$ and $F=-\frac{d\Phi}{dx}$, by the chain rule $F_x=ma_x=\frac{2x}{(1+2x^2+2y^2)^{3/2}}$.  Using unit mass, and repeating the procedure for the y component we get the equations:
\begin{equation}
\frac{d^2x}{dt^2}=\frac{2x}{(1+2x^2+2y^2)^{3/2}}
\end{equation}
\begin{equation}
\frac{d^2y}{dt^2}=\frac{2y}{(1+2x^2+2y^2)^{3/2}}
\end{equation}

These then reduce down to 4 coupled first order equations:
\begin{eqnarray}
  \frac{dx}{dt}=Z_x(t) \\
  \frac{dZ_x(t)}{dt}=\frac{-2x}{(1+2x^2+2y^2)^{3/2}}\\
  \frac{dy}{dt}=Z_y(t) \\
  \frac{dZ_y(t)}{dt}=\frac{-2y}{(1+2x^2+2y^2)^{3/2}}
\end{eqnarray}

In Figures 4 and 5, plots of X vs Y, and Energy vs Time can be seen for both the RK4 method and the Leapfrog method for stepsizes of 1,.5,.25, and .01.  The differences between the two methodes becomes very apparent when comparing these plots.  The stability of the leapfrog system keeps the body in an clean orbit, becoming more circular with smaller stepsizes, where the RK4 method has a very complicated orbit that appears to degrade with time.  This same trend is seen in the plots of Energy vs Time where the RK4 method shows the energy of the system falling to 0 with the larger stepsizes, whereas the Energy of the Leapfrog system oscillates around a nominal value even with large stepsizes.

\section*{Problem 3}
Figure 6 shows a phase-plane plot produced with the RK4 method and stepsizes of 1,.5,.25, and .1.  In this phase-plane, A=1,B=.1,C=1.5,D=.03,d=e=0.  If we use a q$\sim$1.25, both populations drop below $10^{-9}$ before t=100, although the wolves drop below much quicker.

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=.6]{Euler_int.pdf}
\caption{Euler integration for various stepsizes.  Also shown is a loglog plot of the difference between the numerical and exact solutions at the last point as a function of stepsize.}
\end{center}
\end{figure}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=.6]{RK4_int.pdf}
\caption{RK4 integration for various stepsizes.  Also shown is a loglog plot of the difference between the numerical and exact solutions at the last point as a function of stepsize.}
\end{center}
\end{figure}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=.6]{Leapfrog_int.pdf}
\caption{Leapfrog integration for various stepsizes.  Also shown is a loglog plot of the difference between the numerical and exact solutions at the last point as a function of stepsize.}
\end{center}
\end{figure}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=.6]{RK4_orbit_int.pdf}
\caption{RK4 orbit integration for various stepsizes.  Also shown for each stepstize is a plot of the calculated enerygy of the system with time.}
\end{center}
\end{figure}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=.6]{Leapfrog_orbit_int.pdf}
\caption{Leapfrog orbit integration for various stepsizes.  Also shown for each stepstize is a plot of the calculated enerygy of the system with time.}
\end{center}
\end{figure}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale=.7]{Phase_plane.pdf}
\caption{Phase plane diagram for Lotka-Volterra Predadtor-Prey Model where A=1,B=.1,C=1.5,D=.03,d=e=0.  RK4 integrated phase planes are shown for different timesteps.}
\end{center}
\end{figure}
\end{document}


\lab{PageRank Algorithm}{PageRank Algorithm}
\label{lab:PageRank}
\objective{Model a network as a graph and implement the PageRank algorithm based on this model.
Use PageRank to predict the rankings of sports teams. }

\begin{comment}
When you enter keywords into Google's search engine, Google finds every page containing your keywords and lists the pages in order of their \emph{rank}.
The rank of a page reflects many factors, including how often the page is visited and how connected it is to other pages.
\end{comment}
As of 2013, the PageRank algorithm is one of over 200 algorithms that Google uses to determine the \emph{rank}, or relative importance, of a webpage.
Named for Larry Page, cofounder of Google, this algorithm ranks pages based on how many other pages link to them.

\begin{comment}
The PageRank algorithm is also used in applications other than internet search engines.
For example, it has been used to rank graduate institutions and the impact factor of journals, and it has been used in some biological applications.
\end{comment}

\section*{The Internet as a Graph}
The PageRank algorithm models the internet with a directed graph.
Each webpage is a node, and there is an edge from node $i$ to node $j$ if page $i$ links to page $j$.
Let $\In(i)$ be the websites linking to page $i$ and let $\Out(i)$ be the websites that page $i$ links to.
That is, $\In(i)$ is the set of nodes with an arrow to node $i$, and $\Out(i)$ is the set of nodes with an arrow from node $i$.
An example is illustrated in Figure \ref{fig:network1}.

\begin{figure}
\centering
\begin{tikzpicture}[node distance=1.75cm, thick ]

\node[draw=none](2)[]{2};
\node[draw=none](3)[right of=2]{3};
\node[draw=none](4)[right of=3]{4};
\node[draw=none](5)[right of=4]{5};
\node[draw=none](6)[right of=5]{6};
\node[draw=none](1)[above of=3]{1};
\node[draw=none, node distance=2.5cm](0)[right of=1]{0};
\node[draw=none](dummy)[above right of=0]{};
\node[draw=none, node distance=.5cm](7)[below
	of=dummy]{7};

\foreach \x/\y in {3/2, 4/5, 5/6, 1/0, 3/0, 4/0, 5/0} \draw[->,
	>=stealth'](\x)--(\y);
\draw[->, >=stealth'](6)--(0);
\draw[->, >=stealth', shorten >= .1cm](7)edge[bend left=20](0);
\draw[->, >=stealth', shorten <= .1cm](0)edge[bend left=20](7);
\draw[->, >=stealth', shorten <= .1cm](3)edge[bend right=40](6.9,-.25);
\draw[->, >=stealth', shorten <= .1cm](4)edge[bend right](6);

\end{tikzpicture}

\caption{This directed graph describes the links between 8 webpages. In this example, $\In(0)=\{1,3,4,5,6,7\}$ and $\Out(0)=\{7\}$.}
\label{fig:network1}
\end{figure}

The PageRank algorithm ranks pages by how many others link to them.
A link from a more important page counts more than one from a less important page.
For example, in Figure \ref{fig:network1} we would expect node 0 to have a very high rank because every other node links to it.
Consequently, we would expect node 7 to have a fairly high rank because node 0 links to it, even though node 0 is the only node to do so.

\section*{The PageRank Algorithm}
The PageRank algorithm assumes that a surfer chooses a starting webpage randomly.
Then, if the surfer is at page $i$, they randomly select a page from $\Out(i)$ to visit next.
This means that the surfer's chance of being on page $i$ at time $t$ is determined by where they were at time $t-1$.

Suppose the internet has $N$ webpages, and let $p_i(t)$ be the likelihood that the surfer is on page $i$ at time $t$.
Then the probabilities $p_i(t)$ are given by
\begin{equation}\label{equ:pr1}
p_i(0)=\frac{1}{N} \qquad p_i(t+1) = \sum_{j \in \In(i)} \frac{p_j(t)}{\abs{\Out(j)}}.
\end{equation}

For example, in Figure \ref{fig:network1} we have $N=8$, and
\[
p_6(t+1)=\frac{p_3(t)}{3}+\frac{p_4(t)}{3} + \frac{p_5(t)}{2}.
\]

\subsection*{Refining the Model: Pages with No Outbound Links}
A node with no outbound links, such as node 2 in Figure \ref{fig:network1}, is called a \emph{sink}.
According to our model, if the surfer ever visits a sink, they will stay there forever.

This is not very realistic; in this situation, a person would likely select another webpage at random and begin surfing again.
Hence, in our model we replace sinks with nodes linking to every other page.
This means we modify Figure \ref{fig:network1} (where node 2 is a sink) to look like Figure \ref{fig:network2}.

\begin{figure}
\centering
\begin{tikzpicture}[node distance=1.75cm, >=stealth', thick]

\node[draw=none](2)[]{2};
\node[draw=none](3)[right of=2]{3};
\node[draw=none](4)[right of=3]{4};
\node[draw=none](5)[right of=4]{5};
\node[draw=none](6)[right of=5]{6};
\node[draw=none](1)[above of=3]{1};
\node[draw=none, node distance=2.5cm](0)[right of=1]{0};
\node[draw=none](dummy)[above right of=0]{};
\node[draw=none, node distance=.5cm](7)[below
	of=dummy]{7};

\draw[->, color=black!35!](2)--(1);
\draw[->, color=black!35!, shorten >= .2cm](2)--(0);
\foreach \x/\y in {2/3, 2/4, 2/5} \draw[->, color=black!35!](\x)
	edge[bend right](\y);

\draw[->, shorten <= .1cm, color=black!35!](2)edge[bend right=50](7,-.35);
\draw[->, shorten <= .1cm](3)edge[bend right=40](6.9,-.25);
\draw[->, shorten <= .1cm](4)edge[bend right](6);
\draw[->, shorten <=.1cm, color=black!35!](2)edge[bend left=40](7);

\draw[->, shorten >= .1cm](7)edge[bend left=20](0);
\draw[->, shorten <= .1cm](0)edge[bend left=20](7);

\foreach \x/\y in {3/2, 4/5, 5/6, 1/0, 3/0, 4/0, 5/0} \draw[->,
	>=stealth'](\x)--(\y);
\draw[-> ](6)--(0);

\end{tikzpicture}

\caption{Here Figure \ref{fig:network1} has been modified to guarantee that page 2 is no longer a sink. A new link has been added from page 2 to every other page (the added links are grey).}
\label{fig:network2}
\end{figure}

\subsection*{Refining the Model: Adding Boredom}
The equations in \eqref{equ:pr1} assume that the current page must link to the next page.
However, the model is more realistic if we assume that the surfer sometimes gets bored and randomly picks a new starting page.
We will denote the probability that a surfer stays interested at step $t$ by a constant $d$, called the \emph{damping factor}.
Then the probability that the surfer gets bored at time $t$ is $1-d$.
The formulas in \eqref{equ:pr1} then become
\begin{equation}\label{equ:pr2}
p_i(0)=\frac{1}{N} \qquad p_i(t+1) = d\sum_{j \in \In(i)} \frac{p_j(t)}{\abs{\Out(j)}}+\frac{1-d}{N} .
\end{equation}

\subsection*{Matrix Form of the PageRank Algorithm}
We can rewrite \eqref{equ:pr2} as the matrix equation
\begin{equation}\label{equ:pr3}
\mathbf{p}(0)=\frac{1}{N}\mathbf{1} \qquad \mathbf{p}(t+1) = dK\mathbf{p}(t) + \frac{1-d}{N}\mathbf{1}
\end{equation}
where $\mathbf{p}(t)=(p_1(t), p_2(t), \ldots, p_N(t))^T$,
$\mathbf{1}$ is a vector of $N$ ones,
and $K$ is defined by
\[K_{ij} = \begin{cases} \frac{1}{\abs{\Out(j)}} & \mbox{ if j links to i} \\
	0 & \mbox{ otherwise.} \end{cases}\]

\subsection*{Defining Page Rank}
As given by the PageRank algorithm, the \emph{rank} of page $i$ is
\[p_i = \lim_{t\to \infty} p_i(t).\]
In other words, the page ranks are the steady state of the modified Markov chain defined in \eqref{equ:pr3}.

\section*{Implementation in Python}

The adjacency matrix $A$ of a directed graph has $A_{ij}=1$ if there is an edge from node $i$ to node $j$, and $A_{ij}=0$ otherwise.
The adjacency matrix of the graph in Figure \ref{fig:network1} is defined below.
We use a code environment to describe $A$ so you can easily use this example to debug the problems in this lab.
\begin{lstlisting}
A = np.array([[ 0,  0,  0,  0,  0,  0,  0,  1],
              [ 1,  0,  0,  0,  0,  0,  0,  0],
              [ 0,  0,  0,  0,  0,  0,  0,  0],
              [ 1,  0,  1,  0,  0,  0,  1,  0],
              [ 1,  0,  0,  0,  0,  1,  1,  0],
              [ 1,  0,  0,  0,  0,  0,  1,  0],
              [ 1,  0,  0,  0,  0,  0,  0,  0],
              [ 1,  0,  0,  0,  0,  0,  0,  0]])
\end{lstlisting}

\begin{problem}
Write the following function that creates a sparse adjacency matrix from a file.
\begin{lstlisting}
def to_matrix( filename, n ):
    ''' Return the nxn adjacency matrix described by the file.

    INPUTS:
    filename - Name of a .txt file describing a directed graph. Lines
    		    describing edges should have the form
				'<from node>\t<to node>'.
				The file may also include comments.
    n		- The number of nodes in the graph described by datafile

    RETURN:
    Return a SciPy sparse `dok_matrix'.
    '''
\end{lstlisting}
Hints:
\begin{enumerate}
\item The file \texttt{datafile.txt} included with this lab describes the matrix in Figure \ref{fig:network1} and has the adjacency matrix \li{A} given above.
You may use it to test your function.

\item You can open a file in Python using the \li{with} syntax.
Then, you can iterate through the lines using a \li{for} loop.
Here is an example.
\begin{lstlisting}
# Open `datafile.txt' for read-only
with open('./datafile.txt', 'r') as myfile:
    for line in myfile:
        print line
\end{lstlisting}

\item Here is an example of how to process a line of the form in \li{datafile}.
\begin{lstlisting}
>>> line = '0\t4\n'
# strip() removes trailing whitespace from a line.
# split() returns a list of the space-separated pieces of the line.
>>> line.strip().split()
['0', '4']
\end{lstlisting}

\item Rather than testing for lines of \texttt{datafile.txt} that contain comments, put all your string operations in a \li{try} block with an \li{except} block following.
\end{enumerate}
\end{problem}

\begin{info}
It makes sense to initialize $A$ as a sparse matrix, since $A$ is mostly zeros. To make the coding easier, throughout the rest of the lab the algorithms will be coded using non-sparse matrices. Don't forget, however, that in a real-world sparse adjacency matrices are generally much more time-efficient than dense matrices.

To convert from a sparse to a non-sparse matrix, use the syntax \li{A.todense()}.
\end{info}

The next step is to compute $K$ from \eqref{equ:pr3}.
A good strategy for computing $K$ comes from writing
\[
K = (D^{-1}A)^T
\]
where $A$ is the adjacency matrix of the directed graph representing the internet and $D$ is a diagonal matrix with $D_{jj}=\abs{\Out(j)}$.
Modify $A$ so that rows corresponding to sinks have all ones instead of all zeros.
For Figure \ref{fig:network2}, the modified adjacency matrix is defined below.
\begin{lstlisting}
Am = np.array([[ 0,  0,  0,  0,  0,  0,  0,  1],
               [ 1,  0,  0,  0,  0,  0,  0,  0],
               [ 1,  1,  1,  1,  1,  1,  1,  1],
               [ 1,  0,  1,  0,  0,  0,  1,  0],
               [ 1,  0,  0,  0,  0,  1,  1,  0],
               [ 1,  0,  0,  0,  0,  0,  1,  0],
               [ 1,  0,  0,  0,  0,  0,  0,  0],
               [ 1,  0,  0,  0,  0,  0,  0,  0]])
\end{lstlisting}

The matrix $D$ is easily obtained by summing the rows of $A$.
Although  $K=(D^{-1}A)^T$, it is better practice to only store the diagonal entries of $D$ as a vector, and then use array broadcasting to divide $A$ by $D$.

Notice that we need to transpose $D^{-1}A$ to get $K$. This is because $D^{-1}A$ is \emph{row stochastic} (meaning that  the rows sum to 1), but we need to multiply \emph{column stochastic} matrices (where the columns sum to 1) To make $K$ be column stochastic, we have to take a transpose.

For Figure \ref{fig:network2}, the matrix $K$ is as follows.

\begin{lstlisting}
K = np.array([[ 0   ,  1   ,  1./8,  1./3,  1./3,  1./2,  1   ,  1   ],
              [ 0   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  1./3,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  0   ,  1./3,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  1./3,  1./3,  1./2,  0   ,  0   ],
              [ 1   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ]])
\end{lstlisting}
\begin{problem}
Write a function that computes the K matrix given an adjacency matrix.
\begin{enumerate}
\item Compute the diagonal matrix $D$.
\item Compute the modified adjacency matrix where the rows corresponding to sinks all have ones instead of zeros.
\item Compute $K$ using array broadcasting.
\end{enumerate}
\end{problem}

\subsection*{Solving for the Page Ranks}
There are several ways to solve for $\lim_{t \to \infty} \mathbf{p}(t)$.
\subsubsection*{Algebraic Method}
One possibility is to assume the modified Markov chain has a steady state $\mathbf{p}$ and solve for it algebraically:
\begin{equation}\label{equ:matrix_solve}
(I-dK)\mathbf{p} = \frac{1-d}{N} \mathbf{1}.
\end{equation}

We can use SciPy's solver to find the page ranks of the network in Figure \ref{fig:network2}.
\begin{lstlisting}
>>> from scipy import linalg as la
>>> I = np.eye(8)
>>> d = .85
>>> la.solve(I-d*K, ((1-d)/8)*np.ones(8))
array([ 0.43869288,  0.02171029,  0.02786154,  0.02171029,  0.02171029,
        0.02786154,  0.04585394,  0.39459924])
\end{lstlisting}
As expected, node 0 has the highest rank, approximately equal to .44.
Node 7 has a higher rank than node 6, even though $\In(7)=1$ and $\In(6)=3$.
This is because node 7's single in-edge comes from a node that has a very high rank (node 0).

\subsubsection*{Iterative Method}
Solving the system in \eqref{equ:matrix_solve} is feasible for our small working example, but this is not an efficient strategy for very large systems.

One option for large systems is an iterative method.
Starting with a guess for $\mathbf{p}(0)$, we iterate on Equation \eqref{equ:pr3} until $\norm{\mathbf{p}(t)-\mathbf{p}(t-1)}$ is sufficiently small.
At this point we assume we have reached the steady state.

\begin{problem}
\label{prob:pagerank_dense_iter}
Implement the function below, using the iterative method to find the steady state of the PageRank algorithm.
When the argument \li{N} is not \li{None}, work with only the upper $N \times N$ portion of the array \li{adj}.
Test your function against the example in the lab.
\begin{lstlisting}
def iter_solve( adj, N=None, d=.85, tol=1E-5):
    '''
    Return the page ranks of the network described by 'adj' using the iterative method.

    INPUTS:
    adj - A NumPy array representing the adjacency matrix of a directed
            graph
    N     - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.
    tol  - Stop iterating when the change in approximations to the
            solution is less than `tol'. Defaults to 1E-5.

    OUTPUTS:
    Return the approximation to the steady state of p.
    '''
\end{lstlisting}
Hints:
\begin{enumerate}
\item Try making your initial guess for $\mathbf{p}(0)$ a random vector.
\item NumPy can do unexpected things with the dimensions when performing matrix-vector multiplication.
When debugging, check at each iteration that all arrays have the dimensions you expect.
\end{enumerate}
\end{problem}

\subsubsection*{Eigenvalue Method}
Another way to solve this problem is to make it into an eigenvalue problem.
Let $E$ be an $N \times N$ matrix of ones; then $E\mathbf{p}(t) = \mathbf{1}$.
Hence, the matrix equation \eqref{equ:pr3} for $\mathbf{p}(t+1)$ becomes
\[\mathbf{p}(t+1) = \Big(dK + \frac{1-d}{N}E\Big)\mathbf{p}(t).\]
If we write $B = dK + \frac{1-d}{N}E$, this simplifies to $\mathbf{p}(t+1) = B\mathbf{p}(t).$
Thus, the steady state $\mathbf{p}(t)$ is an eigenvector of $B$ corresponding to the eigenvalue 1.

The columns of $B$ sum to 1, and the entries of $B$ are strictly positive (because the entries of $E$ are all positive).
With these hypotheses, the Perron-Frobenius theorem says that 1 is the unique eigenvalue of B of largest magnitude, and the corresponding eigenvector is unique.
In this case, the ``iterative method'' described above is just the power method for finding the eigenvector corresponding to a dominant eigenvalue, introduced in the lab on eigensolvers. %TODO: make sure the reference to another lab is accurate.

We can also compute $\mathbf{p}$ using eigenvalue solvers in SciPy.

%TODO: the output from scipy.linalg.eig is not very accurate, even for the small example in this lab.
\begin{problem}
Implement the function below, using the eigenvalue method to find the steady state of the PageRank algorithm.
\begin{lstlisting}
def eig_solve( adj, N=None, d=.85):
    '''
    Return the page ranks of the network described by `adj`.

    INPUTS:
    adj - A NumPy array representing the adjacency matrix of a directed
            graph
    N     - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.

    OUTPUTS:
    Return the approximation to the steady state of p.
    '''
\end{lstlisting}
Hint: Review the techniques from the Markov chain section of Lab \ref{lab:EigSolve}.
\end{problem}

\section*{Ranking Teams}
This ranking algorithm can be applied not only to webpages, but to any problem with a directed graph structure.
One such application is ranking sports teams.

Suppose we have data about a collection of sports teams, including which teams played each other and who won each match.
We can model this as a directed graph. Each node in the graph represents a team.
An edge between two nodes points from the losing team to the winning team.
If two teams never played each other, there is no edge between them.
Wins and losses do not cancel out; if BYU and Boise played twice, and each team won once, then there is an edge from BYU to Boise and another edge from Boise to BYU. TODO: add a picture.

To simplify our model, edges are not weighted. So if Duke ever beat Harvard, no matter whether they beat them once or 5 times, there is only one edge pointing from Harvard to Duke.

The key here is that edges tend to lead from worse teams to better teams.
So by starting with some team and randomly following edges, we should end up visiting better teams more often.
This is reminiscent of the PageRank algorithm! Given an appropriate dataset, we can use PageRank to estimate team rankings.

Note that in this scenario, the parameter $d$ no longer represents boredom.
It allows us to jump randomly from one team to another, so it could represent a surprise upset, or the random outcome of a game between two teams who have never played each other.

\begin{problem}
By applying the PageRank algorithm to win-loss data from the 2013 NCAA basketball season, produce a comparative ranking of the teams.
\begin{enumerate}
\item The file  \texttt{ncaa2013.csv} contains data on over 5000 basketball games.
The first line is a header.
After the header, each line represents a game and has the winning team followed by the losing team (there are no ties in basketball).

Load this file and use it to create the adjacency matrix $A$, where $A_{ij} = 1$ if team $j$ beat team $i$.
Make sure to ignore the header line.
You will need some way of mapping from team names to the integers and vice versa.
\item Use the iterative method from Problem \ref{prob:pagerank_dense_iter} with $d = 0.7$ to find the steady state.
The steady-state solution is your vector of ranks.
\item Return the ranks sorted from largest to smallest, and the corresponding list of teams sorted from ``best'' to ``worst''.
\end{enumerate}
Hints:
\begin{enumerate}
\item The code below may be helpful for processing the .csv file:
\begin{lstlisting}
>>> with open('./ncaa2013.csv', 'r') as ncaafile:
>>>     ncaafile.readline() #reads and ignores the header line
>>>     for line in ncaafile:
>>>         teams = line.strip().split(',') #split on commas
>>>         print teams
>>> ['Middle Tenn St', 'Alabama St']
>>> ...
>>> ['Mississippi', 'Florida']
\end{lstlisting}
\item Before creating the adjacency matrix, you can get all the unique teams by running through all the matches once and adding every team to a set.
Next, count the number of unique teams and initialize $A$ to be the right size.
Try using dictionaries, lists, or both to map numbers to teams and teams to numbers and fill in $A$.
There is more than one right way to do this.
\item The function \li{np.argsort()} will be useful for sorting the ranks and teams.
\item There should be 347 teams. PageRank should predict that the top five ranked teams are Duke, Butler, Louisville, Illinois, and Indiana (in that order). Use this to check your results.
\end{enumerate}
\end{problem}

\section*{Optional: SNAP Datasets}
The SNAP graph library, located at \url{http://snap.stanford.edu/data/index.html}, provides a variety of medium sized data sets for public use.
These datasets have to do with networks, including road systems, social networks, and online communities. There are some interesting resources here for those wanting to experiment further with the PageRank algorithm on different datasets.
\begin{comment}
The \li{matplotlib.pyplot.spy} command on the adjacency matrix from a SNAP data set yielded the plot shown in Figure \ref{fig:WebSparse}
\end{comment}

\begin{comment}
\begin{figure}
\centering
\includegraphics[width=\textwidth]{sparse_web.png}
\caption{Output of the \li{spy} command on the adjacency matrix corresponding to the websites supported by Notre Dame University in 1999.
Data was taken from the SNAP datasets.}
\label{fig:WebSparse}
\end{figure}
\end{comment}

\begin{problem}
(Optional) Try running the functions you wrote in this lab on a data set downloaded from SNAP.
\begin{enumerate}
\item Begin by running your methods on the first 100 nodes of the data set.
\item Modify your solution to Problem \ref{prob:pagerank_dense_iter} so that it uses only sparse matrices.
With this modification, you should be able to run the function on more nodes.
Hint: Convert the adjacency matrix to a \li{csc_matrix} or a \li{csr_matrix} to perform the computations.
\end{enumerate}
\end{problem}


%#########################################################
\chapter{Diffusion Tensor Imaging}
\label{ch: Diffusion}
%#########################################################
Molecular diffusion in biological tissues is restricted due to interactions with many obstacles, such as fibers and neural tracts.
Therefore diffusion patterns can be used to provide microscopic details about tissue architecture. This information is further used to detect abnormalities in skeletal muscles, heart and brain.
MRI-based diffusion tensor imaging (DTI) is a relatively new modality, which allows the noninvasive \textit{in-vivo} determination of diffusion of water molecules arising from random motions due to the thermal energy of the tissue.
%-new paragraph-%

%-new paragraph-%
This chapter gives a brief introduction into diffusion theory and provides an overview of spin-echo and stimulated echo MR echo planar imaging pulse sequences that were used in the experiments discussed further in chapter~\ref{ch: DiffusionExp}.
%=========================================================
\section{Brownian Motion and Einstein Relation}
%=========================================================
In 1855 Adolf Fick explained diffusion through the flux of particles arising from the gradient of local concentration $c(\mathbf{r},t)$~\cite{Fick}:
%.........................................................
\begin{equation}\label{eq:Fick1}
\mathbf{J}=-D\nabla c(\mathbf{r},t)
\end{equation}
%.........................................................
For the total number of particles to be conserved divergence of the local flux should be related to the rate of change of $c(\mathbf{r},t)$ as $\nabla\mathbf{J}=-\frac{\partial c}{\partial t}$. This gives the diffusion equation:
%.........................................................
\begin{equation}\label{eq:Fick2}
\frac{\partial c}{\partial t}=D\nabla^2c
\end{equation}
%.........................................................
Equations~\ref{eq:Fick1} and \ref{eq:Fick2} are known as Fick's law.
It was obtained for the case of admixture, when particles drift from higher to lower concentration regions, and this process is called \textit{mutual diffusion}.
Albert Einstein in 1905 explained Brownian motion quantitatively~\cite{Einstein} and showed that Fick's laws are also valid for the case of \textit{self-diffusion}.
Einstein described the motion of particles inside a finite volume $\delta V$ considering small displacements $\delta x$ due to the net force $K$, such that free energy is minimized: $\delta F=\delta E - T\delta S=0$.
Utilizing the ideal gas model he related the work done by the particles with the change in entropy $\delta S = -k_B\frac{\partial V}{V}$. Thus the equilibrium condition is given:
%.........................................................
\begin{equation}\label{eq:EQ_Condition}
-Kn+k_BT\frac{\partial c}{\partial x}=0
\end{equation}
%.........................................................
where $k_B$ is Boltzmann's constant. The net flow of particles due to to net force $K$ is $J_{\mathrm{drift}}=\mu K $, where $\mu$ is mobility. Drift is counteracted by diffusive flow,  which according to Fick's law, is $J_{\mathrm{diff}}=-D\frac{\partial c}{\partial x}$.
These flows are balanced and, together with equilibrium condition Equation~(\ref{eq:EQ_Condition}), result in a famous Einstein-Smoluchowski relation:
%.........................................................
\begin{equation}\label{eq:Einstein-Smoluchowski}
D=\mu k_BT
\end{equation}
%.........................................................
which was obtained independently by Albert Einstein~\cite{Einstein} in 1905, and Marian Smoluchowski~\cite{Smoluchowski} in 1906.
For the small spherical particles of radius $r$, mobility is given by Stock's law $\mu=1/{(6\pi\eta r)}$, and Equation~(\ref{eq:EQ_Condition}) can be written in the following form:
%.........................................................
\begin{equation}\label{eq:Einstein-Stocks}
D=\frac{k_BT}{6\pi\eta r}
\end{equation}
%.........................................................
In addition to the expression for the diffusion coefficient, Einstein was able to describe  Brownian motion as a stochastic process~\cite{Einstein}.
He considered conditional probability $P(\mathbf{r}|\mathbf{r'},t)$ for the particle at $\mathbf{r}$ at time $t_0=0$ to be at $\mathbf{r'}$ after time $t$.
Local particle concentration~$c(\mathbf{r'}, t)$~is:
%.........................................................
\begin{equation}\label{eq:probability_concentration}
c(\mathbf{r'},t)=\int d\mathbf{r} \, c(\mathbf{r},0)P(\mathbf{r}|\mathbf{r'},t)
\end{equation}
%.........................................................
Since $c(\mathbf{r'},t)$ satisfies the diffusion equation Equation~(\ref{eq:Fick2}) for arbitrary choice of initial $c(\mathbf{r},0)$, the diffusion equation is also valid for probability $P(\mathbf{r'}|\mathbf{r}, t)$.
%.........................................................
\begin{equation}\label{eq:probability_diffusion}
\frac{\partial}{\partial t}P(\mathbf{r}|\mathbf{r'}t)=D\nabla^2P(\mathbf{r'}|\mathbf{r}, t)
\end{equation}
%.........................................................
With the initial condition $P(\mathbf{r'}|\mathbf{r}, 0)=\delta(\mathbf{r}-\mathbf{r'})$ the solution of the Equation~(\ref{eq:probability_diffusion}) is a Gaussian:
%.........................................................
\begin{equation}\label{eq:probability}
P(\mathbf{r}|\mathbf{r'},t)=\frac{1}{(4\pi Dt)^{3/2}} \mathrm{exp}\left( -\frac{(\mathbf{r'}-\mathbf{r})^2}{4Dt}\right)
\end{equation}
%.........................................................
From this equation, the mean square displacement is liner in time:
%.........................................................
\begin{equation}\label{eq:ms_dr}
\left<(\mathbf{r'}-\mathbf{r})^2\right>=6Dt
\end{equation}
%.........................................................
or in one dimension:
%.........................................................
\begin{equation}\label{eq:ms_dx}
\left<(x'-x)^2\right>=2Dt
\end{equation}
%.........................................................
The following definition of the diffusion coefficient flows from the Equation~(\ref{eq:ms_dx}):
%.........................................................
\begin{equation}\label{eq:D_v}
D=\lim_{t\rightarrow\infty}\frac{1}{2}\frac{\partial\left<\Delta x^2\right>}{\partial t}
\end{equation}
%.........................................................
Since $\Delta x = \int_0^t \mathrm{d}\tau \, v({\tau})$ the mean square displacement is:
%.........................................................
\begin{equation}\label{eq:D_v}
\left<\Delta x^2\right>=\int\limits_0^tdt_1\int\limits_0^tdt_2\left<v(t_1)v(t_2)\right>
\end{equation}
%.........................................................
By taking the derivative of this expression and using time translation invariance one can obtain Green-Kubo relation for diffusion coefficient $D$~\cite{Peliti}:
%.........................................................
\begin{equation}\label{eq:Green-Kubo}
D=\lim_{t \rightarrow \infty}\int\limits_0^\infty \mathrm{d} t \, \left<v(t) v(0)\right>
\end{equation}
%.........................................................
where $\left<v(t) v(0)\right>$ is the autocorrelation function of the molecular velocity $v$. In fact Equation~(\ref{eq:Green-Kubo}) represents a zero frequency component of the diffusion spectrum:
%.........................................................
\begin{equation}\label{eq:diffusion spectrum}
D(\omega)=\int\limits_0^\infty dt \, \left< v(t) v(0)\right> \mathrm{exp}(i\omega t)
\end{equation}
%.........................................................
The correlation time is defined as:
%.........................................................
\begin{equation}\label{eq:correlation_time}
t_c=\int\limits_0^\infty dt \, \frac{\left<v(t) v(0)\right> }{\left< v^2\right>}
\end{equation}
%.........................................................
and provides a timescale over which the fluctuating molecular velocity becomes decorrelated~\cite{DerekKJones}.
This is an important result.
In free solution, the correlation time is short.
However, biological tissue molecules take much longer to thermalize because the length-scales of the spatial heterogeneities are typically much larger than the molecular scale.
Thus in case of restricted diffusion, it's possible to access spectral features of the diffusion coefficient which provide detailed information even in case of complicated tissue architecture~\cite{Tuch:2002ts}.
%=========================================================
\section{Bloch-Torrey Equations and Diffusion Tensor}
%=========================================================
In 1950, just four years after the discovery of the NMR phenomenon by Bloch and Purcell~\cite{Bloch1946}, Ervin Hahn discovered that the spin-echoes he observed are sensitive to the effects of diffusion.
He related the reduction of the signal to a dephasing caused by translational diffusion of spins subjected to local magnetic field gradients~\cite{Hahn}.
In the presence of the diffusion term Bloch equations for the magnetization would change:
%.........................................................
\begin{align}\label{eq:Bloch-Torrey}
\frac{dM_x}{dt}&=\gamma\left(\mathbf{M}\times B_0\right)_x-\frac{M_x}{\mathrm{T_2}}+D\Delta M_x\nonumber\\
\frac{dM_y}{dt}&=\gamma\left(\mathbf{M}\times B_0\right)_y-\frac{M_y}{\mathrm{T_2}}+D\Delta M_y\\
\frac{dM_z}{dt}&=\gamma\left(\mathbf{M}\times B_0\right)_z+\frac{M_0-M_z}{\mathrm{T_1}}+D\Delta(M_z-M_0)\nonumber
\end{align}
%.........................................................
This is so called Bloch-Torrey equations first introduced by Torrey in 1956~\cite{Torrey}.
Here $\mathbf{M}$ is the magnetization of a sample [\si{\ampere/\meter}], $B_0$ is a static magnetic field applied [\si{\tesla}], $\gamma$ is a gyromagnetic ratio [\si{\radian/\second \tesla}], $M_x$, $M_y$, $M_z$ are the $x$, $y$ and $z$ components of $\mathbf{M}, M_0$ is magnetization at thermal equilibrium and $\mathrm{T_1}, \mathrm{T_2}$ are the longitudinal and transversal relaxation times respectively.
After \ang{90} radio frequency (RF) pulse is applied to the system, Bloch-Torrey equations give the following solution for the magnetization in the transverse plane:
%.........................................................
\begin{equation}\label{eq: dif_mag}
\bar{M}=M_0e^{-\frac{t}{T_2}}e^{-bD}
\end{equation}
%.........................................................
As opposed to the solution for the spin-echo Equation~(\ref{eq: dif_mag}) has an additional exponential factor $e^{-bD}$, \textit{b-}value identifies the measurement sensitivity to diffusion and determines the strength and duration of the diffusion gradients. The units of \textit{b-}value are [\si{\second/\milli\meter\squared}]:
%.........................................................
\begin{equation}\label{eq: b-value}
b=\gamma^2\int\limits_{0}^{\mathrm{TE}}dt\left(\int\limits_{0}^{t}dt'G(t')\right)^2
\end{equation}
%.........................................................
where $G$ is time-varying gradient of the magnetic field and $\mathrm{TE}$ is the echo-time.
Typical diffusion-weighted gradient consists of two lobes with equal area. In basic spin-echo sequences, two lobes ($G_{\mathrm{d1,2}}$) have the same polarity and are placed at either side of a refocusing RF-pulse as seen in Figure~\ref{fig: DTISeq}a.
In gradient-echo sequences, however, the two lobes must have opposite polarity as shown in Figure~\ref{fig: DTISeq}b.
%*********************************************************
\begin{figure}[t]
\vspace{+0.2cm}
\centering
\includegraphics[width=1\textwidth]{Figures/DiffusionPS.pdf}
\caption[Diffusion-weightning gradients in spin-echo and gradient echo pulse sequences]{Diffusion-weightning gradients in spin-echo and gradient echo pulse sequences.}\label{fig: DTISeq}
\end{figure}
%*********************************************************
An application of balanced bipolar gradients for the diffusion measurements was offered by Stejskal and Tanner~\cite{Stejskal}.
Although the first gradient creates a spatially dependent phase to each spin the second gradient eliminates this effect for the stationary spins.
Each of the protons experiencing a random diffusive displacement between the application of the two gradient pulses will acquire a phase offset proportional to the magnitude of the displacements.
The result is phase dispersion proportional to the spread of positions. Thus, the diffusion coefficient $D$ along any direction can be measured by comparing MR signals $S(b)=M_0e^{-t/\mathrm{T_2}}e^{-bD}$ and $S(0)=M_0e^{-t/\mathrm{T_2}}$:
\begin{equation}\label{eq: Diffusion from bvalue}
D=-\frac{1}{b}\ln{\frac{S(b)}{S(0)}}
\end{equation}
%.........................................................
Until now diffusion coefficient $D$ was considered as a scalar in anisotropic medium.
Biological tissues with regularly ordered microstructure such as skeletal muscle, spine, tongue, heart, white matter exhibit anisotropic water diffusion.
An anisotropic media can be described in terms of tensor formalism.
A diffusion tensor is a covariant tensor of $\rank$~2 that is described by a $3\times3$ symmetric matrix with 6 unique elements:
%.........................................................
\begin{equation}
\mathbf{D} =\left[
\begin{array}{ccc}
D_{xx} & D_{xy} & D_{xz} \\[4pt]
D_{yx} & D_{yy} & D_{yz} \\[4pt]
D_{zx} & D_{zy} & D_{zz} \\
\end{array}\right]
\end{equation}
%.........................................................
The components of the diffusion tensor can be calculated from the diffusion weighted image (DWI) set collected with the diffusion gradients applied in six or more directions.
For the arbitrary number of gradient directions $n \geq 6$, the following system of liner equations can be written:
%.........................................................
\begin{align}
\begin{cases}
-\ln{\dfrac{S(b_1)}{S(b_0)}}&=\displaystyle\sum_{i,j=1}^3 b_{1ij}D_{ij}\\[2em]
-\ln{\dfrac{S(b_2)}{S(b_0)}}&=\displaystyle\sum_{i,j=1}^3 b_{2ij}D_{ij}\\
&\setbox0\hbox{=}\mathrel{\makebox[\wd0]{\vdots}} \\
-\ln{\dfrac{S(b_n)}{S(b_0)}}&=\displaystyle\sum_{i,j=1}^3 b_{nij}D_{ij}\\
\end{cases}
\end{align}
%.........................................................
where indexes $i$ and $j$ stand for $x,y$ and $z$ components.
Diffusion tensor yields several useful metrics. Mean Apparent Diffusion Coefficient (ADC) which gives the magnitude of the diffusion:
%.........................................................
\begin{equation}
\mathrm{ADC}=\frac{1}{3}\mathrm{Tr}\mathbf{D}
\end{equation}
%.........................................................
And  fractional anisotropy (FA) which is calculated after diffusion tensor eigenvalue decomposition is performed:
%.........................................................
\begin{equation}
\mathrm{FA}=\frac{\sqrt{3\displaystyle\sum_{i=1}^3 (\lambda_i-\mathrm{ADC})^2}}{\sqrt{2\displaystyle\sum_{i=1}^3 \lambda_i^2}}
\end{equation}
%.........................................................
where $\lambda_{1,2,3}$ are the eigenvalues of the diffusion tensor.
FA describes the degree of anisotropy and can take values between zero and one.
Conventionally diffusion tensors are represented by ellipsoids.
%*********************************************************
\begin{figure}[h]
\vspace{+0.2cm}
\centering
\includegraphics[scale=0.8]{Figures/DiffusionFA.pdf}
\caption[Visual representation of the diffusion tensor in the isotropic and anisotropic cases]{Visual representation of the diffusion tensor in the isotropic and anisotropic cases.}\label{fig: AnisIso}
\end{figure}
%*********************************************************
\noindent If $\mathrm{FA} = 1$, then diffusion occurs only along one axis and is fully restricted along all others. $\mathrm{FA} = 0$ means an isotropic diffusion (Figure~\ref{fig: AnisIso}a).
Biological tissues, such as muscle are anisotropic (Figure~\ref{fig: AnisIso}b) and diffusion mostly occurs along one of the eigenvectors, being restricted along the remaining two.
%=========================================================
\section{Echo Planar Imaging (EPI)}
\label{sec: EPI}
%=========================================================
A conventional MRI spin echo sequence acquires \textit{k}-space by sequential repetition of the basic spin echo pulse sequence ($\SI{90}{\degree}$ excitation RF-pulse followed by refocusing $\SI{180}{\degree}$).
This sequence is too long compared to physiological (patient) motion.
Addition of diffusion sensitization to routine spin echo sequence will essentially result in a completely wiped out image.
In order to freeze motion, the established method for performing diffusion weighted imaging is to use a single shot technique, a spin echo echo-planar imaging (EPI) sequence, rather than a conventional spin echo.
%-new paragraph-%

%-new paragraph-%
Echo planar imaging (EPI) is the fastest MRI pulse sequence.
With the modern hardware EPI allows producing of a 2D image as fast as tens of a millisecond.
The main difference of EPI pulse sequence from the conventional pulse sequences (such as spin echo and gradient echo), is that the entire \textit{k}-space is acquired with one RF excitation (single-shot)~\cite{RNDT23}.
This is accomplished by a series of bipolar readout gradients.
To generate a train of gradient echoes (Figure~\ref{fig: EPI}a) with accompanying small phase-encoding gradients ('blips'), each gradient echo is distinctively spatially encoded so that multiple \textit{k}-space lines can be sampled under an RF spin echo (Figure~\ref{fig: EPI}b).
%*********************************************************
\begin{figure}[!h]
\vspace{+0.2cm}
\includegraphics[width=\textwidth]{Figures/EPI.pdf}
\caption[Spin-echo diffusion weighted EPI sequence and \textit{k}-space sampling path]{Spin-echo diffusion weighted EPI sequence and \textit{k}-space sampling path.}\label{fig: EPI}
\end{figure}
%*********************************************************
EPI generates an image in a considerably shorter time than any other MRI sequence.
Typical EPI pulse sequence is capable of producing $\sim 100$ gradient echoes as a result 2D image can be constructed from a single RF excitation~\cite{RNDT24}.
%-new paragraph-%

%-new paragraph-%
Compared to conventional MR imaging pulse sequences, EPI is more prone to a variety of artifacts.
These artifacts arise from the fact the effective bandwidth in the phase-encoding direction is very small so that small differences in the frequency (other than from the phase encode gradient) at different spatial locations can result in severe mismapping.
These frequency differences occur due to eddy currents from the large diffusion gradients, as well as from magnetic field inhomogeneities arising primarily from susceptibility differences in tissues and air.
In my studies several correction techniques for eddy current, strong steady field inhomogeneity magnetic susceptibility were incorporated at the image pre-processing stage.
%-new paragraph-%

%-new paragraph-%
In spin-echo EPI sequence diffusion gradients are applied right before and immediately after $\SI{180}{\degree}$ refocusing pulse.
As a result the range of the diffusion times is very constrained.
Ability to measure diffusion tensor at diffusion times much longer compared to spin-echo EPI sequence provides an extra dimension in the data and a better estimate for micro-structural parameters.
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\subsection{Stimulated Echo Acquisition Mode (STEAM)}
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Compared to spin-echo which uses two RF-pulses Stimulated Echo~\cite{Hahn} uses three $\SI{90}{\degree}$ pulses with the same phase and observes total of five echos: three primary spin echos, one secondary and one simulated echo.
All primary and a secondary echo experience $\mathrm{T_2^*}$ dephasing in between second and third RF-pulses while the magnetization forming stimulated echo is preserved along the longitudinal axis.
The phase evolution of the magnetization in the pulse sequence with three RF-pulses is conveniently described using the diagram from~Figure~\ref{fig: STEAM_phase}.
%*********************************************************
\begin{figure}[!h]
\vspace{+0.2cm}
\centering
\includegraphics[width=\textwidth]{Figures/STEAM_Phase.pdf}
\caption[Spin-phase evolution diagram for the MR pulse sequence with three RF-pulses]{Spin-phase evolution diagram for the MR pulse sequence with three RF-pulses.}\label{fig: STEAM_phase}
\end{figure}
%*********************************************************
 To keep only simulated echo pathway while eliminating signal from all the other four echoes as well as signal from free induction decays a set of four crusher gradients $\phi_{1-4}$ must be introduced.
 For signal to be produced the accumulated phase due to the crusher must cancel out.
 Phase dispersion created by a crusher gradient is directly proportional to its area thus by manipulating the size of the crusher gradients $\phi_{1-4}$ the stimulated echo pathway can be chosen while other echos can be destroyed.
%-new paragraph-%

%-new paragraph-%
 The stimulated echo signal pathway denoted SE in~Figure~\ref{fig: STEAM_phase}: first $\mathrm{RF_1}$ excites magnetization (1), next, phase is accumulated (2) due to gradient $\phi_1$, then $\mathrm{RF_2}$ reverses the phase (3) and locks magnetization longitudinally (4), following $\mathrm{RF_3}$ magnetization is restored in the transverse plane(5) and rephased by crusher gradient $\phi_4$ (6) to form an echo (7).
%-new paragraph-%

%-new paragraph-%
Conditions for stimulated echo are the following:
%.........................................................
\begin{equation}\label{eq: STEAM phase}
{\setstretch{1.0}
\begin{cases}
    \mathrm{E_1} &= -\phi_1 + \phi_2\\
    \mathrm{E_2} &= -\phi_1 + \phi_2 + \phi_3 - \phi_4\\
    \mathrm{E_3} &= -\phi_2 + \phi_3 + \phi_4\\
    \mathrm{E_4} &= -\phi_1 - \phi_2 - \phi_3 + \phi_4\\
    \mathrm{E_{1-4}} &\neq 0\\
    \mathrm{SE} &= -\phi_1 + \phi_4\\
    \phi_2 &\neq 0\\
    \phi_4 &\neq 0\\
\end{cases}
}
\end{equation}
%.........................................................
where first five equations are the conditions to destroy four echos, last two equations are conditions to remove signal originating from free induction decay and the equation for SE is the requirement to preserve stimulated echo.
Diffusion weighting is added into the STEAM sequence by placing two identical diffusion gradients: first is placed in between $\mathrm{RF_1}$ and $\mathrm{RF_2}$ and second after $\mathrm{RF_3}$ pulse.
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\subsection{Correction to the \textit{b}-matrix}
\label{subsection: STEAM b value}
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In STEAM diffusion weighted sequence contribution to the \textit{b-}matrix (Equation~\ref{eq: bmatrix}) from cross-terms between different gradients becomes significant at long diffusion time $\Delta$ and must be calculated from the Equation~\ref{eq: b-value}.
%.........................................................
\begin{equation}\label{eq: bmatrix}
b =\left[
\begin{array}{ccc}
b_{rr} & b_{rp} & b_{rs} \\
b_{pr} & b_{pp} & b_{ps} \\[1pt]
b_{sr} & b_{sp} & b_{ss} \\
\end{array}\right]
\end{equation}
%.........................................................
where indices $r, p, s$ are corresponding to three gradient axis: readout, phase encoding and slice selection respectively.
%-new paragraph-%

%-new paragraph-%
Figure~\ref{fig: STEAM_GE} shows a plot of STEAM pulse sequence programmed in EPIC (GE Medical Systems, Milwaukee, WI, USA).
Important timing parameters marked: diffusion time ($\Delta$) measured as time between diffusion gradients, duration of the diffusion gradients ($\delta$) and mixing time TM, measured as time between centers of the second and third RF-pulses.
%*********************************************************
\begin{figure}[h]
\vspace{+0.2cm}
\centering
\includegraphics[width =0.7\textwidth]{Figures/STEAM_GE.pdf}
\caption[Plot of STEAM pulse sequence programmed for GE scanner]{Plot of STEAM pulse sequence programmed for GE scanner.}\label{fig: STEAM_GE}
\end{figure}
%*********************************************************
To satisfy conditions~\ref{eq: STEAM phase} the following configuration of crusher gradients was chosen:
%.........................................................
\begin{equation}\label{eq: bmatrix}
{\setstretch{1.0}
\begin{cases}
    \phi_1 = \phi_4\\
    \phi_1 \neq \phi_2\\
    \phi_2 \neq 0\\
    \phi_4 \neq 0\\
    \phi_2 \neq - \phi_3\\
    \phi_3 \neq \phi_2 - \phi_1\\
    \phi_3 \neq 2\phi_1 - \phi_2\\
    \phi_3 < \phi_2 < \phi_1
\end{cases}
}
\end{equation}
%.........................................................
Using full analytic form derived in~\cite{RNDT} independent components for \textit{b-}matrix due to diffusion, crusher and slice select gradient interactions are:
%.........................................................
\begin{equation}\label{eq: bmatrix}
{\setstretch{1.0}
\begin{array}{l}
    b_{rr} = \gamma^2\Bigl[G_{\mathrm{d}r}^2\tau_{11} + 2G_{\mathrm{d}r}G_{\mathrm{c}r}\tau_{12} + G_{\mathrm{c}r}^2\tau_{22}\Bigr]\\[10pt]

    b_{pp} = \gamma^2\Bigl[G_{\mathrm{d}p}^2\tau_{11} + 2G_{\mathrm{d}p}G_{\mathrm{c}p}\tau_{12} + G_{\mathrm{c}p}^2\tau_{22} \Bigr]\\[10pt]

    b_{ss} = \gamma^2\Bigl[G_{\mathrm{d}s}^2\tau_{11} +2G_{\mathrm{d}s}G_{\mathrm{c}s}\tau_{12} + G_{\mathrm{c}s}^3\tau_{22} + G_{\mathrm{sl}}^2\tau_{33} \Bigr]\\[10pt]

    b_{rp} = b_{pr} = \gamma^2\Bigl[G_{\mathrm{d}r}G_{\mathrm{d}p}\tau_{11} + \left(G_{\mathrm{c}p}G_{\mathrm{d}r} + G_{\mathrm{c}r}G_{\mathrm{d}p}\right)\tau_{12} + G_{\mathrm{c}r}G_{\mathrm{c}p}\tau_{22} \Bigr]\\[10pt]

    \begin{split}
	b_{rs} &= b_{sr} = \gamma^2\Bigl[G_{\mathrm{d}s}G_{\mathrm{d}r} \tau_{11} + \left( G_{\mathrm{d}s}G_{\mathrm{c}r} + G_{\mathrm{d}r}G_{\mathrm{c}s}\right)\tau_{12} + \\[2pt]
    	 	&\qquad\qquad\qquad\quad  + G_{\mathrm{c}r}G_{\mathrm{c}s}\tau_{22} + G_{\mathrm{sl}}G_{\mathrm{d}r}\tau_{13}+G_{\mathrm{sl}}G_{\mathrm{c}r}\tau_{23}\Bigr]
	\end{split}
    \\[4ex]
    \begin{split}
	b_{ps} &= b_{sp} = \gamma^2 \Bigl[G_{\mathrm{d}s}G_{\mathrm{d}p} \tau_{11} + \left( G_{\mathrm{d}s}G_{\mathrm{c}p} + G_{\mathrm{d}p}G_{\mathrm{c}s}\right)\tau_{12} + \\[2pt]
    	 	&\qquad\qquad\qquad\quad  + G_{\mathrm{c}p}G_{\mathrm{c}s}\tau_{22} + G_{\mathrm{sl}}G_{\mathrm{d}p}\tau_{13}+G_{\mathrm{sl}}G_{\mathrm{c}p}\tau_{23}\Bigr]\\[4pt]
	\end{split}
    \end{array}
}
\end{equation}
%.........................................................
with gradients amplitudes: $G_{\mathrm{d}*}$ and $G_{\mathrm{c}*}$ being diffusion and crusher pulses respectively where $*$ denotes gradient axis ($r,p$ or $s$), $G_{\mathrm{sl}}$ being an amplitude of a slice selection gradient pulse.
Time constants $\tau_{ij}$ for the trapezoid gradient shape approximation are:
%.........................................................
\begin{equation}\label{eq: bmatrix timing}
{\setstretch{1.0}
\begin{array}{l}

	\tau_{11} = \delta_1^2\left( \Delta_1 - \dfrac{1}{3}\delta_1\right) + \dfrac{1}{30}\epsilon_{\mathrm{d}}^3-\dfrac{1}{6}\delta_1\epsilon_{{\mathrm{d}}}^2\\[12pt]
	\tau_{22} = \delta_2^2\left( \Delta_2 - \dfrac{1}{3}\delta_2\right) + \dfrac{1}{30}\epsilon_{\mathrm{c}}^3-\dfrac{1}{6}\delta_2\epsilon_{{\mathrm{c}}}^2\\[12pt]
	\tau_{12} = \delta_1\delta_2\Delta_2\\[12pt]
	\tau_{13} = \dfrac{1}{4}\delta_1\delta_{\mathrm{sl}}^2\\[12pt]
	\tau_{23} = \dfrac{1}{4}\delta_2\delta_{\mathrm{sl}}^2\\[12pt]
	\tau_{33} = \dfrac{1}{12}\delta_{\mathrm{sl}}^3\\[12pt]
\end{array}
}
\end{equation}
%.........................................................
where $\Delta_1$ is diffusion time, time constant $\Delta_2$ and pulse durations $\delta_{1,2}$ defined:
%.........................................................
\begin{equation}\label{eq: bmatrix timing delta}
{\setstretch{1.0}
\begin{array}{l}
	\Delta_{2} = \Delta_{1}-(\delta_{\mathrm{d}}+\delta_{\mathrm{c}})-2(\epsilon_{\mathrm{d}}+\epsilon_{\mathrm{c}})\\[12pt]
	\delta_{1} = \delta_{\mathrm{d}}+\epsilon_{\mathrm{d}}\\[12pt]
	\delta_{2} = \delta_{\mathrm{c}}+\epsilon_{\mathrm{c}}
\end{array}
}
\end{equation}
%.........................................................
$\delta_{\mathrm{d}}$ and $\delta_{\mathrm{c}}$ durations of the diffusion and crusher gradients, $\epsilon_{\mathrm{d}}$ and $\epsilon_{\mathrm{c}}$ rise time of the diffusion and crusher gradients, $\delta_{\mathrm{sl}}$ duration of the slice selection gradient.


\documentclass[11pt]{article}
\usepackage{cctbx_preamble}
\usepackage{amscd}

\title{Restraint Gradients}
\author{\rjgildea}
\date{\today}

\begin{document}
\maketitle

\section{ADP similarity restraint}
\label{ADP:similarity}
The anisotropic displacement parameters of two atoms are restrained to have the
same $U_{ij}$ components.  This is equivalent to a SHELXL SIMU restraint \cite{SHELX:man97}.
The weighted least-squares residual is defined as
\begin{equation}
R = w \sum_{i=1}^3 \sum_{j=1}^3 (U_{A,ij} - U_{B,ij})^2,
\end{equation}
which we note is the square of the Frobenius norm of the matrix of deltas.
But since $\mat{U}$ is symmetric, i.e. $U_{ij} = U_{ji}$, this can be rewritten as
\begin{equation}
R = w \left( \sum_{i=1}^3 (U_{A,ii} - U_{B,ii})^2 + 2 \sum_{i < j} (U_{A,ij} - U_{B,ij})^2 \right) .
\end{equation}
Therefore the gradient of the residual with respect to the diagonal element $U_{A,ii}$ is then
\begin{equation}
\partialder{R}{U_{A,ii}} = 2w(U_{A,ii} - U_{B,ii}).
\end{equation}
Similarly the gradient with respect to the off-diagonal element $U_{A,ij}$ is
\begin{equation}
\partialder{R}{U_{A,ij}} = 4w(U_{A,ij} - U_{B,ij}).
\end{equation}

\section{Rigid-bond restraint}

In a `rigid-bond' restraint the components of the anisotropic displacement parameters
of two atoms in the direction of the vector connecting those two atoms are restrained
to be equal.  This corresponds to Hirshfeld's `rigid-bond' test \cite{Hirshfeld:1976} for testing
whether anisotropic displacement parameters are physically reasonable (see SHELX
manual, DELU restraint \cite{SHELX:man97}).  We must therefore minimise the mean square displacement of
the atom in the direction of the bond.

The weighted least-squares residual is then
\begin{equation}
R = w(z^2_{A,B} - z^2_{B,A})^2,
\end{equation}
where in the Cartesian coordinate system the mean square displacement of atom A
along the vector $\overrightarrow{AB}$, $z^2_{A,B}$, is given by
\begin{equation}
z^2_{A,B} = \frac{\vec{r}^t\mat{U}_{cart,A}\vec{r}}{\norm{\vec{r}}^2},
\end{equation}
where
\begin{equation}
\vec{r} = \begin{pmatrix} x_A - x_B\\y_A - y_B\\z_A - z_B \end{pmatrix}
= \begin{pmatrix} x\\y\\z \end{pmatrix},
\end{equation}
$\vec{r}^t$ is the transpose of $\vec{r}$ (\textit{i.e.} a row vector) and
$\norm{\vec{r}}$ is the length of the vector $\overrightarrow{AB}$.

The derivative of the residual with respect to an element of $\vec{U}_{cart,A}$,
$U_{A,ij}$ is given by (using the chain rule)
\begin{align}
\partialder{R}{U_{A,ij}} &= \partialder{R}{z^2_{A,B}} \partialder{z^2_{A,B}}{U_{A,ij}}\\
&=2w(z^2_{A,B} - z^2_{B,A}) \partialder{z^2_{A,B}}{U_{A,ij}}\label{eqn:r_derivative}
\end{align}

The matrix multiplication in obtaining $z^2_{A,B}$ can be evaluated as follows
(remembering $\vec{U}_{cart}$ is symmetric):
\begin{align}
\vec{r}^t\vec{U}_{cart,A}\vec{r} &=
\begin{pmatrix} x & y & z \end{pmatrix}
\begin{pmatrix} U_{11} & U_{12} & U_{13}\\
  U_{12} & U_{22} & U_{23}\\
  U_{13} & U_{23} & U_{33}\end{pmatrix}
\begin{pmatrix} x\\y\\z\end{pmatrix}\\
&= U_{11}\: x^2 + U_{22}\: y^2 + U_{33}\: z^2 + 2U_{12}\: xy + 2U_{13}\: xz + 2U_{23}\: yz
\end{align}
It then follows that
\begin{equation}
\partialder{z^2_{A,B}}{U_{11}} = \frac{x^2}{\norm{\vec{r}}^2} ,\qquad
\partialder{z^2_{A,B}}{U_{22}} = \frac{y^2}{\norm{\vec{r}}^2} ,\qquad
\partialder{z^2_{A,B}}{U_{33}} = \frac{z^2}{\norm{\vec{r}}^2} ,
\end{equation}
and
\begin{equation}
\partialder{z^2_{A,B}}{U_{12}} = \frac{2xy}{\norm{\vec{r}}^2} ,\qquad
\partialder{z^2_{A,B}}{U_{13}} = \frac{2xz}{\norm{\vec{r}}^2} ,\qquad
\partialder{z^2_{A,B}}{U_{23}} = \frac{2yz}{\norm{\vec{r}}^2} .
\end{equation}
These can be combined with \eqnref{r_derivative} to give us the derivatives
with respect to each $U_{ij}$ component.

\section{Isotropic ADP restraint}
Here we minimise the difference between the Cartesian ADPs, $\mat{U}_{cart}$ and
the isotropic equivalent, $\mat{U}_{eq}$.  As in section \ref{ADP:similarity}, we
must remember that we are dealing with symmetric matrices, and we can therefore
define the weighted least-squares residual as
\begin{equation}
R = w \left( \sum_{i=1}^3 (U_{ii} - U_{eq,ii})^2 + 2 \sum_{i<j} (U_{ij} - U_{eq,ij})^2 \right) ,
\end{equation}
where
\begin{equation}
\mat{U}_{eq} =
\begin{pmatrix} U_{iso} & 0 & 0\\
  0 & U_{iso} & 0\\
  0 & 0 & U_{iso}\end{pmatrix},
\end{equation}
and
\begin{equation}
U_{iso} = \tfrac{1}{3} \mathrm{tr}(\mat{U}_{cart}).
\end{equation}
We expand the summation of the residual as follows
\begin{equation}
R = w \left( (U_{11} - U_{iso})^2 + (U_{22} - U_{iso})^2 + (U_{33} - U_{iso})^2 + 2 U_{12}^2 + 2 U_{13}^2 + 2 U_{23}^2 \right) .
\end{equation}
We can now see by inspection that the derivatives of the residual with respect to the off-diagonal elements are
\begin{equation}
\partialder{R}{U_{ij,i\neq j}} = 4 w U_{ij}.
\end{equation}
The derivatives of the residual with respect to the diagonal elements can be obtained as follows
\begin{align}
\partialder{R}{U_{11}} =& w \left( 2 (U_{11} - U_{iso})\partialder{(U_{11} - U_{iso})}{U_{11}}\right. \nonumber\\
                        &+ 2 (U_{22} - U_{iso})\partialder{(U_{22} - U_{iso})}{U_{22}}\nonumber\\
                        &+ 2 \left. (U_{33} - U_{iso})\partialder{(U_{33} - U_{iso})}{U_{33}} \right) \nonumber\\
                       =& w \left( 2 (U_{11} - U_{iso})(1 - \tfrac{1}{3}) + 2 (U_{22} - U_{iso})(-\tfrac{1}{3}) + 2 (U_{33} - U_{iso})(-\tfrac{1}{3}) \right) \nonumber\\
                       =& w \left( \tfrac{4}{3} U_{11} - \tfrac{2}{3} U_{22} - \tfrac{2}{3} U_{33}\right) \nonumber\\
                       =& 2 w (U_{11} - U_{iso}) .
\end{align}
This can be generalised as
\begin{equation}
\partialder{R}{U_{ii}} = 2 w (U_{ii} - U_{iso}) .
\end{equation}

\bibliography{cctbx_references}

\end{document}


\chapter{Field Theory}
\label{sec:field_theory}
In the last sections, we dealt with discrete and finite dimensional problems. Now, we want to extend our theory to an infinite number of degrees of freedom and apply it on field theory. The main difference is that we can analyse not only several points but every point in space at once. Imagine assigning the amplitude $q(\bar{x})$ to each point $\bar{x} \in \mathbb{R}^3$:

\begin{example}
\begin{align}
L(q_i,\dot{q}_i) = \sum_{i=1}^n \left( \frac{\dot{q}_i^2}{2} - \gamma \frac{q_i^2}{2} \right).
\end{align}
In the finite dimensional case, we just have n oscillating points.
\begin{align}
L[q_{\bar{x}},\dot{q}_{\bar{x}}] = \int d^3 x \left( \frac{\dot{q}^2(\bar{x})}{2} - \gamma \frac{q^2(\bar{x})}{2} \right).
\end{align}
Now, we have oscillators in every point of space.
\end{example}

In field theory, for instance, we want to know the value of the field $\varphi$ in every point $\bar{x}$, see Fig.~\ref{fig:6}. Of course, there is the option to assign more than one field to every point $\bar{x}$.
One can see that the discrete index $i$ is replaced by the continous variable $\bar{x}$.

\begin{figure}[H]
\begin{center}
\includegraphics[scale=0.7]{img/field.png}
\end{center}
\caption{Assigning the value of the field $\varphi$ to each point $\bar{x} \in \mathbb{R}^3$.}
\label{fig:6}
\end{figure}

The difference between finite dimensional and infinite dimensional theory is summerized in the following table:

\begin{table}[H]
\begin{tabular}{>{\centering}p{.45\textwidth} | >{\centering}p{.45\textwidth}}
Discrete case \vspace{5pt} & Continuous case \vspace{5pt} \tabularnewline \hline
\vspace{5pt} $q_i(t), \ \ \ i=1, \dots, n$ & \vspace{5pt} $q_{\bar{x}}(t) = q(\bar{x},t), \ \ \ \bar{x} \in \mathbb{R}^3$ \tabularnewline
\vspace{3pt} $\sum_{i=1}^n$ & \vspace{3pt} $\int d^3 x$ \tabularnewline
\vspace{3pt} $L(q_1, \dots, q_n, \dot{q}_1, \dots, \dot{q}_n)$ & \vspace{3pt} $L[q_{\bar{x}}, \dot{q}_{\bar{x}}]$ \tabularnewline
\end{tabular}
\caption{Comparison between finite dimensional and infinite dimensional theory.}
\end{table}

We want our theory to be relativistic invariant, so we have to make our fields dependent on time $q(t, \vec{x}) = q(x^{\mu})$ and build the Lagrangian out of Lorentz scalars:
\begin{align}
L = \frac{1}{2} \int d^3 x \left( \dot{q}^2 - \gamma q^2 \right) \ \ \longrightarrow \ \ \frac{1}{2} \int d^3 x \left( \partial_{\mu} q \partial^{\mu} q - \gamma q^2 \right),
\end{align}
where $\mu = 0,1,2,3$. If we apply a Lorentz transformation on the old Lagrangian, space and time are mixed together so that we get a different result. The new Lagrangian should be build in a way to compensate this mixing and leave the action invariant under Lorentz transformations. We will use the $(+ - - -)$ signature of Minkowski spacetime. \\
The simplest example for such a relativistic invariant field theory is the Klein-Gordon-Fock field:

\begin{example}[Klein-Gordon-Fock field]
\begin{align}
L[\varphi, \dot{\varphi}] &= \frac{1}{2} \int d^3 x \left( \partial_{\mu} \varphi \partial^{\mu} \varphi - m^2 \varphi^2 \right) \notag \\
&= \frac{1}{2} \int d^3 x \left( \dot{\varphi}^2 - (\nabla \varphi)^2 - m^2 \varphi^2 \right) \notag \\
&= \frac{1}{2} \int d^3 x \left( \dot{\varphi}^2 + \varphi \Delta \varphi - m^2 \varphi^2 \right) \notag \\
&= \frac{1}{2} \int d^3 x \left( \dot{\varphi}^2 + \varphi (\Delta - m^2) \varphi \right).
\end{align}
This Lagrangian is regular, so no constraints appear.
\end{example}

We would like to illustrate the Hamiltonian formulation of field theory with the KGF field. The above form of the Lagrangian will be useful later. \\
The next step is to find the Hamiltonian from a given Lagrangian. In the discrete case, we used to differentiate the Lagrangian with respect to the generalized velocities to find the conjugated momenta. What are the generalized velocities now and how to define the conjugated momenta in field theory? To answer this questions, we will need a mathematical tool which we want to develop in the next section.

\pagebreak

\section{Functional derivative}

The directional derivative for a function $M(q_1, \dots, q_n)$ of $n$ varibles is defined as
\begin{align}
\frac{\partial M}{\partial q_k} = \lim_{a \rightarrow 0} \ \frac{M(q_1, \dots, q_k + a, \dots, q_n) - M(q_1, \dots, q_k, \dots, q_n)}{a}.
\end{align}
We want to generalize the directional derivative for functionals
\begin{align}
M(q_1, \dots, q_n) \ \longrightarrow \ M[q(x)].
\end{align}
The problem is that the functional doesn't depend on the variable $x$ but on the function $q$. For instance, if $x \in [a,b]$, we have to consider every possible path from $a$ to $b$:
\begin{figure}[H]
\begin{center}
\includegraphics[scale=1.8]{img/variation.pdf}
\end{center}
\caption{Variation of a path $q(x)$ with fixed endpoints.}
\label{fig:7}
\end{figure}
Usually, the knowledge of the complete functional for all possible paths is not required. We rather want to know the behavior of the functional in the vicinity of the function $q$, which makes it extremal or stationary. Therefore, we can take the path $q(x)$ and disturb or variate it by adding a small but arbitrary function $\delta q(x)$. One way of generalizing the directional derivative is by using the variation $\delta M$ of the functional $M[q(x)]$:

\begin{definition}[Functional derivative]
\begin{align}
\delta M \equiv M[q(x) + \delta q(x)] - M[q(x)] = \displaystyle\int\limits_{a}^{b} dx \ \frac{\delta M[q]}{\delta q(x)} \ \delta q(x).
\end{align}
\end{definition}

\pagebreak

If we write $\delta q(x) = \varepsilon \ \eta(x)$, where $\eta(x)$ is an arbitrary function and $\varepsilon$ is infinitesimally small, we can Taylor expand our definition and find:
\begin{align}
\left. \frac{d M[q + \varepsilon \eta]}{d\varepsilon} \ \right|_{\varepsilon=0} = \displaystyle\int\limits_{a}^{b} dx \ \frac{\delta M[q]}{\delta q(x)} \ \eta(x).
\end{align}

This is a useful result to calculate the functional derivative, if one has more complicated functionals. Now, let us look at some simple examples to get comfortable with the definition and understand what's going on.
\vspace{15pt}
\begin{example}
Consider the following functional
\begin{align}
M[q(x)] =  \displaystyle\int\limits_{a}^{b} dx \ q(x).
\end{align}
We can use our definition of the functional derivative
\begin{align}
\delta M = \displaystyle\int\limits_{a}^{b} dx \left( q(x) + \delta q(x) \right) - \displaystyle\int\limits_{a}^{b} dx \ q(x) = \displaystyle\int\limits_{a}^{b} dx \ \delta q(x) \overset{!}{=} \displaystyle\int\limits_{a}^{b} dx \ \frac{\delta M}{\delta q(x)} \ \delta q(x)
\end{align}
and read off the result
\begin{align}
\frac{\delta M}{\delta q(x)} = 1.
\end{align}
\end{example}
\vspace{20pt}
\begin{example}
Next, let us take the functional
\begin{align}
K[q(x)] =  \displaystyle\int\limits_{a}^{b} dx \ q^n(x).
\end{align}
Then, we can use our helpful result
\begin{align}
\left. \frac{d K[q + \varepsilon \eta]}{d\varepsilon} \ \right|_{\varepsilon=0} = \displaystyle\int\limits_{a}^{b} dx \ n q^{n-1}(x) \ \eta(x) \overset{!}{=} \displaystyle\int\limits_{a}^{b} dx \ \frac{\delta K}{\delta q(x)} \ \eta(x)
\end{align}
to find
\begin{align}
\frac{\delta K}{\delta q(x)} = n q^{n-1}(x).
\end{align}
\end{example}
\vspace{15pt}

One can see that the functional derivative is very similar to the ordinary derivative. In fact, all the properties of the ordinary derivative (like Leibniz rule, chain rule, ...) are satisfied by the functional derivative. We want to show one last useful tool for working with functional derivatives. It is no coincidence that the functional derivative of the above examples looks like the ordinary derivative. \\

More general, we have the property that
\begin{align}
M[q(x)] = \displaystyle\int\limits_{a}^{b} dx \ f(q(x)) \ \ \Longrightarrow \ \ \frac{\delta M}{\delta q(y)} = f'(q(y)).
\end{align}

\begin{proof}
Consider a parametrized family of functionals $M_z[q(x)]$. For a fixed $q$, we can have different outputs depending on $z$. So it looks much like a function with variable $z$:
\begin{align}
M_z[q(x)] \equiv q(z).
\end{align}
By calculating the variation
\begin{align}
\delta M_z &= M_z[q(x) + \delta q(x)] - M_z[q(x)] = q(z) + \delta q(z) - q(z) = \delta q(z) \notag \\
&= \displaystyle\int\limits_{a}^{b} dx \ \delta(x-z) \delta q(x).
\end{align}
we note that
\begin{align}
\frac{\delta q(z)}{\delta q(x)} \equiv \frac{\delta M_z}{\delta q(x)} = \delta(x-z).
\end{align}
With a bit more work, one can show some more properties of the functional derivative (like changing the order of integration and derivation or the chain rule) and get
\begin{align}
\frac{\delta M}{\delta q(y)} = \displaystyle\int\limits_{a}^{b} dx \ \frac{\delta f(q(x))}{\delta q(y)} = \displaystyle\int\limits_{a}^{b} dx \ f'(q(x)) \frac{\delta q(x)}{\delta q(y)} = f'(q(y)).
\end{align}
\end{proof}

\section{Hamiltonian description}

Now, we are in the position to continue with the Hamiltonian description of field theory. First, we will define the needed quantities in general and see how the Hamiltonian equations generalizes in field theory. After, we will apply it to the KGF field and see how to interpret the results. Let's go ahead and define the conjugated or generalized momentum for a given field $\varphi$ to be:

\begin{definition}[Generalized momentum]
\begin{align}
\pi(\bar{x}) = \frac{\delta L}{\delta \dot{\varphi}(\bar{x})}.
\end{align}
\end{definition}

The Hamiltonian will be given by
\begin{definition}[Hamiltonian]
\begin{align}
H[\varphi(\bar{x}), \pi(\bar{x})] = \int d^3 x \ \dot{\varphi}(\bar{x}) \pi(\bar{x}) \ - \ L.
\end{align}
\end{definition}

The equation of motion (or time-evolution) is calculated like before
\begin{align}
\dot{g}[\varphi(\bar{x}), \pi(\bar{x})] = \left\{ g,H \right\},
\end{align}
where the Poisson bracket is defined analogous to the discrete case by
\begin{definition}[Poisson bracket]
\begin{align}
\left\{ f,g \right\} \equiv \displaystyle\int d^3 z \left( \frac{\delta f}{\delta \varphi(\bar{z})} \frac{\delta g}{\delta \pi(\bar{z})} - \frac{\delta g}{\delta \varphi(\bar{z})} \frac{\delta f}{\delta \pi(\bar{z})} \right).
\end{align}
\end{definition}

The Hamiltonian equations of motion in field theory take the form:
\begin{align}
\dot{\varphi}(\bar{x}) = \left\{ \varphi(\bar{x}),H \right\} = \displaystyle\int d^3 z \ \frac{\delta \varphi(\bar{x})}{\delta \varphi(\bar{z})} \frac{\delta H}{\delta \pi(\bar{z})} &= \displaystyle\int d^3 z \ \delta^3(\bar{x} - \bar{z}) \frac{\delta H}{\delta \pi(\bar{z})} = \frac{\delta H}{\delta \pi(\bar{x})} \\
\dot{\pi}(\bar{x}) = \left\{ \pi(\bar{x}),H \right\} = - \displaystyle\int d^3 z \frac{\delta H}{\delta \varphi(\bar{z})} \frac{\delta \pi(\bar{x})}{\delta \pi(\bar{z})} &= - \displaystyle\int d^3 z \frac{\delta H}{\delta \varphi(\bar{z})} \delta^3(\bar{x} - \bar{z}) = - \frac{\delta H}{\delta \varphi(\bar{x})}.
\end{align}

Let's apply this results to the Klein-Gordon-Fock field. Remembering the Lagrangian for the KGF field, one gets for the generalized momentum
\begin{align}
\pi(\bar{x}) = \frac{\delta L}{\delta \dot{\varphi}(\bar{x})} = \dot{\varphi}(\bar{x}).
\end{align}
The Hamiltonian can be written as:
\begin{align}
H[\varphi(\bar{x}), \pi(\bar{x})] &= \int d^3 x \ \dot{\varphi}(\bar{x}) \pi(\bar{x}) \ - \ L \notag \\
&= \frac{1}{2} \int d^3 x \left( \pi^2(\bar{x}) + (\nabla \varphi)^2 + m^2 \varphi^2 \right).
\end{align}
The Hamiltonian equations of motion are
\begin{align}
\dot{\varphi}(\bar{x}) &= \frac{\delta H}{\delta \pi(\bar{x})} = \pi(\bar{x}) \label{eq:1h} \\
\dot{\pi}(\bar{x}) &= - \frac{\delta H}{\delta \varphi(\bar{x})} = - m^2 \phi(\bar{x}) - \frac{\delta \left[\displaystyle\int (\nabla \varphi)^2 d^3 y \right]}{2 \ \delta \varphi(\bar{x})} \notag \\
&= - m^2 \phi(\bar{x}) - \displaystyle\int d^3 y \ \nabla \varphi(\bar{y}) \ \frac{\delta (\nabla \varphi(\bar{y}))}{\delta \varphi(\bar{x})} \notag \\
&= - m^2 \phi(\bar{x}) - \displaystyle\int d^3 y \ \nabla_y \varphi(\bar{y}) \ \nabla_y \left( \delta^3(\bar{x}-\bar{y}) \right) \notag \\
&= - m^2 \varphi(\bar{x}) + \displaystyle\int d^3 y \ \Delta \varphi(\bar{y}) \ \delta^3(\bar{x}-\bar{y}) \notag \\
&= - m^2 \varphi(\bar{x}) + \Delta \varphi(\bar{x}). \label{eq:2h}
\end{align}

Differentiating equation \eqref{eq:1h} and inserting equation \eqref{eq:2h} yields
\begin{align}
\ddot{\varphi}(t, \bar{x}) = \dot{\pi}(t, \bar{x}) = (\Delta - m^2) \ \varphi(t, \bar{x}),
\end{align}
which is nothing less than the Klein-Gordon-Fock equation.

\begin{definition}[Klein-Gordon-Fock equation]
\begin{align}
(\partial_{\mu} \partial^{\mu} + m^2) \ \varphi(x^{\nu}) = 0.
\end{align}
\end{definition}

\vspace{10pt}

Now, we want to analyse this equation a bit further. Its general solution is:
\begin{align}\label{eq:KGF}
\varphi(t,\bar{x}) = \displaystyle\int d^3 k \ c_1(\bar{k}) \ \text{e}^{i k_{\mu} x^{\mu}} + \displaystyle\int d^3 k \ c_2(\bar{k}) \ \text{e}^{- i k_{\mu} x^{\mu}},
\end{align}
where $c_1(\bar{k}), c_2(\bar{k})$ are complex coefficients for each $\bar{k}$. If $c_2(\bar{k}) = c_1^*(\bar{k})$, the field $\varphi(t,\bar{x})$ is real-valued. One can easily derive a condition which $k_{\mu}$ must satisfy in order to be a solution of the KGF equation. Inserting the plane wave exponential into the KGF equation, we get
\begin{alignat*}{2}
    &\qquad& (\partial_{\mu} \partial^{\mu} + m^2) \text{e}^{i k_{\mu} x^{\mu}} &= 0 \\
    &\Leftrightarrow& \left((i k_0)^2 - (- i \bar{k})^2 + m^2 \right) \text{e}^{i k_{\mu} x^{\mu}} &= 0 \\
    &\Leftrightarrow& \left( - k_0^2 + \bar{k}^2 + m^2 \right) \text{e}^{i k_{\mu} x^{\mu}} &= 0 .
\end{alignat*}
Since the exponential isn't zero, it follows that we must have
\begin{align}
k_0 = \pm \sqrt{m^2 + \bar{k}^2}
\end{align}
for a solution of the KGF equation. This is the relativistic dispersion relation for a particle with
mass $m$. That's why we really can interpret the quadratic terms in the fields like mass terms. From here, it follows also that we have two solutions, one with positive frequency and one with negative frequency. If we change $\bar{k} \rightarrow -\bar{k}$, the sign of the space part does change too but the time part stays the same:
\begin{align}
k_{\mu} x^{\mu} = \sqrt{m^2 + \bar{k}^2} \ t - \bar{k} \bar{x}.
\end{align}
That's why we can split the general solution into two parts like in \eqref{eq:KGF}. Since the KGF Lagrangian is regular, we have no constraints. In electrodynamics, this is not the case anymore and we will see how constraints enter into the Hamiltonian description.


\section{Tricks}
\begin{enumerate}

\item
In a script, line breaking is allowed provided the line breaks occur immediately after operators.
The scanner will automatically go to the next line after an operator.

\item
Setting \verb$trace=1$ in a script causes each line to be printed just before it is evaluated.
This is useful for debugging.

\item
The last result is stored in the symbol $last$.

\item
Use \verb$contract(A)$ to get the mathematical trace of matrix $A$.

\item
Use \verb$binding(s)$ to get the unevaluated binding of symbol $s$.

\item
Use \verb$s=quote(s)$ to clear symbol $s$.

\item
Use \verb$float(pi)$ to get the floating point value of $\pi$.
Set \verb$pi=float(pi)$ to evaluate expressions with a numerical value for $\pi$.
Set \verb$pi=quote(pi)$ to make $\pi$ symbolic again.

\item
Assign strings to unit names so they are printed normally.
For example, setting \verb$meter="meter"$ causes the symbol {\it meter}
to be printed as meter instead of $m_{eter}$.

\item
Use \verb$expsin$ and \verb$expcos$ instead of \verb$sin$ and \verb$cos$.
Trigonometric simplifications occur automatically when exponentials are used.

\item
Use \verb$A==B$ or \verb$A-B==0$ to test for equality of $A$ and $B$.
The equality operator \verb$==$ uses a cross multiply algorithm to eliminate denominators.
Hence \verb$==$ can typically determine equality even when the unsimplified result of $A-B$ is nonzero.
Note: Equality tests involving floating point numbers can be problematic
due to roundoff error.

\item
If local symbols are needed in a function, they can be appended to {\it arg-list}.
(The caller does not have to supply all the arguments.)
The following example uses Rodrigues's formula to
compute an associated Legendre function of $\cos\theta$.
\begin{equation*}
P_n^m(x)=\frac{1}{2^n\,n!}(1-x^2)^{m/2}\frac{d^{n+m}}{dx^{n+m}}(x^2-1)^n
\end{equation*}
Function $P$ below first computes $P_n^m(x)$ for local variable
$x$ and then uses {\it eval} to replace $x$ with $f$.
In this case, $f=\cos\theta$.

\begin{Verbatim}[formatcom=\color{blue}]
x = 123 -- global x in use, need local x in P
P(f,n,m,x) = eval(1/(2^n n!) (1 - x^2)^(m/2) d((x^2 - 1)^n,x,n + m),x,f)
P(cos(theta),2,0) -- arguments f, n, m, but not x
\end{Verbatim}

\noindent
$\displaystyle \tfrac{3}{2} \cos(\theta)^2-\tfrac{1}{2}$

\bigskip
\noindent
Note: The maximum number of arguments is nine.

\end{enumerate}


\chapter{Markov Random Fields}
\emph{“但是你们肯定没我懂”}
\newpage

\section{Markov Random Fields}
    A stochastic process is a collection of random variables $\{X_t|x\in\mathbb{T}\}$ where $\mathbb{T}$ is a subset of reals $\mathbb{R}$. And a \textbf{Random Field} is a generalization of a stochastic process.

    \subsection{Markov Random Fields}
        \begin{definition}[Random Field]
            A \textbf{Random Field} is a collection of random variables $\{X_s:s\in\mathcal{G}\}$, where $\mathcal{G}$ is not necessarily a subset of $\mathbb{R}$. And we will consider $\mathcal{G}$ to be a set of nodes of a \emph{graph}.
        \end{definition}

        \begin{definition}
            Given a graph $\mathcal{G}$ with nodes $V=\{1,2,\dots,n\}$, let $\mathcal{N}(t)$ denote the set of neighbours of $t$. The collection of $\{X_1, \dots, X_n\}$ is a \textbf{Markov Random Field} if
            \[ \forall v \quad \mathbb{P}\left[X_v = i \middle\vert \bigwedge_{w \in V-\{v\}} X_w = j_w\right] = \mathbb{P}\left[X_v = i \middle\vert \bigwedge_{w\in\mathcal{N}(v)}X_w=j_w\right] \]
            i.e. the state of current node is only dependent on its neightbours.
        \end{definition}

    \subsection{Some Notations}
        Let $x$ be the values of random varialbes $[X_1,\dots,X_n]$(a vector of length $n$), where each element $x_v$ (the state of each node) come from a state space $\{1,2,\dots,q\}$. Let $S$ be a subset of $V$, then
        \begin{itemize}
            \item We use $x_S$ to denote the values of random varialbes $X_v \quad v \in S$. (a vector $[x_v]$ of length $|S|$, where $v\in S$)
            \item We use $p(x_S)$ to denote $\mathbb{P}[X_S=x_S]$.
        \end{itemize}
        $p(x_S)$ is the probability that the nodes in $S$ have states in $x_S$.

        Let $S$, $T$ be two subsets of $V$, let $x$ be the vector of nodes in $S$, let $y$ be the vector of nodes in $T$, then
        \begin{itemize}
            \item We use $p(x_S|y_T)$ to denote $\mathbb{P}[X_S=x_S|X_T=y_T]$
        \end{itemize}

        And here comes the problem: In a Markov chain, once the initial distribution and the probability transition matrix are specified, we will be able to construct and simulate the chain. However, for a MRF, even if we know all $\mathbb{P}[X_t=j|\mathcal{N}(j)]$, we may still be unable to construct the MRF, because the joint distribution of all $v \in V$ may not even exist.

        Consider the simplest yet still non-trivial MRF, which consists of only two nodes $X_1$ and $X_2$ taking values in $\{0,1\}$. We have to specify 3 of the 4 values $p(x_1, x_2)$ (only 3 are required because they sum up to 1 so the fourth one is determine once the remaining 3 are specified), such that they satify the 4 specified conditional probabilities $\mathbb{P}[x_1|x_2=1/0]$ and $\mathbb{P}[x_2|x_1=1/0]$. This is generally difficult (just like we cannot generally solve a system of 4 equations containing only 3 unknown variables).

        Therefore in MRF, specifying all conditional probabilities is of little help. Luckily, an alternative exists.

\section{Gibbs Distribution}
    In this section, we discuss the Gibbs Distribution. A random field having the Markov property is equivalent to its having a Gibbs distribution, and the latter one is much more friendly than conditional probabilities.

    \subsection{Completeness}
        \begin{definition}[Complete Graphs]\label{def:CompleteGraph}
            A set of nodes $C$ is \textbf{complete} if all distinct nodes in $C$ are neightbours of each other.
        \end{definition}
        \begin{remark}
            That is, $C$ is not complete if two of its nodes are not neightbours.
        \end{remark}

        \begin{definition}[Clique]\label{def:Clique}
            A \textbf{clique} of a set $C$ is a maximal complete set of nodes in $C$.
        \end{definition}

    \subsection{Gibbs Distribution}
        \begin{definition}[Gibbs Distribution]\label{def:GibbsDistribution}
            Let $\mathcal{G}$ be a finite graph. A \textbf{Gibbs Distribution} with respect to $\mathcal{G}$ is a probability mass function that can be expressed in the form of
            \[ p(x) = \prod_{\text{$C$ complete}}V_C(x) \]
            where $V_C(\cdot)$ is a function that only depends on the values $x_C = [x_v:v \in C]$ of $x$ at the nodes in a clique $C$. That is, if $x_C = y_C$, then $V_C(x) = V_C(y)$.
        \end{definition}

    \subsection{Examples of Gibbs Distibutions}
        \subsubsection{Independent Sets on a Graph}
        Let $\mathcal{G}=(V,E)$ be a graph. We want to generate a mask $\{0,1\}^{|V|}$ such that the result is a independent set. \emph{The uniform distribution of all independent sets of $\mathcal{G}$ has a Gibbs distribution}.

        Let $A$ be any complete sub-graph of $\mathcal{G}$, we define a weight function $w_A$ on $A$
        \[ w_A(x) = \begin{cases}
            0 &\quad |A| = 2, x_A = [1,1]\\
            1 &\quad o.w.
        \end{cases} \]

        Therefore
        \[ p(x) \sim w_A(x) = \prod_{\{i,j\}\in E}\mathbb{I}[x(i) \neq 1 \vee x(j) \neq 1] \]

        \subsubsection{Proper Coloring}
        Consider the coloring of a graph, suppose there are $q$ colors, let $\Omega=\{1,2,\dots,q\}^{|V|}$ be the sample space consisting of all possible color assignment to each node in $V$. Let $c$ be a mapping from $V$ to $\Omega$, which represents a way to color all the nodes. A coloring is \textbf{proper} if no neighbouring nodes have the same color. We will show that \emph{the uniform distribution of all proper colorings on the graph has a Gibbs distribution}.
        \[ \forall \{i,j\} \in E \quad c(i) \neq c(j) \]

        We define a weight function $w_A(x)$ on all complete subgraphs of $\mathcal{G}$ by
        \[ w_A(c) = \begin{cases}
            0 &\quad c(i) = c(j), |A|=2\\
            1 &\quad o.w.
        \end{cases} \]

        And Therefore
        \[ p(c) \sim w_A(c) = \prod_{\{i,j\} \in E}\mathbb{I}[c(i) \neq c(j)] \]

        \subsubsection{Ising Model}
        Consider a graph $\mathcal{G}$ whose nodes form a subset of $\mathbb{Z}^d$. For each $v \in V$ there is a corresponding random variable $X_v \in \{0,1\}$ (sometimes in $\{-1,1\}$). The Ising model gives a joint distribution of these random variables.

        Let $\sigma: V \mapsto \{0,1\}$ be the configuration of each node. We consider a special case: let $m(x)$ be the set of monochromatic edges in $E$. By ``monochromatic'' we mean the configuration of the nodes $i,j$ of edge $e=\{i,j\}$ are the same.
        \[ m(x) = \{\{i,j\}: \sigma(i) = \sigma(j)\} \]

        The Ising model is parametrized by a positive parameter $\beta$ (typically $\beta < 1$). Let
        \[ w(x) = \beta^{|m(x)|} \]

        Then the distribution $\mu(x) \sim w(x)$ is a Gibbs distribution.
        \[ \mu(x) \sim w(x) = \prod_{\{i,j\} \in E}\beta^{\mathbb{I}[x(i)\neq x(j)]} \]

    \subsection{Hammersley-Clifford Theorem}
        \begin{theorem}[Hammersley-Clifford]\label{thm:HammersleyClifford}
            Suppose that $X=(X_1,\dots,X_n)$ has a positive joint probability mass function $\mu(X) > 0 \quad \forall X$. Then $X$ is a Markov random field on $\mathcal{G}$ if and only if $X$ has a Gibbs distribution w.r.t. $\mathcal{G}$.
        \end{theorem}

        The proof of the theorem is skipped due to time limitation. Can be found in References.

\section{Hidden Markov Models}
    A Hidden Markov Model is a Markov random fields in which some random variables are observable, while others are not.

    Suppose a hidden random variable chain is a Markov Chain $\{X_t\}$ with initial distribution $\xi$ and transition probability $A$, and suppose each hidden state $X_t$ emits an observation $Y_t$ according to a probability matrix $B$. We can only oberseve $Y_t$, and we want to estimate $\theta ] \{\xi, A, B\}$.

    Given a sequence of observations $y = (y_1, y_2, \dots ,y_n)$, our goal is to find
    \[ \hat{\theta} = \arg\max_{\theta} p_{\theta}(y) = \sum_x p_{\theta}(x,y) \]

    If we can directly maximize $p_{\theta}(y)$, then we are done. But this is practically impossible because we need to enumerate over all possible $x$.

    \subsection{Expectation Maximization}
        Let $L(\theta) = p_{\theta}(y)$. Maximizing $L(\theta)$ is equivalent to maximizing $\mathcal{L}(\theta) = \log L(\theta)$.

        The EM algorithm maximizes the \emph{Expectation} of $p_{\theta}(X,y)$ due to the complexity of directly maximizing $p_{\theta}(x,y)$. Furthermore, it maximizes $\mathbb{E}[p_{\theta_t}(X,y)]$ using a current estimator $\theta_t$ of $\theta$, and update $\theta_t$ iteratively.

        The general framework for the M-Step of an EM Algorithm is
        \[ \theta_{t+1} = \arg\max_{\theta} \mathbb{E}_{\theta_t}\left[ \log p_{\theta}(X,y)|Y=y \right] \]

        \subsubsection{Proof Of Correctness}
        \begin{definition}[KL Divergence]\label{def:KLDivergence}
            The \textbf{KL-Divergence} is used to measure the difference of two distributions
            \[ D_{KL}(p\|q) = \sum_i p_i\log p_i - \sum_i p_i \log q_i \]
        \end{definition}
        \begin{proposition}[Non-negativity of KL Divergence]\label{prop:PositivityOfKLDivergence}
            \[ D_{KL}(p\|q) \ge 0 \]
            The equality is achieved if and only if $p=q$.
        \end{proposition}

        \begin{lemma}\label{lem:IncreasingLowerBoundOfEMAlgo}
            If there exist $\theta_0$ and $\theta_1$ such that
            \[ \mathbb{E}_{\theta_0}[p_{\theta_1}(X,y)|Y=y] > \mathbb{E}_{\theta_0}[p_{\theta_0}(X,y)|y] \]
            Then
            \[ p_{\theta_1}(y) > p_{\theta_0}(y) \]
        \end{lemma}
        \begin{proof}
            By assuption of the lemma, we move the LHS to RHS,
            \begin{align*}
                0 &< \mathbb{E}_{\theta_0}\left[ \log \frac{p_{\theta_1}(X,y)}{p_{\theta_0}(X,y)} \middle\vert Y=y \right]\\
                &= \sum_x p_{\theta_0}(x|y) \log \frac{p_{\theta_1}(x,y)}{p_{\theta_0}(x,y)} \quad \text{(definition of conditional expectation)}\\
                &= \sum_x p_{\theta_0}(x|y) \log \frac{p_{\theta_1}(y)}{p_{\theta_0}(y)} - \sum_x p_{\theta_0}(x,y)\log\frac{p_{\theta_0}(x|y)}{p_{\theta_1}(x|y)} \quad \text{(乘法定理)} \\
                &\le \log\frac{p_{\theta_1}(y)}{p_{\theta_0}(y)}
            \end{align*}
        \end{proof}

    \subsection{M-Step of HMM}
        \[ p_{\theta}(x,y) = \xi(x_0) \cdot \prod_{t=0}^{n-1}A(x_t,x_{t+1}) \cdot \prod_{t=0}^n B(x_t, y_t) \]
        Taking logarithm and expectation,
        \[ \mathbb{E}_{\theta_0}\left[ \log p_{\theta}(X,y)|y \right] = \mathbb{E}_{\theta_0}[\log\xi(x_0)|y] + \sum_{t=0}^{n-1}\mathbb{E}_{\theta_0}[\log A(x_t,x_{t+1})|y] + \sum_{t=0}^n\mathbb{E}_{\theta_0}[\log B(x_t,y_t)|y] \]

        Notice that the first term only involves $\xi$, the second term only involves $A$ and the last tern only involes $B$.

        \subsubsection{Maximizing Term 1}
        \[ Term1 = \sum_i \mathbb{P}_{\theta_0}[x=i|y]\log\xi(i) \]
        By Property of KL-Divergence \ref{prop:PositivityOfKLDivergence}, $Term1$ is maximized when $\xi(i) = \mathbb{P}_{\theta_0}[x=i|y]$.

        \subsubsection{Maximizing Term 2}
        \[ Term2 = \sum_{t=0}^{n-1}\sum_i\sum_j \mathbb{P}_{\theta_0}[x_t=i,x_{t+1}=j|y] \cdot \log A(i,j) \]
        Exchange the summations,
        \[ \sum_i\sum_j \left(\sum_t \mathbb{P}_{\theta_0}[x_t=i,x_{t+1}=j|y] \cdot \log A(i,j)\right) \]
        Again by \ref{prop:PositivityOfKLDivergence}, $A(i,j)$ is maximized when
        \[ A(i,j) = \frac{\sum_t \mathbb{P}_{\theta_0}[x_t=i,x_{t+1}=j|y]}{\sum_j\sum_t \mathbb{P}_{\theta_0}[x_t=i,x_{t+1}=j|y]} \]

        \subsubsection{Maximizing Term 3}
        \begin{align*}
            Term3 &= \sum_i \sum_t \mathbb{P}_{\theta_0}[x_t=i|y]\cdot\log B(i,y_t)\\
            &= \sum_i \sum_j \sum_{t:y_t=j}\mathbb{P}_{\theta_0}[x_t=i|y]\log B(i,j)
        \end{align*}
        By Mafs, $B(i,j)$ is maximized when
        \[ B(i,j) = \frac{\sum_{t:y_t=j}\mathbb{P}_{\theta_0}[x_t=i|j]}{\sum_j\sum_{t:y_t=j}\mathbb{P}_{\theta_0}[x_t=i|j]} \]

    \subsection{E-Step of HMM}
        All the values required in M-step can be efficiently computed by dynamic programming algorithms. This method is generally known as the \emph{forward-backward algorithm}. For details, please refer to AI2651 Notes.

        We first clarify some notations.
        \begin{itemize}
            \item Let $Y=y$ be a sequence of observation, where $y_t$ denotes the observation at time $t$.
            \item Let $X=x$ be a sequence of hidden states, where $X_t=x_t$ denotes the hidden random variable and its value at time $t$.
            \item We use $y_1^t$ to denote the sequence $(y_1,\dots,y_t)$, and similarly use $y_i^j$ to denote $(y_i, y_{i+1},\dots, y_j)$.
            \item Suppose there are $N$ hidden states.
            \item Suppose the time ranges from $0$ to $T$.
        \end{itemize}

        All the probabilities computed below should be conditioned on (or parametrized by) $\theta_0$, but for brevity, the parameter is intentionally omitted in the following part.

        Let
        \[ \gamma_{(i,j)}(t) \triangleq \mathbb{P}[X_t=i, X_{t+1}=j | Y=y] \]
        be the probability that, given an observation $y$, the $t$-th and $(t+1)$-th hidden state are $i$ and $j$ repectively.

        Let
        \[ \gamma_i(t) \triangleq \mathbb{P}[X_t=i|Y=y] \]
        be the probability that, given an observation $y$, the $t$-th hidden state is $i$.

        By the Law of Total Probability,
        \[ \gamma_i(t) = \sum_{j=1}^N \gamma_{(i,j)}(t) \]

        Then the parameters of the HMM is updated by
        \begin{itemize}
            \item $\xi(i) = \mathbb{P}[X_0=i|Y=y]$.
            \item $A(i,j) = \frac{\sum_{t=1}^T \gamma_{(i,j)}(t)}{\sum_{j'=1}^N\sum_{t=1}^T\gamma_{(i,j')}(t)}$
            \item $B(i,j) = \frac{\sum_{t:y_t=j}\gamma_i(t)}{\sum_{t=1}^T\gamma_i(t)}$
        \end{itemize}

        So the problem is reduced to computing $\gamma_{(i,j)}(t)$ and $\gamma_i(t)$.

        By definition of conditional probability,
        \[ \gamma_i(t) = \mathbb{P}[X_t=i|Y=y] = \frac{\mathbb{P}[X_t=i, Y=y]}{\mathbb{P}[Y=y]} \]

        \[ \gamma_{(i,j)}(t) = \mathbb{P}[X_t=i, X_{t+1}=j|Y=y] = \frac{\mathbb{P}[X_t=i, X_{t+1}=j, Y=y]}{\mathbb{P}[Y=y]} \]

        And the problem becomes computing $\mathbb{P}[X_t=i, Y=y]$, $\mathbb{P}[X_t=i,X_{t+1}=j, Y=y]$ and $\mathbb{P}[Y=y]$.

        Define
        \[ \alpha_i(t) \triangleq \mathbb{P}[X_t=i, Y_0^t=y_0^t] \]
        which is the probability that the hidden state at $t$ is $i$ when we observe $y_0^t$ in time interval $0:t$, where we recall that $y_1^t$ is the sequence $(y_0,y_2,\dots,y_t)$.

        Define
        \[\beta_i(t) \triangleq \mathbb{P}[Y_{t+1}^T=y_{t+1}^T|X_t=i] \]
        which is the probability that we observe $y_{t+1}^T$, given the hidden state at $t$ is $i$.

        \begin{align*}
            \alpha_i(t) &= \mathbb{P}[y_0^t, X_t=i]\\
            &= \sum_{j=1}^{N}\mathbb{P}[y_0^t,X_t=i,X_{t-1}=j] \quad \text{(Law of Total Probability)}\\
            &= \sum_{j=1}^{N}\mathbb{P}[y_0^{t-1},y_t,X_t=i,X_{t-1}=j] \quad (\text{Extract $y_t$ from $y_0^t$})\\
            &= \sum_{j=1}^{N}\mathbb{P}[y_t|y_0^{t-1},X_t=i,X_{t-1}=j]\\
            &\quad \cdot \mathbb{P}[X_t=i|y_0^{t-1},X_{t-1}=j]\\
            &\quad \cdot \mathbb{P}[y_0^{t-1},X_{t-1}=j] \quad \text{(Expand terms using conditional expectation)}\\
            &= \sum_{j=1}^{N}\mathbb{P}[y_t|X_t=i] \cdot \mathbb{P}[X_t=i|X_{t-1}=j] \cdot \mathbb{P}[y_0^{t-1},X_{t-1}=j]\quad\text{(Assumption of HMM/MRF)}\\
            &= B(i,y_t)\sum_{j=1}^{N}A(j,i)\alpha_j(t-1) \quad \text{(Definitions)}
        \end{align*}
        This result gives a recursive relation between $\alpha_i(t)$ and all $\alpha_j(t-1)$, so $\alpha_i(t)$ can be computed by memoized search over $t=0:T$.

        Similarly,
        \begin{align*}
            \beta_i(t) &= \mathbb{P}[y_{t+1}^T|X_t=i]\\
            &= \sum_{j=1}^{N}\mathbb{P}[y_{t+1},y_{t+2}^T,X_{t+1}=j|X_t=i]\\
            &= \sum_{j=1}^{N}\mathbb{P}[y_{t+1}|X_{t+1}=j]\mathbb{P}[X_{t+1}=j|X_t=i]p(y_{t+2}^T|_{t+1}=j)\\
            &= \sum_{j=1}^{N}B(j, y_{t+1})A(i,j)\beta_{j}(t+1)
        \end{align*}
        Again this result is a recursive relation between $\beta_i(t)$ and $\beta_j(t+1)$, so $\beta_i(t)$ can be computed by memoized search over $t=T:0$.

        Notice that the $A(i,j)$ and $B(i,j)$ here are the results computed in the previous iteration.

        Once all $\alpha_i(t)$ and $\beta_i(t)$ are computed, using the law of total probability and enumerate all hidden state at some time $t$, we have
        \[ \mathbb{P}[Y=y] = \sum_{i=1}^N\mathbb{P}[y_0^t, y_{t+1}^T, X_t=i] = \sum_{i=1}^N \mathbb{P}[y_0^t, X_t=i]\mathbb{P}[y_{t+1}^T|X_t=i] = \sum_{i=1}^N \alpha_i(t)\beta_i(t) \quad \forall t \]
        Therefore we should be able to compute $\mathbb{P}[Y=y]$.

        Furthermore,
        \begin{align*}
            \mathbb{P}[X_t=i,X_{t+1}=j, y_0^T] &= \mathbb{P}[y_{t+2}^T, y_{t+1}, y_1^t, X_t=i, X_{t+1}=j] \quad \text{(Split $y_1^T$)}\\
            &= \mathbb{P}[y_{t+2}^T|X_{t+1}=j]\\
            &\quad \cdot \mathbb{P}[y_{t+1}|X_{t+1}=j]\\
            &\quad \cdot \mathbb{P}[X_{t+1}=j|X_t=i]\\
            &\quad \cdot \mathbb{P}[X_t=i, y_0^t] \quad \text{(Conditional Probability and HMM assumptions)}\\
            &= \beta_j(t+1)B(j,y_{t+1})A(i,j)\alpha_i(t)
        \end{align*}

        And similarly
        \begin{align*}
            \mathbb{P}[X_t=i, y_0^T] &= \mathbb{P}[y_0^{t}, y_{t+1}^T, X_t=i]\\
            &= \mathbb{P}[y_{t+1}^T|X_t=i]\mathbb{P}[y_0^t, X_t=i]\\
            &= \beta_i(t)\alpha_i(t)
        \end{align*}

        Therefore, once we have $\alpha_i(t)$ and $\beta_i(t)$, and $A(i,j)$ and $B(i,j)$ in the previous iteration, we will also be able to compute $\mathbb{P}[X_t=i, X_{t+1}=j, y_0^T]$ and $\mathbb{P}[X_t=i,y_0^T]$. Then it follows that we will be able to compute $\gamma_{(i,j)}(t)$ and $\gamma_i(t)$.

        \[ \gamma_i(t) = \frac{\beta_i(t)\alpha_i(t)}{\sum_{i=1}^N \alpha_i(t)\beta_i(t)} \]

        \[ \gamma_{(i,j)}(t) = \frac{\beta_j(t+1)B(j,y_{t+1})A(i,j)\alpha_i(t)}{\sum_{i=1}^N \alpha_i(t)\beta_i(t)} \]
        where $A(i,j)$ and $B(i,j)$ are the parameters in the previous iteration, and $\alpha_i(t)$ and $\beta_i(t)$ can be computed efficiently by DP, using the recursive relationship mentioned above.

        And eventually, we will be able to update the parameters.
        \[ A(i,j) = \frac{\sum_{t=1}^T \gamma_{(i,j)}(t)}{\sum_{j'=1}^N\sum_{t=1}^T\gamma_{(i,j')}(t)} \]
        \[ B(i,j) = \frac{\sum_{t:y_t=j}\gamma_i(t)}{\sum_{t=1}^T\gamma_i(t)} \]
        \[ \xi(i) = \gamma_i(0) \]