\section{\module{random} ---
         Generate pseudo-random numbers}

\declaremodule{standard}{random}
\modulesynopsis{Generate pseudo-random numbers with various common
                distributions.}

This module implements pseudo-random number generators for various
distributions: on the real line, there are functions to compute normal
or Gaussian, lognormal, negative exponential, gamma, and beta
distributions.  For generating distribution of angles, the circular
uniform and von Mises distributions are available.

The \module{random} module supports the \emph{Random Number
Generator} interface, described in section \ref{rng-objects}.  This
interface of the module, as well as the distribution-specific
functions described below, all use the pseudo-random generator
provided by the \refmodule{whrandom} module.

The following functions are defined to support specific distributions,
and all return real values.  Function parameters are named after the
corresponding variables in the distribution's equation, as used in
common mathematical practice; most of these equations can be found in
any statistics text.  These are expected to become part of the Random
Number Generator interface in a future release.

\begin{funcdesc}{betavariate}{alpha, beta}
Beta distribution.  Conditions on the parameters are
\code{\var{alpha} > -1} and \code{\var{beta} > -1}.
Returned values range between 0 and 1.
\end{funcdesc}

\begin{funcdesc}{cunifvariate}{mean, arc}
Circular uniform distribution.  \var{mean} is the mean angle, and
\var{arc} is the range of the distribution, centered around the mean
angle.  Both values must be expressed in radians, and can range
between 0 and \emph{pi}.  Returned values will range between
\code{\var{mean} - \var{arc}/2} and \code{\var{mean} + \var{arc}/2}.
\end{funcdesc}

\begin{funcdesc}{expovariate}{lambd}
Exponential distribution.  \var{lambd} is 1.0 divided by the desired
mean.  (The parameter would be called ``lambda'', but that is a
reserved word in Python.)  Returned values will range from 0 to
positive infinity.
\end{funcdesc}

\begin{funcdesc}{gamma}{alpha, beta}
Gamma distribution.  (\emph{Not} the gamma function!)  Conditions on
the parameters are \code{\var{alpha} > -1} and \code{\var{beta} > 0}.
\end{funcdesc}

\begin{funcdesc}{gauss}{mu, sigma}
Gaussian distribution.  \var{mu} is the mean, and \var{sigma} is the
standard deviation.  This is slightly faster than the
\function{normalvariate()} function defined below.
\end{funcdesc}

\begin{funcdesc}{lognormvariate}{mu, sigma}
Log normal distribution.  If you take the natural logarithm of this
distribution, you'll get a normal distribution with mean \var{mu} and
standard deviation \var{sigma}.  \var{mu} can have any value, and
\var{sigma} must be greater than zero.
\end{funcdesc}

\begin{funcdesc}{normalvariate}{mu, sigma}
Normal distribution.  \var{mu} is the mean, and \var{sigma} is the
standard deviation.
\end{funcdesc}

\begin{funcdesc}{vonmisesvariate}{mu, kappa}
\var{mu} is the mean angle, expressed in radians between 0 and 2*\emph{pi},
and \var{kappa} is the concentration parameter, which must be greater
than or equal to zero.  If \var{kappa} is equal to zero, this
distribution reduces to a uniform random angle over the range 0 to
2*\emph{pi}.
\end{funcdesc}

\begin{funcdesc}{paretovariate}{alpha}
Pareto distribution.  \var{alpha} is the shape parameter.
\end{funcdesc}

\begin{funcdesc}{weibullvariate}{alpha, beta}
Weibull distribution.  \var{alpha} is the scale parameter and
\var{beta} is the shape parameter.
\end{funcdesc}

\begin{seealso}
  \seemodule{whrandom}{The standard Python random number generator.}
\end{seealso}

\subsection{The Random Number Generator Interface
            \label{rng-objects}}

% XXX This *must* be updated before a future release!

The \dfn{Random Number Generator} interface describes the methods
which are available for all random number generators.  This will be
enhanced in future releases of Python.

In this release of Python, the modules \refmodule{random},
\refmodule{whrandom}, and instances of the
\class{whrandom.whrandom} class all conform to this interface.

\begin{funcdesc}{choice}{seq}
Chooses a random element from the non-empty sequence \var{seq} and
returns it.
\end{funcdesc}

\begin{funcdesc}{randint}{a, b}
\deprecated{2.0}{Use \function{randrange()} instead.}
Returns a random integer \var{N} such that
\code{\var{a} <= \var{N} <= \var{b}}.
\end{funcdesc}

\begin{funcdesc}{random}{}
Returns the next random floating point number in the range [0.0
... 1.0).
\end{funcdesc}

\begin{funcdesc}{randrange}{\optional{start,} stop\optional{, step}}
Return a randomly selected element from \code{range(\var{start},
\var{stop}, \var{step})}.  This is equivalent to
\code{choice(range(\var{start}, \var{stop}, \var{step}))}.
\versionadded{1.5.2}
\end{funcdesc}

\begin{funcdesc}{uniform}{a, b}
Returns a random real number \var{N} such that
\code{\var{a} <= \var{N} < \var{b}}.
\end{funcdesc}


% !TeX root = ../../main.tex

\chapter{Background}

\lipsum[1]

\begin{equation}
    E = mc^2
\end{equation}

\lipsum[2]

\section{Celestial Bodies in the Solar System}

\lipsum[3]

\begin{figure}[ht]
	% Figures and tables are centred by default
	\includegraphics[width=0.6\textwidth]{pale-blue-dot}
	\caption[Pale Blue Dot]{{\bf Pale Blue Dot.} The \emph{Voyager 1} spacecraft took this photograph of planet Earth on February 14, 1990 when she was more than six billion kilometers away from our home.}
\end{figure}

\subsection{Inner Planets}

\lipsum[4]

\begin{table}[ht]
    \begin{tabular}{lcccc}
        \toprule
                & \textbf{Mass} & \textbf{Diameter} & \textbf{Density}  & \textbf{Gravity} \\
                & $10^{24}$ kg  & km                & kg/m$^3$          & m/s$^2$ \\
        \midrule
        Mercury & 0.33          & 4,879             & 5,427             & 3.7 \\
        Venus   & 4.87          & 12,104            & 5,243             & 8.9 \\
        Earth   & 5.97          & 12,756            & 5,514             & 9.8 \\
        Mars    & 0.64          & 6,792             & 3,933             & 3.7 \\
        \bottomrule
    \end{tabular}
    \caption[Comparison of Inner Planets of the Solar System]{{\bf Comparison of Inner Planets of the Solar System.} Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus pellentesque dictum neque, sollicitudin accumsan purus porttitor vitae. Phasellus faucibus enim.}
\end{table}

\lipsum[5]

\begin{figure}[p]
    \caption[Inner Planets of the Solar System]{{\bf Inner Planets of the Solar System}}
    \begin{subfigure}[b]{0.5\textwidth}
        % While figures and tables are centred by default, subfigures and subtables are not
        \centering
        \includegraphics[height=2in]{mercury}
        \caption{Mercury}
    \end{subfigure}%
    \begin{subfigure}[b]{0.5\textwidth}
        \centering
        \includegraphics[height=2in]{venus}
        \caption{Venus}
    \end{subfigure}
    % Add space between rows
    \subfigurerow
    \begin{subfigure}[b]{0.5\textwidth}
        \centering
        \includegraphics[height=2in]{earth}
        \caption{Earth}
    \end{subfigure}%
    \begin{subfigure}[b]{0.5\textwidth}
        \centering
        \includegraphics[height=2in]{mars}
        \caption{Mars}
    \end{subfigure}
\end{figure}

\subsection{Asteroid Belt}

\lipsum[6]

\subsection{Outer Planets}

\lipsum[7]

\subsection{Kuiper Belt}

\lipsum[8]

\section{Artificial Objects in Heliocentric Orbits}

\lipsum[9]

\subsection{Tesla Roadster and Starman}

\lipsum[10]

\begin{figure}[ht]
    \includegraphics[width=0.8\textwidth]{starman}
    \caption[Don't Panic]{{\bf Don't Panic.} During the maiden flight of SpaceX's Falcon Heavy launch vehicle on February 6, 2018, a Tesla Roadster electric car with the ``Starman'' mannequin in spacesuit is launched into a heliocentric orbit.}
\end{figure}

\lipsum[11]

\begin{equation}
    \Delta v = v_\text{e} \ln \frac{m_0}{m_f}
\end{equation}

\lipsum[12]


\chapter{Appendix}

\begin{longtable}{ | P{6cm} | P{6cm} | }
    \hline
    \textbf{Multi-layer perceptron} & \textbf{Radial Basis Function} \newline \\ \hline
    Internal representations are distributed representations \newline &
    Hidden unit response are local with respect to the input spaces \newline \\ \hline

    Multi-layer architecture \newline &
    Three layer architecture \newline  \\ \hline

    Weights are determined simultaneously\newline &
    Weights are determined by clustering algorithm while output layer is supervised \newline \\ \hline

    Required less hidden-layer unit &
    Required more hidden-layer units \newline \\ \hline
\end{longtable}


\section{Histogram}
\label{sec:histogram}

\subsection{Purpose}
This program performs a multi-dimensional histogram of photon correlation events. These are defined by the program \program{correlate}, though it is also possible to histogram T3 photons directly. The output is a set of histogram bins and the number of events which fall into each. This output is \textit{not} normalized by bin width or any other factor, and represents the raw number of counts falling into each bin.

For T3 photons, \program{histogram} can be used to build a histogram of events as would be done in interactive mode. To do this, set the mode to T3, and order to 1.

For \gn{n>2}, all time dimensions are defined identically, and all pulse dimensions are defined identically. If distinct time dimensions are required, the modification can be achieved at the stage where bins are defined, without any modification later in the code.

\subsection{Command-line syntax}
%\input{programs/histogram.usage}

\subsubsection{Input}
\paragraph{\gn{1} of T3 data}
The expected input is a stream of T3 photons.

\paragraph{\gn{n\ge 2}}
The expected input is the output of \program{correlate}.

\paragraph{General options}
The main options specified for \program{histogram} define the axes for pulse and time. These are treated equivalently, so we will focus on the time axes.

For \texttt{--time}, the parameters specify the lower and upper bounds of the time axis, as well as the number of bins $n$ to create along that axis. For a linear spacing, the bin width $\Delta t$ is $(t_{\max}-t_{\min})/n$, such that the bins are defined by the ranges
\begin{equation}
\begin{aligned}
&[\time_{\min},\time_{\min}+\Delta\time),\\
&[\time_{\min}+\Delta\time,\time_{\min}+2\Delta\time),\\
&\ldots\\
&[\time_{\min}+(n-1)\Delta\time,\time_{\min}+n\Delta\time)
\end{aligned}
\end{equation}
This linear spacing is the default behavior, but the flag \texttt{--time-scale} can produce two other scales: log, and log-zero. The log scale creates bins with fixed width over the span of $[\log(\time_{\min}),\log(\time_{\max}))$, as if often desired for measurements requiring long and short time correlations. The log-zero scale has identical behavior, except that any zero-time correlations ($\timedelay=0$) are placed into the first bin. Note that the log scale cannot handle zero-time correlations, and neither log not log-zero can handle negative-time correlations. These values will be dropped from the histogram, with an error message indicating this has happened.

\subsubsection{Output}
After the input stream terminates, \program{histogram} outputs the bin definitions and the number of counts associated with that bin. Generically, this format is:
\begin{verbatim}
channel 0, channel 1, bin (1,1) lower, bin (1,1) upper, ...,
    channel 2, ... ,
    counts \n
\end{verbatim}
where the channels are integers, bin edges are floats, and the counts are integers. For every bin in the histogram, one line will be output. For T2 mode, the bin definition has only one dimension (time), so the format is:
\begin{verbatim}
channel 0, channel 1, time 1 lower, time 1 upper,
           channel 2, time 2 lower, ...,
           counts \n
\end{verbatim}
T3 data have an additional dimension (pulse):
\begin{verbatim}
channel 0, channel 1, pulse 1 lower, pulse 1 upper,
                      time 1 lower, time 1 uppper,
           channel 2, ...
           counts \n
\end{verbatim}
See section~\ref{sec:histogram_examples} for specific examples of output in these formats.

\subsection{Examples of usage}
\label{sec:histogram_examples}
\subsubsection{Time-averaged photoluminescence lifetime from T3 data}
In T3 mode, a correlation order \gn{1} is code for interactive-like behavior. Formally this is a correlation of the laser pulse and the system response, but this language is not often used.
\begin{verbatim}
> picoquant --file-in data.pt3 |  \
  histogram --mode t3 --order 1 --channels 2 \
            --time 0,10,500000
0,0.00,50000.00,0
0,50000.00,100000.00,0
...
1,0.00,50000.00,11267838
1,50000.00,100000.00,14947845
1,100000.00,150000.00,1512803
1,150000.00,200000.00,1152498
1,200000.00,250000.00,1037717
1,250000.00,300000.00,973572
1,300000.00,350000.00,932802
1,350000.00,400000.00,899615
1,400000.00,450000.00,12278
1,450000.00,500000.00,0
\end{verbatim}
Note that \texttt{histogram} will operate on channel 0 as well, even though channel 1 is the only channel with any signal. This costs extra memory and some computational overhead at startup, but ultimately the cost is insignificant compared to the cost of processing the data stream.

\subsubsection{\gn{2} from T2 data}
This data represents an electronic sync source (channel 4) and the detection of the laser itself.
\begin{verbatim}
> picoquant --file-in data.ht2 | \
  correlate --mode t2 --order 2 \
            --channels 5 \
            --max-time-distance 1000 | \
  histogram --mode t2 --order 2 \
            --channels 5 \
            --time 0,10,1000
0,0,0.00,100.00,0
...
0,4,0.00,100.00,43
0,4,100.00,200.00,24
0,4,200.00,300.00,38
0,4,300.00,400.00,43
0,4,400.00,500.00,44
...
4,4,900.00,1000.00,0
\end{verbatim}

\subsubsection{\gn{2} from T3 data}
\begin{verbatim}
> picoquant --file-in data.pt3 | \
  correlate --mode t3 --channels 2 \
            --order 2 \
            --max-pulse-distance 3 \
  histogram --mode t3 --channels 2 \
            --order 2 --pulse 0,3,3 \
            --time -500000,2,500000
0,0,0.00,1.00,-500000.00,0.00,0
...
1,1,0.00,1.00,-500000.00,0.00,0
1,1,0.00,1.00,0.00,500000.00,104640
1,1,1.00,2.00,-500000.00,0.00,123289
1,1,1.00,2.00,0.00,500000.00,124676
1,1,2.00,3.00,-500000.00,0.00,231962
1,1,2.00,3.00,0.00,500000.00,231694
\end{verbatim}
As a rough benchmark, on a computer with a dual-core 3\giga\hertz{} processor running 32-bit Linux, this command required 25\second{} of wall time, for \texttt{data.ht3} containing 32.7 million photon records (18.1\kilo\cps) for a total of 0.8 million correlation events.

\subsubsection{\gn{3} from T2 data}
This T2 data is the same as the laser data from before:
\begin{verbatim}
> picoquant --file-in data.ht2 | \
  correlate --mode t2 --order 2 \
            --channels 5 \
            --max-time-distance 1000000 | \
  histogram --mode t2 --order 2 \
            --channels 5 \
            --time 0,1,1000000
0,0,0.00,1000000.00,0,0.00,1000000.00,31
0,0,0.00,1000000.00,1,0.00,1000000.00,55
0,0,0.00,1000000.00,2,0.00,1000000.00,57
...
4,0,0.00,1000000.00,2,0.00,1000000.00,2710
4,0,0.00,1000000.00,3,0.00,1000000.00,2235
4,0,0.00,1000000.00,4,0.00,1000000.00,0
...
\end{verbatim}
Note how much larger the time window must be to catch significant numbers of higher-order events.

\subsection{Implementation details}
The problem of populating and returning histograms for all possible correlations can be broken into a few distinct steps:
\begin{enumerate}
\item Construct a histogram for every permutation of channels.
\item For each correlation event:
  \begin{enumerate}
  \item Identify the histogram corresponding to the permutation of channels.
  \item Identify the bin associated with the parameters of the correlation.
  \item Increment that bin in that histogram.
  \end{enumerate}
\item For every bin of every histogram, print the bin definition and associated counts
\end{enumerate}
As such, each of these will be discussed separately. The discussion will focus on T2 data, but T3 data are handled identically for twice the order of equivalent T2 data.

\subsubsection{The cross-correlations can be enumerated as base-$\abs{\channels}$ numbers}
Each histogram corresponds to a single cross-correlation, as identified by the tuple $\vec{\channel}\in\channels^{n}$, for $\channels$ the set of all channels and correlation order $n$. As such, if we enumerate the channels as whole numbers ($0, 1, \ldots$), it is evident that each element of the tuple can be treated as the coefficient of an $n$-digit number in base $\abs{\channels}$. Mapping these tuples onto the set of whole numbers can thus be achieved by the following formula:
\begin{equation}
\Index(\vec{\channel}) = \sum_{j=0}^{n-1}{\channel_{j}\abs{\channels}^{n-1-j}}
\end{equation}
where the elements $\channel$ refer implicitly to the index of that channel in our enumeration.
If $\abs{\channels}=2$, this is identical to the expression of an $n$-digit binary number.

In the software, all possible cross-correlations are enumerated and the corresponding histograms allocated before beginning the calculation. This requires some computational overhead at instantiation, and if an application requires the calculation of large numbers of histograms it may be more efficient to modify the code to output and reset upon receiving particular signals.

The implementation of these methods can be found in \texttt{histogram\_gn.c} and \texttt{combinations.c}.

\subsubsection{A histogram is function of $n$-dimensional vectors mapping mapping onto integers}
Once the particular cross-correlation has been identified, the task falls to that of incrementing a counter corresponding to the appropriate histogram bin. The purpose of a histogram is to divide some phase space into well-defined smaller blocks, and then to count the number of events which fall into those blocks. We will restrict ourselves to rectangular volumes, so the task of identifying the correct volume of phase space can be reduced to determining the correct index along all axes, and thus each indexing step is identical to that for a single-dimensional histogram.

In one dimension, we first define a range of values into which the events can fall, for example $\timewindow=\left[\timewindow\upminus,\timewindow\upplus\right)\subset\integers$. Next, we define some number $N$ of ranges $\resolution$ whose members collectively span $\timewindow$. These $\resolution$ can be enumerated $0,1,\ldots N-1$, so assign these indices in order to the sorted $\resolution$. Now this one dimension is represented by an $N$-dimensional vector whose dimensions represent some $\resolution$. The task then turns to mapping any value $z\in\integers$ to the appropriate $\resolution$.

For linearly-spaced sub-ranges, this identification can be achieved in O(1) time by:
\begin{equation}
\Index(x) = \timewindow\upminus + x\parens{\frac{\timewindow\upplus-\timewindow\upminus}{N}}
\end{equation}
However, in the code this is not used, and instead a more general binary search algorithm (costing O($\log(N)$)) is implemented to permit arbitrary spacings of these ranges. The binary algorithm determines the placement of an element into a sorted list by iterative division of the list into upper and lower halves, until the appropriate element is define. This is implemented as:
\lstset{language=Python}
\begin{lstlisting}
def binary_search(value, bins):
    upper = len(bins)
    lower = 0

    if value < bins[0] or values > bins[-1]:
        # Not in the bounds
        return(False)

    while True:
        middle = (upper - lower)/2
        if value in bins[middle]:
            # Just right
            return(middle)
        elif value < bins[middle]:
            # Too high
            upper = middle
        else:
            # Too low
            lower = middle
\end{lstlisting}
Because this must be performed for each dimension of the correlation, the search costs O($n\log{(N)}$) time, compared to the O($n$) for the complete linear search.

To see how this is implemented in \program{histogram}, see \texttt{histogram\_gn.c} and \texttt{histogram\_t*.c}.

\subsubsection{Printing the histogram}
Having exhausted the stream of correlation events, the final task is to print all of the histogram bins in a readable format. This is done by iterating over the histograms, then iterating over the bins, printing the counts associated with each bin. How this process is performed should be evident from the preceding discussion, and for the details refer to the routine \texttt{print\_gn\_histogram} in \texttt{histogram\_gn.c}.

Note that the output of $\program{histogram}$ is not normalized in any way: only the number of counts is reported for each bin.


\documentclass{article}

%----------------------------------------------------------------------------------------

\usepackage{listings} % Required for inserting code snippets
\usepackage{geometry}
\geometry{margin=0.7in}
\usepackage[usenames,dvipsnames]{color} % Required for specifying custom colors and referring to colors by name
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{tikz}
\usepackage{enumerate}

\delimitershortfall-1sp
\newcommand\abs[1]{\left|#1\right|}

\definecolor{DarkGreen}{rgb}{0.0,0.4,0.0} % Comment color
\definecolor{highlight}{RGB}{255,251,204} % Code highlight color

%----------------------------------------------------------------------------------------

\begin{document}

%----------------------------------------------------------------------------------------

\begin{flushleft}
\begin{center}
$\displaystyle \lim_{x \to 2} (x^2 - x) = 2$
\end{center}

We know that $c=2$, $L=2$, $f(x) = x^2 - x$. \\

\vspace{.4cm}
\begin{center}
$\abs{f(x) - L} = \abs{x^2-x-2} < \epsilon \implies \abs{\abs{x-2} \abs{x+1}} < \epsilon$ \\
\end{center}
\vspace{.4cm}

We cannot define $\delta_\epsilon$ in terms of $x$, but $\abs{x-2} = \frac{\epsilon}{\abs{x+1}}$ leaves $\delta_\epsilon$ in terms of $x$. We need to find a fixed $M$ such that $\frac{\epsilon}{M} < \frac{\epsilon}{\abs{x+1}}$. We only care about $x$ \textit{around} $2$, so we can restrict $x$ to the open interval $(1,3)$.

\begin{center}
\begin{tabular}{c c}
$1<x<3$ & $1<x<3$ \\
$-1<x-2<1$ & $2<x+1<4$ \\
$\abs{x-2}<1$ & $\frac{1}{2}<\frac{1}{\abs{x+1}}<\frac{1}{4}$ \\
\end{tabular}
\end{center}

\subsection*{Proof}
Let $\epsilon > 0$ be given. Let $\delta_\epsilon=min\{1, \frac{\epsilon}{4}\}$. Then $0<\abs{x-2}<\delta_\epsilon$ implies
\begin{center}
\begin{tabular}{l}
(1) $\abs{x-2} < 1 \implies 2<\abs{x+1}<4$ \\
(2) $\abs{f(x) - L} =\abs{x^2-x-2} = \abs{x-2}\abs{x+1} < \frac{\epsilon}{4}\cdot4 = \epsilon$ \\
\end{tabular}
\end{center}

This completes the proof.
\end{flushleft}

\vspace{.75cm}

\begin{flushleft}
\begin{center}
$\displaystyle \lim_{x \to 0} x^2(sin(x) + cos(x)) = 0$
\end{center}
\vspace{.4cm}
We know that $c=0$, $L=0$, $f(x) = x^2(sin(x) + cos(x))$. \\
\vspace{.4cm}
From the combination of the transcendental functions $\abs{sin(x) + cos(x)}$, we know the maximum value that the functions can reach is $\sqrt{2}$. Therefore, we can modify $f(x)$ to be $\sqrt{2} x^2$ since we know that the $\abs{sin(x) + cos(x)}$ part of the function is bounded above. Thus, we have
\begin{center}
\begin{tabular}{r}
$\abs{\sqrt{2} x^2 - 0} < \epsilon$ \\
$\abs{\sqrt{2} x^2} < \epsilon$ \\
$\sqrt{2} x^2 < \epsilon$ \\
$x < \frac{\sqrt{\epsilon}}{\sqrt[4]{2}}$ \\
\end{tabular}
\end{center}

\subsection*{Proof}

Let $\epsilon > 0$ and $\delta_\epsilon=\frac{\sqrt{\epsilon}}{\sqrt[4]{2}}$. Then, $0 < x < \frac{\sqrt{\epsilon}}{\sqrt[4]{2}}$ implies
\begin{center}
$-\frac{\sqrt{\epsilon}}{\sqrt[4]{2}} < x < \frac{\sqrt{\epsilon}}{\sqrt[4]{2}}$ \\
$x^2 < \frac{\epsilon}{\sqrt{2}}$ \\
$\sqrt{2}x^2 < \epsilon$ \\
\end{center}
Thus, $\abs{\sqrt{2}x^2} < \epsilon$. This satisfies the proof because we stated that $sin(x) + cos(x)$ is bounded above by $\sqrt{2}$. By definition of an upper bound, there can never exist a value that exceeds $\abs{sin(x) + cos(x)x^2}$. Then, $\abs{sin(x) + cos(x)x^2} < \abs{x^2\sqrt{2}} < \epsilon$.
\end{flushleft}

%----------------------------------------------------------------------------------------

\end{document}


\documentclass{article}
\begin{document}
  \section{Block}

  \begin{equation}
    e^{i\pi}=-1
    \label{eq:euler}
  \end{equation}
  Equation~\ref{eq:euler} is some good math.

  \section{Inline}

  some lovely maths \(x^2 + y^2 = z^2\)
\end{document}


\subsection{The EXAFS Equation}
\begin{frame}
  \frametitle{The EXAFS Equation} %% Analysis with {\feff} \&  {\arch} or {\ifeffit} }

  The XAFS Equation used with {\feff} and {\larch}:

  \[
  \chi(k) = \sum_j {{ S_0^2 {\Blue{N_j}} {\Red{f_j(k)}}  e^{-2R_j/\lambda(k)}
      e^{-2k^2{\Blue{\sigma_j^2}}}}\over{k{\Blue{R_j}}^2}}
  {\sin[{2k{\Blue{R_j}} + {\Red{\delta_j(k)}}} ]}
   \]

\begin{itemize}
   \pause
     \item  The sum is over  {\RedEmph{Scattering Paths}} of the
       photo-electron.   Both:

       \begin{description}

       \item[Single Scattering] {\tiny{ absorbing atom $\Rightarrow$ neighbor atom
             $\Rightarrow$ absorbing atom}}

       \item[Multiple Scattering] {\tiny{absorbing atom $\Rightarrow$ neighbor
             atom  $\Rightarrow$ neighbor atom $\Rightarrow$ \ldots  $\Rightarrow$
             absorbing atom}}

       \end{description}

       \pause\item $\Red{f(k)}$ and $\Red{\delta(k)}$ are
       {{photo-electron scattering amplitude and phases}}:

   \begin{itemize}
   \item Energy ($k$) dependent.
   \item $Z$ dependent -- $Z$ of the scattering atoms(s).
   \item non-trivial: must be calculated (or extracted from measured spectra).

   \end{itemize}
 \end{itemize}

   \vmm \hrule \vmm

   \onslide+<4->

   Knowing  $\Red{f(k)}$ and $\Red{\delta(k)}$, we can determine structural information:

     \begin{itemize}
     \item ${\Blue{R}}$ --  near neighbor distance.
     \item ${\Blue{N}}$ -- coordination number.
     \item ${\Blue{\sigma^2}}$ -- mean-square  disorder in ${\Blue{R}}$.
     \end{itemize}

   \vmm

\end{frame}


\documentclass{article}

%%%%%%%%%%%%%%%% 2020 - 11 - 28 %%%%%%%%%%%%%%%%%%
\usepackage{tikz}
\usetikzlibrary{calc}
\usepackage{pgfplots}
\usepackage{graphicx}
\usepackage{url}

\usepackage{amsmath}
\usepackage{algorithm, algpseudocode}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
\algnewcommand{\LeftComment}[1]{\Statex \(\triangleright\) #1}
\usepackage{amsfonts}  % \mathbb{R}
\makeatletter
\newlength{\trianglerightwidth}
\settowidth{\trianglerightwidth}{$\triangleright$~}
\algnewcommand{\LineComment}[1]{\Statex \hskip\ALG@thistlm $\triangleright$ #1}
\algnewcommand{\LineCommentCont}[1]{\Statex \hskip\ALG@thistlm%
  \parbox[t]{\dimexpr\linewidth-\ALG@thistlm}{\hangindent=\trianglerightwidth \hangafter=1 \strut$\triangleright$ #1\strut}}

\algnewcommand{\LeftLineCommentCont}[1]{\Statex \hskip\ALG@thistlm%
  \parbox[t]{\dimexpr\linewidth-\ALG@thistlm}{\leftskip=\algorithmicindent \hangindent=\trianglerightwidth \hangafter=1 \strut$\triangleright$ #1\strut}}

\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{amsthm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newtheorem{theorem}{Theorem}[section]
\newtheorem{corollary}{Corollary}[theorem]
\newtheorem{lemma}[theorem]{Lemma}
\theoremstyle{definition}
\newtheorem{definition}{Definition}[section]
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage{minted}

\usepackage{subfig}
\usepackage[export]{adjustbox}% in preamble
%\subfloat[hlof entryi][hsub-captioni]{%
%hfigurei}
%\subfloat[hlot entryi][hsub-captioni]{%
%htablei}

\usepackage{hhline}
\usepackage{makecell, caption, booktabs}
\usepackage{siunitx}

%%The second way (hacky, and depends on an implementation detail of els-cas)
\makeatletter
\def\redefparbox{\def\@parboxrestore{\@arrayparboxrestore\let\\\@normalcr
  \if@minipage\expandafter\@gobbletwo\fi
  \@firstofone{\centering\casscparboxtest}}}
\def\casscparboxtest#1{%
  \ifx\rightskip#1\relax\expandafter\dimen@\else
    \expandafter\@secondoftwo
  \fi\@gobble{#1}}
\makeatother
%%The second way (hacky, and depends on an implementation detail of els-cas)

%%%%%%%%%%%%%%%% 2020 - 11 - 28 %%%%%%%%%%%%%%%%%%

\usepackage{PRIMEarxiv}

\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc}    % use 8-bit T1 fonts
\usepackage{hyperref}       % hyperlinks
\usepackage{url}            % simple URL typesetting
\usepackage{booktabs}       % professional-quality tables
\usepackage{amsfonts}       % blackboard math symbols
\usepackage{nicefrac}       % compact symbols for 1/2, etc.
\usepackage{microtype}      % microtypography
\usepackage{lipsum}
\usepackage{graphicx}
\graphicspath{{media/}}     % organize your images and other figures under media/ folder

%% Title
\title{ on Polynomial Approximation of Activation Function
%%%% Cite as
%%%% Update your official citation here when published
%\thanks{\textit{\underline{Citation}}:
%\textbf{Authors. Title. Pages.... DOI:000000/11111.}}
}

\author{ \href{https://orcid.org/0000-0003-0378-0607}{\includegraphics[scale=0.06]{orcid.pdf}\hspace{1mm}John Chiang} \\
	\texttt{liyue.sun@mail.nankai.edu.cn, john.chiang.smith@gmail.com} \\
}

\begin{document}
\maketitle

\section{Introduction}
When it comes to applying homomoiphic encryption techqiue to machine learning such as neural network, there is a techquice problem that non-polynomial of activation function  could not be calculated directly in the HE domain. The common way to deal with it is to approximate the activation function using the least-square method, to a polynomial that can be calculated in a HE-based environmental\cite{hadash2018estimate}. Being widely adapted in recent work related to HE\cite{kour2014real}, however, the least-square method maight not be ideal for this task. In this essay, we propose an neat method to approximate the activation function based on the the least squared method.

Note that the idea of this essay did come to the present author, but we can not garnteen that other researches haven't found it (we should give some time to read the book carefully before trying to develope some own method). In conclude, this is a simple idea that can propaply work for approximating activation function of HE. // %比最小二次乘更容易符合同态加密下的要求

\section{Least Square Method}
\label{sec:headings}
For a continuous real-value function $f(x)$, a simple version of least square method to approximate $f(x)$ over range $[a, b]$ by a polynomial of a given degree $n$ is to find a polynomial of at most degree $n$, $p_n(x)$, such that  minimise the $$\int_{a}^{b} (f(x) - p_n(x))^2 dx. $$

For approximating a continuous real-value function $f(x)$ over the range $[a, b]$ by a polynomial of a given degree $n$, a simple version of least square method  is to find a polynomial of at most degree $n$, $p_n(x)$, such that  maximise the $$\int_{a}^{b} (f(x) - p_n(x))^2 dx, $$ which has a unique solution (the best approximation). Least square method is such a common method that many softwares implement this method, such as the function  $polyfit( \cdot )$ in Python, Matlab or Octave. For example, only 3 lines of Octave codes is all it needs to fit the activation function $ReLU(x) = max(0, x) $ over the range $[-8, +8]$ by a polynomial of degree 2, resulting in the figure 1:
\begin{minted}
{octave}
x = [-8:0.000001:+8];
y = max(0,x);
polyfit(x,y,2)
\end{minted}
and we get the polynomial approximation $p_2^{ls}(x) = 0.058594 +  0.500000 \cdot x +  0.750000 \cdot x^2 $

Even through least square method has many advantages, it maight not be the best choice in the sense of a HE-based environment. In the sense of a HE-based environment, we would like to choice a low-degree polynomial due to the expensive HE operations, and also perfer the similiar curve shape between the apprximation polynomial and the function to approximate, even at the cost of losing best appxiromation. Thus, we propose a simple method to conside the gradient(the slope) into the optimal function.

and its limition in the polynomial approximation of activation function
To minimise the gray area of the is not enough, we perhaps would like a polynomial approximation that has a similiar shape to the activation function, even at the cost of some larger gray area.

\section{Our Method}
\label{sec:others}

In this section, we present  a simple method based on the least-square method, which  includes  the graduals (shape) of the Polynomial and the activation into  the least-square method, in order to get the desired polynomial approximation that we describled above.
such that  minimise the
$$\int_{a}^{b} (f(x) - p_n(x))^2 + (f'(x) - p_n'(x))^2 dx. $$

For a toy example, we use this method for the same task above, that is to fit the activation function $ReLU$ over the range $[-8, +8]$ by a polynomial of degree 2 denoted by $p_2^{lg}(x) = c +  b \cdot x +  a \cdot x^2 . $ The question is to minimise the observation loss function
\begin{equation*}
  \begin{aligned}
F(a, b, c)  = & \int_{-8}^{0} (f(x) - p_2^{lg}(x))^2  dx
                +\int_{0}^{+8} (f(x) - p_2^{lg}(x))^2  dx
                +\int_{-8}^{0} (f'(x) - p_2^{lg}{'}(x))^2 dx
                +\int_{0}^{+8} (f'(x) - p_2^{lg}{'}(x))^2 dx   \\
            = & [\frac{4}{3}a^2x^3 + (b^2 + 1 - 2b)x + 2a(b - 1)x^2  ]|_{0}^{+8}  + \\
            &[\frac{1}{5}a^2x^5 + \frac{1}{3}b^2x^3 + bcx^2 - \frac{2}{3}bx^3 + c^2x + \frac{1}{3}x^3 - cx^2 + \frac{1}{2}abx^4 - \frac{1}{2}ax^4 + \frac{2}{3}acx^3 ]|_{0}^{+8} + \\
            & [\frac{4}{3}a^2x^3 + b^2x + 2abx^2  ]|_{-8}^0 + \\
            &[\frac{1}{5}a^2x^5 + \frac{1}{3}b^2x^3 + c^2x + bcx^2  + \frac{1}{2}abx^4 + \frac{2}{3}acx^3 ]|_{-8}^{0} + \\
            = & 14472.5\dot{3}a^2 + 357.\dot{3}b^2 + 16c^2  + 682.\dot{6}ac - 357.\dot{3}b + 178.\dot{6} - 64c - 2176a
 \end{aligned}
\end{equation*}
Minimising $F(a,b,c)$ is a problem of unconstrained optimization. To do that, we need to calcuate  the first-order partial derivatives of $F(a,b,c)$ and have them equal to $0$ :
\begin{equation*}
  \begin{aligned}
F_a(a, b, c) & = 28945.0\dot{6}a +  682.\dot{6}c - 2176 &= 0  \\
F_b(a, b, c) & =  714.\dot{6}b  - 357.\dot{3}  &= 0  \\
F_c(a, b, c) & = 32c  + 682.\dot{6}a - 64 &= 0
 \end{aligned}
\end{equation*}
Solving these equations, we obtain the unique solution and the polynomial approximation, respectly:
\begin{equation*}
  \begin{aligned}
a = 0.0563686709, b = 0.5, c = 0.7974683544, \\
p_n^{lg}(x) = 0.797468 +  0.500000 \cdot x +  0.056369 \cdot x^2
 \end{aligned}
\end{equation*}
We compare the polynomial generated by least square,  $p_n^{ls}(x)$, with that of our method $p_n^{lg}(x)$.
\begin{figure}[ht]
\centering

\begin{tikzpicture}[remember picture]
\begin{axis}[
%legend pos=south east,
%legend style={at={(0.5,-0.17)},anchor=north,legend cell align=left},
legend style={at={(0.5,+0.9)},anchor=north,},
width=0.45\linewidth,
xtick={-8, -6, -4, -2, 0, 2,  4, 6, 8 },
yticklabels={},
at={(0.66\linewidth,0)},
]
%Below the red parabola is defined
\addplot [
    domain=-8:8,
    samples=100,
    color=red,
]
{0.797468 +  0.500000*x +  0.056369*x^2 };
\addlegendentry{$p_2^{lg}(x)$}
%Here the blue parabola is defined
\addplot [
    domain=-8:8,
    samples=100,
    color=blue,
    ]
    {0.058594 +  0.500000*x +  0.750000*x^2 };
\addlegendentry{$p_2^{ls}(x)$};
%ReLU
\addplot [
    domain=-8:8,
    samples=100,
    color=gray,
    ]
    {max(0, x) };
\addlegendentry{$\texttt{ReLU}$};
\end{axis}
\end{tikzpicture}

\iffalse
\begin{tikzpicture}[overlay, remember picture, scale=.5]
\begin{axis}[
hide axis,
%xtick=\empty, ytick=\empty,
width=0.45\linewidth,
%width=3cm,
%height=2.1cm,
%scale only axis,
%xmin=-0.6235,
%xmax=-0.6186,
%xtick={-0.623, -0.621, -0.619},
%ymin=0.1651,
%ymax=0.1655,
%ytick={0.1651, 0.1653, 0.1655},
xshift=2.25cm,yshift=5.75cm,
]
%Below the red parabola is defined
\addplot [
    domain=-2:2,
    samples=20,
    color=red,
]
{0.797468 +  0.500000*x +  0.056369*x^2 };
%Here the blue parabola is defined
\addplot [
    domain=-2:2,
    samples=20,
    color=blue,
    ]
    {0.058594 +  0.500000*x +  0.750000*x^2 };
%ReLU
\addplot [
    domain=-2:2,
    samples=20,
    color=black,
    ]
    {max(0, x) };
\end{axis}
\end{tikzpicture}
\fi

\caption{ Results in the }
\label{fig1}
\end{figure}

\begin{figure}[ht]
\centering

\begin{tikzpicture}[remember picture]
\begin{axis}[
%legend pos=south east,
%legend style={at={(0.5,-0.17)},anchor=north,legend cell align=left},
legend style={at={(0.5,+0.9)},anchor=north,},
width=0.45\linewidth,
xtick={ -6, -4, -2, 0, 2,  4, 6 },
yticklabels={},
at={(0.66\linewidth,0)},
]
%Below the red parabola is defined
\addplot [
    domain=-6:6,
    samples=100,
    color=red,
]
{1.1110537229 + 0.5*x + 0.054235537*x^2 };
\addlegendentry{$p_2^{lg}(x)$}
%ReLU
\addplot [
    domain=-6:6,
    samples=100,
    color=gray,
    ]
    { max(0, x) };
\addlegendentry{$\texttt{ReLU}$};
\end{axis}
\end{tikzpicture}

\caption{ Results in the }
\label{fig2}
\end{figure}
From the figure \ref{fig1}, we can see that our polynomial are much the same as ReLU in the term of shape(slope), and that the least square polynomial doesn't even look like the ReLU funtion.

A more flexiable way to generate teh polynomial approximation is to set add more  parameters to control the weight or propotion of the least square to slope gradial resulting polynomial. To take the $ReLU$  within the domain $[-L, +L]$ as a example, we could optimise the following function:
\begin{equation*}
  \begin{aligned}
F(a, b, c)  =  \lambda_0 \cdot \int_{-L}^{0} (0 - p_2^{lg}(x))^2  dx
             + \lambda_1 \cdot \int_{0}^{+L} (x - p_2^{lg}(x))^2  dx  \\
             + \lambda_2 \cdot \int_{-L}^{0} (0 - p_2^{lg}{'}(x))^2 dx
             + \lambda_3 \cdot \int_{0}^{+L} (1 - p_2^{lg}{'}(x))^2 dx ,
 \end{aligned}
\end{equation*}
where $\lambda_0$, $\lambda_1$, $\lambda_2$ and $\lambda_3$ are four real number greater than zero to control the various weight propotion.

Another example is, supposing that we want to find a polynomial to approximate the relu over the domain $[-6, +6]$ such that it fit the ReLU very well at the both ends at the expense of losing some precsion round the center. In this case, we need to minimise the function (we just set $\lambda_0 = \lambda_1 = \lambda_2 = \lambda_3 = 1$ ):
\begin{equation*}
  \begin{aligned}
F(a, b, c)  =&   \int_{-6}^{-3} (0 - p_2^{lg}(x))^2  dx
             +  \int_{+3}^{+6} (x - p_2^{lg}(x))^2  dx
             +  \int_{-6}^{-3} (0 - p_2^{lg}{'}(x))^2 dx
             +  \int_{+3}^{+6} (1 - p_2^{lg}{'}(x))^2 dx   \\
            =& 3517.2a^2 + 132b^2 + 6c^2  + 54bc + 1917ab + 252ac - 607.5a - 78b - 27c + 12.
 \end{aligned}
\end{equation*}
Solving this  unconstrained optimization, we get the polynomial $p_2^{(1)} = 1.1110537229 + 0.5 \cdot x + 0.054235537 \cdot x^2 $. Figure \ref{fig2} shows that $p_2^{(1)}$ is exactly what polynomial we want.

\section{Conclusion}
Your conclusion here
Our method to approximate activation function is much flexiable compared to least square method in the sense of ml.
Vouriac setting of parmpaters should result in polynomials of different shapes.
%Bibliography
\bibliographystyle{unsrt}
\bibliography{references}

\end{document}

on Polynomial Approximation of Activation Function/templatePRIME.tex


\chapter{The Cubic Nonlinear Schr\"{o}dinger Equation}

%%%%%%%
%Section
%%%%%%%
\section{Background}

The cubic nonlinear Schr\"{o}dinger equation occurs in a variety of areas, including, quantum mechanics, nonlinear optics and surface water waves. A general introduction can be found at \url{http://en.wikipedia.org/wiki/Schrodinger_equation} and \url{http://en.wikipedia.org/wiki/Nonlinear_Schrodinger_equation}. A mathematical introduction to Schr\"{o}dinger equations can be found in Sulem and Sulem~\cite{SulSul99} and Yang~\cite{Yan10}. In this section we will introduce the idea of operator splitting and then go on to explain how this can be applied to the nonlinear Schr\"{o}dinger equation in one, two and three dimensions. In one dimension, one can show that the cubic nonlinear Schr\"{o}dinger equation is subcritical, and hence one has solutions which exist for all time. In two dimensions, it is $H^1$ critical, and so solutions may exhibit blow-up of the $H^1$ norm, that is the integral of the square of the gradient of the solution can become infinite in finite time.  Finally, in three dimensions, the nonlinear Schr\"{o}dinger equation is $L^2$ supercritical, and so the integral of the square of the solution can also become infinite in finite time. For an introduction to norms and Hilbert spaces, see a textbook on partial differential equations or analysis, such as Evans~\cite{Eva10}, Linares and Ponce~\cite{LinPon09}, Lieb and Loss~\cite{LieLos03} or Renardy and Rogers~\cite{RenRog04}. A question of interest is how this blow-up occurs and numerical simulations are often used to understand this; see Sulem and Sulem~\cite{SulSul99} for examples of this. The cubic nonlinear Schr\"{o}dinger equation\footnote{To simplify the presentation, we primarily consider the focusing cubic nonlinear Schr\"{o}dinger equation.} is given by
\begin{equation}
i\psi_t + \Delta \psi \pm\lvert \psi \rvert^2\psi =0,
\end{equation}
where $\psi$ is the wave function and $\Delta$ is the Laplacian operator, so in one dimension it is $\partial_{xx}$, in two dimensions, $\partial_{xx}+\partial_{yy}$ and in three dimensions it is $\partial_{xx}+\partial_{yy}+\partial_{zz}$. The $+$ corresponds to the focusing cubic nonlinear Schr\"{o}dinger equation and the $-$ corresponds to the defocusing cubic nonlinear Schr\"{o}dinger equation. This equation has many conserved quantities, including the ``mass'',
\begin{equation}
\int_{\Omega}\lvert\psi\rvert^2\mathrm{d}^n\bm x
\end{equation}
and the ``energy'',
\begin{equation}
\int_{\Omega}\frac{1}{2}\lvert\nabla \psi\rvert^2\mp\frac{1}{4}\lvert\psi\rvert^4\mathrm{d}^n\bm x
\end{equation}
where $n$ is the dimension and $\Omega$ is the domain of the solution. As explained by Klein~\cite{Kle08}, these two quantities can provide useful checks on the accuracy of numerically generated solutions.

%%%%%%%
%Section
%%%%%%%
\section{Splitting}

We will consider a numerical method to solve this equation known as splitting. This method occurs in several applications, and is a useful numerical method when the equation can be split into two separate equations, each of which can either be solved exactly, or each part is best solved by a different numerical method. Introductions to splitting can be found in Holden et al.~\cite{HolKarLieRis10}, McLachlan and Quispel~\cite{McLQui02}, Thalhammer~\cite{Tha08}, Shen, Tang and Wang~\cite{SheTanWan11}, Weideman and Herbst~\cite{WeiHer86} and Yang~\cite{Yan10}, and also at \url{http://en.wikipedia.org/wiki/Split-step_method}. For those interested in a comparison of time stepping methods for the nonlinear Schr\"{o}dinger equation, see Klein~\cite{Kle08}. To describe the basic idea of the method, we consider an example given in Holden et al.~\cite{HolKarRisTao11}, which is the ordinary differential equation,
\begin{equation}\label{eq:SplitOde}
u_t=u(u-1),\quad u(t=0)=0.8.
\end{equation}
We can solve this equation relatively simply by separation of variables to find that
\begin{equation}
u(t)=\frac{4}{4+\exp(t)}.
\end{equation}
Now, an interesting observation is that we can also solve the equations $u_t=u^2$ and $u_t=-u$ individually. For the first we get that $u(t)=\frac{u(0)}{1-tu(0)}$ and for the second we get that $u(t)=u(0)\exp(-t)$. The principle behind splitting is to solve these two separate equations alternately for short periods of time. We will describe Strang splitting, although there are other forms of splitting, such as Godunov splitting and also additive splittings.  We will not describe these here, but refer you to the previously mentioned references, in particular Holden et al.~\cite{HolKarLieRis10}. To understand how we can solve the differential equation using splitting, consider the linear ordinary differential equation
\begin{equation}
u_t=u+2u,\quad u(0)=1.
\end{equation}
We can first solve $p_t=p$ for a time $\delta t/2$ and then using $q(0)=p(\delta t/2)$, we solve $q_t=2q$ also for a time $\delta t$ to get $q(\delta t)$ and finally solve $r_t=r$ for a time $\delta t/2$ with initial data $r(0)=q(\delta t)$. Thus in this case $p(\delta t)=\exp(\delta t/2)$, $q(\delta t)=p(\delta t/2)\exp(2\delta t)=\exp(5\delta t/2)$ and $u(\delta t)\approx r(\delta t/2)=q(\delta t)\exp(\delta t/2)=\exp(3\delta t)$, which in this case is the exact solution. One can perform a similar splitting for matrix differential equations. Consider solving $\bm u_t = (\bm A + \bm B)\bm u$, where $\bm A$ and $\bm B$ are $n\times n$ matrices, the exact solution is $\bm u=\exp\left((\bm A + \bm B) t\right)\bm u(t=0)$, and an approximate solution produced after one time step of splitting is $u(\delta t)\approx u(0)\exp(\bm A \delta t)\exp(\bm B \delta t)$, which is not in general equal to $u(t=0)\exp\left((\bm A + \bm B) \delta t\right)$ unless the matrices $\bm A$ and $\bm B$ commute\footnote{That is$\bm A\bm B=\bm B\bm A$.}, and so the error in doing splitting in this case is of the form $(\bm A \bm B - \bm B \bm A)\delta t$\footnote{One can derive this by using the series expansion of the exponential function, $\exp(\bm A t)=\sum_{n=0}^{\infty}\frac{(\bm A t)^n}{n!}$, and subtracting $\exp((\bm A+\bm B)\delta t)$ from $\exp(\bm A \delta t)\exp(\bm B \delta t).$}. Listing \ref{lst:OdeStrangMatlab} uses Matlab to demonstrate how to do splitting for eq.\ \eqref{eq:SplitOde}.

\lstinputlisting[style=matlab_style,label=lst:OdeStrangMatlab,caption={A Matlab program which uses Strang splitting to solve an ODE.}]{./CubicNonlinearSchrodinger/Programs/ODEsplittingStrang.m}

%%%%%%%
%Section
%%%%%%%
\section{Exercises}

\begin{enumerate}
\item[1)] Modify the Matlab code to calculate the error at time 1 for several different choices of timestep. Numerically verify that Strang splitting is second order accurate.
\item[2)] Modify the Matlab code to use Godunov splitting where one solves $u1_t=u1$ for a time $\delta t$ and then using $u1(\delta t)$ as initial data solves $u2_t=2u2$ also for a time $\delta t$ to get the approximation to $u(\delta t)$. Calculate the error at time 1 for several different choices of timestep. Numerically verify that Godunov splitting is first order accurate.
\end{enumerate}
%%%%%%%
%Section
%%%%%%%
\section{Serial}

For the nonlinear Schr\"{o}dinger equation
\begin{equation}
i\psi_t\pm\lvert\psi\rvert^2\psi+\Delta\psi=0, \label{eq:NLS}
\end{equation}
we first solve
\begin{equation}
i\psi_t+\Delta\psi=0 \label{eq:NLSsplit1}
\end{equation}
exactly using the Fourier transform to get $\psi(\delta{t}/2,\cdot)$. We then solve
\begin{equation}
i\psi_t\pm\lvert\psi\rvert^2\psi=0  \label{eq:NLSsplit2}
\end{equation}
with $\psi(\delta{t}/2,\cdot)$ as initial data for a time step of $\delta t$. As explained by Klein~\cite{Kle08} and Thalhammer~\cite{Tha08}, this can be solved exactly in real space because in eq.\ \eqref{eq:NLSsplit2}, $\lvert \psi \rvert^2$ is a conserved quantity at every point in space and time. To show this, let $\psi^*$ denote the complex conjugate of $\psi$, so that
\begin{align}
\frac{\mathrm{d}\lvert \psi \rvert^2}{\mathrm{d}t} &{} = \psi^*\frac{\mathrm{d}\psi}{\mathrm{d}t} +\frac{\mathrm{d}\psi^*}{\mathrm{d}t}\psi = \psi^*\left(\pm i\lvert\psi\rvert^2\psi\right)+\left(\pm i\lvert\psi\rvert^2\psi\right)^*\psi = 0.
\end{align}
Another half step using eq.\ \eqref{eq:NLSsplit1} is then computed using the solution produced by solving eq.\ \eqref{eq:NLSsplit2} to obtain the approximate solution at time $\delta t$.  Example Matlab codes demonstrating splitting follow.

\subsection{Example Matlab Programs for the Nonlinear Schr\"{o}dinger Equation}

The program in listing \ref{lst:NlsSplit1DMatlab} computes an approximation to an explicitly known exact solution to the focusing nonlinear Schr\"{o}dinger equation.

\lstinputlisting[style=matlab_style,label=lst:NlsSplit1DMatlab,caption={A Matlab program which uses Strang splitting to solve the one dimensional nonlinear Schr\"{o}dinger equation.}]{./CubicNonlinearSchrodinger/Programs/NLSsplitting1D.m}

\lstinputlisting[style=matlab_style,label=lst:NlsSplit2DMatlab,caption={A Matlab program which uses Strang splitting to solve the two dimensional nonlinear Schr\"{o}dinger equation.}]{./CubicNonlinearSchrodinger/Programs/NLSsplitting2D.m}

\lstinputlisting[style=matlab_style,label=lst:NlsSplit3DMatlab,caption={A Matlab program which uses Strang splitting to solve the three dimensional nonlinear Schr\"{o}dinger equation.}]{./CubicNonlinearSchrodinger/Programs/NLSsplitting3D.m}

%%%%%%%
%Section
%%%%%%%
\section{Example One-Dimensional Fortran Program for the Nonlinear Schr\"{o}dinger Equation}

Before considering parallel programs, we need to understand how to write a Fortran code for the one-dimensional nonlinear Schr\"{o}dinger equation. Below is an example Fortran program followed by a Matlab plotting script to visualize the results. In compiling the Fortran program a standard Fortran compiler and the FFTW library are required. Since the commands required for this are similar to those in the makefile for the heat equation, we do not include them here.

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For1dNLS,caption={A Fortran program to solve the 1D nonlinear Schr\"{o}dinger equation using splitting.}]{./CubicNonlinearSchrodinger/Programs/NLS1dFortran/NLSsplitting.f90}

\lstinputlisting[style=matlab_style,label=lst:NlsSplit1DMatlabPlot,caption={A Matlab program which plots a numerical solution to a 1D nonlinear Schr\"{o}dinger equation generated by listing \ref{lst:For1dNLS}.}]{./CubicNonlinearSchrodinger/Programs/NLS1dFortran/plotcreate.m}

%%%%%%%
%Section
%%%%%%%
\section{Shared Memory Parallel: OpenMP}

We recall that OpenMP is a set of compiler directives that can allow one to easily make a Fortran, C or C++ program run on a shared memory machine -- that is a computer for which all compute processes can access the same globally addressed memory space. It allows for easy parallelization of serial programs which have already been written in one of the aforementioned languages.

We will demonstrate one form of parallelizm for the two dimensional nonlinear Schr\"{o}dinger equation in which we will parallelize the loops using OpenMP commands, but will use the threaded FFTW library to parallelize the transforms for us. The example programs are in listing \ref{lst:For2dNlsOmp1},   A second method to parallelize the loops and Fast Fourier transforms explicitly using OpenMP commands is outlined in the exercises.

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dNlsOmp1,caption={An OpenMP Fortran program to solve the 2D nonlinear Schr\"{o}dinger equation using splitting and threaded FFTW.}]{./CubicNonlinearSchrodinger/Programs/NLS2dFortranThreadFFT/NLSsplitting.f90}

\lstinputlisting[style=make_style,language=make,label=lst:Makefile2dNLSomp2,caption={An example makefile for compiling the OpenMP program in listing \ref{lst:For2dNlsOmp1}. The example assumes one is using Flux and has loaded environments for the GCC compiler as well as the GCC compiled version of FFTW.  To use the Intel compiler to with this code, the OMP stack size needs to be explicitly set to be large enough.  If one is using the the PGI compilers instead of the GCC compilers, change the flag $-fopenmp$ to $-mp$.}]{./CubicNonlinearSchrodinger/Programs/NLS2dFortranThreadFFT/makefile}

\lstinputlisting[style=matlab_style,label=lst:NlsSplit2DMatlabPlot,caption={A Matlab program which plots a numerical solution to a 2D nonlinear Schr\"{o}dinger equation generated by listing \ref{lst:For2dNlsOmp1} or \ref{lst:For2dNlsOmp2}.}]{./CubicNonlinearSchrodinger/Programs/NLS2dFortran/plotcreate.m}

\lstinputlisting[style=bash_style,language=bash,label=lst:Fluxsub2dNLSomp,caption={An example submission script for use on Flux. Change \texttt{your\_username} appropriately.}]{./CubicNonlinearSchrodinger/Programs/NLS2dFortran/fluxsubscript}

%%%%%%%
%Section
%%%%%%%
\section{Exercises}
\begin{enumerate}
\item[1)] Download the example Matlab programs which accompany the pre-print by Klein, Muite and Roidot~\cite{KleMuiRoi11}. Examine how the mass and energy for these Schr\"{o}dinger like equations are computed. Add code to check conservation of mass and energy to the Matlab programs for the nonlinear Schr\"{o}dinger equation.
\item[2)] The Gross-Pitaevskii equation\footnote{\url{http://en.wikipedia.org/wiki/Gross\%E2\%80\%93Pitaevskii\_equation}}  is given by
\begin{equation}
i\psi_t+\lvert\psi\rvert^2\psi +V(\bm x)\psi=0
\end{equation}
where we will take
\begin{equation}
V(\bm x)=\lVert \bm x \rVert^2_{l^2}=\sum_{k=1}^Nx_k^2
\end{equation}
in which $N$ is the space dimension. Show that this equation can be solved by splitting it into
\begin{equation}
i\psi_t+\Delta\psi=0 \label{eq:GPsplit1}
\end{equation}
and
\begin{equation}
i\psi_t+\lvert\psi\rvert^2\psi+V(\bm x)\psi=0.  \label{eq:GPsplit2}
\end{equation}
Be sure to explain how eqs.\ \eqref{eq:GPsplit1},\eqref{eq:GPsplit2} are solved.
\item[3)] Modify the Matlab codes to solve the Gross-Pitaevskii equation in one, two and three dimensions.
\item[4)] Modify the serial Fortran codes to solve the Gross-Pitaevskii equation in one, two and three dimensions.
\item[5)] Listings \ref{lst:For2dNlsOmp2} and \ref{lst:Makefile2dNLSomp2} give an alternate method of parallelizing an OpenMP program. Make the program in listing \ref{lst:For2dNlsOmp1} as efficient as possible and as similar to that in \ref{lst:For2dNlsOmp2}, but without changing the parallelization strategy. Compare the speed of the two different programs. Try to vary the number of grid points and cores used. Which code is faster on your system? Why do you think this is?

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dNlsOmp2,caption={An OpenMP Fortran program to solve the 2D nonlinear Schr\"{o}dinger equation using splitting.}]{./CubicNonlinearSchrodinger/Programs/NLS2dFortran/NLSsplitting.f90}

\lstinputlisting[style=make_style,language=make,label=lst:Makefile2dNLSomp2,caption={An example makefile for compiling the OpenMP program in listing \ref{lst:For2dNlsOmp2}.  The example assumes one is using Flux and has loaded environments for the intel compiler as well as the Intel compiled version of FFTW.  If one is using the freely available GCC compilers instead of the Intel compilers, change the flag $-openmp$ to $-fopenmp$.}]{./CubicNonlinearSchrodinger/Programs/NLS2dFortran/makefile}

\item[6)] Modify the OpenMP Fortran codes to solve the Gross-Pitaevskii equation in two and three dimensions.
\item[7)] \footnote{This question is due to a project by Joshua Kirschenheiter.} Some quantum hydrodynamic models for plasmas are very similar to the nonlinear Schr\"{o}dinger equation and can also be numerically approximated using splitting methods.  A model for a plasma used by Eliasson and Shukla~\cite{EliShu09} is
$$i\Psi_t+\Delta \Psi + \phi\Psi - \lvert \Psi \rvert^{4/D}\Psi=0$$
and
$$ \Delta \phi =\lvert \Psi \rvert^2-1,$$
where $\Psi$ is the, $\phi$ the and $D$ the dimension, typically 1,2 or 3. This equation can be solved in a similar manner to the Davey-Stewartson equations in Klein, Muite and Roidot~\cite{KleMuiRoi11}. Specifically, first solve
$$i\Psi_t+\Delta \Psi=0$$
using the Fourier transform so that
$$\Psi(\delta t)=\exp\left(-i\Delta^2 \delta t\right)\Psi(0)$$
Then solve
$$\phi=\Delta^{-1}\left(\lvert \Psi \rvert^2-1\right)$$
using the Fourier transform.  Finally, solve
$$i\Psi_t+ \phi\Psi - \lvert \Psi \rvert^{4/D}\Psi=0$$
using the fact that at each grid point $\phi\Psi - \lvert \Psi \rvert^{4/D}$ is a constant, so the solution is
$$\Psi=\exp\left[ i\left(\phi-\lvert \Phi \rvert^{4/D}\right)\delta t\right].$$
\item[8)] \footnote{This question is due to a project by Kohei Harada and Matt Warnez.}The operator splitting method can be used for equations other than the nonlinear Schr\"{o}dinger equation. Another equation for which operator splitting can be used is the complex Ginzburg-Landau equation
$$\frac{\partial A}{\partial t}=A+(1+i\alpha)\Delta A- (1+i\beta)|A|^2A,$$
where $A$ is a complex function, typically of one, two or three variables. An example one dimensional code is provided in listing \ref{lst:Nls16thOrderMatlab}, based on an earlier finite difference code by Blanes, Casa, Chartier and Miura, using the methods described in Blanes et al.~\cite{BlaCasChaMur12}. By using complex coefficients,  Blanes et al.~\cite{BlaCasChaMur12} can create high order splitting methods for parabolic equations. Previous attempts to do this have failed since if only real coefficients are used, a backward step which is required for methods higher than second order leads to numerical instability. Modify the example code to solve the complex Ginzburg-Landau equation in one, two and then in three spatial dimensions. The linear part
$$\frac{\partial A}{\partial t}=A+(1+i\alpha)\Delta A$$
can be solved explicitly using the Fourier transform. To solve the nonlinear part,
$$\frac{\partial A}{\partial t}=- (1+i\beta)|A|^2A$$
consider
$$\frac{\partial |A|^2}{\partial t}=\frac{\partial A}{\partial t}A^*+\frac{\partial A^*}{\partial t}A=2|A|^4$$
and solve this exactly for $|A|^2$. To recover the phase, observe that
$$\frac{\partial \log(A)}{\partial t}=- (1+i\beta)|A|^2$$
which can also be integrated explicitly since $|A|^2(t)$ is known.

\lstinputlisting[style=matlab_style,label=lst:Nls16thOrderMatlab,caption={A Matlab program which uses 16th order splitting to solve the cubic nonlinear Schr\"{o}dinger equation.}]{./CubicNonlinearSchrodinger/Programs/NLSsplitting1d16Order.m}
\end{enumerate}

%%%%%%%
%Section
%%%%%%%
\section{Distributed Memory Parallel: MPI}

For this section, we will use the library 2DECOMP{\&}FFT available from \url{http://www.2decomp.org/index.html}. The website includes some examples which indicate how this library should be used, in particular the sample code at \url{http://www.2decomp.org/case_study1.html} is a very helpful indication of how one converts a code that uses FFTW to one that uses MPI and the aforementioned library.

Before creating a parallel MPI code using 2DECOMP{\&}FFT, we will generate  a serial Fortran code that uses splitting to solve the 3D nonlinear Schr\"{o}dinger equation. Rather than using loop-based parallelization to do a sequence of one dimensional fast Fourier transforms, we will use FFTW's three dimensional FFT, so that the serial version and MPI parallel version have the same structure. The serial version is in listing \ref{lst:For3dNls}. This file can be compiled in a similar manner to that in \ref{lst:MakefileHeat}.

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For3dNls,caption={A Fortran program to solve the 3D nonlinear Schr\"{o}dinger equation using splitting and FFTW.}]{./CubicNonlinearSchrodinger/Programs/NLS3dFortran/NLSsplitting.f90}

In comparison to the previous programs, the program in listing \ref{lst:For3dNls} writes out its final data as a binary file. This is often significantly faster than writing out a text file, and the resulting file is usually much smaller in size. This is important when many such files are written and/or if individual files are large. Due to the formatting change, the binary file also needs to be read in slightly differently. The Matlab script in listing \ref{lst:NlsSplit3DMatlabPlot} shows how to do this.

\lstinputlisting[style=matlab_style,label=lst:NlsSplit3DMatlabPlot,caption={A Matlab program which plots a numerical solution to a 3D nonlinear Schr\"{o}dinger equation generated by listings \ref{lst:For3dNls} or \ref{lst:For3dNlsMPI}.}]{./CubicNonlinearSchrodinger/Programs/NLS3dFortran/plotcreate.m}

We now modify the above code to use MPI and the library 2DECOMP{\&}FFT.  The library 2DECOMP{\&}FFT hides most of the details of MPI although there are a few commands which it is useful for the user to understand. These commands are:
\begin{itemize}
\item \texttt{USE mpi} or \texttt{INCLUDE 'mpif.h'}
\item \texttt{MPI\_INIT}
\item \texttt{MPI\_COMM\_SIZE}
\item \texttt{MPI\_COMM\_RANK}
\item \texttt{MPI\_FINALIZE}
\end{itemize}

The program is listed in listing \ref{lst:For3dNlsMPI}, please compare this to the serial code in \ref{lst:For3dNls}. The library 2DECOMP{\&}FFT does a domain decomposition of the arrays so that separate parts of the arrays are on separate processors.  The library can also perform a Fourier transform on the arrays even though they are stored on different processors -- the library does all the necessary message passing and transpositions required to perform the Fourier transform. It should be noted that the order of the entries in the arrays after the Fourier transform is not necessarily the same as the order used by FFTW. However, the correct ordering of the entries is returned by the structure \texttt{decomp} and so this structure is used to obtain starting and stopping entries for the loops. We assume that the library 2DECOMP{\&}FFT has been installed in an appropriate location.

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For3dNlsMPI,caption={A Fortran program to solve the 3D nonlinear Schr\"{o}dinger equation using splitting and 2DECOMP{\&}FFT.}]{./CubicNonlinearSchrodinger/Programs/NLS3dFortranMPI/NLSsplitting.f90}

%%%%%%%
%Section
%%%%%%%
\section{Exercises}

\begin{enumerate}
\item[1)] Write an MPI  code using 2DECOMP{\&}FFT to solve the Gross-Pitaevskii equation in three dimensions.
\item[2)] Learn to use either VisIt (\url{https://wci.llnl.gov/codes/visit/}) or Paraview (\url{http://www.paraview.org/}) and write a script to visualize two and three dimensional output in a manner that is similar to the Matlab codes.
\end{enumerate}