\documentclass{memoir}
\usepackage{notestemplate}

%\logo{~/School-Work/Auxiliary-Files/resources/png/logo.png}
%\institute{Rice University}
%\faculty{Faculty of Whatever Sciences}
%\department{Department of Mathematics}
%\title{Class Notes}
%\subtitle{Based on MATH xxx}
%\author{\textit{Author}\\Gabriel \textsc{Gress}}
%\supervisor{Linus \textsc{Torvalds}}
%\context{Well, I was bored...}
%\date{\today}

%\makeindex

\begin{document}

% \maketitle

% Notes taken on

\subsection{Meromorphic Functions and Residues}
\label{sub:meromorphic_functions_and_residues}

\begin{defn}[Meromorphic Function]
	Let \(\Omega \subset \C\) be an open subset. Suppose that a function \(f\) has poles at \(\left\{ z_i \right\}_{i=1}^{\infty}\). Then \(f\) is \textbf{meromorphic} if \(f\) is holomorphic on \(\Omega \setminus\left\{ z_i \right\}_{i=1}^{\infty}\).
\end{defn}

\begin{anki}
TARGET DECK
Complex Qual::Complex Analysis
START
MathJaxCloze
Text: Let \(\Omega \subset \C\) be an open subset. Suppose that a function \(f\) has {{c1::poles at \(\left\{ z_i \right\}_{i=1}^{\infty}\)}}. Then \(f\) is **meromorphic** if {{c1::\(f\) is holomorphic on \(\Omega \setminus\left\{ z_i \right\}_{i=1}^{\infty}\)}}.
Tags: analysis complex_analysis singularities_residues defn
<!--ID: 1626125582286-->
END
\end{anki}

\begin{exmp}[Polynomials]
	Let \(p \in \C[z]\) be a polynomial. Then \(f(z) = \frac{1}{p(z)}\) is a meromorphic function with poles at the zeros of \(p\).
\end{exmp}

Earlier, we showed that a function holomorphic on \(\Omega \) is analytic on \(\Omega \), and hence is equal to a unique power series within \(\Omega \). Because poles are isolated, if \(f\) is meromorphic on a region \(\Omega \), then there exists a Laurent expansion at each pole \(\left\{ z_i \right\}_{i=1}^{\infty}\) which is valid on some open neighborhood of each pole. The Laurent expansion carries useful information about the pole, which we capture with the notion of \textit{residues}.

\begin{defn}[Residue]
	Let \(\Omega \subset \C\) be an open subset and let \(f\) be meromorphic on \(\Omega \) with a pole at \(z_0\) of order \(k\), with Laurent expansion
	\begin{align*}
		f(z) = \sum_{n=-k}^{\infty} a_n (z-z_0)^{n}.
	\end{align*}
	The coefficient \(a_{-1}\) is the \textbf{residue} of \(f\) at \(z_0\), which we will denote \(\textrm{Res}_{z_0}f := a_{-1}\). Furthermore, the partial sum
	\begin{align*}
		\sum_{n=-k}^{-1} a_n (z-z_0)^{n}
	\end{align*}
	is called the \textbf{principal part} of \(f\) at \(z_0\).
\end{defn}

We can use residues to calculate integrals rather easily.

\begin{anki}
START
MathJaxCloze
Text: Let \(\Omega \subset \C\) be an open subset and let \(f\) be meromorphic on \(\Omega \) with a pole at \(z_0\) of order \(k\), with Laurent expansion
\(\begin{align*}
  	f(z) = \sum_{n=-k}^{\infty} a_n (z-z_0)^{n}.
  \end{align*}\)
The {{c1::coefficient \(a_{-1}\)}} is the **residue** of \(f\) at \(z_0\), which we will denote {{c1::\(\textrm{Res}_{z_0}f := a_{-1}\)}}. Furthermore, the partial sum
{{c1::\(\begin{align*}
      	\sum_{n=-k}^{-1} a_n (z-z_0)^{n}
        \end{align*}\)}}
is called the **principal part** of \(f\) at \(z_0\).
Tags: analysis complex_analysis singularities_residues defn
<!--ID: 1626125582303-->
END
\end{anki}

\begin{thm}
	Let \(z_0 \in \C\) be given, and suppose \(f\) is a complex-valued function meromorphic on \(D_r(z_0)\) for some \(r>0\) with exactly one pole at \(z_0\). Then
	\begin{align*}
		\int_{\partial D_r(z_0)}f \,d t = 2\pi i \textrm{Res}_{z_0}f.
	\end{align*}
\end{thm}

We can extend this to a more general residue formula.

\begin{cor}[Residue Formula]
	Let \(\Omega \subset \C\) be an open set and let \(\gamma \) be a closed piecewise-smooth curve in \(\Omega \) homologous to a point. Suppose \(f\) is a meromorphic function on \(\Omega \) with poles \(\left\{ z_i \right\}_{i=1}^{n}\). Then
	\begin{align*}
		\int_{\gamma } f \,d t = 2\pi i \sum_{j=1}^{n} n(\gamma ,z_j) \textrm{Res}_{z_j}f.
	\end{align*}
\end{cor}

\begin{anki}
START
MathJaxCloze
Text: Let \(\Omega \subset \C\) be an open set and let \(\gamma \) be a closed piecewise-smooth curve in \(\Omega \) homologous to a point. Suppose \(f\) is a meromorphic function on \(\Omega \) with poles \(\left\{ z_i \right\}_{i=1}^{n}\). Then
{{c1::\(\begin{align*}
        	\int_{\gamma } f \,d t = 2\pi i \sum_{j=1}^{n} n(\gamma ,z_j) \textrm{Res}_{z_j}f.
        \end{align*}\)::residue formula}}
Tags: analysis complex_analysis singularities_residues
<!--ID: 1626125582321-->
END
\end{anki}

\begin{exmp}
	\(\int_{-\infty}^{\infty} \frac{1}{1+x^2}\,d x = \pi \)
\end{exmp}

\begin{exmp}

\end{exmp}

\begin{lemma}[Tools to Help Calculate Residues]
	\begin{itemize}
		\item Let \(f\) be a meromorphic function with a simple pole at \(z_0\), and suppose \(g\) is a function holomorphic at \(z_0\). Then
			\begin{align*}
				\textrm{Res}_{z_0}(fg) = g(z_0) \textrm{Res}_{z_0}(f).
			\end{align*}
		\item Let \(f\) be a function with a simple zero at \(z_0\). Then \(\sfrac{1}{f}\) has a simple pole at \(z_0\) with residue \(\sfrac{1}{f'(z_0)}\).
		\item Let \(f\) be a function with a pole of order \(n\) at \(z_0\). Then
			\begin{align*}
				\textrm{Res}_{z_0}f = \lim_{z \to z_0} \frac{1}{(n-1)!} \left( \frac{d}{\,d z} \right)^{n-1}(z-z_0)^{n}f(z).
			\end{align*}
	\end{itemize}
\end{lemma}

\begin{anki}
START
MathJaxCloze
Text:
* Let \(f\) be a meromorphic function with a simple pole at \(z_0\), and suppose \(g\) is a function holomorphic at \(z_0\). Then
 {{c1::\(\begin{align*}
         	\textrm{Res}_{z_0}(fg) = g(z_0) \textrm{Res}_{z_0}(f).
         \end{align*}\)::residue of product}}
* Let \(f\) be a function with a simple zero at \(z_0\). Then {{c2::\(\sfrac{1}{f}\)}} has a simple pole at \(z_0\) with residue {{c2::\(\sfrac{1}{f'(z_0)}\)}} .
* Let \(f\) be a function with a pole of order \(n\) at \(z_0\). Then
{{c1::\(\begin{align*}
        	\textrm{Res}_{z_0}f = \lim_{z \to z_0} \frac{1}{(n-1)!} \left( \frac{d}{\,d z} \right)^{n-1}(z-z_0)^{n}f(z).
        \end{align*}\)::residue of point via higher derivatives}}
Tags: analysis complex_analysis singularities_residues
<!--ID: 1626125582338-->
END
\end{anki}

%\begin{defn}[Bounding]
%	A cycle \(\gamma\) is said to bound the region \(M\) if and only if \(n(\gamma,a)\) is defined and equal to \(1\) for all points \(a \in M\) and either undefined or equal to zero for all points \(a\) not in \(M\).
%\end{defn}

\subsection{Argument Principle}
\label{sub:argument_principle}

There is a deep connection between residues, winding numbers, and the complex logarithm. It turns out that all of these notions capture in some form the change in argument of a meromorphic function-- the theorem that ties these ideas together is what we refer to as the \textit{argument principle}.\\

Recall that the goal of the complex logarithm is to provide an inverse function for \(e^{x}\). In other words, we define the logarithm so that
\begin{align*}
	\ln(\left| z \right| e^{i \theta }) = \ln_\R (\left| z \right| ) + i \theta .
\end{align*}
We refer to \(\theta \) as \(\textrm{Arg}(z)\) more generally. The imaginary term alone captures in its entirety the angle of the input \(z\)-- this observation leads us to utilize the logarithm as an intermediate tool to capture more generally the change in angle of a function.

\begin{general}[Logarithmic Derivative]
	Let \(f\) be a complex-valued meromorphic function. Consider the composition \(\ln(f)\). Observe that
	\begin{align*}
		\frac{\partial }{\partial z} \ln(f(z)) = \frac{f'(z)}{f(z)}.
	\end{align*}
	We refer to \(\frac{f'}{f}\) as the \textbf{logarithmic derivative} of \(f\). \\

	Notably, the logarithmic derivative carries an additive formula:
	\begin{align*}
		\frac{\partial }{\partial z} \ln\left( \prod_{n=\infty}^{N} f_n  \right) = \sum_{n=1}^{N} \frac{f'_n}{f_n}.
	\end{align*}
\end{general}

\begin{anki}
START
MathJaxCloze
Text: Let \(f\) be a complex-valued meromorphic function. Consider the composition \(\ln(f)\). Observe that
 {{c1::\(\begin{align*}
         	\frac{\partial }{\partial z} \ln(f(z)) = \frac{f'(z)}{f(z)}.
         \end{align*}\)}}
We refer to {{c1::\(\frac{f'}{f}\)}} as the **logarithmic derivative** of \(f\).

Notably, the logarithmic derivative carries an additive formula:
{{c1::\(\begin{align*}
        	\frac{\partial }{\partial z} \ln\left( \prod_{n=\infty}^{N} f_n  \right) = \sum_{n=1}^{N} \frac{f'_n}{f_n}.
        \end{align*}\)}}
Extra: What is the connection between the logarithm and residues? Suppose \(f\) is holomorphic with a zero of order \(n\) at \(z_0\). Then
\(\begin{align*}
  	f(z) = (z-z_0)^{n}g(z)
  \end{align*}\)
for \(g\) holomorphic and non-vanishing in a neighborhood of \(z_0\). The logarithmic derivative of \(f\) is hence
\(\begin{align*}
  	\frac{f'(z)}{f(z)} = \frac{n}{z-z_0}+ \frac{g'(z)}{g(z)}
  \end{align*}\)
by the additive property of the logarithmic derivative. But \(g\) and \(g'\) are holomorphic and non-vanishing-- and hence the logarithmic derivative has transformed \(f\) into a meromorphic function with a simple pole at \(z_0\) with residue \(\textrm{Res}_{z_0} = n\). Likewise, if \(f\) is meromorphic with a pole of order \(n\) at \(z_0\), then
\(\begin{align*}
  	f(z) = (z-z_0)^{-n}h(z)
  \end{align*}\)
for \(h\) holomorphic and non-vanishing. One can see the above follows but with \(-n\).
Tags: analysis complex_analysis singularities_residues defn
<!--ID: 1626125582356-->
END
\end{anki}

At this point, the connection between the logarithm and winding number is clear-- by performing a \(w\)-substitution with \(w = f(z)\), we see that that the logarithmic derivative is merely another way to write
\begin{align*}
	\frac{f'}{f} \,d z = \frac{dw}{w}
\end{align*}
and hence when integrated on a closed curve, captures the winding numbers of singularities of \(f\) inside the curve. Furthermore, we have the identity
\begin{align*}
	\int_\gamma \,d \ln(f(z)) = \int_\gamma \,d \left[ \ln \left| f(z) \right| + i \textrm{arg}(f(z)) \right] = \int_\gamma \,d \ln\left| f(z) \right| + \int_\gamma \,d \textrm{arg}(f(z)).
\end{align*}
Because the first integral always evaluates to zero (verify this), we see that the logarithmic derivative allows us to capture the change in argument of a function.\\

What is the connection between the logarithm and residues? Suppose \(f\) is holomorphic with a zero of order \(n\) at \(z_0\). Then
\begin{align*}
	f(z) = (z-z_0)^{n}g(z)
\end{align*}
for \(g\) holomorphic and non-vanishing in a neighborhood of \(z_0\). The logarithmic derivative of \(f\) is hence
\begin{align*}
	\frac{f'(z)}{f(z)} = \frac{n}{z-z_0}+ \frac{g'(z)}{g(z)}
\end{align*}
by the additive property of the logarithmic derivative. But \(g\) and \(g'\) are holomorphic and non-vanishing-- and hence the logarithmic derivative has transformed \(f\) into a meromorphic function with a simple pole at \(z_0\) with residue \(\textrm{Res}_{z_0} = n\). Likewise, if \(f\) is meromorphic with a pole of order \(n\) at \(z_0\), then
\begin{align*}
	f(z) = (z-z_0)^{-n}h(z)
\end{align*}
for \(h\) holomorphic and non-vanishing. One can see the above follows once more but with \(-n\).\\

In summary-- the logarithmic derivative transforms \textit{singularities of algebraic order \(n\)} into \textit{simple poles with residue \(n\)}. Thus, a contour integral of the logarithmic derivative will sum the algebraic orders of the singularities contained. This remarkable conclusion is the argument principle.

\begin{thm}[Argument Principle]
	If \(f\) is meromorphic in \(\Omega\subset \C \) with zeros \(\left\{ z_j \right\}\) and the poles \(\left\{ w_k \right\} \), then
	\begin{align*}
		\frac{1}{2\pi i} \int_{\gamma} \frac{f'}{f} \,d t = \sum_{j} n(\gamma,z_j) - \sum_{k} n(\gamma,w_k)
	\end{align*}
	for every closed piecewise-smooth curve \(\gamma\) which is homologous to zero in \(\Omega \) and does not pass through any of the zeros or poles.
\end{thm}
In light of our earlier discussion, we refer to
\begin{align*}
	\frac{1}{2\pi i} \int_\gamma \frac{f'}{f}\,d t
\end{align*}
as the \textbf{winding number of \(f\) along \(\gamma \)}. We urge the reader to appreciate how remarkable it is that the change in angle of a meromorphic function can be captured so simply via its zeros and poles!

\begin{anki}
START
MathJaxCloze
Text: **Argument Principle**
If \(f\) is meromorphic in \(\Omega\subset \C \) with zeros \(\left\{ z_j \right\}\) and the poles \(\left\{ w_k \right\} \), then
{{c1::\(\begin{align*}
        	\frac{1}{2\pi i} \int_{\gamma} \frac{f'}{f} \,d t = \sum_{j} n(\gamma,z_j) - \sum_{k} n(\gamma,w_k)
        \end{align*}\)}}
	for every closed piecewise-smooth curve \(\gamma\) which is homologous to zero in \(\Omega \) and does not pass through any of the zeros or poles.
Extra: We refer to
\(\begin{align*}
  	\frac{1}{2\pi i} \int_\gamma \frac{f'}{f}\,d t
  \end{align*}\)
as the \textbf{winding number of \(f\) along \(\gamma \)}.
Tags: analysis complex_analysis singularities_residues
<!--ID: 1626125582372-->
END
\end{anki}

\begin{cor}[Rouche's Theorem]
	Let \(\Omega \subset \C\) be an open bounded subset with \(\partial \Omega \) piecewise-smooth. Let \(f,g\) be holomorphic functions on \(\overline{\Omega }\) with \(\left| g(z) \right| < \left| f(z) \right| \) for all \(z \in \partial\Omega \). Then \(f\) and \(f+g\) have the same number of zeros in \(\Omega \).
\end{cor}
We can interpret \(g\) as a "holomorphic perturbation" of \(f\) in \(\Omega \)-- and hence the theorem really states that if the perturbation is bounded above by \(f\) along the boundary, then it cannot perturb any zero outside \(\Omega \). To see why this should be true, recall that the maximum principle tells us that \(g\) globally and locally achieves its maxima on the boundary. If \(g\) were to "push" a zero \(z_0\) of \(f\) outside \(\Omega \), then \(g\) would have to exceed \(f\) somewhere on the boundary for the maximum principle to hold. Formalizing this argument is difficult, however. The argument principle simplifies this process greatly.
\begin{proof}
	Note that the hypothesis implicitly requires that \(f\) and \(f+g\) are non-zero on \(\gamma \). Thus, we can rewrite
	\begin{align*}
		f+g &= f\left( 1+ \frac{g}{f} \right)
	\end{align*}
	Applying the logarithmic derivative gives us
	\begin{align*}
		\int_\gamma \frac{(f+g)'}{f+g}\,d t = \int_\gamma \frac{f'}{f} \,d t + \int_{\gamma }\frac{(1+\sfrac{g}{f})'}{1+\sfrac{g}{f}} \,d t.
	\end{align*}
	But we have that \(\left| \frac{g}{f} \right|<1\) on \(\gamma \), and so there are no zeros of \(\frac{g}{f}\) within \(\gamma \), and likewise no poles (because \(\frac{g}{f}\) is holomorphic). Hence, the argument principle tells us that the right-hand integral is zero, and hence
	\begin{align*}
		\int_\gamma \frac{(f+g)'}{f+g} \,d t = \int_\gamma \frac{f'}{f}\,d t
	\end{align*}
	which implies they must have the same zeros.
\end{proof}

\begin{anki}
START
MathJaxCloze
Text: **Rouche's Theorem**
Let \(\Omega \subset \C\) be an open bounded subset with \(\partial \Omega \) piecewise-smooth. Let \(f,g\) be holomorphic functions on \(\overline{\Omega }\) with \(\left| g(z) \right| < \left| f(z) \right| \) for all \(z \in \partial\Omega \). Then {{c1::\(f\) and \(f+g\) have the same number of zeros in \(\Omega \)}}.
Extra: We can interpret \(g\) as a "holomorphic perturbation" of \(f\) in \(\Omega \)-- and hence the theorem really states that if the perturbation is bounded above by \(f\) along the boundary, then it cannot perturb any zero outside \(\Omega \).
Tags: analysis complex_analysis singularities_residues
<!--ID: 1626125582389-->
END
\end{anki}

Now we will look at some applications of the two powerful theorems here to evaluating integrals.

%% Deal with later

% \subsection{Definite Integrals}
% \label{sec:definite_integrals}
%
% First, note that all integrals of the form
% \begin{align*}
% 	\int_{0}^{2\pi} R(\cos(\theta),\sin(\theta)) \,d \theta
% \end{align*}
% where the integrand is a rational function of the two trigonometric functions can be done via residues. Substituting \(z = e^{i\theta}\) yields
% \begin{align*}
% 	-i \int_{\left| z \right| =1} R \left[ \frac{1}{2}\left( z+ \frac{1}{z} \right) , \frac{1}{2i}\left( z-\frac{1}{z} \right)  \right] \frac{\,d z}{z}.
% \end{align*}
%
% --
%
% An integral of the form
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x) \,d x
% \end{align*}
% converges if and only if the rational function \(R(x)\) the degree of the denomination is at least 2 degrees higher tahn taht of the numerator, and if there is no pole ON the real axis. We do this by integrating the complex function \(R(z)\) over a closed curve consisting of a line segment \((-p,p)\) and the semicircle from \(p,-p)\) in the upper half plane. Choosing \(p\) large enough  encloses all poles in the upper half plane, and so the integral is equal to \(2\pi i\) times the sum of the residues in the upper half plane. So,
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x) \,d x = 2\pi i \sum_{y>0} \textrm{Res}R(z)
% \end{align*}
%
% -
%
% We can do this same method for integrals of the form
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x) e^{ix} \,d x
% \end{align*}
% whose real and imaginary parts determine the integrals
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x) \cos(x) \,d x, \quad \int_{-\infty}^{\infty} R(x) \sin(x) \,d x
% \end{align*}
% Because \(e^{-y}\) is bounded in the upper half plane, so the integral over the semicircle tends to zero (as long as \(R(z)\) has a zero of at least order two at infinity). Thus
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x) e^{ix}\,d x = 2\pi i \sum_{y>0} \textrm{Res}R(z) e^{ix} .
% \end{align*}
% This holds when \(R(z)\) only has order one zero at infinity, but not by the semicircle argument.\\
%
% Note that we assumed that \(R(z)\) has no poles on the real axis; however, if it coincides with zeros of \(\sin(x)\), then it very well can be evaluated! It will work out to yield
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x)e^{ix}\,d x = 2\pi i \sum_{y>0} \textrm{Res}R(z) e^{iz} + \pi i \sum_{y=0} \textrm{Res}R(z)e^{iz}
% \end{align*}
% Often, integrals containing powers of cosine and sine can be written as linear combinations via double angle identities, and hence
% \begin{align*}
% 	\int_{-\infty}^{\infty} R(x) e^{imx}\,d x = \frac{1}{m} \int_{-\infty}^{\infty} R\left( \frac{x}{m} \right) e^{ix} \,d x.
% \end{align*}
%
% -
%
% Now consider
% \begin{align*}
% 	\int_{0}^{\infty} x^{\alpha}R(x) \,d x
% \end{align*}
% with \(\alpha \in (0,1)\subset \R\). This only converges if \(R(z)\) has a zero of at least order two at \(\infty\) and at most a simple pole at the origin. Using the substitution of \(x = t^2\) and then omitting the negative imaginary axis, we can apply the residue theorem to yield
% \begin{align*}
% 	(1-e^{2\pi i \alpha}) \int_{0}^{\infty} z^{2\alpha+1}R(z^2) \,d z
% \end{align*}
% Thus we determine the residues of the integrand in the upper hlaf plane. This is the same as the residues of \(z^{\alpha}R(z)\) in the whole plane.
% \printindex
\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% UMB-CS110-2015S: Introduction to Computing
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/UMB-CS110-2015S
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\def \topDirectory {.}
\def \texDirectory {\topDirectory/src/main/tex}

\documentclass[12pt,letterpaper,twoside]{article}
\usepackage{\texDirectory/template/style/directives}
\usepackage{\texDirectory/template/style/assignment}
\input{\texDirectory/template/config}

\begin{document}

\doc{title}{Lab Session Problems}
\doc{points}{0}

\prepare{header}

\subsection*{Week 4}
\hfill \textbf{March 3, 2015}

Write a program that asks a number $N$ using Scanner Class and does the following.

\begin{enumerate}[itemsep=0pt]
\item Prints all integer numbers from $1$ to $N$.
\item Calculates a random number M from $0$ to $N$ and prints numbers from $M$ to $N$.
\item Asks for number $M$ and generates $M$ random numbers in range $0$ to $N$ and prints their average.
\end{enumerate}

\end{document}


\section{Axiom of union}


%%========================================================
%% Section 0.02: Overview of the ILW Direct Scattering Map
%%========================================================

\documentclass[../dissertation.tex]{subfiles}

\begin{document}
\section{Overview of the ILW Direct Scattering Map}\label{sec0:DM}

While there are a series of papers from the late 1970's and early 1980's
culminating in a paper by Y. Kodama, M.J. Ablowitz and J. Satsuma
\cite{Kodama1982} and a subsequent paper by P.M. Santini, M.J. Ablowitz and A.S.
Fokas \cite{Santini1984} which formally describe the Inverse Scattering
Transform (IST) for the ILW, little research has been done to place the Inverse
Scattering Method for the ILW equation on a rigorous mathematical footing.
At the time of writing this dissertation, the author found no results in the
literature showing that the IST for the ILW equation is actually well-defined
or bi-Lipschitz continuous\textemdash{}even for small data.

As described in Section \ref{sec0:IST}, using the Inverse
Scattering Method to solve the ILW entails constructing
an invertible, bi-Lipschitz
continuous map from initial data to the corresponding scattering data in such
a way that linearizes the flow\textemdash{}\textit{i.e.} the time dependence
of the output of this map applied to initial data is determined by a linear
differential equation. The ``forward
direction'' of this map which takes initial data to scattering data is referred to
as the ``direct scattering map,'' and its inverse
is referred to as the ``inverse scattering map.'' This distinction is made as the process for
constructing the direct scattering map for a given equation is often very different from
the process for constructing the corresponding inverse scattering map. As previously mentioned,
the combination of this direct scattering map with the corresponding inverse
scattering map and the linear differential equation used to propagate the scattering data in
time is called an Inverse Scattering Transform for the ILW.

What allows us to construct an IST for the ILW
is
fact that the ILW is an isospectral flow for the linear spectral problem
\begin{align}\label{eq0:SpecProb}
	L_\delta (\Psi)
		:= \frac{1}{i} \frac{\partial}{\partial x} \Psi^+
			- \zeta \left(\Psi^+ - \Psi^-\right) = u \Psi^+,
\end{align}
which is a part of the Lax pair\footnote{Please see the appendix titled
\hyperref[app:Lax]{\textit{Lax Representation}} for a comparison of
\eqref{eq0:Lax} with the ILW Lax pair typically given in the literature.}
\begin{subequations}\label{eq0:Lax}
	\begin{align}
		\frac{1}{i} \frac{\partial}{\partial x} \Psi^+
				- \zeta \left(\Psi^+ - \Psi^-\right)
			 &= u \Psi^+
			 \label{eq0:LaxX} \\
		\frac{1}{i} \frac{\bd}{\bd t} \Psi^\pm + 2 i
				\( \zeta-\frac{1}{2\delta} \) + \Psi_{xx}
			&= \left[ \pm i u_x  - T u_x + \eta  \right] \Psi^\pm,
			\label{eq0:LaxT}
\end{align}
\end{subequations}
where $\Psi$ is a function analytic in the complex strip\label{sym:Sdelta}
\[
	\mathcal S_\delta := \{ z \in \mathbb C \, : \, 0 < \im z < 2 \delta\}
\]
with respective lower and upper boundary values\label{sym:bndries}
\begin{align}\label{eq0:bndryvaluedefn}
	\Psi^+(x):= \lim_{y \searrow 0} \Psi(x + i y), \qquad
	\text{and}\qquad
	\Psi^-(x):= \lim_{y \nearrow 2\delta} \Psi(x + i y),
\end{align}
where we use the superscript notation $f^\pm$ throughout this dissertation to
indicate lower and upper boundary values as shown above of functions $f$
analytic on the complex strip $\mathcal S_\delta$.
The spectral parameter $\zeta \in (0, \infty)$\label{sym:zeta} is itself parametrized by a second spectral
parameter $\lambda \in \mathbb R$\label{sym:lambda} as
\[
	\zeta(\lambda; \delta) := \frac{\lambda}{1 - e^{-2 \delta \lambda}},
\]
where we use the notation $f(\dotarg; t_1, \ldots, t_n):=f_{t_1, \ldots, t_n}$ to denote a
family of functions ``indexed'' in a (possibly) uncountable sense by $t_1, \ldots, t_n$.
The output of direct scattering map $\mathscr D$ for the ILW is determined by the
lower boundary values
$M_1^+$, $M_e^+$, $N_1^+$, $N_e^+$ of eigenfunctions $M_1$, $M_e$, $N_1$, $N_e$
of $L_\delta$ which satisfy the asymptotic conditions given in \eqref{eq0:JostDEasymp}.
Such eigenfunctions are referred to in the literature as ``Jost solutions,'' and, given their
importance in the construction of the direct scattering map $\mathscr D$, we explicitly
define Jost solutions as follows:
\begin{defn}[Jost solutions]\label{defn0:jost}
	The Jost solutions $M_1$, $M_e$, $N_1$, $N_e$ are solutions to the linear
	spectral problem \eqref{eq0:SpecProb} whose lower boundary values
	$M_1^+$, $M_e^+$, $N_1^+$, $N_e^+$ as defined in \eqref{eq0:bndryvaluedefn}
	obey the following asymptotic conditions
	\begin{subequations}\label{eq0:JostDEasymp}
		\begin{align}
			\lim_{x\to -\infty}
					\inn{x}
					\left(
						M_1^+(x; \lambda, \delta) - 1
					\right)
				&= \lim_{x\to \infty}
						\inn{x}
						\left(
							N_1^+(x; \lambda, \delta) - 1
						\right)
				= 0 \\
			\lim_{x\to -\infty}
					\inn{x} \left(
						M_e^+(x; \lambda, \delta) - e^{i\lambda x}
					\right)
				&= \lim_{x\to \infty}
						\inn{x}
						\left(
							N_e^+(x; \lambda, \delta) - e^{i\lambda x}
						\right)
				= 0,
		\end{align}
	\end{subequations}
	where we use the notation $\inn{x}:=\sqrt{1+|x|^2}$\label{sym:xbracket} to
	indicate a linear weight.

	Additionally, we require the upper boundary values $M_{(\dotarg)}^-$, $N_{(\dotarg)}^-$
	(where $(\dotarg)$ represents either the subscript $1$ or $e$) of
	$M_{(\dotarg)}$, $N_{(\dotarg)}$ to have a decomposition
	\begin{align*}
		M_1^- - 1 &= M_1^{(1)} + M_1^{(2)} \qquad &\text{and}& \qquad
		&N_1^- - 1 &= N_1^{(1)} + N_1^{(2)} \\
		M_e^- - e^{i\lambda x}\,e^{-2\delta\lambda} &= M_e^{(1)} + M_e^{(2)} \qquad &\text{and}& \qquad
		&N_e^- - e^{i\lambda x}\,e^{-2\delta\lambda} &= N_e^{(1)} + N_e^{(2)} \\
	\end{align*}
	satisfying
	\begin{align*}
		\inn{x}^{1+\upsilon} \left|M_{(\dotarg)}^{(1)}(x)\right| \lesssim 1
			\quad (\text{for } x \ll -1), \qquad
		\inn{x}^{1+\upsilon} \left|N_{(\dotarg)}^{(1)}(x)\right| \lesssim 1
			\quad (\text{for } x \gg 1),
	\end{align*}
	and
	\begin{align*}
		\inn{\dotarg}^\tau M_{(\dotarg)}^{(2)},
			~\inn{\dotarg}^\tau N_{(\dotarg)}^{(2)} \in L^2(\mathbb R)
	\end{align*}
	for any $\upsilon \in \left(0,\frac{1}{2}\right)$ and $\tau \in [0,1)$, where we
	use the notation  $a\lesssim b$ to indicate $a \leq C \, b$ for some constant $C>0$.
	\label{sym:lesssim}
	% \begin{subequations}\label{eq0:JostDEasymp}
	% 	\begin{align}
	% 		\left\{
	% 			\begin{aligned}
	% 				&\lim_{x\to -\infty}	M_1^+(x; \lambda, \delta) - 1
	% 					= \lim_{x\to \infty} N_1^+(x; \lambda, \delta) - 1
	% 					= 0 \\
	% 				&\lim_{x\to -\infty} \big( M_e^+(x; \lambda, \delta) - e^{i\lambda x}\big)
	% 					= \lim_{x\to \infty} \big( N_e^+(x; \lambda, \delta) - e^{i\lambda x}\big)
	% 					= 0
	% 			\end{aligned}
	% 		\right.
	% 		\qquad (\lambda \ne 0)
	% 	\end{align}
	% 	and
	% 	\begin{align}
	% 		\left\{
	% 			\begin{aligned}
	% 				&\lim_{x\to -\infty}
	% 						\inn{x}^{-1} \big(M_1^+(x; \lambda, \delta) - 1\big)
	% 						\\
	% 				&\qquad\qquad= \lim_{x\to \infty} \inn{x}^{-1}
	% 						\big(N_1^+(x; \lambda, \delta) - 1\big)
	% 					= 0 \\
	% 				&\lim_{x\to -\infty}\inn{x}^{-1}
	% 						\big( M_e^+(x; \lambda, \delta) - e^{i\lambda x}\big)
	% 						\\
	% 					&\qquad\qquad= \lim_{x\to \infty}\inn{x}^{-1}
	% 						\big( N_e^+(x; \lambda, \delta) - e^{i\lambda x}\big)
	% 					= 0
	% 			\end{aligned}
	% 		\right.
	% 		\qquad (\lambda = 0)
	% 	\end{align}
	% \end{subequations}
\end{defn}

For a given
$u(x)$, the corresponding output $r = \mathscr D u$ of direct scattering map is
given by
$r(\lambda; \delta) = b(\lambda;\delta) / a(\lambda; \delta)$,\label{sym0:reflection}
where
\begin{align*}
	b(\lambda)
		&:=
			\frac{i}{1-2\delta\zeta(-\lambda)}
			\int_{\mathbb R} e^{-i\lambda x}
				u(x) \, M_1^+(x; \lambda,\delta)
			\, \mathrm{d}x
			\\
	a(\lambda)
		&:=
			1
			+ \frac{i}{1-2\delta \zeta(\lambda)}
				\int_{\mathbb R}
					u(x) \, M_1^+(x; \lambda,\delta)
				\, \mathrm{d}x.
\end{align*}
In this dissertation, we prove the following result:
\begin{thm}\label{thm0:MainResult}
	For sufficiently small $c_0 > 0$ the map
	\begin{align*}
		\mathscr D : B_X(0, c_0) &\to L^\infty(\mathbb R) \\
		                       u &\mapsto r
	\end{align*}
	is well-defined for all real $\lambda$, where
	$X$ denotes the space $\inn{\dotarg}^{-4} L^2(\mathbb R)$, and
	$B_X(0, c_0)$ \label{sym:ball}
	is the ball in the
	space $X$ about zero with radius $c_0$.\footnote{See Theorem \ref{thm4:Dwelldefined}
	in Section \ref{sec4:DM} of Chapter \ref{cptr04:DM}.}
	Further, $\mathscr D$\label{sym0:DSM} is
	Lipschitz continuous as a map from $B_X(0, c_0)$ to
	$L^\infty\big((-\infty, k]\cup[k, \infty)\big)$ for each fixed
	$k>0$.\footnote{See Theorem \ref{thm4:DlipR} in Section \ref{sec4:DM} of Chapter
	\ref{cptr04:DM}.}
\end{thm}

A crucial first step to showing that $\mathscr D$ is well-defined is showing that
for each given $u \in B_X$ the corresponding Jost solutions both exist and are
unique. To do so, one uses
the (formal) Green's Functions $G_L$ and $G_R$, given by
\begin{subequations}
	\label{eq0:GFs}
	\begin{align}
		\label{eq0:GFL}
		G_L(z; \lambda, \delta)
			&=
				\lim_{\varepsilon \searrow 0}
				\frac{1}{2\pi}
				\int\limits_{\mathbb R - i\varepsilon}
					\frac{e^{iz\xi}}
						{\xi -  \zeta(\lambda) \left(1- e^{-2 \delta \xi}\right) }
				\, \mathrm{d}\xi,
				\qquad \left(z\in \overline{\mathcal{S}}_\delta \right)\\
		\intertext{and}
		\label{eq0:GFR}
		G_R(z; \lambda, \delta)
			&=
				\lim_{\varepsilon \searrow 0}
				\frac{1}{2\pi}
				\int\limits_{\mathbb R + i\varepsilon}
					\frac{e^{iz\xi}}
						{\xi -  \zeta(\lambda) \left(1- e^{-2 \delta \xi}\right) }
				\, \mathrm{d}\xi,
				\qquad \left(z\in \overline{\mathcal{S}}_\delta \right)
	\end{align}
\end{subequations}
to rewrite \eqref{eq0:SpecProb} with asymptotic conditions \eqref{eq0:JostDEasymp}
as the integral equations
\begin{subequations}
	\label{eq0:JostIE}
	\begin{align}
		\label{eq0:JostIEleft}
		\begin{pmatrix}
			M_1^+(x; \lambda, \delta) \\
			M_e^+(x; \lambda, \delta)
		\end{pmatrix}
			&=
				\begin{pmatrix}
					1 \\
					e^{i\lambda x}
				\end{pmatrix}
				+ \int_{\mathbb R} G_L^+(x - x'; \lambda, \delta)
					u(x')
					\begin{pmatrix}
						M_1^+(x'; \lambda, \delta) \\
						M_e^+(x'; \lambda, \delta)
					\end{pmatrix}
					\, \mathrm{d}x' \\[0.3\baselineskip]
		\label{eq0:JostIEright}
		\begin{pmatrix}
			N_1^+(x; \lambda, \delta) \\
			N_e^+(x; \lambda, \delta)
		\end{pmatrix}
			&=
				\begin{pmatrix}
					1 \\
					e^{i\lambda x}
				\end{pmatrix}
				+ \int_{\mathbb R} G_R^+(x - x'; \lambda, \delta)
					u(x')
					\begin{pmatrix}
						N_1^+(x'; \lambda, \lambda) \\
						N_e^+(x'; \lambda, \lambda)
					\end{pmatrix}
					\, \mathrm{d}x',
	\end{align}
\end{subequations}
where we again use the ``$+$'' superscript to indicate the lower boundary
values of functions analytic in the complex strip $\mathcal S_\delta$.

Formally, assuming that the solutions to equations \eqref{eq0:JostIE}
have analytic extensions to the strip $\mathcal S_\delta$ with upper
boundary values
$M_1^-$, $M_e^-$, $N_1^-$, and $N_e^-$, one can show through a simply
heuristic computation that solutions to the integral equations \eqref{eq0:JostIE}
should satisfy the spectral problem \eqref{eq0:SpecProb} with asymptotic
conditions \eqref{eq0:JostDEasymp}. However, as discussed in Section
\ref{sec1:RootsOfP},the integrand of $G_L$ and $G_R$ has exactly two poles
along the real line as shown in \textemdash{}namely $\xi = 0$ and
$\xi = \lambda$\textemdash{}and countably many poles in the complex plane.
Worse, the Fourier symbol $p(\xi)$ of $G_L$, $G_R$ does not belong to any
standard symbol class due to its radically different asymptotic behavior
as $\xi \to -\infty$, and $\xi \to +\infty$, respectively. As such,
it is hardly obvious that $G_L$ and $G_R$ are even remotely well
defined as convolution operators. As such, before we can even begin to prove
that the direct scattering map $\mathscr D$ is well-defined, we require a
thorough understanding of the Green's functions $G_L$, $G_R$ as convolution
operators\textemdash{}indeed, such is the focus of Chapters \ref{cptr01:GF}
through \ref{cptr03:xContin} of this dissertation.

We begin our study of the Green's functions $G_L$, $G_R$ in Chapter
\ref{cptr01:GF} by analyzing the properties of the lower boundary
values $G_L^+$, $G_R^+$ as functions.
Using a combination of a contour shift
and several dyadic decompositions, we show
that the boundary values $G_L^+$ and $G_R^+$ can be represented as
\begin{align}\label{eq0:GFrep}
	G_L^+(x; \lambda, \delta)
		&=
			\begin{cases}
				K^+(x; \lambda, \delta), &x < 0 \\
				K^+(x; \lambda, \delta)
				+ i \alpha(\lambda; \delta)
				+ i \beta(\lambda; \delta) \, e^{i \lambda \, x},
				&x >0
			\end{cases}
\end{align}
where $G_L(z; \lambda, \delta) = \overline{G_R(-\re z +i\im z; \lambda, \delta)}$,
\begin{align*}
	\alpha(\lambda; \delta)
		= \frac{1}{1-2\delta\lambda(\lambda)},
	\qquad
	\beta(\lambda; \delta)
		= \frac{1}{1 - 2\delta\lambda(-\lambda) e^{-2\delta\lambda}},
\end{align*}
\label{sym0:residues}and the function $K^+$\label{sym0:K} satisfies the properties
\begin{itemize}
	\item[(i)] $K^+(x) = \mathcal O \left( \frac{e^{-\pi|x|}}{x} \right)$
		for $|x| \geq 1$, and
	\item[(ii)] $|K^+(x)| \leq C + C \log\left(\frac{1}{|x|}\right)$ for $|x| < 1$.
\end{itemize}

Our study of the boundary values $G_L^+$, $G_R^+$ continues in Chapter
\ref{cptr02:GFmapping} as we use \ref{eq0:GFrep} to understand the mapping
properties of $G_L^+$, $G_R^+$ as convolution operators. More specifically,
we study the operators $T_{L, \lambda, u}$, $T_{R, \lambda, u}$ given by
\begin{align} \label{eq0:Tstar}
	T_{L, \lambda, u} f:= G_L^+(\dotarg; \lambda, \delta)*(uf)
	\qquad \text{and} \qquad
	T_{R, \lambda, u} f:= G_R^+(\dotarg; \lambda, \delta)*(uf)
\end{align}
and show that the are bounded operators acting the space
$\inn{\dotarg}L^\infty(\mathbb R)$ whose operator norms depend only
on the $\nm{u}_X$ and not on $\lambda$. We further show in Chapter
\ref{cptr02:GFmapping} that for every
$f \in \inn{\dotarg} L^\infty(\mathbb R)$
and $u \in X$ the operators $T_{L, \lambda, u}$, $T_{R, \lambda, u}$
satisfy the asymptotic behavior
\begin{align*}
	\lim_{x\to-\infty} T_{L, \lambda, u} f(x)
		= \lim_{x\to-\infty} T_{R, \lambda, u} f(x)
		= 0
\end{align*}
when real $\lambda \ne 0$, and
\begin{align*}
	\lim_{x\to-\infty} \inn{x}^{-1} T_{L, \lambda, u} f(x)
		= \lim_{x\to-\infty} \inn{x}^{-1} T_{R, \lambda, u} f(x)
		= 0
\end{align*}
for every $\lambda \in \mathbb R$. That $T_{L, \lambda, u}$,
$T_{R, \lambda, u}$ satisfy the above limits is a property we use later to
prove solutions to the integral equations \eqref{eq0:JostIE} satisfy the
asymptotic conditions in \eqref{eq0:JostDEasymp}.

The focus for our final chapter on the the Green's functions, Chapter
\ref{cptr03:xContin}, is analytically extending $G_L^+$, $G_R^+$ in
the variable $x$ to $G_L$ and
$G_R$ defined on the complex strip $\mathcal S_\delta$ and showing that $G_L$, $G_R$
have upper boundary values $G_L^-$, $G_R^-$. Analytically extending $G_L^+$,
$G_R^+$ is important as it allows us to analytically extend the solutions
$M_1^+$, $M_e^+$, $N_1^+$, $N_e^+$ to the integral equations \ref{eq0:JostIE},
which is a prerequisite to showing that solutions to the integral
equations \ref{eq0:JostIE} are Jost solutions. While, extending $G_L^+$, $G_R^+$
(as convolution operators) to the open strip $\mathcal S_\delta$ is straight
forward, showing the existence of the upper boundaries $G_L^-$, $G_R^-$ is
considerably more involved, as it involves working with a singular operator
that is reminiscent of the Hilbert transform. In fact, nearly the entirety of
Chapter \ref{cptr03:xContin} is devoted to proving the existence (in an $L^2$
sense) of $G_L^-$ and $G_R^-$.

Our analysis of the Green's functions in \ref{cptr01:GF} through
\ref{cptr03:xContin} allows us to finally prove in Section \ref{sec4:equiv}
of Chapter \ref{cptr04:DM} the equivalence of Jost solutions and solutions to
the integral equations \eqref{eq0:JostIE}. The big pay-off in proving this
equivalence is that it allows us to consider the Jost solutions as solutions
to Volterra type integral equations instead of an ordinary differential
equation on a complex strip involving complex boundary values. Being able to do
so is invaluable as the theory of Volterra type integral equations is far better
understood (at least by this author) than the theory of such complex ordinary
differential equations. Indeed, it is precisely by treating the Jost solutions
as solutions to integral equations \eqref{eq0:JostIE} that we are ultimately
able in Chapter \ref{cptr04:DM} to prove that $\mathscr D$ is well-defined and,
at least for real $\lambda$ values away from zero, $\mathscr D$ is also
Lipschitz continuous.
\end{document}

% {\color{red}
% Using \eqref{eq0:GFrep}, we further show in Chapter \ref{cptr01:GF} that, when
% taken as a convolution operator, $G_L^+$ and $G_R^+$ are bounded and continuous
% operators on $L^1\cap L^p$ for $p \in (1, 2]$. This allows us
% to show in Chapter \ref{cptr03:xContin} that taken as convolution
% operators, $G_L$, $G_R$ are analytic in the open strip
% $\mathcal S_\delta = \{ z\in \mathbb C \,:\, 0 < z < 2\delta \}$.

% Using techniques from harmonic analysis, we show in Chapter \ref{cptr03:xContin} the
% existence of the upper boundary values $G_L^-$, $G_R^-$, and study their mapping
% properties as convolution operators. Doing so is important as it allows us to
% then prove the equivalence between solutions of the differential equation
% \eqref{eq0:SpecProb} satisfying
% asymptotic conditions \eqref{eq0:JostDEasymp} and solutions of the integral equations
% \eqref{eq0:JostIE}. Proving this equivalence allows us to use the integral equations
% \eqref{eq0:JostIE} to prove the existence and uniqueness of solutions to the linear
% spectral problem \eqref{eq0:SpecProb} governed by asymptotic conditions
% \eqref{eq0:JostDEasymp}.
% Further, by studying the asymptotics of the solutions to \eqref{eq0:JostIE}, we
% are then able to determine in Section \ref{sec3:IE} that the scattering is then given by the integral
% formulas
% \reversemarginpar
% \marginnote{\color{red} Need to prove $a$, $b$ continuous in $u$.
% Want to characterize $a$, $b$ and specify spaces for $a$, $b$.}
% \begin{align*}
% 	a(\lambda)
% 		&=	1 + i \alpha(\lambda)
% 			\int_{-\infty}^x u(x') M_1^+(x',\lambda) \, \mathrm{d}x'	\\
% 	\intertext{and}
% 	b(\lambda)
% 		&=	i \beta(\lambda)
% 			\int_{-\infty}^x e^{-i\lambda x'} u(x') M_1^+(x',\lambda) \, \mathrm{d}x'.
% \end{align*}
% \sout{Since we are able to use the properties of $M_1^+$ to prove that the above integral
% equations are uniquely solvable}, this completes the Direct Map of the ILW's
% Inverse Scattering Transform.

% The first step in constructing the inverse scattering map involves setting up and
% solving a Riemann-Hilbert problem (RHP) involving the scattering data.
% However, in order to construct the correct
% RHP, we need to first understand the analytic (in $\lambda$) properties of the
% scattering data, which means also understanding the analytic (in $\lambda$)
% properties of the previously mentioned Jost solutions and Green's functions.
% While this dissertation focuses primarily on the Direct Map, a discussion
% of the analytic properties of the scattering data is provided at the end of
% this disseration in {\color{red}?Chapter/Section? ??.??} as a staging point for
% future research into the ILW's Inverse Scattering Transform.}


    \filetitle{'...!!...'}{Beginning of aliasing inside descriptions and labels}{modellang/alias}

	\paragraph{Syntax in descriptions of variables, shocks, and
parameters}\label{syntax-in-descriptions-of-variables-shocks-and-parameters}

\begin{verbatim}
'Description !! Alias' Name
\end{verbatim}

\paragraph{Syntax in equations labels}\label{syntax-in-equations-labels}

\begin{verbatim}
'Label !! Alias' Equation;
\end{verbatim}

\paragraph{Description}\label{description}

When used in descriptions of variables, shocks, and parameters, or in
equation labels, the double exclamation mark starts an alias (but the
exlamation marks are not included in it). The alias can be used to
specify, for example, a LaTeX code associated with the variable, shock,
parameter, or equation. The aliases can be retrieved from the model code
by using the appropriate query in the function
\href{model/get}{\texttt{model/get}}.

\paragraph{Example}\label{example}

\begin{verbatim}
!transition_variables
    'Output gap !! $\hat y_t$` Y_GAP
\end{verbatim}

In the resulting model object, the description of the variables
\texttt{Y\_GAP} will be

\begin{verbatim}
Output gap
\end{verbatim}

while its alias will be

\begin{verbatim}
$\hat y_t$.
\end{verbatim}


\begin{document}
	\chapter{Network Representation}

	\section{Complex Networks}

	Communication networks, transportation systems, social studies, biology and neuroscience all presents some characteristic elements that can be related to a social network. Social networks are structures were social actors interact with each other and through the \textit{social network analysis} it is possible to undergo deeply on the characterization of the network itself. While social network emerged as a prominent sociology topic in 1908 by Georg Simmel, it was later re-discovered by physicists and mathematically formalized in late 1950s. Through the years many researchers developed newer and more fine-grained models for these networks (Barabási–Albert\cite{Barabasi1999}, Erdős–Rényi\cite{Erdos1959}, Watts–Strogatz\cite{Watts1998}) producing a vast literature on the topic and several unique models that would capture essential properties of different scenarios.

	Social networks are usually complex networks, networks whose structure features non-trivial topology. Scale-free networks are complex networks of particular interest as they are random graphs (graphs whose degree distribution is regulated by a probability function) governed by a power-law probability function. The first scale-free network to be ever observed and formalized was the WWW network by Barabàsi, A.L. and Albert, R. \cite{Barabasi1999} which were the first to notice the lack of expressive power of the existing models that were failing to capture essential properties such as the growth factor or the preferential attachment which are characteristic features of real networks.

	The fundamental results of their work in WWW and, later, in many other real networks, is that some nodes manifest a degree that can be order of magnitudes with respect to other nodes of the network. The terms scale-free was coined by Barabàsi and collaborators to indicate network whose probability distribution looks the same regardless the size of the network.

	Such probability density function brings some characteristic features unique to scale-free networks that can't be observed on random graph models such as Erdős–Rényi, where degree distribution responds to a Poisson distribution, or Watts-Strogatz model that produces a degree distribution that follows a Dirac delta function. Typical features of a scale-free network are:
	\begin{itemize}
		\item Preferential attachment: it's the likelihood expressed by a node of receiving new edges proportional to its degree k and it's often referred as $\Pi(k)$. In their work, the BA-model assumes the preferential attachment to be a linear probability defined as $$\Pi(k_i) = \frac{k_i}{\sum_{j}k_j}$$ and express the probability that a new node will be connected to node $i$, based on its degree $k$ and it has been demonstrated that with this preferential attachment, the degree probability distribution converges to $$P(k) \sim k^{\gamma}, \gamma = 3$$
		As proved in \cite{Krapivsky2000} by Krapivsky, Redner and Leyvraz, for every $\Pi(k)$ that is asymptotically linear, the graph is scale free with degree distribution $$P(k) \sim k^{\gamma}, 2 < \gamma < \infty $$

		\item  Small World Property: expresses the fact that the length of any shortest path between a pair of nodes is always small compared to the size of the network. Cohen and Havlin proved that a power-law graph with $2 < \gamma < 3$ will have diameter $d \sim \ln\ln N$ where $N$ is the order of the network \cite{Cohen2002}.

		\item Clustering: depending on the network topology, scale-free networks may presents many clusters or cliques inside of which the density of links is very high with respect to the number of links that connect the various clusters.

		\item Network resilience: disconnecting random nodes from the network, even a large portion, does not impact the connectedness of the graph. Usually high-degree vertices take place in the middle of the network, while lower-degree nodes occupy the peripheral regions.
	\end{itemize}

	\section{Dynamic Networks}

	The nature of the Lightning Network implicitly suggests that the network itself should not be considered as a static entity. Instead, due to time constraints over the contracts that compose the links of the network, the system is subject to the phenomenon of \textit{churn}, which refers to the behavior of some nodes to arrive or depart from the network at time intervals and that has been addressed recently in many systems whose composition evolves through time \cite{Baldoni2010} \cite{Ko2008} with a particular attention to P2P systems \cite{Liben-Nowell2002} \cite{Mostefaoui}.

	The strict relationship between time and money in the Lightning Network puts it in an incredibly interesting spot regarding the dynamic properties of it: there must exist channels connecting pair of nodes along a payment path and every node involved in the payment path must be online in the moment a transaction is performed. A third constraint on a payment procedure is that every node involved in the payment process must have enough funds to transfer.

	The three constraints play a key role in the modeling of the network and put in evidence the dynamic feature of the network itself. Therefore the network is not a static entity but a dynamic one, and a lots of its feature will be shown have similarities with social networks. The dynamicity of a network has lately seen intensive research efforts and a vast literature is available, yet it often results to be very problem-specific and lacks of a universal formal description. An important contribution to this research area was the formalization of \textit{Time-Varying Graphs}\cite{Casteigts2012} that is a unified framework that address the main characteristics of a dynamic network, and whose goal is to put in evidence and formally define important concepts that were present in other research areas (delay-tolerant networks, opportunistic networks, real-world complex networks) but not related each other.

	\section{Time Varying Graph}

	A Time-Varying Graph (TVG) is described by a quintuple \(\text{TVG} = (V, E, T, \rho, \zeta) \):
	\begin{itemize}
		\item a set of nodes \(V\).

		\item a set of relations between nodes \(E\) (edges).

		\item an alphabet \(L\) (optional).

		\item a relation \(E \subseteq V \times V \times L\) where \(L\) are domain-specific labels and can be used (or omitted) to enrich the description of the network.

		\item a time span \( \mathcal{T} \subseteq \mathbb{T}\) called \textit{lifespan} with \(\mathbb{T}\) representing the domain of time which usually coincides with the \(\mathbb{N}\) if the system is discrete, \(\mathbb{R}^+\) otherwise.

		\item a function \(\rho : E \times \mathcal{T} \to \{0, 1\} \) called \textit{presence} to track the availability of an edge at a given time.

		\item a function \(\zeta : E \times \mathcal{T} \to \mathbb{T}\) called latency function that indicates the time needed to cross an edge at a given date.
	\end{itemize}

	The model can also be enriched with two more functions that can capture the nodes dynamic behavior in a way similar to \(\rho\) and \(\zeta\).
	\begin{itemize}
		\item a function \(\psi : V \times \mathcal{T} \to \{0, 1\}\) called \textit{node presence function}, expressing the availability of a node at a given time T.

		\item a function \(\phi : V \times \mathcal{T} \to \mathbb{T}\) called
	\end{itemize}

	Inside the definition of a time varying graph we have the notion of \textit{underlying graph}, that is the graph \(G = (V, E)\) which is the backbone of the dynamic network. One important thing to notice is that a connected component G doesn't imply the connectivity of the TVG, in fact the TVG could be disconnected for every time instant of its lifespan.

	\begin{figure}
		\begin{subfigure}{0.5\textwidth}
			\centering
			\includegraphics[scale=0.65]{example_tvg}
			\caption{The visual representation of a TVG.}
		\end{subfigure}
		\begin{subfigure}{0.5\textwidth}
			\centering
			\includegraphics[scale=0.65]{example_underlying}
			\caption{The underlying graph.}
		\end{subfigure}
		\begin{subfigure}{0.5\textwidth}
			\centering
			\includegraphics[scale=0.65]{example_tvg_0_1}
			\caption{Edges availability according to presence function at time interval [0, 1].}
		\end{subfigure}
		\begin{subfigure}{0.5\textwidth}
			\centering
			\includegraphics[scale=0.65]{example_tvg_2_3}
			\caption{Edges availability according to presence function at time interval [2, 3].}
		\end{subfigure}
		\caption{Layers of a Time Varying Graph. Image courtesy of \cite{Casteigts2012}.}
	\end{figure}

	\begin{itemize}
		\item \textit{Edge-centric evolution}: every edges has associated a set of the union of all the dates in which an edge \(e\) is available. Such set is denoted by \(I(e)\) and every element of \(I\) is such that \(I(e) = \{t \in \mathcal{T} | \rho(e,t) = 1\}\), that is, the elements of the set are of the kind \(I(e) = \{t_1, t_2, t_3 ... \}\) and since they express time intervals it is possible to subdivide them in two sets \(App(e)\) and \(Dis(e)\), \textit{appearance and disappearance}, where \(App(e) = \{t_n | t_n \in I(e), n = 2N + 1\}\) and \(Dis(e) = \{t_n | t_n \in I(e), n = 2N\}\), i.e. every \(t\) in an even position is a date of appearance while \(t\) in odd position are dates of disappearance.

		\item \textit{Vertex-centric evolution}: nodes spawn and de-spawn causes changes in the configuration of the neighbor for some nodes, and it is expressed through \textit{sequences of neighborhoods} \(N_{t_1}(v), N_{t_2}(v), N_{t_3}(v) ...\). This kind of configuration is not very common in the literature.

		\item \textit{Graph-centric evolution}: it is the situation in which graphs are subject to events (edges appearance and disappearance) that make their topology change. Each of this events happens in a \textit{characteristic date}, and these dates are sorted in chronological such that \(S_{\mathcal{T}}(\mathcal{G}) = sort(\cup\{S_{\mathcal{T}}(e) : e \in E\})\) with \( \mathcal{G}\) being a TVG. From a global point of view, it is possible to have a look at the evolution of the graph as a sequence of graph snapshot \(S_{\mathcal{G}} = G_1, G_2, G_3\) where \(G_i\) is a static snapshot of \(\mathcal{G}\) at time \(i\).
	\end{itemize}

	The time-varying graph framework also put emphasis on the different peculiarity of the different networks that can be modeled as a TVG by providing a classification method based on some particular properties. They are thirteen in total and are \textit{minimum reachability, all-nodes-reachability, connectivity over time, round connectivity, recurrent connectivity, recurrence of edges, time bounded recurrence of edges, periodicity of edges, constant connectivity, T-interval connectivity, eventual instant-connectivity, eventual instant-routability, complete graph of interaction}.
	Such a classification should help to categorize better a problem, and to help navigating the literature with well defined properties that have been investigated in other works.
\end{document}


\documentclass{article}

\setlength{\evensidemargin}{.25in}
\setlength{\oddsidemargin}{.25in}
\setlength{\textwidth}{6.0in}
%\setlength{\parindent}{0.25in}
\setlength{\topmargin}{-0in}
\setlength{\textheight}{8.0in}
%\newtheorem{theorem}{Theorem}
%\newtheorem{proposition}{Proposition}
%\newtheorem{lemma}{Lemma}
%\newtheorem{corollary}{Corollary}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
\bibliographystyle{siam}

\title{\bf An implementation of the Volume Algorithm}
\author{Francisco Barahona}
\date{June 22, 2000}
\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Here we describe an implementation of the Volume algorithm (VA) originally
presented in \cite{BA}. We focus on the {\it uncapacitated facility location
problem} (UFLP) as an example, see \cite{BC} for some of the theoretical
issues. As a first step, a new user should be able to run our code ``as is''.
This can also be used as a framework for Lagrangian relaxation. The user would
have to modify the files {\tt ufl.hpp}, {\tt ufl.cpp}, {\tt ufl.par}, and {\tt
data}, to produce an implementation for a different problem. The files {\tt
volume.hpp} and {\tt volume.cpp} are specific to the VA, so the user should
not have to change them. We hope to receive reports about bugs and/or
successful experiences.

\smallskip

Initially this directory contains the files: {\tt INSTALL, Makefile,
  volume.hpp, volume.cpp, ufl.hpp, ufl.cpp, ufl.par, data, doc.ps}. See the
  {\tt INSTALL} file for instructions on installation.

Now we present the linear program used in \cite{BC}. This is
\begin{eqnarray}
\min \sum c_{ij} x_{ij} & + & \sum f_i y_i  \label{fp1} \\
\sum_i x_{ij} & = & 1, \hbox{ for all } j,  \label{fp2} \\
x_{ij} & \leq & y_i, \hbox{ for all } i, j, \label{fp3} \\
x_{ij} & \ge  & 0, \hbox{ for all } i, j,   \label{fp4} \\
y_i    & \le  & 1, \hbox{ for all } i.      \label{fp5}
\end{eqnarray}

Here the variables $y$ correspond to the locations, and the variables $x$
represent connections between customers and locations. Let $u_j$ be a set of
Lagrange multipliers for equations (\ref{fp2}). When we dualize equations
(\ref{fp2}), we obtain the {\it lagrangian problem}
\begin{eqnarray*}
L(u) & = & \min \sum \bar c_{ij} x_{ij} + \sum \bar f_i y_i + \sum u_j, \\
x_{ij} & \le & y_i, \hbox{ for all } i, j, \\
x_{ij} & \ge & 0, \hbox{ for all } i, j, \\
y_i    & \le & 1, \hbox{ for all } i.
\end{eqnarray*}

\noindent Where the {\it reduced costs} $\bar c_{ij}=c_{ij}-u_j$, and
$\bar f_i = f_i$. We apply the VA to maximize $L(\cdot)$ and to produce a
primal vector $(\bar x, \bar y)$ that is an approximate solution of
(\ref{fp1})-(\ref{fp5}). Using this primal information we run a heuristic
that gives an integer solution.

In what follows we describe the different files in this directory.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{ufl.par}

This file contains a set of parameters that control the algorithm and contain
some information about the data. Each line has the format

{\tt keyword=value}

\noindent where {\tt keyword} should start in the first column. If we add any
other character in the first column, the line is ignored or considered as a
comment. The file looks as below

\bigskip
\begin{verbatim}
fdata=data
*dualfile=dual.txt
dual_savefile=dual.txt
int_savefile=int_sol.txt
h_iter=100

printflag=3
printinvl=5
heurinvl=10

greentestinvl=1
yellowtestinvl=4
redtestinvl=10

lambdainit=0.1
alphainit=0.1
alphamin=0.0001
alphafactor=0.5
alphaint=50

maxsgriters=2000
primal_abs_precision=0.02
gap_abs_precision=0.
gap_rel_precision=0.01
granularity=0.
\end{verbatim}

The first group of parameters are specific to the UFLP and the user should
define them. {\tt fdata} is the name of the file containing the data. {\tt
dualfile} is the name of a file containing an initial dual vector. If we add
an extra character at the beginning ({\tt *dualfile}) this line is ignored,
this means that no initial dual vector is given. {\tt dual\_savefile} is the
name of a file where we save the final dual vector. If this line is missing,
then the dual vector is not saved. {\tt int\_savefile} is the name of a file
to save the best integer solution found by the heuristic procedure, if this
line is missing, then this vector is not saved. {\tt h\_iter} is the number of
times that the heuristic is run after the VA has finished.

The remaining parameters are specific to the VA. See the {\tt VOL\_parms.html}
file for their documentation (this file will be created after doing a `make
doc` or it's available on the Internet).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{data}

The file {\tt data} has the following format. On the first line we have the
number of possible locations and the number of customers. On the next lines,
the cost of opening each location appears, one cost per line. Then each of the
remaining lines is like

$
i \quad j \quad d_{ij},
$

\noindent where $i$ refers to a location, $j$ refers to a customer, and
$d_{ij}$ is the cost of serving customer $j$ from location $i$. The indices
$i$ and $j$ start from 1. If a pair $i,j$ is missing then the cost $d_{ij}$ is
set to $10^7$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{ufl.hpp}

This file contains C++ classes specific to the UFLP.

First we have a class of parameters specific to the UFLP. The description
of these parameters appears in the preceding section.

\begin{verbatim}
class UFL_parms {
public:
   string fdata;         // file with the data
   string dualfile;      // file with an initial dual solution
   string dual_savefile; // file to save final dual solution
   string int_savefile;  // file to save primal integer solution
   int h_iter;           // number of times that the primal heuristic will be
                         // run after termination of the volume algorithm

   UFL_parms(const char* filename);
   ~UFL_parms() {}
};
\end{verbatim}

Before the next class we should mention the classes {\tt VOL\_dvector} and
{\tt VOL\_ivector} defined in {\tt Volume.hpp}. The pseudo-code below
illustrates their use.

\begin{verbatim}
int n=100;
VOL_dvector x(n);  // a double vector with n entries
x=0.;              // sets to 0. all entries of x
VOL_dvector y;     // a double vector, it size remains to be set
y.allocate(n);     // size is set
y=x;               // copy each entry of x into y
VOL_dvector z(y);  // a double vector of the same size as y,
                   // all entries of y are copied into z
x[0]=-1;           // first entry of x is set to -1
y[0]=x[0];         // copy first entry of x into first entry of y
\end{verbatim}

The class {\tt VOL\_ivector} is used for vectors of {\tt int}. One can do the
same operations as for {\tt VOL\_dvector}.

Then we have a class containing the data for the UFLP.

\begin{verbatim}
class UFL\_data { // original data for uncapacitated facility location
public:
   VOL_dvector fcost; // cost for opening facilities
   VOL_dvector dist;  // cost for connecting a customer to a facility
   VOL_dvector fix;   // vector saying if some variables should be fixed
                      // if fix=-1 nothing is fixed
   int ncust, nloc;   // number of customers, number of locations
   VOL_ivector ix;    // best integer feasible solution so far
   double      icost; // value of best integer feasible solution
public:
   UFL_data() : icost(DBL_MAX) {}
   ~UFL\_data() {}
};
\end{verbatim}

Then we have

\begin{verbatim}
class UFL_hook : public VOL_user_hooks {
public:
   // for all hooks: return value of -1 means that volume should quit
   // compute reduced costs
   int compute_rc(void * user_data,
                  const VOL_dvector& u, VOL_dvector& rc);
   // solve lagrangian problem
   int solve_subproblem(void * user_data,
                        const VOL_dvector& u, const VOL_dvector& rc,
                        double& lcost, VOL_dvector& x, VOL_dvector&v,
                        double& pcost);
   // primal heuristic
   // return DBL_MAX in heur_val if feas sol wasn't/was found
   int heuristics(void * user_data, const VOL_problem& p,
                  const VOL_dvector& x, double& heur_val);
};
\end{verbatim}

Here the function {\tt compute\_rc} is used to compute reduced costs. In the
function {\tt solve\_subproblem} we solve the lagrangian problem. In {\tt
heuristics} we run a heuristic to produce a primal integer solution.

Finally in this file we have {\tt UFL\_parms::UFL\_parms(const char
*filename)}, where we read the values for the members of {\tt UFL\_parms}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{ufl.cpp}

This file contains several functions that we describe below.

First we have {\tt int main(int argc, char* argv[])}. In here we initialize
the classes described in {\tt ufl.hpp}, and read the data. Then {\tt
volp.psize()} is set to the number of primal variables, and {\tt volp.dsize()}
is set to the number of dual variables. Then we check if a dual solution is
provided and if so we read it.

For the UFLP all relaxed constraints are equations, so the dual variables are
unrestricted. In this case we do not have to set bounds for the dual
variables. If we have inequalities of the type $ax \ge b$, then we have to set
the lower bounds of their dual variables equal to 0. If we had constraints of
the type type $ax \le b$, then we have to set the upper bounds of their
variables equal to 0. This would be done as in the pseudo-code below.

\begin{verbatim}
// first the lower bounds to -inf, upper bounds to inf
volp.dual_lb.allocate(volp.dsize);
volp.dual_lb = -1e31;
volp.dual_ub.allocate(volp.dsize);
volp.dual_ub = 1e31;
// now go through the relaxed constraints and change the lb of the ax >= b
// constrains to 0, and change the ub of the ax <= b constrains to 0.
for (i = 0; i < volp.dsize; ++i) {
   if ("constraint i is '<=' ") {
      volp.dual_ub[i] = 0;
   }
   if ("constraint i is '>=' ") {
      volp.dual_lb[i] = 0;
   }
}
\end{verbatim}

The function {\tt volp.solve} invokes the VA. After completion we compute the
violation of the fractional primal solution obtained. This vector is {\tt
psol}. Then we check if the user provided the name of a file to save the dual
solution. If so, we save it. Then we run the primal heuristic using {\tt psol}
as an input. Notice that this heuristic has also been run periodically during
the execution of the VA. Then if the user has provided the name of a file to
save the integer heuristic solution, we do it. Finally the values of the
solutions and some statistics are printed.

The next function is {\tt void UFL\_read\_data(const char* fname, UFL\_data\&
data)}, where we read the data. {\tt data.nloc} is the number of locations,
{\tt data.ncust} is the number of customers. {\tt data.fcost} is a vector
containing the cost of opening each location. {\tt data.dist} is a vector
containing the cost of serving customers from facilities. All entries are
initialized to $10^7$ and then particular entries are being set with the
statement

{\tt dist[(i-1)*ncust + j-1]=cost;}

\noindent where {\tt i} is the index of a location and {\tt j} is the index
of a customer. Here the indices start from 1. Finally we have a vector {\tt
data.fix} associated with the locations. A particular entry is set to 0 if the
location should be closed, it is set to 1 if it should be open, and it is set
to -1 if this variable is free. Initially all entries are set to -1.

In the function

{\tt double solve\_it(void * user\_data, const double* rdist,
VOL\_ivector\& sol)}

\noindent we solve the lagrangian problem.
We receive the data and reduced costs as input and return a primal vector. The
solution is in the vector {\tt sol}. Its first {\tt n} entries correspond to
the locations, then all remaining entries correspond to connections between
locations and customers.

In the function

{\tt int UFL\_hook::compute\_rc(void * user\_data, const VOL\_dvector\& u,
VOL\_dvector\& rc)}

\noindent we compute the reduced costs. They will be used to solve the
lagrangian problem.

In the function

\begin{verbatim}
   int
   UFL_hook::solve_subproblem(void *user_data,
                              const VOL_dvector& u, const VOL_dvector& rc,
                              double& lcost, VOL_dvector& x,
                              VOL_dvector& v, double& pcost)
\end{verbatim}

\noindent we compute the lagrangian value, we call {\tt solve\_it}, we compute
the objective value and the vector $v$ defined as follows. If $\hat x$ is the
primal solution given by {\tt solve\_it}, and $Ax \sim b$ is the set of
relaxed constraints, then the difference $v$ is $$v = b - A \hat x.$$

The last function in this file is
\begin{verbatim}
   int
   UFL_hook::heuristics(void * user_data, const VOL_problem& p,
                        const VOL_dvector& x, double& icost)
\end{verbatim}

\noindent where we run the following simple heuristic.
Given a fractional solution $(\bar x, \bar y)$, let $\bar y_i$ be the variable
associated with location $i$. We pick a random number $r \in [ 0, 1 ]$ and if
$r < \bar y_i$ facility $i$ is open, and closed otherwise. We repeat this for
every facility, then given the set of open facilities we find a minimum cost
assignment of customers. This function is invoked periodically in the VA and
by the main program after the VA has finished.

\bibliography{ufldoc}

\end{document}


\documentclass[aspectratio=169]{beamer}
\usetheme[]{msubeamer}
\usepackage[T1]{fontenc}
\usepackage[backend=bibtex,sorting=none]{biblatex}
\addbibresource{mybeamer.bib}
\usepackage{multirow}
\usepackage[ruled,linesnumbered]{algorithm2e}
\usepackage{algorithmic}
\usepackage{subfigure}
\usepackage{float}
\usepackage{epstopdf}
%\setbeamerfont{footnote}{size=\small}
\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\xdefinecolor{dred}{rgb}{0.6, 0.0, 0.0}
\setbeamercolor{math text}{fg=dred}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\title[Adam-CBO]{A consensus-based global optimization method with adaptive momentum estimation}
\maketitleframe
\begin{frame}
\frametitle{Machine learning tasks}

Highly nonconvex unconstrained optimization problem
\begin{align*}
x^* = \arg\min_{x\in \mathbb{R}^d} f(x)
\end{align*}
with the loss function
\begin{equation*}
f(x) = \frac{1}{n}\sum_{i=1}^n f_i(x) = \frac{1}{n}\sum_{i=1}^n\|\mathcal{N}_x(\hat{x})-\hat{y}\|
\end{equation*}
\begin{itemize}
	\item[] $x$ is the parameter vector
	\item[] $\mathcal{N}_x$ represents a neural network representation
	\item[] $(\hat{x}_i,\hat{y}_i)_{i=1}^n$ is a set of labeled data
	\item[] $\|\cdot\|$ is the $L^2$ distance
	\item[] \textcolor{red}{$d\gg 1$}
\end{itemize}

\end{frame}

\begin{frame}<beamer>
\frametitle{\textbf{Outline}}
\tableofcontents[]
\end{frame}

\section{Optimization methods: Zero-order or first-order?}

\begin{frame}
\frametitle{First-order methods}

\begin{itemize}
	\item gradient descent method
	\begin{equation*}
	x^{t+1} = x^t - \alpha \nabla f(x^t)
	\end{equation*}
	with $\alpha$ being the learning rate
	\item stochastic gradient descent (SGD) method
	\begin{equation*}
		x^{t+1} = x^t - \alpha \nabla f_i(x^t)
	\end{equation*}
	\item SGD method with momentum term \footfullcite{Qian1999Jan}
	\begin{align*}
		&x^{t+1} = x^t - m^t\\
		&m^t = -\gamma m^{t-1} + \alpha \nabla f_i(x^t)
	\end{align*}
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Cont'd}

\begin{itemize}
	\item Adaptive momentum method (Adam) \footfullcite{kingma2014adam}
	\begin{align*}
	&x^{t+1} = x^t - \gamma \frac{\hat{m}^t}{\sqrt{\hat{v}^t}+\epsilon}\\
	&m^t = \beta_1 m^{t-1} + (1-\beta_1) \nabla f(x^t), \quad \hat{m}_t = \frac{m_t}{1-\beta_1^t}\\
	&v^t = \beta_2 v^{t-1} + (1-\beta_2) \nabla^2 f(x^t), \quad \hat{v}_t = \frac{v_t}{1-\beta_1^t}
	\end{align*}
	where $0<\beta_1,\beta_2<1$
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{First-order methods}

\begin{itemize}
	\item mostly have problems with loss functions containing large noise or non-differentiable units
	\item gradient tends to explode or vanish as the neural network gets deeper\footfullcite{hanin2018neural}
	\item are easily influenced by the loss landscape\footfullcite{liu2020bad}
\end{itemize}

\end{frame}

\begin{frame}
	\frametitle{Zero-order methods: Gradient-free}

	\begin{itemize}
		\item Nelder-Mead method
		\item genetic algorithm
		\item simulated annealing method
		\item particle swarm optimization
		\item \textcolor{red}{consensus based optimization (CBO) method} \footfullcite{carrillo2018analytical}\footfullcite{totzeck2018numerical}\footfullcite{pinnau2017consensus}
	\end{itemize}
\end{frame}

\section{CBO method}
\begin{frame}
	\frametitle{Original CBO method}

   Interacting particles during the dynamic evolution
   \begin{itemize}
   	\item tend to their weighted average
   	\item undergo fluctuation due to the random noise
   \end{itemize}

	N particles $X^i$, $i =1, \cdots N$
   \begin{equation*}
   		\dot X^i = -\lambda (X^i - \bar{x}^*) +\sigma \textcolor{red}{|X^i-\bar{x}^*|} \dot W^i_t
   \end{equation*}
   \begin{itemize}
   	\item[] weighted average $\bar{x}^* = \frac{1}{\sum_{i=1}^N e^{-\beta L(X^i)}}\sum_{i=1}^N X^i e^{-\beta L(X^i)}$
   	\item[] cost (loss) function $L(x)$ to be optimized
   	\item[] white noise $\dot W_t$
   \end{itemize}
   Discretization of the above system with unit stepsize
   \begin{equation*}
   		X^i_{t+1} =  X^i_{t} -\lambda (X^i - \bar{x}^*) +\sigma \textcolor{red}{|X^i-\bar{x}^*|}  dW^i_t
   \end{equation*}
\end{frame}

\begin{frame}
\frametitle{Curse of dimensionality (CoD)}

\begin{itemize}
	\item Exponential convergence rate under dimension-dependent conditions \footfullcite{carrillo2018analytical}
	\item The larger the dimension, the smaller the learning rate (\textcolor{red}{CoD})
	\item Replacement of the isotropic geometric Brownian motion with the component-wise one \footfullcite{carrillo2019consensus}
	\begin{equation*}
	X^i_{t+1} =  X^i_{t} -\lambda (X^i - \bar{x}^*) +\sigma \textcolor{red}{(X^i-\bar{x}^*)}  dW^i_t
	\end{equation*}
	\item[] Random mini-batch: $\mathcal{O}(N) \rightarrow \mathcal{O}(\frac{N}{M})$
	\item Convergence to the global minimizer with dimension-independent parameters \footfullcite{ha2019convergence}
\end{itemize}
\end{frame}

\begin{frame}
	\frametitle{Some practical issues in CBO}
	\begin{itemize}
		\item \textcolor{red}{the initial data need to be well-chosen}
		\item difficult to optimize high dimensional no-convex function (Rastrigin Function over $20$ dimension)
		\item difficult to optimize deep neural networks with many parameters
	\end{itemize}
\end{frame}
\section{Adam-CBO Method}
\begin{frame}
	\frametitle{First-order momentum}

	\begin{itemize}
		\item[] The same system without random term but with inertial effect
		\begin{equation*}
		\sigma \ddot X^i_t + \dot X^i_t = -(X_t- x^*) , \quad i=1,\cdots,N
		\end{equation*}
		\item[] An equivalent first-order system
		\begin{align*}
		&\dot X_t^i = -M_t^i \\
		&\sigma \dot M_t^i + M_t^i = X_t^i - x^*
		\end{align*}
		\item[] Discretization
		\begin{align*}
		&X^i_{t+1} = X_t^i -\delta t M^i_{t+\frac{1}{2}}\\
		&M^i_{t+\frac{1}{2}} = \frac{\sigma - \delta t}{\sigma+ \delta t}M_{t-\frac{1}{2}} + \frac{2\delta t}{\sigma  + \delta t } (X_t^i - x^*)
		\end{align*}
	\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Cont'd}

\begin{itemize}
	\item[] Relabel $M^i_{t+\frac{1}{2}}$ by $M^i_{t+1}$
	\begin{equation*}
	\begin{aligned}
	&X^i_{t+1} = X_t^i - \lambda M^i_{t+1}  \\
	\ &M^i_{t+1} = \beta_1 M^i_{t} + (1-\beta_1) (X_t^i - \bar{x}^*)
	\end{aligned}
	\end{equation*}
	with $\lambda = \delta t $ and $\beta_1 =\frac{\sigma - \delta t}{\sigma+ \delta t} =1 - \frac{2\delta t }{\sigma  + \delta t }  $
	\item[] $\beta_1$ is near 1 ($=0.9$ in practice) since $\delta t$ is small
	\item[] Add the stochastic term
		\begin{equation*}
	\begin{aligned}
	&X^i_{t+1} = X_t^i - \lambda M^i_{t+1} + \sigma_t  W^i_t \\
	\ &M^i_{t+1} = \beta_1 M^i_{t} + (1-\beta_1) (X_t^i - \bar{x}^*)
	\end{aligned}
	\end{equation*}
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Expectation}

\begin{itemize}
	\item[] A recursive argument of $M_t^i$ yields
	\begin{equation*}
	\begin{aligned}
	M^i_t & = \beta_1 M^i_{t-1} + (1-\beta_1)(X^i_{t-1} - x^*)\\
	& = \beta_1 (\beta_1 M^i_{t-2} + (1-\beta_1) (X_{t-1}^i - x^*)) + (1-\beta_1) (X^i_{t-2} - x^*)\\
	& = (1-\beta_1) \sum_{k=0}^{t-1} \beta_1^{t-k} (X_{k}^i - x^*).
	\end{aligned}
	\end{equation*}
	\item[] Stationary assumption of $X_{k}^i - x^*$ w.r.t. $k$ leads to
	\begin{equation*}
	\begin{aligned}
	\mathbb{E} [M^i_t] & =(1-\beta_1)   \mathbb{E} [\sum_{k=0}^t \beta_1^{t-k} (X_{k}^i - x^*)] \\
	&= (1-\beta_1^t)  \mathbb{E}[ X_{t}^i - x^* ]
	\end{aligned}
	\end{equation*}
	\item[] Unbiased estimation of first-order moment
	\[
	\hat{M}^i_{t+1} = \frac{M^i_{t+1}}{(1-\beta_1^t)}
	\]
	\end{itemize}

\end{frame}

\begin{frame}
	\frametitle{Second-order momentum $\mathbb{E}(|X^i_t-x^*|^2)$}

	\begin{itemize}
		\item Define $V^i_t  = \beta_2 V^i_{t-1} + (1-\beta_2) |X^i_t - x^*|^2$
		\item[] Application of the same argument for $\mathbb{E}[X^i_t]$ yields
		\begin{equation*}
		\mathbb{E}[V^i_t] = (1-\beta_2^t) \mathbb{E}[|X^i_t-x^*|^2]
		\end{equation*}
		\item[] Unbiased estimation of $\mathbb{E}(|X^i_t-x^*|^2)$
		\[\hat{V}^i_t = \frac{V^i_t}{1-\beta_2^t}\]
		\item Modify the model
		\begin{equation*}
			X^i_{t+1} = X_t^i - \frac{\lambda \hat{M}^i_{t+1}}{\sqrt{\hat{V}^i_{t+1}} + \epsilon} + \sigma^ t  W_t^i
		\end{equation*}
		with a small $\epsilon$ ($1e-8$) to avoid the vanishing of denominator
	\end{itemize}

\end{frame}

\section{Linear stability analysis of Adam-CBO}
\begin{frame}
\small{
\begin{algorithm}[H]
\KwIn{$\lambda$, $N$, $M$, $t_N$, $\beta_1$, $\beta_2$}

  Initialize $X^i_0$, $i = 1,\cdots N$ by the uniform distribution;

  Initial $M^i_0,V^i_0 =0$;
  \tcc{Initialize first order and second order moments.}

  \For{$t = 0$ \textbf{to} $t_N$ }{
    Generate a random permutation of index $\{1,2,\cdots,N\}$ to form set  $P_k$;

    Generate batch set of particles in order of $P_k$ as $B^1,\cdots B^{\frac{N}{M}}$ with each batch having $M$ particles;

    \For{$j = 0$  \textbf{to} $\frac{N}{M}$}{
        Update $x^* =\sum\limits_{k\in B^j} \frac{X_t^k\mu^k_t}{\sum\limits_{i\in B^j} \mu_t^i }$, where $\mu_t^i = \omega_f^\alpha (X_t^i)$;

        Update $X^i_t$ for $j \in B^j$ as follows

        $
        M^i_{t+1} = \beta_1 M_{t}^i +(1-\beta_1) (X^i_t-x^*) \quad \quad \hat{M}^i_{t+1} = M^i_{t+1}/(1-\beta_1^t)
        $;

        $
        V^i_{t+1} = \beta_2 V_{t}^i +(1-\beta_2) (X^i_t-x^*)^2 \quad \quad  \hat{V}^i_{t+1} = V^i_{t+1}/(1-\beta_2^t)
        $;

        $
        X^i_{t+1} = X^i_t - \lambda \hat{M^i_t}/(\sqrt{\hat{V^i_t}}+\epsilon) + \sigma^t \sum_{k = 1}^d \vec{e}_k z_i  %\quad z_i\; \text{ is a random variable}
        $.
    }
    }
    \KwOut{$X_{t_N}^i, \quad i = 1\cdots N$}
\end{algorithm}
}
\end{frame}

\begin{frame}
	\frametitle{Linear stability analysis of Adam-CBO}

	 Continuous formulation without the stochastic term
	\begin{align*}
&\dot m = (\beta_1 -1) m + (1-\beta_1) (x-\bar{x})\\
&\dot v = (\beta_2 -1) v + (1-\beta_2) (x-\bar{x})^2\\
&\hat{m} = \frac{m}{1-\beta_1^t} \quad  \hat{v} = \frac{v}{1-\beta_2^t} \\
&\dot x =  - \lambda \frac{\hat{m}}{\sqrt{\hat{v}}+\epsilon}
	\end{align*}

\end{frame}

\begin{frame}
\frametitle{Linearization around $m = 0, x = \bar{x}, v = 0 $}
	\begin{align*}
	&\dot m = -(1-\beta_1) m + (1-\beta_1) \tilde{x}\\
	&\dot v = -(1-\beta_2)v \\
	&\dot {\tilde{x}} = -\frac{\lambda }{(1-\beta_1^t)\epsilon}m \rightarrow -\frac{\lambda}{\epsilon} m = -\mu m  \quad (t \rightarrow \infty)
	\end{align*}
	with $\tilde{x} = x - \bar{x}$ and $\mu=\lambda/\epsilon$, and in a vector form
	\begin{equation*}
	\begin{aligned}
	\mathrm{d}_t	\left(\begin{matrix}
			m \\ v \\ \tilde x
		\end{matrix}\right) =
	\left(\begin{matrix}
		-(1-\beta_1) & 0 & 1-\beta_1\\
		0 &  -(1-\beta_2) & 0  \\
		 -\mu & 0 & 0
	\end{matrix}\right)
	\left(\begin{matrix}
	m \\ v \\ \tilde x
	\end{matrix}\right)
	\end{aligned}
	\end{equation*}
\end{frame}

\begin{frame}
\begin{theorem}
	The Adam-CBO method generates a sequence that converges to the optimal solution with rates independent of the learning rate $\lambda$.
	\begin{proof}
	Eigenvalues of the matrix on the right-hand side are $\beta_2-1$ and $\frac{1}{2}( \beta_1 - 1 \pm i  \sqrt{1-\beta_1}\sqrt{\beta_1-1+4\mu})$ (typically $1-\beta_1\ll 4\mu$), respectively. Thus, $m,v,\tilde{x}$ decay to $0$  exponentially with rate $\beta_2 -1 $ when $\beta_1>2\beta_2+1$ and with rate $\frac{1}{2}(\beta_1-1)$ when $\beta_1 < 2 \beta_2 + 1$ in an oscillatory way.
	\end{proof}
\end{theorem}

\begin{itemize}
	\item $\beta_1 = 0.9$ and $\beta_2 = 0.99$
	\item Continuous formuation of CBO without random noise
	\begin{equation*}
	\dot x = - \lambda (x- \bar{x})
	\end{equation*}
	\item The decay rate of the CBO method depends exponentially on the learning rate $\lambda$
\end{itemize}

\end{frame}

\section{Numerical results}
\subsection{Rastirgin function}
\begin{frame}
	\frametitle{Rastrigin function}
\begin{columns}
\column{0.5\textwidth}
\begin{equation*}
\begin{aligned}
		f(x) &= \frac{1}{d} \sum_{i=1}^d \left[(x_i-B)^2 \right.\\
		 &\left.-10\cos(2\pi (x_i-B)) + 10\right] + C
\end{aligned}
\end{equation*}
with $B = \arg \min f(x)$ and $C= \min f(x)$
\column{0.5\textwidth}
\begin{figure}[ht]
	\centering
	\includegraphics[width=1.1\linewidth]{Figure/R_function}
	%\caption{$d=2$ and $B=C=0$}
	\end{figure}
\end{columns}
\end{frame}

\begin{frame}
\frametitle{Massive local minima of Rastrigin function}

\begin{itemize}
	\item Exponential growth of the number of local minima: $5^d$
	\item Number of minima is $5^{1000}\approx 10^{690}$, when $d=1000$
	\begin{table}[ht]

		\centering\begin{tabular}{|c|c|c|c|c|c|}
			\hline
			d & 1 & 2 & 30 & 100 & 1000 \\
			\hline
			Number of local minima & $5$ & $5^2$ & $5^{30}$ & $5^{100}$ & $5^{1000}$\\
			\hline
		\end{tabular}
		\caption{Number of local minima in terms of dimension}
	\end{table}
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Comparison with different random processes}
\begin{table}
	\centering
	\begin{tabular}{|c|c|c|c|c|c|}
		\hline
		\multirow{2}*{$d$}&
		\multirow{2}*{$N$}&
		\multirow{2}*{$M$}&\multicolumn{3}{c|}{CBO}\\
		\cline{4-6}
		~& ~ & ~ &  $\mathcal{N}(0,1)$&$\mathcal{U}(-1,1)$ &Wiener process  \\
		\hline
		2  & 50 & 40 & 100\% & 100\% & 99\% \\
		10 & 50 & 40 & 100\% & 100\% & 2\%  \\
		20 & 50 & 40 & 98\%  & 22\%  & 0\%  \\
		20 & 50 & 20 & 66\%  & 2\%   & 0\%  \\
		30 & 50 & 40 & 26\%  & 0\%   & 0\%  \\
		30 & 500&5   &  0\%  & 0\%   & 0\%  \\
		%30 & 500 & 400 &     & 0\%   &  \%  \\
		\hline
		\multirow{2}*{$d$}&
		\multirow{2}*{$N$}&
		\multirow{2}*{$M$}&\multicolumn{3}{c|}{Adam-CBO}\\
		\cline{4-6}
		~& ~ & ~ &  $\mathcal{N}(0,1)$& $\mathcal{U}(-1,1)$ &Wiener process  \\
		\hline
		30 & 500  & 5  & 99\% & 100\% & 0\% \\
		100& 5000 & 5  & 100\% & 100\% & 0\%\\
		1000& 8000 & 50 & 92\% & 20\% &0\%\\
		\hline
	\end{tabular}
\end{table}
\end{frame}
\begin{frame}
\frametitle{$\lambda = 0.1$, and $\sigma^t= 0.99^{\frac{t}{20}}$}

\begin{table}
	\centering
	\begin{tabular}{|c|c|c|c|c|}
		\hline
		\multirow{2}*{$d$}&
		\multirow{2}*{$N$}&
		\multirow{2}*{$M$}&\multicolumn{2}{c|}{Adam-CBO}\\
		\cline{4-5}
		~& ~ & ~ &  $\mathcal{N}(0,1)$& $\mathcal{U}(-1,1)$\\
		\hline
		1000& 8000 &  50  & 92\% & 20\% \\
		1000& 10000 & 50  & 100\% & 28\% \\
		1000& 12000 & 50  & 100\% & 28\% \\
		1000& 14000 & 50  & 100\% & 32\% \\
		1000& 16000 & 50  & 100\% & 32\%  \\
		\hline
	\end{tabular}
	%\caption{Different numbers of particles when the dimension is $1000$}
	\label{tbl:p_N sr}
\end{table}

\end{frame}
\subsection{Machine learning tasks}
\begin{frame}
\frametitle{Spectrail bias \cite{Rahaman2018}/Frequency principle \cite{xu2019frequency}}

	\begin{columns}
	\column{0.5\textwidth}

	\begin{itemize}
		\item[] \begin{align*}
		u(x) = \sin(2\pi x) + \sin(8 \pi x ^2)
		\end{align*}
		\item Network width $ = 50$, depth $ = 3$, and $2701$ parameters
		\item $\lambda=0.2$
		\item $N = 500$ and $M=5$ in the first $50000$ iterations
		\item Afterwards the random term is ignored and $M = 10$
	\end{itemize}

	\column{0.5\textwidth}
	\begin{figure}[ht]
		\centering
		\includegraphics[width=0.7\linewidth]{Figure//Fprinciple_exm1}
	\end{figure}
	\end{columns}

\end{frame}
\begin{frame}
\begin{columns}
\column{0.5\textwidth}
	\begin{align*}
	&u(x) = \left\{\begin{matrix}
	1   & x < -\frac{7}{8}, x> \frac{7}{8}, -\frac{1}{8}<x<\frac{1}{8}\\
	-1  & \frac{3}{8}< x< \frac{5}{8} , -\frac{5}{8}< x<- \frac{3}{8}\\
	0 & \text{otherwise}
	\end{matrix} \right.
	\end{align*}
	The same setup as in the previous slide
	\column{0.5\textwidth}
	\begin{figure}[ht]
		\centering
		\includegraphics[width=0.7\linewidth]{Figure//Fprinciple_exm2}
	\end{figure}
\end{columns}
\end{frame}
\begin{frame}
\frametitle{Gradient exploding or vanishing: DNN with fixed width $10$ and different depths}

\begin{equation*}
	u(x) = \sin(k\pi x ^k)
\end{equation*}
$N = 500$, $M = 5$
\begin{table}
	\centering
	\begin{tabular}{|c|c|c|c|c|}
	\hline
		 depth & Num of parameters & k = 2 &  k = 3 & k = 4  \\
		\hline
		4   & 141 & 6.62 e-03 & 1.32 e-02 & 1.71 e-01\\
		7   & 471 & 4.78 e-03 & 1.42 e-02 & 7.54 e-03\\
		12  & 1021 & 7.44 e-03 & 1.30 e-02 & 5.32 e-02\\
		22  & 2121 & 1.00 e-02 & 1.01 e-02 & 1.21 e-01\\
		\hline
	\end{tabular}
	\caption{Absolute $L^2$ norm in terms of network depth when $k=2, 3, 4$}
\end{table}

SGD or Adam fails to converges well (with final error around $0.3$) when the network depth is $4$ and $10$, respectively

\end{frame}

\begin{frame}
\frametitle{Solving PDEs by DNNs: Deep Ritz method \footfullcite{weinan2018deep}}

\begin{equation*}
\left\{
\begin{aligned}
&-\nabla \cdot ( A(x) \nabla u) = - \sum_{i=1}^d\delta(x_i) & x\in \Omega=[-1,1]^d\\
&u(x) = g(x)  & x\in \partial \Omega
\end{aligned}\right.
\end{equation*}
with
\begin{equation*}
	A(x)=
	\left[\begin{matrix}
		(x_1^2)^{\frac{1}{4}} & & \\
		& \ddots & &\\
		& &  (x_d^2)^{\frac{1}{4}}
	\end{matrix}\right].
\end{equation*}
\begin{itemize}
	\item Exact solution $u(x)= \sum_{i=1}^d|x_i|^{\frac{1}{2}}$ is only in $H^{1/2}(\Omega)$
	\item Derivatives have singularities at $x_i =0$
\end{itemize}

\end{frame}
\begin{frame}
\frametitle{Loss function in Deep Ritz method}

\begin{equation*}
\begin{aligned}
	I[u] =& \int_{\Omega}\frac{1}{2}(\nabla u)^T  A(x)  \nabla u(x)\mathrm{d}x + \sum_{i=1}^d\int_{-1}^{1}\delta(x_i)u(x)\mathrm{d}x_i \\ & + \eta \int_{\partial \Omega} (u(x)-g(x))^2 \mathrm{d}x,
	\end{aligned}
\end{equation*}
\tiny{
\begin{table}[H]
	\centering
	\begin{tabular}{|c|c|c|c|c|c|}
		\hline
		d & n & m &  Activation-Optimizer & $L^2$ error & $L^{\infty}$ error\\
		\hline
		\multirow{4}*{2}&\multirow{4}*{20} & \multirow{4}*{2} & ReLu-Adam & 1.23 e-02 & 9.91 e-02\\
		& &  & ReQu-Adam & 2.22 e-02  & 4.21 e-01 \\
		& &  & sigmoid-Adam & 2.19 e-02 & 3.14 e-01\\
		& &  & $|x|^{0.5}$ - Adam-CBO & 3.96 e-03 & 2.09 e-02\\
		\hline
		\multirow{4}*{4}&\multirow{4}*{40} & \multirow{4}*{2} & ReLu-Adam & 6.72 e-03 & 3.70 e-01\\
		& &  & ReQu-Adam & 1.43 e-02 & 1.10 e -00\\
		& &  & sigmoid-Adam & 7.90 e-03 & 7.66 e -02\\
		& &  & $|x|^{0.5}$ -Adam-CBO & 3.13 e-03 & 9.52 e -02\\
		\hline
	\end{tabular}
\caption{Errors in $L^2$ and $L^{\infty}$ norms by Adam and Adam-CBO methods}
\end{table}
}
\end{frame}

\begin{frame}
\frametitle{Cont'd}

	\begin{figure}
	\subfigure[$L^{\infty}$ error]{
		\includegraphics[width=0.45\linewidth]{Figure//singular_PDE_4D_L_infty_error}
	}
	\subfigure[$L^2$ error]{
		\includegraphics[width=0.45\linewidth]{Figure//singular_PDE_4D_L2_error}
	}
	\caption{Training process of Adam and Adam-CBO methods when $d=4$}
\end{figure}

\end{frame}
\begin{frame}
\frametitle{Singularities}

\begin{figure}
	\subfigure[$x_2=x_3=x_4=0$]{
		\includegraphics[width=0.45\linewidth]{Figure//singular_PDE_4D_Landscape_x1}
	}
	\subfigure[$x_1=x_3=x_4=0$]{
		\includegraphics[width=0.45\linewidth]{Figure//singular_PDE_4D_Landscape_x2}
	}
	\caption{One-dimensional solution profiles at the intersection}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{Conclusion}

Adam-CBO is
\begin{itemize}
	\item able to find the global minimizer in high dimensions
	\item free of curse of dimensionality
	\item suitable for machine learning tasks with
	\begin{itemize}
		\item gradient explosion or vanishing
		\item non-different activation functions
	\end{itemize}
\end{itemize}
{\huge\medskip
\begin{center}
	Thank you for your attention!
\end{center}
}
\end{frame}

\end{document}


%&context

\section[sct_match]{Finding subtrees in other trees}

\match{} tries to match a (typically smaller) "pattern" tree to one or more
"target" tree(s). If the pattern matches the target, the target tree is
printed. Intuitively, a pattern matches a target if one can superimpose it onto
the target without "breaking" either. More accurately, the following happens
(in both trees):
\startitemize[n]
	\item leaves with labels found in both trees are kept, the other ones are
		pruned
	\item inner labels are discarded
	\item both trees are ordered (as done by \order{}, see  \in{}[sct_order])
	\item branch lengths are discarded
\stopitemize
At this point, the modified pattern tree is compared to the modified target, and if the \nw{} strings are identical, the match is successful.

\subsubsection{Example: finding trees with a specified  subtree topology}

File \filename{hominoidea.nw} contains seven trees corresponding to successive
theories about the phylogeny of apes (these were taken from
\from[URL:Hominoidea]). Let us see which of them group
humans and chimpanzees as a sister clade of gorillas (which is the current
hypothesis).

\page[no]
Here are small images of each of the trees in \filename{hominoidea.nw}: \\

\startcombination[2*4]
{\externalfigure[homino_0][scale=700]} {1 (until 1960)}
{\externalfigure[homino_1][scale=700]} {2 (Goodman, 1964)}
{\externalfigure[homino_2][scale=700]} {3 (gibbons as outgroup)}
{\externalfigure[homino_3][scale=700]} {4 (Goodman, 1974: orangs as outgroup)}
{\externalfigure[homino_4][scale=700]} {5 (resolving trichotomy)}
{\externalfigure[homino_5][scale=700]} {6 (Goodman, 1990: gorillas as outgroup)}
{\externalfigure[homino_6][scale=700]} {7 (split of {\em Hylobates})}
\stopcombination

Trees \#6 and \#7 match our criterion, the rest do not. To look for matching trees in \filename{hominoidea.nw}, we pass the pattern on the command line:

\typefile{match_1_txt.cmd}
\page[no]
\typefile{match_1_txt.out}

Note that only the pattern tree's topology matters: we would get the
same results with pattern \code{((Homo,Pan),Gorilla);},
\code{((Pan,Homo),Gorilla);}, etc., but not with
\code{((Gorilla,Pan),Homo);} (which would select trees \#1, 2, 3, and 5. In
future versions I might add an option for strict matching.

The behaviour of \match{} can be reversed by passing option \code{-v} (like
\code{grep -v}): it will print trees that {\em do not} match the pattern.
Finally, note that \match{} only works on leaf labels (for now), and assumes
that labels are unique in both the pattern and the target tree.


\appendix %% Start the appendices.
\chapter{Appendix}

\section{Comparison of Distance Measures and Prototype Selection Strategies on UCR datasets}
\label{sec:app-ucr}

As a part of this thesis, we tested multiple combinations of different distance measures and Feature DTW transformations, including various strategies for prototype selection on UCR datasets \cite{exp:UCRArchive2018}. In our implementation, we are using five-split cross-validation and comparing the average accuracy of each algorithm. For complete results, please see our repository \footnote{\url{https://github.com/H00N24/visual-analysis-of-big-time-series-datasets}}. Based on our results (Fig.~\ref{app:results-datasets}, Fig.~\ref{app:results-rank}), we make these conclusions:
\begin{itemize}
    \item The overall best distance measure is a combination of constrained DTW computed on the original time series and their derivations.
    \item Prototyped Feature DTW outperforms Prototyped Feature DTW.
    \item Random selection of prototypes can be a useful starting strategy.
\end{itemize}

\subsection*{Notation}
\subsubsection*{Distance measures:}
\begin{itemize}
    \item $dtw$ - Dynamic Time Warping \cite{met:dtw}
    \item $fdtw$ - Fast Dynamic Time Warping \cite{met:FastDTW}
    \item $sakoe\_chiba$ - DTW with Sakoe-Chiba constraint \cite{met:dtw-window}
    \item $itakura$ - DTW with Itakura constraint \cite{met:dtw-itakura}
    \item $dd\_distance\_measure\_\alpha$ - a combination of distance measure using the  original time series and their first derivatives ($ (1 - \alpha) * dist(x, y) + \alpha * dist(x', y') $) \cite{met:fDTW}
\end{itemize}
\subsubsection*{Prototype selection + Classification methods:}
\begin{itemize}
    \item $Random\_X$ - Randomly selecting $X\%$ of training data as prototypes ($Random\_10$, $Random\_30$, $Random\_50$) + Linear SVC
    \item $LassoSVC$ - Selecting prototypes with the highest importance from the Linear Support Vector Machine classifier with l1 penalization + Linear SVC
    \item $Tree$ - Selecting prototypes with the highest importance from the Extra Tree Classifier + Linear SVC
    \item $SVC$ - Using all prototypes + Linear SVC
    \item $1NN$ - Nearest Neighbor classifier using one closest neighbor
\end{itemize}
\begin{figure}[h]
    \centering
    \includegraphics[width=\textwidth]{img/ucr_accuracy.png}
    \caption{Average accuracy of distance measures, prototype selection strategies, and classification algorithms on UCR datasets.}
    \label{app:results-datasets}
\end{figure}
\begin{figure}[h]
    \centering
    \includegraphics[width=\textwidth]{img/ucr_avg_rank.png}
    \caption{Average rank of combinations of distance measures and prototype selection strategies on UCR datasets.}
    \label{app:results-rank}
\end{figure}