%!TEX root=ClassNotes.tex

\section{Proofs}

We'll start by learning about the various kinds of proofs that we'll encounter in this class.\footnote{This section in taken almost verbatim from \url{http://zimmer.csufresno.edu/~larryc/proofs/proofs.html}.}

Proofs are the heart of mathematics. You must come to terms with proofs -- you must be able to read, understand
and write them. What is the secret? What magic do you need to know? The short answer is: there is no secret, no
mystery, no magic. All that is needed is some common sense and a basic understanding of a few trusted and easy
to understand techniques.

\subsection*{The Structure of a Proof}
The basic structure of a proof is easy: it is just a series of statements, each one being either
\begin{itemize}
	\item
	An assumption or
	\item
	A conclusion, clearly following from an assumption or previously proved result
\end{itemize}
And that is all. Occasionally there will be the clarifying remark, but this is just for the reader and has no logical
bearing on the structure of the proof.

A well written proof will flow. That is, the reader should feel as though they are being taken on a ride that takes
them directly and inevitably to the desired conclusion without any distractions about irrelevant details.

Each step
should be clear or at least clearly justified. A good proof is easy to follow.
When you are finished with a proof, apply the above simple test to every sentence: is it clearly
\begin{enumerate}
	\item an assumption
	\item a justified conclusion?
\end{enumerate}
If the sentence fails the test, maybe it doesn't belong in the proof.

\subsection*{Example}
In order to write proofs, you must be able to read proofs. See if you can follow the proof below. Don't worry
about how you would have (or would not have) come up with the idea for the proof. Read the proof with an eye
towards the criteria listed above. Is each sentence clearly an assumption or a conclusion? Does the proof flow?
Was the theorem in fact proved?

\begin{theorem}
	\label{theorem:irrationality_of_sqrt2}
	 The square root of 2 is an irrational number.
\end{theorem}
\begin{proof}
	Let's represent the square root of 2 by $s$. Then, by definition, $s$ satisfies the equation
	\begin{align*}
		s^2 = 2.
	\end{align*}
	If $s$ were a rational number, then we could write $s = p/q$
where $p$ and $q$ are a pair of integers. In fact, by dividing out the common multiple if necessary, we may even assume $p$ and $q$ have no common multiple (other than 1). If we now substitute this into the first equation we obtain, after a little algebra, the equation
\begin{align*}
		p^2 = 2 q^2.
\end{align*}

But now, 2 must appear in the prime factorization of the number $p^2$ (since it appears in the same number $2 q^2$). Since 2 itself is a prime number, 2 must then appear in the prime factorization of the number $p$. But then, $2 \cdot 2$ would appear in the prime factorization of $p^2$, and hence in $2 q^2$. By dividing out a 2, it then appears that 2 is in the prime factorization of $q^2$. Like before (with $p^2$) we can now conclude 2 is a prime factor of $q$. But now we have $p$ and $q$ sharing a prime factor, namely 2. This violates our assumption above (see if you can find it) that $p$ and $q$ have no common multiple other than 1.
\end{proof}

\subsection{Direct Proofs}
Most theorems that you want to prove are either explicitly or implicitly in the form \begin{align*}
	\mbox{ ``If \textit{P}, then \textit{Q}". }
\end{align*} This is the standard form of a
theorem (though it can be disguised). A direct proof should be thought of as a flow of implications beginning
with \textit{P} and ending with \textit{Q}.
	\begin{align*}
		{P} \implies \dots \implies {Q}
	\end{align*}
Most proofs are (and should be) direct proofs. Always try direct proof first, unless you have a good reason not to. If you find a simple proof, and you are convinced of its correctness, then don't be shy about. Many times proofs
are simple and short.

\begin{exercise}
	\label{question:ex1}
	Prove each of the following.
	\begin{enumerate}
		\item If $a$ divides $b$ and $a$ divides $c$ then $a$ divides $b + c$, where $a$, $b$, and $c$ are positive integers.
		\item For all real numbers $a$ and $b$, $a^2 + b^2 \ge 2 a b$.
		\item If $a$ is a rational number and $b$ is a rational number, then $a+b$ is a rational number.
	\end{enumerate}
\end{exercise}

\subsection{Proof by Contradiction}
\label{section:proofByContradiction}
In a proof by contradiction we assume, along with the hypotheses, the logical negation of the result we wish to
prove, and then reach some kind of contradiction. That is, if we want to prove ``If \textit{P}, then \textit{Q}'', we assume \textit{P} and
Not \textit{Q}. The contradiction we arrive at could be some conclusion contradicting one of our assumptions, or
something obviously untrue like 1 = 0. The proof of Theorem \ref{theorem:irrationality_of_sqrt2} is an example of this.

\begin{exercise}
	\label{question:ex2}
	Use the method of Proof by Contradiction to prove each of the following.
	\begin{enumerate}
		\item The cube root of 2 is irrational.
		\item If $a$ is a rational number and $b$ is an irrational number, then $a+b$ is an irrational number.
		\item There are infinitely many prime numbers.\footnote{There are dozens of proofs of this theorem, originally due to Euclid. Feel free to look one up online.}
	\end{enumerate}
\end{exercise}

\subsection{Proof by Contrapositive}
\label{section:proofByContrapositive}
	Proof by contrapositive takes advantage of the logical equivalence between ``\textit{P} implies \textit{Q}'' and ``Not \textit{Q} implies Not \textit{P}''. For example, the assertion ``If it is my car, then it is red'' is equivalent to ``If that car is not red, then it is not mine''. So, to prove ``If \textit{P}, then \textit{Q}'' by the method of contrapositive means to prove
	\begin{align*}
		\mbox{``If Not \textit{Q}, then Not \textit{P}''.}
	\end{align*}

\noindent \textbf{How Is This Different From Proof by Contradiction?}
The difference between the Contrapositive method and the Contradiction method is subtle. Let's examine how the two methods work when trying to prove ``If \textit{P}, then \textit{Q}".
\begin{description}
	\item[Method of Contradiction:] Assume \textit{P} and Not \textit{Q} and prove some sort of contradiction.
	\item[Method of Contrapositive:] Assume Not \textit{Q} and prove Not \textit{P}.
\end{description}
The method of Contrapositive has the advantage that your goal is clear: Prove Not P. In the method of Contradiction, your goal is to prove a contradiction, but it is not always clear what the contradiction is going to be at the start.

\begin{exercise}
	\label{question:ex3}
	Use the method of Proof by Contrapositive to prove each of the following.
	\begin{enumerate}
		\item If the product of two integers is even, then at least one of the two must be even.
		\item If the product of two integers is odd, then both must be odd.
		\item If the product of two real numbers is an irrational number, then at least one of the two must be an irrational number.
	\end{enumerate}
\end{exercise}

\subsection{Converse}
The converse of an assertion in the form ``If \textit{P}, then \textit{Q}" is the assertion
\begin{align*}
	\mbox{``If \textit{Q}, then \textit{P}"}.
\end{align*}
A common logical fallacy is to assume that if an assertion is true then so is its converse.

\subsubsection{If and Only If}
Many theorems are stated in the form ``\textit{P}, if, and only if, \textit{Q}". Another way to say the same thing is: ``\textit{Q} is necessary, and sufficient for \textit{P}". This means two things:
\begin{center}
	``If \textit{P}, Then \textit{Q}" \qquad and \qquad ``If \textit{Q}, Then \textit{P}".
\end{center} So to prove an ``If, and Only If" theorem, you must prove the theorem and also its \textit{converse}.

\begin{exercise}
	Go back to the problems in Exercises \ref{question:ex1}, \ref{question:ex2}, \ref{question:ex3} and find the ones which are of the form ``If \textit{P}, Then \textit{Q}". For each of these:
	\begin{itemize}
		\item State the converse.
		\item Prove or disprove the converse (by providing either a proof or a counterexample).
		\item For the problems where the converse is also true rewrite the assertion as an "If, and Only If" statement.
	\end{itemize}
\end{exercise}

\subsection{Quantifiers}
It is extremely important in mathematics to be able to formulate very precise statements. To prevent ambiguous statements and logical fallacies the vocabulary used is very limited and every term has a well-defined meaning. The two concepts we need to get used to are {\it logical operators} and {\it quantifiers}.\\

{\bf Logical Operators:} Operators allow us to form complex statements by combining simpler ones. The basic logical operators are {\it and}, {\it or}, and {\it not}. Their usage is the same as in everyday language. The more complicated logical operator that we'll be using a lot is {\it If - then -}. We've already encountered several examples of its usage in the previous sections.\\

{\bf Quantifiers:} Quantifiers allow us to make abstract statements which are ``universally true'' without having to specify a concrete element. There are two commonly used	quantifiers:
\begin{align*}
	\mbox{\textbf{For all/every}} && \mbox{\textbf{There exists}}
\end{align*}
Understanding and formulating complex statements using these requires a lot of practice. You'll get used to these as the course progresses. Here are a few examples,
\begin{example}$ $
	\begin{enumerate}
		\item \emph{For every} odd integer $a$, the integer $a+1$ is even.
		\item An integer $a$ is even if and only if \emph{there exists} an integer $b$ such that $a = 2b$.
		\item There \emph{do not exist} integers $p,q$ such that $\frac{p}{q} = \sqrt{2}$.
		\item \emph{For every} non-zero rational number $x$ \emph{there exists} a rational number $y$ such that $x \cdot y = 1$.
	\end{enumerate}
\end{example}

\subsubsection{Nesting Quantifiers}
	When dealing with statements involving multiple quantifiers the order really matters; changing the order changes the meaning of a statement completely. When you're using multiple quantifiers in a single sentence you should always pause and check if you're using the right quantifiers in the right order.

\begin{exercise}
	For each of the following pairs, explain how changing the order of the quantifiers changes the meaning of the statement.\footnote{It's ok to a bit vague in your answers to this question.}
	\begin{enumerate}
		\item \begin{enumerate}
			\item \emph{For every} even integer $a$ \emph{there exists} an integer $b$ such that $a = 2b$.
			\item \emph{There exists} an integer $b$ such that \emph{for every} even integer $a$, $a = 2b$.
		\end{enumerate}
		\item \begin{enumerate}
			\item \emph{For every} positive integer $n$ \emph{there exists} a positive real number $\epsilon$ such that $ \epsilon < 1/n$.
			\item \emph{There exists} a positive real number $\epsilon$ such that \emph{for every} positive integer $n$, $ \epsilon < 1/n$.\footnote{Such an $\epsilon$ is called an {\bf infinitesimal}.	There are number systems, for example the \textbf{hypperreal numbers}, which extend the real numbers by incorporating infinitesimals.}
		\end{enumerate}
		\item \begin{enumerate}
			\item \emph{For every} even integer $a$, \emph{for every} odd integer $b$, $a + b$ is odd.
			\item \emph{For every} odd integer $b$, \emph{for every} even integer $a$, $a + b$ is odd.
			\end{enumerate}
		\item Let $f: \R \rightarrow \R$ be a function and let $\epsilon$ be a positive real number. \begin{enumerate}
			\item {\it For every} $x$, {\it there exists} a $\delta > 0$ such that {\it for every} $y$, if $|x-y| < \delta$ then $|f(x) - f(y)| < \epsilon$.
			\item {\it There exists} a $\delta > 0$ such that {\it for every} $x$, {\it for every} $y$, if $|x-y| < \delta$ then $|f(x) - f(y)| < \epsilon$.
		\end{enumerate}
	\end{enumerate}
\end{exercise}

	\subsubsection{Negating Nested Quantifiers}
	As we saw in Sections \ref{section:proofByContradiction} and \ref{section:proofByContrapositive} we often need to negate mathematical statements.

	In order to systematically negate complicated logical expressions, we observe that we can negate simple statements in the following manner. (These are called {\it De Morgan's laws}.)
	\begin{center}
		\begin{tabular}{|p{0.475\textwidth}|p{0.475\textwidth}|}
			\hline Statement
				&
					Negation \\ \hline & \\
			For every $x$ the statement $P$ is true.
				&
					There exists an $x$ such that the statement $P$ is false.
					\\& \\ \hline &\\
			There exists an $x$ such that the statement $P$ is true.
				&
					For every $x$ the statement $P$ is false.
					\\& \\ \hline &\\
			$P$ is true and $Q$ is true
				&
					$P$ is false or $Q$ is false.
					\\& \\ \hline &\\
			$P$ is true or $Q$ is true
				&
					$P$ is false and $Q$ is false.
					\\& \\ \hline &\\
			If $P$ then $Q$
				&
					$P$ but not $Q$.
					\\& \\ \hline
	\end{tabular}
	\end{center}
	If we have multiple quantifiers in a statement then we start with the outermost quantifier and recursively move inwards.
	\begin{example}
		The negation of {\it ``For every $x$, there exits a $y$ such that, the statement $P$ is true''} equals {\it ``There exists an $x$ such that, for every $y$, the statement $P$ is false.''}
	\end{example}
	\begin{exercise} Negate each of the following.
		\begin{enumerate}
			\item \emph{For every} even integer $a$ \emph{there exists} an integer $b$ such that $a = 2b$.
			\item \emph{There exists} an integer $b$ such that \emph{for every} even integer $a$, $a = 2b$.
			\item \emph{For every} positive integer $n$ \emph{there exists} a positive real number $\epsilon$ such that $ \epsilon < 1/n$.
			\item \emph{There exists} a positive real number $\epsilon$ such that \emph{for every} integer $n$, $ \epsilon < 1/n$.
			\item \emph{For every} even integer $a$, \emph{for every} odd integer $b$, $a + b$ is odd.
			\item {\it For every} $\epsilon > 0$, {\it for every} $x$, {\it there exists} a $\delta > 0$ such that {\it for every} $y$, if $|x-y| < \delta$ then $|f(x) - f(y)| < \epsilon$.
			\item {\it For every} $\epsilon > 0$, {\it there exists} a $\delta > 0$ such that {\it for every} $x$, {\it for every} $y$, if $|x-y| < \delta$ then $|f(x) - f(y)| < \epsilon$.
		\end{enumerate}
	\end{exercise}

	\noindent {\bf Word of caution:} When the word ``and'' is used in a list it is not considered a logical operator (this is a deficiency of the English language). For example, negating the following expression
	\begin{center}
		\emph{For every} even integer $a$ and \emph{for every} odd integer $b$, $a + b$ is odd,
	\end{center}
	does not change the {\it and} to an {\it or} (what happens if you do change the {\it and} to an {\it or}?).


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\hyphenation{DCPVAL DMDVAL}
\begmath 11.2 Evaluation, Integration, and Differentiation of Polynomials

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

This set of subroutines will evaluate, integrate, or differentiate
polynomials. The polynomials may be represented by coefficients relative to
either the monomial or Chebyshev basis. The data structure and
parameterization used to represent a polynomial is the same as that used by
the least-squares polynomial curve fit subroutines described in Chapter~11.1.
Special procedures for evaluation of polynomials expressed using the
Legendre and Laguerre bases are described in Chapters~2.11 and~2.12
respectively.

\subsection{Usage}

\subsubsection{Usage for Evaluation}

\paragraph{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf NDEG}

\item[REAL]  \ {\bf P}$(\geq $NDEG+3){\bf , X, Y, SCPVAL, SMPVAL}
\end{description}
Assign values to NDEG, X, and P($i$), $i = 1$, NDEG$+3$. If the Chebyshev
basis is being used, use the statement:
$$
\fbox{{\bf Y = SCPVAL(P, NDEG, X)}}
$$
If the monomial basis is being used, use the statement:
$$
\fbox{{\bf Y = SMPVAL(P, NDEG, X)}}
$$
Following the appropriate one of these two statements the value of the
polynomial at the argument X will be stored in Y.

\paragraph{Argument Definitions}
\begin{description}
\item[P()]  \ [in] An array containing NDEG$+3$ parameters that define a
polynomial as described in Section D.

\item[NDEG]  \ [in] Degree of the polynomial.

\item[X]  \ [in] Argument value at which the polynomial is to be evaluated.

\item[SCPVAL]  \ [out] The value of the polynomial evaluated at X assuming
the Chebyshev basis representation.

\item[SMPVAL]  \ [out] The value of the polynomial evaluated at X assuming
the monomial basis representation.
\end{description}
\subsubsection{Usage for Integration}

\paragraph{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf NDEGA, NDEGB}

\item[REAL]  \ {\bf A}($\geq $NDEGA+3){\bf , B}($\geq $NDEGA+4)
\end{description}
Assign values to NDEGA and A($i$), $i = 1$, ..., NDEGA$+3$. If the Chebyshev
basis is being used, use the statement:
$$
\fbox{{\bf CALL SCPINT (A, NDEGA, B, NDEGB)}}
$$
If the monomial basis is being used, use the statement:
$$
\fbox{{\bf CALL SMPINT (A, NDEGA, B, NDEGB)}}
$$
Following the appropriate one of these two call statements the results will
be stored in B() and NDEGB.

\paragraph{Argument Definitions}
\begin{description}
\item[A()]  \ [in] An array containing NDEGA$+3$ parameters that define the
input polynomial, say $p(x)$. See Section D for the specification of the
parameterization.

\item[NDEGA]  \ [in] Degree of the input polynomial, $p(x).$

\item[B()]  \ [out] On return B() will contain NDEGB$+3$ parameters defining
the output polynomial, say $q(x)$, which is the indefinite integral of the
input polynomial $p(x)$. Mathematically the constant term of $q(x)$ is an
arbitrary constant of integration. This subroutine will set the constant
term, B($3)$, to zero. The storage locations occupied by A() and B() must be
distinct.

\item[NDEGB]  \ [out] The subroutine sets NDEGB $= \text{NDEGA}+1$ to indicate
the degree of the output polynomial. The storage locations occupied by NDEGA
and NDEGB must be distinct.
\end{description}
\subsubsection{Usage for Differentiation}

\paragraph{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf NDEGC, NDEGD}

\item[REAL]  \ {\bf C} $(\geq $NDEGC+3){\bf , D}($\geq \max(3,\text{NDEGC}+2%
))$
\end{description}
Assign values to NDEGC and C($i$), $i = 1$, ..., NDEGC$+3$. If the Chebyshev
basis is being used, use the statement:
$$
\fbox{{\bf CALL SCPDRV (C, NDEGC, D, NDEGD)}}
$$
If the monomial basis is being used, use the statement:
$$
\fbox{{\bf CALL SMPDRV (C, NDEGC, D, NDEGD)}}
$$
Following the appropriate one of these two call statements the results will
be stored in D() and NDEGD.

\paragraph{Argument Definitions}
\begin{description}
\item[C()]  \ [in] An array containing NDEGC$+3$ parameters that define the
input polynomial, say $p(x)$. See Section D for the specification of the
parameterization.

\item[NDEGC]  \ [in] Degree of the input polynomial, $p(x).$

\item[D()]  \ [out] On return D() will contain NDEGD$+3$ parameters defining
the output polynomial, say $q(x)$, which is the derivative of the input
polynomial $p(x)$. The storage locations occupied by C() and D() must be
distinct.
\newpage

\item[NDEGD]  \ [out] The subroutine sets NDEGD $=\max (0$, NDEGC$-$1) to
indicate the degree of the output polynomial. The storage locations occupied
by NDEGC and NDEGD must be distinct.
\end{description}
\subsubsection{Usage for Double Precision Evaluation, Integration or
Differentiation}

For DOUBLE PRECISION usage change the REAL type statements to DOUBLE
PRECISION and change the initial ``S'' of the function and subroutine names
to a ``D.'' Note particularly that if the function names DCPVAL or DMPVAL
are used they must be typed DOUBLE PRECISION either explicitly or via an
IMPLICIT statement.

\subsection{Examples and remarks}

Let a cubic polynomial $p(x)$ be defined relative to the Chebyshev basis as $%
p(x)=10+8T_1(u)+6T_2(u)+4T_3(u)$ where $u=(x-5)/2$. The DRSCPVAL program
computes the indefinite integral of $p(x)$ calling it $q(x)$. This
computation is checked by computing $r(x)$ as the derivative of $q(x)$. Note
that $r(x)$ agrees with $p(x)$. Finally the program evaluates the definite
integral%
\begin{equation*}
z=\int_4^6p(x)\,dx=q(6)-q(4)=10
\end{equation*}
The output from this program is shown in ODSCPVAL.

\subsection{Functional Description}

In typical expected usage the polynomial parameter vector input to any of
the subprograms of this set will have been produced by the library curve
fitting subroutine SPFIT (or DPFIT) or an integration or differentiation
subroutine of this set. The subprograms of this set are thus intended to
let the user do the operations of evaluation, integration or differentiation
of polynomials without being concerned with the details of the parametric
representation of polynomials or algorithmic details.

The following description is provided for those who wish to know more of the
internal details.

For the purposes of this set of subprograms a polynomial of degree $n$, say $%
p(x)$, is represented by a set of $n+3$ parameters, say $a_1$, ..., $a_{n+3}$%
. The first two parameters define a linear transformation of the independent
variable
\begin{equation*}
u=(x-a_1)/a_2
\end{equation*}
The remaining $n+1$ parameters are coefficients of an $n^{th}$ degree
polynomial in the transformed variable $u$. If the Chebyshev basis is used
this polynomial is%
\begin{equation*}
p(x)=q(u)=\sum_{i=0}^na_{i+3}T_i(u)
\end{equation*}
whereas if the monomial basis is used the polynomial is%
\begin{equation*}
p(x)=q(u)=\sum_{i=0}^na_{i+3}u^i
\end{equation*}
The Chebyshev polynomials $T_i(u)$ are defined by the equations%
\begin{gather*}
T_0(u)=1,\quad T_1(u)=u\\
T_i(u)=2uT_{i-1}(u)-T_{i-2}(u),\quad i=2,3,...
\end{gather*}
The formulas for differentiation and integration of polynomials expressed
using the Chebyshev basis may be derived from the following standard
identities:
\begin{align*}
0&=dT_0(u)/du\\
T_0(u)&=dT_1(u)/du\\
T_1(u)&=\frac 14 dT_2(u)/du\\
T_i(u)&=\frac 12\frac d{du}\left[ \frac{T_{i+1}(u)}{i+1}-
\frac{T_{i-1}(u)}{i-1}\right] ,\quad i=2,3,...
\end{align*}
The algorithms used by this set of subprograms are specified as follows:

\subsubsection{Monomial Basis Evaluation, SMPVAL or DMPVAL}

Given an $n^{th}$ degree polynomial $p$ represented by the parameters $p_i$,
$i=1$,\ ..., $n+3$ and an argument $x$, compute $y=p(x).$%
\begin{align*}
u&=(x-p_1)/p_2\\
z_n&=p_{n+3}\\
z_i&=u\,z_{i+1}+p_{i+3},\quad i=n-1,n-2,...,0\\
y&=z_0.
\end{align*}
\subsubsection{Monomial Basis Integration, SMPINT or DMPINT}

Given an $n^{th}$ degree polynomial $p$ represented by the parameters $a_i$,
$i=1$,\ ..., $n+3$, compute the parameters $b_i$ that represent a polynomial
$q$ that for arbitrary $u$ and $v$ satisfies%
\begin{equation*}
\int_u^vp(x)\,dx=q(v)-q(u).
\end{equation*}
The formulas used are%
\begin{gather*}
b_1=a_1,\quad b_2=a_2,\quad b_3=0\\
b_{i+3}=a_2a_{i+2}/i,\ \ \ \ \ i=1,...,n+1
\end{gather*}
\subsubsection{Monomial Basis Differentiation, SMPDRV or DMPDRV}

Given an $n^{th}$ degree polynomial $p$ represented by the parameters $c_i$,
$i=1$,\ ..., $n+3$, compute the parameters $d_i$ that represent the
polynomial $q$ satisfying%
\begin{equation*}
\frac d{dx}p(x)=q(x).
\end{equation*}
The formulas used are%
\begin{gather*}
d_1=c_1,\quad d_2=c_2\\
d_{i+3}=(i+1)c_{i+4}/c_2,\quad i=0,...,n-1
\end{gather*}
with a special case of $d_3=0$ if $n=0.$

\subsubsection{Chebyshev Basis Evaluation, SCPVAL or DCPVAL}

Given an $n^{th}$ degree polynomial $p$ represented by the parameters $p_i$,
$i=1$, ..., $n+3$ and an argument $x$ compute $y=p(x).$%
\begin{align*}
u&=(x-p_1)/p_2\\
z_n&=p_{n+3}\\
z_{n-1}&=2uz_n+p_{n+2}\\
z_i&=2uz_{i+1}-z_{i+2}+p_{i+3},\quad i=n-2,...,1\\
y&=uz_1-z_2+p_3
\end{align*}
\subsubsection{Chebyshev Basis Integration, SCPINT or DCPINT}

Given an $n^{th}$ degree polynomial $p$ represented by the parameters $a_i$,
$i = 1, $..., $n+3$, compute the parameters $b_i$ that represent a
polynomial $q$ that for arbitrary $u$ and $v$ satisfies%
\begin{equation*}
\int_u^vp(x)\,dx=q(v)-q(u).
\end{equation*}
The formulas used are
\begin{gather*}
b_1=a_1,\quad b_2=a_2,\quad b_3=0\\
b_4=a_2\left[ a_3-(1/2)a_5\right]\\
b_{i+3}=a_2(a_{i+2}-a_{i+4})/(2i),\quad i=2,...,n+1,
\end{gather*}
where $a_i$ for $i>n+3$ is taken to be zero.

\subsubsection{Chebyshev Basis Differentiation, SCPDRV or DCPDRV}

Given an $n^{th}$ degree polynomial $p$ represented by the parameters $c_i$,
$i=1$, ..., $n+3$, compute the parameters $d_i$ that represent the
polynomial $q$ satisfying%
\begin{equation*}
\frac d{dx}p(x)=q(x)
\end{equation*}
The formulas used are%
\begin{gather*}
d_1=c_1,\quad d_2=c_2\\
d_{i+3}=2(i+1)c_{i+4}/c_2,\quad i=n-1,n-2\\
d_{i+3}=d_{i+5}+2(i+1)c_{i+4}/c_2,\quad i=n-3,...,1\\
d_3=d_5/2 + c_4 / c_2
\end{gather*}
with a special case of $d_3=0$ if $n=0.$

\subsection{Error Procedures and Restrictions}

The degree of the input polynomial must be zero or positive. If it is
negative the subprograms in this set will issue an error message
using the error processor in Chapter 19.2 at level 0 and
return.

The given values of P($2)$, A(2), and C($2)$ must be nonzero.  These
conditions are not tested.

The storage locations for the input quantities in the integration and
differentiation subroutines must be distinct from the storage locations for
the output quantities.

Since DCPVAL and DMPVAL are FUNCTION type subprograms, their names must be
typed DOUBLE PRECISION in any program that uses them.

\subsection{Supporting Information}

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DCPDRV & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DCPDRV, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
DCPINT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DCPINT, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
DCPVAL & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DCPVAL\rule[-5pt]{0pt}{8pt}}\\
DMPDRV & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DMPDRV, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
DMPINT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DMPINT, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
DMPVAL & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DMPVAL\rule[-5pt]{0pt}{8pt}}\\
SCPDRV & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
ERFIN, ERMSG, IERM1, IERV1, SCPDRV\rule[-5pt]{0pt}{8pt}}\\
SCPINT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
ERFIN, ERMSG, IERM1, IERV1, SCPINT\rule[-5pt]{0pt}{8pt}}\\
SCPVAL & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
SCPVAL\rule[-5pt]{0pt}{8pt}}\\
SMPDRV & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
ERFIN, ERMSG, IERM1, IERV1, SMPDRV\rule[-5pt]{0pt}{8pt}}\\
SMPINT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
ERFIN, ERMSG, IERM1, IERV1, SMPINT\rule[-5pt]{0pt}{8pt}}\\
SMPVAL & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
SMPVAL}\\
\end{tabular}

The source language is Fortran~77.

Designed by C. L. Lawson, JPL, 1970. Programmed by Lawson and D. Campbell,
JPL, 1970. Adapted to Fortran~77 by Lawson and S. Y. Chiu, JPL, 1984.

\begcodenp
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRSCPVAL}\vspace{10pt}
\lstinputlisting{\codeloc{scpval}}

\vspace{30pt}\centerline{\bf \large ODSCPVAL}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{scpval}}
\end{document}


\chapter{Principal Component Analysis}
\label{ch:pca}

Which of the following three scatter plots (showing x vs. y, x vs. z and y vs. z) for the same three-dimensional data gives us the best picture about the actual layout of the data in space?

\begin{figure*}[h]
    \centering
    \includegraphics[scale=0.25]{pca1.png}
    \hspace{0.5cm}
    \includegraphics[scale=0.25]{pca2.png}
    \hspace{0.5cm}
    \includegraphics[scale=0.25]{pca3.png}
    \caption{$\;$}
\end{figure*}

Yes, the first scatter plot looks very useful: it tells us that x and y are highly correlated and that we have three clusters of somewhat irregular shape. But remember: this data is three dimensional. What is we saw it from another, perhaps better perspective?

Let's make another experiment. Go to \url{https://in-the-sky.org/ngc3d.php}, disable Auto-rotate and Show labels and select Zoom to show Local Milky Way. Now let's rotate the picture of the galaxy to find the layout of the stars.

Think about what we've done. What are the properties of the best projection?

% not the best example, but couldn't reproduce the original
\begin{wrapfigure}{o}{0.8\textwidth}
    \includegraphics[scale=0.4]{pca-3d.png}
    \label{fig:pca3d}
\end{wrapfigure}

We want the data to be as spread out as possible. If we look from the direction parallel to the galactic plane, we see just a line. We lose one dimension, essentially keeping just a single coordinate for each star. (This is unfortunately exactly the perspective we see on the night sky: most stars are in the bright band we call the milky way, and we only see the outliers.) Among all possible projections, we attempt to find the one with the highest spread across the scatter plot. This projection may not be (and usually isn't) orthogonal to any axis; it may be a projection to an arbitrary plane.

\clearpage
\newpage

We again talk about two dimensional projection only for the sake of illustration. Imagine that we have ten thousand dimensional data and we would like, for some reason, keep just ten features. Yes, we can rank the features and keep the most informative, but what if these are correlated and tell us the same thing? Or what if our data does not have any target variable: with what should the "good features" be correlated? And what if the optimal projection is not aligned with the axes at all, so "good" features are combinations of the original ones?

We can do the same reasoning as above: we want to find a 10-dimensional (for the sake of examples) projection in which the data  points are as spread as possible.

How do we do this? Let's go back to our everyday's three dimensional world and think about how to find a two-dimensional projection.

Imagine you are observing a swarm of flies; your data are their exact coordinates in the room, so the position of each fly is described by three numbers. Then you discover that your flies actually fly in a formation: they are (almost) on the same line. You could then describe the position of each fly with a single number that represents the fly's position along the line. Plus, you need to know where in the space the line lies. We call this line the first principal component. By using it, we reduce the three-dimensional space into a single dimension.

After some careful observation, you notice the flies are a bit spread in one other direction, so they do not fly along a line but along a band. Therefore, we need two numbers, one along the first and one along the — you guessed it — second principal component.

It turns out the flies are actually also spread in the third direction. Thus you need three numbers after all.

Or do you? It all depends on how spread they are in the second and in the third direction. If the spread along the second is relatively small in comparison with the first, you are fine with a single dimension. If not, you need two, but perhaps still not three.

Let's step back a bit: why would one who carefully measured expressions of ten thousand genes want to throw most data away and reduce it to a dozen dimensions? The data, in general, may not and does not have as many dimensions as there are features. Say you have an experiment in which you spill different amounts of two chemicals over colonies of amoebas and then measure the expressions of 10.000 genes. Instead of flies in a three-dimensional space, you now profile colonies in a 10,000-dimensional space, the coordinates corresponding to gene expressions. Yet if expressions of genes depend only on the concentrations of these two chemicals, you can compute all 10,000 numbers from just two. Your data is then just two-dimensional.

\newpage

A technique that does this is called Principle Components Analysis, or \widget{PCA}. The corresponding widget is simple: it receives the data and outputs the transformed data.

\begin{wrapfigure}{o}{0.6\textwidth}
    \includegraphics[width=\linewidth]{workflow-scatterplot.png}
    \label{fig:wf1}
\end{wrapfigure}

The widget allows you to select the number of components and helps you by showing how much information (technically: explained variance) you retain with respect to the number of components (brownish line) and the amount of information (explained variance) in each component.

\begin{figure}[h]
    \centering
    \includegraphics[width=\linewidth]{pca-scree.png}
    \caption{$\;$}
\end{figure}

The PCA on the left shows the scree diagram for brown-selected data. Set like this, the widget replaces the 80 features with just seven - and still keeping 82.7\% of information. (Note: disable "Normalize data" checkbox to get the same picture.) Let us see a scatter plot for the first two components.

\begin{figure}[h!]
    \centering
    \includegraphics[width=\linewidth]{scatterplot-gray.png}
    \caption{$\;$}
\end{figure}

\newpage

The axes, PC1 and PC2, do not correspond to particular features in the original data, but to their linear combination. What we are looking at is a projection onto the plane, defined by the first two components. When you consider only two components, you can imagine that PCA puts a hyperplane into multidimensional space and projects all data into it.

Note that this is an unsupervised method: it does not care about the class. The classes in the projection may be well separated or not. Let's add some colors to the points and see how lucky we are this time.

\begin{figure}[h]
    \centering
    \includegraphics[width=\linewidth]{scatterplot-color.png}
    \caption{$\;$}
\end{figure}

The data separated so well that these two dimensions alone may suffice for building a good classifier. No, wait, it gets even better. The data classes are separated well even along the first component. So we should be able to build a classifier from a single feature!

\begin{figure*}[h]
    \centering
    \newcommand{\wf}{\includegraphics[scale=0.6]{workflow-testandscore.png}}
    \newcommand{\distrib}{\includegraphics[scale=0.6]{distributions.png}}
    \infinitewidthbox{
    \stackinset{r}{-0.4\linewidth}{t}{+0.00\linewidth}{\distrib}{\wf}\hspace{8cm}
    }
\end{figure*}

\newpage

In the above schema we uses the ordinary Test and Score widget, but renamed it to "Test on original data" for better understanding of the workflow.

On the original data, logistic regression gets 98\% AUC and classification accuracy. If we select just a single component in PCA, we already get a 93\%, and if we take two, we get the same result as on the original data.

PCA is thus useful for multiple purposes. It can simplify our data by combining the existing features to a much smaller number of features without losing much information. The directions of these features may tell us something about the data. Finally, it can find us good two-dimensional projections that we can observe in scatter plots.


%!TEX root = main.tex
\section{Introduction}
Artificial neural networks have dozends of hyperparameters which influence
their behaviour during training and evaluation time. One parameter is the
choice of activation functions. While in principle every neuron could have a
different activation function, in practice networks only use two activation
functions: The softmax function for the output layer in order to obtain a
probability distribution over the possible classes and one activation function
for all other neurons.

Activation functions should have the following properties:
\begin{itemize}
    \item \textbf{Non-linearity}: A linear activation function in a simple feed
          forward network leads to a linear function. This means no matter how
          many layers the network uses, there is an equivalent network with
          only the input and the output layer. Please note that \glspl{CNN} are
          different. Padding and pooling are also non-linear operations.
    \item \textbf{Differentiability}: Activation functions need to be
          differentiable in order to be able to apply gradient descent. It is
          not necessary that they are differentiable at any point. In practice,
          the gradient at non-differentiable points can simply be set to zero
          in order to prevent weight updates at this point.
    \item \textbf{Non-zero gradient}: The sign function is not suitable for
          gradient descent based optimizers as its gradient is zero at all
          differentiable points. An activation function should have infinitely
          many points with non-zero gradient.
\end{itemize}

One of the simplest and most widely used activation functions for \glspl{CNN}
is \gls{ReLU}~\cite{AlexNet-2012}, but others such as
\gls{ELU}~\cite{clevert2015fast}, \gls{PReLU}~\cite{he2015delving}, softplus~\cite{7280459}
and softsign~\cite{bergstra2009quadratic} have been proposed.

Activation functions differ in the range of values and the derivative. The
definitions and other comparisons of eleven activation functions are given
in~\cref{table:activation-functions-overview}.

\section{Important Differences of Proposed Activation Functions}
Theoretical explanations why one activation function is preferable to another
in some scenarios are the following:
\begin{itemize}
    \item \textbf{Vanishing Gradient}: Activation functions like tanh and the
          logistic function saturate outside of the interval $[-5, 5]$. This
          means weight updates are very small for preceding neurons, which is
          especially a problem for very deep or recurrent networks as described
          in~\cite{bengio1994learning}. Even if the neurons learn eventually,
          learning is slower~\cite{AlexNet-2012}.
    \item \textbf{Dying ReLU}: The dying \gls{ReLU} problem is similar to the
          vanishing gradient problem. The gradient of the \gls{ReLU} function
          is~0 for all non-positive values. This means if all elements of the
          training set lead to a negative input for one neuron at any point in
          the training process, this neuron does not get any update and hence
          does not participate in the training process. This problem is
          addressed in~\cite{maas2013rectifier}.
    \item \textbf{Mean unit activation}: Some publications
          like~\cite{clevert2015fast,BatchNormalization-2015} claim that mean
          unit activations close to 0 are desirable. They claim that this
          speeds up learning by reducing the bias shift effect. The speedup
          of learning is supported by many experiments. Hence the possibility
          of negative activations is desirable.
\end{itemize}

Those considerations are listed
in~\cref{table:properties-of-activation-functions} for 11~activation functions.
Besides the theoretical properties, empiric results are provided
in~\cref{table:CIFAR-100-accuracies-activation-functions,table:CIFAR-100-timing-activation-functions}.
The baseline network was adjusted so that every activation function except the
one of the output layer was replaced by one of the 11~activation functions.

As expected, \gls{PReLU} and \gls{ELU} performed best. Unexpected was that the
logistic function, tanh and softplus performed worse than the identity and it
is unclear why the pure-softmax network performed so much better than the
logistic function.
One hypothesis why the logistic function performs so bad is that it cannot
produce negative outputs. Hence the logistic$^-$ function was developed:
\[\text{logistic}^{-}(x) = \frac{1}{1+ e^{-x}} - 0.5\]
The logistic$^-$ function has the same derivative as the logistic function and
hence still suffers from the vanishing gradient problem.
The network with the logistic$^-$ function achieves an accuracy which is
\SI{11.30}{\percent} better than the network with the logistic function, but is
still \SI{5.54}{\percent} worse than the \gls{ELU}.

Similarly, \gls{ReLU} was adjusted to have a negative output:
\[\text{ReLU}^{-}(x) = \max(-1, x) = \text{ReLU}(x+1) - 1\]
The results of \gls{ReLU}$^-$ are much worse on the training set, but perform
similar on the test set. The result indicates that the possibility of hard zero
and thus a sparse representation is either not important or similar important as
the possibility to produce negative outputs. This
contradicts~\cite{glorot2011deep,srivastava2014understanding}.

A key difference between the logistic$^-$ function and \gls{ELU} is that
\gls{ELU} does neither suffers from the vanishing gradient problem nor is its
range of values bound. For this reason, the S2ReLU activation function, defined
as
\begin{align*}
  \StwoReLU(x) &= \ReLU \left (\frac{x}{2} + 1 \right ) - \ReLU \left (-\frac{x}{2} + 1 \right)\\
  &=
  \begin{cases}-\frac{x}{2} + 1 &\text{if } x \le -2\\
               x &\text{if } -2\le x \le 2\\
               \frac{x}{2} + 1&\text{if } x > -2\end{cases}
\end{align*}
This function is similar to SReLUs as introduced in~\cite{jin2016deep}. The
difference is that S2ReLU does not introduce learnable parameters. The S2ReLU
was designed to be symmetric, be the identity close to zero and have a smaller
absolute value than the identity farther away. It is easy to compute and easy to
implement.

Those results --- not only the absolute values, but also the relative
comparison --- might depend on the network architecture, the training
algorithm, the initialization and the dataset. Results for MNIST can be found
in~\cref{table:MNIST-accuracies-activation-functions} and for HASYv2
in~\cref{table:HASYv2-accuracies-activation-functions}. For both datasets, the
logistic function has a much shorter training time and a noticeably lower test
accuracy.

\glsunset{LReLU}
\begin{table}[H]
    \centering
    \begin{tabular}{lccc}
    \toprule
    \multirow{2}{*}{Function} & Vanishing  & Negative Activation & Bound \\
                  & Gradient       & possible & activation \\\midrule
    Identity      & \cellcolor{green!25}No    & \cellcolor{green!25}  Yes    & \cellcolor{green!25}No  \\
    Logistic      & \cellcolor{red!25} Yes    & \cellcolor{red!25}   No      & \cellcolor{red!25}  Yes \\
    Logistic$^-$  & \cellcolor{red!25} Yes    & \cellcolor{green!25}  Yes    & \cellcolor{red!25}  Yes \\
    Softmax        & \cellcolor{red!25} Yes    & \cellcolor{green!25}  Yes    & \cellcolor{red!25}  Yes \\
    tanh          & \cellcolor{red!25} Yes    & \cellcolor{green!25}  Yes    & \cellcolor{red!25}  Yes \\
    Softsign      & \cellcolor{red!25} Yes    & \cellcolor{green!25}Yes      & \cellcolor{red!25}   Yes \\
    ReLU          & \cellcolor{yellow!25}Yes\footnotemark & \cellcolor{red!25} No & \cellcolor{yellow!25}Half-sided \\
    Softplus      & \cellcolor{green!25}No    & \cellcolor{red!25}   No      & \cellcolor{yellow!25}Half-sided \\
    S2ReLU        & \cellcolor{green!25}No    & \cellcolor{green!25}Yes      & \cellcolor{green!25} No \\
    \gls{LReLU}/PReLU   & \cellcolor{green!25}No    & \cellcolor{green!25}Yes      & \cellcolor{green!25} No \\
    ELU           & \cellcolor{green!25}No    & \cellcolor{green!25}Yes      & \cellcolor{green!25} No \\
    \bottomrule
    \end{tabular}
    \caption[Activation function properties]{Properties of activation functions.}
    \label{table:properties-of-activation-functions}
\end{table}
\footnotetext{The dying ReLU problem is similar to the vanishing gradient problem.}

\begin{table}[H]
    \centering
    \begin{tabular}{lccclllll}
    \toprule
    \multirow{2}{*}{Function} & \multicolumn{2}{c}{Inference per}                                & Training                            & \multirow{2}{*}{Epochs} & Mean total        \\\cline{2-3}
                              & 1 Image                        & 128                             & time                                &                         & training time     \\\midrule
    Identity                  & \SI{8}{\milli\second}          & \SI{42}{\milli\second}          & \SI{31}{\second\per\epoch}          & 108 -- \textbf{148}     &\SI{3629}{\second} \\
    Logistic                  & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \SI{24}{\second\per\epoch}          & \textbf{101} -- 167     &\textbf{\SI{2234}{\second}} \\
    Logistic$^-$              & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \textbf{\SI{22}{\second\per\epoch}} & 133 -- 255              &\SI{3421}{\second} \\
    Softmax                   & \SI{7}{\milli\second}          & \SI{37}{\milli\second}          & \SI{33}{\second\per\epoch}          & 127 -- 248              &\SI{5250}{\second} \\
    Tanh                      & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \SI{23}{\second\per\epoch}          & 125 -- 211              &\SI{3141}{\second} \\
    Softsign                  & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \SI{23}{\second\per\epoch}          & 122 -- 205              &\SI{3505}{\second} \\
    \gls{ReLU}                & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \SI{23}{\second\per\epoch}          & 118 -- 192              &\SI{3449}{\second} \\
    Softplus                  & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \SI{24}{\second\per\epoch}          & \textbf{101} -- 165     &\SI{2718}{\second} \\
    S2ReLU                    & \textbf{\SI{5}{\milli\second}} & \SI{32}{\milli\second}          & \SI{26}{\second\per\epoch}          & 108 -- 209              &\SI{3231}{\second} \\
    \gls{LReLU}               & \SI{7}{\milli\second}          & \SI{34}{\milli\second}          & \SI{25}{\second\per\epoch}          & 109 -- 198              &\SI{3388}{\second} \\
    \gls{PReLU}               & \SI{7}{\milli\second}          & \SI{34}{\milli\second}          & \SI{28}{\second\per\epoch}          & 131 -- 215              &\SI{3970}{\second} \\
    \gls{ELU}                 & \SI{6}{\milli\second}          & \textbf{\SI{31}{\milli\second}} & \SI{23}{\second\per\epoch}          & 146 -- 232              &\SI{3692}{\second} \\
    \bottomrule
    \end{tabular}
    \caption[Activation function timing results on CIFAR-100]{Training time and
             inference time of adjusted baseline models trained with different
             activation functions on GTX~970 \glspl{GPU} on CIFAR-100. It was
             expected that the identity is the fastest function. This result is
             likely an implementation specific problem of Keras~2.0.4 or
             Tensorflow~1.1.0.}
    \label{table:CIFAR-100-timing-activation-functions}
\end{table}

\begin{table}[H]
    \centering
    \begin{tabular}{lccccc}
    \toprule
    \multirow{2}{*}{Function} & \multicolumn{2}{c}{Single model}              & Ensemble & \multicolumn{2}{c}{Epochs}\\\cline{2-3}\cline{5-6}
                              & Accuracy             & std                    & Accuracy & Range & Mean \\\midrule
    Identity                  & \SI{99.45}{\percent} & $\sigma=0.09$          & \SI{99.63}{\percent} & 55 -- \hphantom{0}77  & 62.2\\%TODO: Really?
    Logistic                  & \SI{97.27}{\percent} & $\sigma=2.10$          & \SI{99.48}{\percent} & \textbf{37} -- \hphantom{0}76  & \textbf{54.5}\\
    Softmax                   & \SI{99.60}{\percent} & $\boldsymbol{\sigma=0.03}$& \SI{99.63}{\percent} & 44 -- \hphantom{0}73  & 55.6\\
    Tanh                      & \SI{99.40}{\percent} & $\sigma=0.09$          & \SI{99.57}{\percent} & 56 -- \hphantom{0}80  & 67.6\\
    Softsign                  & \SI{99.40}{\percent} & $\sigma=0.08$          & \SI{99.57}{\percent} & 72 -- 101             & 84.0\\
    \gls{ReLU}                & \textbf{\SI{99.62}{\percent}} & \textbf{$\sigma=0.04$} & \textbf{\SI{99.73}{\percent}} & 51 -- \hphantom{0}94 & 71.7\\
    Softplus                  & \SI{99.52}{\percent} & $\sigma=0.05$          & \SI{99.62}{\percent} & 62 -- \hphantom{0}\textbf{70}  & 68.9\\
    \gls{PReLU}               & \SI{99.57}{\percent} & $\sigma=0.07$          & \textbf{\SI{99.73}{\percent}} & 44 -- \hphantom{0}89 & 71.2\\
    \gls{ELU}                 & \SI{99.53}{\percent} & $\sigma=0.06$          & \SI{99.58}{\percent} & 45 -- 111 & 72.5\\
    \bottomrule
    \end{tabular}
    \caption[Activation function evaluation results on MNIST]{Test accuracy of
             adjusted baseline models trained with different activation
             functions on MNIST.}
    \label{table:MNIST-accuracies-activation-functions}
\end{table}
\glsreset{LReLU}


%!TEX root = ../thesis.tex

\chapter{Conclusions and recommendation}
\section{Summary}
\paragraph{}
In this thesis, a systematic numerical method where all procedures are conducted without human involvement for an arbitrary geometric input in both 2D and 3D situations has been developed based on the SBFEM.
The SBFEM is a semi-analytical method which combines the main advantages of the finite element method and the boundary element method but also has unique features of its own.
In contrast to the FEM, only the boundary is discretized using the conventional FEM interpolating function which leads to a decline in the number of unknowns.
It also allows solving the problem involving bimaterial interfaces and crack faces without the discretization of them.
Compared to the BEM, the fundamental solution is no longer required.
The infinite boundary can be achieved naturally as the radiation condition at infinity is satisfied in the SBFEM .
As only the boundary information is required in SBFEM and hence provides a seamless integration with the CAD modeling compared to conventional Isogeometric Method.
The discretization of the boundary can be based on the standard finite element interpolation functions for computational efficiency or NURBS basis function for exact geometry.

\paragraph{}
The preprocessing of the proposed method is conducted automatically without losing exact geometric representation.
In 2D problems, the geometric information from the IGES file exported from CAD design is extracted and a quad-tree based mesh will be generated automatically.
High mesh quality and reduced computational cost in the calculation of the element stiffness matrix can be expected.
For 3D cases, both of the STL and the IGES files are exported from the CAD design and an octree based mesh will be generated based on STL file.
Since the STL file represents the triangulation of the geometric surfaces, intersections calculated during cutting stage is supposed to be located on the triangular plane instead of the original surface.
As a consequence, a point projection algorithm is adopted to move the points back to the NURBS surfaces after the mesh has been generated.
Then, an arbitrary geometric shape in 2D and 3D can be meshed automatically with high quality mesh and solved by the SBFEM.
The geometric exact can also be achieved by replacing the distance calculation with finding that between the edge of the element and the NURBS surface directly.

\paragraph{}
After the solution is determined from the SBFEM, a mesh refinement may be necessary to improve the accuracy and check the convergence.
In order to find out the scaled boundary finite element which improves the accuracy significant after refinement, an adaptive mesh refinement algorithm is required.
Expressions related to the eigenvalues of the SBFEM formulation are adopted as the physical error indicator to prevent extra work such as stress recovery.
Some key geometric error estimator including area of the subdomain, minimal angel and so on are also included.
The decision based on numerous error estimators are conducted by the help of a machine learning algorithm.
The proposed adaptive mesh refinement exhibits higher rate of convergence compared to an uniform mesh refinement using SBFEM.

\paragraph{}
The thesis commences with an introduction to the research topic in Chapter 1.
A background and motivation for the research are presented followed by the objective and outline of the research.
In Chapter 2, the linear theory of the Isogeometric analysis, together with a brief introduction on the NURBS and its mathematical backgrounds, potentials and limitations are summarized.
A brief introduction on the IGES file is also presented.
Due to the dimensional mismatch between the FEM and the CAD, the SBFEM is the technique which can provide a seamless integration with the CAD modeling.
The concept of adaptive mesh refinement and its limitation is presented as well.
The MLP then is introduced to overcome this limitation by the adoption of multiple error indicators.
Finally, algorithms used to generate octree mesh in 3D from STL file is presented.

\paragraph{}
In Chapter 3, the NURBS basis functions are employed to approximate the unknown fields in the circumferential direction within the framework of the SBFEM.
The accuracy, effectiveness and the convergence properties of the proposed method are demonstrated with benchmark problems in linear elasticity and linear elastic fracture mechanics.
From the numerical studies, it can be observed that the NURBS basis functions yield superior accuracy when compared to Lagrange basis functions of the same order.
The proposed method overcomes the disadvantages of both the isogeometric finite element analysis and the isogeometric boundary element method.
When applied to problems with singularities, the proposed method does not require additional functions to span the solution space.
Moreover, the proposed framework does not require internal discretization to study the dynamic response at high frequencies.
However, for complicated geometries, to meet the star convexity, subdivision into smaller sub-domains is required.

\paragraph{}
In Chapter 4, the IGES file is employed directly from the CAD output to represent the geometry during the preprocessing.
The proposed methods provides a systematic and automatic mesh generation algorithm  where high quality mesh is produced efficiently.
The CAD design file can be used directly and the exact geometry can also be retained, which largely reduce the human efforts involved in numerical analysis.
Moreover, it helps to reduce the analytical error as the difference in the geometric representation is minimized.
Computational efficiency is improved via utilizing the pattern of the quadtree to prevent repeated calculation.
Points projection algorithm is accelerated using the strong convex hull property of the NURBS curve.
The accuracy, effectiveness and the convergence properties of the proposed method are demonstrated with benchmark problems in linear elasticity mechanics.
From the numerical studies, it can be observed that the quadtree mesh yield better accuracy when compared to uniform mesh with same degree of freedoms.

\paragraph{}
Chapter 5 adopts a machine learning algorithm to develop an extensible and flexible error indicator.
Any other error estimators can be added to the existing framework and their effects can be detected based on the performance indicators in machine learning.
A MLP trained error estimator that concludes expressions related to the eigenvalues of the SBFEM formulation and some key geometric properties of the scaled boundary finite element gives a higher convergence rate compared to the uniform refinement.
In order to improve the learning effectiveness of the MLP, regularization methods including bagging and dropout are utilized.
Due to the lack of the eigenvalue error indicator in the first order triangular element, method that eliminates these situation is developed.
A matrix representation of the NURBS curves is presented to achieve a higher efficiency and stability in calculating the intersections between edge of the element and the NURBS curve.

\paragraph{}
In Chapter 6, a projection is conducted after the mesh is generated from STL file using octree or calculating the intersection between the edge of the element and the original NURBS surface to retain the exact geometry.
In order to utilize the strong hull property of the NURBS surface, a quick hull algorithm in 3D is introduced.
As a consequence, the computational efficiency of the points projection and the intersection calculation are improved significantly.
The proposed method is able to generate the mesh from an arbitrary geometric input and shows higher accuracy compared to conventional method.

\section{Future work}
\paragraph{}
As discussed in Chapter 3, there is no exact numerical integration for the NURBS basis function at the moment as its rational property currently.
Furthermore, the computational cost of calculating a value of NURBS basis function can be $10$ times higher than that of the traditional Legendre polynomials.
In chapters after Chapter 3, the conventional shape functions are adopted instead of the NURBS basis functions to avoid too many expensive computational operation when the number of the elements grow drastically.
Future work can be performed on derive an exact and efficient numerical integration quadrature to prevent extra error.
Algorithm that evaluate the NURBS basis function can also be optimized so that the whole programme can be much more efficient as it is involved in almost all NURBS operation.

\paragraph{}
As discussed in Chapter 4, one of the most outstanding advantages of using machine learning algorithm to train the model responsible for making decision whether an element need to be refined lies in the allowance of unlimited number of error indicators.
The effectiveness of the deep neural network is closely related to the quantity of the training data.
In Chapter 4, less than a thousand of the training data is used, which limits the depth of the neural network and the number of the error indicators.
Future work can concentrate on building a platform so the cloud computing can be utilized and the service can be available to the whole community.
By having a platform, number of the training data can be solved naturally as any request of calculation contributes to one extra training data.
A large training set then motive the scholars adding any error indicator.


Parametrized, ring valued
Tutte equations and functions (with empty port set), and
the universal ported Tutte invariant (called the big Tutte
polynomial by Las Vergnas) have been studied.  So has
the (reduced) Laplacian matrix's determinant for a graph--This
determinant, according to the famous matrix tree theorem, equals
the number of bases in the corresponding graphic matroid.  It is
well-known that the number of bases in a Tutte invariant defined
for all matroids.  More strongly, the following homogeneous generating
function for the bases
\[
F(\mathcal{N}(\emptyset,E))=
\sum_{B\in\mathcal{B}(\mathcal{N})}
\left(\prod_{e\in B} g_e
\right)
\left(\prod_{e\in E\setminus B} r_e
\right)
=
\sum_{B\in\mathcal{B}(\mathcal{N})}
g_B\;\;r_{\scomp{B}}
\]
is a Tutte function (with $P=\emptyset$).
These background topics will be surveyed and extended as needed.

This paper's main topic is a parametrized function from certain
(ported) extensors to extensors.  An extensor is a
(fully or completely\footnote{We will follow authors such as
\cite{JacobsonI} in omitting this qualifier for such decomposible elements})
decomposible element in an exterior algebra (i.e., algebra of antisymmetric
tensors).  Our main result (Theorem \textbf{FILLIN}) is that
if extensor deletion and contraction are defined in a way that is
consistant with our function definition, then our function obeys
identities in exterior algebra that are analogs of the Tutte
equations.

One corollary applies to the class of ported unimodular oriented
matroids:  Our function, restricted to the unimodular extensors
which represent this class, yields an extensor-valued Tutte function
that sometimes distinguishes different orientations of the same
matroid.

When the corollary is applied with the
parameters all set to $1$, we get new Tutte-like invariants
that can sometimes distinguish different orientations of the same matroid.

\subsection{Ports and Parameters}

The idea to restrict deletion/contraction decomposition so it does
not apply to some distinguished elements
is a natural one.  It was applied to invariants of
matroid morphisms (or strong maps) by Las
Vergnas\cite{MR0419272,SetPointedLV,EtienneLasVergnasMorphismVectorial}.
The equations for
ported Tutte invariants (i.e., $F$ with the $r_e=g_e=1$ and
invariant under port-preserving matroid isomorphisms) determine
the
\textbf{big Tutte polynomial} defined by Las Vergnas in \cite{SetPointedLV}.
Whereas the traditional Tutte polynomial has only two variables,
which correspond to the isomorphism classes of the indecomposible
loop and coloop matroids, the generalization adds additional
variables corresponding to the additional indecomposible matroids
on sets of port elements.

Our motivation there and here
comes from distinguishing as ports some edges
in a directed graph model of an electrical network.  The ports
model two-terminal plugs or sockets where the network interacts
with an enviroment; the other edges model resistors.

In this paper we add the observation that if ported Tutte
decomposition is done on an \emph{oriented} matroid, the indecomposible
minors include connected \emph{oriented} matroids on sets of port elements.
Therefore, if $\mathcal{N}_1$ and $\mathcal{N}_2$ are different orientations
of the same orientable matroid, then
$F(\mathcal{N}_1)\neq F(\mathcal{N}_2)$ is possible for a ported
Tutte function $F$.


% !TEX root = ../thesis.tex

\chapter{Beam Propagation using the Hankel Transform}
% \label{cha:beam_propagation}

In this chapter I will briefly show how the trapping geometry presented before could be realised in an experiment.
The two ring beams can be produced with axicon lenses. I have programmed a simple beam propagation for cylindrical beams. This tool can be found on my personal GitHub page\footnote{\url{github.com/AvonHaaren}} and was programmed in \Cpp and Python.

\section{Theory -- Fourier Optics}
Fourier optics in a mathematical approach to the propagation of light through free space and variuos media. In Fourier optics, the light field is decomposed into a sum of plane waves, each of which can be propagated through an arbitrary linear optical system independently. Afterwards, the components are superimposed to give the light field after the system. In two dimensions, this is done by using the Fourier transform
\begin{align}
    \mathcal{F}\left\{f(x,y)\right\} = F(\nu_x,\nu_y) &= \iint \diff{x} \diff{y}\; f(x,y) \exp\!\left(2\pi i (x\nu_x + y\nu_y)\right) \\
    f(x,y) &= \iint \diff{\nu_x} \diff{\nu_y}\; F(\nu_x,\nu_y) \exp\!\left(-2\pi i (x\nu_x + y\nu_y)\right)
\end{align}
In the case of free space propagation, the transfer function $H(\nu_x,\nu_y,\Delta z)$ is fully expressed by the spatial frequencies $\nu_x$ and $\nu_y$. Using this, we can express the electric field after a distance $\Delta z$ through
\begin{align}
    E(x,y,\Delta z) = \iint \diff{\nu_x} \diff{\nu_y}\; H(\nu_x,\nu_y,\Delta z) \mathcal{F}\left\{E(x,y,0)\right\} \exp\!\left(-2\pi i (x\nu_x + y\nu_y)\right)
\end{align}

\subsection{Cylindrical Symmetry}
If the light field and optical system are cylindrically symmetric, the Fourier transform in the equation above can be replaced by the Hankel transform of zero order:
\begin{align}
    G(\rho) &= 2\pi\int_0^\infty \diff{r}\; g(r) r J_0(2\pi r \rho) \\
    g(r) &= 2\pi \int_0^\infty \diff{\rho}\; G(\rho) \rho J_0(2\pi r \rho)
\end{align}
Here, $g(r)$ is the field in real space, $G(\rho)$ the transformed field in reciprocal space and $J_0$ is the Bessel function of zero order. $\rho$ is the spatial frequency in radial direction, where $2\pi\rho = \kappa$ is the radial wavenumber. For numerical calculations, this reduction to a one-dimensional problem is highly beneficial.

\subsection{Transfer Functions of Optical Elements}
\paragraph{Spherical Lens} ~\\
The transfer function of a spherical lens is given by
\begin{equation*}
    H_\text{TL}(r) = \exp\!\left(-i\pi\frac{r^2}{f\lambda}\right)
\end{equation*}
where $f$ is the focal length and $\lambda$ the wavelength of the light.

\paragraph{Axicon} ~\\
An axicon has a transfer function
\begin{equation*}
    H_\text{A}(r) = \exp\!\left(-\frac{2\pi r}{\lambda}(n - 1)\tan(\alpha)\right)
\end{equation*}
where $n$ is the refractive index of the axicon and $\alpha$ is the opening angle, as shown in \cref{fig:axicon}. A constant phase factor depending on the thickness of the axicon has been omitted.
\begin{figure}[htbp]
    \centering
    \input{TexContents/Figures/Evap/Axicon/axicon.tikz}
    \caption{Axicon Lens}
    \label{fig:axicon}
\end{figure}

\paragraph{Free Space} ~\\
The free space transmission function depends only on the coordinates in reciprocal space:
\begin{equation}
    H_\text{FS}(\rho, \Delta z) = \exp\!\left(\frac{2\pi i \Delta z}{\lambda}\sqrt{1 - (\lambda \rho)^2}\right)
\end{equation}
Notably, for $\rho^{-1} > \lambda$, transmission is exponentially suppressed. This expresses the fact that structures smaller than a wavelength are not resolvable.

\subsection{Numerical Evaluation}
To calculate the Hankel transform and its inverse, we discretise real and reciprocal space up to maximum values $r_\text{max}$ and $\rho_\text{max}$ in regular intervals with spacings $\delta r$ and $\delta \rho$ given by
\[
    \delta r = \frac{r_\text{max}}{N}, \quad \delta \rho = \frac{\rho_\text{max}}{N}
\]
where $N$ is the grid size.
Then we have
\begin{align}
    G(\rho) &= 2\pi \delta r \sum_{i = 0}^N r_i  g(r_i) J_0(2\pi r_i \rho) \\
    g(r) &= 2\pi \delta \rho \sum_{i = 0}^N \rho_i  G(\rho_i) J_0(2\pi r \rho_i)
\end{align}
To save computation time, the Bessel function can be pre-evaluated on a $N\times N$ grid. However, at high resolutions, the memory consumption of this becomes too large, also resulting in higher cache miss rates. We therefore only evaluate the Bessel function on a grid of size $M$ with maximum $2\pi r_\text{max} \rho_\text{max}$. Interpolation between these values is much faster than calculating the exact value of $J_0(x)$ in the transformation.

\section{Tested Setup and Results}
To test an optical system with an arbitrary number of elements, the initial electric field is propagated to the $n$-th optical element using the Hankel transform and the free space propagator. At the location of the optical element, the field is transformed back into real space, the transfer function of the element is applied and then the field can be propagated to the next element etc.

We adapt the setup from Hueck et al. \cite{PhysRevLett.120.060402}, using one axicon \cite{McLeod} to split the initial gaussian into a ring, another axicon in combination with a long focal length spherical lens to invert it and then a third, movable axicon to produce a collimated ring that can have a varying radius. This is shown in \cref{fig:axicon_setup}. In the setup that we want to use and that is simulated, there is an additional $5\times$ demagnification stage after the third axicon.
\begin{figure}[htbp]
    \centering
    \input{TexContents/Figures/Evap/AxiconSetup/AxiconSetupNew.tikz}
    \caption[Optical setup to create a ring beam with dynamically adjustable radius]{Adapted from \cite{axiconSM}. Optical setup to create a ring beam with dynamically adjustable radius. Not shown here is an additional microscope that demagnifies the beam from the image plane to the target plane.}
    \label{fig:axicon_setup}
\end{figure}
The position of the first axicon is defined as $z=0$. The \SI{400}{mm} lens is positioned at $z=\SI{85}{mm}$ with the second axicon behind it at $z=\SI{95}{mm}$. The third axicon can be moved between $z=\SI{585}{mm}$ and $z=\SI{625}{mm}$ \todo{check if correct}.
The initial gaussian beam has a wavelength of \SI{770}{nm} the position of its waist relative to $z=0$ as well as the waist size can be varied.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=\textwidth]{Evap/AxiconBeam/plot}
    \caption{Example of a beam resulting from the setup above. \textbf{PRELIMINARY, WILL BE SWAPPED OUT FOR THE FINAL VERSION}}
    % \label{fig:axiconBeam}
\end{figure}

\begin{figure}[htbp]
    \centering
    \begin{subfigure}[t]{0.49\textwidth}
        \centering
        \missingfigure[figwidth=.9\linewidth]{Closeup of the focus region}
        \caption{Close-up image of the beam in the focus region}
    \end{subfigure}
    \begin{subfigure}[t]{0.49\textwidth}
        \centering
        \missingfigure[figwidth=.9\linewidth]{Intensity Plot at the focus, with fitted power law}
        \caption{Resulting ring intensity at the focus position. A power-law curve has been fitted to the data. \textbf{VALUE MISSING}}
    \end{subfigure}
    \caption{Ring beam at the focus position with a resulting diameter of \textbf{VALUE MISSING}.}
    % \label{fig:axiconBeamResult1}
\end{figure}


\section{Appendix}

\subsection{VWM Network model}

\begin{notation}
	Treat 1D tensors as column vectors and 2D tensors as matrices, where appropriate.
	We use lower case to represent both 1D and 2D tensors but occasionally use upper case
	for 2D tensors where matrix operations are involved.

	\begin{enumerate}
		\item Let
		$\Delta^d = \{ (x_0, x_1, \dots, x_d) : x_0 + x_1 + \dots + x_d = 1, x_i \ge 0, i = 0, 1, \dots, d\}$ denote the standard $d$-simplex.

		\item  Let $\circ$ denote concatenation of two tensors with identical shape except possibly
		for their last dimensions $d_1$ and $d_2$, respectively,
		resulting in a tensor with last dimension of $d_1+d_2$.

		\item Let $\odot$ denote  element-wise product of two tensors of same shape,
		i.e., Hadamard product for vectors/matrices.

		\item Let $\otimes$ denote tensor product of two tensors,
		i.e. Kronecker product for vectors/matrices.
	\end{enumerate}
\end{notation}

\subsection{Basic layers/modules}

\colorbx{Linear (Affine) Layer}

\begin{description}
	\item[Inputs:] A tensor $x$ with last dimension $n$.
	\item[Parameter:] An affine function $\cG: \Reals^n \to \Reals^m$ with
	weight and bias parameters.

	\item[Output:] A tensor $y$ with last dimension $m$, and remaining dimensions
	same as that of $x$, obtained by applying $\cG$ to each 1D slice of $x$
	along the last dimension.
\end{description}

\colorbx{Attention Module}

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Query: $q \in \Reals^d$
		\item Keys: $K \in \Reals^{N \times d}$
		\item Values: $V \in \Reals^{N \times d}$. By default $V=K$, unless mentioned explicitly.
	\end{enumerate}

	\item[Parameter:] Weight $w \in \Reals^d$

	\item[Outputs:]
	\begin{enumerate}
		\item[]
		\item Content vector: $h =  V^{\T} u \in \Reals^d$
		\item Attention vector:  $w = \softmax(K(w \odot q)) \in \Reals^N$
	\end{enumerate}
\end{description}

\colorbx{Interaction Module}

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Base object: $b \in \Reals^d$
		\item Feature objects: $f \in \Reals^{M \times d}$
	\end{enumerate}

	\item[Parameters:]
	\begin{enumerate}
		\item[]
		\item Base object projection linear layer: $\cG: \Reals^d \to \Reals^d$
		\item Feature objects projection linear layer : $\cK: \Reals^{M \times d} \to \Reals^{M \times d}$
		\item Modifier linear layer:  $\cH: \Reals^{M \times 2d} \to \Reals^{M \times d}$
	\end{enumerate}

	\item[Output:] Modified feature objects
	$f' =  \cH( \cK(f) \odot ( \vone \otimes \cG(b))) \in  \Reals^{M \times d}$
\end{description}

\hrulefill

\subsection{SAM cell}

The SAM recurrent cell is executed for $T$ reasoning steps for every frame in
the temporal order.  Within a single frame, the cell state at the end of each reasoning step
$t=1,2, \dots, T$ is denoted by $(c_t, M_t, o_t)$, where:
\begin{enumerate}
	\item $c_t \in \Reals^d$ is the control state;
	\item $M_t \in  \Reals^{N \times d}$ is the visual working memory with $N$ slots;
	\item $w_t \in  \Reals^N$ is the write head; and
	\item $so_t  \in \Reals^d$ is the summary visual object.
\end{enumerate}
The initial state is such that both $c_0$ and $so_0$ are initialized
to a fixed value at the start of each frame. However $M_0$ is initialized only once at
the start of the first frame and otherwise taken to be the value of $M_T$ at the
end of the previous frame.

%The number of slots $N$ for the SAM $M_t$ is not fixed because the neural network
%parameters do not depend on it. It can be variable across the different datasets used
%for training, validation and test as well as within each dataset. This, for example, enables a
%form of transfer learning where we can train on an easy dataset for one value of $N$
%and study its generalization to a hard dataset using a larger value of $N$.

\colorbx{Question-driven Controller}

%The Question-driven Controller plays an important role in the reasoning process.
%It drives the attention over the question and produces the new control states. Each new control state defines a new reasoning operation. The inputs of this unit are the past control state, the question encoding and the contextual words (see Question Encoding Unit). It uses the dot product attention between the contextual words
%and the combination of the past control states and the question encoding.  This attention layer produces the new control state.
%
%This unit also outputs the temporal class weights that will be used in the Reasoning Unit. It gives access to a temporal information for the current words (last, latest, now, none temporal).

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Reasoning step $t = 1,2, \dots, T$
		\item Previous control state: $c_{t-1} \in \Reals^d$
		\item Contextual words: $cw \in \Reals^{L \times d}$
		\item Question encoding: $q \in \Reals^d$
	\end{enumerate}

	\item[Parameters:]
	\begin{enumerate}
		\item[]
		\item Reasoning step-dependent linear layer: $\cG_t: \Reals^d \to \Reals^d$, depending on $s$
		\item Concatenation linear layer: $\cH: \Reals^{2d} \to \Reals^d$
		\item Attention module $\cA$
		\item Temporal classifier:  $\cK: \Reals^d \to \Delta^3$. A two-layer feedforward
		network with ELU activation in the hidden layer of $d$ units.
		The classes for the temporal context are labeled ``last'', ``latest'', ``now'', as well as
		a fourth class label ``none`` indicating no temporal context.
		If $\tau \in \Delta^3$ is the output of the classifier, we denote the components by
		$\tlast$, $\tlatest$, $\tnow$ and $\tnone$.
	\end{enumerate}

	\item[Outputs:]
	\begin{enumerate}
		\item[]
		\item Control state $c_t \in \Reals^d$
		\item Control attention $ca_t \in \Reals^L$
		\item Temporal class weights $\tau_t \in \Reals^4$
	\end{enumerate}

	\item[Equations:]
	\begin{enumerate}
		\item[]
		\item Modulation: $y = \cH\bigl([c_{t-1}, \cG_t(q)]\bigr)$
		\item Control state and attention: $c_t,  ca_t= \cA(y, cw)$
		\item Temporal classification: $\tau_t = \cK(c_t)$
	\end{enumerate}
\end{description}

\colorbx{Visual Retrieval Unit}

%The visual retrievial unit is responsible to extract visual information from the current image given a control state coming from the Question-driven Controller. It is first projecting the past summary object and the feature maps together using the interaction module.
%It is then using the attention module as follow. The query are the control states and the keys and are the feature maps coming from the image encoder. The results of this attention is applied on the modified features maps coming from the interaction module.
%This unit outputs the extracted object and the visual attention.

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Control state: $c_t \in \Reals^d$
		\item Previous summary object: $so_{t-1} \in \Reals^d$
		\item Feature map of current frame: $F \in \Reals^{H \times W \times d}$
	\end{enumerate}

	\item[Parameters:]
	\begin{enumerate}
		\item[]
		\item Interaction module $\cI$
		\item Attention module $\cA$
	\end{enumerate}

	\item[Outputs:]
	\begin{enumerate}
		\item[]
		\item Visual object: $vo_t \in  \Reals^d$
		\item Visual attention: $va_t  \in \Reals^{H \times W \times d}$
	\end{enumerate}

	\item[Equations:]
	\begin{enumerate}
		\item[]
		\item Modified feature map: $\hat{F} = \cI(so_{t-1}, F)$
		\item Visual object and attention: $vo_t, va_t = \cA(y, \hat{F}, M_{t-1})$
	\end{enumerate}
\end{description}

\begin{note}
	Appropriate flatten/unflatten operations are performed to match the signature
	of the modules.
\end{note}

\colorbx{Memory Retrieval Unit}

%The role of the memory retrieval unit is to read and extract object from memory).
%As the Visual Retrieval Unit, it uses the combination of the two following submodules. The interaction module blends together the extracted object and the content of the memory. The attention module then extract the corresponding object in memory if present. This unit outputs the extracted object and its corresponding location, we call it the "read head".

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Control state: $c_t \in \Reals^d$
		\item Previous summary object: $so_{t-1} \in  \Reals^d$
		\item Previous SAM $M_{t-1} \in \Reals^{N \times d}$
	\end{enumerate}

	\item[Parameters:]
	\begin{enumerate}
		\item[]
		\item Interaction module $\cI$
		\item Attention module $\cA$
	\end{enumerate}

	\item[Outputs:]
	\begin{enumerate}
		\item[]
		\item Memory object: $mo_t \in \Reals^d$
		\item Read head: $\rhead_t \in \Reals^N$
	\end{enumerate}

	\item[Equations:]
	\begin{enumerate}
		\item[]
		\item Modified SAM: $\hat{M}_t = \cI(so_{t-1}, M_{t-1})$
		\item Memory object and attention: $mo_t, \rhead_t = \cA(y, \hat{M}_t, M_{t-1})$
	\end{enumerate}
\end{description}

\colorbx{Reasoning Unit}

%control_state, visual_object, memory_object, temporal_class_weights

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Control state: $c_t \in \Reals^d$
		\item Visual object: $vo_t \in \Reals^d$
		\item Memory object: $mo_t \in \Reals^d$
		\item Temporal class weights $\tau \in \Delta^3$
	\end{enumerate}

	\item[Parameters:] Validator modules $\cG, \cK: \Reals^{2d} \to \Reals$.
	Both $\cG, \cK$ are two-layer networks of $2d$ hidden units,
	using ELU activation in the hidden layer, and sigmoid in the output layer.

	\item[Output:] Predicate gates for the current reasoning step
	\begin{enumerate}
		\item Object match predicate gates (i) image: $\imatch_t \in [0,1]$ and
		(ii) memory: $\mmatch_t \in [0,1]$.

		\item Memory update predicate gates (i) add: $\doadd_t \in [0,1]$ and
		(ii) replace: $\doreplace_t \in [0,1]$
	\end{enumerate}

%	\item[Equations:]
%	\begin{enumerate}
%		\item[]
%		\item $\imatch_t \in [0,1]$:
%		It's true if there is a valid visual object in the current temporal context.
%
%		\item $\mmatch_t \in [0,1]$:
%		It's true if there is a valid memory object in the current temporal context.
%
%		\item $\doadd_t \in [0,1]$:
%		It's true if the current visual object should be added as a new object to memory.
%
%		\item $\doreplace_t \in [0,1]$:
%		It's true if the current visual object should replace an existing object in memory.
%	\end{enumerate}
\end{description}

\colorbx{Memory Update Unit}

%This unit is meant to update the content of the memory.
%
%Three actions can happen:
%
%\begin{itemize}
%	\item There is no object to be added to memory, the memory remains unchanged
%	\item There is one object that needs to be added to memory, but a similar object is already in memory at a given location. The new object will replace the old object at this location
%
%	\item There is one object that needs to be added to memory, and  it is a new object. It is added at the write head location.
%\end{itemize}
%
%
%This module also updates the position of the write head. If a new object as been added to the current write head position, the right head shifts right to a new empty slot. If the object has been replaced, the write head doesn't move.
% visual_object, visual_working_memory, read_head, write_head, do_replace, do_add_new

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Visual object: $vo_t \in \Reals^d$
		\item Memory object: $mo_t \in \Reals^d$
		\item Memory update predicate gates:  $\doadd_t, \doreplace_t \in [0,1]$
		\item Read head: $\rhead_t \in \Reals^N$
		\item Previous SAM $M_{t-1} \in \Reals^{N \times d}$
		\item Previous write head: $\whead_{t-1} \in \Reals^N$
	\end{enumerate}

	\item[Outputs:]
	\begin{enumerate}
		\item[]
		\item SAM $M_t \in \Reals^{N \times d}$
		\item Read head: $\rhead_t \in \Reals^N$
		\item Write head: $\whead_t \in \Reals^N$
	\end{enumerate}

\end{description}

\colorbx{Summary Object Update Unit}

%The  Summary Unit is the last unit of the SAMCell. It is responsible to output the new summary object. It first picks which object is relevant between the object extracted from memory and the visual object extracted from the image. Once the relevant object is picked, it is combined with the former summary object through a linear layer to become the new summary object. It is the final step of the SAMCell reasoning cycle.
%
%The image encoder, question encoder and output unit are described in the appendix.

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item[]
		\item Previous summary object: $so_{t-1} \in \Reals^d$
		\item Visual object: $vo_t \in \Reals^d$
		\item Memory object: $mo_t \in \Reals^d$
		\item Object predicate gates: $\imatch_t, \mmatch_t \in [0,1]$
	\end{enumerate}

	\item[Parameters:] Concatenation linear layer $\cH$

	\item[Output:]
	New summary object:
	$so_t = \cH\bigl([so_{t-1}, (\imatch_t * vo_t + \mmatch_t * mo_t)]\bigr) \in \Reals^d$

\end{description}

\noindent\makebox[\linewidth]{\rule{\paperwidth}{1pt}}

\colorbx{Image Encoder}

\begin{description}
	\item[Inputs:]
	Images

	\item[Output:]
	Features maps:
	$F_k$
\end{description}

The Image Encoder unit is responsible to preprocess the sequence of images one at the time. It is a 4 layer convolutional neural network. Every layer is composed of the following sequence of operations:

\begin{enumerate}
	\item 2D convolution ($stride=1$)
	\item 2D max pooling
	\item 2D batch normalization
	\item Relu (only for the first 3 layers)
\end{enumerate}

The first layer has 3 inputs channels and 32 output channels. Layer 2 has 32 and 64. Layer 3 has 64 and 64. Finally, layer 4 has 64 inputs channels and 128 output channels .
The resulting tensor (feature maps) are of dimensions [batch size, $H*W =7*7$, $dim$=128].

\colorbx{Question Encoder}

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item question string $question$
		\item question lengths $question_l$

	\end{enumerate}

	\item[Output:]
	\begin{enumerate}
		\item contextual word embeddingg $cw$
		\item question encoding $q$

	\end{enumerate}

\end{description}

The strings questions are first embedded ($torch.NN.Embeddings$) and then passed through a $d$-dimensional biLSTM ($d$=128).
The final hidden state of the biLSTM becomes the question encoding representation $q$. Whereas the biLSTM outputs are projected via a linear layer to become the final words encodings named contextual words $cw$.

\colorbx{Output Unit}

\begin{description}
	\item[Inputs:]
	\begin{enumerate}
		\item question encoding $q$
		\item summary object $so_t$

	\end{enumerate}

	\item[Output:]
	prediction
\end{description}

The output unit predicts the final answer to the question. It is a 2-linear-layer-ELU classifier that produces a distribution over the possible candidates. It is based on two inputs, the final summary object $so_t$ and the question encoding $q$ that are concatenated together.

\subsection{Training and Implementation Details}

\subsubsection{Implementation Details}

SAMNet is implemented using MI-Prometheus~\cite{kornuta2018accelerating}, a framework based on Pytorch.
We trained all our models using NVIDIA’s GeForce GTX TITAN X GPUs. SAMNet was trained using 8 reasoning steps and a hidden state size of 128. The external memory has 128-bit slots for all experiments. We trained our model until convergence but we also have set a training time limit of 80 hours.

\subsubsection{Training and testing Methodology}

We compared our model to the original COG model  ~\cite{yang2018dataset} using their implementation (https://github.com/google/cog) and scores provided by the authors through personal communications. We used the same training parameters detailed in the original paper and reproduced their results.  For the generalization experiments from canonical to hard, we used the verified model and obtained new results that were not reported in the reference paper.   In Table 1 COG section shows 4 columns divided into two parts: "paper" and "ours" which distinguish between the results reported in the paper vs. our own experiments.

Our experiments focused on the 22 classification tasks provided by the COG dataset. More details about the dataset are given in Table 2. First we evaluated SAMNet's performance on the canonical setting and compared it with the COG Model. As shown in Table 1 we could achieve a small improvement in accuracy, from 97.6\% for the COG model to 98\% for SAMNet. Next we focused on the hard setting of the dataset which increases the number of distractors from 1 to 10 and the number of frames from 4 to 8.

The first approach was to train a model on the hard training set, and test it on the hard test set. This is the same approach used by the COG paper ~\cite{yang2018dataset} to evaluate performance on the hard dataset. We achieve a test accuracy of 96.1 \% which represents a 16\% improvement from the COG model score (see Table 1).
%It shows that SAMNet's design choices make a difference when it come to harder tasks with longer sequences and more distractors objects.

The second approach was to see if the models can generalize from the easy to the hard setting. For this experiment, we trained a model on the canonical dataset, and directly tested on the hard dataset.  This experiment highlighted the most significant difference between SAMNet and the baseline COG model.

Finally we trained a model on the canonical data set, fine-tuned it on the hard data set using only 25k iterations, and tested on the hard dataset. Thanks to fine-tuning, we can observe a significant improvement from 91.6\%  to 96.5\% test accuracy which represents the state of the art accuracy for the hard setting (classification tasks).
After a short fine-tuning process, the transferred model could generalize well to harder tasks and even surpass the accuracy obtained in the first approach. We note that the third approach is also twice faster than the first one, and it is more effective in terms of accuracy.

A more granular analysis of accuracy per task shows a major improvement for the two hardest tasks, AndCompareShape and AndCompareColor. Those two tasks represent a higher level of difficulty due to the number of objects to be remembered in order to answer the question correctly.
As we can see in Table 1 we could achieve a 12\% improvement for the canonical data set and almost a 30\% improvement for the hard dataset.
The large improvement in these memory-intensive tasks indicate that the SAMNet's external memory plays a crucial role in our results.
The training and implementation details are in appendix.

\begin{table}[htb]
\caption{COG test set accuracies for SAMNet \& COG models. Below `paper' denotes results from~\cite{yang2018dataset}
	while `code' denotes results of our experiments using their implementation~\cite{yang2018implement}}
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tabular}{lcccccccccc}
	\toprule
	Model & & SAMNet & && && COG&& \\
	\cmidrule{2-5} \cmidrule{7-11}
	&&&&& & paper & ours & ours & paper&\\
	\cmidrule{7-9} \cmidrule{10-11}
	Trained on       & canonical & canonical & canonical & hard &           &  canonical  & canonical  & canonical & hard \\
	Fine tuned on  & - & - & hard  & - &           & -   & - & hard & - \\
	Tested on        & canonical & hard & hard & hard &            &canonical  & hard & hard & hard  \\
	\midrule

	Overall accuracy & 98.0 & 91.6 & 96.5  & 96.1 &           & 97.6  & 65.9 & 78.1& 80.1 \\

	\midrule

	AndCompareColor			&	93.5	&	82.7	&	89.2	&	80.6	&	&	81.9	&	57.1	&	60.7	&	51.4	 \\
	AndCompareShape			&	93.2	&	83.7	&	89.7	&	80.1	&	&	80.0	&	53.1	&	50.3	&	50.7 \\
	AndSimpleCompareColor		&	99.2	&	85.3	&	97.6	&	99.4	&	&	99.7	&	53.4	&	77.1	&	78.2 \\
	AndSimpleCompareShape		&	99.2	&	85.8	&	97.6	&	99.2	&	&	100.0	&	56.7	&	79.3	&	77.9 \\
	CompareColor			&	98.1	&	89.3	&	95.9	&	99.7	&	&	99.2	&	56.1	&	67.9	&	50.1 \\
	CompareShape			&	98.0	&	89.7	&	95.9	&	99.2	&	&	99.4	&	66.8	&	65.4	&	50.5	 \\
	Exist				&	100.0	&	99.7	&	99.8	&	99.8	&	&	100.0	&	63.5	&	96.1	&	99.3 \\
	ExistColor			&	100.0	&	99.6	&	99.9	&	99.9	&	&	99.0	&	70.9	&	99	&	89.8 \\
	ExistColorOf			&	99.9	&	95.5	&	99.7	&	99.8	&	&	99.7	&	51.5	&	76.1	&	73.1 \\
	ExistColorSpace			&	94.1	&	88.8	&	91.0	&	90.8	&	&	98.9	&	72.8	&	77.3	&	89.2 \\
	ExistLastColorSameShape		&	99.5	&	99.4	&	99.4	&	98.0	&	&	100.0	&	65.0	&	62.5	&	50.4 \\
	ExistLastObjectSameObject	&	97.3	&	97.5	&	97.7	&	97.5	&	&	98.0	&	77.5	&	61.7	&	60.2 \\
	ExistLastShapeSameColor		&	98.2	&	98.5	&	98.8	&	97.5	&	&	100.0	&	87.8	&	60.4	&	50.3 \\
	ExistShape			&	100.0	&	99.5	&	100.0	&	100.0	&	&	100.0	&	77.1	&	98.2	&	92.5 \\
	ExistShapeOf			&	99.4	&	95.9	&	99.2	&	99.2	&	&	100.0	&	52.7	&	74.7	&	72.70 \\
	ExistShapeSpace			&	93.4	&	87.5	&	91.1	&	90.5	&	&	97.7	&	70	&	89.8	&	89.80 \\
	ExistSpace			&	95.3	&	89.7	&	93.2	&	93.3	&	&	98.9	&	71.1	&	88.1	&	92.8 \\
	GetColor			&	100.0	&	95.8	&	99.9	&	100.0	&	&	100.0	&	71.4	&	83.1	&	97.9 \\
	GetColorSpace			&	98.0	&	90.0	&	95.0	&	95.4	&	&	98.2	&	71.8	&	73.	&	92.3 \\
	GetShape			&	100.0	&	97.3	&	99.9	&	99.9	&	&	100.0	&	83.5	&	89.2	&	97.1	 \\
	GetShapeSpace			&	97.5	&	89.4	&	93.9	&	94.3	&	&	98.1	&	78.7	&	77.3	&	90.3 \\
	SimpleCompareShape		&	99.9	&	91.4	&	99.7	&	99.9	&	&	100.0	&	67.7	&	96.7	&	99.3 \\
	SimpleCompareColor		&	100.0	&	91.6	&	99.80	&	99.9	&	&	100.0	&	64.2	&	90.4	&	99.3 \\
	\bottomrule
\end{tabular}
\end{adjustbox}
\label{results}
\end{table}

\begin{table}[htb]
\caption{COG Dataset parameters for the canonical setting and the hard setting  }
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tabular}{ccccccc}
	\toprule

	Dataset    &  	number of frames  &  	maximum memory duration & number of distractors & size of training set & size of validation/test set    \\
	\midrule

	Canonical setting & 4 & 3 & 1 & 10000320 & 500016 &   \\
	\midrule

	Hard  setting & 8 & 7& 10 & 10000320 & 500016  \\
	\bottomrule
\end{tabular}
\end{adjustbox}
\label{tab:parameters}
\end{table}


\problemname{Assistant Ranking}
The online retailer Amagoogsoftbook currently offers $N$ different so-called ``home assistants'', which it wants to recommend to its customers.
For this recommendation, they wish to rank all the assistants.
The quality of this ranking is not very important -- multiple assistants may even be assigned the same rank -- but they wish to maximize the number of distinct ranks assigned, to lessen the decision fatigue of their customers.

To ensure that the ranking is not \emph{completely} arbitrary, they have collected for each assistant $i$ two measurements $a_i$ and $b_i$ -- the quality of the jokes the assistant can tell and how nice are the compliments the assistant is able to give (clearly these are the two most important aspects).
These measurements are of course somewhat subjective, so we wish to ignore small differences in them.
However, if for two given assistants $i$ and $j$ we have that $a_i + K < a_j$ or $b_i + K < b_j$, the ranking of assistant $j$ must be the same or higher than the ranking of assistant $i$.
This rule may force two products to be given the same ranking, for example if an assistant $i$ gives much better puns than assistant $j$, while assistant $j$ gives the superior self-esteem boosts.

What is the maximum number of distinct ranks, taken over all possible rankings?

\section*{Input}
The first line contains the integers $1 \le N \le 100\,000$ and $0 \le K \le 10^9$ -- the number of assistants and the measurement difference limit as described in the statement.
The next line contains the $N$ integers $a_1, a_2, \dots, a_N$.
The next line contains the $N$ integers $b_1, b_2, \dots, b_N$.

All measurements are between $0$ and $10^9$.

\section*{Output}
Output a single integer: the maximum number of distinct ranks.


\subsubsection{Beta Model (\betamodel)}

The \betamodel\ is an analytic shape function widely used in X-ray
analysis to fit the radial density profile:

\begin{equation}
n_e(x) = {{n_e}_0\over{\left[1 + x^2\right]^{(3\beta/2)}}}
\end{equation}

The line-integral of this function is therefore commonly used to fit the
projected 2D cluster profiles in SZ observations:

\begin{equation}
p(x) = {1\over{\left[1 + x^2\right]^{(1-3\beta/2)}}}
\end{equation}

The \betamodel\ can bs used in \climax\ by invoking \code{addmodel}
with \code{type = betamodel}.


\section{Voting Protocol}

% eventually consider moving to contributions subsection if needed
The voting protocol closely follows the cryptographic
setup described in Schoenmaker et. al.'s~\cite{Cramer:1997:SOE:1754542.1754554}
multi authority election scheme.  Given the different threat model,
the threshold encryption multi authority features are eliminated
and replaced with a single pollster service.

% Threat model table?

This is justified on the grounds that neither preventing malicious voters from
voting nor defending against malicious authorities is a
goal of this solution.

The malicious voter handling aspect is left to the human level pollsters
and human authorities which can presumably scale according to the level
of interest in the specific poll.
Our protocol ensures that the set of voting parties, the subset of eligible
voters selected after ballot casting, and the decisions about
accepted votes are publicly verifiable.
Another aspect of differentiation is \emph{coercion-freeness}, which
is an important requirement for a trustable voting infrastucture,
as described in~\cite{Karlof:2005:CVP:1251398.1251401}.
This property follows from the fact that only the aggregate totals are ever
decrypted, thereby providing voters deniability about their vote,
which in turn makes the
election coercion-free by allowing the voter to lie about the vote
they had cast\footnote{Ideally one would add a legal clause at the bottom of the
  ballot stating that it is your right, and under some circumstances,
  perhaps your duty, to lie about your vote. Such a disclaimer
  would make the deniability based coercion freeness obvious to
  unsophisticated voters.  Achieving coercion freeness requires both social and
  technological actions.  Our protocol provides the latter.}.

Malicious authorities are not considered as a threat because we
propose to establish authority honesty via open source implementation as well
as massive public validation of the algorithms and artifacts involved.
The authority supports the fully open source voting protocol in an
open verifiable manner.  The voting protocol is explained below in terms of
its phases:

\subsection{Poll creation}
On receiving a request to create a poll, the Pollster service
chooses a large prime $p$, a generator $g$, and an integer private key $s < p$
large enough such that an $O(\sqrt s)$ time algorithm is computationally
infeasible.
The public key $h \equiv g^s\bmod p$ is computed from the private key.
The values $p$, $g$ and $h$
are exposed on the public bulletin board.

This step shall also disclose the non-cryptographic
attributes of the poll. For example, the expected number of voters,
the duration of the poll, as well as the
locations where a vote may be cast and the location of the
bulletin board where the results shall be available.

This stage can also be used to cryptographically commit the
polling parameters.  Accordingly the hash of the contents of the
initial bulletin board shall be stored on a blockchain
in a future version of our implementation.

\subsection{Casting encrypted ballots}

The specific values sent out in the encrypted ballots depends on
whether the vote is positive or negative.  The values of the variables
used in casting the universally verifiable ballot are shown in
Table~\ref{table:ballotparams}.  The values are expressed in terms of
a generator $G$ and random numbers $\alpha$, $r_1$ and $r_2$.
% TODO explain is G a new generator and if so why

\begin{table}[t]
\caption{Values of ballot parameters}
\begin{center}
  \begin{tabular}{c|c|c}
    Variable & Vote $+1$ & Vote $-1$ \\\hline
    $x$   &  $g^\alpha$ &   $g^\alpha$ \\
    $y$   &  $h^\alpha G$ & $\frac{h^\alpha}{G}$  \\
    $a_1$ &  $g^{r_1}x^{d_1}$ & $g^\omega$  \\
    $b_1$ &  $h^{r_1}(yG)^{d_1}$ & $h^\omega$  \\
    $a_2$ &  $g^\omega$ &  $g^{r_2}x^{d_2}$ \\
    $b_2$ &  $h^\omega$ & $h^{r_2} \big( \frac{y}{G} \big)^{d_2}$ \\
\end{tabular}
\label{table:ballotparams}
\end{center}
\end{table}

% TODO : get fig from jcn and data from page in yellow book
The universally veriable ballot to be cast by the voter $v_i$
can be expressed as a 3-tuple $\{T,C,D\}$, where:
\begin{equation} \label{eq3}
\begin{split}
T & = \{ x, y, a_1, b_1, a_2, b_2 \} \\
C & = H( T || v_i ) \\    % subtle conceptual bug - v_i is voter not verifier but see intro
D & = \{d_1, d_2, r_1, r_2 \}
\end{split}
\end{equation}

Any verifier needs to establish that i)
the vote itself is valid, i.e. is a $+1$ or $-1$, and ii) that the ElGamal
pair $(x,y) \equiv (g^\alpha, h^\alpha m)$ is properly constructed using the
publicly known values $g$ and $h$.  These properties follow from testing
the following on the encrypted ballot:

\begin{equation} \label{eq4}
\begin{split}
C & = d_1 + d_2  \\
a_1 & = g^{r_1} x^{d_1}  \\
b_1 & = h^{r_1} (yG)^{d_1} \\
a_2 & = g^{r_2} x^{d_2} \\
b_2 & = h^{r_2}(\frac{y}{G})^{d_2} \\
\end{split}
\end{equation}

\subsection{Counting ballots}

Consider the encrypted vote part of the ballot $(x,y) \equiv (g^\alpha, h^\alpha G^m)$
where $m \in {+1, -1}$.
The homomorphic property of the encryption allows us to take the product(modulo
the large prime) of the ElGamal pairs, and the resultant values can be decrypted to
arrive at the total.   Consider several encrypted ballots
$(x_i,y_i) \equiv (g^{r_i} , h^{r_i} G^{m_i})$, and let
\[
(X,Y) \equiv (\prod x_i , \prod y_i)
\]
Then it follows that:
\begin{equation} \label{eq5}
\begin{split}
  \frac{Y}{X^s} & = \frac{\prod h^{r_i} G^{m_i}}{ \prod (g^{r_i})^s } \\
  & =  \frac{\prod g^{sr_i} G^{m_i}}{ \prod (g^{sr_i}) } \\
  & = \prod G^{m_i} \\
   & = G^{\sum m_i}
\end{split}
\end{equation}
Hence the total vote count is given as $\log_G \frac{Y}{X^s}$, which can be
calculated by the pollster service as it knows the private key $s$ corresponding
to this poll.

% zkp of this total is correct