% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\date{\today}
\title{Notes from Neural Networks and Deep Learning}
\hypersetup{
 pdfauthor={},
 pdftitle={Notes from Neural Networks and Deep Learning},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.1.9)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Using neural nets to recognize handwritten digits}
\label{sec:org75be78e}
\subsection{The sigmoid function}
\label{sec:orgcbcab66}
The sigmoid function is defined as:
\begin{equation} \label{eqn:sigmoid}
\sigma(z) = \frac{1}{1 + e^{-z}} = \frac{e^z}{e^z + 1}
\end{equation}

\subsection{The cost function}
\label{sec:org8ed9cda}
The cost function is defined as:
\begin{equation} \label{eqn:cost}
C(w,b) \equiv \frac{1}{2n} \sum_{n=1} ||y(x) - a||^2
\end{equation}

The sum over all the training data of the square of each difference between the correct (or desired) output
and the network's output.

\subsection{Gradient descent}
\label{sec:org85a25e0}
Calculus dictates that \(C\) changes as follows:
\begin{equation} \label{eqn:deltacost}
\Delta C \approx \frac{\partial C}{\partial v_1} \Delta v_1 + \frac{\partial C}{\partial v_2} \Delta v_2
\end{equation}
We define the gradient vector as:
\begin{equation}
\nabla C \equiv (\frac{\partial C}{\partial v_1}, \frac{\partial C}{\partial v_2})^T
\end{equation}
And by defining the vector of changes to all the inputs:
\begin{equation}
\Delta v \equiv (\Delta v_1, \Delta v_2)^T
\end{equation}
We can now rewrite Equation \ref{eqn:deltacost} as:
\begin{equation} \label{eqn:deltacostconcise}
\Delta C \approx \nabla C \cdot \Delta v
\end{equation}

Now we can define our movement of weights and biases to be in the opposite direction of the gradient
of the cost function. This is the concept of the ball rolling down the hill. If we define:
\begin{equation}
\Delta v = - \eta \nabla C
\end{equation}
Where \(\eta\) is the \emph{learning rate}, we can rewrite Equation \ref{eqn:deltacostconcise} as:
\begin{equation}
\Delta C = - \eta ||\nabla C||^2
\end{equation}
We know \(||\nabla C||^2\) will always be positive, and have chosen \(\eta\) as a positive constant, so it
is guaranteed that \(\Delta C\) will always decrease, as desired.

\textbf{Stochastic gradient descent} is a cut down version of gradient descent, which reduces computation time by
randomly choosing a \emph{mini batch} from the training data, rather than using all of it.
\begin{itemize}
\item n.b. a mini batch of size 1 is equivalent to on-line learning.
\end{itemize}

\subsection{Code}
\label{sec:org5ef75ad}
The code centres around the \textbf{Network} class, with the following variables:
\begin{itemize}
\item \textbf{num\(_{\text{layers}}\)}: the number of layers the network has
\item \textbf{sizes}: a list of integers, with the nth integer defining the number of nodes in the nth layer
\item \textbf{biases}: the biases of the network
\item \textbf{weights}: the weights of the links
\end{itemize}
For now we initialise the biases and weights to be completely random. Both biases and weights are numpy matrices, so weights[1] is a matrix storing the weights of the links between the second and third row of neurons.

Looking at that particular matrix, which we will name \(w\), let us denote the weight of the link joining the \(k^{\text{th}}\) neuron in the second layer with the \(j^{\text{th}}\) neuron in the third layer: \(w_{jk}\). While this notation seems to have \(j\), and \(k\) around the wrong way, it allows for a very concise notation for the activations of the neurons in the 3rd layer:
\begin{equation} \label{eqn:activation}
a' = \sigma ( wa+b )
\end{equation}
where:
\begin{itemize}
\item a is the vector of activations from the second layer of neurons
\item so \(a'\) is obtained by multiplying the previous layer's activations by the link weights, and adding the biases. Simple!
\end{itemize}

The \textbf{SGD} function does a lot of the work here. Each epoch it randomly shuffles the training data, then partitions it into mini batches of the specified size. For each mini-batch, it makes one gradient descent step by using the function update\(_{\text{mini}}\)\(_{\text{batch}}\), updating the network's weights and biases

The \textbf{update\(_{\text{mini}}\)\(_{\text{batch}}\)} function also does a lot of work, as it performs the actual update. Within this function, the \textbf{self.backrop} function does most of the actual work, as it computes the gradient of the cost function. \textbf{update\(_{\text{mini}}\)\(_{\text{batch}}\)} really works by computing the gradient for every training example in the mini-batch, and updatng self.weights and self.biases. The backprop function is detailed in the next chapter, for now, all that is necessary is an understanding of what it is doing.

\clearpage

\section{How the back propagation algorithm works}
\label{sec:org5a70fa5}
\subsection{Using matrices to compute the output of a neural network}
\label{sec:org3fe5676}
In chapter 1 we introduced the notation \(w_{jk}\) leading up to equation \ref{eqn:activation}. We now extend this slightly, such that \(w^l_{jk}\) refers to the weight of the link between the \(k^{\text{th}}\) neuron in the \((l-1)^{\text{th}}\) layer and the \(j^{\text{th}}\) neuron in the \(l^{\text{th}}\) layer. Similarly for biases, \(b^l_j\) is the bias of the \(j^{\text{th}}\) neuron in the \(l^{\text{th}}\) layer.

Using these notations, the activation \(a^l_j\) of the \(j^{\text{th}}\) neuron in the \(l^{\text{th}}\) layer is given by:
\begin{equation} \label{eqn:alj}
a^l_j = \sigma ( \sum_k w^l_{jk} a^{l-1}_k + b^l_j )
\end{equation}
The sum here is over all neurons, \(k\), in the \((l-1)^{\text{th}}\) layer.

\vspace{0.3cm}

To utilise matrix form we define a \textbf{weights matrix}, \(w^l\), for the weights connecting the neurons in the \(l^{\text{th}}\) layer to those in the \((l-1)^{\text{th}}\) layer. \(w^l\) will be defined such that the entry in the \(j^{\text{th}}\) row and the \(k^{\text{th}}\) column is \(w^l_{jk}\).

Similarly, we define a \textbf{bias vector}, \(b^l\), which follows naturally as the biases for each neuron in layer \(l\). And finally, we define the \textbf{activation vector}, \(a^l\), whose components are the activations \(a^l_j\).

The final step towards writing Equation \ref{eqn:alj} in matrix form is vectorising a function, such as \(\sigma\). This simply refers to performing the \(\sigma\) operation elementwise.

\vspace{0.3cm}

Equation \ref{eqn:alj} can now be written as:
\begin{equation} \label{eqn:almatrix}
a^l = \sigma(w^l a^{l-1} + b^l)
\end{equation}

We will actually compute the intermediate quantity:
\begin{equation}
  z^l = w^l a^{l-1} + b^l
\end{equation}
It turns out that this quantity will be useful enough (in time) to be worth naming. We call \(z^l\) the \emph{weighted input} to the neurons in layer \(l\). Naturally, Equation \ref{eqn:almatrix} can be written as:
\begin{equation}
a^l = \sigma(z^l)
\end{equation}

\subsection{The two assumptions we need about the cost function}
\label{sec:orgc244547}
The goal, as discussed, of back propagation is to compute the partial derivatives \(\frac{\partial C}{\partial w}\) and \(\frac{\partial C}{\partial b}\) of the cost function \(C\) with respect to any weight or bias in the network.

As the title of this section suggests, we need to make two assumptions for back propagation to work. Before we state those assumptions, we should define a cost function. We return to the quadratic cost function defined in Equation \ref{eqn:cost}:
\begin{equation*}
C(w,b) \(\equiv\) \frac{1}{2n} \(\sum_{\text{n=1}}\) ||y(x) - a\(^{\text{L}}\)(x)||\(^{\text{2}}\)
\end{equation}
We have made some small notational changes, such as the superscript \(L\), but it means the same thing.
\begin{itemize}
\item \(y(x)\) is the expected (or correct) outputs of the network
\item \(a^L(x)\) is the vector of activations of the output layer
\end{itemize}
\vspace{0.3cm}

Okay, on to the assumptions.

\textbf{Assumption 1} is that the cost function is an average \(C = \frac{1}{n} \sum_{x} C_x\) over cost functions \(C_x\), for individul training examples, \(x\). This is the case for the quadratic cost function, where the cost of a single training example is \(C_x = \frac{1}{2} ||y - a^L||^2\). This assumtion will hold true for all cost functions to be introduced.

This assumption is important, because the backpropagation algorithm will actually calculate \(\frac{\partial C_x}{\partial w}\) and \(\frac{\partial C_x}{\partial b}\) for a single training example. We then get \(\frac{\partial C}{\partial w}\) and \(\frac{\partial C}{\partial b}\) by averaging over training examples.

\textbf{Assumption 2} is that the cost function can be written as a function of the outputs of the network. That is, for output layer activations \(a^L\), the cost function can be expressed as:
\begin{equation}
cost C = C(a^L)
\end{equation}
This is true of the quadratic cost function, as the expected network outputs can be thought of as a fixed expression (i.e. not a variable because the weights and biases of the network do not change them), and as such it is a function only of \(a^L\).

\subsection{The Hadamard product}
\label{sec:org421f816}
The Hadamard product (\(\circ\)) is like the dot product but it works for matrices of multiple dimensions rather than just vectors.

\subsection{The four fundamental equations behind backpropagation}
\label{sec:org07111f6}
As has been previousy discussed, backpropagation is concerned with how a change in the weights and biases of the network affects the cost function. This ultimately means calculating \(\frac{\partial C}{\partial w^l_{jk}}\) and \(\frac{\partial C}{\partial b^l_j}\), but first we will define an intermediate quantity, \(\delta^l_j\), known as the \emph{error} in the \(j^{\text{th}}\) neuron of the \(l^{\text{th}}\) layer.

Suppose a demon sits at the \(j^{\text{th}}\) neuron in the \(l^{\text{th}}\) layer, and makes a small change \(\Delta z^l_j\) to the neuron's weighted input (\(z^l_j\)), such that the neuron now outputs \(\sigma (z^l_j + \Delta z^l_j)\) instead of \(\sigma (z^l_j)\). This change then propagates through the network, producing a final \textbf{change} in cost of \(\frac{\partial C}{\partial z^l_j} \Delta z^l_j\).

The demon is our friend, however, and is trying to make a change that will minimise the cost function. Suppose \(\frac{\partial C}{\partial z^l_j}\) has a large magnitude (positive or negative), the demon would then choose a value of \(\Delta z^l_j\) that has the opposite sign of \(\frac{\partial C}{\partial z^l_j}\). In contrast, if \(\frac{\partial C}{\partial z^l_j}\) small in magnitude, the demon would assume the neuron is near optimal, as he cannot easily change the cost function by perturbing the weighted input. From this we can derive a heuristic sense in which \(\frac{\partial C}{\partial z^l_j}\) is a measure of error in the neuron.

Motivated by the demon's story, we will define the error of neuron \(j\) in layer \(l\) as:
\begin{equation}
\delta^l_j \equiv \frac{\partial C}{\partial z^l_j}$
\end{equation}
And like we have done before, we will use \(\delta^l\) to define the vector of errors in layer \(l\). Backpropagation then gives us a way of computing \(\delta^l\) in each layer, and relating those vectors to \(\frac{\partial C}{\partial w^l_{jk}}\) and \(\frac{\partial C}{\partial b^l_j}\).

\subsubsection{Error in the output layer, \(\delta^L\) \label{orgc4d77df}}
\label{sec:org317e585}
The components of \(\delta^L\) are given by:
\begin{equation} \label{eqn:bp1}
\delta^L_j = \frac{\partial C}{\partial a^L_j} \sigma' (z^L_j)
\end{equation}
\begin{itemize}
\item \(\frac{\partial C}{\partial a^L_j}\) measures how fast the cost is changing as a function of the \(j^{\text{th}}\) output activation.
\item \(\sigma' (z^L_j)\) measures how fast the activation function is changing at \(z^L_j\)
\end{itemize}

To rewrite Equation \ref{eqn:bp1} in matrix form:
\begin{equation} \label{eqn:bp1a}
\delta^L = \nabla_a C \circ \sigma' (z^L)
\end{equation}
\begin{itemize}
\item \(\nabla_a C\) is a vector whose components are the partials \(\frac{\partial C}{\partial a^L_j}\)
\end{itemize}

In the case of the quadratic cost function, \(C = \frac{1}{2} \sum_j(y_j - a_j)^2\):
\begin{align*}
\frac{\partial C}{\partial a^L_j} &= 2 \times \frac{1}{2} \times (y_j - a_j) \times -1 \\
&= (a_j - y_j) \\
\nabla_a C &= (a^L - y)
\end{align*}
So now we can rewrite Equation \ref{eqn:bp1a} as:
\begin{equation}
\delta^L = (a^L - y) \circ \sigma' (z^L)
\end{equation}

\subsubsection{Error \(\delta^l\) in terms of error in the next layer, \(\delta^{l+1}\)}
\label{sec:org852e550}
\begin{equation} \label{eqn:bp2}
\delta^l = ((w^{l+1})^T \delta^{l+1}) \circ \sigma'(z^l)
\end{equation}
\begin{itemize}
\item \((w^{l+1})^T\) is the transpose of the weights matrix for the \(l+1^{\text{th}}\) layer.
\end{itemize}

Suppose we know the error \(\delta^{l+1}\) at the \(l+1^{\text{th}}\) layer. We can think of applying the transpose matrix to this error as moving the error backward through the network, giving us a sort of measure of the error at the \(l^{\text{th}}\) layer of the network. We then take the hadamard product \(\circ \sigma'(z^l)\), which moves the error back through the activation function in layer \(l\), giving us the error in layer \(l\), \(\delta^l\).
\vspace{0.3cm}

It follows then, that by applying Equation \ref{eqn:bp1a} and then repeatedly applying Equation \ref{eqn:bp2} for the remaining layers in the network, that we can compute the error for any layer in the network.

\subsubsection{The rate of change of cost with respect to any bias in the network}
\label{sec:orgad59eba}
\begin{equation} \label{eqn:bp3}
\frac{\partial C}{\partial b^l_j} = \delta^l_j
\end{equation}
That is, the error \(\delta^{\text{l}}_{\text{j}}\) is \emph{exactly} the rate of change we are after. Naturally we can write this shorthand as:
\begin{equation}
\frac{\partial C}{\partial b} = \delta
\end{equation}
where it is understood that \(\delta\) is being evaluated at the same neuron as \(b\).

\subsubsection{The rate of change of cost with respect to any weight in the network}
\label{sec:org3f4cfe2}
\begin{equation} \label{eqn:bp4}
\frac{\partial C}{\partial w^l_{jk}} = a^{l-1}_k \delta^l_j
\end{equation}
We already know how to compute \(a^{l-1}_k\) and \(\delta^l_j\), yay!

We can write Equation \ref{eqn:bp4} in a less index-intensive way as:
\begin{equation}
\frac{\partial C}{\partial w} = a_{in} \delta_{out}
\end{equation}
where it is understood that:
\begin{itemize}
\item \(a_{in}\) is the activation of the neuron input to the weight \(w\), and \(\delta_{out}\) is the error in the neuron output from the weight \(w\). One interesting conclusion from this result is that weights output from low activation neurons learn more slowly than those output from higher activation neurons.
\end{itemize}

\subsection{Proofs for the four fundamental equations}
\label{sec:orgb9ab6d3}
The proofs for the four fundamental equations are mostly derived from the chain rule.
\subsubsection{Error in the output layer}
\label{sec:org855fe57}
Recall that:
\begin{equation*}
\delta^L_j = \frac{\partial C}{\partial z^L_j}
\end{equation*}
As earlier discussed, the cost function is a function only of output activations \(a^L\), so by using the chain rule to break this down we get:
\begin{equation*}
\delta^L_j = \frac{\partial C}{\partial a^L_j} \frac{\partial a^L_j}{\partial z^L_j}
\end{equation*}
We can break the first partial down quite easily when using the quadratic cost function, but we have already done this in Section \ref{orgc4d77df}. The second partial is, by definition:
\begin{align*}
a^l &= \sigma(z^l) \\
\frac{\partial a^L_j}{\partial z^L_j} &= \sigma'(z^l)
\end{align*}
Naturally this leaves us with the desired result:
\begin{equation*}
\delta^L_j = \frac{\partial C}{\partial a^L_j} \sigma' (z^L_j)
\end{equation*}

\subsubsection{Error in layer \(l\), in terms of error in layer \(l+1\)}
\label{sec:org726fed4}
To formulate this equation, we want to write \(\delta^l_j = \frac{\partial C}{\partial z^l_j}\) in terms of \(\delta^{l+1}_k = \frac{\partial C}{\partial z^{l+1}_k}\). We do this using the chain rule:
\begin{align*}
\delta^l_j &= \frac{\partial C}{\partial z^l_j} \\
&= \sum_k \frac{\partial C}{\partial z^{l+1}_k} \frac{\partial z^{l+1}_k}{\partial z^l_j} \\
&= \sum_k \delta^{l+1}_k \frac{\partial z^{l+1}_k}{\partial z^l_j} \\
\end{align*}
Now to evaluate the remaining partial term, we use the definition that:
\begin{equation*}
z^{l+1}_k = \sum_j w^{l+1}_{kj} a^l_j + b^{l+1}_k = \sum_j w^{l+1}_{kj} \sigma(z^l_j) + b^{l+1}_k
\end{equation*}

\subsubsection{The rate of change of cost with respect to any bias in the netowrk \label{org85b26af}}
\label{sec:org26126cb}
Recalling the demon example, he was perturbing the \textbf{weighted input} to the system. The weighted input is given by:
\begin{equation*}
z^l_j = \sum_k w^l_{jk} a^{l-1}_k + b^l_j
\end{equation*}

We have defined the error as:
\begin{equation*}
\delta^l_j \equiv \frac{\partial C}{\partial z^l_j}$
\end{equation*}

Now we use partials:
\begin{align*}
\frac{\partial C}{\partial b^l_j} &= \frac{\partial C}{\partial z^l_j} \frac{\partial z^l_j}{\partial b^l_j} \\
&= \delta^l_j \times 1
\end{align*}

As required.

\subsubsection{The rate of change of cost with respect to any weight in the network}
\label{sec:org26c7d14}
Using the same approach as in Section \ref{org85b26af}, we will break down the desired product using partials.
\begin{align*}
\frac{\partial C}{\partial w^l_{jk}} &= \frac{\partial C}{\partial z^l_j} \frac{\partial z^l_j}{\partial w^l_{jk}} \\
&= \delta^l_j \frac{\partial z^l_j}{\partial w^l_{jk}}
\end{align*}

Now we break down the second partial
\begin{align*}
z^l_j &= \sum_k w^l_{jk} a^{l-1}_k + b^l_j \\
\frac{\partial z^l_j}{\partial w^l_{jk}} &= a^{l-1}_k
\end{align*}

Therefore we get, as desired:
\begin{equation*}
\frac{\partial C}{\partial w^l_{jk}} = \delta^l_j a^{l-1}_k
\end{equation*}

\subsection{The backpropagation algorithm}
\label{sec:org91c6ab6}
\subsubsection{For an individual training example}
\label{sec:org70fa9e6}
Now that the equations are established, we can define an algorithm that conducts backpropagation.
\begin{enumerate}
\item \textbf{Input} \(x\): get the activations of the first layer, \(a^1\)
\item \textbf{Feedforward}: For each \(l = 2,3,...,L\) compute \(z^l = w^la^{l-1} + b^l\) and \(a^l = \sigma(z^l)\)
\item \textbf{Output error \(\delta^L\)}: Compute the vector \(\delta^L = \nabla_a C \circ \sigma'(z^L)\)
\item \textbf{Backpropagate the error}: For each \(l = L-1, L-2, ... ,2\) compute \(\delta^l = ((w^{l+1})^T \delta^{l+1}) \circ \sigma'(z^l)\)
\item \textbf{Output}: The gradient of the cost function is given by \(\frac{\partial C}{\partial w^l_{jk}} = a^{l-1}_k \delta^l_j\) and \(\frac{\partial C}{\partial b^l_j} = \delta^l_j\)
\end{enumerate}

It is clear now why this is called \emph{backpropagation}. The desired products are the partial derivatives of cost with respect to weights and biases, which we can find from the layer errors, which we find by propagating backward through the network from the output layer.
\vspace{0.3cm}

\subsubsection{For a mini-batch of training examples \label{orgf844af8}}
\label{sec:org7a947e7}
The steps above outline the algorithm for a single training example, however in practice it is common to combine backpropagation with a learning algorithm such as stochastic gradient descent, in which the gradient is computed for a mini-batch of training examples at a time, and then a learning step is taken. The \textbf{SGD} algorithm is detailed below, for a mini-batch of \(m\) training examples.
\begin{enumerate}
\item \textbf{Input} a set of training examples
\item \textbf{For each training example \(x\):} set the corresponding input activation \(a^{x,1}\), and perform the following steps:
\begin{enumerate}
\item \textbf{Feedforward:} for each \(l = 2, 3, ... , L\) compute \(z^{x,l} = w^l a^{x,l-1} + b^l\) and \(a^{x,l} = \sigma(z^{x,l})\)
\item \textbf{Output error \(\delta^{\text{x,L}}\):} compute the vector \(\delta^{x,L} = \nabla_a C_x \circ \sigma'(z^{x,L})\)
\item \textbf{Backpropagate the error:} for each \(l = L-1, L-2, ... , 2\) compute \(\delta^{x,l} = ((w^{l+1})^T \delta^{x,l+1}) \circ \sigma'(z^{x,l})\)
\end{enumerate}
\item \textbf{Gradient descent:} For each \(l = L, L-1, ... , 2\) update the weights according to the rule \(w^l \rightarrow w^l - \frac{\eta}{m} \sum_x \delta^{x,l} (a^{x,l-1})^T\), and the biases according to the rule \(b^l \rightarrow b^l - \frac{\eta}{m} \sum_x \delta^{x,l}\)
\end{enumerate}

\subsection{Code}
\label{sec:orgbc98a3f}
Following on from the code from the last chapter, we add the \texttt{update\_mini\_batch} and \texttt{backprop} methods.

The \texttt{update\_mini\_batch} method, as its name suggests, updates the network's weights and biases for a single mini-batch of training examples, by computing the gradient for the mini-batch. The work is mostly done by calling the \texttt{backprop} method, which computes \(\frac{\partial C}{\partial w^l_{jk}}\) and \(\frac{\partial C}{\partial b^l_j}\). \textbf{Note: The algorithm exploits the useful feature of Python where a negative index in a list gives that index counting from the end of the list.}

\subsubsection{Modification to use matrix input}
\label{sec:org49cb826}
To leverage efficient linear algebra methods that are present in most programming languages, we will modify \texttt{network.py} to apply an activation matrix whose columns are mini-batches. This script is called \texttt{network\_matrix.py}. It involves relativey few changes:
\begin{itemize}
\item Multiplying the weights matrix by the activations matrix automatically gives us a matrix of the correct number of rows, and \(n\) columns for \(n\) training examples
\item The bias vector must become a bias matrix, with \(n\) columns (each a copy of the bias vector) for \(n\) training examples.
\item The sigmoid function and cost derivative work without any modification as they can both be applied elementwise to both vectors and matrices
\item \(\delta^l\) is now a matrix, but the operations still work the same way
\item \textbf{The only difference outside the \texttt{backprop\_matrix} function is that we sum each row of nabla\(_{\text{b}}\) as it is returned currently.} This could be done within the backprop\(_{\text{matrix}}\) function too
\end{itemize}

\clearpage

\section{Improving the way neural networks learn}
\label{sec:org34ae88c}
This chapter introduces some new concepts that will help our networks learn faster and be more applicable to data outside our training data, as well as a better way of initialising weights in the network, and heuristics for choosing the network's hyperparameters.

\subsection{The cross-entropy cost function}
\label{sec:org6c949ab}
When one is learning something new, being badly wrong about it tends to lead to very rapid learning. We touched on this in the last chapter, and of course we want the neurons in our network to act the same way. When they are far off producing the correct output, we would like them to rapidly correct themselves. This, however, does not always happen when using the quadratic cost function. The cost derivative, used to modify the weights and biases of a network, hinges upon the sigmoid function, which is nearly flat for a big chunk of its domain. The derivative in these areas is naturally flat, so the learning rate is slow. If the optimal weight or bias is far from the initial value, this can mean training takes a long time.
\vspace{0.3cm}

Let us consider a very basic model network: a single neuron with multiple inputs, \(x_1, x_2, ... x_k\) and bias \(b\). The output of the neuron is still \(a = \sigma(z)\), where \(z = \sum_j w_j x_j + b\). We define the \emph{cross-entropy cost function} as:
\begin{equation} \label{eqn:cross-entropy-cost}
C = - \frac{1}{n} \sum_x [y \ln a + (1-y) \ln (1-a)]
\end{equation}
where \(n\) is the number of items of training data, the sum is over all training inputs, \(x\), and \(y\) is the corresponding desired output for input \(x\).
\vspace{0.3cm}

It is not immediately clear that this expression is even a valid cost function, let alone one that fixes the learning slowdown problem! Let us first discuss the former point.

Firstly, this cost function is non-negative. Both of the expressions in the summed term are negative by definition, as both \(y\) and \(a\) are bound to the interval \([0,1]\). Paired with the negative sign at the beginning of the expression, this cost function must only give positive values.

Secondly, if the neuron's actual output is close to the desired output for all training inputs, \(x\), then the cross-entropy cost will be close to zero. Suppose we have a training example with \(y = 0\) and \(a \approx 0\). The first term disappears as \(y = 0\), and the second term does too as \(\ln(1) = 0\). A similar logic holds for \(y = 1\) and \(a \approx 1\). This does assume that the the desired putputs are 0 or 1, as is often the case in classification problems. With that said, at intermediate values of \(y\) (and \(a\)) between 0 and 1, the cross-entropy cost function is still minimised. This can be proven with calculus, or simply by plugging the cross-entropy cost function into a graphing calculator such as geogebra.
\vspace{0.3cm}

We will now discuss the learning slowdown problem. We substitute \(a = \sigma(z)\) into Equation \ref{eqn:cross-entropy-cost}, and apply the chain rule twice.
\begin{align*}
C &= - \frac{1}{n} \sum_x [y \ln \sigma(z) + (1-y) \ln (1-\sigma(z))] \\
\frac{\partial C}{\partial w_j} &= \frac{\partial C}{\partial \sigma} \frac{\partial \sigma}{\partial w_j} \\
&= \frac{\partial C}{\partial \sigma} \frac{\partial \sigma}{\partial z} \frac{\partial z}{\partial w_j} \\
&= - \frac{1}{n} \sum_x [\frac{y}{\sigma(z)} + \frac{1-y}{1-\sigma(z)}](-1) \sigma'(z) x_j \\
\frac{\partial C}{\partial w_j} &= \frac{1}{n} \sum_x \frac{\sigma'(z)x_j}{\sigma(z)(1-\sigma(z))} (\sigma(z)-y)\\
\end{align*}

Using the very helpful fact that \(\sigma'(z) = \sigma(z)(1-\sigma(z))\) (relatively easily proven using the quotient rule and Equation \ref{eqn:sigmoid}), we can simplify this experssion to the very neat:
\begin{equation}
\frac{\partial C}{\partial w_j} = \frac{1}{n} \sum_x x_j (\sigma(z) - y)
\end{equation}

This expression tells us that the cost is controlled \textbf{only} by the error in the output, where the analogous expression for the quadratic cost function also relates to \(\sigma'(z)\), which caused the learning slowdown. Yay!
\vspace{0.3cm}

Similarly to the expression for \(w_j\), the partial derivative of cost with respect to the bias is found as follows (skipping some steps, the derivation is quite simple and can easily be computed in full on paper):
\begin{align*}
\frac{\partial C}{\partial b} &= \frac{\partial C}{\partial \sigma} \frac{\partial \sigma}{\partial z} \frac{\partial z}{\partial b} \\
&= -\frac{1}{n} \sum_x[\frac{\sigma(z) - y}{\sigma(z)(\sigma(z)-1)}]\sigma'(z) \\
\end{align*}
Which simplifies nicely down to:
\begin{equation} \label{del_CEcost_bias}
\frac{\partial C}{\partial b} = \frac{1}{n} \(\sum_{\text{x}}\) [\(\sigma\)(z) - y]
\end{euation}

It is also worth mentioning that the learning rate (\(\eta\)) can be much lower for the cross-entropy cost function than for the quadratic cost function, but this doesn't really mean much.

\subsubsection{Cross-entropy cost for a network of many neurons}
\label{sec:org5d73456}
Equation \ref{eqn:cross-entropy-cost} is relevant to a network of one neuron, but can easily be generalised for networks of many neurons. With \(y = y_1, y_2, ... y_k\) as the desired outputs of the system, and \(a^L = a^L_1, a^L_2, ... , a^L_k\) as the final layer activations (actual outputs) of the system, we define cross entropy as:
\begin{equation}
C = -\frac{1}{n} \sum_x \sum_j [y_j \ln a^L_j + (1-y_j) \ln (1-a^L_j)]
\end{equation}

This is the same as equation \ref{eqn:cross-entropy-cost}, just summing over the output neurons.

\subsubsection{When should I use cross-entropy cost?}
\label{sec:org6c9c449}
Cross-entropy cost is better than the quadratic cost function for almost every network, so long as \textbf{the sigmoid function is used}.

\subsection{Softmax}
\label{sec:org8228ed9}
While we will not discuss softmax layers again until chapter 6, they are an interesting aside. A softmax layer replaces the sigmoid activation function with the so-called \emph{softmax function}. The weighted input is still calculated the same way. According to this function, the activation \(a^L_j\) is given by:
\begin{equation}
a^L_j = \frac{e^{z^L_j}}{\sum_k e^{z^L_k}}
\end{equation}
Where the denominator is summed over all output neurons.
\vspace{0.3cm}

The point of a softmax layer is that it makes the output of the neurons form a kind of probability distribution. That is to say, their activations sum to 1. This can be verified algebraically but can also be seen intuitively.
\vspace{0.3cm}

Softmax layers can also be used to address the learning slowdown problem. To understand this, we define the \emph{log-likelihood} cost function. Using the usual notation where \(x\) is a training input to the network and \(y\) is the corresponding desired output, we define the log-likelihood cost as:
\begin{equation} \label{eqn:log-likelihood-cost}
C \equiv -\ln a^L_y
\end{equation}
Note that \(a^L_y\) is the output \textbf{only from the neuron we want to fire}. Using our MNIST classification as an example, suppose we want to classify a 7, then \(a^L_y\) will be the activation of the 7\(^{\text{th}}\) neuron. If the network is doing well and \(a^L_7 \approx 1\), then the cost will be low. We need not consider the other activations in the cost function, as the softmax function already does this. If \(a^L_7 \approx 1\), then by definition, all other activations will be quite low.
\vspace{0.3cm}

Now, if we find the partial derivatives of the log-likelihood cost with respect to individual weights and biases in the network, we will find that they, like those of the cross-entropy cost function, are only related to the difference between desired output and the output layer activations.
\begin{align}
\frac{\partial C}{\partial b^L_j} &= a^L_j - y_j \\
\frac{\partial C}{\partial w^L_{jk}} &= a^{L-1}_k (a^L_j - y_j)
\end{align}

Note that here \(y\) refers to the vector of output activations, where above we were using it to refer to a single output activation.

\subsection{Overfitting}
\label{sec:org0af57ba}
Overfitting is a phenomenon where the network becomes very good at classifying the training data, but worse at generalising to test data or real world data. It is often referred to as \emph{overtraining}, as it is caused by training for too long and/or on a dataset that is too small. In general the best way to avoid overfitting is to use as big a training dataset as possible, however this is not always pratical.

\subsubsection{Detecting overfitting}
\label{sec:org21a999c}
There are some telltale signs of overfitting that we can use to help us detect it. Given our propensity for understanding data when visualised, most of them involve plotting accuracy.
\begin{itemize}
\item \textbf{Plotting the accuracy on the test data} - if we see a plateau of acuracy, this usually suggests the network has stopped learning. The cost on training data plotted over the same range of epochs will likely continue to decrease, a clear sign of overfitting.
\item \textbf{Plotting accuracy on the training data} - Overfitting is characterised by accuracy on training data reaching (or getting very close to) 100%.
\item \textbf{Plotting both of these together} - A network that has been overfitted to the training data will show a divergence in the previous two graphs when overfitting really kicks in.
\end{itemize}

\subsubsection{An aside on the MNIST validation data}
\label{sec:orgbc2289d}
Until now we have not used the \texttt{validation\_data} that the MNIST data loading function provides. The intention of this dataset is to be used to tune the network's hyper-parameters (network structure, no. of epochs, mini-batch size, etc.) without testing on the test data. That is to say, we use the validation data as test data while we tune the hyper-parameters, and only once we have finished tuning do we test on the test data. This prevents us from having a network that is biased toward the test data.

\subsection{Regularization}
\label{sec:org887a347}
There are ways to prevent overfitting other than just having a bigger dataset. We can always reduce the size of our network, though this reduces the power of the network as well, so we would only do this as a last resort.

Fortunately we can employ \emph{regularization} techniques to help reduce the instance of overfitting. We will now intrduce \emph{weight decay}, also known as \emph{L2 regularization}. It works by adding a term to the cost function, called the regularization term. The regularized cross entropy is:
\begin{equation} \label{eqn:regularized-cross-entropy}
C = -\frac{1}{n} \sum_{xj}[y_j \ln a^L_j + (1-y) \ln (1-a^L_j)] + \frac{\lambda}{2n} \sum_w w^2
\end{equation}

The regularization term here is the sum of the squares of all the weights in the network, scaled by \(\frac{\lambda}{2n}\), where \(\lambda > 0\) is the \emph{regularization parameter}, and \(n\) (as usual) is the number of training examples. Note here that the regularization term does not include biases.

The same thing can be done for the quadratic cost function, using exactly the same regularization term. We can use this fact to write the regularized cost function as:
\begin{equation}
C = C_0 + \frac{\lambda}{2n} \sum_w w^2
\end{equation} \label{eqn:regularized-lazy}
where $C_0$ is the original cost function.
\vspace{0.3cm}

Intuitively, regularization makes the network prefer to learn small weights. Large weigts will only be allowed if they considerably reduce the cost function. The relative importance of the two terms in the cost function is varied with the regularization parameter, $\lambda$. Small $\lambda$ conveys a preference for minimised cost, and large $\lambda$ conveys a preference for minimised weights.
\vspace{0.3cm}

Let us now show algebraically that regularization works. As usual, we are required to compute $\frac{\partial C}{\partial w}$ and $\frac{\partial C}{\partial b}$ for each weight and bias in the network. We will take the partial derivatives of Equation \ref{eqn:regularized-lazy}:
\begin{align*}
\frac{\partial C}{\partial w} &= \frac{\partial C_0}{\partial w} + \frac{\lambda}{n} w \\
\frac{\partial C}{\partial b} &= \frac{\partial C_0}{\partial b}
\end{align*}

We have already computed the intermediate terms $\frac{\partial C_0}{\partial w}$ and $\frac{\partial C_0}{\partial b}$, so we simply add $\frac{\lambda}{n} w$ to the partial derivative of all the weight terms.
\vspace{0.3cm}

The partial derivatives with respect to the biases have not changed, so the gradient descent learning rule for biases remains:
\begin{equation*}
b \rightarrow b - \eta \frac{\partial C_0}{\partial b}
\end{equation*}
and the rule for weights becomes:
\begin{align}
w &\rightarrow w - \eta \frac{\partial C_0}{\partial w} - \frac{\eta \lambda}{n} w \\
&= (1-\frac{\eta \lambda}{n}) w - \eta \frac{\partial C_0}{\partial w}
\end{align}
Exactly the same as the usual rule, except we first rescale the weight by a factor $1-\frac{\eta \lambda}{n}$. This rescaling is referred to as /weight decay/. It appears from the equation that this will drive the weights unstoppably toward 0, but this is not the case. If increasing the weight will reduce the cost function $C_0$ sufficiently, this term will be overpowered.
\vspace{0.3cm}

The learning rule for /stochastic/ gradient descent for biases remains:
\begin{equation*}
b \rightarrow b - \frac{eta}{m} \sum_x \frac{\partial C_x}{\partial b}
\end{equation*}
And the equivalent for weights becomes:
\begin{equation}
w \rightarrow (1 - \frac{\eta \lambda}{n}) w - \frac{\eta}{m} \sum_x \frac{\partial C_x}{\partial w}
\end{equation}
Where:
\begin{itemize}
\item $m$ is the size of the mini-batch
\item $n$ is the size of the full training set
\item The sum is over training examples $x$ in the mini-batch
\end{itemize}

Interestingly, regularization also helps to avoid getting stuck in local minima of the cost function.

\subsubsection{Why does regularization work?}
\label{sec:org162219a}
The standard explanation for this (which is well explained in the online book with diagrams) involves Ockham's Razor. Suppose we have 9 data points in a mostly linear arrangement, and are trying to deduce the model of \(y\) in terms of \(x\). We \textbf{can} perfectly fit a 9\(^{\text{th}}\) order polynomial to the data points, but it will become dominated by the \(x^9\) term once we move far beyond the data points. A linear model, on the other hand, will not pass through all points, but logically provides a much better prediction of the model outside the range of the data points, if we assume the original data was polluted with some kind of noise. We "know" this because of Ockham's Razor.
\vspace{0.3cm}

Now let's think of this in terms of neural networks. Suppose we have a network with mostly small weights, as a regularized cost function generates. The smalless of the weights means that a few random inputs here and there (like noise) will have limited effect on the behaviour of the network. This makes it hard for the network to learn the effects of local noise in the data. It's analogous to think of this as a way of making single pieces of evidence matter less to the output of the network, while a type of evidence seen often across a network will elicit a greater response from the network.

\subsubsection{Why not regularize biases?}
\label{sec:org3d0d07d}
We can regularize biases in our networks, but it often has little to no effect. This is partly because large biases do not make a neuron sensetive to its inputs in the same way as large weights. Large biases also allow neurons to saturate, which is actually desirable behaviour.

\subsubsection{Other techniques for regularization}
\label{sec:orgab39453}
We will here discuss three other regularization techniques, though there are many, many more.

\begin{enumerate}
\item \textbf{L1 regularization}
\label{sec:org87a6c6b}
is similar to L2 regularization, but we add the sum of the absolute values of the weights rather than their squares.
\begin{equation}
C = C_0 + \frac{\lambda}{n} \sum_w |w|
\end{equation}

While similar to L2 regularization, this will behave slightly differently. Let us look at the partial derivatives of the cost function now, to see if we can establish how differently it will behave.
\begin{equation}
\frac{\partial C}{\partial w} = \frac{\partial C_0}{\partial w} + \frac{\lambda}{n} sgn(w)
\end{equation}
Where \(sgn(w)\) is the sign of w, i.e. \(1\) if \(w\) is positive, and \(-1\) if \(w\) is negative. Now we can consider the gradient descent learning rules for L1 regularization:
\begin{equation}
w \rightarrow w' - \frac{\eta \lambda}{n} sgn(w) - \eta \frac{\partial C_0}{\partial w}
\end{equation}
As usual, we can replace the final term with an average over a mini-batch if we wish.
\vspace{0.3cm}

If we compare the gradient descent rules for L1 and L2 regularization, we see that L1 regularization drives the weights down by a constant amount, where L2 regularization drives them down by an amount proportional to the size of the weight. This means that when a particular weight is large, it will be driven down by L2 regularization faster than by L1 regularization; but if that weight is small it will be driven down by L1 regularization faster. This tends to concentrate the weight of the network in a small number of important connections.

We also must note that the derivative \(\frac{\partial C}{\partial w}\) is not defined at \(w = 0\), because the function \(|w|\) has a sharp corner at \(w = 0\). This is okay though, as we can simply define \(sgn(0) = 0\), which will work because regularization is already trying to reduce weights, and it can't reduce a weight that is already 0.

\item \textbf{Dropout}
\label{sec:org9a8bbb4}
has a very different mechanism of action to L1 and L2 regularization, in that it does not modify the cost function, but the network itself. For each mini-batch, half of the hidden neurons (selected at random) are temporarily removed. The training examples are fed through the stripped down network and then backpropagated, and the weights and biases (that have not been removed) are updated as usual. This is repeated, with a new random selection of weights and biases each time. This does mean that running the full network will result in twice the neurons activating, so we halve all weights outgoing from the hidden neurons. The mechanism by which dropout reduces overfitting is analogous to averaging the outputs from multiple networks. Actually doing that is unduly expensive, but does reduce the effect of overfitting. Empirically, dropout is very effective as a regularization tool, especially in deep neural networks.

\item \textbf{Atrificially expanding the training data}
\label{sec:orge3b0b6b}
is a surprisingly effective tool for reducing overfitting. We know already that larger training datasets are less prone to overfitting than smaller ones, but collecting more data is expensive. We can, however, make small changes to our training data to make it look different on a pixel-by-pixel basis, while retaining the same desired network output. The simplest way to do this is by rotating the input image slightly. This changes the locations of the black and white pixels substantially, but is still the same image.
\end{enumerate}

\subsection{Weight Initialisation \label{org5a09953}}
\label{sec:org7b6359d}
In our work thus far we have initialised our networks' weights and biases with independent Gaussian random variables, normalised to have mean 0 and standard deviation 1. It turns out we can actually initialise our weights and biases wuite a bit better than this. Let us look at an example network to demonstrate the problem with our current initialisation.
\vspace{0.3cm}

Suppose we have a network with 1000 input neurons, with normalised Gaussians used to initialise the weights and biases. Let us focus on the 1000 weights connecting the input neurons and the first neuron in the first hidden layer. Suppose also that we have an input where half of the input neurons are on and half are off. Now consider the weighted sum \(z = \sum_j w_j x_j + b\) of inputs to our hidden neuron. Of course, 500 terms in this sum vanish, so we are left with 500 weight terms and a bias term. Therefore \(z\) itself is distributed as a Gaussian with mean 0 and standard deviation \(\sqrt{501} \approx 22.4\). That is, \(z\) has a broad gaussian distribution, and it is highly likely that \(z\) wil be much greater than 1 or much less than -1. This means the chance of saturating our hidden neuron is alarmingly high, and as we know a saturated neuron will learn very slowly due to small changes in weights having almost no effect.
\vspace{0.3cm}

This is very similar to the problem we faced earlier with saturated output neurons, which we solved by implementing a better cost function. Unfortunately, a better choice of cost function does not help at all with saturated hidden neurons.
\vspace{0.3cm}

Considering the cause of the excessively large standard deviation in the example above, for a network with \(n_{in}\) input weights, we will initialise our weights as Gaussian random variables with mean 0 and standard deviation \(\frac{1}{\sqrt{n_{in}}}\). We will still initialise our biases with mean 0 and standard deviation 1, because it really doesn't matter what biases start as. Some people initialise them all to 0.

\subsection{Code}
\label{sec:org1c4acf5}
The code for our updated network, \texttt{network2.py}, is quite similar to \texttt{network.py}. We will cover the important changes here.

\subsubsection{\texttt{default\_weight\_initializer}}
\label{sec:org3febd75}
This is the function that initialises the weights as discussed in Section \ref{org5a09953}. It is the same as the \texttt{large\_weight\_initializer}, except it divides the weights by the square root of the number of connections input to that neuron (i.e. the number of neurons in the previous layer).

\subsubsection{\texttt{CrossEntropyCost}}
\label{sec:org4b347fc}
The cost is now implemented as a class rather than a function. This is because different cost functions provide different \(\delta\) functions, so each cost class (we also have \texttt{QuadraticCost}) has two functions within it. Two important notes about this class are:
\begin{itemize}
\item \texttt{np.nan\_to\_num()} ensures that we handle the log of numbers very close to 0 appropriately.
\item \texttt{@staticmethod} tells the Python interpreter that the function that follows does not depend on the object in any way, and it is for this reason that both functions in the cost class do not take \texttt{self} as the first argument.
\end{itemize}

\subsubsection{L2 Regularization}
\label{sec:org4f2c35f}
The change from L2 regularization is hard to detect, but it is there! In the 4\(^{\text{th}}\) last line of the \texttt{update\_mini\_batch} method, un updating the weights, we have the weight decay term.

\subsection{How to choose hyper-parameters}
\label{sec:org1cf1d1a}
Without an intuition for appropriate values for a neural network's hyper-parameters, it can be extremely difficult to just pull appropriate values out of a hat, so to speak.

\subsubsection{Broad strategy}
\label{sec:orgfc9ef7c}
When using neural networks to attack a new problem, the first step is to achieve any non-trivial result, i.e. anything better than chance. This can be surprisingly difficult, especially when confronting a new kind of classification problem. There are some strategies we can adopt to help us overcome this, which mostly boil down to training faster so that we can try many different network hyper-parameters. Some techniques for this are covered below:
\vspace{0.3cm}

\textbf{Remove all training examples that aren't ones or zeros}, which will reduce our training and test sets to one fifth of their original size, which provides a training speedup by a factor of 5.
\vspace{0.3cm}

\textbf{Strip the network down to the simplest network that will do meaningful learning}. We may decide that by removing the hidden layer(s) or cutting down their size, our network can still learn in a meaningful way. This will be much faster, but of course this is inappropriate if we are trying to find the correct number of hidden layers or neurons for our network.
\vspace{0.3cm}

\textbf{Increase the frequency of monitoring}. Running \texttt{network2.py} on a laptop (without matrix optimisation) takes about 10 seconds per epoch. This isn't a huge issue, but when doing a lot of testing it can get very tiresome. Monitoring every epoch means monitoring every 50000 images. We can monitor more frequently than every epoch, or we could also reduce the training set size to achieve a similar effect. Similarly, we could reduce the validation set size from 10000 to, say, 100. \textbf{Note:} If we decrease the number of training examples we should proportionally decrease \(\lambda\).
\vspace{0.3cm}

It is worth mmentioning here that it can be tempting to discard these methods, on the assumption that we will get a result sooner or later, but this is not always true, and implementing these kinds of methods can save an immense amount of time.
\vspace{0.3cm}

We will now discuss some specific recommendations for setting parameters of our networks, focusing on the learning rate \(\eta\), the L2 regularization parameter \(\lambda\), and the mini-batch size. Many of the remarks will also apply to other hyperparameters, including those of network architecture, other regularization parameters, and some hyper-parameters we are yet to meet, such as momentum co-efficient.

\subsubsection{Learning rate}
\label{sec:org75a3ac8}
As we have seen empirically so far, a value of \(\eta\) too low will cause very slow learning, while a value of \(\eta\) too high will cause no or very erratic learning. A process to determine a good value for \(\eta\) is:

\begin{enumerate}
\item Estimate a threshold value for \(\eta\) at which the cost on the training data immediately begins decreasing, rather than oscillating or increasing.
\label{sec:orga3d3786}
This needs not be accurate, just as an order of magnitude. We could start at \(\eta = 0.01\) and then increase to 0.1 and 1 in turn until we find a value for eta at which the cost oscillates or increases. The same applies in reverse if our initial guess is too high, and the cost oscillates or increases, we should decrease it to 0.001 and 0.0001 in turn until we find a value at which the cost decreases over the first few epochs. This procedure will give us an order of magnitude estimate for the threshold value of \(\eta\).

\item Suppose we landed on \(\eta = 0.1\) as our threshold value.
\label{sec:org17cd02e}
We could then optionally bump up the value by 0.1 until we find the threshold at which the cost starts oscillating, let's say \(\eta = 0.5\).

\item Of course, the actual value for \(\eta\) should be no greater than the threshold value,
\label{sec:org80f678b}
our network wouldn't learn that way. The ideal learning rate should be something like a factor of two below the threshold value, so in our case this would be \(\eta = 0.25\). In the case of the MNIST dataset, this strategy led to these exact figures for the learning rate. Over 30 eopchs, \(\eta = 0.5\) works perfectly well also. \\

\item Holup, why are we measuring this based on the cost rather than accuracy on validation data?
\label{sec:org9924578}
We will use accuracy on validation data to adjust all of the other parameters, and the choice to use cost to quantify this is really just a preference. This preference is rooted in the fact that learning rate is intended to control the step size in gradient descent, and only incidentally affects the classification accuracy of the network. The other parameters are directly intended to improve classification accuracy.
\end{enumerate}

\subsubsection{Use early stopping to determine the number of epochs to train for}
\label{sec:org958eb52}
Using early stopping eliminates the number of epochs parameter altogether, but necessarily introduces another parameter to determine when to stop. A common example, at least for MNIST, is the point at which no improvement is seen in 10 epochs. Even at this point, we could be missing future learning (if we are unlucky), but this helps to control the training time until the point at which we have come to know our network well. Once the other parameters are better set, we can relax this imposition to no improvement for, say, 20 epochs or even 50 epochs.

\subsubsection{Learning rate schedule}
\label{sec:org2e75205}
So far we have been holding the learning rate \(\eta\) constant, but it is often desirable to vary the learning rate. Intuitively, we want a higher learning rate earlier on in our program when our weights are badly wrong, and later we want a lower learning rate to fie tune the weights, avoiding overshooting the local minima of the cost function. \\

A common way to implement this is to use a similar idea to early stopping, hold the learning rate constant until the validation accuracy gets worse. At this pint, we decrease the learning rate by, say, a factor of two or ten. We repeat this until our learning rate is a factor of 1024 or 1000 below its original value, then we terminate.\\

This can lead to a world of headaches, so it is usually best to start with a fixed learning rate while we get to know our network, then implement a learning rate schedule.

\subsubsection{The regularization parameter}
\label{sec:org63f9a94}
It is best to start with \(\lambda = 0\) and find a good value for \(\eta\) first. Once this is done, start at \(\lambda = 1.0\) (which is a completely arbitrary choice btw) and increase or decrease by factors of 10 as required to improve performance on validation data. Once the order of magnitude is found, \(\lambda\) can be fine tuned. Once that is done, return to \(\eta\) and make any necessary adjustments.

\subsubsection{Mini-batch size}
\label{sec:org7c35b98}
To answer the question of how to set mini-batch size, consider online learning (i.e. mini-batch size 1). A concern about online learning is that it will provide an inaccurate estimate of the gradient of the cost function. In reality this isn't all that important, so long as our gradient estimate still tends to decrease the cost. The positive aspect of online learning is that we are constantly updating our weights, so each new estimate is based on the previous, slightly improved one. \\

The optimal solution is, unsurprisingly, somewhere between online learning and learning with enormous mini-batches. A mini-batch size too small doesn't make good use of optimised matrix algebra libraries that speed up the backpropagation process, and one too big doesn't stop to learn often enough. Fortunately, mini-batch size is independent of other network hyper-parameters (except network architecture), so once acceptable values for the oher hyper-parameters have been found, a little bit of trial and error can be employed to optimise the mini-batch size.

\subsection{Other techniques}
\label{sec:orgbf8043e}
\subsubsection{Variations on stochastic gradient descent}
\label{sec:orgafddd6e}
\begin{enumerate}
\item \textbf{The Hessian Technique:}
\label{sec:orgd25996b}
If we imagine our cost function as a function of the weights of the system (which it is), so \(C = C(w)\) for \(w = w_1, w_2, ... ,w_n\), we can approximate the cost function near a point using a Taylor approximation:
\begin{equation}
C(w + \Delta w) = C(w) + \sum_j \frac{\partial C}{\partial w_j} \Delta w_j + \frac{1}{2} \sum_{jk} \Delta w_j \frac{\partial^2 C}{\partial w_j \partial w_k} \Delta w_k + ...
\end{equation}

Discarding any higher order terms, this can be written more compactly as:
\begin{equation} \label{eqn:compact-hessian}
C(w + \Delta w) = C(w) + \nabla C \dot \Delta w + \frac{1}{2} \Delta w^T H \Delta w
\end{equation}

Where \(\nabla C\) is the usual gradient vector, and \(H\) is a atrix known as the \emph{Hessian matrix}, whose \textit{jk}\(^{\text{th}}\) entry is \(\frac{\partial^2 C}{\partial w_j \partial w_k}\). We can use calculus to show that the expression can be minimised by choosing
\begin{equation}
\Delta w = -H^{-1} \nabla C
\end{equation}
Providing that Equation \ref{eqn:compact-hessian} is a good approximate expression for the cost function, we would expect that moving from point \(w\) to \(w + \Delta w = w - H^{-1} \nabla C\) should significantly reduce the cost function. This suggests we can so something very similar to gradient descent, starting with random weights \(w\), then updating the weights:
\begin{equation}
w' = w - \eta H^{-1} \nabla C
\end{equation}
Where \(\eta\) is the learning rate. \\

There are theoretical and empirical results showing that this hessian technique converges in fewer steps than standard gradient descent, which is largely a result of incorporating second order changes in the cost function. So why aren't we using it? Despite its qualities, it is \textbf{very difficult to apply in practice}, partly due to the sheer size of the Hessian matrix. A neural network with 10\(^{\text{7}}\) weights and biases will have a corresponding Hessian matrix with 10\(^{\text{14}}\) entries! There are, however, variations on gradient descent inspired by the Hessian technique, which avoid the problem of overly large matrices. One example is momentum-based gradient descent.

\item \textbf{Momentum-based gradient descent:}
\label{sec:orge676e1f}
Thinking back to the notion of the ball rolling down the hill, it was important to be aware that gradient descent didn't behave exactly like a ball rolling down a hill. If we stick with this analogy, the advantage of the Hessian technique is its ability to capture the velocity of the ball, not just its position on the hill. Momentum- based gradient descent emulates this by adding an element of velocity to the parameters we're trying to optimise, as well as an element of friction, giving the ball a kind of momentum. The gradient acts to change the velocity, not (directly) the position, and the friction gradually reduces the velocity. \\

For a more precise definition, we introduce the variables \(v = v_1, v_2, ... , v_n\), one for each \(w_j\) variable. then we replace the gradient descent update rule \(w \rightarrow w' = w - \eta \nabla C\) with:
\begin{align}
v \rightarrow v' &= \mu v - \eta \nabla C \\
w \rightarrow w' &= w + v'
\end{align}
Where \(\mu\) is a hyper-parameter which controls the amount of damping, or friction, in the system. \\

To build up an understanding of how this works, imagine the case where \(\mu = 1\), which corresponds to no friction. The "force" term \(\nabla C\) is modifying the velocities, and the velocities are controlling the rate of change of the weights. We build up the velocity by repeatedly adding gradient terms to it, which means if the gradient is roughly the same through several rounds of learning, we could build up quite a considerable velocity. \\

This enables us the momentum technique to work considerably faster than vanilla gradient descent, but what happens when we get to the bottom? With all that velocity we could easily overshoot. With \(\mu = 0\) (maximum friction), the equations reduce to vanilla gradient descent. In practice, a value between 1 and 0 will have the optimal behaviour of building velocity while minimising overshoot. \\

The name, incidentally, of the hyper-parameter \(\mu\), is the poorly chosen \emph{momentum coefficient}. Poorly chosen because it much more closel affects friction than momentum, but that's the name.
\end{enumerate}
\subsubsection{Other models of artificial neuron}
\label{sec:org2a2ad72}
In principle, a network built from sigmoid neurons can compute any function. In practice, however, networks built from different neurons can outperform sigmoid neuron networks. Let us look at some other neuron models in use today.

\begin{enumerate}
\item \textbf{tanh neuron:}
\label{sec:org4b99878}
The tanh neuron (pronounced "tanch") replaces the sigmoid function with the hyperbolic tangent function. It still takes in the same input, \(wx + b\), and looks very similar to the sigmoid function both graphically and algebraically. The tanh function is defined as:
\begin{equation}
\tanh(z) \equiv \frac{e^z - e^{-z}}{e^z + e^{-z}}
\end{equation}
The tanh function is shaped very similarly to the sigmoid function, with the main difference being that its range is \([-1,1]\) rather than \([0,1]\). This means that the outputs of (and potentially inputs to) the system may have to be normalised differently to those with sigmoid neurons. \\

\item \textbf{Rectified linear neuron:}
\label{sec:orgdd5b432}
The output of a rectified linear neuron is given by:
\begin{equation}
\max(0, w x + b)
\end{equation}
Plotting the activation \(a\) of a rectified linear neuron as a function of \(Z\), it looks like the function \(a = Z\), but only in the domain \([0,\infty)\), and is not defined in the domain \((-\infty,0)\). \\

Clearly this is quite different to the sigmoid and tanh activation functions, but it can still be used to compute any function, and can still be trained using backpropagation and gradient descent. The primary advantage of the rectified linear neuron is that it doesn't saturate, but at the same time, any negative input to a rectified linear neuron won't induce any learning at all. \\

Both tanh neurons and rectified linear neurons suffer from a lack of research into the most appropriate circumstances in which to use them, and the same goes for all types of neuron. Generally speaking, sigmoid neurons will perform just fine, but it is worth knowing about the different alternatives that exist for developing our networks with.
\end{enumerate}

\section{A visual proof that neural nets can compute any functon}
\label{sec:org16ddd46}
See the online book, this is all very straightforward. The takeaway is that a network with a single hidden layer can compute any continuous function to an arbitrary degree of accuracy.
\clearpage

\section{Why are deep neural networks hard to train?}
\label{sec:org6dd5478}
There is an intuitive sense in which deep networks should be able to learn better than shallow ones. The first layer can detect basic patterns, like edges and corners and such, with the second detecting compounds of these patterns and so on. Along with this are theoretical results showing that deep neural networks are intrinsically more powerful than shallow ones. Unfortunately, there are often situations where our network won't train as effectively as we hope, manifesting particularly in different layers training at different speeds. This is often due to our gradient based learning techniques.

\subsection{The vanishing gradient problem}
\label{sec:orgedca5ba}
The vanishing gradient problem is the phenomenon where the last layer in a network trains the fastest, and the first layer in the network trains the slowest, with intermediate layers following in a linear fashion. The gradient tends to get smaller as we move backward through the hidden layers, which causes this learning slowdown.

\subsection{The cause of the vanishing gradient problem}
\label{sec:org44aa619}
Consider a network with only one neuron in each layer, and three hidden layers. Using our regular nomenclature, we denote the weight of the link between the input neuron and the first hidden neuron \(w_1\), with its bias \(b_1\), and so on throughout the network. We will study the gradient \(\frac{\partial C}{\partial b_1}\) associated with the first hidden neuron. \\

Suppose we make a small change \(\Delta b_1\) in the bias \(b_1\). That will trigger a cascade of changes in the rest of the network:
\begin{itemize}
\item First it will cause a change \(\Delta a_1\) in the activation of the first hidden neuron
\item Then it will cause a change \(\Delta z_2\) in the weighted input to the second hidden neuron
\item This will cause a change \(\Delta a_2\) in the activation of the second hidden neuron
\item etc.
\item Finally, this will cause a change \(\Delta C\) in the cost at the output.
\end{itemize}
We can say:
\begin{equation} \label{eqn:partial_approx_delta}
\frac{\partial C}{\partial b_1} \approx \frac{\Delta C}{\Delta b_1}
\end{equation}

Which suggests that we can find an expression for \(\frac{\partial C}{\partial b_1}\) by carefully tracking the effect of each step in this cascade. Let us start by considering the first of the above dot points. We have \(a_1 = \sigma(z_1) = \sigma(w_1 a_0 + b_1)\), so
\begin{align}
\Delta a_1 &\approx \frac{\partial \sigma(w_1 a_0 + b_1)}{\partial b_1} \Delta b_1 \\
&= \sigma'(z_1) \Delta b_1
\end{align}

This change \(\Delta a_1\) then causes a change in the weighted input \(z_2 = w_2 a_1 + b_2\) to the second hidden neuron:
\begin{align}
\Delta z_2 &\approx \frac{\partial z_2}{\partial a_1} \Delta a_1 \\
&= w_2 \Delta a_1
\end{align}

Combining the terms from the above two expressions, the change \(\Delta b_1\) in \(b_1\) causes \(z_2\) to change as:
\begin{equation}
\Delta z_2 \approx \sigma'(z_1) w_2 \Delta b_1
\end{equation}

We can continue in this fashion, tracking the way changes propagate through the network. At each neuron we pick up a \(\sigma'(z_j)\) term, and through each weight we pick up a \(w_j\) term. We also have the term \(\frac{\partial C}{\partial a_4}\) relating the cost to the final neuron's activation at the end of the expression, because the cost is a function of the final activation. This leaves us with:
\begin{equation}
\Delta C \approx \sigma'(z_1) w_2 \sigma'(z_2) w_3 \sigma'(z_3) w_4 \sigma'(z_4) \frac{\partial C}{\partial a_4} \Delta b_1
\end{equation}
Dividing by \(\Delta b_1\) and invoking Equation \ref{eqn:partial_approx_delta}, we get:
\begin{equation} \label{eqn:cost_wrt_b1}
\frac{\partial C}{\partial b_1} = \sigma'(z_1) w_2 \sigma'(z_2) w_3 \sigma'(z_3) w_4 \sigma'(z_4) \frac{\partial C}{\partial a_4}
\end{equation}

\subsubsection{Why the vanishing gradient problem occurs}
\label{sec:org97ad96f}
With the exception of the final term, Equation \ref{eqn:cost_wrt_b1} is a product of the terms \(w_j \sigma'(z_j)\). Looking at this term's components:
\begin{itemize}
\item \(\sigma'(z)\) is a normal (ish) distribution, which reaches a maximum at \(z = 0\) of \(\sigma'(0) = \frac{1}{4}\).
\item With our new way of initialising weights and biases, weights are initialised as gaussian variables with mean 0 and standard deviation 1, so the weights will usually satisfy \(|w| < 1\)
\end{itemize}
Combining these two observations, it is clear that \(w_j \sigma'(z_j)\) will usually satisfy \(|w_j \sigma'(z_j)| < \frac{1}{4}\). When we take the product of many such terms, the product will tend to exponentially decrease. \\

For comparison, we will consider the equivalent of Equation \ref{eqn:cost_wrt_b1} for \(b_3\). We haven't explicitly calculated this expression, but we can just remove terms from \Equation \ref{eqn:cost_wrt_b1}.
\begin{equation} \label{eqn:cost_wrt_b3}
\frac{\partial C}{\partial b_3} = \sigma'(z_3) w_4 \sigma'(z_4) \frac{\partial C}{\partial a_4}
\end{equation}
These two equations are very similar, but \Equation \ref{eqn:cost_wrt_b3} has two fewer terms of the form \(w_j \sigma'(z_j)\), and so the gradient \(\frac{\partial C}{\partial b_1}\) will be a factor of 16 (or more) smaller than \(\frac{\partial C}{\partial b_3}\). \textbf{This is essentially the origin of the vanishing gradient problem.} \\

This whole argument hingeson the fact that \(|w_j \sigma'(z_j)| < \frac{1}{4}\), but if the weights grow during training (or are differently initialised), to the point that \(|w_j \sigma'(z_j)| > 1\), then we will no longer have a vanishing gradient, rather the gradient will exlplode! This is called, of course, the \emph{exploding gradient problem}.

\section{Deep learning}
\label{sec:org0cfdf86}
\subsection{Introducing convolutional networks}
\label{sec:org72a8644}
In our earlier networks, every neuron in layer \(n\) is connected to every neuron in layer \(n+1\) and every neuron in layer \(n-1\). At the input layer, this architecture treats pixels that are far apart the same as it treats adjacent pixels. In reality, it is much more likely that the relationship between adjacent pixels is more important than that of pixels on opposite ends of the image, when we are trying to classify images. Convolutional neural networks use a special architecture which is much better adapted to classifying images, and as such is the most common form of network used in image recognition. \\

Convolutional neural networks use three basic ideas: \emph{local receptive fields}, \emph{shared weights}, and \emph{pooling}.

\subsubsection{Local receptive fields}
\label{sec:orge998e1b}
Previously we have visualised the input neurons in a straight vertical line, but let us now view them as a \(28 \times 28\) grid. Rather than each neuron in the first hidden layer being connected to all of the input neurons, we will connect a small region, say a \(5 \times 5\) square in the top left corner, corresponding to 25 input neurons, to the first neuron in the first hidden layer. This region is called the \emph{local receptive field}, and we slide it one neuron across and connect all of the input neurons in our new region to the second neuron in the first hidden layer, and so on sliding across until we reach the other side of the image, then jump one input neuron down and repeat, until we have connected all local receptive fields to a neuron in the second hidden layer. This exact architecture (\(5 \times 5\) region jumping one neuron at a time) will lead to a hidden layer of size \(24 \times 24\) The size of the region and the number of neurons to jump by can both be varied if desired.

\subsubsection{Shared weights and biases}
\label{sec:orgd54da36}
Naturally, each of the \(24 \times 24\) neurons in the first hidden layer will have a bias and a \(5 \times 5\) array of weights connecting to it. Unlike previous networks, however, we are going to use \textbf{the same} bias and array of weights for each neuron in the first hidden layer. This means that for the \(j,k^{\text{th}}\) hidden neuron, the output is:
\begin{equation} \label{eqn:convolution}
\(\sigma\)(b + \(\sum_{\text{l=0}}^{\text{4}}\) \(\sum_{\text{m=0}}^{\text{4}}\) w\(_{\text{l,m}}\) a\(_{\text{j+l,k+m}}\))
\end{equation}

This means that all the neurons in the first hidden layer detect exactly the same feature, just at different locations across the image. For this reason, we somtimes call the map from the input layer to the hidden layer a \emph{feature map}. We call the weights defining the feature map the \emph{shared weights}, and the bias defining the feature map the \emph{shared bias}. The shared weights and bias are often said to define a kernel, or filter. \\

Our hidden layer detects a feature in the image, but we will often want to detect many features, so most convolutional networks will have many hidden layers in paralel, all connected to the input layer. Because of the shared weights and biases, we have only 26 parameters per feature map, so having many in paralel isn't a big problem at all. By comparison, our original network has \(784 \times 30\) weights and 30 biases, a total of 23550 parameters! Of course the direct comparison of both networks isn't really valid, but it's good for framing the difference mentally. \\

The name \emph{convolutional} comes from the fact that Equation \ref{eqn:convolution} is sometimes known as a convolution. More concisely, people sometimes write that operation as \(a^1 = \sigma(b + w * a^0)\), where \(a^1\) denotes the set of output activations from a feature map, \(a^0\) is the set of input activations, and \(*\) is called the convolution operation.

\subsubsection{Pooling layers}
\label{sec:org0b4b06c}
In addition to the convolutional layers, convolutional networks contain \emph{pooling layers}, which are usually used immediately after convolutional layers. They exist to simplify the information in the output of the convolutional layer by condensing a region (say, \(2 \times 2\)) into a single neuron. \\

One common procedure for pooling is known as \emph{max-pooling}, where the pooling unit simply outputs the maximum activation in the \(2 \times 2\) input region. We apply \emph{max-pooling} to each feature map separately, so there is a max-pooling layer for each one. We can think of it of a way of asking our network if there is a feature in the rough region, throwing away exact positional information but drastically cutting the number of parameters needed in later layers. \\

Max pooling isn't the only technique for pooling. Another common approach used is called \emph{L2 pooling}, where we take the square root of the sum of the squares of the activations in the \(2 \times 2\) region.

\subsubsection{Putting it all together}
\label{sec:orgf4321ae}
We can now complete the construction of our convolutional neural network. We add to the end of our existing structure 10 output neurons, corresponding to the 10 possible values of a MNIST digit.
\subsection{Convolutional networks in practice}
\label{sec:org8888709}
We will now be itroducing \texttt{network3.py}, which we will use to build a convolutional neural network. Our earlier programs constructed neural networks from first principles, but this time we will use a library known as \texttt{Theano}. \texttt{Theano} makes it easy to implement backpropagation for convolutional neural networks, as it automatically computes all of the mappings involved, and does so quite a bit faster than our easy to read code. It also allows us to use a GPU, if one is available.
\end{document}


\documentclass{article}
\usepackage[top=1.0in,bottom=1.0in,left=1.0in,right=1.0in]{geometry}
\usepackage{amsmath,amssymb,amsthm,amsfonts}
\usepackage[utf8]{inputenc}
\usepackage{hyperref}

\newtheorem{definition}{Definition}[section]
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem*{remark}{Remark}

\title{Non-negligible Functions and Reduction Proofs}
\author{coinstudent2048}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present a lemma about non-negligible functions that is helpful in reduction proofs in cryptography. We also provide a reduction proof as a demonstration.
\end{abstract}

\section{The Thing}
Let $\mathbb{R}_{\ge 0}$ be the set of non-negative real numbers. Let us define the concept of \textit{negligible function} first:

\begin{definition}\label{negl}
A function $f:\mathbb{N}\rightarrow\mathbb{R}_{\ge 0}$ is \textbf{\em negligible} if for all polynomial $p(\cdot)$ there exists an $N\in\mathbb{N}$ such that for all integers $n>N$ it holds that $f(n)<\frac{1}{p(n)}$.
\end{definition}
\noindent Definition \ref{negl} is from Katz \& Lindell \cite{katz-lindell}. We now prove the following lemma:

\begin{lemma}\label{non-negl-exp}
If $f:\mathbb{N}\rightarrow\mathbb{R}_{\ge 0}$ is non-negligible, then $g(\cdot)=f(\cdot)^m$ for any $m\in\mathbb{N}$ and $m>1$ is non-negligible.
\end{lemma}
\begin{proof}
The function $f$ being not negligible means that there exists a polynomial $p(\cdot)$ such that for all $N\in\mathbb{N}$, there exists an $n>N$ such that $f(n)\ge\frac{1}{p(n)}$. Let $p_f(\cdot)$ be such polynomial and $n_f$ be such $n>N$. Then setting $p_g(\cdot)=p_f(\cdot)^m$ and $n_g=n_f$ suffices for non-negligibility of $g$ because $f(n_f)\ge\frac{1}{p_f(n_f)}\implies f(n_f)^m\ge\frac{1}{p_f(n_f)^m}$.
\end{proof}
\noindent Lemma \ref{non-negl-exp} justifies the usage of finite number of ``breaks'' of one hardness assumption in reduction proofs. For a start, the probability of breaking the hardness assumption $\textsf{HA}$ is a function of the security parameter $\lambda$. Just here we denote this as $\textsf{Pr}[\textsf{HA}(\lambda)]$. Hence, for $m>1$, the probability for breaking $\textsf{HA}$ $m$ times, $\textsf{Pr}[\wedge_{i=1}^{m}{\textsf{HA}_i(\lambda)}]\ge\textsf{Pr}[\textsf{HA}(\lambda)]^m$. Now Lemma \ref{non-negl-exp} says that if $\textsf{Pr}[\textsf{HA}(\lambda)]$ is non-negligible (or equivalently, for all negligible function $\textsf{negl}(\lambda)$, $\textsf{Pr}[\textsf{HA}(\lambda)]\ge\textsf{negl}(\lambda)$), then $\textsf{Pr}[\textsf{HA}(\lambda)]^m$ must also be non-negligible and hence $\textsf{Pr}[\wedge_{i=1}^{m}{\textsf{HA}_i(\lambda)}]$ is also non-negligible.

\section{The Demo}
Let $\mathbb{G}$ be a cyclic group where the Discrete Logarithm (DL) assumption holds, and $\mathbb{F}$ be its scalar field. We now present a hardness assumption used in Bulletproofs \cite{bp}, Bulletproofs+ \cite{bp-plus}, and Halo \cite{halo}:

\begin{definition}[Discrete Logarithm Relation Assumption]
	DL Relation assumption holds relative to $\emph{\textsf{Setup}}$ if for all $n \ge 2$ and  $\emph{\textsf{PPT}}$ adversary $\mathcal{A}$, there exists a negligible function $\emph{\textsf{negl}}(\lambda)$ such that
	\begin{align*}
		\emph{\textsf{Pr}}\left[
		\begin{array}{c|c}
			\begin{gathered}
				\exists i \in \{1, \ldots, n\}: x_i \ne 0 \\
				\wedge \sum_{i=1}^{n} x_i G_i = 0
			\end{gathered}
			&
			\begin{gathered}
				(\mathbb{G}, \mathbb{F})\leftarrow\emph{\textsf{Setup}}(1^{\lambda}); \\
				\{G_i\}_{i=1}^n \xleftarrow{\$}\mathbb{G}^n; \\
				\{x_i\}_{i=1}^n \leftarrow\mathcal{A}(\mathbb{G}, \mathbb{F}, \{G_i\}_{i=1}^n) \\
			\end{gathered}
		\end{array}
		\right]
		\le \emph{\textsf{negl}}(\lambda).
	\end{align*}
\end{definition}
\noindent Note that the $\sum_i x_i G_i$ operation is also called \textit{multi-scalar multiplication}.

\begin{theorem}\label{equiv-demo}
DL relation assumption holds if and only if DL assumption holds.
\end{theorem}
\begin{proof}
The forward direction is trivial. For the backward direction, we prove by induction on $n$:

\textit{Base case ($n=2$)}: Assume that $\mathcal{A}$ breaks DL relation: with non-negligible probability, for $G_1, G_2 \xleftarrow{\$} \mathbb{G}$, $\mathcal{A}$ outputs $x_1, x_2 \in \mathbb{F}$ such that $x_1 G_1 + x_2 G_2 = 0$. Then $G_1 = (-x_2 / x_1) G_2$, breaks DL assumption.

\textit{Inductive case}: Assume that the backward direction of Theorem \ref{equiv-demo} holds for case $n$. Then we prove the same for case $n+1$. Assume that $\mathcal{A}$ breaks DL relation for case $n+1$. By Lemma \ref{non-negl-exp}, $\mathcal{A}$ can break it \textit{twice}: with non-negligible probability, for $\{G_i\}_{i=1}^{n+1} \xleftarrow{\$} \mathbb{G}^{n+1}$, $\mathcal{A}$ outputs $\{x_i\}_{i=1}^{n+1}$ \textit{and} $\{x'_i\}_{i=1}^{n+1}$ such that both satisfy the multi-scalar multiplication with $\{G_i\}_{i=1}^{n+1}$ to zero. Now observe that
\begin{align*}
x'_1 \sum_{i=1}^{n+1} x_i G_i = x'_1 \cdot 0 = 0\ \wedge\ x_1 \sum_{i=1}^{n+1} x'_i G_i = x_1 \cdot 0 = 0 \\
\implies \sum_{i=1}^{n+1} x'_1 x_i G_i - \sum_{i=1}^{n+1} x_1 x'_i G_i = 0 - 0 = 0 \\
\implies \sum_{n=1}^{n+1} (x'_1 x_i - x_1 x'_i) G_i = 0 \\
\implies  \sum_{n=2}^{n+1} (x'_1 x_i - x_1 x'_i) G_i = 0
\end{align*}
with the last implication because $x'_1 x_1 - x_1 x'_1 = 0$. Now the last implication has only $n$ addends, hence this breaks DL relation assumption for case $n$. From the above assumption of the backward direction of Theorem \ref{equiv-demo} holding for case $n$, this must also break DL assumption.
\end{proof}
\bibliographystyle{plain}
\bibliography{non-negl}
\end{document}


\section{Scenarios}\label{sec:scenarios}

In order to ease the analysis, we have defined three scenarios with different
values for \(A\), \(\frac{X}{Y}\) and \(N\). Inside each scenario, we will vary
all other factors and we will evaluate the impact of these factors in the
performances of the network. As said in \secref{sec:factors}, we will also vary
the number of users \(N\) but we will not consider this a tunable factor to
improve the performances of the network.

The scenarios defined are the following:
\begin{enumerate}
	\item \(A = 22500m^2\), \(\frac{X}{Y} = 1\) \idest{\(150m\times150m\)
		square} and \(N = 1125\mathit{users}\) \idest{\(\frac{N}{A} =
		\frac{1125\mathit{users}}{22500m^2} = 0.05\mathit{users}/m^2 =
		\frac{1}{20}\mathit{users}/m^2\)}. This is the ``high density
		scenario''.
	\item \(A = 250000m^2\), \(\frac{X}{Y} = 1\) \idest{\(500m\times500m\)
		square} and \(N = 1250\mathit{users}\) \idest{\(\frac{N}{A} =
		\frac{1250\mathit{users}}{250000m^2} = 0.005\mathit{users}/m^2 =
		\frac{1}{200}\mathit{users}/m^2\)}. This is the ``low density
		scenario''.
	\item \(A = 30000m^2\), \(\frac{X}{Y} = 3\) \idest{\(300m\times100m\)
		rectangle} and \(N = 1500\mathit{users}\) \idest{\(\frac{N}{A} =
		\frac{1500\mathit{users}}{30000m^2} = 0.05\mathit{users}/m^2
		\simeq \frac{1}{20}\mathit{users}/m^2\)}. This is the
		``rectangular scenario'' (high density).
\end{enumerate}

\input{design/scenarios/calibration}


\subsection{Example: Inverted slider-crank Mechanism}

\begin{frame}
	\begin{block}{Example 3: Inverted slider-crank Mechanism}
		\begin{table}
			\begin{minipage}{0.5\linewidth}
				\begin{tabular}{l|l}
					      & $l_{AD}=l_1=0.35m$\\
					Given & $l_{BC}=l_2=0.20m$\\
					      & $l_{AC}=0.15m$\\
					      & $\theta_1=60^{\circ}$\\\hline
					Find  & $\vb{r}{B}$, $\vb{r}{D}$
				\end{tabular}
			\end{minipage}\hfill
				\begin{minipage}{0.5\linewidth}
					\includegraphics[width=30mm]{images/Inverted-R-RRT.png}
				\end{minipage}
		\end{table}
	\end{block}
\emph{Solution}\vskip2.5mm
Position of joint $B$: $\displaystyle \vb{r}{B} = x_B\ih + y_B\jh = l_{AB}\cos{\theta_1}\ih + l_{AB}\sin{\theta_1}\jh$\\
Position of joint $C$: $\displaystyle \vb{r}{C} = x_C\ih + y_C\jh = 0.15\ih$\\
Position of joint $D$: $\displaystyle \vb{r}{D} = x_D\ih + y_D\jh = l_1\cos{\theta_1}\ih + l_1\sin{\theta_1}\jh$
\end{frame}

\begin{frame}
\emph{Solution}\vskip2.5mm
Position of joint $B$: $\displaystyle \vb{r}{B} = x_B\ih + y_B\jh = l_{AB}\cos{\theta_1}\ih + l_{AB}\sin{\theta_1}\jh$\\
Position of joint $C$: $\displaystyle \vb{r}{C} = x_C\ih + y_C\jh = 0.15\ih$\\
Position of joint $D$: $\displaystyle \vb{r}{D} = x_D\ih + y_D\jh = l_1\cos{\theta_1}\ih+l_1\sin{\theta_1}\jh$
\[
\Rightarrow\displaystyle(x_B-x_C)^2+y_B^2=l_2^2
\]
\[
\text{ or } (l_{AB}\cos{60^\circ}-0.15)^2+l_{AB}\sin{60^\circ}=l_2^2
\]
Solving the system of equations yields $l_{AB}>0$ and $l_{AB}<0$. Then, choose $l_{AB}>0$ and substitute the result into $\vb{r}{B}$
\end{frame}

\begin{frame}{MATLAB R2019a code}
\lstinputlisting[style=Matlab-editor, basicstyle=\mlttfamily]{codes/Inverted-RRRT-position.m}
\end{frame}
\begin{frame}{Plotting using MATLAB R2019a}
\lstinputlisting[style=Matlab-editor, basicstyle=\mlttfamily]{codes/Inverted-RRRT-plot.m}
\end{frame}
\begin{frame}{Output figure}
\centering
\includegraphics[width=100mm]{images/Inverted-RRRT-plot.png}
\end{frame}
\begin{frame}{Trajectory plotting using MATLAB R2019a}
\lstinputlisting[style=Matlab-editor, basicstyle=\mlttfamily]{codes/Inverted-RRRT-trajectory.m}
\end{frame}
\begin{frame}{Output figure}
\centering
\includegraphics[width=100mm]{images/Inverted-RRRT-trajectory.png}
\end{frame}