%!TEX root=../labNotes.tex

\subsection*{06 October 2014}

\begin{enumerate}
	\item RIO3 Rotavirus activation and series dilution for P63A plated cells
		\begin{enumerate}
			\item Combined $400\mu$L SA11 reclaimed from RIO3-transfected cells (vial 3 of 5) and $2\mu$L trypsin
			\item Incubated for 1 hour in a $37^{\circ}$C water bath
			\item Added $2.7$mL serum-free M199 to each of 8 $15$mL tubes
			\item To the first tube, added $300\mu$L rotavirus solution to a final concentration of $10^{-1}$
			\item Mixed contents by vertex
			\item To the next tube, added $300\mu$L rotavirus solution from the previous tube to a final concentration of $10^{-2}$
			\item Mixed contents by vertex
			\item Repeated serially for the remaining tubes ending with a concentration of $10^{-8}$ in the final tube
			\item Stored at $4^{\circ}$C
		\end{enumerate}
	\item RIO3 Rotavirus infection of P63A plated cells
		\begin{enumerate}
			\item Washed monolayer in each well of 2 6-well plates twice with serum-free M199 by dumping
			\item Added $\sim 1$mL rotaviral solution to wells in concentrations from $10^{-3}$ to $10^{-8}$ with 2 wells being infected at each concentration of rotavirus
			\item Spread cells evenly by shaking
			\item Incubated at $37^{\circ}$C for 1 hour
		\end{enumerate}
	\item NSV Rotavirus activation and series dilution for P63A plated cells
		\begin{enumerate}
			\item Combined $400\mu$L SA11 reclaimed from NSV-transfected cells (vial 4 of 5) and $2\mu$L trypsin
			\item Incubated for 1 hour in a $37^{\circ}$C water bath
			\item Added $2.7$mL serum-free M199 to each of 8 $15$mL tubes
			\item To the first tube, added $300\mu$L rotavirus solution to a final concentration of $10^{-1}$
			\item Mixed contents by vertex
			\item To the next tube, added $300\mu$L rotavirus solution from the previous tube to a final concentration of $10^{-2}$
			\item Mixed contents by vertex
			\item Repeated serially for the remaining tubes ending with a concentration of $10^{-8}$ in the final tube
			\item Stored at $4^{\circ}$C
		\end{enumerate}
	\item NSV Rotavirus infection of P63A plated cells
		\begin{enumerate}
			\item Washed monolayer in each well of 2 6-well plates twice with serum-free M199 by dumping
			\item Added $\sim 1$mL rotaviral solution to wells in concentrations from $10^{-3}$ to $10^{-8}$ with 2 wells being infected at each concentration of rotavirus
			\item Spread cells evenly by shaking
			\item Incubated at $37^{\circ}$C for 1 hour
		\end{enumerate}
	\item Agarose overlay of P63A plated cells
		\begin{enumerate}
			\item Liquefied 1.2\% agarose by microwave and equilibrated to $55^{\circ}$C
			\item For each plate of cells (4 total), prepared 1 vial of $10$mL agarose, $10$mL 2x EMEM, $5\mu$L $2$mg/mL trypsin
			\item Aspirated viral inoculant from each well
			\item To each well, added $\sim3$mL agarose solution
			\item Let agarose solidify
			\item Incubated at $37^{\circ}$C
		\end{enumerate}
	\item Plated MA104 cells --- Passage 64, Flask A
		\begin{enumerate}
			\item Aspirated cell culture medium
			\item Rinsed cells in $10$mL 1x PBS; aspirated PBS
			\item Rinsed cells in $5$mL $0.05$\% trypsin; aspirated trypsin
			\item Bathed cells in $5$mL $0.05$\% trypsin
			\item Incubated cells at $37^{\circ}$C until all cells detached from flask
			\item Added $20$mL complete M199 to flask\footnote{I goofed and added $5$mL too much complete M199. Small goof, but FYI.}
			\item Took cell count by combining $10\mu$L cell mixture with $10\mu$L trypan blue:

				\begin{align*}
				% Initial
				\text{[cells]} &= \frac{1.10\e{5}\text{ cells}}{1\text{mL}} \\
				% Cells / flask
				\frac{\text{cells}}{\text{flask}} &= \frac{1.10\e{5}\text{ cells}}{1\text{mL}} \cdot 25\text{mL} &= \frac{2.75\e{6}\text{ cells}}{25\text{mL}}\\
				% Cells / 10mL
				\frac{\text{cells}}{20\text{mL cell mix}} &= \frac{1.10\e{5}\text{ cells}}{1\text{mL}}\cdot 20\text{mL} &= \frac{2.20\e{6}\text{ cells}}{20\text{mL}}\\
				% Cells / conical vial
				\frac{\text{cells}}{150\text{mL vial}} &= \frac{2.20\e{6}\text{ cells}}{150\text{mL}} &= \frac{1.47\e{4}\text{ cells}}{\text{mL}}\\
				% Cells / well
				\frac{\text{cells}}{3\text{mL well}} &= \frac{1.47\e{4}\text{ cells}}{\text{mL}} \cdot 3\text{mL} &= \frac{4.40\e{4}\text{ cells}}{\text{well}}\\
				\end{align*}
			\item Added $130$mL complete M199 and $20$mL cell mixture to $250$mL conical vial for final volume of $150$mL
			\item Transferred $3$mL solution to each well of 8 6-well plates
			\item Spread cells evenly by shaking
			\item Incubated at $37^{\circ}$C
		\end{enumerate}
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%
% Tuesday Plan
%%%%%%%%%%%%%%%%%%%%
% All's quiet...
%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%
% Wednesday Plan
%%%%%%%%%%%%%%%%%%%%
% Still nothing but crickets...
%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%
% Thursday Plan
%%%%%%%%%%%%%%%%%%%%
% Neutral red overlay of P63A
% plated cells
%
% Split P64C
%
% Plate P64B cells for infection
% with remaining samples from
% transfected cells
%
% Infect P64A plates with samples:
% RIO3 #1, 2; NSV #3, 5
%%%%%%%%%%%%%%%%%%%%

\subsection*{09 October 2014}

\begin{enumerate}
	\item Disposal of P64A Plated Cells

	An unknown error occurred when plating cells on 06 Oct. causing none to survive. The culture medium was aspirated from each well and the plates disposed of appropriately.

	\item Neutral red overlay of P63A plated cells (NSV)
		\begin{enumerate}
			\item Prepared a vial with $7.5$mL 1.2\% agarose, $7.5$mL 2x EMEM, and $750\mu$L neutral red
			\item To each well of 2 6-well plates, added $\sim 1$mL prepared agarose solution
			\item Let agarose solution solidify
			\item Incubated at $37^{\circ}$C for 4 hours
			\item Observed $\text{titer}=2\e{8}\text{ PFU/mL}$
		\end{enumerate}
	\item Neutral red overlay of P63A plated cells (RIO3)
		\begin{enumerate}
			\item Prepared a vial with $7.5$mL 1.2\% agarose, $7.5$mL 2x EMEM, and $750\mu$L neutral red
			\item To each well of 2 6-well plates, added $\sim 1$mL prepared agarose solution
			\item Let agarose solution solidify
			\item Incubated at $37^{\circ}$C for 4 hours
			\item Observed $\text{titer}=5.5\e{8}\text{ PFU/mL}$
		\end{enumerate}

	\item Plated MA104 cells --- Passage 64, Flask B
		\begin{enumerate}
			\item Aspirated cell culture medium
			\item Rinsed cells in $10$mL 1x PBS; aspirated PBS
			\item Rinsed cells in $5$mL $0.05$\% trypsin; aspirated trypsin
			\item Bathed cells in $5$mL $0.05$\% trypsin
			\item Incubated cells at $37^{\circ}$C until all cells detached from flask
			\item Added $15$mL complete M199 to flask\
			\item Took cell count by combining $10\mu$L cell mixture with $10\mu$L trypan blue:

				\begin{align*}
				% Initial
				\text{[cells]} &= \frac{3.46\e{5}\text{ cells}}{1\text{mL}} \\
				% Cells / flask
				\frac{\text{cells}}{\text{flask}} &= \frac{3.46\e{5}\text{ cells}}{1\text{mL}} \cdot 20\text{mL} &= \frac{6.92\e{6}\text{ cells}}{20\text{mL}}\\
				% Cells / conical vial
				\frac{\text{cells}}{150\text{mL vial}} &= \frac{6.92\e{6}\text{ cells}}{150\text{mL}} &= \frac{4.61\e{4}\text{ cells}}{\text{mL}}\\
				% Cells / well
				\frac{\text{cells}}{3\text{mL well}} &= \frac{4.61\e{4}\text{ cells}}{\text{mL}} \cdot 3\text{mL} &= \frac{1.38\e{5}\text{ cells}}{\text{well}}\\
				\end{align*}
			\item Added $130$mL complete M199 and $20$mL cell mixture to $250$mL conical vial for final volume of $150$mL
			\item Transferred $3$mL solution to each well of 8 6-well plates
			\item Spread cells evenly by shaking
			\item Incubated at $37^{\circ}$C
		\end{enumerate}

\end{enumerate}

%%%%%%%%%%%%%%%%%%%%
% Friday Plan
%%%%%%%%%%%%%%%%%%%%
% Split P64C flask
%
% Prepare more complete M199,
% incomplete M199
%%%%%%%%%%%%%%%%%%%%

\subsection*{10 October 2014}

\begin{enumerate}
	\item MA104 cell split --- Passage 65, from Passage 64, Flask C
		\begin{enumerate}
			\item Aspirated cell culture medium
			\item Rinsed cells in $10$mL 1x PBS; aspirated PBS
			\item Rinsed cells in $5$mL $0.05$\% trypsin; aspirated trypsin
			\item Bathed cells in $5$mL $0.05$\% trypsin
			\item Incubated cells at $37^{\circ}$C until all cells detached from flask
			\item Added $15$mL complete M199 to flask
			\item Added to $4$ T150 flasks $20$mL complete M199/$5$mL cell mix; $20$mL complete M199/$5$mL cell mix; $22.5$mL complete M199/$2.5$mL cell mix; and $22.5$mL complete M199/$2.5$mL cell mix. Respectively, flasks A, B, C, and D
			\item Gently shook flasks to distribute cells evenly
			\item Incubated at $37^{\circ}$C
		\end{enumerate}
	\item Prepared complete Medium 199
		\begin{enumerate}
				\item Supplemented $500$mL incomplete M199 with $5$mL penicillin/streptomycin stock ($100$U/mL penicillin; $100\mu$g/mL streptomycin final concentration) and $1$mL $250\mu$g/mL amphotericin B stock ($0.25\mu$g/mL amphotericin B final concentration), $55$mL fetal bovine serum
				\item Stored at $4^{\circ}$C
		\end{enumerate}
	\item Prepared serum-free Medium 199
		\begin{enumerate}
			\item Supplemented $500$mL incomplete M199 with $5$mL penicillin/streptomycin stock ($100$U/mL penicillin; $100\mu$g/mL streptomycin final concentration) and $1$mL $250\mu$g/mL amphotericin B stock ($0.25\mu$g/mL amphotericin B final concentration)
			\item Stored at $4^{\circ}$C
		\end{enumerate}
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%
% Sunday Plan
%%%%%%%%%%%%%%%%%%%%
% Infect P64B plated cells with
% RIO3 #1, 2; NSV #4, 5
%%%%%%%%%%%%%%%%%%%%


\chapter{Attention Mechanism}
\section{Introduction}
In this section, we will focus on the deep learning models, the first one being a bidirectional LSTM and the second one an attention layer is added to this LSTM. But it is need to use another text embedding in order to work with LSTM. Indeed, tf-idf create a sparse matrix with each row corresponding to a value for a given word. This means that the order of the words are lost. In order to solve this, word2vec\cite{Mikolov2013} is used. It allows matching words to continuous vectors of a given size with interesting properties. Another method, which consists in making word embedding as tuning parameters will be used.
\section{Text to Vectors}
\subsection{Word2Vec}
Word2Vec comes in two fashions: continuous bag of words (CBOW) and skip gram. It is originally designed to predict a word given a context. For instance, given two previous words and the next two words, which word is the most likely to take place between them. But it appears that the hidden representation of these words works well as word embedding and has very interesting properties such that words with similar meaning have similar vector representation. It is also possible to perform arithmetic that captures information such as singular, plural or even capital and countries. For example, we have that $dog - dogs \approx cat - cats$ but also $Paris - France \approx Berlin - Germany$. \\

It is possible to visualize these relationships by using t-SNE for projecting high dimensions word vectors in 2D space. The results of various relationships can be seen at \textbf{Figure \ref{fig:chap4:word2vec}}.
\begin{figure*}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/linear-relationships}
 \caption{Relationships between different words with t-SNE dimensionality reduction. }
 \label{fig:chap4:word2vec}
\end{figure*}
\subsubsection{How does it work?}
As the original authors did not intend this kind of result, Xin Rong\cite{Rong2014} did a good job explaining how it works.
Let V be the size of the vocabulary and that there is only one word in the CBOW model, it give \textbf{Figure \ref{fig:chap4:CBOW}} models.
\begin{figure*}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/CBOW}
 \caption{A simple CBOW model with only one word in the context}
 \label{fig:chap4:CBOW}
\end{figure*}
Each word is encoded as a one-hot vector of size V. That means that it is a sparse vector full of zeros except for the position assigned to that word which is one. The hidden layer is computed as
\begin{equation}
 \mathbf{h} = \mathbf{W^Tx}
\end{equation}
Where $\mathbf{W^{V \times N}}$ is the weight matrix to optimize over.
The output layer values are computed as
\begin{equation}
 \mathbf{Y} = \mathbf{W'^Th}
\end{equation}
As before $\mathbf{W'^{N \times V}}$ is also a weight matrix to optimize. The loss can be computed as softmax cross entropy. \\

It is also possible to make the opposite: predicting the context given a single input word. This is the skip-gram model. In this case the loss becomes \textbf{Equation \ref{eq:loss}}.
\begin{equation}
 E = - \sum_{c=1}^C u_{j^*_c} + C \cdot \sum_{j' = 1}^V \exp(u_{j'}) \ \label{eq:loss}
\end{equation}
$j_c^*$ is the index of the cth output context word and $u_{j^*_c}$ is the score of the jth word in the vocabulary for the cth context word. Finally, the embedding that is used are the value of the hidden layers produced for a given word.
\begin{figure*}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/skip-gram}
 \caption{Skip-gram model with multiple outputs.}
 \label{fig:chap4:skip-gram}
\end{figure*}
\section{LSTM}
LSTM or Long Short Term Memory\cite{Hochreiter1997LongSM} is a kind of recurrent neural network that fits well to temporal or sequential input such as texts. A RNN is a type of neural network where the hidden state is fed in a loop with the sequential inputs. There are usually shown as unrolled version of it (\textbf{Figure \ref{fig:chap4:RNN_unroll}}). Each of the $X_i$ being one value in the sequence.\\

In this case, $X_i$ values are word vectors. There are two possibilities, either use pre-trained vector with word2vec or make $X_i$ inputs a parameter to learn in the same way as it works for the Word2Vec algorithm, having a one-hot encoding of the word and a matrix of weights to tune. Each method will be used. \\

Recurrent Neural Networks do not works very well with long-term dependencies, that is why LSTM have been introduced. It is made of an input gate, an output gate and a forget gate that are combined in \textbf{Equation \ref{eq:LSTM}}.
\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/RNN-unrolled.png}
 \caption{Unrolled RNN (Understanding LSTM Networks, \url{https://colah.github.io/posts/2015-08-Understanding-LSTMs/)}}
 \label{fig:chap4:RNN_unroll}
\end{figure}
\begin{align} \label{eq:LSTM}
 f_t &= \sigma_g(\mathbf{W}_f x_t + \mathbf{U}_fh_{t-1} + b_f)\\
 i_t &= \sigma_g(\mathbf{W}_i x_t + \mathbf{U}_ih_{t-1} + b_i)\\
 o_t &= \sigma_g(\mathbf{W}_o x_t + \mathbf{U}_oh_{t-1} + b_o)\\
 c_t &= f_t \circ c_{t-1} + i_t \circ \sigma_c(\mathbf{W}_cx_t + \mathbf{U}_c h_{t-1} + b_c)\\
 h_t &= o_t \circ \sigma_h (c_t)
\end{align}
\textbf{Figure \ref{fig:chap4:LSTM-gates}} shows how it works.
\begin{figure}
 \centering
 \includegraphics[width=0.6\textwidth]{images/chapitre4/LSTM1.jpeg}
 \caption{LSTM gates, \\ \url{https://hackernoon.com/understanding-architecture-of-lstm-cell-from-scratch-with-code-8da40f0b71f4)}}
 \label{fig:chap4:LSTM-gates}
\end{figure}
A bidirectional LSTM works the same way, but the input is fed in the two directions, from the start to the end and from the end to the start.
\section{Attention Mechanism}
Attention mechanism\cite{zhou-etal-2016-attention,Vaswani2017AttentionIA} adds an extra layer between LSTM outputs and the final output of the network. It merges word-level features into sentence features using a weight vector. \\

\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/attention.png}
 \caption{Bidirectional LSTM Model With Attention}
 \label{fig:chap4:attention}
\end{figure}
Outputs sequence of the LSTM is summed element-wise in order to merge them. We have that $h_i = [\overrightarrow{h_i} + \overleftarrow{h_i}]$, $\overrightarrow{h_i}$ and $\overleftarrow{h_i}$ begin the outputs i of sequence in each direction as show at \textbf{Figure \ref{fig:chap4:attention}}.\\
Let’s $H$ be a matrix of the concatenation of all the $h_i$,
\begin{equation}
 H = [h_1,h_2,...,h_T]
\end{equation}
Where T is the sequence length.
Then we define
\begin{align}
 M &= \tanh(H)\\
 \alpha &= softmax(w^TM) \\
 r &= H \alpha^T
\end{align}
Finally, we compute $h^* = \tanh(r)$.
For the classification, it uses a softmax classifier as $\hat{p}(y|S) = softmax(W^Sh^* + b)$. Originally the loss function is the negative log likelihood, but as in this case it is a binary classification I used binary cross entropy.
\section{Results}
\subsection{Methodology}
In order to train the models and perform hyper parameters optimization grid search have been used when it was possible (on the liar-liar dataset) and knwoleadge acquired there have been used in order to tune parameters for the networks on the \textbf{Fake News Corpus}. In addition, in order to find the best parameters among all tested with gird search, for each metric, the training epochs having the highest validation value for those metrics have been chosen. \\

All the models have been trained using adam optimizer and initialized using a normal distribution for the weights. \\

As SMOTE cannot be used on the \textbf{Fake News Corpus} dues to the size of the corpus, in order to rebalance the dataset the minority class have been over sampled by feeding multiple times the same input by looping through them.
\subsection{Liar-Liar dataset results}
As explained earlier, both models have been trained using different embedding: the first one being pre-trained word2vec vectors of size 300 and the second one being a tunable parameter with different embedding size.
\subsubsection{LSTM}
When it comes to LSTM trained on liar-liar dataset, it simply does not works. It classifies almost all the texts as being from the same class. Although, it reaches a good score on the training data, it does not manage to generalize correctly. \textbf{Figure \ref{fig:chap4:lstm2}} shows the recall, precision and f1-score and loss for training and testing set of the best models for the LSTM using word2vec. We can see that even if the training score increase, the testing values oscillate. \\

\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/lstm2}
 \caption{Best LSTM With word2vec}
 \label{fig:chap4:lstm2}
\end{figure}
When training the models with word embedding as tunable parameters, the results slightly improve, with an average precision between $55\%$ and $60\%$. This can be seen at \textbf{Figure \ref{fig:chap4:lstm1}}. \\
The training was stopped after 200 iterations because the validation score was not improving anymore.
\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/lstm1}
 \caption{Best LSTM with word embedding as tunable parameters.}
 \label{fig:chap4:lstm1}
\end{figure}

\subsection{Attention Mechanism}
We could think that adding an extra layer to such a bad model would not make any improvement and be useless, but adding an attention layer does improve a little the results. When using word2vec embedding, the best epoch reaches up to $62.9595\%$ of average accuracy, which is better than the simple LSTM. Because there are a lot of models with different parameters, it is interesting to look at the distribution of the results for the best epochs by fixing parameters one by one. \textbf{Figure \ref{fig:chap4:att1:confInt}} shows the $95\%$ confidence interval for precision for a fixed parameter. \\
\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/confInt_precision_liar_attention_word2vec}
 \caption{Confidence Interval of Precision for Each Parameter Value}
 \label{fig:chap4:att1:confInt}
\end{figure}
It shows that it is better to use fewer hidden units in the model, and only a single layer. The sequence length has a very small impact on the precision. Actually, the best model uses a sequence length of 20. \\\
The precision of the different models range from $53\%$ to $63\%$ (\textbf{Figure \ref{fig:chap4:att1:distPrecision}}.)
\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/distplot_precision_liar_attention_word2vec}
 \caption{Distribution of the precision of best epochs for all the models trained with word2vec embedding.}
 \label{fig:chap4:att1:distPrecision}
\end{figure}
The training plot of the model that reaches the maximum precision can be seen at \textbf{figure \ref{fig:chap4:att1:train}}. It shows that after the 25th iteration, the validation values start to decrease, which is a sign of overfitting. \\

\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/attention1}
 \caption{Training and validation of the model with top precision trained with word2vec embedding.}
 \label{fig:chap4:att1:train}
\end{figure}
Finally, there is the models where the embedding is a tunable parameter. The \textbf{Figure \ref{fig:chap4:att3:confInt2}} shows that in this case, the longer the sequence the better, and that as before using few hidden units perform better. In this case, variation has a wider range than when using word2vec.
\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/confInt_precision_liar_attention_200}
 \caption{Training and Validation of the Model With top Precision}
 \label{fig:chap4:att3:confInt2}
\end{figure}
There are a few models that have top precision higher than $75\%$, but looking at the training plot (\textbf{Appendix \ref{Appendix2}, Figure \ref{appendix2:training_plot1}}) shows that the model does not perform well. Because in particular case precision in not a good indicator of how well a model perform, f1-score will be used instead, as it is a balance between precision and recall. \\
And the best f1-score obtained is $0.55384$, which is quite smaller than the $0.63$ for the model using word2vec. The training plot is at \textbf{Figure \ref{fig:chap4:att3:f1}}. We can see that there is still room for improvement, the next step is to see what happens when training on more epochs. \\

\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/attention-f1}
 \caption{Training and validation of the model with top f1-score}
 \label{fig:chap4:att3:f1}
\end{figure}
Training on 1000 epochs rather than 200 does not improve validation score, but it does for training (\textbf{Figure \ref{fig:chap4:att3:f1.1}}). \\
\begin{figure}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/attention3}
 \caption{Training on 1000 epochs rather than 200}
 \label{fig:chap4:att3:f1.1}
\end{figure}
\subsection{Result Analysis}
The previous section shows a few things
\begin{itemize}
 \item LSTMs do not work well,
 \item Adding attention layer improve LSTM results,
 \item Using word2vec rather than training the embedding gives better results.
\end{itemize}
It also shows that despite reaching a very good precision, recall and f1-score on the training set it does not perform well on the validation set. This is a sign of overfitting. In order to avoid this, multiple methods have been applied without showing any improvement. \\

The following methods have been applied:
\begin{itemize}
 \item Dropout\cite{srivastava2014dropout},
 \item batch-normlaziation\cite{Ioffe2015},
 \item reducing network capacity (fewer hidden layers, lower embedding dimensions, less training parameters with word2vec),
 \item Early stopping of training.
\end{itemize}
The highest gain was from using word2vec embedding. This significantly reduces the amount of training parameters, secondly dropout also helped a little.
\subsection{Testing}
The same way as in \textbf{Chapter \ref{chap3}}, the models will be trained on the parameters that produced the best results on the training set, and trained on training and validation set, and tested on testing set. \\
The parameters used for training are given at \textbf{Table \ref{table:chap4:param}}.
\begin{table}[h]
 \begin{tabular}{|c|ccccc|}
 \hline
 model & embedding size & Sequence Length & num hiddens & dropout & Early Stop\\
 \hline
 LSTM & 300 & 10 & 50 & 0.75 & 126\\
 LSTM + word2vec & 300 & 10 & 50 & 0.0 & 160\\
 Attention & 10 & 20 & 10 & 0.75 & 400\\
 Attention + word2vec & 300 & 20 & 5 & 0.75 & 25\\
 \hline
 \end{tabular}
 \caption{Parameters used for training}
 \label{table:chap4:param}
\end{table}
The results for all four models are given at \textbf{Table \ref{table:chap4:results}}. It shows that the model that works the best is attention network using word2vec embedding, with an accuracy of $61\%$, which is equivalent to ridge classifiers and linear svm. The three other models do not perform well, all having a average precision around $55\%$, which is close to being a random classifier.
\begin{table}
\centering
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.440574 &     0.649551 &  0.569061 &     0.545062 &      0.558340 \\
precision &     0.508274 &     0.599526 &  0.569061 &     0.553900 &      0.559698 \\
recall    &     0.388788 &     0.708683 &  0.569061 &     0.548736 &      0.569061 \\
support   &  1106.000000 &  1428.000000 &  0.569061 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Simple LSTM}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
 \midrule
 f1-score  &     0.481724 &     0.623040 &  0.563536 &     0.552382 &      0.561361 \\
 precision &     0.500000 &     0.606906 &  0.563536 &     0.553453 &      0.560245 \\
 recall    &     0.464738 &     0.640056 &  0.563536 &     0.552397 &      0.563536 \\
 support   &  1106.000000 &  1428.000000 &  0.563536 &  2534.000000 &   2534.000000 \\
 \bottomrule
 \end{tabular}
 \caption{LSTM + word2vec}
\end{subtable}
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.486636 &     0.615597 &  0.560379 &     0.551116 &      0.559310 \\
precision &     0.496241 &     0.606803 &  0.560379 &     0.551522 &      0.558546 \\
recall    &     0.477396 &     0.624650 &  0.560379 &     0.551023 &      0.560379 \\
support   &  1106.000000 &  1428.000000 &  0.560379 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Attention network}
\end{subtable}
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.511397 &     0.676721 &  0.610892 &     0.594059 &      0.604563 \\
precision &     0.565789 &     0.636252 &  0.610892 &     0.601021 &      0.605497 \\
recall    &     0.466546 &     0.722689 &  0.610892 &     0.594618 &      0.610892 \\
support   &  1106.000000 &  1428.000000 &  0.610892 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Attention Network + word2vec}
\end{subtable}
\caption{Results for the differents models trained with parameters given at \textbf{Table \ref{table:chap4:param}}.}
\label{table:chap4:results}
\end{table}
\section{Attention Mechanism on fake news corpus}
\subsection{Model Selection}
The two models using word2vec embedding have shown to work better than their counterparts, this why only these two methods will be tested on \textbf{Fake News Corpus} for comparison as very good results have already been obtained. \\

It shows out that in this case LSTMs works better than Attention Mechanism, but as in previous section does not reach machine learning results. \\

The best LSTM obtained use sequence of 200 words and 200 hidden layers, with an average precision of $0.929376$ on the validation set. The training plots of this particular model are shown at \textbf{Figure \ref{chap4:fig:lstm5.1}}.\\
\begin{figure*}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/lstm5}
 \caption{Training plots of the best LSTM using word2vec embedding.}
 \label{chap4:fig:lstm5.1}
\end{figure*}

In the case of attention mechanism, training on the \textbf{Fake News Corpus} has shown to be harder than on the \textbf{Liar-Liar Corpus} as using too large learning rate would lead to oscillating loss and too small learning rate lead to halting the loss decrease. This can be seen at \textbf{Appendix \ref{appendix2:training_plot2}}. \\

The same parameters as for the LSTM will be used for training the Attention Network. Its training plot is available at \textbf{Figure \ref{chap4:fig:attention5}}.
\begin{figure*}
 \centering
 \includegraphics[width=\textwidth]{images/chapitre4/attention5}
 \caption{Training plots of the best attention network using word2vec embedding.}
 \label{chap4:fig:attention5}
\end{figure*}
The final results are given at \textbf{Table \ref{table:chap4:results2}}. It shows that for the same parameters LSTM works better than Attention Network on this particular dataset. It shows that LSTM place below Linear SVM and Ridge Classifier and above Decision Tree and Na\"{i}ve-Bayes in terms of accuracy.
\begin{table}
 \begin{subtable}{\textwidth}
  \begin{tabular}{lrrrrr}
  \toprule
  {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
  \midrule
  f1-score  &      0.856568 &      0.947577 &  0.923217 &      0.902073 &      0.924724 \\
  precision &      0.806655 &      0.969503 &  0.923217 &      0.888079 &      0.928611 \\
  recall    &      0.913066 &      0.926621 &  0.923217 &      0.919843 &      0.923217 \\
  support   &  17496.000000 &  52181.000000 &  0.923217 &  69677.000000 &  69677.000000 \\
  \bottomrule
  \end{tabular}
  \caption{LSTM + word2vec results on \textbf{Fake News Corpus}}
 \end{subtable}
 \begin{subtable}{\textwidth}
  \begin{tabular}{lrrrrr}
  \toprule
  {} &      reliable &          fake &  accuracy &     macro avg &  weighted avg \\
  \midrule
  f1-score  &      0.850296 &      0.687493 &  0.797566 &      0.768894 &      0.809416 \\
  precision &      0.952876 &      0.561344 &  0.797566 &      0.757110 &      0.854562 \\
  recall    &      0.767655 &      0.886774 &  0.797566 &      0.827215 &      0.797566 \\
  support   &  52181.000000 &  17496.000000 &  0.797566 &  69677.000000 &  69677.000000 \\
  \bottomrule
  \end{tabular}
  \caption{Attention Network + word2vec on \textbf{Fake News Corpus}}
 \end{subtable}
 \caption{Final result on testing set for LSTM and attention network using word2vec.}
 \label{table:chap4:results2}
\end{table}
It is likely to be possible to reach results as well as LSTM or even better for the Attention Network, but due to technical and time constraints I was not able to experiment further. For instance, using longer sequence length and more hidden units with a smaller learning rate might have overcome this problem.
\section{Conclusion}
In this chapter I have investigated how state-of-the-art deep learning models work on fake news detection, and it shows that for the particular case of fake news detection it does not outperform traditional machine learning methods. I have also made some addition to the original model that improves the performances by a few percent by replacing the tunable word embedding by constant one using word2vec. It shows out that it helps reduce overfitting and increase result on the testing set. \\

A hypothesis to explain why these two deep learning methods do not works as well as machine learning methods is the fact that in this case text are required to be the same size. Which means that some of them require some padding and the other are srunk. In the second case, information is lost. \\

In addition, it shows that \textbf{Liar-Liar Corpus} is hard to work on, with $60\%$ precision, when \textbf{Fake News Corpus} still have good results.


\chapter{6. Vehicle and Control Tests}

In this chapter the control strategy is applied to the real scale PEV and an stability and robustness analysis is presented as well. Due to the possibility of controlling the PEV remotely (through the Bluetooth module and the Android app) two models were studied simultaneously. After the proper analysis and validation in MATLAB, the control strategy was implemented in the Arduino boards and the PEV was tested several times. Here the results of these tests are presented.

\section{Control Strategy Analysis}

The control strategy works in parallel with the dynamic model of the vehicle. It receives the sensor data as inputs; that information, along with with the gains and the actual speed of the vehicle, gives result to the output signal $M_{t}$ that tilts the vehicle.

Regarding the sensors, the rotary encoder in the rear wheel and the pair of inertial motion units provide the necessary data. The longitudinal speed given by the rotary encoder is used to calculate the gains of the model as well. The actuated motor is the tilting NIDEC motor, since the steering motor only responses to the driver's inputs and the handle bar motor only does the haptic feedback to the driver.

\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/strategy3}
	\caption{Workflow of the control strategy: input from IMU and output to tilting actuator}
	\label{strategy}
\end{figure}

\newpage
\subsection{Model Parameters}

To calculate the gains of the control strategy it is first needed to obtain or measure the \textbf{parameters to complete the dynamic model}. This includes the vehicle and the wheels variables:

$\hspace{2cm} m \enspace L_{f} \enspace L_{r} \enspace I_{x} \enspace I_{z} \enspace C_{f} \enspace C_{r} \enspace \lambda_{f} \enspace \lambda_{r} \enspace ...$.

Therefore, the model presented in Chapter 3 needs to be calculated for the PEV characteristics. Both the vehicle with and without the driver were modelled in CAD software (Solidworks). By indicating the material of each body the necessary properties for the control model were extracted.

The geometrical parameters -- $h \enspace L_{f} \enspace L_{r} \enspace R_{wf}\enspace R_{wr}\enspace b$ -- were measured directly from the model. The dynamic parameters -- $m \enspace I_{x} \enspace I_{z} \enspace ...$ --, on the other side, were extracted from the mass properties.

\begin{table*}[h!]
\centering
	\begin{tabular}{l|cc|l|l}
	%\hline
	\textbf{Name}      &  \textbt{\begin{tabular}[c]{@{}l@{}}Driver\\ included\end{tabular}} & \textbt{\begin{tabular}[c]{@{}l@{}}No driver\\ included\end{tabular}}   & \textbf{Units}         & \textbf{Description}                                   \\
	\hline
	$m$                  & 115                       & 35                          & kg                     & Total mass of the vehicle                              \\
	$h$                 & 0.91                      & 0.36                        & m                      & Position of the center of gravity G on the z axis      \\
	$L_f$                & 0.718                     & 0.518                       & m                      & Distance from center of gravity to front axle          \\
	$L_r$               & 0.61                      & 0.81                        & m                      & Distance from center of gravity to rear axle           \\
	$m_f$               & 52.82                     & 21.35                       & kg                     & Mass supported by the front wheels                     \\
	$m_r$                & 62.18                     & 13.65                       & kg                     & Mass supported by the rear wheel                       \\
	$I_x$               & 25.14                     & 4.00                        & kg m^2				  & Vehicle roll moment of inertia                         \\
	$I_z$                & 17.8                      & 10.93                       & kg m^2				  & Vehicle yaw moment of inertia                          \\
	$I_{wheel\,f\,\theta}$ & 0.05                      & 0.05                        & kg m^2				  & Tilting inertia of each front wheel about its own axis \\
	$I_{wheel\,f\,rot}$  & 0.11                      & 0.11                        & kg m^2				  & Inertia of each front wheel about its rotating axis    \\
	$I_{wheel\,f\,\phi}$ & 0.05                      & 0.05                        & kg m^2				  & Yaw inertia of each front wheel about its own axis     \\
	$I_{wheel\,r\,\theta}$ & 0.16                      & 0.16                        & kg m^2				  & Tilting inertia of the rear wheel about its own axis   \\
	$I_{wheel\,r\,rot}$  & 0.32                      & 0.32                        & kg m^2				  & Inertia of the rear wheel about its rotating axis      \\
	$I_{wheel\,r\,\phi}$ & 0.16                      & 0.16                        & kg m^2				  & Yaw inertia of the rear wheel about its own axis       \\
	$R_{w\,f}$          & 0.218                     & 0.218                       & m                      & Front wheel radius                                     \\
	$R_{w\,r}$           & 0.315                     & 0.315                       & m                      & Rear wheel radius                                      \\
	$F_{z\,f}$          & 259.10                    & 104.71                      & N                      & Vertical reaction in each front wheel                  \\
	$F_{z\,r}$          & 609.95                    & 133.93                      & N                      & Vertical reaction in the rear wheel                    \\
	$C_f$               & 7277.45                   & 2690.05                     & N/rad                  & Cornering stiffness of each front wheel                \\
	$C_r$               & 20454.62                  & 3501.36                     & N/rad                  & Cornering stiffness of the rear wheel                  \\
	$\lambda_f$          & 231.90                    & 79.70                       & N/rad                  & Camber stiffness of each front wheel                   \\
	$\lambda_r$          & 731.44                    & 105.33                      & N/rad                  & Camber stiffness of the rear wheel                     \\
	$b$                  & 0.92                      & 0.92                        & m                      & Width of the vehicle in the base                       \\
	$M_{max}$            & 518.949                   & 157.941                     & Nm                     & Maximum tilt torque before rolling over    			   \\
	%\hline
	\end{tabular}
	\\[20pt]
	\caption{PEV parameters extracted from Solidworks model}
\end{table*}

\subsection{Driver Modelling}

\begin{marginfigure}[5cm]
	\includegraphics[width=1\linewidth]{figs/06/driver}
	\caption{Driver model in the PEV}
\end{marginfigure}
The driver was modeled as a 80kg person seated and in driving position. Due to the location of the body, the center of gravity and the moments of inertia changed considerably. The variability in the weight and the position of the driver will be overcome by the uncertainly analysis presented below.

\subsection{Tire Stiffness}

In regards to the tires, the stability and handling depends strongly on the tire properties -- cornering stiffness and camber stiffness. Mathematical models are available to predict vehicle handling. However, very little data is available on properties of bicycle and tricycle tires currently on the market.

In Windes et. al., 2013\cite{windes2013experimental}, the authors focus on the experimental determination of the cornering and camber stiffness of several different
types of bicycle and tricycle tires. Indeed, they designed and built an machine for measuring cornering and camber stiffness using the back-to-back method. In this way, the cornering and camber stiffness were obtained over a range of vertical loads.

Knowing the mass balance of the PEV -- $m_{f} \enspace m_{r}$ -- , the vertical loads of each wheel were estimated: $F_{z\,f} \enspace F_{z\,r}$. Then, using the quadratic relationships presented in Windes et. al., the cornering and camber stiffness were obtained.

The PEV is mounting a Continental Grand Prix 4000 SII tire, which is not modeled in the mentioned paper. Nevertheless, an average estimation from those selected tires was calculated and the relation between the stiffness and the vertical load was estimated for this tire: \[C=\frac{F_{z}^2}{3690}+\frac{F_{z}}{2.38} \quad\quad \lambda=\frac{F_{z}^2}{66085}+\frac{F_{z}}{85.47}\]
\begin{figure*}[b]
	\includegraphics[width=0.95\linewidth]{figs/06/tire}
	\caption{Cornering and camber stiffness in function of the vertical load}
	\\[-1cm]
\end{figure*}
\newpage

\subsection{Simulink Model}

The control strategy stability and robustness was studied in MATLAB and Simulink. The model is illustrated in Figure \ref{model}.

\begin{figure*}[!h]
	\includegraphics[width=1\linewidth]{figs/06/model}
	\caption{Simulink model}
	\label{model}
\end{figure*}
\begin{marginfigure}[5cm]
	\includegraphics[width=1.2\linewidth]{figs/06/control2}
	\caption{Control strategy schematic}
\end{marginfigure}
The steering input from the driver is include as a disturbance in the system, whose effect on the perceived lateral acceleration $a_{per}$ should be canceled by the tilt torque $M_{t}$. This fact is taken into account with the feedforward gains $K_{d}=\Big[K_{\delta} \quad K_{\dot{\delta}}\Big]^{T}$

The vehicle is modeled with the differential equation \[\dot{x_{i}}=\Big[A_{i}\Big]x_{i}+\Big[B_{id}\Big]d+\Big[B_{iu}\Big]u\]
This differential equation is integrated to get the state vector $x_{i}$. The regulator problem is then completed with the feedback loop of the state vector, optimized to minimize the perceived lateral acceleration $a_{per}$.

For the next sections, it is fundamental to do a distinction between the open and the closed loop systems:

\minipage{0.4\textwidth}
	\hspace{0.25cm}--Open-loop system
\endminipage\hfill
\minipage{0.6\textwidth}
	\[\dot{x_{s}}=\Big[A_{s}\Big]x_{s}+\Big[B_{s}\Big]u \]\[y=\Big[I\Big]_{7x7}x_{s}\]
\endminipage\hfill
\minipage{0.4\textwidth}
	\hspace{0.25cm}--Closed-loop system
\endminipage\hfill
\minipage{0.6\textwidth}
	\[\dot{x_{s}}=\Big(\big[A_{s}\big]-\big[B_{s}\big]\big[K\big]\Big)x_{s}+\big[B_{s}\big]u \]\[y=\big[I\big]_{7x7}x_{s}\]
\endminipage\hfill

\hspace{0.5cm}with $x_{s}=\begin{bmatrix}\dot{y} & \dot{\psi} & \theta & \dot{\theta}	& a_{per} & \delta & \dot{\delta} \end{bmatrix}^{T}$

\newpage
\subsection{Feedback and Feedforward Gains}

Following the methodology explained in \textit{Chapter 3 - Control Strategy Summary}, the feedback and feedforward gains are obtained. The influence of the inverse velocity term in the gain scheduling stage was also studied.

Once the dynamic model is completed with the PEV parameters, the gains are obtained from the Riccati and Sylvester equations:

\begin{itemize}
\begin{itemize}
\item Feedback Gains: Riccati equation

$\hspace{1.5cm} M_{1}\,A_{i} + A_{i}^{T}\,M_{1} - M_{1}\,B_{iu}\,R_{u}^{-1}B_{iu}^{T}M_{1} + Q_{x}=0$

$\hspace{3.5cm} K=R_{u}^{-1}\,B_{iu}^{T}\,M_{1}$

\item Feedback Gains: Slyvester equation

$\hspace{1cm}M_{2}\,A_{e}+(A_{i}^{T}-M_{1}\,B_{iu}\,R_{u}^{-1}\,B_{iu}^{T})M_{2}+M_{1}\,B_{id}\,C_{e}=0$

$\hspace{3.5cm} K_{d}=R_{u}^{-1}\,B_{iu}^{T}\,M_{2}$

\end{itemize}
\end{itemize}

\marginnote{\begin{tabular}{l|ccc|}
                       & $K_{C}$ & $K_{V}$ & $K_{1/V}$ \\[5pt] \hline \\[-5pt]
$K_{a_{per}}^{'}$      & -0.56   & -0.27   & 0.52      \\[5pt]
$K_{\dot{\psi}}^{'}$   & 19.14   & -4.09   & -16.63    \\[5pt]
$K_{\theta}^{'}$       & 242.92  & -2.51   & 5.54      \\[5pt]
$K_{\dot{\theta}}^{'}$ & 64.40   & -0.46   & 0.84      \\[5pt]
$K_{a_{per}^{I}}^{'}$  & -0.32   & 0       & 0         \\[5pt]
$K_{\delta}^{'}$       & 1228.56 & -270.84 & -1121.79  \\[5pt]
$K_{\dot{\delta}}^{'}$ & 220.54  & -59.70  & -201.26   \\[5pt] \hline
\end{tabular}}

The obtained gains are transformed so that the state vector is fully measurable:
\begin{eqnarray}
K_{a_{per}}^{'}=\frac{K_{\dot{y}}}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})} \\[10pt]
K_{\dot{\psi}}^{'}=\frac{K_{\dot{\psi}}(a_{11}+ha_{41})-K_{\dot{y}}(a_{12}+ha_{42}+V_{x})}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})} \\[10pt]
K_{\theta}^{'}=\frac{K_{\theta}(a_{11}+ha_{41})-K_{\dot{y}}(a_{13}+ha_{43}-g)}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})} \\[10pt]
K_{\dot{\theta}}^{'}=\frac{K_{\dot{\theta}}(a_{11}+ha_{41})-K_{\dot{y}}(a_{14}+ha_{44})}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})} \\[10pt]
K_{a_{per}^{I}}^{'}=\frac{K_{a_{per}^{I}}(a_{11}+ha_{41})}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})} \\[10pt]
K_{\delta}^{'}=\frac{K_{\delta}(a_{11}+ha_{41})-K_{\dot{y}}(b_{\delta 1}+hb_{\delta 4})}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})} \\[10pt]
K_{\dot{\delta}}^{'}=\frac{K_{\dot{\delta}}(a_{11}+ha_{41})}{a_{11}+ha_{41}+K_{\dot{y}}\,(b_{u1}+hb_{u4})}
\end{eqnarray}

\[K^{'}=\begin{bmatrix}
K_{a_{per}}^{'} & K_{\dot{\psi}}^{'} & K_{\theta}^{'} & K_{\dot{\theta}}^{'} & K_{a_{per}^{I}}^{'} & K_{\delta}^{'} & K_{\dot{\delta}}^{'}
\end{bmatrix} \]

\newpage
\begin{figure*}[!h]
	\includegraphics[width=0.8\linewidth]{figs/06/control/model_gains}
	\caption{Gains relation with velocity. Velocity inverse term is not considered}
	\label{model_gains}
	\\[-0.5cm]
\end{figure*}

\newpage
\begin{figure*}[!h]
	\includegraphics[width=0.8\linewidth]{figs/06/control/model_gains_inverse}
	\caption{Gains relation with velocity. Velocity inverse term is considered}
	\label{model_gains_inverse}
	\\[-0.5cm]
\end{figure*}

\newpage
In Figures \ref{model_gains} and \ref{model_gains_inverse} the gains of the PEV without driver have been represented. The gains curves are estimated as a least square estimation of some discrete points. As was indicated in previous chapters, the gain scheduling as function of the longitudinal velocity was designed to depend on three terms $K_{c}$, $K_{V}$ and $K_{1/V}$  with:

$\hspace{2.5cm}\begin{pmatrix} K_{c} \\ K_{V} \\ K_{1/V} \end{pmatrix}=(M^{T}\,M)^{-1}\,M^{T}\,K_{M}$

If the inverse term $K_{1/V}$ is not considered, then the gain scheduling only depends linearly on the velocity:

$\hspace{2.5cm}\begin{pmatrix} K_{c} \\ K_{V} \end{pmatrix}=(M^{T}\,M)^{-1}\,M^{T}\,K_{M}$

\subsection{Stability}

If the system is studied without feedback control, it is inherently unstable. The instability can be visualized through different indicators.

The system matrix $\big[A_{s}\big]$ is non-negative defined, meaning that it has positive eigenvalues. The eigenvalues of $\big[A_{s}\big]$ represent the open poles of the system, and if a pole is in the positive side of the real numbers, then it means that the system is unstable.

 \[eig(\big[A_{s}\big])=\begin{bmatrix}0 \\ -92.47 \\ -43.38 \\ -4.06 \\ 3.61 \\ -0.5 \\ -1 \end{bmatrix}\]

The eigenvalues are usually represented in the pole-zero map. The poles (represented with a cross $X$) are the roots of denominator of the system transfer function, while the zeros (represented with a circle $O$) are the roots of the numerator. In figure \ref{pole_zero_map_1}, a pole-zero map has been included for each system transfer function. The input is the torque $M_{t}$ and the outputs are the remaining systems variables: $a_{per} \quad \dot{\psi} \quad \theta \quad \dot{\theta} \quad \a_{per}^{I}$.

All transfer functions share a common pole in the positive real axis, which makes the system unstable. The transfer functions can also share some poles and zeros.

\newpage
\begin{figure}[!h]
	\includegraphics[width=1.15\linewidth]{figs/06/control/pole_zero_map_1}
	\caption{Pole zero map of the open--loop system (unstable)}
	\label{pole_zero_map_1}
	\\[-1cm]
\end{figure}

Simulating the response of the system to a disturbance $\delta,\,\dot{\delta}$ makes the state variables unstable and tend to infinite values:

\begin{figure}[!h]
	\includegraphics[width=1.15\linewidth]{figs/06/control/response_1}
	\caption{Unstable response of the open--loop system, without control feedback}
	\label{response_1}
	\\[-1.3cm]
\end{figure}

Looking at the ranks of the observability and controllability matrix of the system: \[Ob=\begin{bmatrix}C & C\,A & C\,A^{2} & ... & C\,A^{6}\end{bmatrix}^{T}\] \[Co=\begin{bmatrix}B & A\,B & A^{2}\,B & ... & A^{6}\,B\end{bmatrix}\]
The system is observable if $Ob$ has full rank ($A_{7x7}$) and is controllable if $Co$ has also full rank ($A_{7x7}$). Therefore, the open loop system is observable ($rank(Ob)=7$), but not controllable ($rank(Ob)=7$)

If the system is controlled with the feedforward and feedback gains, then the system stabilizes. The matrix $\big[A_{s}\big]$ is definite non-positive, with all its eigenvalues located in the negative side of the pole map. However, a pair of conjugate imaginary poles and positive zeros are introduced to effectively stabilize the system.
 \[eig(\big[A_{s}\big]-\big[B_{s}\big]\big[K\big])=\begin{bmatrix}-81.87 \\-44.56 + 11.53i \\-44.56 - 11.53i \\-4.917\\-5.478\\-0.50\\-1\end{bmatrix}\]
The pole-zero map and the response of the system are indeed stable:

\begin{figure}[!h]
	\includegraphics[width=1.15\linewidth]{figs/06/control/pole_zero_map_2}
	\caption{Pole zero map of the closed--loop system (stable)}
	\\[-0.5cm]
\end{figure}

\begin{figure}[!h]
	\includegraphics[width=1.15\linewidth]{figs/06/control/response_2}
	\caption{Stable response of the closed--loop system, with control feedback}
	\\[-5cm]
\end{figure}

\newpage
\subsection{Robustness}

The $\mu$-analysis makes it possible to carry out a posteriori robust studies with respect to the parametric variations of the model, or the neglected dynamics, taking into account the structure and the nature of the uncertainties.

According to the generalized small gains theorem, checking the robustness of the stability of the closed loop, amounts to calculating the singular values of the transfer function $\mu(H(jw))$. The system is stable if these values are under 1 $\mu(H(jw))<1$ and if in addition $\mu(H(jw))$ is small, the stability of the system is robust.

\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/control/singular_values_1}
	\caption{Singular values of the open--loop system}
	\\[-1cm]
\end{figure}

High-gain feedback in low-frequency ranges is a way to deal with the effects of unknown biases and disturbances acting on the process output. The control feedback increases the singular values (the principal gains of the frequency response) at 10 to 100 Hz range, but it remains below $\mu=0\,dB=20 \log (1)$.
\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/control/singular_values_2}
	\caption{Singular values of the closed--loop system}
	\\[-5cm]
\end{figure}

\newpage
Apart from being stable, the system has deal with the effects of uncertainty. Reducing the effects of some forms of uncertainty (initial conditions, low-frequency disturbances) without increasing the effects of the sensor noise or the model uncertainty is the primary job of the feedback control system.

At the heart of robust control is the concept of an uncertain LTI system. Model uncertainty arises when system gains or other parameters are not precisely known, or can vary over a given range. The following table summarizes the model parameters and their variation over the nominal value.

\begin{table}
	\centering
	\begin{tabular}{c|cc|c}
	 & \textbf{Value} & \textbf{Deviation\%} &      \\ \hline
	$L_{f}$       & 0.51          & 5         & $m$       \\
	$L_{r}$       & 0.81          & 5         & $m$       \\
	$h$           & 0.36          & 15        & $m$       \\
	$m$           & 35            & 20        & $kg$      \\
	$I_{z}$       & 11            & 50        & $kg\,m^2$ \\
	$I_{x}$       & 4             & 50        & $kg\,m^2$ \\
	$C_{f}$       & 3500          & 25        & $N/rad$   \\
	$C_{r}$       & 3000          & 25        & $N/rad$   \\
	$\lambda_{f}$ & 200           & 25        & $N/rad$   \\
	$\lambda_{r}$ & 200           & 25        & $N/rad$   \\ \hline
	\end{tabular}
	\caption{Parameters Deviations}
	\\[6pt]
\end{table}

Once formulated, high-level system robustness tools can help to analyze the potential degradation of stability and performance of the closed-loop system brought on by the system model uncertainty.

The parameter uncertainties mean uncertain pole and zero locations. Fortunately, among the represented poles, none of them is moving towards the positive real part, which will produce instabilities.

\begin{figure}[!h]
	\includegraphics[width=1.1\linewidth]{figs/06/control/pole_zero_map_uncertainty_1}
	\caption{Uncertainty in the pole zero map of the open--loop system}
	\\[-5cm]
\end{figure}
\newpage
\begin{figure}[!h]
	\includegraphics[width=1.1\linewidth]{figs/06/control/pole_zero_map_uncertainty_2}
	\caption{Uncertainty in the pole zero map of the closed--loop system}
	\\[-1cm]
\end{figure}

Regarding the singular values of the closed-loop system, these values are not affected by the parameters uncertainties.

\begin{figure*}[!h]
	\includegraphics[width=1\linewidth]{figs/06/control/singular_values_uncertainty}
	\caption{Uncertainty in the singular values of the open and closed--loop system}
\end{figure*}

%\begin{figure}[!h]
%	\includegraphics[width=1\linewidth]{figs/06/control/singular_values_uncertainty_1}
%	\caption{Uncertainty in the singular values of the open--loop system}
%\end{figure}
%\begin{figure}[!h]
%	\includegraphics[width=1\linewidth]{figs/06/control/singular_values_uncertainty_2}
%	\caption{Uncertainty in the singular values of the closed--loop system}
%\end{figure}

Notions such as gain and phase margins help to quantify the sensitivity of stability and performance in the face of model uncertainty, which is the imprecise knowledge of how the control input directly affects the feedback variables.

\begin{marginfigure}[1cm]
	\includegraphics[width=0.8\linewidth]{figs/06/control/margin}
	\caption{Gain and phase margin}
\end{marginfigure}
The gain margin is the amount of gain increase or decrease required to make the loop gain unity at the frequency where the phase angle is $–180\degree$. Similarly, the phase margin is the difference between the phase of the response and $–180\degree$ when the loop gain is 1.
%\begin{lstlisting}[style=codematlab2]
%GainMargin: 0.0707
%GMFrequency: 4.4670
%PhaseMargin: 76.0715
%PMFrequency: 50.6347
%DelayMargin: 0.0262
%DMFrequency: 50.6347
%\end{lstlisting}

\newpage

If the stability robustness margin is greater than 1 it means that the uncertain system is stable for all values of its modeled uncertainty. Being less than 1 implies that certain allowable values of the uncertain elements lead to instability. In our system only bounds on the exact stability margin were computed. The exact robust stability margin is guaranteed to lie in between these upper and lower bounds.
\begin{lstlisting}[style=codematlab2]
		LowerBound: 0.9099
		UpperBound: 1.9902
		DestabilizingFrequency: 0.1005
\end{lstlisting}
A nominally stable uncertain system is generally unstable for specific values of its uncertain elements. Determining the values of the uncertain elements closest to their nominal values for which instability occurs is a robust stability calculation.
%\begin{marginfigure}[-5cm]
%	\includegraphics[width=0.65\linewidth]{figs/06/control/destabilizing}
%	\caption{Destabilizing Parameters}
%\end{marginfigure}
\begin{lstlisting}[style=codematlab2]
Destabilizing Parameters:
		C_f: 5.2414e+03
		C_r: 1.5074e+03
		I_x: 0.0196
		I_z: 0.0540
		L_f: 0.5607
		L_r: 0.7294
		h: 0.2525
		landa_f: 299.5095
		landa_r: 100.4905
		m: 48.9313
\end{lstlisting}

If the uncertain system is stable for all values of uncertain elements within their allowable ranges, the uncertain system is robustly stable. Conversely, if there is a combination of element values that cause instability, and all lie within their allowable ranges, then the uncertain system is not robustly stable.
\begin{lstlisting}[style=codematlab2,frame=single]
	Uncertain system is possibly not robustly stable to modeled uncertainty.
	-- It can tolerate up to 91% of the modeled uncertainty.
	-- A destabilizing combination of 199% of the modeled uncertainty was found.
	-- This combination causes an instability at 0.1 rad/seconds.
	-- Sensitivity with respect to the uncertain elements are:
	'C_f' is 27%. Increasing 'C_f' by 25% leads to a 7% decrease in the margin.
	'C_r' is 17%. Increasing 'C_r' by 25% leads to a 4% decrease in the margin.
	'I_x' is 78%. Increasing 'I_x' by 25% leads to a 20% decrease in the margin.
	'I_z' is 28%. Increasing 'I_z' by 25% leads to a 7% decrease in the margin.
	'L_f' is 6%.  Increasing 'L_f' by 25% leads to a 2% decrease in the margin.
	'L_r' is 9%.  Increasing 'L_r' by 25% leads to a 2% decrease in the margin.
	'h' is 8%.    Increasing 'h' by 25% leads to a 2% decrease in the margin.
	'landa_f' is 5%. Increasing 'landa_f' by 25% leads to a 1% decrease in the margin.
	'landa_r' is 5%. Increasing 'landa_r' by 25% leads to a 1% decrease in the margin.
	'm' is 12%. Increasing 'm' by 25% leads to a 3% decrease in the margin.
\end{lstlisting}

%\begin{figure}[!h]
%	\includegraphics[width=1\linewidth]{figs/06/control/open_loop_responses_input_sensitivities_uncertainty}
%	\caption{Open loop responses and input sensitivities}
%\end{figure}

\newpage
\section{Experiments}

Once the control system was developed and the PEV was fully fabricated, the test and validation stage started. First, some tests were run to validate the sensor data. Second, the electronics were tested, along with the live streaming of data through Bluetooth. Finally, the control feedback was implemented in Arduino and tested several times.

\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/P1060127}
	\caption{PEV outside the MIT Media Lab}
	\\[-1.2cm]
\end{figure}
\subsection{Test Setup}

\begin{marginfigure}[6cm]
	\includegraphics[width=1\linewidth]{figs/06/circuit}
	\caption{Testing setup schematic}
\end{marginfigure}
With the intention of comparing the experimental results with the carried out simulations, the PEV was tested under some controlled conditions. The 6th floor of the MIT Media Lab was selected as the test field due to the available space and the accessibility of the room. The geometry of the experiment was limited by two circumferences of known radius ($R=2.5m$). These circumferences were delimited by some cardboard boxes laid on the floor.

As has been stated below, the first tests were run to validate the data coming from the sensors. The throttle potentiometer did not implicate any problem, neither did the rotary encoder at the rear wheel. On the contrary, the pair of IMUs (in the frame and in the handle bar) presented some problems.
\begin{itemize}
\begin{itemize}
\item In the calibration stage, the IMUs seemed to lose the 'north pole' every time the PEV was transported from one floor to another. However, when testing in the same floor, these problem did not appear. To solve this issue, the IMUs had to be calibrated again with each test.

\item This calibration problem also affected considerably the initial absolute orientation of both IMUs, meaning that the relative angle between them --for the $\delta$ angle, for example-- had a initial drift.

\item The noise coming from the accelerometer and the gyroscope was excessive for a good control feedback. Each signal coming from the IMU was filtered with a simple one dimensional Kalman filter. Some preliminary tests were necessary to estimate the parameters of each Kalman filter: $p,\, q,\,k$.

\end{itemize}
\end{itemize}

The data was gathered using one of the Bluetooth modules, that sent the data to the computer, where the selected variables were saved in a .csv file. Afterwards, the data and the video from the front camera were a synchronized using an script in Matlab.

\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/live}
	\caption{Live streaming through Bluetooth module}
	\\[-1cm]
\end{figure}

The PEV was tested with and without the tilting mechanism, thanks to the modularity of the front suspension design. The non-tilting PEV was used for the tests where the sensor data was being validated and for the tests where the control strategy was implemented. After all the systems had been validated, the PEV was modified to allow the tilting and the final experiments were carried out.

\begin{figure*}[b]
	\includegraphics[width=1\linewidth]{figs/06/test_1}
	\caption{Testing setup}
	\\[-1.5cm]
\end{figure*}

\newpage

\subsection{Test Results}

The improvement of the PEV from the first to the last test was evident. The vehicle was exhaustively tested to improve the steering response as well as the remote control from the Android app. The mechanical robustness of the vehicle was put to the test during these experiments.

In the next set of pictures (Figure \ref{scene_2}), two turns to the circumference are illustrated. Before running any test, the PEV team consensually decided not to put in risk the integrity of any member, so it was decided to experiment only with driverless PEV. The main focus of these tests was the validation of the mechanics, the electronics and the control strategy.

It is important to recall that at the test stage any false step can imply the breaking of the PEV or in the worst case scenario, the harm of the researches. A testing vehicle can suppose a danger if it gets out of control, so some safety measures, both at the hardware and the software level were implemented.

Therefore, since it was a driverless test, the PEV was remotely controlled, both at the steering and at the throttle. In this experiment, the control of the PEV was smooth, and the data was correctly gathered for the afterwards analysis.

With regards to the tilting mechanism, it successfully worked properly, meaning that the front suspension geometry was appropriate and that the selection of the components was satisfactory. In addition, the self limitation of the geometry limited the tilting angle to a maximum, thus preventing the PEV from rolling over when tilting.

\begin{figure*}[!h]
	\includegraphics[width=1\linewidth]{figs/06/scene_2}
	\caption{Testing scene}
	\label{scene_2}
\end{figure*}

\newpage

\begin{marginfigure}
	\includegraphics[width=1.15\linewidth]{figs/06/scene00151}
	\caption{PEV tilting during the test}
\end{marginfigure}
Even though the control strategy was not fully validated, the implementation of the feedback and feedforward gains showed that the torque $M_{t}$ was giving good inputs for the tilting.

The main problem with the control strategy was focused on the integration of the perceived lateral acceleration ($a_{per}^{I}$). The integration of a signal coming from the IMU was reviewed in the chapter 4, where the velocity of the MiniPEV was estimated from the longitudinal linear acceleration. The similarities between the two challenges seem obvious. That is why the control strategy had to be finally reduced to the output variable  $a_{per}$ instead of $a_{per}^{I}$

In figures \ref{Picture1},\ref{Picture2}, \ref{Picture3} and \ref{Picture4} a sample of the carried out test has been included. This experiment was carried out allowing the tilting of the vehicle, and the control strategy was almost similar to the one presented previously in this chapter. The only difference can be found in the output variable, that is the $a_{per}$ instead of $a_{per}^{I}$.

As was already mentioned, lateral stability is obtained when $a_{per}=0$. The control of the integrated value of $a_{per}$ avoids the model errors due to parameters uncertainty, neglected dynamics or linearization of the model. Therefore, these are the errors that can appear if only the $a_{per}$ is controlled. In this case the process to obtain the gains is completely similar.

The Figure \ref{Picture1} represents the disturbance of the model, that is, the input from the driver. The steering angle indicates the desired direction. In this case, the vehicle does turn to the left before turning to the right. The reader can visualize the inherent noise in the steering rate $\dot{\delta}$. Even though the Kalman filter revokes some of the signal noise, it is important to remind that the gyroscope data is very prone to vibrations.
\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/tests/Picture1b}
	\caption{Disturbance signal; input from the driver $\delta$ and $\dot{\delta}$}
	\label{Picture1}
\end{figure}
\newpage

The velocity of the vehicle has been included in Figure \ref{Picture2}. This test was carried out at low speed, in order to have the vehicle under control if necessary. The lateral perceived acceleration $a_{per}$ is noisy as well, whereas the yaw rate $\dot{\psi}$ is more smooth. Anyway, the data confirms that both variables are properly calculated.

Before any input from the driver --when the vehicle is going in a straight line-- both the $a_{per}$ and the $\dot{\psi}$ are null. Just when the driver turns the handle bar and steers the front wheels, the yaw rate starts to increase (meaning a left turn) and the perceived acceleration takes negative values synchronously. Just when the steering angle is changed to the opposite direction, these two signals invert their values.

\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/tests/Picture2b}
	\caption{Vehicle variables: longitudinal velocity, perceived lateral acceleration and yaw rate}
	\label{Picture2}
\end{figure}

The response of the control strategy is to apply a torque $M_{t}$ so that the effect of the disturbance $\delta$ on the perceived lateral acceleration $a_{per}$ is canceled. Using the gains calculated previously, the torque required from the tilting motor is represented in the Figure \ref{Picture4}.

The control objective is obtained with very satisfying performances; $a_{per}$ does not exceed $1m/s^2$ and the maximum tilting torque value is 10 Nm. The fact that more torque is required to the right turn is due to the fact that the steering input is higher in this case, so that demanding more torque.
\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/tests/Picture4b}
	\caption{Torque required by the control system to tilt the PEV}
	\label{Picture4}
	\\[-3cm]
\end{figure}

\newpage
The applied torque in the tilting motor leans the PEV and as a consequence, there is a change in the tilting angle $\theta$ and in the tilting rate $\dot{\theta}$. In this case the maximum tilting angle goes over $0.15 rad=8 deg.$, which is not very high due to the low speed of the test.

\begin{figure}[!h]
	\includegraphics[width=1\linewidth]{figs/06/tests/Picture3b}
	\caption{Control strategy results}
	\label{Picture3}
\end{figure}

Overall, the carried out tests were satisfactory in almost every way. The exclusion of  the integrate of the perceived lateral acceleration from the control strategy changed the properties and the robustness of the system, but far from ruining the developed control model, this fact introduced a much simpler and effective model. While it is true that some errors can rise --parameters uncertainty, neglected dynamics or model linearization -- the reduced model successfully tilt the vehicle.

%\begin{figure*}[!h]
%	\includegraphics[width=1\linewidth]{figs/06/tests/Picture5}
%	\caption{Control strategy results}
%	\label{Picture5}
%\end{figure*}

\begin{figure}[b]
	\includegraphics[width=1\linewidth]{figs/06/test_2}
	\caption{PEV testing}
\end{figure}


\documentclass{scrartcl}
\usepackage[a4paper,left=1in,right=1in,top=1.2in,bottom=1in]{geometry}
\usepackage{siunitx}
\usepackage{graphicx}
\usepackage{mathtools}
\setkomafont{disposition}{\normalfont\bfseries}
\newcommand*\diff{\mathop{}\!\mathrm{d}}
\newcommand*\Diff[1]{\mathop{}\!\mathrm{d^#1}}
\newcommand*\colvec[3][]{
    \begin{pmatrix}\ifx\relax#1\relax\else#1\\\fi#2\\#3\end{pmatrix}
}

%title
\title{Exercise 04:\\Supervised Learning}
\subtitle{Theoretical Neuroscience II}
\author{Johannes G\"atjen \and Lorena Morton}

%use these for structure/overview
\newcommand\Question{%
  \textbf{Question:}%
}
\newcommand\Answer{%
  \textbf{Answer:}%
}

\begin{document}
\maketitle

\section{Training of feed-forward connections}
\begin{figure}[h]
\centering
\includegraphics[trim = {0.8cm 0 0.5cm 0.2cm}, width=0.48\textwidth, clip]{../pics/lin_mean}\includegraphics[trim = {0.8cm 0 0.5cm 0.2cm}, width=0.48\textwidth, clip]{../pics/lin_noise}\\
\includegraphics[trim = {0.8cm 0 0.5cm 0.2cm}, width=0.48\textwidth, clip]{../pics/cons_mean}\includegraphics[trim = {0.8cm 0 0.5cm 0.2cm}, width=0.48\textwidth, clip]{../pics/cons_noise}
\caption{Left: Mean response of selected neurons to different frequencies (tuning curve). Right:~Ensemble of actual noisy responses of the same neurons. Note the larger scale on the y axis. Top: Standard deviation for tuning curve increases linearly with frequency ($\kappa = 0.22$). Bottom: Constant standard deviation ($\sigma = 30\si{Hz}$).}
\end{figure}

\begin{figure}
\centering
\includegraphics[trim = {1cm 0 1cm 0.3cm}, width = 0.9\textwidth, clip]{../pics/cov}
\caption{Covariance matrices for the activity of the linear and constant standard deviation populations. Left: Calculated from mean activity. Right: Calculated from noisy activity. It can be seen, that neurons close to each other are positively correlated and neurons further apart are anti-correlated.}
\end{figure}
\clearpage

\section{Decoding the downstream activity}

\begin{figure}[h]
\centering
\includegraphics[trim = {0.7cm 0 0.5cm 0.2cm}, width = 0.7\textwidth, clip]{../pics/tuning}
\caption{The decoded downstream activity plotted against the input ($\sigma=30\si{Hz}$, $\kappa=0.22$, $N=M=40$). In a perfectly tuned network the points would form a straight line on the diagonal ($x=y$). Here aside from some noise, the tuning fits well for low to medium high frequency ranges. For very low inputs the decoded frequency is very inaccurate, ranging up to over 800 \si{Hz}. For inputs larger than ca.\ 750\si{Hz} the decoded frequencies are too low, due to the inaccurate input coming from the upstream population.}
\end{figure}

\begin{figure}
\centering
\includegraphics[trim = {0.7cm 0 0.5cm 0.2cm}, width = 0.7\textwidth, clip]{../pics/sigma}
\caption{The tuning (in)accuracy measured by the root mean square error (where error is taken as difference between input and decoded frequency) for $10000$ random inputs as a function of the $\sigma$ parameter ($\kappa=0.22$, $N=M=40$). The error grows linearly with $\sigma$.}
\end{figure}

\begin{figure}
\centering
\includegraphics[trim = {0.7cm 0 0.5cm 0.2cm}, width = 0.7\textwidth, clip]{../pics/numN}
\caption{Tuning error as a function of the number of neurons in both the upstream and downstream populations ($\sigma=30\si{Hz}$, $\kappa=0.22$, $N=M$). A higher number of neurons allows slightly more accurate decoding.}
\end{figure}

\begin{figure}
\centering
\includegraphics[trim = {0.7cm 0 0.5cm 0.2cm}, width = 0.7\textwidth, clip]{../pics/kappa}
\caption{Tuning error as a function of $\kappa$ ($\sigma=30\si{Hz}$, $N=M=40$). For $\kappa < 0.5$ the error grows quickly with increasing $\kappa$, for larger values the growth in error is less pronounced.}
\end{figure}

\end{document}


\section{Conclusion}
\label{conclusion}
Contingency analysis is a security function to assess the ability of a power grid to sustain various combinations of power grid component failures at energy control centers. To date, there exists no such work to examine the power grid resilience in Lebanon, a country which is still reeling under the effect of a brutal civil war, and recently, bearing an additional burden associated with the spillover from the Syrian war. This neighbouring conflict has exposed Lebanon to a number of random terrorist attacks, and caused it to become one of the major hosts of Syrian refugees, in addition to Iraqi and Palestinian refugees in former years. The strains on Lebanon's vital infrastructure and economical resilience have also been affecting the host community itself, where electric power supply is becoming increasingly drained. Our analysis of the resilience of Lebanese power grid captures all of the network down to its finest spatial coordinates. The computational burden associated with the resulting big graph is alleviated using a Spark-based, BSP modeled algorithm that balances the work optimally and incurs linear communication cost and a constant amount of synchronisation among threads. Our analysis exploits a high level of decentralisation in the Lebanese power grid, and reveals a high level of redundancy, thanks to Lebanon's widespread reliance on diesel generators for surviving daily power blackouts. Our analysis also reveals the loss of connectivity in the power grid as a function of failures. This function behaves analogously to the case of the North American grid, as can be seen from \cite{2000Natur.406..378A}, for example.

In addition to the functional requirements described in this paper, the nonfunctional requirements our contingency analysis software provide for all of the following:
\begin{enumerate}
\item Usability: The targeted audience of the system includes governmental and other vital organisations. Our tool is intended to assist governments in mitigating any possible exploitation of the network and the networks that are incumbent on it, for example, the road network, commercial network, telephony.
\item Portability: The system can run on any Hadoop configured machine, which can be a serial machine, a multi-core, or a cluster of machines.
\item Availability: The system is an open source project and can be downloaded from the developer's Github repository at \url{https://github.com/okm02/power-grid-analysis}.
\item Capacity: The capacity of the system adapts to the size of the graph. No further amendments are required, provided the input graph decomposes into a set of connected components each of which can be processed locally on a serial computer.
\end{enumerate}