\section{Experiment 1: Simulated Data from Multivariate Normal Distribution}

	In the first simulation experiment, we focused on an ideal setting where data come from a known multivariate normal
	distribution and imputation is required to estimate the means, variances, and covariances of six items with
	missing values.
	We investigated the relative performance of the methods described above across a set of conditions defined by two
	experimental factors: the number of columns in the dataset $p$, taking values 50 or 500;
	and the proportion of \emph{per} variable missing cases $pm$, taking values 0.1 or 0.3.
	Table \ref{tab:condExp1} summarizes the four crossed conditions.
	Data with sample size $n=200$ were independently generated $S = 1,000$ times for each condition.
	For each $s$-th replicate, missing values were imposed and then all the missing data treatment methods described above
	were used to obtain estimates of the item means, variances, and covariances.

\begin{table}
	\centering
	\begin{tabular}{l | r | r | r | r }
		condition & label & n & p & pm \\
		\hline
		1 & low-dim-low-pm   & 200 & 50  & .1 \\
		2 & high-dim-low-pm  & 200 & 500 & .1 \\
		3 & low-dim-high-pm  & 200 & 50  & .3 \\
		4 & high-dim-high-pm & 200 & 500 & .3 \\
	\end{tabular}
	\caption{\label{tab:condExp1}Summary of conditions for Experiment 1.}
\end{table}

%\FloatBarrier

\subsection{Simulation Study Procedure}

\subsubsection{Data Generation}
	At every replication, a data matrix $\bm{Z}_{n \times p}$ was generated according to a multivariate normal model
	centered around a mean of 0 with a covariance matrix $\bm{\Sigma}_0$, with diagonal elements (variances) equal to 1.
	The off-diagonal elements of $\bm{\Sigma}_0$ were used to define three blocks of variables:
	the first five variables were highly correlated among themselves ($\rho = .6$);
	variables 6 to 10 were weakly correlated with variables in block 1 and among themselves ($\rho = .3$),
	and all the remaining $p-10$ variables were uncorrelated.
	Items were rescaled to have mean of 5.

\subsubsection{Missing Data Imposition} \label{sub_missing}

	Missing values were imposed on six items in $\bm{Z}$: three variables in the block of
	highly correlated variables ($z_j$ with $j = 1,2,3$), and three in the block of lowly correlated variables ($z_j$
	with $j = 6,7,8$).
	Item non-response was imposed by sampling from a Bernoulli distribution with individual missing probabilities
	defined by
%
	\begin{equation} \label{eqn:rm}
		p_{miss} = p(z_{i,j} = miss | \tilde{Z}) = \frac{ exp(\gamma_0 + \tilde{Z}_{i}\bm{\gamma}) }
								{ 1 + exp(\gamma_0 + \tilde{Z}_{i}\bm{\gamma}) }
	\end{equation}
%
	where $z_{i,j}$ is the $i$-th subject's response on the $j$-th variable target of missing data imposition,
	$\tilde{Z}_{i}$ is a vector of responses to the set of predictors involved in the missing data mechanism
	for the $i$-th individual, $\gamma_0$ is an intercept parameter, and $\bm{\gamma}$ is a vector of slope
	parameters for the linear term.
	$\tilde{Z}$ was specified to include two fully observed variables from the highly correlated set, and two
	from the lowly correlated set ($z_r$ with $r = 4,5,9,10$).
	The probability of observing a response on a target variable did not depend on the variable itself,
	to avoid imputation under MNAR.
	Furthermore, when all the features in the data are provided to the MI procedures, the predictors in $\tilde{Z}$
	are allowed to be part of the imputation models and the MAR assumption can be met.
	All slopes in $\bm{\gamma}$ were fixed to 1, while the value of $\gamma_0$ was chosen with an optimization
	algorithm that minimized the difference between the actual and desired proportion of missing values.

\subsubsection{Imputation}

	Missing values were treated with all the methods described in Section 2.
	Convergence of the imputation models was assessed in a preprocessing step by observing trace plots.
	The imputation algorithms were considered to have converged after 50 iterations, after which 10 imputed data
	sets were stored and used for the subsequent standard complete-data analysis and pooling.
	The only exception was blasso, which required approximately 2,000 iterations for convergence.

	The ridge penalty used in the bridge algorithm was fixed across iterations.
	The value used in the simulation was determined by means of cross-validation in a preprocessing phase.
	The ridge penalty values $10^{-1}, 10^{-2}, ..., 10^{-8}$ were used to impute data with bridge
	and we selected the value that resulted in the smallest average Fraction of Missing Information (FMI)
	\citep[eq. 3.1.10]{rubin:1987} across the analysis model parameters.

	Both IURR and DURR could have been specified with a variety of penalty parameters.
	For example, one could use any of the following: ridge penalty \citep{hoerlKennard:1970}, lasso penalty
	\citep{tibshirani:1996}, elastic net penalty \citep{zouHastie:2005}, adaptive lasso \citep{zou:2006}.
	In this study, we specified the lasso penalty for regularization as it is computationally efficient,
	and it performed well for imputation in \cite{zhaoLong:2016} and \cite{dengEtAl:2016}.
	A 10-fold cross-validation procedure was used at every iteration of DURR and IURR to choose the penalty parameter.

	For blasso, in order to maintain consistency with previous research, the hyper-parameters in equations
	\eqref{eqn:sigprior}, \eqref{eqn:tauprior}, and \eqref{eqn:rhoprior} were specified as in \cite{zhaoLong:2016}:
	$(a,b)=(0.1, 0.1)$, $(r,s)=(0.01, 0.01)$, and $(g,h)=(1,1)$.
	In the MI-PCA algorithm, enough components were extracted to explain 50\% of the total variance in the data.
	To impute data with the single imputation random forest approach we used the missForest R package
	\citep{missForest} implementing algorithm 1 proposed by \cite{stekhovenBuhlmann:2011}.
	The stopping criterion for the missForest algorithm was usually met within the first 10 iterations,
	but to make a conservative choice we fixed the maximum number of iterations to 20.
	\cite{stekhovenBuhlmann:2011} showed that increasing the number of trees grown in each forest has
	stagnating effects on the imputation error while linearly increasing the computation time.
	In their paper, the authors recommend growing 100 trees per forest, which offers a good compromise
	between imputation precision and computation time.
	Therefore, we used this value in our study.

\subsubsection{Analysis}
	The substantive model of interest in Experiment 1 was a saturated model that estimated means,
	variances, and covariances of the six variables with missing values.
	This resulted in estimating six means, six variances, and 15 covariances.

\subsection{Comparison Criteria} \label{criteria}

	We compared methods in terms of bias and confidence interval coverage.

	\paragraph{Bias}

	For a given parameter of interest $\theta$ (e.g., mean of item 1, variance of item 2), we used the
	Percent Relative Bias (PRB) to quantify the estimation bias introduced by the imputation procedures:
%
	\begin{equation} \label{eqn:prb}
		PRB = \frac{\bar{\hat{\theta}} - \dot{\theta}}{\dot{\theta}} \times 100
	\end{equation}
%
	where $\dot{\theta}$ is the true value of the focal parameter defined as
	$\sum_{s=1}^{S} \hat{\theta}_{s}^{GS}/S$
	, with
	$\hat{\theta}_{s}^{GS}$
	being the Gold Standard parameter estimate for the $s$-th repetition.
	The averaged focal parameter estimate under a given missing data treatment is computed as
	$\bar{\hat{\theta}} = \sum_{s=1}^{S} \hat{\theta}_{s}/S$,
	with
	$\hat{\theta}_{s}$ being the estimate obtained after having treated the missing values in the
	$s$-th repetition.
	Following \cite{muthenEtAl:1987}, $|\text{PRB}| > 10\%$ was considered indicative of problematic
	estimation bias.

	\paragraph{Confidence Intervals Coverage}
	To assess the correctness of hypothesis testing, the Confidence Interval Coverage (CIC) of the true parameter
	value was defined as:
%
	\begin{equation} \label{eqn:cic}
		CIC =  \frac{ \sum_{s=1}^{S} I(\dot{\theta} \in \widehat{CI}_s ) }{S}
	\end{equation}
%
	where $\widehat{CI}_s$ is the confidence interval of the parameter estimate $\hat{\theta}_{s}$ in a given repetition,
	and $I(.)$ is the indicator function that returns 1 if the argument is true and 0 otherwise.

	CICs below 0.9 are usually considered problematic for 95\% confidence intervals \cite[p. 52]{vanBuuren:2018}
	as they imply inflated Type I error rates.
	A high coverage (e.g., 0.99) may indicate confidence intervals that are too wide, implying inflated Type II error rates.
	Therefore, Confidence Intervals were considered to show severe under-coverage (over-coverage) if they were
	below 0.9 (above 0.99).

	Following \cite{burtonEtAl:2006}, in simulation studies, a CIC can be considered as significantly different from the
	nominal coverage rate if it falls outside two Standard Errors of the nominal coverage probability ($SE(p)$) from the
	nominal coverage rate.
	The standard error of nominal coverage probability is defined as $SE(p) = \sqrt{p (1-p)/S}$, with $p$ indicating the
	chosen nominal coverage probability.
	Therefore, for $S = 1000$, 95\% CI coverages ($p = 0.95$) outside the range (0.94, 0.96) were considered as significantly
	different from the nominal coverage rate.

\subsection{Results}

	Both PRB and CIC were computed for all the 27 parameters in the analysis model (six means, six item variances,
	and 15 covariances).
	To summarize the results, we focus on the typical and extreme values of these measures.
	In Figures \ref{fig:exp1bias} and \ref{fig:exp1cir}, we report the average, minimum, and maximum absolute PRB
	and CIC achieved by the missing data treatment methods for each parameter parameter type.
	In the supplementary material, we included figures reporting the raw PRB and CIC for every parameter estimate.
%
	\paragraph{Means}
	Focusing first on the item means (top rows), the largest $|\text{PRB}|$ is below 10 percentage points for all
	imputation methods.
	However, looking at relative performances, IURR and MI-PCA resulted in smaller bias than all other methods,
	except MI-OP.
	In the conditions with high proportion of missing values (columns 3 and 4 in the figures), all methods showed
	significant deviations from nominal coverage rates, with all CICs outside of the interval $(0.94, 0.96)$.
	The only exceptions was MI-PCA which showed non-significant deviations from nominal coverage for
	almost all estimates, with both the lowest and highest CIC falling within (0.94, 0.96) in all conditions.
	The tree-based MI methods, missForest, and CC lead to CICs significantly different from nominal coverage rates
	in all conditions.
%
	\paragraph{Variances}
	Moving to the item variances (central rows), IURR, blasso, and the MI tree-based methods resulted in the lowest
	biases across all conditions, even in the high-dim-high-pm condition.
	These low biases were mostly paired with low deviations from nominal coverage, except for the high-dim-high-pm
	condition where they resulted in significant under-coverage of the true item
	variances (highest $CIC < 0.94$).
	Nevertheless, apart from MI-OP, blasso was the method with best coverage in this final condition.

	DURR showed poor performance with regard to the item variances: in all conditions but the first, it led to
	large bias accompanied by significant CI under-coverage.
	Bridge was the only MI method showing larger bias than DURR in all the high-dimensional conditions (columns 2 and 4),
	with even the minimum $|\text{PRB}|$ exceeding the 20\% threshold.
	MI-PCA also showed poor performance with noticeable item variance bias in all conditions that became extreme in the
	high-dim-high-pm condition (column 4), where even the smallest $|\text{PRB}|$ exceeded 20\%.
	This poor performance was reflected in extreme confidence interval under-coverage of the true item variances in
	the final experimental condition.
	Finally, missForest and CC led to substantial bias and CI under-coverage for all
	item variances, even in condition 1.
%
	\paragraph{Covariances}
	IURR performed noticeably better than most other methods, with negligible covariance bias
	(maximum $|\text{PRB}| < 10\%$) and acceptable coverage (average $CIC \sim 0.94$) in the high-dim-low-pm
	condition.
	However, it struggled with a large covariance bias and extreme under-coverage in the high-dim-high-pm
	condition (average $|\text{PRB}| > 10\%$ and average $CIC < 0.90$).
	MI-PCA showed negligible bias for all the covariance estimates (with the maximum $|\text{PRB}| < 10\%$),
	and performed as well as MI-OP in all but the high-dim-high-pm condition.
	Furthermore, MI-PCA resulted in virtually no deviation from nominal coverage, except in the last condition.
	In the high-dim-high-pm condition, MI-PCA led to negligible bias, and mild significant \emph{over}-coverage
	of the items covariances: the average CIC was greater than 0.96, but smaller than 0.99.

	Bridge displayed low bias and acceptable coverage in the low dimensional conditions (columns 1 and 3), but
	extreme bias and low CI coverage in all the high dimensional conditions (columns 2 and 4).
	MissForest and CC showed extreme bias and under-coverage for all the covariances (minimum $|\text{PRB}|>10\%$),
	even in condition 1.
	All other methods, including DURR, showed absolute covariance PRBs larger than the 10\% threshold in all but the
	first condition, with persistently significant CI under-coverage of the true values.

\begin{figure}
\centering
\includegraphics{\pathFIG/exp1_bias_summy.pdf}
\caption{\label{fig:exp1bias}
	Maximum, average, and minimum absolute Percent Relative Bias ($|\text{PRB}|$) for item means, variances,
	and covariances in Experiment 1.
	For each method, the maximum, average, and minimum $|\text{PRB}|$ are reported in this order.
	}
\end{figure}

\begin{figure}
\centering
\includegraphics{\pathFIG/exp1_CI_summy.pdf}
\caption{\label{fig:exp1cir}
	Maximum, average, and minimum CIC for item means, variances, and covariances in Experiment 1.
	For each method, the maximum, average, and minimum are reported in this order.
	}
\end{figure}

%\FloatBarrier % stops fig:exp1cir to leave its section


\lab{Application}{Cracking Blackjack}{Cracking Blackjack}
\label{Ch:BJ}

\objective{This section teaches about how toexploite the weaknesses of a pseudorandom number generator that uses Linear Congruience}

\section*{Blackjack}

%Lab \ref{BJ}

Black Jack is a card game that involes the use of randomness. The game is simple, the dealer deals the player and himself each two cards. He flips over his first card so the player can see it. The player has to chose to take another card ("hit") or not ("stand"). If the player hits he gets another card and again has the choice to hit or stand.

The goal is to get your hand to be at or as close to 21 without going over. Face cards are worth 10 points. Aces can count either as 11 or 1. The of value other cards are equal to number on the card.

Once the player has decided to stand the dealer flips over his second card and deals himself cards until his hand value is 17 or greater.

If the player value goes above 21 he automaticly loses. If his value is 21 and below and dealer has above 21 then the player wins. If they both have 21 or under than whose hand has the highest value, wins. If both hands have the same value, the game is a tie.

\section*{Shuffling Algorithms}

One use of Psuedorandom Number Generators (PRNGs) is to shuffle cards. The main goal of these algorithms is that the card order are random - so that no one player has advatage based on order. Often online gambling sites will post their algorithms online. The only thing they do not post is their seed values. Often the time in miliseconds is used as the seed value.

\section*{Cracking Blackjack}
For these next problems you will need three files that are provided:Black.py, BlackEasy.py, and bjHelp.py.  Black.py and BlackEasy.py are are programs that run games of Blackjack that use a Linear Congruentail Generator (LCG) to shuffle the cards. They print out 52 numbers and the argsort of those numbers is the order of the cards.The parameters for BlackEasy.py are a$=2521$, c$=13$, mod$=2^{16}$; For Black.py they are a$=25214903917$, c$=11$, mod$=2^{48}$. In order to play them python name (Black.py or BlackEasy.py) numberofgames. They are both seeded initially by the time.

bjHelp.py contains two functions that will help you "predict" the cards:
SuffleHack(n,a,c,mod,seed) gives the first n card shuffles given the parameters for a LCG. The shuffles are represented by numbers
Hacker(Stats,['card','card','card']) Stats is the output of SuffleHack and takes a list of 3 cards (see below.) It prints all shuffles as a list of cards in Stats that have the same first three cards as the inputed list.

The trick to being able to "predict" the cards is to find the initial seed value.

Cards- A, 2-10, J, Q, or K combined with heart, diamond, club, or spade in single quotes. Examples: '6diamond', 'Kclub'

\begin{problem}
Play 10 games of BlackEasy.py and by the 5th game be able to predict the cards. You can write your own functions or use the ones in bjHelp.py. You will want to open two comand promts, one to play the game and one to use to predict the cards.
\end{problem}

Not too hard. That is because there is only $2^{16}$ seed values. This next one you will have to look at more hands until you can find out the initial seed value.

\begin{problem}
Play 20 games of Black.py and by the 15th game be able to predict the cards.
\end{problem}


% !TEX root=../main.tex

We now present our one-class Neural Network (OC-NN) model for unsupervised anomaly detection.
The method can be seen as designing a neural architecture using an  OC-SVM equivalent loss function.
Using OC-NN we will be able to exploit and refine features obtained from unsupervised tranfer learning specifically for anomaly
detection. This in turn will it make it possible to discern anomalies in complex data sets where the decision boundary between normal and anomalous
is highly nonlinear.
\vspace{-0.1cm}
\subsection{One-Class Neural Networks (OC-NN)}
\label{sec:oc-nn}
We design a simple feed forward network with one hiden layer having linear or sigmoid activation $g(\cdot )$ and one output node. Generalizations to deeper
architectures is straightforward. The OC-NN objective can be formulated as:
\begin{equation}
	\label{eqn:oc-nn}
	\min_{w, V, \bias} \frac{1}{2} \| w \|_{2}^2 + \frac{1}{2} \| V \|_F^2 + \frac{1}{\nu} \cdot \frac{1}{N} \sum_{n= 1}^N \max( 0, \bias - \langle w, g( V  \X_{n :} ) \rangle ) - \bias
\end{equation}

where $w$ is the scalar output obtained from the hidden to output layer,
$V$ is the weight matrix from input to hidden units.
Thus the \underline{key insight} of the paper is to replace the dot product $\mathbf{\langle w,\Phi(\X_{n :}) \rangle}$ in OC-SVM with the dot product $\mathbf{\langle w,g(V\X_{n :})\rangle}$. This change will make it possible to leverage transfer learning features obtained using an autoencoder and create an additional layer to refine
the features for anomaly detection. However, the price for the change is that the objective becomes non-convex and thus the resulting algorithm
for inferring the parameters of the model will not lead to a global optima.

%which is randomly initialized from the uniform distribution within the range $[-1, 1]$.  $\nu \in (0,1)$, is the parameter that
%equates to the percentage of anomalies within data, found via grid search in the experiments. $\bias$ is the equivalent maximum margin separating plane from the origin.

%%%%
\begin{comment}
\subsection{Connection to OC-SVM}
\label{sec:connection_to_OC-SVM}

Clearly, Equation~\ref{eqn:oc-nn} is the special case of the Equation~\ref{eqn:ocsvm-objective} presented in the Section~\ref{sec:ocsvm}, where the weights $V$  in Equation~\ref{eqn:oc-nn} are randomly initialised, and not optimised, and $g$ is the either a $linear$ and $sigmoid$ activations.
More generally, for a deep network with parameters $w$, we have

\begin{equation}
\label{eqn:gen-oc-nn}
 \min_{w,\bias} \frac{1}{2} \| w \|_{2}^2 +\frac{1}{\nu} \cdot \frac{1}{N} \sum_{i = 1}^N \max( 0, \bias - \langle w, \Phi( \X_{n :} w ) \rangle ) - \bias
\end{equation}
\end{comment}

%%%%
\vspace{-0.1cm}
\subsection{Training the model}
\label{sec:training}
We can optimize Equation~\ref{eqn:oc-nn} using an alternate minimization approach: We first fix $r$ and optimize for $w$ and $V$. We then use
the new values of $w$ and $V$ to optimize $r$. However, as we  will show, the optimal value of $r$ is just the $\upsilon$-quantile of
the array $\langle w,g(Vx_{n})\rangle$. We first define the objective to solve for $w$ and $V$ as
% \vspace{-0.1cm}
\begin{equation}
                \label{eqn:minimize_w_V}
                 \underset{w, V}{\argmin} \frac{1}{2} \| w \|_{2}^2 + \frac{1}{2} \| V \|_F^2 + \frac{1}{\upsilon} \cdot \frac{1}{N} \sum_{n = 1}^N \ell( y_n, \hat{y}_n( w, V ) )
                 \end{equation}
                where
                \begin{align*}
                        \ell( y, \hat{y} ) &= \max( 0, y - \hat{y} ) \\
                        y_n       &= \bias \\
                        \hat{y}_n( w, V ) &= \langle w, g( V x_n ) \rangle
                \end{align*}

Similarly the optimization problem for $\bias$ is
\begin{equation}
\label{eqn:minimize_r}
\underset{\bias}{\argmin} \left( \frac{1}{N\upsilon} \cdot \sum_{n = 1}^N \max( 0, \bias - \hat{y}_n ) \right) - r
\end{equation}
\begin{comment}
%%%
\begin{enumerate}[(1)]
	\item Initialise $w^{(0)}, V^{(0)}, r^{(0)}$ randomly
	\item for $t = 1, 2, \ldots$
	\begin{enumerate}[(a)]
		\item find $(w^{(t+1)}, V^{(t+1)})$ that minimise the objective through Backpropagation (BP) algorithm.
		\begin{equation}
		\label{eqn:minimize_w_V}
		 \underset{w, V}{\argmin} \frac{1}{2} \| w \|_{2}^2 + \frac{1}{2} \| V \|_F^2 + \frac{1}{\upsilon} \cdot \frac{1}{N} \sum_{n = 1}^N \ell( y_n, \hat{y}_n( w, V ) )
		 \end{equation}
		where
		\begin{align*}
			\ell( y, \hat{y} ) &= \max( 0, y - \hat{y} ) \\
			y_n       &= \bias^{(t)} \\
			\hat{y}_n( w, V ) &= \langle w, g( V x_n ) \rangle
		\end{align*}

		\item pick $\bias^{(t)}$ to be the $\upsilon^{\text{th}}$ quantile of $\{ \hat{y}_n \}_{n = 1}^N$, where
		$$ \hat{y}_n^{(t+1)} = \langle w^{(t+1)}, g( V^{(t+1)} x_n ) \rangle. $$
	\end{enumerate}

\end{enumerate}

To justify step 2(b), let us observe that the optimisation with respect to $\bias$ is\\
%
\begin{equation}
\label{eqn:minimize_r}
\underset{\bias}{\argmin} \left( \frac{1}{N\upsilon} \cdot \sum_{n = 1}^N \max( 0, \bias - \hat{y}_n ) \right) - r
\end{equation}
\end{comment}
\begin{theorem}
Given $w$ and $V$ obtained from solving Equation~\ref{eqn:minimize_w_V}, the solution to Equation~\ref{eqn:minimize_r} is given
by the $\upsilon^{\text{th}}$ quantile of $\{ \hat{y}_n \}_{n = 1}^N$, where
                $$ \hat{y}_n = \langle w, g(V x_n ) \rangle. $$
\end{theorem}
\begin{proof}
 We can rewrite Equation~\ref{eqn:minimize_r} as:
\begin{align*}
%\underset{\bias}{\argmin} \left( \frac{1}{N\upsilon} \cdot \sum_{n = 1}^N \max( 0, \bias - \hat{y}_n ) \right) - r \\
\underset{\bias}{\argmin} \left( \frac{1}{N\upsilon} \cdot \sum_{n = 1}^N \max( 0, \bias - \hat{y}_n ) \right) - \left( \bias - \frac{1}{N} \sum_{n = 1}^{N} \hat{y}_n \right) \\
= \underset{\bias}{\argmin} \left( \frac{1}{N\upsilon} \cdot \sum_{n = 1}^N \max( 0, \bias - \hat{y}_n ) \right) - \left( \frac{1}{N} \sum_{n = 1}^{N} \left[ \bias - \hat{y}_n \right] \right) \\
= \underset{\bias}{\argmin} \left( \sum_{n = 1}^N \max( 0, \bias - \hat{y}_n ) \right) - \upsilon \cdot \left( \sum_{n = 1}^{N} \left[ \bias - \hat{y}_n \right] \right) \\
= \underset{\bias}{\argmin} \sum_{n = 1}^N \left[ \max( 0, \bias - \hat{y}_n ) - \upsilon \cdot \left( \bias - \hat{y}_n \right) \right] \\
= \underset{\bias}{\argmin} \sum_{n = 1}^N
\begin{cases} (1 - \upsilon) \cdot \left( \bias - \hat{y}_n \right) & \text{ if } \bias - \hat{y}_n > 0 \\ - \upsilon \cdot \left( \bias - \hat{y}_n \right) & \text{ otherwise}
\end{cases}
\end{align*}
% \vspace{0.4cm}
\vspace{-0.1cm}
We can observe that the derivative with respect to $r$ is
$$ F'( r ) = \sum_{n = 1}^N \begin{cases} (1 - \upsilon) & \text{ if } \bias - \hat{y}_n > 0 \\ -\upsilon & \text{ otherwise. } \end{cases} $$
Thus, by F'( r ) = 0 we obtain
%
\begin{align*}
	(1 - \upsilon) \cdot \sum_{n = 1}^N \indicator{ \bias - \hat{y}_n > 0 } &= \upsilon \cdot \sum_{n = 1}^N \indicator{ \bias - \hat{y}_n \leq 0 } \\
	&= \upsilon \cdot \sum_{n = 1}^N (1 - \indicator{ \bias - \hat{y}_n > 0 }) \\
	&= \upsilon \cdot N - \upsilon \cdot \sum_{n = 1}^N \indicator{ \bias - \hat{y}_n > 0 },
\end{align*}
or
\begin{equation}
 \frac{1}{N} \sum_{n = 1}^N \indicator{ \bias -\hat{y}_n > 0 }= \frac{1}{N} \sum_{n = 1}^N \indicator{ \hat{y}_n < r } = \upsilon
\end{equation}\\
This means we would require the $\nu^{\text{th}}$ quantile of $\{ \hat{y}_n \}_{n = 1}^N$.
\end{proof}
% \vspace{0.3cm}
%Algorithm
\vspace{-0.3cm}
\subsection{OC-NN Algorithm}
\label{sec:algorithm}
We summarize the solution in Algorithm~\ref{alg1}. We initialize $\bias^{(0)}$ in Line 2. We learn the parameters($w,V$) of the neural network
using the standard Backpropogation(BP) algorithm (Line 7). In the experiment section, we will train the model using features extracted from
an autoencoder instead of raw data points. However this has no impact on the OC-NN algorithm. As show  in Theorem 3.1, we solve for $\bias$
using the $\upsilon$-quantile of the scores $\langle y_{n} \rangle$. Once the convergence criterion is satisfied, the data points are
labeled normal or anomalous using the decision function $S_{n} = sgn(\hat{y}_{n} - r)$.

\begin{algorithm}
\caption{one-class neural network (OC-NN) algorithm}\label{alg:oc-nn}
\label{alg1}
\begin{algorithmic}[1]
\State{
\textbf{Input:} Set of points $\X_{n :},\hspace{0.2cm} n: 1,...,N$
}
\State{
\textbf{Output:} A Set of decision scores $S_{n :}=\hat{y}_{n :}$, n: 1,...,N for X
}

\State Initialise $\bias^{(0)}$
\State $t \gets 0$
\While{(no convergence achieved)}
\State Find $(w^{(t+1)}, V^{(t+1)})$ \Comment Optimize Equation~\ref{eqn:minimize_w_V} using BP.
\State $r^{t+1} \gets \nu^{\text{th}}$ quantile of $\{ \hat{y}_n^{t+1} \}_{n = 1}^N$
\State $t \gets t + 1$
\EndWhile \label{endwhile}
\State \textbf{end}
\State Compute decision score $S_{n :}=  \hat{y}_n -r  \mbox{ for each }  \X_{n :}$
\If{($S_{n :}$ $\geq$ 0)}
    \State $\X_{n :}$ is normal point
   \Else
    \State $\X_{n :}$ is anomalous
\EndIf

\State \textbf{return} $\{S_n\}$

\end{algorithmic}
\end{algorithm}
% \vspace{-0.05cm}
%
\noindent
{\bf Example:} We give a small example to illustrate that the minimum of the function
\[f(r) =  \left( \frac{1}{N\upsilon} \cdot \sum_{n = 1}^N \max( 0, \bias - y_n ) \right) - r\] occurs at the
the $\upsilon$-quantile of the set $\{y_{n}\}$. \\

Let $y =\{1,2,3,4,5,6,7,8,9\}$ and  $\upsilon = 0.33$. Then the minimum will occur at $f(3)$ as detailed in
the table below.
\[
\begin{array}{|r|l|r|}  \hline \hline
r & f(r) (\mbox{expr}) & f(r) (\mbox{value}) \\ \hline \hline
1 & \frac{1}{9*.33}[0 + 0 +\ldots 0] - 1 & -1.00 \\ \hline
2 & \frac{1}{9*.33}[1 + 0 +\ldots 0] - 2 & -1.67 \\ \hline
\rowcolor{green!20}
3 & \frac{1}{9*.33}[2 + 1 +\ldots 0] - 3 & -1.99  \\ \hline
4 & \frac{1}{9*.33}[3 + 2 + 1 +\ldots 0] - 4 & -1.98  \\ \hline
5 & \frac{1}{9*.33}[4 + 3 + 2 + 1\ldots 0] - 5 & -1.63  \\ \hline
6 & \frac{1}{9*.33}[5 + 4  + 3 + 2 + 1\ldots 0] - 6 & -0.94  \\ \hline
7 & \frac{1}{9*.33}[6 + 5  + 4 + 3 + 2 + 1\ldots 0] - 7 & 0.07  \\ \hline
8 & \frac{1}{9*.33}[7 + 6  + 5 + 4 + 3 + 2 + 1\ldots 0] - 8 & 1.43  \\ \hline
9 & \frac{1}{9*.33}[8 + 7  + 6 + 4 + \ldots 0] - 9 & 3.12  \\ \hline
\end{array}
\]

\begin{comment}
\subsection{Predicting with the model}

One convenient property of our model is that the anomaly detector will be inductive, i.e.\ it can generalise to unseen data points.
One can interpret the model as learning a non linear projection  of the input;
such a representation should thus be able to accurately model any unseen points that lie on the same manifold as the data used to train the model. Formally, $f( x ) = \langle w, \Phi( \X_{n :} w ) \rangle$  computes decision score of the given data point.
The "negative" or "closer" $\langle w, \Phi( \X_{n :} w ) \rangle$ is, to origin and the more likely the point is deemed to be anomalous.

%
\subsection{Relation to existing models}
Our contribution is breaking-new grounds, to integrate the OC-SVM equivalent objective into the neural
network architecture, utilizing the features extracted through unsupervised tranfer representation learning for anomaly detection.
Some previous works have employed deep networks for anomaly detection~\cite{chalapathy2017robust,zhou2017anomaly}. For example, the recent work of \cite{zhou2017anomaly} employed an autoencoder-inspired objective to train a robust deep autoencoder network, with extensions to capture structured noise within the data; Our method is distinct to robust deep autoencoders (RDA),  wherein we introduce a novel one-class neural network objective function illustrating the analytic derivation of $\nu^{th}$ quantile of the sorted output scores represents, the maximum margin distance,  separating normal data from anomalies. Although, nonlinear projections~\cite{bingham2001random,rathore2017ensemble} have been explored  for anomaly detection. We are unaware of prior usage of transfer learnt autoencoder networks integrated with one-class neural network objective to detect anomalies in unsupervised fashion.
\end{comment}


\appendix
\section{Cantilever beam dynamics}

Consider an eVTOL wing with multiple electric motors and rotors mounted along the spar, shown in Fig.~\ref{fig:wing_diagram}. The wing structure is idealized as a cantilever beam with non-structural lumped masses placed along the span, shown in Fig.~\ref{fig:cantilever}.

\begin{figure}[H]
     \centering
         \includegraphics[width=0.7\textwidth]{images/wing_diagram.png}
        \caption{Rotor layout for an example eVTOL wing}
         \label{fig:wing_diagram}
\end{figure}

\begin{figure}[H]
         \centering
         \includegraphics[width=0.7\textwidth]{images/cantilever.png}
        \caption{Idealization of eVTOL wing structure}
         \label{fig:cantilever}
\end{figure}

To expand the generality of the analysis, consider a tapered circular spar running along the length of the wing. The cross-section of the spar is shown in Fig.~\ref{fig:taper_beam}. The tube thickness is equal to $t$, and is uniform along the spar length. The radius of the circular cross-section varies from $r_1$ at the root of the beam to $r_2$ at the tip. The mass per unit span and bending stiffness of the spar is given by
\begin{align*}
m(x) \quad &= \quad 2 \pi \rho r(x) t \\
EI_{yy}(x) \quad &= \quad E \pi r(x)^3 t
\end{align*}

\begin{figure}
\begin{center}
\includegraphics[width=0.45\textwidth]{images/taper_beam.png}
\vspace{-0.1cm}
\caption{Linearly tapered circular spar}
\label{fig:taper_beam}
\end{center}
\end{figure}

The first bending natural frequency is estimated using a Rayleigh-Ritz approximation, with a mode shape set by the static deflection of the cantilever due to a tip-load, i.e.,
\begin{equation}
w''(x) \quad = \quad P (L - x)
\end{equation}
Integrate twice along the span and apply the cantilever boundary condition to obtain the deflected bending slope as
\begin{equation}
w'(x) \quad = \quad \frac{P}{E\pi t}\left[\frac{a_2}{r(x)^2} \spc+\spc \frac{a_1}{r(x)} \spc+\spc a_0\right]
\end{equation}
The constants are
\begin{align*}
a_2 \quad =& \quad -\frac{L}{2r'} \spc-\spc \frac{r_1}{2{r'}^2} \\
a_1 \quad =& \quad -\frac{1}{{r'}^2} \\
a_0 \quad =& \quad \frac{L}{2r'r_1^2} \spc-\spc \frac{1}{2{r'}^2 r_1}
\end{align*}
Integrate the bending slope along the span to obtain the deflected mode shape as
\begin{equation}
w(x) \quad = \quad \frac{P}{E\pi t}\left[b_0 \spc + \spc b_1 x \spc+\spc \frac{b_m}{r(x)} \spc+\spc b_l \log_e r(x)\right]
\end{equation}
The constants are
\begin{align*}
b_0 \quad =& \quad -\frac{a_2}{r_1 r'} \spc-\spc \frac{a_1}{r'} \log_e(r_1)\\
b_1 \quad =& \quad a_0 \\
b_m \quad =& \quad -\frac{a_2}{r'}  \\
b_l \quad =& \quad \frac{a_1}{r'}
\end{align*}
\subsection{Potential energy}
The maximum potential energy is given by
\begin{equation*}
U \quad = \quad \frac{1}{2} \int_0^L \frac{P^2(L-x)^2}{EI_{yy}(x)} dx
\end{equation*}
Substitute for the bending stiffness to obtain
\begin{equation*}
U \quad = \quad \frac{1}{2} \frac{P^2}{E\pi t} [A + B + C](\lambda)
\end{equation*}
The term $\lambda$ is the spar taper ratio, given by
\begin{equation}
\lambda \quad = \quad \frac{r_2}{r_1}
\end{equation}
The sum $A+B+C$ is well-represented by a cubic polynomial in $\lambda$ as
\begin{equation*}
A+B+C\quad \approx \quad (1.6261 \spc - \spc 0.4826 \lambda \spc + \spc 2.1197 \lambda^2 \spc - \spc 0.5957 \lambda^3) \frac{1}{K^3}
\end{equation*}
The constant $K$ is the inverse of the spar aspect ratio, given by
\begin{equation}
K \quad = \quad \frac{2 (\frac{t}{c})}{AR_{\rm wing}}
\end{equation}
Here, $\frac{t}{c}$ is the wing thickness to chord ratio, and $AR_{\rm wing}$ is the wing aspect ratio.

\subsection{Kinetic energy}
The maximum kinetic energy of the beam has contributions from three types of masses:
\begin{enumerate}
\item \textbf{Structural masses}: This term deals with the kinetic energy of the spar structure. This contribution to kinetic energy is
\begin{equation*}
KE_{s} \quad = \quad \int_0^L \frac{1}{2} \spc 2 \pi r(x) t \rho \spc w(x)^2 \spc \omega_n^2 \spc dx \quad = \quad \frac{P^2}{E^2 \pi t} \rho \omega_n^2 (RHS)
\end{equation*}
The term $RHS$ is a function of the taper ratio $\lambda$, length $L$ and spar taper ratio inverse $K$ only. Further, the dependence on wing taper $\lambda$ can be approximated using a cubic polynomial in $\lambda$. Thus, the kinetic energy coefficient $RHS$ is approximated as
\begin{equation*}
RHS \quad \approx \quad \frac{L^2}{K^5} (0.04823 \spc + \spc 0.269 \lambda \spc + \spc 0.15178 \lambda^2 \spc + \spc 0.36917 \lambda^3)
\end{equation*}
\item \textbf{Non-structural distributed masses}: This term deals with the kinetic energy due to the skin and other components (e.g., wires) that do not contribute significantly to bending stiffness. The kinetic energy term is
\begin{equation*}
KE_{\rm NS} \quad = \quad \int_0^L \frac{1}{2} \spc m_{\rm NS}(x)  \spc w(x)^2 \spc \omega_n^2 \spc dx
\end{equation*}
The non-structural mass due to wing skin is given by
\begin{equation*}
m_{\rm NS}(x) \quad = \quad \frac{2 M_{\rm NS}}{A (\frac{t}{c})} r(x)
\end{equation*}
The term $\frac{M_{\rm NS}}{A}$ is the non-structural mass per unit plan-form area of the skin. Define an intermediate quantity $\beta$ given by
\begin{equation*}
\beta \quad = \quad \frac{M_{\rm NS}}{A (\frac{t}{c})}
\end{equation*}
The expression for kinetic energy of non-structural masses simplifies to
\begin{equation*}
KE_{\rm NS} \quad = \quad \beta \spc \frac{P^2 \omega_n^2}{E^2 \pi^2 t^2} \spc RHS
\end{equation*}
\item \textbf{Non-structural lumped mass}: This term refers to the kinetic energy of the electric motor/speed controller unit, rotor hubs and rotor blades. These discrete lumped masses are mounted along the wing spar, and the kinetic energy of these masses is given by
\begin{equation*}
KE_{\rm lump} \quad = \quad \mathlarger{\mathlarger{\Sigma}}_i^{N_{\rm M}}\spc \frac{1}{2} \spc M_i \spc w(x_i)^2 \quad = \quad \frac{P^2 \omega_n^2}{E^2 \pi^2 t^2} \Delta RHS
\end{equation*}
The coefficient $\Delta RHS$ is
\begin{equation*}
\Delta RHS \quad = \quad \mathlarger{\mathlarger{\Sigma}}_i^{N_{\rm M}} \frac{M_i}{2} \left[b_0 \spc + \spc b_1 x_i \spc + \spc b_l \log_e r(x_i) \spc + \spc \frac{b_m}{r(x_i)} \right]^2
\end{equation*}
\end{enumerate}
\subsection{Frequency tuning}
The goal is to determine the spar mass by setting a target first natural frequency $\omega_n$ for the cantilever beam with distributed and lumped non-structural masses. Equating the maximum total kinetic energy and maximum potential energy for the first bending mode, we obtain
\begin{equation}
\frac{1}{2} \frac{P^2}{E^2 \pi t} \spc LHS \quad = \quad \omega_n^2 \left[\frac{P^2}{E^2 t} (RHS) \spc + \spc \frac{P^2}{E^2 \pi^2 t^2} \beta (RHS) \spc + \spc \frac{P^2}{E^2 \pi^2 t^2} \Delta RHS \right]
\end{equation}
Solve for spar thickness as
\begin{equation*}
t \quad = \quad \left(\frac{E \pi}{2 \omega_n^2} LHS \spc - \spc \rho \pi RHS \right)^{-1} \left[\beta (RHS) \spc + \spc \Delta RHS \right]
\end{equation*}
The total spar mass is therefore
\begin{equation}
M_{\rm spar} \quad = \quad \pi \rho \spc t (1 + \lambda) r_1
\end{equation}

For validation of the Rayleigh-Ritz solution, several cases were evaluated with and without lumped non-structural masses.

\subsection{Validation}
\subsubsection{Case 1: cantilever beam}
Consider a cantilever beam with a hollow circular cross-section of constant wall thickness, and the cross-section radius is linearly tapered from root to tip, with a taper ratio of $\lambda$ varying from 0.5 to 1.2. The beam length is 2.9 m, mean tube radius is 0.1411 m. The first bending natural frequency predictions from the Rayleigh-Ritz approximation are compared against a beam Finite Element Method analysis. These two frequencies are plotted in Fig.~\ref{fig:beam_freq1}. The Rayleigh-Ritz approximation exhibits about 2\% over-prediction, but otherwise captures the trends accurately. In this example, no non-structural masses were considered.

\begin{figure}
     \centering
	\subfigure[Natural frequency vs. taper ratio]{\includegraphics[width=0.45\textwidth]{images/beam_freq1b.png}}
	\subfigure[Rayleigh-Ritz vs. FEM]{\includegraphics[width=0.45\textwidth]{images/beam_freq1.png}}
     \caption{Natural frequency: Rayleigh-Ritz vs. FEM predicted natural frequencies for a tapered cantilever beam}
     \label{fig:beam_freq1}
\end{figure}

\begin{figure}
     \centering
	\subfigure[Natural frequency vs. taper ratio]{\includegraphics[width=0.45\textwidth]{images/beam_freq2b.png}}
	\subfigure[Rayleigh-Ritz vs. FEM]{\includegraphics[width=0.45\textwidth]{images/beam_freq2.png}}
     \caption{Natural frequency: Rayleigh-Ritz vs. FEM for a tapered cantilever beam with distributed non-structural masses}
     \label{fig:beam_freq2}
\end{figure}

\subsubsection{Case 2: distributed non-structural mass}
This case is representative of an aircraft half-wing with skin, and no lumped masses. The non-structural masses are represented as a mass per unit area, i.e., $\large\frac{M}{A}$ kg/$m^2$. Three different values of the non-structural mass per unit wing area were evaluated: 0, 2.0 and 4.9 kg/m$^2$. The resulting natural frequencies were calculated from both Rayleigh-Ritz and FEM, and are plotted in Fig.~\ref{fig:beam_freq2}. Excellent agreement is obtained.

\subsubsection{Case 3: half-wing with lumped masses}
This case is representative of an eVTOL wing. The dimensions of the beam are the same as in the previous examples, and the distributed skin mass per unit area is 4.9 kg/m$^2$. Two lumped masses (21.3 kg, 27 kg) are placed along the beam at 0.5$L$ and $L$, respectively. The wing layout and natural frequencies are shown in Fig.~\ref{fig:beam_freq3}.

\begin{figure}
     \centering
	\subfigure[Layout of eVTOL wing]{\includegraphics[width=0.3\textwidth]{images/wing_layout.png}}
	\subfigure[Rayleigh-Ritz vs. FEM]{\includegraphics[width=0.45\textwidth]{images/beam_freq3b.png}}
     \caption{Natural frequency: Rayleigh-Ritz vs. FEM for an example eVTOL wing}
     \label{fig:beam_freq3}
\end{figure}


% Copyright 2021 Google LLC
%
% Use of this source code is governed by an MIT-style
% license that can be found in the LICENSE file or at
% https://opensource.org/licenses/MIT.

%!BIB program = biber
%!TeX program = lualatex
%!TeX spellcheck = en-US

\documentclass[hctr2.tex]{subfiles}
\begin{document}
\section{Specification}\label{specification}
\subfile{algorithm.tex}
\subsection{Notation}
\begin{itemize}
    \item $\abs{X}$: length of $X \in \bin^{*}$ in bits
    \item $\lambda$: the empty string $\abs{\lambda} = 0$
    \item $X[a;l]$: the substring of $X$ of length $l$ starting at the 0-based index $a$
    \item $\Concat$: bitstring concatenation
    \item \(\xor\): bitwise XOR
    \item \(n\): block size in bits
    \item $\fromint_l: \{0 \ldots 2^l-1\} \rightarrow \bin^l$:
    little-endian conversion of integers to binary;
    \(\fromint(x)\) means \(\fromint_n(x)\)
    \item $\pad(X) = X \Concat 0^v$
    where $v$ is the least integer $\geq 0$ such that $n$ divides $\abs{X} + v$
    \item \(x, x^2, \ldots\): elements of the finite field \(\GF(2^n)\)
    \item \(E: \mathcal{K} \times \bin^n \rightarrow \bin^n\):
    \(n\)-bit block cipher with keyspace \(\mathcal{K}\);
    our concrete proposal uses AES\cite{aes},
    so \(n=128\) and \(\mathcal{K}\) is
    \(\bin^{128}\), \(\bin^{192}\), or \(\bin^{256}\)
    \item \(\mathcal{T}\): the set of permissible tweaks
    \(\mathcal{T} = \bigcup_{i \in \{0\ldots2^{n-1}-2\}}\bin^i\)
    \item \(\mathcal{M}\): the set of permissible messages
    \(\mathcal{M} = \bigcup_{i \in \{n\ldots n + 2^{n-1}-2\}}\bin^i\)
\end{itemize}
We map bytes to bitstrings with \(\fromint_8\). Subscripts may denote partial application; if we define $f: A \times B \rightarrow C$ and
$a \in A$ then $f_a: B \rightarrow C$, and if $f_a^{-1}$ exists then $f_a^{-1}(f_a(b)) = b$.

\subsection{Polynomial hash function}\label{hashspec}
We interpret \(n\)-bit blocks as little-endian field elements of \(\GF(2^n)\),
so \(001 \Concat 0^{n-3}\) is interpreted as the element \(x^2\).
Per \cite{aes_gcm_siv,aes_gcm_siv_rfc} we define
\begin{align*}
    \POLYVAL(\hgen, \lambda) & = 0^n\\
    \POLYVAL(\hgen, A \Concat B) & = (\POLYVAL(\hgen, A) \xor B) \otimes \hgen \otimes x^{-n}
\end{align*}
where \(\abs{\hgen} = \abs{B} = n\) and
\(\otimes\) is multiplication in the finite field.
In our concrete proposal, \(n=128\), we reduce by
\(x^{128} + x^{127} + x^{126} + x^{121} + 1\),
and the value of the field element \(x^{-n}\)
is equal to \(x^{127} + x^{124} + x^{121} + x^{114} + 1\).

For hash key \(\hgen \in \bin^n\), tweak \(T\) and message \(M\), we define:
\begin{align*}
    & H_{\hgen}(T, M) \\
    \defeq &
    \begin{cases}
        \POLYVAL(\hgen, \fromint(2\abs{T} + 2) \Concat \pad(T) \Concat M) &
        \text{if } n \text{ divides } \abs{M} \\
        \POLYVAL(\hgen, \fromint(2\abs{T} + 3) \Concat \pad(T) \Concat \pad(M \Concat 1)) &
        \text{otherwise}
    \end{cases}
\end{align*}

\subsection{XCTR mode}
HCTR and HCTR2 use an unusual mode of stream encryption,
which we name \emph{XCTR mode}:
\begin{displaymath}
    \XCTR_k(S) = E_k(S \xor \fromint(1)) \Concat E_k(S \xor \fromint(2)) \Concat E_k(S \xor \fromint(3)) \Concat \cdots
\end{displaymath}
Generating the first \(m\) bits \(\XCTR_k(S)[0; m]\) takes \(\ceil{m/n}\) block cipher calls.

\subsection{HCTR2}
HCTR2 encryption, defined in \autoref{pseudocode},
takes a tweak and a plaintext,
and returns a ciphertext of the same length as the plaintext.
HCTR2 decryption (\autoref{decryptionpseudocode})
recovers the plaintext given the same tweak and the ciphertext, ie
for \(k \in \mathcal{K}\), \(T \in \mathcal{T}\)
and \(P \in \mathcal{M}\),
\(\textsc{Decrypt}(k, T, \textsc{Encrypt}(k, T, P)) = P\).
\end{document}


\chapter{Maximum Likelihood Estimation for Multiparameter Models}\label{S:MultiParamEst}

\section{Introduction}\label{S:MultiParamEstIntro}
When two or more parameters index a statistical experiment we want to estimate the vector-valued parameter $\theta^* := (\theta^*_1,\ldots,\theta^*_k)$.
Here we will find the maximum likelihood estimates of vector-valued parameters.

The maximum likelihood estimator (MLE) of a possibly unknown but fixed parameter  $\theta^* := (\theta^*_1,\ldots,\theta^*_k)$  in a multi-parametric experiment, i.e.~$\theta^* \in \BB{\Theta} \subset \Rz^k$ with $1 < k < \infty$ is defined analogously to \hyperref[D:LklFn]{Definition~\ref*{D:MLE}} with the exception that we allow the parameter to be a vector.  We take an excursion in multi-dimensional optimisation before finding the MLE of a parametric experiment involving two parameters.

 \section{Practical Excursion in Multi-dimensional Optimisation}\label{S:PracticalMultiDimOptimization}
The basic idea involves multi-dimensional iterations that attempt to converge on a local maximum close to the starting vector $\theta^{(0)} \in \BB{\Theta}$ (our initial guess).  We can employ {\sc Matlab}'s built-in function {\tt fminsearch} to find the MLE of vector-valued parameters such as in the $\lognormal$ model with two parameters, i.e.~$\theta=(\lambda,\zeta) \in \BB{\Theta} \subset \Rz^2$.  The  function {\tt fminsearch} is similar to {\tt fminbnd} except that it handles a given function of many variables, and the user specifies a starting vector $\theta^{(0)}$ rather than a starting interval.  Thus, {\tt fminsearch} tries to return a vector $\theta^{(*)}$ that is a local minimiser of, $-\log(L(x_1,x_2,\ldots,x_n; \theta)$, the negative log-likelihood function of the vector-valued parameter $\theta$, near this starting vector $\theta^{(0)}$.
We illustrate the use of {\tt fminsearch} on a more challenging target called the Levy density:

{\scriptsize
\begin{equation}\label{E:LevyDensity}
f(x,y)   =  \exp \left(-\frac{1}{50} \left( \left( \sum_{i=1}^5 {i \cos{((i-1)x+i)} } \right) \left( \sum_{j=1}^5 {j \cos{((j+1)y+j)} } \right)+ (x + 1.42513 )^2 + (y + 0.80032)^2\right) \right)
\end{equation}
}

\begin{figure}[htpb]
\caption{Plot of Levy density as a function of the parameter $(x,y) \in [-10,10]^2$ scripted in \hyperref[Mf:LevyDensityPlot]{Labwork \ref*{Mf:LevyDensityPlot}}.\label {F:LevyDensityPlot}}
\begin{center}
\makebox{\includegraphics[width=6.750in]{figures/LevyDensityT50}}
\end{center}
\end{figure}

{\tt fminsearch} uses the simplex search method [Nelder, J.A., and Mead, R.~1965, Computer Journal, vol.~7, p.~308-313].  For an animation of the method and more details, please visit \href{http://en.wikipedia.org/wiki/Nelder-Mead_method}{\url{http://en.wikipedia.org/wiki/Nelder-Mead_method}}.
%For a more recent treatment see Lagarias, J.C., J. A. Reeds, M. H. Wright, and P. E. Wright, {\em Convergence Properties of the Nelder-Mead Simplex Method in Low Dimensions}, SIAM Journal of Optimisation, Vol. 9 Number 1, pp. 112-147, 1998.
An advantage of the method is that it does not use numerical (finite differencing) or analytical (closed-form expressions) gradients but relies on a direct search method.  Briefly, the simplex algorithm tries to ``tumble and shrink'' a simplex towards the local valley of the function to be minimised.  If $k$ is the dimension of the parameter space or domain of the function to be optimised, a $k$-dimensional simplex is specified by its $k+1$ distinct vertices each of dimension $k$.  Thus, a simplex is a triangle in a two-dimensional space and a pyramid in a three-dimensional space. At each iteration of the algorithm:
\begin{enumerate}
\item A new point inside or nearby the current simplex is proposed.
\item The function's value at the newly proposed point is compared with its values at the vertices of the simplex.
\item One of the vertices is typically replaced by the proposed point, giving rise to a new simplex.
\item The first three steps are repeated until the diameter of the simplex is less than the specified tolerance.
\end{enumerate}

A major limitation of {\tt fminsearch}, as demonstrated with the Levy target (encoded in \hyperref[Mf:NegLevyDensity]{Labwork~\ref*{Mf:NegLevyDensity}}) is that it  can only give local solutions.  The {\bf global maximiser} of the Levy function $f(x,y)$ is $(-1.3069, -1.4249)$ and the {\bf global maximum} is $f (-1.3069, -1.4249)=33.8775$ For instance, if we start the search close to, say $(x^{(0)},y^{(0)})=(-1.3,-1.4)$, as shown below, then the simplex algorithm converges as desired to the solution $(-1.3068, -1.4249)$.
\begin{VrbM}
>> [params, fvalue, exitflag, output] = fminsearch('NegLevyDensity',[-1.3 -1.4],options)
params =   -1.3068   -1.4249
fvalue =  -33.8775
exitflag =     1
output =
    iterations: 24
     funcCount: 46
     algorithm: 'Nelder-Mead simplex direct search'
       message: [1x194 char]
\end{VrbM}
However, if we start the search further away, say $(x^{(0)},y^{(0)})=(1.3,1.4)$, as shown below, then the algorithm converges to the {\bf local maximiser} $(1.1627,1.3093)$ with a {\bf local maximum} value of $f(1.1627,1.3093)=0.9632$, which is clearly smaller than the global maximum of $33.8775$.
\begin{VrbM}
>> [params, fvalue, exitflag, output] = fminsearch('NegLevyDensity',[1.3 1.4],options)
params = 1.1627    1.3093
fvalue = -0.9632
exitflag = 1
output =
   iterations: 29
     funcCount: 57
     algorithm: 'Nelder-Mead simplex direct search'
       message: [1x194 char]
\end{VrbM}
Therefore, we have to be extremely careful when using point-valued, iterative,  local optimisation algorithms, implemented in floating-point arithmetic to find the global maximum.  Other examples of such algorithms include:
\begin{itemize}
\item {\bf Conjugate Gradient Method}:\\ \href{http://en.wikipedia.org/wiki/Conjugate_gradient_method}{\url{http://en.wikipedia.org/wiki/Conjugate_gradient_method}}
\item {\bf Broyden-Fletcher-Goldfarb-Shanno (BFGS) method}:\\ \href{http://en.wikipedia.org/wiki/BFGS_method}{\url{http://en.wikipedia.org/wiki/BFGS_method}}
\item {\bf Simulated Annealing}:\\ \href{http://en.wikipedia.org/wiki/Simulated_annealing}{\url{http://en.wikipedia.org/wiki/Simulated_annealing}}
\end{itemize}
In general, we have no guarantee that the output of such local optimisation routines will indeed be the global optimum.  In practice, you can start the search at several distinct starting points and choose the best local maximum from the lot.

\begin{figure}[hbpt]
\caption{Plot of the ``well-behaved'' (uni-modal and non-spiky) $\log(L((x_1,x_2,\ldots,x_{100});\lambda,\zeta))$, based on $100$ samples $(x_1,x_2,\ldots,x_{100})$ drawn from the $\lognormal(\lambda^*=10.36,\zeta^*=0.26)$ as per \hyperref[Mf:LogNormalLogLklPlot]{Labwork \ref*{Mf:LogNormalLogLklPlot}}.\label {F:LogNormalLogLklPlot}}
\begin{center}
\makebox{\includegraphics[width=4.0in]{figures/LogNormalLogLklPlot}}
\end{center}
\end{figure}

When the target function is ``well-behaved,'' i.e.~uni-modal or single-peaked and not too spiky, the optimisation routine can be expected to perform well.  Log-likelihood functions are often well-behaved.  Let us generate $100$ samples from an RV $C\sim \lognormal(\lambda^*=10.36, \zeta^*=0.26)$ by exponentiating the samples from the $\normal(10.36,0.26^2)$ RV, and then compute the corresponding MMEs and MLEs for parameters $(\lambda,\zeta)$ using the formulae in \hyperref[T:MMEMLE]{Table~\ref*{T:MMEMLE}}.
\begin{VrbM}
>> rand('twister',001); % set the fundamental sampler
>> % draw 100 samples from the Lognormal(10.36,0.26) RV
>> Cs = exp(arrayfun(@(u)(Sample1NormalByNewRap(u,10.36,0.26^2)),rand(1,100)));
>> MLElambdahat = mean(log(Cs)) % maximum likelihood estimate of lambda
MLElambdahat =   10.3397
>> MLEzetahat = sqrt(mean( (log(Cs)-MLElambdahat) .^ 2)) % max. lkl. estimate of zeta
MLEzetahat =    0.2744
>> MMEzetaahat = sqrt(log(var(Cs)/(mean(Cs)^2) + 1)) % moment estimate of zeta
MMEzetaahat =    0.2624
>> MMElambdahat = log(mean(Cs))-(0.5*MMEzetaahat^2) % moment estimate of lambda
MMElambdahat =   10.3417
\end{VrbM}
Let us try to apply the simplex algorithm to find the MLE numerically.  We first encode the negative log-likelihood function of the parameters $(\lambda,\zeta) \in (0,\infty)^2$ for the given data $x$, as follows:
{\VrbMf[label=NegLogNormalLogLkl.m]{scripts/NegLogNormalLogLkl.m}}
Here is how we can call {\tt fminsearch} and find the MLE after the re-transformation.
\begin{VrbM}
>> [params, fvalue, exitflag, output] = ...
fminsearch(@(params)(NegLogNormalLogLkl(Cs,params)),[log(5), log(1)])
params =    2.3360   -1.2931
fvalue =  -1.0214e+03
exitflag =     1
output =
    iterations: 74
     funcCount: 131
     algorithm: 'Nelder-Mead simplex direct search'
       message: [1x194 char]
>> % But we want exp(params) since we defined lambda and zeta as exp(params)
exp(params)
ans =   10.3397    0.2744
\end{VrbM}
Note that the MLEs $(\widehat{\lambda}_{100},\widehat{\zeta}_{100}) = (10.3397,0.2744)$ from $74$ iterations or ``tumbles'' of the `Nelder-Mead simplex (triangle)' and the MLEs agree well with the direct evaluations {\tt MLElambdahat} and {\tt MLEzetahat} based on the formulae in \hyperref[T:MMEMLE]{Table~\ref*{T:MMEMLE}}.

%%\newpage
%\begin{labwork}
%Recall \hyperref[LW:lognormal]{labwork~\ref*{LW:lognormal}} where you simulated $1000$ samples directly from the RV $C$ (in part 3.).  Pretend that you do not know the true parameters used in this particular simulation from RV $C$ and do the following:
%\begin{enumerate}
%\item Store the first 10, the first 100 and all 1000 samples in data arrays named {\tt x10}, {\tt x100} and {\tt x1000} [Please don't do this manually!].
%\item Report the MLE of the two parameters for each of the three sub-arrays of data, namely {\tt x10}, {\tt x100} and {\tt x1000}, i.e.~report the six point estimates: $\widehat{\lambda}_{10},\widehat{\zeta}_{10},\widehat{\lambda}_{100},\widehat{\zeta}_{100},\widehat{\lambda}_{1000},\widehat{\zeta}_{1000}$.
%\item Report the MME of the two parameters for each of the three sub-arrays of data, namely {\tt x10}, {\tt x100} and {\tt x1000}, i.e.~report the six point estimates: $\widehat{\lambda}_{10},\widehat{\zeta}_{10},\widehat{\lambda}_{100},\widehat{\zeta}_{100},\widehat{\lambda}_{1000},\widehat{\zeta}_{1000}$.
%\item Discuss in a short paragraph what you can deduce from the two sets of point estimates.  Explain how the maximum likelihood (ML) and method of moments (MM) estimates of the parameters are related to the true parameters used in the simulation as the sample size increases in powers of $10$.
%\item {\bf There is no credit for this part}: Try to use {\tt fminsearch} to numerically find the MLE for your data and make a 2D-plot of the log-likelihood function being maximised.
%\end{enumerate}
%\end{labwork}

\subsubsection*{Summarizing Table of Point Estimators}
Using the sample mean $\overline{X}_n$ and sample standard deviation $S_n$ defined in \eqref{E:SampleMeanRV} and \eqref{E:SampleStdDevRV}, respectively, we summarise the two point estimators of the parameters of some common distributions below.  For some cases, the MLE is the same as the MME (method of moments) and can be solved analytically.
\begin{center}
\begin{table}[htbp]
\caption{Summary of the Method of Moment Estimator (MME) and the Maximum Likelihood Estimator (MLE) for some IID Experiments. \label{T:MMEMLE}}
\begin{tabular}{l | r | r}
\hline
Statistical Experiment & MLE & MME \\ \hline
$X_1,X_2,\ldots,X_n \overset{IID}{\sim} \bernoulli(\theta)$ & $\widehat{\theta}=\overline{X}_n$ & same as MLE \\ \hline
$X_1,X_2,\ldots,X_n \overset{IID}{\sim} \exponential(\lambda)$ & $\widehat{\lambda}={1}/{\overline{X}_n} $ & same as MLE \\ \hline
$X_1,X_2,\ldots,X_n \overset{IID}{\sim} \normal(\mu,\sigma^2)$ & $\widehat{\mu}=\overline{X}_n, \widehat{\sigma} = \sqrt{\frac{n-1}{n}S^2_n} $ & $\widehat{\mu}=\overline{X}_n, \widehat{\sigma} = S_n $ \\ \hline
$X_1,X_2,\ldots,X_n \overset{IID}{\sim} \lognormal(\lambda,\zeta)$ & $\widehat{\lambda}=\frac{1}{n}{\sum_{i=1}^n \log(X_i)} $ & $\widehat{\lambda} = \log(\overline{X}_n) - \frac{1}{2} {\widehat{\zeta}} \ ^2$ \\ %\\
 & $\widehat{\zeta} = \sqrt{\frac{1}{n} \sum_{i=1}^n{(\log(X_i)-\widehat{\lambda})^2}} $ & $\widehat{\zeta} = \sqrt{\log \left({S_n^2}/{\overline{X}_n^2} +1 \right)}$ \\
\hline
\end{tabular}
\end{table}
\end{center}

\section{Confidence Sets for Multiparameter Models}\label{S:ConfSetsMultiParamModels}
We will extend the Fisher Information and Delta method to models with more than one parameter:
\[
X_1,X_2,\ldots,X_n \overset{IID}{\sim} f(x;\theta^*), \qquad \theta^* := (\theta_1^*, \theta_2^*, \ldots, \theta_k^*) \in \BB{\Theta} \subset \Rz^k  \ .
\]
Let, the ML estimator of the fixed and possibly unknown vector-valued parameter $\theta^*$ be:
\[
\widehat{\Theta}_n := \left( \widehat{\Theta}_{1,n}, \widehat{\Theta}_{2,n}, \ldots, \widehat{\Theta}_{k,n} \right), \qquad  \widehat{\Theta}_n := \widehat{\Theta}_n(X_1,X_2,\ldots,X_n) : \Xz_n \to \BB{\Theta}
\]
and the ML estimate based on $n$ observations $x_1,x_2,\ldots,x_n$ be:
\[
\widehat{\theta}_n := \left( \widehat{\theta}_{1,n}, \widehat{\theta}_{2,n}, \ldots, \widehat{\theta}_{k,n} \right), \qquad  \widehat{\theta}_n := \widehat{\theta}_n(x_1,x_2,\ldots,x_n) \in \BB{\Theta} \ .
\]
Let the log-likelihood function and its Hessian matrix $H = (H_{i,j})_{i,j=1,2,\ldots,k}$ of partial derivatives be:
\[
\ell_n(\theta) := \ell_n(\theta_1,\theta_2,\ldots,\theta_k) := \sum_{i=1}^n \log(f(x_i; (\theta_1,\theta_2,\ldots,\theta_k))), \qquad
H_{i,j} := \frac{\partial}{\partial \theta_i} \frac{\partial}{\partial \theta_j} \ell_n(\theta_1,\theta_2,\ldots,\theta_k) \ ,
\]
respectively, provided the log-likelihood function is sufficiently smooth.
\begin{definition}[Fisher Information Matrix]  The Fisher Information matrix is:
\begin{equation}\label{E:FisherInfoMat}
I_n(\theta) := I_n(\theta_1,\theta_2,\ldots,\theta_k) =
-
\begin{bmatrix}
\E_{\theta}(H_{1,1}) & \E_{\theta}(H_{1,2}) & \cdots & \E_{\theta}(H_{1,k}) \\
\E_{\theta}(H_{2,1}) & \E_{\theta}(H_{2,2}) & \cdots & \E_{\theta}(H_{2,k}) \\
\vdots & \vdots & \ddots & \vdots \\
\E_{\theta}(H_{k,1}) & \E_{\theta}(H_{k,2}) & \cdots & \E_{\theta}(H_{k,k})
\end{bmatrix}
\end{equation}
and its matrix inverse is denoted by $I_n^{-1}(\theta)$.
\end{definition}
\begin{prop}[Asymptotic Normality of MLE in Multiparameter Models]
Let
$$X_1,X_2,\ldots,X_n \overset{IID}{\sim} f(x_1;\theta_1^*,\theta_2^*,\ldots,\theta_k^*), \qquad \theta^*=(\theta_1^*,\theta_2^*,\ldots,\theta_k^*)\in \BB{\Theta}\subset \Rz^k \ ,$$
for some fixed and possibly unknown $\theta^* \in \BB{\Theta}\subset \Rz^k$.  Then, under appropriate regularity conditions:
\[
{\widehat{\Theta}_n} := \left( \widehat{\Theta}_{1,n}, \widehat{\Theta}_{2,n}, \ldots, \widehat{\Theta}_{k,n} \right) \rightsquigarrow \normal(\theta^*,I_n^{-1})
\]
In other words, the vector-valued estimator $\widehat{\Theta}_n$ converges in distribution to the multivariate $\normal$ distribution centred at the unknown parameter $\theta^*$ with the variance-covariance matrix given by inverse Fisher Information matrix $I_n^{-1}$.  Furthermore, let $I_n^{-1}(j,j)$ denote the $j^{\text{th}}$ diagonal entry of $I_n^{-1}$.  In this case:
% and let $$\widehat{\mathsf{se}}_n ( \widehat{\Theta}_{j,n}) = \sqrt{ I_n^{-1}(j,j)}$$  be the standard error of the estimator of $\theta_j^*$
\[
\frac{\widehat{\Theta}_{j,n} - \theta^*_j}{\sqrt{I_n^{-1}(j,j)}} \rightsquigarrow \normal(0,1)
\]
and the approximate covariance of $\widehat{\Theta}_{i,n}$ and $\widehat{\Theta}_{j,n}$ is:
\[
{\sf Cov}({\Theta}_{i,n},{\Theta}_{j,n}) \approxeq I_n^{-1}(i,j) \ .
\]
\end{prop}
Now, let us look at a way of obtaining ML estimates and confidence sets for functions of $\theta$.  Suppose the real-valued function $g(\theta)=\psi:\BB{\Theta} \to \BB{\Psi}$ maps points in the $k$-dimensional parameter space $\BB{\Theta} \subset \Rz^k$ to points in $\BB{\Psi} \subset \Rz$.  Let the gradient of $g$ be
\[
\bigtriangledown g(\theta)
:=
\bigtriangledown g (\theta_1,\theta_2,\ldots,\theta_k)
=
\begin{pmatrix}
\frac{\partial} {\partial \theta_1}{g(\theta_1,\theta_2,\ldots,\theta_k)} \\
\frac{\partial} {\partial \theta_2}{g(\theta_1,\theta_2,\ldots,\theta_k)} \\
\vdots \\
\frac{\partial} {\partial \theta_k}{g(\theta_1,\theta_2,\ldots,\theta_k)} \\
\end{pmatrix} \ .
\]
\begin{prop}[Multiparameter Delta Method]
Suppose:
\begin{enumerate}
\item $X_1,X_2,\ldots,X_n \overset{IID}{\sim} f(x_1;\theta_1^*,\theta_2^*,\ldots,\theta_k^*), \qquad \theta^*=(\theta_1^*,\theta_2^*,\ldots,\theta_k^*)\in \BB{\Theta}\subset \Rz^k$,
\item Let  $\widehat{\Theta}_n$ be a ML estimator of $\theta^* \in \BB{\Theta}$ and let $\widehat{\theta}_n$ be its ML estimate, and
\item Let $g(\theta)=\psi:\BB{\Theta} \to \BB{\Psi} \subset \Rz$ be a smooth function such that $\bigtriangledown g(\widehat{\theta}_n) \neq 0$.
\end{enumerate}
Then:
\begin{enumerate}
\item $\widehat{\Psi}_n = g(\widehat{\Theta}_n)$ is the ML estimator and  $\widehat{\psi}_n = g(\widehat{\theta}_n)$ is the ML estimate of of $\psi^*=g(\theta^*) \in \BB{\Psi}$,
\item The standard error of the ML estimator of $\psi^*$ is:
$$\widehat{\sf{se}}_n(\widehat{\Psi}_n) = \sqrt{ \left( \bigtriangledown g(\widehat{\theta}_n) \right)^T I_n^{-1}(\widehat{\theta}_n) \left( \bigtriangledown g(\widehat{\theta}_n) \right)}, $$
\item The ML estimator of $\psi^*$ is asymptotically normal, i.e.:
$$\frac{\widehat{\Psi}_n - \psi^*}{\widehat{\sf{se}}_n(\widehat{\Psi}_n)} \rightsquigarrow \normal(0,1) \  ,$$
\item And a $1-\alpha$ confidence interval for $\psi^*$ is:
\[
\widehat{\psi}_n \pm z_{\alpha/2} {\widehat{\sf{se}}_n(\widehat{\Psi}_n)}
\]
\end{enumerate}
\end{prop}
Let us put the theory to practice in the problem of estimating the coefficient of variation from samples of size $n$ from an RV.
\begin{example}[Estimating the Coefficient of Variation of a $\normal(\mu^*,{\sigma^*}^2)$ RV]\label{EX:CoeffOfVarForBivNormal}
Let
$$
\psi^* = g(\mu^*,\sigma^*)=\sigma^*/\mu^*, \qquad X_1,X_2,\ldots,X_n \overset{IID}{\sim} \normal(\mu^*,{\sigma^*}^2) \ .
$$
We do not know the fixed parameters $(\mu^*,\sigma^*)$ and are interested in estimating the coefficient of variation $\psi^*$ based on $n$ IID samples $x_1,x_2,\ldots,x_n$.  We have already seen that the ML estimates of $\mu^*$ and $\sigma^*$ are:
\[
\widehat{\mu}_n =  \overline{x}_n := \frac{1}{n}\sum_{i=1}^n x_i,  \qquad
\widehat{\sigma}_n = s_n := \sqrt{\frac{1}{n} \sum_{i=1}^n (x_i-\widehat{\mu}_n)^2} \ .
\]
Thus, the ML estimate of $\psi^*=\sigma^*/ \mu^*$ is:
\[
\widehat{\psi}_n =  \frac {\widehat{\sigma}_n}{\widehat{\mu}_n} = \frac {s_n}{\overline{x}_n}
\]
We can now derive the standard error of the ML estimator $\widehat{\Psi}_n$ by first computing $I_n(\mu,\sigma)$, $I_n^{-1}(\mu,\sigma)$, and $\bigtriangledown g(\mu,\sigma)$.  A careful computation shows that:
\[
I_n(\mu,\sigma) =
\begin{bmatrix}
\frac{n}{\sigma^2} & 0\\
0 & \frac{2n}{\sigma^2}
\end{bmatrix},
\qquad \qquad
I_n^{-1}(\mu,\sigma) =
\frac{1}{n}
\begin{bmatrix}
{\sigma^2} & 0\\
0 & \frac{\sigma^2}{2}
\end{bmatrix},
\qquad \qquad
\bigtriangledown g(\mu,\sigma) =
\begin{pmatrix}
-\frac{\sigma}{\mu^2} \\
\frac{1}{\mu}
\end{pmatrix} \ .
\]
Therefore, the standard error of interest is:
\[
\widehat{\sf{se}}_n(\widehat{\Psi}_n) = \sqrt{ \left( \bigtriangledown g(\widehat{\theta}_n) \right)^T I_n^{-1}(\widehat{\theta}_n) \left( \bigtriangledown g(\widehat{\theta}_n) \right)} =
\frac{1}{\sqrt{n}} \sqrt{\frac{1}{\widehat{\mu}_n^4} + \frac{\widehat{\sigma}_n^2}{2 \widehat{\mu}_n^2}}
\]
and the $95\%$ confidence interval for the unknown coefficient of variation $\psi^*$ is:
\[
\widehat{\psi}_n \pm z_{\alpha/2} {\widehat{\sf{se}}_n(\widehat{\Psi}_n)}  =
\frac{s_n}{\overline{x}_n} \pm z_{\alpha/2} \left( \frac{1}{\sqrt{n}} \sqrt{\frac{1}{\widehat{\mu}_n^4} + \frac{\widehat{\sigma}_n^2}{2 \widehat{\mu}_n^2}} \right)
\]
\end{example}

Let us get our hands dirty in the machine with Labwork~\ref{LW:CoeffOfVarForBivNormal} next.
\begin{labwork} [Computing the coefficient of variation of a $\normal(\mu^*,{\sigma^*}^2)$ RV]\label{LW:CoeffOfVarForBivNormal}
Let us apply these results to $n=100$ simulated samples from $\normal(100,10^2)$ as follows.
{\VrbMf[label=CoeffOfVarNormal.m]{scripts/CoeffOfVarNormal.m}}
\begin{VrbM}
>> CoeffOfVarNormal
Muhat =  100.3117
Sigmahat =   10.9800
Psihat =    0.1095
Sehat =    0.0077
ConfInt95 =    0.0943    0.1246
\end{VrbM}
\end{labwork}


\section{M/M/m Queues}
\label{sec:M-M-m-Queues}

A $M/M/m$ is a queue where
(i) the arrival process is Poissonian with rate $\lambda$,
(ii) the service process is Exponential with rate $\mu$,
(iii) there are $m$ servers,
(iv) the buffer has infinite capacity,
(v) the scheduling policy is FCFS.

%\begin{figure}[tp]
%\label{fig:M-M-m-Queue}
%	\centering
%	\includegraphics{fig/M-M-m-Queue}
%	\caption{An M/M/m queue and its corresponding CTMC.}
%\end{figure}

The key question in these types of systems is determining the \textit{Queue Probability} $P_{Q}$, that is the probability that an arriving job is enqueued.

\begin{definition}[Utilization]
\label{def:M-M-m-Utilization}
	For any $M/M/m$, the system utilization is

	\begin{equation}
	\label{eqn:M-M-m-Utilization}
	\varrho = \frac{\lambda}{m \mu}
	\end{equation}
\end{definition}

\begin{definition}[Resource Requirement]
\label{def:M-M-m-Resource-Requirement}
	For any $M/M/m$, the resource requirement is

	\begin{equation}
	\label{eqn:M-M-m-Resource-Requirement}
	R = \frac{\lambda}{\mu}
	\end{equation}
\end{definition}

The resource requirement $R$ is
(i) the minimum number of servers needed to keep the system stable,
(ii) the expected number of busy servers, and
(iii) the expected number of jobs in service.

\begin{theorem}[State Probability]
\label{thm:M-M-m-Probability-State}
	For any $M/M/m$, the state probability is

	\begin{equation}
	\label{eqn:M-M-m-Probability-State}
	\pi_{i} = \frac{m^{q} \varrho^{i}}{q!} \cdot \pi_{0}
	\end{equation}

	with

	\begin{equation}
	\label{eqn:M-M-m-Probability-State-Zero}
	\pi_{0} = \Big[ \sum_{i=0}^{m-1} \frac{(m \varrho)^{i}}{i!} + \frac{(m \varrho)^{m}}{m! (1- \varrho)} \Big]^{-1}
	\end{equation}

	where

	$q=\min\{i,m\}$ is the number of busy servers.

	\begin{proof}
		For a formal demonstration, see \cite{harchol2013performance} on page 258-259.
	\end{proof}
\end{theorem}

\begin{theorem}[Queue Probability]
	\label{thm:M-M-m-Probability-Queue}
	For any $M/M/m$, the queue probability is

	\begin{equation}
	\label{eqn:M-M-m-Probability-Queue}
	P_{Q} = \frac{(m \varrho)^{m}}{m!(1-\varrho)} \cdot \pi_{0}
	\end{equation}

	\begin{proof}
		For a formal demonstration, see \cite{harchol2013performance} on page 260.
	\end{proof}
\end{theorem}

\Cref{thm:M-M-m-Probability-Queue} is called \textit{Erlang-C Formula}.

\begin{theorem}[Mean Queue Jobs]
	\label{thm:M-M-m-Mean-Queue-Jobs}
	For any $M/M/m$, the expected number of jobs in queue is

	\begin{equation}
	\label{eqn:M-M-m-Mean-Queue-Jobs}
	\expected{N_{Q}} = \frac{\varrho}{(1-\varrho)} \cdot P_{Q}
	\end{equation}

	\begin{proof}
		For a formal demonstration, see \cite{harchol2013performance} on page 262.
	\end{proof}
\end{theorem}

The remaining metrics ($\expected{N},\expected{T},\expected{T_{Q}}$) could be determined by applying the Little's Law, the basic definitions
$\expected{N}=\expected{N_{Q}}+\expected{N_{S}}$,
$\expected{N_{S}}=R$,
$\expected{T}=\expected{T_{Q}}+\expected{T_{S}}$, and
$\expected{T_{S}}=\frac{1}{\mu}$.

In particular we obtain:

\begin{equation}
\label{eqn:M-M-m-Delay}
\expected{T_{Q}} = \frac{1}{\lambda} P_{Q} \frac{\varrho}{(1-\varrho)}
\end{equation}

\begin{equation}
\label{eqn:M-M-m-Reponse-Time}
\expected{T} = \frac{1}{\lambda} P_{Q} \frac{\varrho}{(1-\varrho)} + \frac{1}{\mu}
\end{equation}

\begin{equation}
\label{eqn:M-M-m-System-Jobs}
\expected{N} = P_{Q} \frac{\varrho}{(1-\varrho)} + m\varrho
\end{equation}

\subsection{Comparison between M/M/m and M/M/1}
\label{sec:Comparison-M-M-m-And-M-M-1}

Let us consider
(i) a $M/M/m$ with average arrival rate $\lambda$ and average service rate $\mu$, and
(ii) a $M/M/1$ with average arrival rate $\lambda$ and average service rate $m \mu$.

Under light load, in (i) jobs are served by few servers with rate $\mu$; whereas in (ii) jobs are served with rate $m \mu$.
Under high load, in (i) the state is always greater than $m$ and it behaves just like (ii).

Hence, if $\lambda^{M/M/1} = \lambda^{M/M/m}$ and $\mu^{M/M/1} = m \mu^{M/M/m}$, then $M/M/1$ is always at least $m$ time faster than $M/M/m$.

\subsection{Comparison between M/M/m/m and M/M/m}
\label{sec:Comparison-M-M-m-m-And-M-M-m}

It is interesting to compare the block probability of a $M/M/m/m$ with the queue probability of an equivalent $M/M/m$.
The following theorem compares $P_{block}^{M/M/m/m}$ with $P_{Q}^{M/M/m}$.

\begin{theorem}[Relation $P_{block}^{M/M/m/m}$ with $P_{Q}^{M/M/m}$]
\label{thm:Relation-Probability-Block-Probability-Queue}
	For any $M/M/m/m$ with block probability $P_{block}$ and any $M/M/m$ with queue probability $P_{Q}$, we have that

	\begin{equation}
	\label{eqn:Relation-Probability-Block-Probability-Queue}
	P_{block} = \frac{(1-\varrho)P_{Q}}{1-\varrho P_{Q}}
	\end{equation}

	\begin{proof}
		For a formal demonstration, see \cite{harchol2013performance} on page 261.
	\end{proof}
\end{theorem}


\section{Calculations and Results}
From the following equaitions,

$$ \tau_z = I\beta_z $$
$$ I_{ab} = I_A + I_B $$
$$ M_\mu = -I_1\beta_1 $$
$$ T = m(g-a)$$

We can derive
$$ m(g-R\beta_2)R-M_\mu=I_1\beta_2 $$
Then we find
$$ I_1=\frac{mR(g-R\beta_2)}{\beta_2-\beta_1} $$
Similarly, if a rigid body with an unknown moment of inertia is placed on the
turntable, we may find
$$ I_2=\frac{mR(g-R\beta_4)}{\beta_4-\beta_3} $$
Using the fact that the moment of inertia is an additive quantity, the moment of
inertia of the rigid object placed on the turntable, with respect to the axis of
rotation, may be found as the difference
$$ I_3 = I_2 - I_1 $$

\subsection{Measurement of Angular Acceleration}
Angular acceleration can be derive by investigating the measurement data (k,t),
the corresponding angular position is
$$ \theta = k\pi = \omega_0 t + \frac{1}{2}\beta t^2 $$

\subsection{In Lab Data}

\begin{table}[H]
  \centering
  \begin{tabularx}{\textwidth}{|p{6cm}|X|X|X|X|}
    \hline
    Object & 1 & 2 & 3 & 4 \\
    \hline
    Disk $[cm] \pm 0.002[cm]$& 24.098 & 24.094 & 24.094 & 24.094 \\
    Hoop 1 $[cm] \pm 0.002[cm]$& 20.982 & 20.918 & 20.966 & 20.976 \\
    Hoop 2 $[cm] \pm 0.002[cm]$& 23.998 & 24.000 & 24.000 & 24.002 \\
    Cylinder A $[cm] \pm 0.002[cm]$& 2.994 & 2.994 & 2.994 & 2.994 \\
    Cylinder B $[cm] \pm 0.002[cm]$& 2.994 & 2.994 & 2.994 & 2.994 \\
    Cone pulley $[cm] \pm 0.002[cm]$& 5.022 & 5.020 & 5.008 & 5.008 \\
    Hole 1 d $[cm] \pm 0.002[cm]$& 3.978 & 5.534 & & \\
    Hole 2 d $[cm] \pm 0.002[cm]$& 3.982 & 5.540 & & \\
    Hole 3 d $[cm] \pm 0.002[cm]$& 5.524 & 6.500 & & \\
    Hole 4 d $[cm] \pm 0.002[cm]$& 5.520 & 6.520 & & \\
    \hline
  \end{tabularx}
  \caption{Calliper measurements}
  \end{table}

\begin{table}[H]
  \centering
  \begin{tabularx}{\textwidth}{|X|X|}
    \hline
    Object & Mass\\
	\hline
    Disk $[g] \pm 0.1 [g] $ & 493.1\\
    Hoop $[g] \pm 0.1 [g] $ & 422.5\\
    Cylinder A $[g] \pm 0.1 [g] $ & 165.8\\
    Cylinder B $[g] \pm 0.1 [g] $ & 165.8\\
    Weight $[g] \pm 0.1 [g] $ & 59.1 \\
    \hline
  \end{tabularx}
  \caption{Mass measurements}
  \end{table}
% begin real data
\begin{table}[H]
  \centering
\begin{tabular}{|p{2cm}|p{1.5cm}|l|l|l|l|l|l|l|l|l|}
\hline
Situation & A or D & k & 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 \\
\hline
Empty & Dec & $t[s]$ & 0.2958 & 0.5924 & 0.8899 & 1.1881 & 1.4873 & 1.7871 & 2.0879 & 2.3895 \\
Empty & Acc & $t[s]$ & 0.9005 & 1.5186 & 2.0204 & 2.4557 & 2.8448 & 3.1996 & 3.5280 & 3.7038 \\
With disk & Dec &  $t[s]$ & 0.3178 & 0.6362 & 0.9554 & 1.2752 & 1.5958 & 1.9170 & 2.2390 & 2.5616 \\
With disk & Acc &  $t[s]$ & 0.8957 & 1.5808 & 2.1582 & 2.6666 & 3.1264 & 3.5491 & 3.9428 & 4.3322 \\
With hoop & Dec &  $t[s]$ & 0.2450 & 0.4903 & 0.7359 & 0.9818 & 1.2281 & 1.4746 & 1.7216 & 1.9688 \\
With hoop & Acc &  $t[s]$ & 1.0035 & 1.7614 & 2.3967 & 2.9547 & 3.4589 & 3.9216 & 4.3521 & 4.7560 \\
A 1 B 2 & Dec &  $t[s]$ & 0.4491 & 0.9003 & 1.3536 & 1.8089 & 2.2666 & 2.7263 & 3.1883 & 3.6524 \\
A 1 B 2 & Acc &  $t[s]$ & 1.3586 & 2.0905 & 2.6595 & 3.1424 & 3.5692 & 3.9562 & 4.3125 & 4.6448 \\
A 3 B 4 & Dec &  $t[s]$ & 0.4539 & 0.9099 & 1.3678 & 1.8279 & 2.2899 & 2.7541 & 3.2204 & 3.6888 \\
A 3 B 4 & Acc &  $t[s]$ & 1.3748 & 2.1298 & 2.7179 & 3.2171 & 3.6585 & 4.0587 & 4.4273 & 4.7711 \\
\hline
\end{tabular}
\caption{ Time measurements}
\end{table}

According to  United States Department of Commerce, the standard gravitational
acceleration is
$$ g =  9.80665 m/s^2 $$

% ==================================================================
% ==================================================================
For the Empty turntable,
\newcommand{\EFWwr}{17cm}
\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/etd}
\end{figure}

$$ \beta_1 = -0.0970 radius/s^2$$ (with 95\% confidence bounds)

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/eta}
\end{figure}

$$ \beta_2 = 2.6580 radius/s^2$$ (with 95\% confidence bounds)

Thus,
$$ I_1 = \frac{59.1 g \times 5.0145 cm \times (9.80665 m/s^2 - 5.0145 cm \times (-0.0970) radius/s^2 )}{2.6580 radius/s^2 -(-0.0970 radius/s^2) } = 0.0105 kg\times m^2 $$

% ==================================================================
% ==================================================================
For turntable with disk,
\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/wdd}
\end{figure}

$$ \beta_1 = -0.0668 radius/s^2$$ (with 95\% confidence bounds)

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/wda}
\end{figure}

$$ \beta_2 = 1.2970 radius/s^2$$ (with 95\% confidence bounds)

$$ I_2 = \frac{59.1 g \times 5.0145 cm \times (9.80665 m/s^2 - 5.0145 cm \times (-0.0668) radius/s^2 )}{1.2970 radius/s^2 -(-0.0668 radius/s^2) } = 0.0213 kg\times m^2 $$

% ==================================================================
% ==================================================================
For turntable with hoop,

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/whd}
\end{figure}

$$ \beta_1 = -0.0689 radius/s^2$$ (with 95\% confidence bounds)

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/wha}
\end{figure}

$$ \beta_2 = 1.1444 radius/s^2$$ (with 95\% confidence bounds)

$$ I_3 = \frac{59.1 g \times 5.0145 cm \times (9.80665 m/s^2 - 5.0145 cm \times (-0.0689) radius/s^2 )}{1.1444 radius/s^2 -(-0.0689 radius/s^2) } =  0.0240 kg\times m^2 $$

% ==================================================================
% ==================================================================
For Cylinder A in hole 1, B in hole 2

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/cabd}
\end{figure}

$$ \beta_1 = -0.0710 radius/s^2$$ (with 95\% confidence bounds)

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/caba}
\end{figure}

$$ \beta_2 = 1.8752 radius/s^2$$ (with 95\% confidence bounds)

$$ I_4 = \frac{59.1 g \times 5.0145 cm \times (9.80665 m/s^2 - 5.0145 cm \times (-0.0710) radius/s^2 )}{1.8752 radius/s^2 -(-0.0710 radius/s^2) } =  0.0149 kg\times m^2 $$

% ==================================================================
% ==================================================================
For Cylinder A in hole 3, B in hole 4

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/ccdd}
\end{figure}

$$ \beta_1 = -0.0661 radius/s^2$$ (with 95\% confidence bounds)

\begin{figure}[H]
\centering
\includegraphics[width=\EFWwr]{matlab/ccda}
\end{figure}

$$ \beta_2 = 1.7496 radius/s^2$$ (with 95\% confidence bounds)

$$ I_5 = \frac{59.1 g \times 5.0145 cm \times (9.80665 m/s^2 - 5.0145 cm \times (-0.0661) radius/s^2 )}{ 1.7496 radius/s^2 -(-0.0661 radius/s^2) } = 0.0160 kg\times m^2 $$

Thus, the moment of inertia for disk is
$$ I_{disk} = I_2 - I_1 = 0.0213  kg\times m^2 - 0.0105 kg\times m^2 =  0.0108  kg\times m^2 $$

The moment of inertia for hoop is
$$ I_{hoop} = I_3 - I_1 = 0.0240  kg\times m^2 - 0.0105 kg\times m^2 =  0.0135  kg\times m^2 $$

The moment of inertia for Cylinder A in hole 1 and Cylinder B in hole 2 is
$$ I_{A1B2} = I_4 - I_1 = 0.0149  kg\times m^2 - 0.0105 kg\times m^2 =  0.0044  kg\times m^2 $$

The moment of inertia for Cylinder A in hole 3 and Cylinder B in hole 4 is
$$ I_{A3B4} = I_5 - I_1 = 0.0160  kg\times m^2 - 0.0105 kg\times m^2 =  0.0055  kg\times m^2 $$

$$ I_{A3B4} -I_{A1B2} =0.0055  kg\times m^2 -   0.0044  kg\times m^2 = 0.0011 kg\times m^2  $$
$$ md^2 =  165.8 g \times  (4.7560 cm - 6.0120cm )^2 + 165.8 g \times  (4.7610 cm - 6.0200cm )^2  = 0.0052436 kg\times m^2 $$

Thus, the relative uncertainty is $$ \frac{ 0.0055  kg\times m^2 - 0.0052436 kg \times m^2}{0.0055  kg\times m^2} = 4.66 \% $$


\documentclass[11pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage[english]{babel}
\usepackage{fancyhdr}
\usepackage[margin=.7in]{geometry}
\usepackage{exsheets}
\usepackage{enumerate}

\usepackage{./Style/linearProgramsStyle} % This is a set of commands for maths content.

\pagestyle{fancy}

\renewcommand{\headrulewidth}{2pt}
\fancyhead[L]{EPITA\_ING1\_BING\_2020\_S5\_PROL}
\fancyhead[R]{March 2018}

\fancyfoot[C]{\textbf{\thepage}}
\fancyfoot[L]{}

\pretitle{\vspace{-.5\baselineskip} \begin{center}}
\title{%
  { \huge Linear Programming}\\ %
  \vspace{.5\baselineskip}
  {\Large \textit{Complemetary Exam} }
}
\posttitle{
\end{center}
  \begin{flushleft}
    \vspace{2\baselineskip} \textit{
      \!\!\emph{Duration: 1h30.}\\
      \! \emph{No documents are allowed.}  \\
      Only \emph{non-programmable} pocket calculators are allowed.
    }
  \end{flushleft}
  \rule{\textwidth}{1.5pt}
  \vspace{-5\baselineskip}
}
\author{}
\date{}

\pdfinfo{
   /Author (Bashar Dudin)
   /Title  (PROL Exam -- 2018)
   /Subject (Linear Programming)
}

\begin{document}

\maketitle\thispagestyle{fancy}

We're considering the following list of linear programs respectively
given the names $(A)$, $(B)$, $(C)$, $(D)$, $(E)$ and $(F)$.
\[
(A) \;
\left\{
  \begin{linearProg}{
      maximize
    }{
      $3x_1 + 2x_2 + 3x_3$
    }{
      \systeme{
        -x_1 - 2x_2 - 3x_3 \leq 5,
        2x_1 + x_2 + x_3 \leq 3,
        -x_1 - x_2 + x_3 \leq 1
      }
    }{
      $x_1$, $x_2$, $x_3 \geq 0.$
    }
  \end{linearProg}
\right.
\qquad (B)\;
\left\{
  \begin{linearProg}{
      minimize
    }{
      $ x_1 + x_2 - x_3$
    }{
      \systeme{
        -x_1 - 2x_2 + 3x_3 \leq 1,
        - x_1 - x_2 + 2x_3 \geq 2
      }
    }{
      $x_1$, $x_2$, $x_3 \geq 0.$
    }
  \end{linearProg}
\right.
\]
\[
(C) \;
\left\{
  \begin{linearProg}{
      maximize
    }{
      $3x_1 + 2x_2 + 3x_3$
    }{
      \systeme{
        -x_1 - 2x_2 - 3x_3 \leq 5,
        2x_1 + x_2 + x_3 \leq 3,
        -x_1 - x_2 + x_3 \leq 1
      }
    }{
      $x_1$, $x_2 \geq 0.$
    }
  \end{linearProg}
\right.
\qquad (D) \;
\left\{
  \begin{linearProg}{
      maximize
    }{
      $-x_1 - x_2 + x_3$
    }{
      \systeme{
        -x_1 + 2x_2 - 3x_3 + x_4 = 1,
         x_1 + x_2 - 2x_3 \leq -2
      }
    }{
      $x_1$, $x_2$, $x_3$, $x_4 \geq 0.$
    }
  \end{linearProg}
\right.
\]
\[
\qquad (E)\;
\left\{
  \begin{linearProg}{
      maximize
    }{
      $ -x_1 - x_2 + x_3$
    }{
      \systeme{
        -x_1 - 2x_2 + 3x_3 \leq 1,
        - x_1 - x_2 + 2x_3 \geq 2
      }
    }{
      $x_1$, $x_2$, $x_3 \geq 0.$
    }
  \end{linearProg}
\right.
\qquad (F)\;
\left\{
  \begin{linearProg}{
      maximize
    }{
      $6x_1 + 4x_2 + 6x_3$
    }{
      \systeme{
        -x_1 - 2x_2 - 3x_3 \leq 5,
        2x_1 + x_2 + x_3 \leq 3,
        x_1 + x_2 - x_3 \geq -1
      }
    }{
      $x_1$, $x_2 \geq 0.$
    }
  \end{linearProg}
\right.
\]

\section{Equivalence and Duality}

\begin{question}{3}
  Which previous programs are equivalent to each others? Sort them
  into groups of equivalent classes.
\end{question}

\begin{question}{2}
  Give standard form of $(F)$. What is the slack form of $(B)$?
\end{question}

\begin{question}{2}
  What are the dual programs of $(A)$ and $(E)$?
\end{question}

\begin{question}{2}
  Guess feasible solutions of $(A)$ and $(D)$.
\end{question}

\section{Simplex Algorithm}

\begin{question}{4}
  Solve linear program $(A)$ using the simplex algorithm.
\end{question}

\begin{question}{2}
  Out of previous solution of $(A)$ give an optimal point of its
  dual. What is its optimal objective value?
\end{question}

\begin{question}{5}
  Solve dual of linear program $(E)$.
\end{question}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\chapter{Newton and quasi-Newton methods}
\label{chapter:Newton}

\section{Basics of nonlinear iterations}
\begin{intro}
  The efficient solution of nonlinear problems is an important
  ingredient to implicit timestepping schemes as well as shooting
  methods. Without attempting completeness, we present some important
  facts about iterative methods for this problem. We introduce the two
  generic schemes, Newton and gradient methods, discuss their
  respective pros and cons and combine their features in order to
  obtain better methods.
\end{intro}

\input{definitions/nonlinear}
\input{definitions/iteration-order}
\input{definitions/newton}
\input{theorems/newton-kantorovich}

\begin{remark}
  Instead of proving the Newton-Kantorovich theorem, we discuss its
  main assumptions and features. First, we note that it does not
  require that the initial value be close to a root, or even assumes
  the existence of a root. The theorem is actually an existence proof.

  The Lipschitz condition on $\nabla f$ can be seen as the deviation
  of $f$ from being linear. Indeed, if $f$ were linear, then $L=0$ and
  provided $M\neq 0$ the method converges in a single step for any
  initial value.

  The larger the constant $M$, the smaller wone of the eigenvalues of % wone?
  the Jacobian. Therefore, the function becomes flat in that
  direction and the root finding problem becomes unstable.

  If we have convergence due to $\beta_0 \le 1/2$ (the proof shows
  contraction) there holds $\beta_1 := LM \left\| f\left(x^{(1)}\right)
  \right\| < 1/2$ and we have quadratic convergence from the second step on.
\end{remark}

\input{definitions/gradient-method}
\input{theorems/gradient-method}

% \begin{proof}
%   First, we observe that in any point $x$ with $\nabla F(x) \neq 0$,
%   there exists $\epsilon > 0$ such that
%   \begin{gather*}
%     F(x) - F(x-\epsilon\nabla F(x))
%     = \epsilon \abs{\nabla F(x)}^2 > 0.
%   \end{gather*}
%   Thus, $F(x-\epsilon\nabla F(x)) < F(x)$. We conclude, that for such
%   $x$, the line search obtains a positive value of $\alpha$. Thus, the
%   sequence of the gradient iteration is monotonically decreasing and
%   stays within the set $K$.
% \end{proof}

\section{Globalization}

\begin{intro}
  The convergence of the Newton method is only local, and it is the
  faster, the closer to the solution we start. Thus, finding good
  initial guesses is an important task. A reasonable initial guess in
  a one-step method seems to be $y_0$, but on closer inspection, this
  is true only if the time step is small. Therefore, the convergence
  requirements of Newton's method would insert a new time step
  restriction, which we want to avoid in the context of implicit
  methods. Therefore, and for other cases like the shooting
  methods of chapter~\ref{chapter:rwa}, we present
  methods which extend the domain of convergence.

  As a rule, Newton'a method should never be implemented without some
  globalization strategy!
\end{intro}

\input{definitions/newton-line-search}
\input{definitions/newton-step-size}

\begin{remark}
  The step size control algorithm can be implemented with very low
  overhead. In fact, in each Newton step we only have to compute the
  norm of the residual, which is typically needed for the stopping
  criterion anyway. Additional work is only needed if the residual
  grows. But this is the case, when the original method was likely to
  fail.

  The convergence proof does not guarantee that the values of $j$
  remain bounded. Practically, this is irrelevant, since typically the
  step size control only triggers within the first few steps, then the
  quadratic convergence of the Newton method starts.
\end{remark}

\input{definitions/descent-methods}

\begin{remark}
  Obviously, the gradient method is a descent method, where the direction $s$
  is chosen parallel to $\nabla F(x^{(k)})$ and $\mu$ is chosen in an
  optimal way. It is also called the method of \define{steepest
    descent}.
\end{remark}

\begin{Lemma}{newton-descent}
  The Newton method applied to the function $f(x)$ is a descent method
  applied to the functional $F(x) = \abs{f(x)}^2$. The same holds for
  the Newton method with line search or step size control.
\end{Lemma}

\begin{proof}
  By the product rule, there holds
  \begin{gather*}
    \nabla F(x) = 2 f^T(x) \nabla f(x).
  \end{gather*}
  The search direction of the Newton method is
  \begin{gather*}
    s = -\frac{d^{(k)}}{\abs{d^{(k)}}}
    = \frac{\bigl(\nabla f(x^{(k)})\bigr)^{-1}f(x^{(k)})}{\abs{\dots}}
  \end{gather*}
  Thus, omitting the arguments $x^{(k)}$, we obtain
  \begin{gather*}
    \frac{\nabla F\, s}{\norm{\nabla F}}
    = \frac{f^T \nabla f(x)\bigl(\nabla f\bigr)^{-1} f}%
    {\norm{\bigl(\nabla f\bigr)^{-1} f}\,\norm{f^T \nabla f(x)}}
    \ge \frac{\abs{f}^2}{\norm{\bigl(\nabla f\bigr)^{-1}}\;
      \norm{f}^2 \norm{\nabla f(x)} }
    = \frac1{\operatorname{cond}_2 (\nabla f(x))},
  \end{gather*}
  where we used the operator norm $\norm{.}$ of matrices with respect
  tot the Euclidean norm of $\R^d$. $\operatorname{cond}_2(A)$ is the
  spectral condition of $A$, namely
  \begin{gather*}
    \operatorname{cond}_2(A) = \norm{A} \, \norm{A^{-1}}.
  \end{gather*}
  With ~\ref{eq:descent-methods:1} we conclude that $s\in \mathcal S_\gamma(\nabla F)$ for any
  $\gamma$ with
  \begin{gather*}
    \gamma \le \frac1{\operatorname{cond}_2 (\nabla f(x))}.
  \end{gather*}
  The different variants of the Newton method are only distinguished by
  a different choice of the scaling parameter $\mu$.
\end{proof}

\begin{Lemma}{downhill}
  Let $F: \R^d \to \R$ be continuously differentiable. For a given
  point $x$, assume $\nabla F = \nabla F(x) \neq 0$.  Then, there is a
  constant $\lambda > 0$ such that for any
  $s\in \mathcal S_\gamma(\nabla F(x))$ and any
  $0 \le \mu \le \lambda$ there holds
  \begin{gather}
    \label{eq:downhill:1}
    F(x-\mu s) \le F(x) - \frac{\gamma\mu}2 \abs{\nabla F(x)}.
  \end{gather}
  In particular, a positive scaling factor $\mu$ for the descent method can
  always be found.
\end{Lemma}

% Stoer/Bulirsch I, 4. Auflage, p. 238
\begin{proof}
  First, define
  \begin{gather*}
    U_1(x) = \bigl\{ y\in \R^d \big\vert \abs{\nabla F(y)-\nabla F(x)}
    \le \tfrac{\gamma}{2} \abs{\nabla F(x)}\bigr\}.
  \end{gather*}
  Since $\nabla F$ is continuous and $\nabla F(x) \neq 0$, this set is
  a nonempty neighborhood of $x$.
% Similarly, the set
%  \begin{gather*}
%    U_2(x) =
%    \bigl\{ y\in \R^d \big\vert \mathcal S_\gamma(\nabla F(y)) \subseteq
%    \mathcal S_{\nicefrac{\gamma}{2}}(\nabla F(x))\bigr\}
%  \end{gather*}
%  is a nonempty neighborhood of $x$.
Choose now $\lambda$ such that
  \begin{gather*}
    B_{\lambda}(x) \subseteq U_1(x),% \cap U_2(x).
  \end{gather*}
  Hence, for any $\mu\in(0,\lambda)$ and
  $s\in \mathcal S_\gamma(\nabla F(x))$, there holds by the mean value
  theorem with $0<\theta<1$
  \begin{multline*}
    F(x)-F(x-\mu s) = \mu \nabla F(x-\theta\mu s)s \\
    = \mu \Bigl(\bigl(\nabla F(x-\theta\mu s) - \nabla F(x)\bigr)
    +\nabla F(x)\Bigr).
  \end{multline*}
  Using the definitions of $U_1(x)$ and $U_2(x)$, we obtain
  \begin{align*}
    F(x)-F(x-\mu s)
    &\ge -\tfrac{\gamma\mu}{2}\abs{\nabla F(x)}
      + \mu DF(x)s\\
    &\ge -\tfrac{\gamma\mu}{2}\abs{\nabla F(x)}
      + \mu\gamma\abs{\nabla F(x)}\\
    &= \tfrac{\gamma\mu}{2}\abs{\nabla F(x)}.
  \end{align*}
\end{proof}

\section{Practical considerations}

\begin{intro}
  Quadratic convergence is an asymptotic statement, which for any
  practical purpose can be replaced by ``fast'' convergence. Most of
  the effort spent in a single Newton step consists of setting up the
  Jacobian $J$ and solving the linear system in the second line
  of~\eqref{eq:newton-def:1}. Therefore, we will consider techniques
  here, which avoid some of this work. We will have to consider two
  cases
  \begin{enumerate}
  \item Small systems with $d\lesssim 1000$. For such systems, a
    direct method like $LU$- or $QR$-decomposition is advisable in
    order to solve the linear system. To this end, we compute the
    whole Jacobian and compute its decomposition, an effort of order
    $d^3$ operations. Comparing to $d^2$ operations for applying the
    inverse and order $d$ for all other tasks, this must be avoided as
    much as possible.
  \item Large systems, where the Jacobian is typically sparse (most of
    its entries are zero). For such a system, the effort of order
    $d^2$ for a full matrix vector multiplication is already not
    affordable. Therefore, the linear problem is solved by an
    iterative method and we will not have to compute the Jacobian at
    all.
  \end{enumerate}
\end{intro}

\begin{remark}
  In order to save numerical effort constructing and inverting
  Jacobians, the following strategies have been successful.
  \begin{itemize}
  \item Fix a threshold $0<\eta<1$ which will be used as a bound for
    error reduction. In each Newton step, first compute the update
    vector $\widehat d$ using the Jacobian $\widehat J$ of the
    previous step. This yields the modified method
    \begin{gather}
      \label{eq:newton:1}
      \begin{alignedat}2
        &&J_k &= J_{k-1} \\
        &&\widehat x &= x^{(k)} - J_{k}^{-1}f(x^{(k)})\\
        \text{If }\abs{f(\widehat x)} &\le \eta
          \abs{f(x^{(k)})} \quad& x^{(k+1)} &= \widehat x\\
          \text{Else } J_k &= \bigl(\nabla f(x^{(k)}\bigr)^{-1}
          \quad & x^{(k+1)} &=x^{(k)}- J_{k}^{-1}f(x^{(k)}).
      \end{alignedat}
    \end{gather}
    Thus, an old Jacobian and its inverse are used until convergence
    rates deteriorate. This method is again a quasi-Newton method
    which will not converge quadratically. However, we can obtain linear
    convergence at any rate $\eta$.
  \item If Newton's method is used within a time stepping scheme, the
    Jacobian of the last Newton step in the previous time step is
    often a good approximation for the Jacobian of the first Newton
    step in the new time step. This holds in particular for small time
    steps and constant extrapolation. Therefore, the previous method
    should also be extended over the bounds of time steps.
  \item An improvement of the method above can be achieved by so
    called rank-1 updates. Given $x^{(k)}$ and $x^{(k-1)}$, compute
    \begin{gather}
      \label{eq:newton:2}
      \begin{split}
        p &= x^{(k)} - x^{(k-1)}\\
        q &= f(x^{(k)}) - f(x^{(k-1)})\\
        J_k &= J_{k-1} + \frac1{\abs{p}^2}
        \left(q - J_{k-1} p\right)p^T
      \end{split}
    \end{gather}
    The fact that the rank of $J_k - J_{k-1}$ is at most one can be
    used to obtain a decomposition of $J_k$ in a cheap way from one
    for $J_{k-1}$.
  \end{itemize}
\end{remark}

\begin{remark}
  For problems leading to large, sparse Jacobians, typically space
  discretizations of partial differential equations, computing
  inverses of $LU$-decompositions is infeasible. These matrices
  typically only feature a few nonzero elements per row, while the
  inverse and the $LU$-decomposition is fully populated, thus
  increasing the amount of memory from $d$ to $d^2$.

  Linear systems like this are often solved by iterative methods,
  leading for instance to so called Newton-Krylov methods. Iterative
  methods approximate the solution of a linear system
  \begin{gather*}
    J d = f
  \end{gather*}
  only using multiplications of a vector with the matrix $J$. On
  the other hand, for any vector $v\in \R^d$, the term $Jv$ denotes
  the directional derivative of $f$ in direction $J$. Thus, it can be
  approximated easily by
  \begin{gather*}
    J v \approx \frac{f\left(x^{(k)}+\epsilon v\right) -
      f\left(x^{(k)}\right)}{\epsilon}.
  \end{gather*}
  The term $f\left(x^{(k)}\right)$ must be calculated anyway as it is
  the current Newton residual. Thus, each step of the iterative linear
  solver requires one evaluation of the nonlinear function, and no
  derivatives are computed.

  The efficiency of such a method depends on the number of linear
  iteration steps which is determined by two factors: the gain in
  accuracy and the contraction speed. It turns out that typically
  gaining two digits in accuracy is sufficient to ensure fast
  convergence of the Newton iteration. The contraction number is a
  more difficult issue and typically requires preconditioning, which
  is problem-dependent and as such must be discussed when needed.
\end{remark}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "notes"
%%% End:


%This is my super simple Real Analysis Homework template

\documentclass{article}
\usepackage[margin=1in]{geometry}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage[]{amsthm} %lets us use \begin{proof}
\usepackage[]{amssymb} %gives us the character \varnothing
\usepackage{amsmath}

\title{Debye length}
\author{John ``Jack'' Brooks}
\date\today
%This information doesn't actually show up on your document unless you use the maketitle command below

\begin{document}

\maketitle %This command prints the title based on information entered above

\section{Debye length derivation}
The Debye length is the scale length associated with plasma screening out external electric fields, $\mathbf{E}$.  To show this, we start with Gauss's law and convert $\mathbf{E}$ to potential, $\phi$ with $\mathbf{E} = - \nabla \phi$.

\begin{equation}
\begin{split}
	%\label{eq:asdf}
	\nabla^2 \phi &= \frac{1}{\epsilon_0}\left( \rho_{plasma} \right) \\
	&= \frac{1}{\epsilon_0}\left[ q n_0 \, \textrm{exp} \left( \frac{-q \phi (r)}{k_b T} \right)\right]
\end{split}
\end{equation}
where and the plasma charge is assuming a Boltzmann distribution.

If we assume that $k_bT \gg q\phi$, then we can Taylor expand the exponential term to get
\begin{equation}
\begin{split}
%\label{eq:asdf}
\nabla^2 \phi = \frac{1}{\epsilon_0}\left[ q n_0 \,  \left(1- \frac{q \phi (r)}{k_b T} \right)\right]
\end{split}
\end{equation}
Assuming that we have equal ion and electron charge density, $q_e n_{oe}$ cancels with $q_i n_{oi}$ leaving
\begin{equation}
\begin{split}
%\label{eq:asdf}
\nabla^2 \phi = \frac{1}{\epsilon_0}\left[ -   \frac{q^2 n_0 \phi (r)}{k_b T}  \right].
\end{split}
\end{equation}
The solution to this is
\begin{equation}
\begin{split}
\phi &= \phi_0 \, \textrm{exp} \left( -r / \left( \frac{q^2n_0}{\epsilon_0 k_b T} \right)^{1/2} \right) \\
 &= \phi_0 \, \textrm{exp} \left( -r /\lambda_D^{1/2} \right)
\end{split}
\end{equation}
where the scale length,
\begin{equation}
\begin{split}
 \lambda_D =  \left( \frac{q^2n_0}{\epsilon_0 k_b T} \right)^{1/2},
\end{split}
\end{equation}
is called the Debye length.

\section{Debye length screening}

If we now include a test charge, $Q_{tc}$, to our Gauss's law equation,
\begin{equation}
\begin{split}
%\label{eq:asdf}
\nabla^2 \phi &= \frac{1}{\epsilon_0}\left( \rho_{plasma} - \rho_{tc}\right) \\
&= \frac{1}{\lambda_D^2}\phi - \frac{Q_{tc}}{\epsilon_0}\delta (r-r_0)
\end{split}
\end{equation}
we can now investigate how the plasma screens out this charge.

TODO

\end{document}


\documentclass{article}

\begin{document}

Author: Carl Cortright

\section{Problem 10.1}

Training Data:\\

\begin{tabular}{c|cccc}

  Class & CornFlakes & Frosties & Sugar Puffs & Branflakes \\
  \hline
  Greater than 60 & 1 & 0 & 0 & 0\\
  Greater than 60 & 1 & 0 & 0 & 1\\
  Greater than 60 & 1 & 1 & 1 & 1\\
  Greater than 60 & 0 & 0 & 0 & 1\\
  Less than 60 & 0 & 1 & 1 & 0\\
  Less than 60 & 1 & 1 & 1 & 0\\

\end{tabular}

\subsection{(a)}

$P(Less than 60 | <0,1,1,0>) = \frac{P(Less than 60) * P(<0,1,1,0>|Less than 60)}{P(<0,1,1,0>)} = \frac{\frac{1}{3} * \frac{1}{2}}{\frac{1}{6}} = 1$

\subsection{(b)}

$P(age) = \frac{ P(age) * P(<a,b,c,d> | age) }{ P(<a,b,c,d>) }$

\section{Problem 10.5}

\subsection{}

$P(C=1) = \sum_{i=0}^N P(X_i | C=1)$ \\

$P(X_i = 1 | C = 1) = \frac{P(C=1) * P(C=1 | X_i=1) }{P(X_i = 1}$ \\

$P(X_i=1 | C = 0) =  \frac{P(C=0) * P(C=0 | X_i=1) }{P(X_i = 1}$ \\

\subsection{}

To form the classifier, we must take the probability of an input vector $\vec{x}$ and class $C$ and divide it by the probability of the input vector:\\

$P(C|\vec{x}) = \frac{P(\vec{x},C)}{P(\vec{x})}$

\subsection{}

If the word viagra doesn't appear in the training set, then $P(Viagra | C) = 0$ meaning via Bayes rule $P(C | Viagra) = 0$ . Via intuition, we know this must not be true. To fix this, we need to use a technique called smoothing, which basically means adding some arbitrary value $z$ to all of our probabilities to insure that they are always non-zero.

\section{Dirichlet Distributions}

\subsection{(a)}

One situation might be where we want to model classes of classes. An example would be if we were to classify spam or not spam, and then classify the sender of that particular email.

\subsection{(b)}

From Lowest to highest entopy:\\
$Dir(2,2,2)$\\
$Dir(1,1,1)$\\
$Dir(0.1,0.1,0.1)$\\

\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{examples}

\begin{document}

\section*{The Gauss relation for the curvature of a hypersurface}

\begin{cadabra}
   {a,b,c,d,e,f,g,i,j,k,l,m,n,o,p,q,r,s,t,u#}::Indices.

   \nabla_{#}::Derivative.

   K_{a b}::Symmetric.
   g^{a}_{b}::KroneckerDelta.

   # Define the projection operator

   hab:=h^{a}_{b} -> g^{a}_{b} - n^{a} n_{b}.

   # 3-covariant derivative obtained by projection on 4-covariant derivative

   vpq:=v_{p q} -> h^{a}_{p} h^{b}_{q} \nabla_{b}{v_{a}}.

   # Compute 3-curvature by commutation of covariant derivatives

   vpqr:= h^{a}_{p} h^{b}_{q} h^{c}_{r} ( \nabla_{c}{v_{a b}} - \nabla_{b}{v_{a c}} ).

   substitute (vpq,hab)
   substitute (vpqr,vpq)

   distribute   (vpqr)
   product_rule (vpqr)
   distribute   (vpqr)
   eliminate_kronecker(vpqr)

   # Standard substitutions

   substitute (vpqr,$h^{a}_{b} n^{b} -> 0$)
   substitute (vpqr,$h^{a}_{b} n_{a} -> 0$)
   substitute (vpqr,$\nabla_{a}{g^{b}_{c}} -> 0$)
   substitute (vpqr,$n^{a} \nabla_{b}{v_{a}} -> -v_{a} \nabla_{b}{n^{a}}$)
   substitute (vpqr,$v_{a} \nabla_{b}{n^{a}} -> v_{p} h^{p}_{a}\nabla_{b}{n^{a}}$)
   substitute (vpqr,$h^{p}_{a} h^{q}_{b} \nabla_{p}{n_{q}} -> K_{a b}$)
   substitute (vpqr,$h^{p}_{a} h^{q}_{b} \nabla_{p}{n^{b}} -> K_{a}^{q}$)

   # Tidy up and display the results

   {h^{a}_{b},\nabla_{a}{v_{b}}}::SortOrder.

   sort_product   (vpqr)
   rename_dummies (vpqr)
   canonicalise   (vpqr)
   factor_out     (vpqr,$h^{a?}_{b?}$)
   factor_out     (vpqr,$v_{a?}$)       # cdb(gauss,vpqr)
\end{cadabra}

\subsection*{The Gauss relation for the curvature of a hypersurface}

\begin{align*}
   D_{r}(D_{q}v_p) - D_{q}(D_{r}v_p) = \cdb{gauss}
\end{align*}

\vspace{15pt}

\begin{latex}
   \begin{align*}
      D_{r}(D_{q}v_p) - D_{q}(D_{r}v_p) = \cdb{gauss}
   \end{align*}
\end{latex}

\end{document}