% http://www.seas.ucla.edu/~vandenbe/236C/lectures/gradient.pdf

\section{Optimization Methods}

In order to explain the \emph{convergence rates} of the following optimization methods, we need to introduce some preliminary definitions about \emph{convexity} and the \emph{Lipschitz continuity} of a function~\cite{boyd2004convex}.

\begin{definition}[Convexity] \label{def:convexity}
	\hfill
	\begin{enumerate}[(i)]
		\item We say that a function $f: \Re^m \rightarrow \Re$ is convex if:
		$$
			(\lambda x + (1 - \lambda) y ) \leq \lambda f(x) + (1 - \lambda) f(y) \ \forall \ x, y \in \Re^m, \lambda \in [0,1]
		$$
		\item We say that a differentiable function $f: \Re^m \rightarrow \Re$ is convex if:
		$$
			f(y) \geq f(x) + \langle \nabla f(x), y - x \rangle \ \forall \ x, y \in \Re^m
		$$
		\item We say that a twice differentiable function $f: \Re^m \rightarrow \Re$ is convex iff:
		$$
			\nabla^2 f(x) \succeq 0 \ \forall \ x \in \Re^m
		$$ i.e., the Hessian matix is \emph{positive semidefinite}.
	\end{enumerate}
\end{definition}

\begin{definition}[Strict Convexity] \label{def:strict_convexity}
	\hfill
	\begin{enumerate}[(i)]
		\item We say that a function $f: \Re^m \rightarrow \Re$ is strictly convex if:
		$$
			(\lambda x + (1 - \lambda) y ) < \lambda f(x) + (1 - \lambda) f(y) \ \forall \ x, y \in \Re^m, x \neq y, \lambda \in (0,1)
		$$
		\item We say that a differentiable function $f: \Re^m \rightarrow \Re$ is strictly convex if:
		$$
			f(y) > f(x) + \langle \nabla f(x), y - x \rangle \ \forall \ x, y \in \Re^m, x \neq y
		$$
		\item We say that a twice differentiable function $f: \Re^m \rightarrow \Re$ is strictly convex iff:
		$$
			\nabla^2 f(x) \succ 0 \ \forall \ x \in \Re^m
		$$ i.e., the Hessian matix is \emph{positive definite}.
	\end{enumerate}
\end{definition}

% http://xingyuzhou.org/blog/notes/strong-convexity

\begin{definition}[Strong Convexity] \label{def:strong_convexity}
We say that a function $f: \Re^m \rightarrow \Re$ is $\mu$-strongly convex if the function:
$$
g(x) = f(x) - \frac{\mu}{2} \| x \|^2
$$
is convex for any $\mu > 0$.
If $f$ is differentiable this is also equivalent to:
$$
f(y) \geq f(x) + \langle \nabla f(x), y - x \rangle + \frac{\mu}{2} \| y - x \|^2 \ \forall \ x, y \in \Re^m
$$
and, if $f$ is a twice differentiable function then $f$ is $\mu$-strongly convex iff:
$$
\nabla^2 g(x) \succ 0 \ \forall \ x \in \Re^m
$$
i.e., the Hessian matix is \emph{positive definite}, which is:
$$
\nabla^2 f(x) \succeq \mu I \ \forall \ x \in \Re^m
$$
i.e., all the eigenvalues of the Hessian matrix are lowerbounded by $\mu I$.
\end{definition}

% https://angms.science/doc/CVX/CVX_alphabeta.pdf
% https://xingyuzhou.org/blog/notes/Lipschitz-gradient

\begin{definition}[$L_f$-Lipschitz continuity] \label{def:l_f_lipschitz_continuity}
We say that a function $f: \Re^m \rightarrow \Re$ is $L_f$-Lipschitz continuous if:
$$
| f(x) - f(y) | \leq L_f \| x - y \| \ \forall \ x, y \in \Re^m
$$
meaning that $f$ is bounded above and below by a linear function.

Intuitively, $L$ is a measure of how fast the function can change.

Finally, we say that a function $f: \Re^m \rightarrow \Re$ is locally $L_f$-Lipschitz continuous if for every $x$ in $\Re^m$ there exists a neighborhood $U$ of $x$ such that $f$ restricted to $U$ is $L_f$-Lipschitz continuous.
Every convex function is locally $L_f$-Lipschitz continuous.
\end{definition}

\begin{definition}[L-Lipschitz continuity] \label{def:l_lipschitz_continuity}
We say that a function $f: \Re^m \rightarrow \Re$ is L-Lipschitz gradient continuous if $f$ is differentiable and:
$$
\| \nabla f(x) - \nabla f(y) \| \leq L \| x - y \| \ \forall \ x, y \in \Re^m
$$
that is equivalent to:
$$
| f(y) - f(x) - \langle \nabla f(x), y - x \rangle | \leq \frac{L}{2} \| y - x \|^2 \ \forall \ x, y \in \Re^m
$$
meaning that $f$ is bounded above and below by a quadratic function.

Also, if $f$ is a twice differentiable function this is equivalent to:
$$
\nabla^2 f(x) \preceq L I \ \forall \ x \in \Re^m
$$
i.e., all the eigenvalues of the Hessian matrix are upperbounded by $L$.

Note that if $f$ is a $\mu$-strongly convex function, we give the following Hessian bounds:
$$
0 \prec \mu I \preceq \nabla^2 f(x) \preceq L I \ \forall \ x \in \Re^m
$$
i.e., all the eigenvalues of the Hessian matrix are lowerbounded by $\mu I$ and upperbounded by $L$.

Finally, we say that a function $f: \Re^m \rightarrow \Re$ is locally L-Lipschitz gradient continuous if for every $x$ in $\Re^m$ there exists a neighborhood $U$ of $x$ such that $f$ restricted to $U$ is L-Lipschitz gradient continuous.
\end{definition}

\begin{definition}[Subgradient] \label{def:subgradient}
Given a function $f: \Re^m \rightarrow \Re$ and $x \in \Re^m$, we define a subgradient $g \in \Re^m$ at $x$ to be any point satisfying:
$$
	f(y) \geq f(x) + \langle g, y - x \rangle \ \forall \ y \in \Re^m
$$
Subgradients always exist for convex function.
\end{definition}

% https://www-ljk.imag.fr/membres/Anatoli.Iouditski/cours/convex/chapitre_3.pdf

\begin{theorem}[\href{http://web.mit.edu/dimitrib/www/Convex_Theory_Entire_Book.pdf\#page=196}{$L_f$-Lipschitz continuity for convex functions}] \label{def:l_f_lipschitz_convex}
Let $f: \Re^m \rightarrow \Re$ be a convex function and let $K$ be a closed and bounded set contained in the relative interior of the domain of $f$, i.e., $K \subset \Re^m$. Then $f$ is $L_f$-Lipschitz continuous on $K$, i.e.,:
$$
| f(x) - f(y) | \leq L_f \| x - y \| \ \forall \ x, y \in K
$$
In particular, $f$ is bounded on $K$.
\end{theorem}

% http://www.athenasc.com/convexsol4.pdf#page=7

\begin{proof}
Let $x$ and $y$ be any two points in the set $K$. Since $\partial f(x)$ is nonempty, by using the subgradient inequality~\ref{def:subgradient}, it follows that:
$$
	f(y) \geq f(x) + \langle g, y - x \rangle \ \forall \ g \in \partial f(x)
$$
implying that:
$$
	f(x) - f(y) \leq \|g\| \|x-y\| \ \forall \ g \in \partial f(x)
$$
By definition, the set $\cup_{x \in K} \partial f(x)$ is nonempty and bounded, so that for some constant $L > 0$, we have:
$$
	\|g\| \leq L_f \ \ \ \forall g \in \partial f(x) \ \ \ \forall x \in K
$$
and therefore:
$$
	f(x) - f(y) \leq L_f \|x-y\|
$$
By exchanging the roles of $x$ ad $y$, we similarly obtain:
$$
	f(y) - f(x) \leq L_f \|x-y\|
$$
and by combining the preceding two relations, we see that:
$$
	|f(x) - f(y)| \leq L_f \|x-y\|
$$
showing that $f$ is $L_f$-Lipschitz continuous over $K$.
\end{proof}

Note that this proof shows how to determine the Lipschitz constant $L_f$: it is the maximum subgradient norm, over all subgradients in $\cup_{x \in K} \partial f(x)$.

Strong convexity and L-Lipschitz continuity are related by Fenchel duality according to the following theorem, which proof is given in~\cite{zhou2018fenchel}.

% https://arxiv.org/pdf/1803.06573.pdf
% https://www.stat.purdue.edu/~vishy/introml/notes/Optimization.pdf#page=8

\begin{theorem}[\href{https://arxiv.org/pdf/1803.06573.pdf}{$\mu$-strong convexity and L-Lipschitz continuity for convex functions}]\label{def:strong_cvx_l_lipschitz_convex}
A function $f$ and its Fenchel dual $f^*$ satisfy the following assertions:
\begin{enumerate}[(i)]
		\item if $f$ is $\mu$-strongly convex, then $f^*$ is $\displaystyle \frac{1}{\mu}$-Lipschitz continuous.
		\item if $f$ is convex and L-Lipschitz continuous, then $f^*$ is $\displaystyle \frac{1}{L}$-strongly convex.
\end{enumerate}
\end{theorem}

Note that since $f$ is convex and its epigraph is a closed convex set, $f^* = f$.

\pagebreak

\subsection{Gradient Descent for primal formulations}

The Gradient Descent algorithm is the simplest \emph{first-order optimization} method that exploits the orthogonality of the gradient wrt the level sets to take a descent direction. In particular, it performs the following iterations:

\begin{algorithm}[H]
	\caption{Gradient Descent}
	\label{alg:gd}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Function{GradientDescent}{$f,\alpha$}
			\State Initialize weight vector $x_0$
			\State $t = 0$
			\While{$not\_convergence$}
				\State $x_{t+1} = x_t - \alpha \partial f(x_t)$ \Comment if $f$ is differentiable then $\partial f(x_t) = \nabla f(x_t)$
				\State $t = t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

Gradient Descent is based on full gradients, since at each iteration we compute the average gradient on the whole dataset:
$$
\partial f(x) = \frac{1}{n} \sum_{i=1}^n \partial f_i(x)
$$
The downside is that every step is very computationally expensive, $\mathcal{O}(nm)$ per iteration, where $n$ is the number of samples in our dataset and $m$ is the number of dimensions.

Since \emph{Gradient Descent} becomes impractical when dealing with large datasets we introduce a stochastic version, called \emph{Stochastic Gradient Descent}, which does not use the whole set of examples to compute the gradient at every step. By doing so, we can reduce computation all the way down to $\mathcal{O}(m)$ per iteration.

\begin{algorithm}[H]
	\caption{Stochastic Gradient Descent}
	\label{alg:sgd}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Require{Batch size $k$}
		\Function{StochasticGradientDescent}{$f,\alpha,k$}
			\State Initialize weight vector $x_0$
			\State $t \gets 0$
			\While{$not\_convergence$}
				\State Sample $(i_1,\dots,i_k) \sim \mathcal{U}^k(1,\dots,n)$
				\State $\displaystyle x_{t+1} \gets x_t - \alpha \frac{1}{k} \sum_{j=1}^k \partial f_{i_j}(x_t)$ \Comment if $f$ is differentiable then $\partial f_{i_j}(x_t) = \nabla f_{i_j}(x_t)$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

Note that in expectation, we converge like GD, since $\displaystyle \mathbb{E}_{i \sim \mathcal{U}(1,\dots,n)}[\partial f_i(x_t)] = \partial f(x_t)$, therefore, the expected iterate of SGD converges to the optimum.

% https://erwanscornet.github.io/teaching/Optimization.pdf
% https://www.math.univ-toulouse.fr/~agarivie/sites/default/files/8_optimization.pdf

Now, consider the SGD algorithm introduced previously but where each iteration is projected into the ball $\mathcal{B}(0, R)$ with radius $R > 0$ fixed. So, the following lower bounds on convergence rates are given.

\begin{theorem}[\href{https://erwanscornet.github.io/teaching/Optimization.pdf\#page=62}{Stochastic Gradient Descent convergence for convex functions}] \label{thm:cvx_sgd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous convex function and assume that exists $b > 0$ satisfying:
$$
\| f_i(x) \| \leq b \ \forall \ x \in \mathcal{B}(0, R)
$$
Besides, assume that all minima of $f$ belong to $\mathcal{B}(0, R)$. Then the Stochastic Gradient Descent with step size $\displaystyle \alpha = \frac{2R}{b\sqrt{k}}$ satisfies:
$$
\mathbb{E}\Bigg[f\Bigg(\frac{1}{k} \sum_{t=1}^k x_t\Bigg)\Bigg] - f(x^*) \leq \frac{3Rb}{\sqrt{k}}
$$
\end{theorem}

\begin{theorem}[\href{https://erwanscornet.github.io/teaching/Optimization.pdf\#page=63}{Stochastic Gradient Descent convergence for strongly convex functions}] \label{thm:str_cvx_sgd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous, $\mu$-strongly convex function and assume that exists $b > 0$ satisfying:
$$
\| f_i(x) \| \leq b \ \forall \ x \in \mathcal{B}(0, R)
$$
Besides, assume that all minima of $f$ belong to $\mathcal{B}(0, R)$. Then the Stochastic Gradient Descent with step size $\displaystyle \alpha = \frac{2}{\mu(k+1)}$ satisfies:
$$
\mathbb{E}\Bigg[f\Bigg(\frac{2}{k(k+1)} \sum_{t=1}^k t x_{t-1}\Bigg)\Bigg] - f(x^*) \leq \frac{2b^2}{\mu(k+1)}
$$
\end{theorem}

SGD’s \emph{convergence rate} for L-Lipschitz continuous convex functions is $\displaystyle \mathcal{O}\bigg(\frac{1}{\sqrt{t}}\bigg)$ and $\displaystyle \mathcal{O}\bigg(\frac{1}{t}\bigg)$ for L-Lipschitz continuous and strongly convex functions. More iterations are needed to reach the same accuracy as GD, but the iterations are far cheaper.

% http://www.princeton.edu/~yc5/ele522_optimization/lectures/subgradient_methods.pdf

\subsubsection{Nonsmooth}

First, consider a nonsmooth, i.e., nondifferentiable, convex function. So, the following lower bounds on convergence rates are given.

\begin{theorem}[\href{http://www.princeton.edu/~yc5/ele522_optimization/lectures/subgradient_methods.pdf\#page=34}{Subgradient Descent convergence for convex functions with Polyak's stepsize}] \label{thm:cvx_polyak_subgd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a $L_f$-Lipschitz continuous convex function. Then the Subgradient Descent with Polyak's step size $\displaystyle \alpha_t = \frac{f(x_t) - f(x^*)}{\| g_t \|^2}$ satisfies:
$$
f(x_t) - f(x^*) \leq \frac{L \| x_0 - x^* \|^2}{\sqrt{t+1}}
$$
\end{theorem}

Unfortunately, Polyak’s stepsize rule requires knowledge of $f(x^*)$, which is often unknown a priori, so we might often need simpler rule for setting stepsizes.

\begin{theorem}[\href{http://www.princeton.edu/~yc5/ele522_optimization/lectures/subgradient_methods.pdf\#page=39}{Subgradient Descent convergence for convex functions}] \label{thm:cvx_subgd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a $L_f$-Lipschitz continuous convex function. Then the Subgradient Descent with step size $\displaystyle \alpha_t = \frac{1}{\sqrt{t}}$ satisfies:
$$
f(x_t) - f(x^*) \leq \frac{\| x_0 - x^* \|^2 + L^2 \log t}{\sqrt{t}}
$$
\end{theorem}

\begin{theorem}[\href{http://www.princeton.edu/~yc5/ele522_optimization/lectures/subgradient_methods.pdf\#page=41}{Subgradient Descent convergence for strongly convex functions}] \label{thm:str_cvx_subgd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a $L_f$-Lipschitz continuous and $\mu$-strongly convex function. Then the Subgradient Descent with step size $\displaystyle \alpha_t = \frac{2}{\mu(t+1)}$ satisfies:
$$
f(x_t) - f(x^*) \leq \frac{2L^2}{\mu} \frac{1}{t+1}
$$
\end{theorem}

In summary, the following \emph{convergence rates} and \emph{iterations complexities} are given:

\begin{table}[H]
\centering
\caption{Subgradient Descent convergence rates and iterations complexities}
\label{subgradient_rates}
\begin{tabular}{lrrr}
\toprule
	& stepsize rule & convergence rate & iteration complexity \\
 	& 		& 		& 		\\
\midrule
convex and $L_f$-Lipschitz & $\displaystyle \alpha = \frac{1}{\sqrt{t}}$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{\sqrt{t}}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{\epsilon^2}\Bigg)$ \\
strongly convex and $L_f$-Lipschitz & $\displaystyle \alpha = \frac{1}{t}$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{t}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{\epsilon}\Bigg)$ \\
\bottomrule
\end{tabular}
\end{table}

Among algorithms that only use subgradient, these \emph{convergence rates} cannot be futher improved.

% http://www.princeton.edu/~yc5/ele522_optimization/lectures/grad_descent_unconstrained.pdf
% http://www.stat.cmu.edu/~larry/=sml/optrates.pdf

\subsubsection{Smooth}

Now, consider a smooth, i.e., differentiable, convex function. So, the following lower bounds on convergence rates are given.

\begin{theorem}[\href{https://erwanscornet.github.io/teaching/Optimization.pdf\#page=44}{Gradient Descent convergence for convex functions}] \label{thm:cvx_gd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous convex function. Then the Gradient Descent with step size $\alpha = 1/L$ satisfies:
$$
f(x_t) - f(x^*) \leq \frac{L \| x_0 - x^* \|^2}{2 t}
$$
\end{theorem}

\begin{theorem}[\href{https://erwanscornet.github.io/teaching/Optimization.pdf\#page=47}{Gradient Descent convergence for strongly convex functions}] \label{thm:str_cvx_gd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous and $\mu$-strongly convex function. Then the Gradient Descent with step size $\alpha = 1/L$ satisfies:
$$
\begin{aligned}
	f(x_t) - f(x^*) \leq & \bigg(1 - \frac{\mu}{L}\bigg)^t \| f(x_0) - f(x^*) \|^2 \\
						= & \bigg(1 - \frac{1}{\kappa}\bigg)^t \| f(x_0) - f(x^*) \|^2
\end{aligned}
$$
where $\kappa = L/\mu$.
\end{theorem}

\begin{theorem}[\href{https://angms.science/doc/CVX/CVX_HBM.pdf\#page=5}{Gradient Descent convergence for convex quadratic functions}] \label{thm:quad_gd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous and $\mu$-strongly convex quadratic function. Then the Gradient Descent with step size $\alpha = \displaystyle \frac{2}{L + \mu}$ and momentum $\beta = \max{\{|1 - \alpha \mu|, |1 - \alpha L|\}}$ satisfies:
$$
\begin{aligned}
	\| x_t - x^* \| = \bigg(\frac{\kappa-1}{\kappa+1}\bigg)^t \| x_0 - x^* \|
\end{aligned}
$$
where $\kappa = L/\mu$.
\end{theorem}

In summary, the following \emph{convergence rates} and \emph{iterations complexities} are given:

\begin{table}[H]
\centering
\caption{Gradient Descent convergence rates and iterations complexities}
\label{gradient_rates}
\begin{tabular}{lrrr}
\toprule
	& stepsize rule & convergence rate & iteration complexity \\
 	& 		& 		& 		\\
\midrule
convex and L-Lipschitz & $\displaystyle \alpha = \frac{1}{L}$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{t}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{\epsilon}\Bigg)$ \\
strongly convex and L-Lipschitz & $\displaystyle \alpha = \frac{1}{L}$ & $\displaystyle \mathcal{O}\Bigg(\Bigg(1 - \frac{1}{\kappa}\Bigg)^t\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\kappa \log \frac{1}{\epsilon}\Bigg)$ \\
\bottomrule
\end{tabular}
\end{table}

% http://www.princeton.edu/~yc5/ele522_optimization/lectures/accelerated_gradient.pdf

\subsubsection{Momentum}

To mitigate the pathological zig-zagging by speeding up the \emph{convergence rate} of the SGD method, we introduce two accelerated methods~\cite{polyak1964some} and~\cite{nesterov1998introductory, nesterov1983method} that exploits information from the history, i.e., past iterates, to add some inertia, i.e., the momentum, to yield smoother trajectory.

In the Polyak's method~\cite{polyak1964some} the velocity vector $v_t$ is calculated by applying the $\beta$ momentum to the previous $v_{t-1}$ displacement, and subtracting the gradient step to $x_t$.

% http://mitliagkas.github.io/ift6085-2019/ift-6085-lecture-6-notes.pdf

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.5]{img/momentum}
  	\caption{Polyak's and Nesterov's momentum}
  	\label{fig:momentum}
\end{figure}

\begin{algorithm}[H]
	\caption{Polyak's Accelerated Gradient Descent or Polyak Heavy-Ball method}
	\label{alg:pag}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Require{Momentum $\beta \in [0,1)$}
		\Function{PolyakAcceleratedGradientDescent}{$f,\alpha,\beta$}
			\State Initialize weight vector $x_1 \gets x_0$ and velocity vector $v_0 \gets 0$
			\State $t \gets 1$
			\While{$not\_convergence$}
				\State $v_t = \beta v_{t-1} + \alpha \nabla f(x_t)$
				\State $x_{t+1} = x_t - v_t$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

% https://angms.science/doc/CVX/CVX_HBM.pdf

\begin{theorem}[\href{https://angms.science/doc/CVX/CVX_HBM.pdf\#page=21}{Polyak's Accelerated Gradient Descent convergence for convex quadratic functions}] \label{thm:quad_pag_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous and $\mu$-strongly convex quadratic function. Then the Polyak's Accelerated Gradient Descent with step size $\alpha = \displaystyle \frac{4}{(\sqrt{L} + \sqrt{\mu})^2}$ and momentum $\beta = \max{\{|1 - \sqrt{\alpha \mu}|, |1 - \sqrt{\alpha L}|\}^2}$ satisfies:
$$
\begin{aligned}
	\| x_t - x^* \| = \bigg(\frac{\sqrt{\kappa}-1}{\sqrt{\kappa}+1}\bigg)^t \| x_0 - x^* \|
\end{aligned}
$$
where $\kappa = L/\mu$.
\end{theorem}

Leveraging the idea of momentum introduced by Polyak, Nesterov introduced a slightly altered update rule that has been shown to converge not only for quadratic functions, but for general convex functions. In the Nesterov's method~\cite{nesterov1998introductory}, instead, the velocity vector $v_t$ is calculated by applying the $\beta$ momentum to the previous $v_{t-1}$ displacement, and subtracting the gradient step to $x_t + \beta v_{t-1}$, which is the point where the momentum term leads from $x_t$.

\begin{algorithm}[H]
	\caption{Nesterov's Accelerated Gradient Descent or Nesterov Heavy-Ball method}
	\label{alg:nag}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate $\alpha > 0$}
		\Require{Momentum $\beta \in [0,1)$}
		\Function{NesterovAcceleratedGradientDescent}{$f,\alpha,\beta$}
			\State Initialize weight vector $x_1 \gets x_0$ and velocity vector $v_0 \gets 0$
			\State $t \gets 1$
			\While{$not\_convergence$}
				\State $\hat{x}_t \gets x_t + \beta v_{t-1}$
				\State $v_t \gets \beta v_{t-1} + \alpha \nabla f(\hat{x}_t)$
				\State $x_{t+1} \gets x_t - v_t$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

Comparing the algorithm~\ref{alg:pag} with the algorithm~\ref{alg:nag}, we can see that Polyak’s method evaluates the gradient before adding momentum, whereas Nesterov’s algorithm evaluates it after applying momentum, which intuitively brings us closer to the minimum $x^*$, as shown in figure~\ref{fig:momentum}.

\begin{theorem}[\href{http://www.princeton.edu/~yc5/ele522_optimization/lectures/accelerated_gradient.pdf\#page=31}{Nesterov's Accelerated Gradient Descent convergence for convex functions}] \label{thm:cvx_nag_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous convex function. Then the Nesterov's Accelerated Gradient Descent with step size $\alpha = 1/L$ and momentum $\beta_{t+1} = t / (t+3)$ satisfies:
$$
f(x_t) - f(x^*) \leq \frac{2L \| x_0 - x^* \|^2}{(t+1)^2}
$$
\end{theorem}

\begin{theorem}[\href{http://www.princeton.edu/~yc5/ele522_optimization/lectures/accelerated_gradient.pdf\#page=38}{Nesterov's Accelerated Gradient Descent convergence for strongly convex functions}] \label{thm:str_cvx_nag_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-Lipschitz continuous and $\mu$-strongly convex function. Then the Nesterov's Accelerated Gradient Descent with step size $\alpha = 1/L$ and momentum $\beta = \displaystyle \frac{\sqrt{\kappa}-1}{\sqrt{\kappa}+1}$ satisfies:
$$
\begin{aligned}
	f(x_t) - f(x^*) \leq & \Bigg(1 - \sqrt{\frac{\mu}{L}}\Bigg)^t \Bigg(f(x_0) - f(x^*) + \frac{\mu \| x_0 - x^* \|^2}{2}\Bigg) \\
						= & \Bigg(1 - \frac{1}{\sqrt{\kappa}}\Bigg)^t \Bigg(f(x_0) - f(x^*) + \frac{\mu \| x_0 - x^* \|^2}{2}\Bigg)
\end{aligned}
$$
where $\kappa = L/\mu$.
\end{theorem}

% https://blogs.princeton.edu/imabandit/2014/03/06/nesterovs-accelerated-gradient-descent-for-smooth-and-strongly-convex-optimization/

In summary, the following \emph{convergence rates} and \emph{iterations complexities} are given:

\begin{table}[H]
\centering
\caption{Nesterov's Accelerated Gradient Descent convergence rates and iterations complexities}
\label{nesterov_gradient_rates}
\begin{tabular}{lrrr}
\toprule
	& stepsize rule & convergence rate & iteration complexity \\
 	& 		& 		& 		\\
\midrule
convex and L-Lipschitz & $\displaystyle \alpha = \frac{1}{L}$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{t^2}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{1}{\sqrt{\epsilon}}\Bigg)$ \\
strongly convex and L-Lipschitz & $\displaystyle \alpha = \frac{1}{L}$ & $\displaystyle \mathcal{O}\Bigg(\Bigg(1 - \frac{1}{\sqrt{\kappa}}\Bigg)^t\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\sqrt{\kappa}\log \frac{1}{\epsilon}\Bigg)$ \\
\bottomrule
\end{tabular}
\end{table}

Note that in case of L-Lipschitz continuous and strongly convex functions, Nesterov's momentum gives the acceleration that we had with Polyak’s momentum for quadratic functions. This is great because we get the guarantee for a more general class of functions, but these \emph{convergence rates} cannot be futher improved only using first-order information.

\pagebreak

\subsection{Sequential Minimal Optimization for Wolfe dual formulations}

The \emph{Sequential Minimal Optimization (SMO)}~\cite{platt1998sequential} method is the most popular approach for solving the SVM QP problem without any extra $Q$ matrix storage required by common QP methods. The advantage of SMO lies in the fact that it performs a series of two-point optimizations since we deal with just one equality constraint, so the Lagrange multipliers can be solved analitically.

\subsubsection{Classification}

At each iteration, SMO chooses two $\alpha_i$ to jointly optimize, let $\alpha_1$ and $\alpha_2$, finds the optimal values for these multipliers and update the SVM to reflect these new values. In order to solve for two Lagrange multipliers, SMO first computes the constraints over these and then solves for the constrained minimum. Since there are only two multipliers, the box-constraints cause the Lagrange multipliers to lie within a box, while the linear equality constraint causes the Lagrange multipliers to lie on a diagonal line inside the box. So, the constrained minimum must lie there as shown in~\ref{fig:smo_lagrange_multipliers}.

\begin{figure}[h!]
	\centering
	\includegraphics[scale=0.5]{img/smo_multipliers}
	\caption{SMO for two Lagrange multipliers}
	\label{fig:smo_lagrange_multipliers}
\end{figure}

In case of classification the ends of the diagonal line segment, i.e., the lower and upper bounds, can be espressed as follow if the target $y_1 \ne y_2$:

\begin{equation} \label{eq:smo_svc_bounds_update1}
	\begin{aligned}
		& L = max(0, \alpha_2 - \alpha_1) \\
		& H = min(C, C + \alpha_2 - \alpha_1)
	\end{aligned}
\end{equation}

or, alternatively, if the target $y_1 = y_2$:

\begin{equation} \label{eq:smo_svc_bounds_update2}
	\begin{aligned}
		& L = max(0, \alpha_2 + \alpha_1 - C) \\
		& H = min(C, \alpha_2 + \alpha_1)
	\end{aligned}
\end{equation}

The second derivative of the objective quadratic function along the diagonl line can be expressed as:

\begin{equation} \label{eq:smo_eta}
	\eta = K(x_1, x_1) + K(x_2, x_2) - 2K(x_1, x_2)
\end{equation}

that will be grather than zero if the kernel matrix will be positive definite, so there will be a minimum along the linear equality constraints that will be:

\begin{equation} \label{eq:smo_svc_a2_new}
	\alpha_2^{new} = \alpha_2 + \frac{y_2(E_1 - E_2)}{\eta}
\end{equation}

where $E_i = y_i - y'_i$ is the error on the $i$-th training example and $y'_i$ is the output of the SVC for the same.

Then, the box-constrained minimum is found by clipping the unconstrained minimum to the ends of the line segment:

\begin{equation} \label{eq:smo_svc_a2_new_clipped}
    \alpha_2^{new,clipped} =
        \begin{cases}
            H & \text{if} \ \alpha_2^{new} \geq H \\
            \alpha_2^{new} & \text{if} \ L < \alpha_2^{new} < H \\
            L & \text{if} \ \alpha_2^{new} \leq L \\
        \end{cases}
\end{equation}

Finally, the value of $\alpha_1$ is computed from the new clipped $\alpha_2$ as:

\begin{equation} \label{eq:smo_svc_a1_new}
	\alpha_1^{new} = \alpha_1 + s (\alpha_2 - \alpha_2^{new,clipped})
\end{equation}

where $s = y_1 y_2$.

Since the \emph{Karush-Kuhn-Tucker} conditions are necessary and sufficient conditions for optimality of a positive definite QP problem and the KKT conditions for the classification problem~\eqref{eq:svc_min_wolfe_dual} are:

\begin{equation} \label{eq:svc_smo_kkt}
	\begin{aligned}
		\alpha_i = 0 & \Leftrightarrow y_i y'_i \geq 1 \\
		0 < \alpha_i < C & \Leftrightarrow y_i y'_i = 1 \\
		\alpha_i = C & \Leftrightarrow y_i y'_i \leq 1
	\end{aligned}
\end{equation}

the steps described above will be iterate as long as there will be an example that violates them.

After optimizing $\alpha_1$ and $\alpha_2$, we select the threshold $b$ such that the KKT conditions are satisfied for $x_1$ and $x_2$. If, after optimization, $\alpha_1$ is not at the bounds, i.e., $0 < \alpha_1 < C$, then the following threshold $b_{up}$ is valid, since it forces the SVC to output $y_1$ when the input is $x_1$:

\begin{equation} \label{eq:smo_svc_b1}
	b_{up} = E_1 + y_1 (\alpha_1^{new} - \alpha_1) K(x_1,x_1) + y_2 (\alpha_2^{new,clipped} - \alpha_2) K(x_1,x_2) + b
\end{equation}

similarly, the following threshold $b_{low}$ is valid if $0 < \alpha_2 < C$:

\begin{equation} \label{eq:smo_svc_b2}
	b_{low} = E_2 + y_1 (\alpha_1^{new} - \alpha_1) K(x_1,x_2) + y_2 (\alpha_2^{new,clipped} - \alpha_2) K(x_2,x_2) + b
\end{equation}

If, after optimization, both $0 < \alpha_1 < C$ and $0 < \alpha_2 < C$ then both these thresholds are valid, and they will be equal; else, if both $\alpha_1$ and $\alpha_2$ are at the bounds, i.e., $\alpha_1 = 0$ or $\alpha_1 = C$ and $\alpha_2 = 0$ or $\alpha_2 = C$, then all the thresholds between $b_{up}$ and $b_{low}$ satisfy the KKT conditions, so we choose the threshold to be halfway in between $b_{up}$ and $b_{low}$. This gives the complete equation for $b$:

\begin{equation} \label{eq:smo_svc_b}
	b =
        \begin{cases}
            b_{up} & \text{if} \ 0 < \alpha_1 < C \\
            b_{low} & \text{if} \ 0 < \alpha_2 < C \\
            \displaystyle \frac{b_{up}+b_{low}}{2} & \text{otherwise} \\
        \end{cases}
\end{equation}

\newpage

\begin{breakablealgorithm}
	\caption{Sequential Minimal Optimization for Classification}
	\label{alg:smo_classifier}
	\begin{algorithmic}
		\Require{Training examples matrix $X \in \Re^{n \times m}$}
		\Require{Training target vector $y \in \pm1^n$}
		\Require{Kernel matrix $K \in \Re^{n \times n}$}
		\Require{Regularization parameter $C > 0$}
		\Require{Tolerance value $tol$ for stopping criterion}
		\Function{SMOClassifier}{$X,y,K,C,tol$}
			\State Initialize the Lagrange multipliers vector $\alpha \in \Re^n, \alpha \gets 0$
			\State Initialize the empty set $I0 \gets \{i : 0 < \alpha_i < C\}$
			\State Initialize the set $I1 \gets \{i : y_i = +1, \alpha_i = 0\}$ to contain all the indices of the training examples of class $+1$
			\State Initialize the empty set $I2 \gets \{i : y_i = -1,	 \alpha_i = C\}$
			\State Initialize the empty set $I3 \gets \{i :  y_i = +1,	 \alpha_i = C\}$
			\State Initialize the set $I4 \gets \{i : y_i = -1, \alpha_i = 0\}$ to contain all the indices of the training examples of class $-1$
			\State Initialize $b_{up} \gets -1$
			\State Initialize $b_{low} \gets +1$
			\State Initialize the error cache vector $errors \in \Re^n, errors \gets 0$
			\While {$num\_changed > 0$ \OR $examine\_all = True$}
				\State $num\_changed \gets 0$
				\State $examine\_all \gets True$
				\If {$examine\_all = True$}
					\For {$i \gets 0$ to $n$} \Comment loop over all training examples
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
					\EndFor
				\Else
					\For {$i$ in $I0$} \Comment loop over examples where $\alpha_i$ are not already at their bounds
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
						\If {$b_{up} > b_{low} - 2 tol$} \Comment check if optimality on $I0$ is attained
							\State {$num\_changed \gets 0$}
							\Break
						\EndIf
					\EndFor
				\EndIf
				\If {$examine\_all = True$}
					\State $examine\_all \gets False$
				\ElsIf {$num\_changed = 0$}
					\State $examine\_all \gets True$
				\EndIf
			\EndWhile
			\State Compute $b$ by~\eqref{eq:smo_svc_b}
			\State \Return $\alpha,b$
		\EndFunction
	\end{algorithmic}

	\newpage

	\begin{algorithmic}
		\Require{$i2$-th Lagrange multiplier}
		\Function{ExamineExample}{$i2$}
			\If {$i2$ in $I0$}
				\State $E_2 \gets errors_{i2}$
			\Else
				\State Compute $E_2$
				\State $errors_{i2} \gets E_2$
				\State Update $(b_{low}, i_{low})$ or $(b_{up}, i_{up})$ using $(E_2, i2)$
			\EndIf
			\If {optimality is attained using current $b_{low}$ and $b_{up}$}
				\State \Return 0
			\Else
				\State Find an index $i1$ to do joint optimization with $i2$
				\If {$\Call{TakeStep}{i1,i2}$ = True}
					\State \Return 1
				\Else
					\State \Return 0
				\EndIf
			\EndIf
		\EndFunction
	\end{algorithmic}

	\newpage

	\begin{algorithmic}
		\Require{$i1$-th Lagrange multiplier}
		\Require{$i2$-th Lagrange multiplier}
		\Function{TakeStep}{$i1,i2$}
			\If {$i1 = i2$}
				\State \Return False
			\EndIf
			\State Compute $L$ and $H$ using~\eqref{eq:smo_svc_bounds_update1} or~\eqref{eq:smo_svc_bounds_update2}
			\If {$L = H$}
				\State \Return False
			\EndIf
			\State Compute $\eta$ by~\eqref{eq:smo_eta} \Comment we assume that $\eta > 0$, i.e., the kernel matrix $K$ is positive definite
			\If {$\eta < 0$}
				\State Choose $\alpha_2^{new,clipped}$ between $L$ and $H$ according to the largest value of the objective function at these points
			\Else
				\State Compute $\alpha_2^{new}$ by~\eqref{eq:smo_svc_a2_new}
				\State Compute $\alpha_2^{new,clipped}$ by~\eqref{eq:smo_svc_a2_new_clipped}
			\EndIf
			\If {changes in $\alpha_2^{new,clipped}$ are larger than some eps}
				\State Compute $\alpha_1^{new}$ by~\eqref{eq:smo_svc_a1_new}
				\State Update $\alpha_2^{new,clipped}$ and $\alpha_1^{new}$
				\For {$i$ in $I0$}
					\State Update $errors_i$ using new Lagrange multipliers
				\EndFor
				\State Update $\alpha$ using new Lagrange multipliers
				\State Update $I0, I1, I2, I3$ and $I4$
				\State Update $errors_{i1}$ and $errors_{i2}$
				\For {$i$ in $I0 \cup \{i1,i2\}$}
					\State Compute $(i_{low}, b_{low})$ by $b_{low} = \max\{errors_i : i \in I0 \cup I3 \cup I4\}$
					\State Compute $(i_{up}, b_{up})$ by $b_{up} = \min\{errors_i : i \in I0 \cup I1 \cup I2\}$
				\EndFor
				\State \Return True
			\Else
				\State \Return False
			\EndIf
		\EndFunction
	\end{algorithmic}
\end{breakablealgorithm}

\newpage

\subsubsection{Regression}

In case of regression the bounds and the new multipliers $\alpha_1^{+,new}$ and $\alpha_2^{+,new}$ can be expressed as follows if ($\alpha_1^+ > 0$ or ($\alpha_1^- = 0$ and $ E_1 - E_2 > 0$)) and ($\alpha_2^+ > 0$ or ($\alpha_2^- = 0$ and $ E_1 - E_2 < 0$)):

\begin{equation} \label{eq:smo_svr_bounds_update1}
	\begin{aligned}
		& L = max(0, \gamma - C) \\
		& H = min(C, \gamma)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new1}
	\alpha_2^{+,new} = \alpha_2^+ - \frac{E_1 - E_2}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new1}
	\alpha_1^{+,new} = \alpha_1^+ - (\alpha_2^{+,new,clipped} - \alpha_2^+)
\end{equation}

or, if ($\alpha_1^+ > 0$ or ($\alpha_1^- = 0$ and $ E_1 - E_2 > 2 \epsilon$)) and ($\alpha_2^- > 0$ or ($\alpha_2^+ = 0$ and $ E_1 - E_2 > 2 \epsilon$)):

\begin{equation} \label{eq:smo_svr_bounds_update2}
	\begin{aligned}
		& L = max(0, -\gamma) \\
		& H = min(C, -\gamma + C)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new2}
	\alpha_2^{-,new} = \alpha_2^- + \frac{(E_1 - E_2) - 2 \epsilon}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new2}
	\alpha_1^{+,new} = \alpha_1^+ + (\alpha_2^{-,new,clipped} - \alpha_2^-)
\end{equation}

or, if ($\alpha_1^- > 0$ or ($\alpha_1^+ = 0$ and $ E_1 - E_2 < - 2 \epsilon$)) and ($\alpha_2^+ > 0$ or ($\alpha_2^- = 0$ and $ E_1 - E_2 < - 2 \epsilon$)):

\begin{equation} \label{eq:smo_svr_bounds_update3}
	\begin{aligned}
		& L = max(0, \gamma) \\
		& H = min(C, C + \gamma)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new3}
	\alpha_2^{+,new} = \alpha_2^+ - \frac{(E_1 - E_2) + 2 \epsilon}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new3}
	\alpha_1^{-,new} = \alpha_1^- + (\alpha_2^{+,new,clipped} - \alpha_2^+)
\end{equation}

or, finally, if ($\alpha_1^- > 0$ or ($\alpha_1^+ = 0$ and $ E_1 - E_2 < 0$)) and ($\alpha_2^- > 0$ or ($\alpha_2^+ = 0$ and $ E_1 - E_2 > 0$)):

\begin{equation} \label{eq:smo_svr_bounds_update4}
	\begin{aligned}
		& L = max(0, -\gamma - C) \\
		& H = min(C, -\gamma)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new4}
	\alpha_2^{-,new} = \alpha_2^- + \frac{E_1 - E_2}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new4}
	\alpha_1^{-,new} = \alpha_1^- - (\alpha_2^{-,new,clipped} - \alpha_2^-)
\end{equation}

where $\gamma = \alpha_1^+ - \alpha_1^- + \alpha_2^+ - \alpha_2^-$. Note that $\eta$ and $\alpha_2^{+,new,clipped}$ or $\alpha_2^{-,new,clipped}$ are identical to~\eqref{eq:smo_eta} and~\eqref{eq:smo_svc_a2_new_clipped} respectively.

The KKT conditions for the regression problem~\eqref{eq:svr_min_wolfe_dual} are:

\begin{equation} \label{eq:svr_smo_kkt}
	\begin{aligned}
		\alpha_i^+ - \alpha_i^- = 0 & \Leftrightarrow | y_i - y'_i | < \epsilon \\
		-C < \alpha_i^+ - \alpha_i^- < C & \Leftrightarrow | y_i - y'_i | = \epsilon \\
		\alpha_i^+ + \alpha_i^- = C & \Leftrightarrow | y_i - y'_i | > \epsilon
	\end{aligned}
\end{equation}

so, the steps described above will be iterate as long as there will be an example that violates them.

In case of regression we select the threshold $b$ as follows:

\begin{equation} \label{eq:smo_svr_b1}
	b_{up} = E_1 + ((\alpha_1^+ - \alpha_1^-) - (\alpha_1^{+,new} - \alpha_1^{-,new})) K(x_1,x_1) + ((\alpha_2^+ - \alpha_2^-) - (\alpha_2^{+,new,clipped} - \alpha_2^{-,new,clipped})) K(x_1,x_2) + b
\end{equation}

\begin{equation} \label{eq:smo_svr_b2}
	b_{low} = E_2 + ((\alpha_1^+ - \alpha_1^-) - (\alpha_1^{+,new} - \alpha_1^{-,new})) K(x_1,x_2) + ((\alpha_2^+ - \alpha_2^-) - (\alpha_2^{+,new,clipped} - \alpha_2^{-,new,clipped})) K(x_2,x_2) + b
\end{equation}

\begin{equation} \label{eq:smo_svr_b}
	b =
        \begin{cases}
            b_{up} & \text{if} \ 0 < \alpha_1^+, \alpha_1^- < C \\
            b_{low} & \text{if} \ 0 < \alpha_2^+, \alpha_2^- < C \\
            \displaystyle \frac{b_{up}+b_{low}}{2} & \text{otherwise} \\
        \end{cases}
\end{equation}

\bigskip
\bigskip

The improvements described in~\cite{keerthi2001improvements, shevade1999improvements} for classification and regression respectively are about the definition of subsets of multipliers to efficiently update them at each iteration by separating the multipliers at the bounds from those who can be further minimized.

\newpage

\begin{breakablealgorithm}
	\caption{Sequential Minimal Optimization for Regression}
	\label{alg:smo_regression}
	\begin{algorithmic}
		\Require{Training examples matrix $X \in \Re^{n \times m}$}
		\Require{Training target vector $y \in \Re^n$}
		\Require{Kernel matrix $K \in \Re^{n \times n}$}
		\Require{Regularization parameter $C > 0$}
		\Require{Epsilon-tube value $\epsilon \geq 0$ within which no penalty is associated in the epsilon-insensitive loss function}
		\Require{Tolerance value $tol$ for stopping criterion}
		\Function{SMORegression}{$X,y,K,C,\epsilon,tol$}
			\State Initialize the Lagrange multipliers vector $\alpha^+ \in \Re^n, \alpha^+ \gets 0$
			\State Initialize the Lagrange multipliers vector $\alpha^- \in \Re^n, \alpha^- \gets 0$
			\State Initialize the empty set $I0 \gets \{i : 0 < \alpha^+_i, \alpha^-_i < C\}$
			\State Initialize the set $I1 \gets \{i : \alpha^+_i = 0, \alpha^-_i = 0\}$ to contain all the indices of the training examples
			\State Initialize the empty set $I2 \gets \{i : \alpha^+_i = 0, \alpha^-_i = C\}$
			\State Initialize the empty set $I3 \gets \{i : \alpha^+_i = C, \alpha^-_i = 0\}$
			\State Initialize $i_{up} \gets 0$ \Comment or any other target index $i_{up}$ from the training examples
			\State Initialize $i_{low} \gets 0$ \Comment or any other target index $i_{low}$ from the training examples
			\State Initialize $b_{up} \gets y_{i_{up}} + \epsilon$
			\State Initialize $b_{low} \gets y_{i_{low}} - \epsilon$
			\State Initialize the error cache vector $errors \in \Re^n, errors \gets 0$
			\While {$num\_changed > 0$ \OR $examine\_all = True$}
				\State $num\_changed \gets 0$
				\State $examine\_all \gets True$
				\If {$examine\_all = True$}
					\For {$i \gets 0$ to $n$} \Comment loop over all training examples
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
					\EndFor
				\Else
					\For {$i$ in $I0$} \Comment loop over examples where $\alpha^+_i$ and $\alpha^-_i$ are not already at their bounds
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
						\If {$b_{up} > b_{low} - 2 tol$} \Comment check if optimality on $I0$ is attained
							\State {$num\_changed \gets 0$}
							\Break
						\EndIf
					\EndFor
				\EndIf
				\If {$examine\_all = True$}
					\State $examine\_all \gets False$
				\ElsIf {$num\_changed = 0$}
					\State $examine\_all \gets True$
				\EndIf
			\EndWhile
			\State Compute $b$ by~\eqref{eq:smo_svr_b}
			\State \Return $\alpha^+,\alpha^-,b$
		\EndFunction
	\end{algorithmic}

	\newpage

	\begin{algorithmic}
		\Require{$i1$-th Lagrange multiplier}
		\Require{$i2$-th Lagrange multiplier}
		\Function{TakeStep}{$i1,i2$}
			\If {$i1 = i2$}
				\State \Return False
			\EndIf
			\State $finished = False$
			\While {\NOT $finished$}
				\State Compute $L$ and $H$ using~\eqref{eq:smo_svr_bounds_update1},~\eqref{eq:smo_svr_bounds_update2},~\eqref{eq:smo_svr_bounds_update3} or~\eqref{eq:smo_svr_bounds_update4}
				\If {$L < H$}
					\State Compute $\eta$ by~\eqref{eq:smo_eta} \Comment we assume that $\eta > 0$, i.e., the kernel matrix $K$ is positive definite
					\If {$\eta < 0$}
						\State Choose $\alpha_2^{+,new,clipped}$ or $\alpha_2^{-,new,clipped}$ between $L$ and $H$ according to the largest value of the objective function at these points
					\Else
						\State Compute $\alpha_2^{+,new}$ or $\alpha_2^{-,new}$ using~\eqref{eq:smo_svr_a2_new1},~\eqref{eq:smo_svr_a2_new3} or~\eqref{eq:smo_svr_a2_new2},~\eqref{eq:smo_svr_a2_new4} respectively
						\State Compute $\alpha_2^{+,new,clipped}$ or $\alpha_2^{-,new,clipped}$ by~\eqref{eq:smo_svc_a2_new_clipped}
					\EndIf
					\State Compute $\alpha_1^{+,new}$ or $\alpha_1^{-,new}$ using~\eqref{eq:smo_svr_a1_new1},~\eqref{eq:smo_svr_a1_new2} or~\eqref{eq:smo_svr_a1_new3},~\eqref{eq:smo_svr_a1_new4} respectively
					\If {changes in $\alpha_2^{+,new,clipped}, \alpha_2^{-,new,clipped}, \alpha_1^{+,new}$ or $\alpha_1^{-,new}$ are larger than some eps}
						\State Update $\alpha_2^{+,new,clipped}, \alpha_2^{-,new,clipped}, \alpha_1^{+,new}$ or $\alpha_1^{-,new}$
					\EndIf
				\Else
					\State $finished = True$
				\EndIf
			\EndWhile
			\If {changes in $\alpha_2^{+,new,clipped}, \alpha_2^{-,new,clipped}, \alpha_1^{+,new}$ or $\alpha_1^{-,new}$ are larger than some eps}
				\For {$i$ in $I0$}
					\State Update $errors_i$ using new Lagrange multipliers
				\EndFor
				\State Update $\alpha^+$ and $\alpha^-$ using new Lagrange multipliers
				\State Update $I0, I1, I2$ and $I3$
				\State Update $errors_{i1}$ and $errors_{i2}$
				\For {$i$ in $I0 \cup \{i1,i2\}$}
					\State Compute $(i_{low}, b_{low})$ by $b_{low} = \max\{errors_i : i \in I0 \cup I1 \cup I2\}$
					\State Compute and $(i_{up}, b_{up})$ by $b_{up} = \min\{errors_i : i \in I0 \cup I1 \cup I3\}$
				\EndFor
				\State \Return True
			\Else
				\State \Return False
			\EndIf
		\EndFunction
	\end{algorithmic}
\end{breakablealgorithm}

\pagebreak

\subsection{AdaGrad for Lagrangian dual formulations}

Due to the sparsity of the weight vector of the \emph{Lagrangian dual}, i.e., the Lagrange multipliers, we might end up in a situation where some components of the gradient are very small and others large. This, in terms of \emph{conditioning number}, i.e., $\kappa = L/\mu \gg 1$, means that the level sets of $f$ are ellipsoid, i.e., we are dealing with an ill-conditioned problem. So, given a learning rate, a standard gradient descent approach might end up in a situation where it decreases too quickly the small weights or too slowly the large ones.

Another method, that is usually deprecated in ML applications due to its increased computational complexity, is Newton’s method. Newton’s method favors a much faster \emph{convergence rate}, i.e., number of iterations, at the cost of being more expensive per iteration. For convex problems, the recursion is similar to the gradient descent algorithm:

$$
x_{t+1} = x_t - \alpha H^{-1} \nabla f(x_t)
$$

where $\alpha$ is often close to one (damped-Newton) or one, and $H^{-1}$ denotes the Hessian of $f$ at the current point, i.e., $\nabla^2 f(x_t)$.

The above suggest a general rule in optimization: find any preconditioner, in convex optimization it has to be positive semidefinite, that improves the performance of gradient descent in terms of iterations, but without wasting too much time to compute that precoditioner. The above result into:

$$
x_{t+1} = x_t - \alpha P^{-1} \nabla f(x_t)
$$

where $P$ is the preconditioner. This idea is the basis of the BFGS quasi-Newton method.

The \emph{AdaGrad}~\cite{duchi2011adaptive} algorithm is just a variant of preconditioned gradient descent, where $P$ is selected to be a diagonal preconditioner matrix and is updated using the gradient information, in particular it is the diagonal approximation of the inverse of the square roots of gradient outer products, until the $k$-th iteration. The above lead to the algorithm:

\begin{algorithm}[H]
	\caption{AdaGrad}
	\label{alg:adagrad}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Require{Offset $\epsilon > 0$ to ensures not divide by 0}
		\Function{AdaGrad}{$f,\alpha,\epsilon$}
			\State Initialize weight vector $x_0$ and the squared accumulated gradients vector $s_t \gets 0$
			\State $t = 1$
			\While {$not\_convergence$}
				\State $g_t \gets \partial f(x_t)$ \Comment if $f$ is differentiable then $\partial f(x_t) = \nabla f(x_t)$
				\State $s_t \gets s_{t-1} + g_t^2$
				\State $x_{t+1} \gets x_t - \alpha P_t^{-1} g_t = x_t - \displaystyle \frac{\alpha}{\sqrt{s_t + \epsilon}} \odot g_t \ \text{where} \ P_t \gets diag(s_t + \epsilon)^{1/2}$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

In practical terms, \emph{AdaGrad} addresses the problem of the sparse optimal by adaptively scaling the learning rate for each dimension with the magnitude of the gradients. Coordinates that routinely correspond to large gradients are scaled down significantly, whereas others with small gradients receive a much more gentle treatment.

\pagebreak

\subsection{Losses properties}

Several losses and objectives have been presented in section~\ref{section:svc} and~\ref{section:svr}. In our experiments, we will consider the following.

For what about the loss functions, two of them are nonsmooth convex functions, i.e., the \emph{hinge} and the \emph{epsilon-insensitive} losses for \emph{classification} and \emph{regression} tasks respectively, and linearly penalizes the misclassified points, i.e., $\mathcal{L}_1$-SVM, meanwhile, their two \emph{squared} versions are smooth, i.e., $\mathcal{L}_2$-SVM, and quadratically penalizes the misclassified points.

% http://juliaml.github.io/LossFunctions.jl/stable/losses/distance/#LPDistLoss-1

% http://juliaml.github.io/LossFunctions.jl/stable/losses/margin/#L1HingeLoss-1
% http://juliaml.github.io/LossFunctions.jl/stable/losses/margin/#SmoothedL1HingeLoss-1
% http://juliaml.github.io/LossFunctions.jl/stable/losses/margin/#ModifiedHuberLoss-1

% http://juliaml.github.io/LossFunctions.jl/stable/losses/distance/#L1EpsilonInsLoss-1
% http://juliaml.github.io/LossFunctions.jl/stable/losses/distance/#L2EpsilonInsLoss-1

Also, both the \emph{margin-based} losses, i.e., the \emph{hinge} and the \emph{squared hinge} losses, are $L_f$-Lipschitz continuous; meanwhile, among the \emph{distance-based} losses, the \emph{epsilon-insensitive} loss is $L_f$-Lipschitz continuous but the \emph{squared epsilon-insensitive} is not $L_f$-Lipschitz continuous, however it is convex and for this reason is locally $L_f$-Lipschitz continuous.

Also the regularization term, i.e., $\displaystyle \frac{1}{2} \| w \|^2$, is not $L_f$-Lipschitz continuous since it becomes arbitrarily steep as $w$ approaches infinity, but it is strictly convex and for this reason is locally $L_f$-Lipschitz continuous. Clearly, its gradient, i.e., $w$, is not bounded since, again, they go to infinity as $w$ goes to infinity, so this function is not $L$-Lipschitz continuous.

Since for our purposes, we need to show that our $\mathcal{L}_1$-SVM objectives are $L_f$-Lipschitz continuous and the $\mathcal{L}_2$-SVM objectives are $L$-Lipschitz continuous for the applicability of the convergence theorems, we will use the theorem~\ref{def:l_f_lipschitz_convex} and~\ref{def:strong_cvx_l_lipschitz_convex} respectively.

In general, if the objective function of a quadratic programming problem is strictly convex, i.e., the associated Hessian matrix is positive definite, the solution is unique. Meanwhile, if the objective function is convex, there may be cases where the solution is nonunique.

Assume that the hard margin SVM has a solution, i.e., the given problem is separable in the feature space. Then, since the objective function of the primal problem is $\displaystyle \frac{1}{2} \| w \|^2$, which is strongly convex, the primal problem has a unique solution for $w$ and $b$.

Since the $\mathcal{L}_1$-SVM linearly penalizes the misclassified points, the primal objective function is convex. Likewise, the Hessian matrix of the dual objective function is positive semidefinite. Thus the primal and dual solutions may be nonunique. Meanwhile, the objective function of the primal problem for the $\mathcal{L}_2$-SVM is strictly convex, due to the quadratic penalization of the misclassified points. Therefore, $w$ and $b$ are uniquely determined if we solve the primal or dual problem.

In summary, the following properties for the SVM's objectives are given:

% http://www.lib.kobe-u.ac.jp/repository/90000231.pdf
% https://hal.archives-ouvertes.fr/hal-00708243v1/file/appendixMNBCI.pdf

\begin{table}[H]
\centering
\caption{SVM's objectives properties for primal formulations}
\label{primal_svm_objectives_props}
\begin{tabular}{lrrr}
\toprule
	& smooth & \vtop{\hbox{\strut Lipschitz}\hbox{\strut continuous}} & convexity \\
objective & 		& 		& 		\\
\midrule
$\mathcal{L}_1$-SVC~\eqref{eq:reg_bias_primal_l1_svc} & no & $L_f$-Lipschitz & convex \\
$\mathcal{L}_2$-SVC~\eqref{eq:reg_bias_primal_l2_svc} & yes & L-Lipschitz & \vtop{\hbox{\strut 1-strongly}\hbox{\strut convex}} \\
\midrule
$\mathcal{L}_1$-SVR~\eqref{eq:reg_bias_primal_l1_svr} & no & $L_f$-Lipschitz & convex \\
$\mathcal{L}_2$-SVR~\eqref{eq:reg_bias_primal_l2_svr} & yes & L-Lipschitz & \vtop{\hbox{\strut 1-strongly}\hbox{\strut convex}} \\
\bottomrule
\end{tabular}
\end{table}

And, according to the theoretical analysis, the following \emph{convergence rates} are given for the primal and \emph{Lagrangian dual} formulations respectively:

% https://homepage.cs.uiowa.edu/~tyng/acml15-tutorial.pdf

\begin{table}[H]
\centering
\caption{SVM's objectives convergence rates for primal formulations}
\label{primal_svm_objectives_rates}
\begin{tabular}{lrrr}
\toprule
	& \vtop{\hbox{\strut SGD}\hbox{\strut convergence rate}} & \vtop{\hbox{\strut Polyak SGD}\hbox{\strut convergence rate}} & \vtop{\hbox{\strut Nesterov SGD}\hbox{\strut convergence rate}} \\
objective & 		& 		& 		\\
\midrule
$\mathcal{L}_1$-SVM~(\ref{eq:reg_bias_primal_l1_svc},~\ref{eq:reg_bias_primal_l1_svr}) & $\displaystyle \mathcal{O}\Bigg(\frac{m}{\sqrt{t}}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{m}{\sqrt{t}}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{m}{\sqrt{t}}\Bigg)$ \\
$\mathcal{L}_2$-SVM~(\ref{eq:reg_bias_primal_l2_svc},~\ref{eq:reg_bias_primal_l2_svr}) & $\displaystyle \mathcal{O}\Bigg(\frac{m}{t}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{m}{t}\Bigg)$ & $\displaystyle \mathcal{O}\Bigg(\frac{m}{t^2}\Bigg)$ \\
\bottomrule
\end{tabular}
\end{table}

\begin{table}[H]
\centering
\caption{SVM's objectives convergence rate for Lagrangian dual formulations}
\label{dual_svm_objectives_props}
\begin{tabular}{lr}
\toprule
	& \vtop{\hbox{\strut AdaGrad}\hbox{\strut convergence rate}} \\
objective & 		\\
\midrule
$\mathcal{L}_1$-SVM~(\ref{eq:l1_svc_aug_lagrangian_dual},~\ref{eq:l1_svr_aug_lagrangian_dual}) or~(\ref{eq:l1_svc_bcqp_aug_lagrangian_dual},~\ref{eq:l1_svr_bcqp_aug_lagrangian_dual}) & $\displaystyle \mathcal{O}\Bigg(\frac{nm}{\sqrt{t}}\Bigg)$ \\
$\mathcal{L}_2$-SVM~(\ref{eq:l2_svc_aug_lagrangian_dual},~\ref{eq:l2_svr_aug_lagrangian_dual}) or~(\ref{eq:l2_svc_lb_aug_lagrangian_dual},~\ref{eq:l2_svr_lb_aug_lagrangian_dual}) & $\displaystyle \mathcal{O}\Bigg(\frac{nm}{t}\Bigg)$ \\
\bottomrule
\end{tabular}
\end{table}


% Copyright (C) He Guanyuming 2020
% The file is licensed under the MIT license.

\section{Mathematical Logic}

\subsection{Mathematical Statements}

\declareexercise{a.1.1}
It is ( (both $X, Y$ are false) or (both $X, Y$ are true) ).

\declareexercise{a.1.2}
It is ( ($Y$ can be true even if $X$ is false) or ($Y$ can be false even if $X$ is true) ).

\declareexercise{a.1.3}
Yes. That's the definition of logical equivalent.

\declareexercise{a.1.4}
No. It is still possible that (even if $X$ is false, $Y$ is still true).

Consider a statement $Y$ that satisfies:
\begin{enumerate}
\item If $X$, then $Y$.
\item If $X$ is false, then $Y$ or (exclusively) $Y$ is false.
\end{enumerate}

$X,Y$ satisfy the description in the exercise, but they are not logical equivalent.

\declareexercise{a.1.5}
Yes. (Now I'm using the symbols defined in the A.2 for the sake of simplification)
$X \Longleftrightarrow Y$ means $X \Longrightarrow Y \wedge \neg X \Longrightarrow \neg Y$. So does $Y$ and
$Z$. So
\begin{align*}
(X \Longrightarrow Y \Longrightarrow Z \wedge \neg X \Longrightarrow \neg Y \Longrightarrow \neg Z)
&\Longrightarrow \\
(X \Longrightarrow Z \wedge \neg X \Longrightarrow \neg Z)
\end{align*}
, which means $X$ and $Z$ are logical equivalent.

(Note that $A \Longrightarrow B$ can also be interpreted as a statement, meaning ``If $A$ is true, then $B$
is true'', just like we did in this example.)

\declareexercise{a.1.6}
Yes. $(X \Longrightarrow Y \Longrightarrow Z) \Longrightarrow (X \Longrightarrow Z)$.

Now we are proving that
$Z \Longrightarrow X \equiv \neg X \Longrightarrow \neg Z$. Assume that $\neg X \wedge Z$. Since
$Z \Longrightarrow X$, we have a contradiction: $X \wedge \neg X$.

So $X \Longrightarrow Z \wedge \neg X \Longrightarrow \neg Z$. Therefore, $X,Z$ are logical equivalent.
Besides, we can conclude that $Y \Longrightarrow X$. Thus $X,Y$ are also logical equivalent.

\subsection{Implication}
Why did Tao say
\begin{quotation}
If $X$, then $Y$ can also be written as ``$X$ can only be true when $Y$ is true''
\end{quotation}?

Assume the $X \wedge \neg Y$, but $X \Longrightarrow Y$. So we have a contradiction
$Y \wedge \neg Y$.

Define ``when $x \neq 2$, $X:x=2 \Longrightarrow x^2=4$ is vacuously true'' to ensure that $X$ is
always true regardless of the value of $x$.

\paragraph{My Own Exercise}
Most of the time, rules of implication are intuitive. But they can be confusing some times. So hereby I
introduce an example which I encountered, and which has confused me for a short time.

\begin{prop}
Let $P,Q,R$ be statements, thus
\[
P \Longrightarrow (Q \Longrightarrow R) \equiv (P \wedge Q) \Longrightarrow R
\]
\end{prop}
\begin{proof}
In order to ascertain that two statements in the form of implication are logically equivalent, we must
deeply understand what they are. At one time (that is, when all variables have definite value), a
statement can only be either true or false, not both. And for a statement in the form of implication:
$X \Longrightarrow Y$, it is true iff (\emph{If} $X$, then $Y$). We do not need to check it if $X$ is
not true.

Now back to the subject. To prove that the two are logically equivalent, we need to show that both
(if the former is true, then the latter is true) and (if the latter is true, then the former is true).

Now suppose that $P \Longrightarrow (Q \Longrightarrow R)$ is true. That is, if $P$, then
(if $Q$, then $R$). To show that under this condition the latter is true, we need to show that if $P,Q$
are both true, then $R$ is true. Suppose that $P \wedge Q$. Since $P$ is true, (if $Q$, then $R$) is
true. And we know that $Q$ is true, so $R$ is true. so the latter is true.

Now suppose that the latter is true. We need to verify that the former is also true under this
condition. Suppose $P$ is true, then we need to show $Q \Longrightarrow R$ is true, that is, if $Q$,
then $R$, and we furthermore suppose that $Q$ is true. Now $P \wedge Q$ is true, so we have $R$ is true.
\end{proof}

\subsection{Nested Quantifiers}
\declareexercise{a.5.1}
\textbf{(a)} Let $P$ be $y^2=x$ is true for each positive number $y$. And this statement means $P$ is
true for each positive number $x$.

\emph{Gaming metaphor}: Me and my friend each randomly pick up a positive, say $x$ and $y$, and check
if $y^2=x$.

The statement is false.

\textbf{(b)} There is at least one positive number $x$ such that for every positive number $y$,
$y^2=x$.

\emph{Gaming metaphor}: I have to pick up a positive number $x$ such that whatever positive number $y$
my friend picks up, $y^2=x$ is always true.

The statement is false.

\textbf{(c)} There is at least two positive numbers $x,y$ such that $y^2=x$.

\emph{Gaming metaphor}: Me and my friend each have to pick up a positive number, say $x$ and $y$, such
that $y^2=x$.

The statement is true. For example, $1^2=1$.

\textbf{(d)} The statement $\exists x > 0, y^2=x$ is true for every $y>0$.

\emph{Gaming metaphor}: For each positive number $y$ my friend picks up, I have to pick up a positive
number $x$ such that $y^2=x$.

The statement is true, because $y^2$ is also positive.

\textbf{(e)} There is at least one positive number $y$ such that for every positive number $x$, $y^2=x$
is always true.

\emph{Gaming metaphor}: I have to find a number $y>0$ such that regardless of what number $x$ my friend
picks up, $y^2=x$ is always true.

The statement is false.

\subsection{Equality}
\declareexercise{a.7.1}
\begin{proof}
Let $F(x) := x+c$. By axiom 4, $F(a)=F(b)$. That is, $a+c=b+c$. Similarly, by letting $G(x) := a+x$,
we have $a+c=a+d$, which, according to axiom 2, becomes $a+d=a+c$. Now we have $a+d=a+c, a+c=b+c$.
According to axiom 3, we can conclude that $a+d=b+c$.
\end{proof}


\chapter{Integral}

\section{Measure}

A \emph{Jordan content} is...

\index{content (measure theory)}%
A \emph{content} \(m : 2^A \to \Real\) satisfies
\begin{enumerate*}[label={(\arabic*)}]
    \item \(\forall x \in A : m~x \ge 0\),
    \item \(m~\emptyset = 0\), and
    \item \(\forall X \subseteq A, Y \subseteq A, X \cap Y = \emptyset : m~(X \cup Y) = m~X + m~Y\).
\end{enumerate*}

\index{measure}%
\index{countably additive}%
\index{countable additivity}%
A \emph{measure} \(m\) on a set \(A\) is a content that is \emph{countably additive}.

\index{Lebesgue measure}%
\index{measure!Lebesgue}%
A \emph{Lebesgue measure} for \(\Real\) is
\(m~(a,b) = m~[a,b) = m~(a,b] = m~[a,b] = |b-a|\).
From a measure \(m : \Real \to \Real\)
we can define a measure \(\mu : \Real^n \to \Real\)
where
\(\mu \left( \prod_{i=1}^n X_i \right) = \prod_{i=1}^n m(X_i)\).
\index{measure space}%
\index{measure!space}%
\index{space!measure}%
A \emph{measure space} is a set and a measure on that set.

\index{measurable function}%
\index{function!measurable}%
A \emph{measurable function} is ...

\section{Integral}

\index{integral}%
\index{integral!Lebesgue}%
\index{integral!Riemann}%
\index{Lebesgue integral}%
\index{Riemann integral}%
Let \((\Real,m)\) be a measure space and \(f\) be a function.
Let \(X \subseteq \Real\) and \(Y = \{ f(x) ~|~ x \in X\}\).
Let \(X_1,\ldots,X_n\) be a partitioning of \(X\) and \(Y_1,\ldots,Y_n\) be a partitioning of \(Y\).
For each \(k\), let \(x_k \in X_k\) and \(y_k \in Y_k\).
An \emph{integral of \(f\) in \(X\)}
is \(\int_X f = \lim_{n\to\infty} \sum_{k=1}^{n} a_k\)
where each \(a_k = m(X_k) \cdot m(Y_k)\) is a rectangular part of the total area.
See Table \ref{tab:integral}.

\begin{table}[h]
    \caption{How integrals partition spaces}
    \label{tab:integral}
    \centering
\begin{tabular}{lll}
    Name & \(X_k\) & \(Y_k\)
    \\
    \hline
    Riemann integral & \( [x_k,x_{k+1}]\) & \( [0,f(x_k)]\)
    \\
    Stieltjes integral with respect to \(g\) & \( [g(x_k),g(x_{k+1})]\) & \( [0,f(x_k)]\)
    \\
    Lebesgue integral & \( \{x ~|~ f(x) \in Y_k\}\) & \( [y_k,y_{k+1}]\)
\end{tabular}
\end{table}

\section{Calculus}

\index{fundamental theorem of calculus}%
\emph{Fundamental theorem of calculus}:
Iff \(D(F) = f\), then \(I(f) = F + c\) where \(c\) is a constant function,
\(D\) is the derivative operator, \(I\) is the antiderivative operator.


\section*{Exercises}

\begin{ex}
  Encrypt the message ``Rendezvous at dawn'' using the Hill cipher with
  block size 3 and encryption matrix
  \begin{equation*}
    A ~=~ \begin{mymatrix}{ccc}
      2 & 1 & 1 \\
      1 & 3 & 1 \\
      1 & 1 & 4 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    Ciphertext: $\q{ZRUJPZVAEJTWOXGJZV}$.
  \end{sol}
\end{ex}

\begin{ex}
  Decrypt the message $\q{ERM DXYBJUWW.JWQLD,HL}$ using the Hill
  cipher with block size $3$ and encryption matrix
  \begin{equation*}
    A ~=~ \begin{mymatrix}{ccc}
      2 & 1 & 1 \\
      1 & 3 & 1 \\
      1 & 1 & 4 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    The decryption matrix is
    \begin{equation*}
      A^{-1} ~=~ \begin{mymatrix}{ccc}
        16 & 22 &  5 \\
        22 & 26 & 17 \\
        5  & 17 &  2 \\
      \end{mymatrix}.
    \end{equation*}
    Plaintext: ``Spies are at the gate''.
  \end{sol}
\end{ex}

\begin{ex}
  Eve intercepts the following encrypted message sent by Bob:
  \begin{center}
    \q{TGVXKHGSW,JU,JHYJSCDSBQIRPEV}
  \end{center}
  Eve knows that Alice uses a Hill cipher with block length 2, but she
  does not know the secret encryption matrix. Eve also knows that
  Bob begins all of his letters with ``Hello''. Decrypt the message.
  \begin{sol}
    The first two plaintext blocks are $(8,5),(12,12)$ and the first
    two ciphertext blocks are $(20,7),(22,24)$. Eve solves the equation
    \begin{equation*}
      A^{-1} \begin{mymatrix}{cc} 20 & 22 \\ 7 & 24 \end{mymatrix}
      = \begin{mymatrix}{cc} 8 & 12 \\ 5 & 12 \end{mymatrix}
    \end{equation*}
    to find the secret decryption matrix
    \begin{equation*}
      A^{-1} = \begin{mymatrix}{cc} 3 & 5 \\ 1 & 2 \end{mymatrix}.
    \end{equation*}
    The plaintext is ``Hello, password is kiwifruit''.
  \end{sol}
\end{ex}

\begin{ex}
  Encrypt the message ``Lost contact'' using the block cipher of
  Definition~\ref{def:toy-block-cipher} and the key
  $2,3,4,1,1,1,5,5,5,4,3,2$.
  \begin{sol}
    Ciphertext: $\q{VXUAJYY,AFJE}$.
  \end{sol}
\end{ex}

\begin{ex}
  Decrypt the message $\q{NRQEUAPOM GLFN}$, using the block cipher of
  Definition~\ref{def:toy-block-cipher} and the key
  $1,1,1,2,2,2,3,3,3,4,4,4$. \textbf{Hint:} To decrypt, we must
  perform all the encryption steps in reverse. To undo a key mixing
  step, we subtract the relevant key components. To undo an S-box
  application, we apply the S-box in reverse. To undo a diffusion
  step, multiply by the inverse of the diffusion matrix.
  \begin{sol}
    The inverse of the diffusion matrix is
    \begin{equation*}
      A^{-1} = \begin{mymatrix}{ccc}
        11 & 2 & 21 \\
        21 & 11 & 2 \\
        2 & 21 & 11 \\
      \end{mymatrix}.
    \end{equation*}
    The plaintext is: ``Eat more fruit''.
  \end{sol}
\end{ex}


\subsection{Partial fraction decomposition}

We have:
\(\dfrac{1}{A.B}\)

We want this in the form of:

\(\dfrac{a}{A}+\dfrac{b}{B}\)

First, lets define \(M\) as the mean of these two numbers, and define \(\delta=M-B\). Then:

\(\dfrac{1}{AB}=\dfrac{1}{(M+\delta)(M-\delta)}=\dfrac{a}{M+\delta}+\dfrac{b}{M-\delta}\)

We can rearrange the latter two to find:

\(1=a(M-\delta)+b(M+\delta)\)

Now we need to find values of \(a\) and \(b\) to choose.

Let's examine \(a\).

\(a=\dfrac{1-b(M+\delta)}{M-\delta}\)

\(a=-\dfrac{bM+b\delta -1}{M-\delta}\)

\(a=-\dfrac{bM+b\delta -1}{M-\delta}\)

For this to divide neatly we need both the numerator to be a constant multiplier of the denominator. This means the ratio the multiplier for the left hand side of the denominator is equal to the right:

\(\dfrac{bM}{M}=\dfrac{b\delta -1}{-\delta}\)

\(b=\dfrac{b\delta -1}{-\delta}\)

\(b=\dfrac{1}{2\delta}\)

We can do the same for \(a\).

\(a=-\dfrac{1}{2\delta}\)

We can plug these back into our original formula:

\(\dfrac{1}{(M+\delta)(M-\delta)}=\dfrac{-\dfrac{1}{2\delta}}{M+\delta}+\dfrac{\dfrac{1}{2\delta}}{M-\delta}\)

\(\dfrac{1}{(M+\delta)(M-\delta)}=\dfrac{1}{2\delta}[\dfrac{1}{M-\delta}-\dfrac{1}{M+\delta}]\)


\documentclass{article}
\usepackage{amsmath}
\usepackage[margin=1.0in]{geometry}
\usepackage{xcolor}

\begin{document}

\noindent
Does $\displaystyle \sum_{n=1}^\infty \frac{n}{n!}$
diverge, converge absolutely, or converge conditionally?

\subsection*{Solution}

\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \frac{n+1}{(n+1)!} \cdot\frac{n!}{n} \right|\\
&= \lim_{n \to \infty} \left| \frac{n+1}{(n+1) \cdot n!} \cdot\frac{n!}{n} \right|\\
&= \lim_{n \to \infty} \left| \frac1n \right|\\
&= \lim_{n \to \infty}  \frac1n \\
&= 0
\end{align*}
Since $L < 1$, by the Ratio Test, the series $\displaystyle \sum_{n=1}^\infty \frac{n}{n!}$ converges absolutely.

\end{document}%%%%%%%%%%%%%%%%%

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \right|}\\
\end{align*}

\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \right|\\
\end{align*}

\begin{align*}
\lim_{n \to \infty} a_n
&= \lim_{n \to \infty} \\
\end{align*}

Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty AAAAAAAAAAAAAA$ converges absolutely.

Since $|r| < 1$, the series ...  converges by the Geometric Series Test.

Since $|r| \geq 1$, the series ...  diverges by the Geometric Series Test.

The function $f(x)=\frac{}{}$ is continuous, positive, and decreasing on $[1,\infty)$.

\subsection*{Solution}


\lab{Eigenvalue Solvers And Markov Chains}{Eigenvalue Solvers And Markov Chains}
\objective{Implement the power method and QR algorithm for finding eigenvalues, and use the power method to find the stationary distributions of Markov chains.}
\label{lab:EigSolve}

\section*{Computing eigenvalues}
The eigenvalues of a matrix are the roots of its characteristic polynomial.
Thus, to find the eigenvalues of an $n \times n$ matrix, we must compute the roots of a degree-$n$ polynomial.
This is easy for small $n$.
For example, if $n=2$ the quadratic equation can be used to find the eigenvalues.
However, Abel's Impossibility Theorem says that no such formula exists for the roots of a polynomial of degree 5 or larger.

\begin{theorem}[Abel's Impossibility Theorem]
There is no general algebraic solution for solving a polynomial equation of degree $n\geq5$.
\label{thm:Abel}
\end{theorem}

Thus, it is impossible to write an algorithm that will exactly find the eigenvalues of an arbitrary matrix.
(If we could write such an algorithm, we could also use it to find the roots of polynomials, contradicting Abel's theorem.)
This is a significant result.
It means that we must find eigenvalues with \emph{iterative methods}, methods that generate sequences of approximate values converging to the true value.

\subsection*{The power method}
There are many iterative methods for finding eigenvalues.
The power method finds an eigenvector corresponding to the \emph{dominant} eigenvalue of a matrix, if such an eigenvalue exists.
The dominant eigenvalue of a matrix is the unique eigenvalue of greatest magnitude.

To use the power method on a matrix $A$, begin by choosing a vector $\x_0$ such that $\|\x_0\|=1$. Then recursively define
\[
x_{k+1}=\frac{Ax_k}{\norm{Ax_k}}.
\]
If
\begin{itemize}
\item $A$ has a dominant eigenvalue $\lambda$, and
\item the projection of $\x_0$ into the subspace spanned by the eigenvectors corresponding to $\lambda$ is nonzero,
\end{itemize}
then the vectors $\x_0, \x_1, \x_2, \ldots$ will converge to an eigenvector of $A$ corresponding to $\lambda$.
(See [TODO: ref textbook] for a proof when $A$ is semisimple, or [TODO: ref something else] for a proof in the general case.)

If all entries of $A$ are positive, then $A$ will always have a dominant eigenvalue (see [TODO: ref something!] for a proof).
There is no way to guarantee that the second condition is met, but if we choose $\x_0$ randomly, it will almost always satisfy this condition.

Once you know that $\x$ is an eigenvector of $A$, the corresponding eigenvalue is equal to the \emph{Raleigh quotient}
\[
\lambda = \frac{\langle Ax, x \rangle}{\|\x\|^2}.
\]

\begin{problem}
Write a function that implements the power method to compute an eigenvector. Your function should
\begin{enumerate}
\item Accept a matrix and a tolerance \li{tol}.
\item Start with a random vector.
\item Use the 2-norm wherever a norm is needed (use \li{la.norm()}).
\item Repeat the power method until the vector changes by less than the tolerance. In mathematical notation, you are defining $x_0, x_1, \ldots x_k$, and your function should stop when $\|x_{k+1}-x_k\| < \text{tol}$.
\item Return the found eigenvector and the corresponding eigenvalue (use \li{np.inner()}).
\end{enumerate}
Test your function on positive matrices.
\end{problem}

\begin{comment}
An overview of the proof of the method is that you can write a matrix in Jordan Conical form $A=VJV^{-1}$ where $V$ is the matrix of the generalized eigenspaces.
But the first column is is the eigenvector corresponding to largest eigenvalue and $J$ is a upper trianglar matrix of eigenvalues and ones.
Note that $A^k=VJ^kV^{-1}$. The limit as $k \rightarrow \infty$ of $(\frac{1}{\lambda_1}J)^k$ is a matrix of all zeros except for a one in the upper right hand corner.
So $(\frac{A}{\norm{A}})^k \approx VJ^kV^{-1}$ So the largest eigenvalue dominates.
\end{comment}

\subsection*{The QR algorithm}
The disadvantage of the power method is that it only finds the largest eigenvector and a corresponding eigenvalue.
To use the QR algorithm, let $A_0=A$. Then let $Q_kR_k$ be the QR decomposition of $A_k$, and recursively define
\[
A_{k+1}=R_kQ_k.
\]
Then $A_0, A_1, A_2, \ldots $ will converge to a matrix of the form
\begin{equation*}
\label{eq:Schur form}
S =
     \begin{pmatrix}
          S_1 &* & \cdots & * \\
           0     &S_2  &  \ddots & \vdots \\
           \vdots  & \ddots & \ddots & *  \\
           0 & \cdots & 0 & S_m
    \end{pmatrix}
\end{equation*}
where $S_i$ is a $1\times1$ or $2\times2$ matrix.\footnote{If $S$ is upper triangular (i.e., all $S_i$ are $1\times1$ matrices), then $S$ is the \emph{Schur form} of $A$.
If some $S_i$ are $2\times2$ matrices, then $S$ is the \emph{real Schur form} of $A$.}
The eigenvalues of $A$ are the eigenvalues of the $S_i$.

This algorithm works for three reasons. First,
\[
Q_k^{-1}A_kQ_k = Q_k^{-1}(Q_kR_k)Q_k = (Q_k^{-1}Q_k)(R_kQ_k) = A_{k+1},
\]
so $A_k$ is similar to $A_{k+1}$.
Because similar matrices have the same eigenvalues, $A_k$ has the same eigenvalues as $A$.
Second, each iteration of the algorithm transfers some of the ``mass'' from the lower to the upper triangle.
This is what makes $A_0, A_1, A_2, \ldots$ converge to a matrix $S$ which has the described form.
Finally, since $S$ is block upper triangular, its eigenvalues are just the eigenvalues of its diagonal blocks (the $S_i$).

A $2 \times 2$ block will occur in $S$ when $A$ is real but has complex eigenvalues.
In this case, the complex eigenvalues occur in conjugate pairs, each pair corresponding to a $2 \times 2$ block on the diagonal of $S$.

\subsubsection*{Hessenberg preconditioning}
Often, we ``precondition'' a matrix by putting it in upper Hessenberg form before passing it to the QR algorithm.
This is always possible because every matrix is similar to an upper Hessenberg matrix (see Lab \ref{}).
Hessenberg preconditioning is done for two reasons.

First, the QR algorithm converges much faster on upper Hessenberg matrices because they are already close to triangular matrices.

Second, an iteration of the QR algorithm can be computed in $\mathcal{O}(n^2)$ time on an upper Hessenberg matrix, as opposed to $\mathcal{O}(n^3)$ time on a regular matrix.
This is because so many entries of an upper Hessenberg matrix are 0.
If we apply the QR algorithm to an upper Hessenberg matrix $H$, then this speed-up happens in each iteration of the algorithm, since if $H = QR$ is the QR decomposition of $H$ then $RQ$ is also upper Hessenberg.

\begin{problem}
Write a function that implements the QR algorithm with Hessenberg preconditioning as described above.
Do this as follows.
\begin{enumerate}
\item Accept a matrix \li{A}, a number of iterations \li{niter}, and a tolerance \li{tol}
\item Put \li{A} in Hessenberg form using \li{la.hessenberg()}.
\item Compute the matrix $S$ by performing the QR algorithm \li{niter} times.
Use the function \li{la.qr()} to compute the QR decomposition.
\item Iterate through the diagonal of $S$ from top to bottom to compute its eigenvalues.
For each diagonal entry,
\begin{enumerate}
\item If this is the last diagonal entry, then it is an eigenvalue.
\item If the entry below this one has absolute value less than \li{tol}, assume this is a $1\times 1$ block.
Then the current entry is an eigenvalue.
\item Otherwise, the current entry is at the top left corner of a $2 \times 2$ block.
Calculate the eigenvalues of this block.
Use the \li{sqrt} function from the scimath library to find the square root of a negative number.
You can import this library with the line \li{from numpy.lib import scimath}.
\end{enumerate}
\item Return the (approximate) eigenvalues of \li{A}.
\end{enumerate}
You can check your function on the matrix
\[
\begin{pmatrix}
 4 &  12 & 17 &  -2 \\
-5.5& -30.5 & -45.5 &  9.5\\
 3. &  20. & 30. &  -6. \\
1.5 &  1.5&   1.5&   1.5
       \end{pmatrix},
\]
which has eigenvalues $1+2i, 1-2i, 3$, and 0. You can also check your function on random matrices against \li{la.eig()}.
\label{prob:qr_solver}
\end{problem}

\begin{comment}
\begin{problem}
\label{prob:QR_eig_hessenberg}
Write a version of the QR algorithm that performs the QR algorithm by computing the Hessenberg form of a matrix, then computing various QR decompositions of the Hessenberg form of the matrix.
Use your solutions to \ref{prob:hessenberg} (where you computed the Hessenberg form of a matrix) and Problem \ref{prob:givens_hessenberg_modified} to do the necessary computations (where you computed the QR decomposition of a Hessenberg matrix and wrote code for multiplication by $Q$ that works in $\mathcal{O} \left( n^2 \right)$ time).
The solution to Problem \ref{prob:givens_hessenberg_modified} is especially important because it allows the compution of each QR decomposition and each $R Q = \left( Q^T R^T \right)$ in $\mathcal{O} \left( n^2 \right)$ time.
\end{problem}
\end{comment}

\begin{comment}
\begin{problem}
If $A$ is normal, its Schur form is diagonal.
For normal $A$, have your function additionally output the eigenvector corresponding to each eigenvalue.
Hint 1: Test your function on Hermitian and real symmetric matrices; they are both normal.
Hint 2: Your work in Problem \ref{problem:similarity proof} will help.
You have already made all the necessary calculations, you just need to store the information correctly.
\end{problem}
\end{comment}

\begin{comment}
\begin{problem}
Test your implementation with random matrices.
Try real-valued and symmetric matrices.
Compare your output to the output from the eigenvalue solver.
How many iterations are necessary?
How large can $A$ be?
\end{problem}
\end{comment}

The QR algorithm as described in this lab is not often used.
Instead, modern computer packages use the implicit QR algorithm, which is an improved version of the QR algorithm.

Lastly, iterative methods besides the power method and QR method are often used to find eigenvalues.
Arnoldi iteration is similar to the QR algorithm but exploits sparsity.
Other methods include the Jacobi method and the Rayleigh quotient method.

\section*{Markov chains (application of the power method)}
A Markov chain is a collection of states with specified probabilities for transitioning from one state to another.
Markov chains are characterized by the fact that future behavior of the system depends only on its current state.

This lab is a very brief introduction to Markov chains. To learn more, see [TODO: ref something].

\subsection*{An example}
Suppose Fredo the frog is jumping between the three lily pads 1, 2, and 3.
These three pads are the \emph{possible states} of the system, and the lily pad on which Fredo is presently sitting is the \emph{current state}.
If Fredo is on lily pad 1 and jumps, there is a 25\% chance that it will land back on lily pad 1, a 25\% chance that it will land on lily pad 2, and a 50\% chance that it will land on lily pad 3.
These probabilities are the \emph{transition probabilities}.
Figure \ref{fig:markov1} is a \emph{transition diagram} that depicts all transition probabilities.

\begin{figure}
\begin{tikzpicture}[normalcircle/.style={draw, circle, minimum size=1cm, fill=shadecolor, thick, node distance=1.5cm} ]

\node[normalcircle](circle1)[]{};
\node[draw=none](one)[]{1};

\node[draw=none, node distance=2.5cm](dummy)
	[below of=circle1]{};

\node[normalcircle](circle2)[left of=dummy]{};
\node[draw=none, node distance= 1.5cm](two)[left
	of=dummy]{2};
\node[normalcircle](circle3)[right of=dummy]{};
\node[draw=none, node distance= 1.5cm](three)
	[right of=dummy]{3};

\foreach \s /\t in {circle2/circle1, circle1/circle3, circle3/circle2}
	{\path[draw,bend left=20, thick, ->, >=stealth'] (\s)edge(\t);}
\foreach \s /\t in {circle1/circle2, circle3/circle1, circle2/circle3}
	{\path[draw,bend left=20, thick, ->, >=stealth'] (\s)edge(\t);}

\draw[thick,->, >=stealth'](-1.9,-2.8) arc (325:40:.4 and .5);
\draw[thick,->, >=stealth'](1.9,-2.2) arc (500:215:.4 and .5);
\draw[thick,->, >=stealth'](-.4,.3) arc (220:-35:.5 and .4);

\node[draw=none, node distance=1.6cm](dummy2)
	[below of=circle1]{};
\node[draw= none, node distance=.3cm](midvalues)
	[above left of=dummy2]{$\frac{1}{4}$};
\node[draw= none, node distance=.3cm](midvalues2)
	[above right of=dummy2]{$\frac{1}{2}$};
\node[draw= none, node distance=.25cm](midvalues3)
	[below of=dummy2]{$\frac{1}{3}$};
\node[draw=none, node distance=1.6cm](outsidevalue)
	[above left of=midvalues3]{$\frac{1}{2}$};
\node[draw=none, node distance=1.6cm](outsidevalue2)
	[above right of=midvalues3]{$\frac{1}{2}$};
\node[draw=none, node distance=1.35cm](outsidevalue3)
	[below of=midvalues3]{$\frac{1}{2}$};
\node[draw=none, node distance=1.25cm](circlevalue)
	[above of=circle1]{$\frac{1}{4}$};
\node[draw=none, node distance=2.8cm](circlevalue2)
	[left of=dummy]{$\frac{1}{6}$};
\node[draw=none, node distance=2.8cm](circlevalue3)
	[right of=dummy]{$0$};

\end{tikzpicture}
\caption{Transition diagram for Fredo the Frog.}
\label{fig:markov1}
\end{figure}

We can convert our transition diagram into a \emph{transition matrix} The $(i,j)$-entry of the transition matrix is the probability that Fredo jumps from lily pad $j$ to lily pad $i$.
Fredo's transition matrix is
\[
A = \begin{pmatrix}
1/4 & 1/2 & 1/2\\
1/4 & 1/6 & 1/2\\
1/2 & 1/3 & 0
\end{pmatrix}.
\]

At any time, the chances that Fredo is on each lily pad is encoded by a \emph{state distribution vector} $\x = (x_1, x_2, x_3)^T$, where $x_i$ is the probability that Fredo is on lily pad $i$.
For $\x$ to be a state distribution vector, we require $x_i\geq 0$ and $\|\x\|_1=1$ (recall that the 1-norm of a column vector is the sum of the magnitudes of its entries).
Then $A\x$ will be another state distribution vector, which tells us the probability that Fredo is on each lily pad after one jump.

Thus, we can use Fredo's transition matrix to find where he will be after $k$ jumps. In fact, the $(i,j)$-entry of $A^k$ is the probability that Fredo goes from lily pad $j$ to lily pad $i$ in $k$ jumps.
In our case,

\[
A^2 \approx \begin{pmatrix}
0.4375 & 0.3750 & 0.3750\\
0.3542 & 0.3194 & 0.2083\\
0.2083 & 0.3056 & 0.4167
\end{pmatrix}.
\]
Therefore, if Fredo starts on lily pad 1, there is a 43.75\% chance it will still be on lily pad 1 after two jumps.
Maybe Fredo jumped from 1 to 1 to 1, denoted $1 \rightarrow 1 \rightarrow 1$, or perhaps it jumped to one of the other lily pads and then back again, that is, either $1 \rightarrow 2 \rightarrow 1$ or $1 \rightarrow 3 \rightarrow 1$.

In addition, there is a 35.42\% chance Fredo will be on lily pad 2 and a 20.83\% chance that it will be on lily pad 3.
We can type our transition matrix into Python and see where Fredo is likely to be after any number of jumps.

\begin{lstlisting}
# The 1.'s in the numerator force floating point division.
>>> A = np.array([[1./4,1./2,1./2],[1./4,1./6,1./2],[1./2,1./3,0]])
>>> np.linalg.matrix_power(A,10)
array([[ 0.40000057,  0.39999962,  0.39999962],
       [ 0.30002369,  0.29999268,  0.29997574],
       [ 0.29997574,  0.3000077 ,  0.30002464]])
\end{lstlisting}

In fact, as we take higher and higher powers of $A$, it appears that
\[
\lim_{k \rightarrow \infty} A^k = \begin{pmatrix}
0.4 & 0.4 & 0.4\\
0.3 & 0.3 & 0.3\\
0.3 & 0.3 & 0.3
\end{pmatrix}.
\]
This means that Fredo's state distribution approaches $(0.4, 0.3, 0.3)^T$ after many jumps, regardless of his initial state distribution.

Moreover, suppose Fredo's initial state distribution is $(0.4, 0.3, 0.3)^T$.
We can use Python to compute his state distribution after 1 jump.
\begin{lstlisting}
>>> A.dot(np.array([0.4,0.3,0.3]))
array([ 0.4,  0.3,  0.3])
\end{lstlisting}
Fredo's state distribution after a jump stays ``fixed.'' We call the vector $(0.4, 0.3, 0.3)^T$ a \emph{stable fixed point} for Fredo.

\subsection*{General Markov chains}
Let us generalize this example. A Markov chain is a collection of states with the probabilities that we will move from one state to another.
These transition probabilities are encoded in a transition matrix, whose $(i-j)$-entry is the probability of moving from state $j$ to state $i$.
Each column of such a matrix will necessarily have entries that sum to 1.

%When we have a Markov chain, we want to know what the state distribution is after our system has been running for some time. We are especially interested if the state distributions converge to something that is independent of the initial state, as was the case for Fredo. That is, we want to know about stable fixed points.

Let $A$ be the transition matrix of a Markov chain.
Then a state distribution vector $\x$ is a \emph{stable fixed point} if $A\x=\x$.
So $\x$ is a stable fixed point if and only if $\x$ is a positive unit eigenvector of $A$ corresponding to the eigenvalue 1.

Every Markov chain has at least one stable fixed point.
If in addition we assume some power $A^k$ of $A$ has all positive (nonzero) entries, then the stable fixed point is unique.
In this case, $A^k$ will converge to a matrix whose columns are all equal to the unique stable fixed point.

Note that Fredo's transition matrix does not have positive entries, but its square does.
So Fredo has the unique stable fixed point $(0.4, 0.3, 0.3)^T$.

\subsection*{Finding stable fixed points}
Calculating stable fixed points is an important problem in Markov chain analysis.
Suppose a Markov chain has a transition matrix $A$ such that $A^k$ has strictly positive entries for some $k$.
Then the \emph{Perron-Frobenius theorem} says that 1 is the unique eigenvalue of $A$ of largest magnitude, and the corresponding eigenvector is unique.
This means that we can use the power method to find the unique stable fixed point of $A$.

Let us look at what the power method is doing in the example of Fredo the frog.
We will use the 1-norm.

Suppose we know Fredo starts on lily pad 1. Then we begin with the state distribution vector
\[
\x_0 = \begin{bmatrix}
1\\
0\\
0
\end{bmatrix}
\]
because we know for certainty (100\%) that Fredo is in the first state.
The next iteration of the power method is $\x_1=A\x_0/\|A\x_0\|_1$. But since $\|A\x_0\|_1=1$, this is just
\[
\x_1 = A \x_0 = \begin{bmatrix}
0.25\\
0.25\\
0.50
\end{bmatrix},
\]
which is exactly Fredo's state distribution after 1 jump.
After two jumps, Fredo's state distribution is
\[
\x_2 = A \x_1 = A^2 \x_0 = \begin{bmatrix}
0.4375\\
0.3542\\
0.2083
\end{bmatrix},
\]
which is also the second iteration of the power method.
After a large number of jumps, we have
\[
\x_n = A \x_{n-1} = \dots = A^n \x_0 \approx \begin{bmatrix}
0.4\\
0.3\\
0.3
\end{bmatrix}.
\]
Thus, the limiting vector of the power method is exactly the unique stable fixed point of the Markov chain.

\subsection*{A final example}
Consider the Markov chain with transition matrix
\[
A = \begin{pmatrix}
0.5 & 0.3 & 0.4\\
0.2 & 0.2 & 0.3\\
0.3 & 0.5 & 0.3
\end{pmatrix}.
\]

Because all entries of $A$ are positive, the Markov chain has a unique stable fixed point.
We could find this fixed point with the power method as outlined above, or we can do it by computing eigenvalues and eigenvectors in Python, as shown below.
\begin{lstlisting}
>>> from scipy import linalg as la
>>> A = np.array([[.5,.3,.4],[.2,.2,.3],[.3,.5,.3]])
>>> evals, evecs = la.eig(A)
>>> evals
array([ 1.        ,  0.14142136, -0.14142136])
\end{lstlisting}
We are interested in the eigenvalue 1, which is the first one outputted in this case.
The corresponding eigenvector is the first column of \li{evecs}; let us call it \li{x}.
\begin{lstlisting}
>>> x = evecs[:,0]
\end{lstlisting}
Now, the one-norm of \li{x} will probably not be 1. To make it 1, we divide by the one-norm of \li{x}.
\begin{lstlisting}
>>>x = x/np.sum(x)
\end{lstlisting}
Finally, let us check that \li{x} is a stable fixed point of $A$. There are two things to check.
\begin{lstlisting}
# Check Ax = x
>>> np.allclose(A.dot(x),  x)
True

# Check ||x||_1 = 1
>>> np.sum(x)==1
True
\end{lstlisting}

\begin{problem}
Write a function that accepts as input a transition matrix, a vector representing the initial state, and a number of iterations \li{niter}. Your function should
\begin{enumerate}
\item Assume that the input matrix has a unique stable fixed point.
\item Calculate the unique stable fixed point by computing eigenvectors and eigenvalues in Python.
\item Return the current state of the Markov chain after \li{niter} iterations and the stable fixed point.
\end{enumerate}
\label{prob:markov}
\end{problem}

\begin{problem}
Suppose a basketball player's success at shooting free throws can be described with the following Markov chain
\[
A = \begin{pmatrix}.75&.50\\.25&.50\end{pmatrix}
\]
where the first state corresponds to success and the second state to failure. Use the function you wrote in Problem \ref{prob:markov} to answer the following questions.
\begin{enumerate}
\item If the player makes his first free throw, what is the probability that he also makes his third one?
(That is, what is the probability that this system starts in state 1 and is still in state 1 after 3 steps?)
\item What is the player's average free throw percentage?
(This is equal to the success-component of the stable fixed point.)
\end{enumerate}
\label{prob:markov_freethrow}
\end{problem}

\begin{comment}
\begin{problem}
Consider the Markov process given by the transition diagram in Figure \ref{fig:markov2}.
\begin{figure}[H]
\includegraphics[width=\textwidth]{markov2}
\caption{Transition diagram}
\label{fig:markov2}
\end{figure}

\begin{enumerate}
\item Find the transition matrix.
\item If the Markov process is in state 1 initially, find the probability that it is in state 2 after two transitions.
\item Find the stable fixed point if it exists.
\end{enumerate}
\label{prob:markov_stablept}
\end{problem}
\end{comment}


\section{Spherical Bessel Functions}

\label{sphericalbess}
Matlab does not have built-in routines for spherical Bessel functions. Below are several variants of spherical Bessel functions and derivatives encountered in scattering problems.

%Log derivatives.

%\bgroup
%\def\arraystretch{3}
%\begin{table}[htdp]
%\caption{default}
%\begin{center}
%\begin{tabular}{|c|c|c|}
%\hline
%Function & Equation & Comments \\
%\hline
%\texttt{jinc(x)}& $\textrm{jinc}(x) = \dfrac{J_1(x)}{x}$ & $\lim_{x\rightarrow0} \textrm{jinc}(x) = \dfrac{1}{2}$ \\
%\hline
%\texttt{sbesselj(n,x)} & $j_n(x) = \sqrt{\dfrac{\pi}{2x}}J_{n+1/2}(x)$ & $\lim_{x\rightarrow0}j_n(x) = \left\{ \begin{array}{c} 1, \quad n=0 \\ 0, \quad n \ge 1\end{array}  \right.$  \\
%\hline
%\end{tabular}
%\end{center}
%\label{default}
%\end{table}
%\egroup

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Sombrero function: $\textrm{jinc}(x)$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

The \textrm{jinc}(x) function, or Sombrero function, is defined:
\begin{equation}
\textrm{jinc}(x) = \dfrac{J_1(x)}{x}
\end{equation}
%
%\noindent where
%\begin{equation}
%\lim_{z\rightarrow0} \textrm{jinc}(z) = \dfrac{1}{2}
%\end{equation}

\noindent where small arguments are computed with the Taylor series
\eq{ \textrm{jinc}(x) \approx \dfrac{1}{2} - \dfrac{x^2}{16} + O(x^4)}

{\footnotesize
\VerbatimInput{\code/BesselFunctions/jinc.m}
}

%This is slow in Matlab and we'd like to avoid the division.  Start with the series representation for the Bessel function of complex argument
%\eq{J_{\nu}(z) = \left(\dfrac{z}{2}\right)^\nu \sum_{k=0}^{\infty} \dfrac{ (-1)^k \left( \dfrac{z}{2}\right)^{2k} }{ k! \Gamma(\nu + k + 1)}}
%
%Evaluating $\nu =1$, and dividing by $z$,
%\eq{\textrm{jinc}(z) = \dfrac{1}{2} \sum_{k=0}^{\infty}(-1)^k a_k }
%\eq{a_k = \dfrac{  \left( \dfrac{z}{2}\right)^{2k} }{ k! \Gamma(k + 2)} }
%
%The log transform is used to avoid multiplying and dividing by large and small numbers, facilitate fast recursive of the factorials, and take care of the limit when $z = 0$.
%\eq{a_k = e^{\ln a_k}}
%
%where, using $\Gamma(n) = (n-1)!$,
%\ea{\ln a_k &=& 2k (\ln z - \ln 2) - \ln k! - \ln (k + 1)! }
%
%Evaluating at $k+1$, separating a factor of $\ln a_k$, and shifting the index down by 1, we get the recursion for the $k$th term of the series
%\ea{\ln a_{0} &=& 0 \\
%\ln a_{k} &=& \ln a_{k-1} + 2 (\ln z - \ln 2) - \ln k - \ln (k + 1) }
%
%%\ea{\ln a_{k+1} &=& 2(k+1) (\ln z - \ln 2) - \ln (k+1)! - \ln (k + 2)! \\
%%\ &=& 2k (\ln z - \ln 2) + 2 (\ln z - \ln 2) - (\ln k! + \ln (k+1)) - ( \ln (k + 1)!  + \ln (k + 2) ) \\
%%\ &=& \ln a_k + 2 (\ln z - \ln 2) - \ln (k+1) - \ln (k + 2) }
%
%The code is as follows.

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Bessel function: $j_n(x)$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

The spherical Bessel function is

\begin{equation}
j_n(x) = \sqrt{\dfrac{\pi}{2x}}J_{n+1/2}(x)
\end{equation}

\noindent where

\begin{equation}
\lim_{x\rightarrow0}j_n(x) = \left\{ \begin{array}{c} 1 - \dfrac{x^2}{6} + O(x^4) \quad n=0 \\ 0, \quad n \ge 1\end{array}  \right.
\end{equation}

The routine \texttt{sbesselj} returns the spherical Bessel function for matching arrays of $n$ and $x$ in the format of \texttt{besselj}. It uses the Taylor series near $x = 0$ for $n=0$, and substitutes 0 when $x=0$ for $n>1$.

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesselj.m}
}

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Bessel function: $j_n(x)/x$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

This variant of the spherical Bessel function has $j_n(x)$ divided by $x$:
\begin{equation}
\dfrac{j_n(x)}{x} = \dfrac{1}{x^{3/2}}\sqrt{\dfrac{\pi}{2}}J_{n+1/2}(x)
\end{equation}

\noindent where
\begin{equation}
\lim_{x\rightarrow0}j_n(x) = \left\{ \begin{array}{c} \infty, \quad n=0 \\ \dfrac{1}{3} - \dfrac{x^2}{30} + O(x^4), \quad n = 1 \\ 0, \quad n \ge 2\end{array}  \right.
\end{equation}

This is computed in the routine \texttt{sbesselj2}, which takes the same inputs as \texttt{sbesselj}.  We let the routine return \texttt{Inf} for $x=0$ when $n=0$.  This variant is found in electromagnetic scattering when usually the monopole term ($n=0$) is not needed.

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesselj2.m}
}

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Bessel function derivative: $j_n'(x)$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

The derivative of the spherical Bessel function with respect to $x$ is given by the recurrence relation
\begin{equation}
j_n'(x) = -j_{n+1}(x) + \dfrac{n}{x}j_n(x)
\end{equation}

This is computed in \texttt{sbesseljp} with \texttt{sbesselj} and \texttt{sbesselj2}, which takes care of $x=0$.

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesseljp.m}
}

\clearpage
\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Bessel function derivative: $[xj_n(x)]'$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

This variant occurs often enough in scattering to have its own function, \texttt{sbesseljp2},
\ea{[xj_n(x)]' &=& j_n(x) + x j'_n(x) \\
\ &=&  (1+n)j_n(x) - x j_{n+1}(x)}

This variant is found in the Mie scattering solution for spheres.

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesseljp2.m}
}

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Hankel function: $h_n^{(1)}(x)$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

The spherical Hankel function is given by
\begin{equation}
h_n^{(1)}(x) = \sqrt{\dfrac{\pi}{2x}}H_{n+1/2}(x)
\end{equation}

This is always irregular at the origin, and computed in the routine \texttt{sbesselh}

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesselh.m}
}

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Hankel function derivative: ${h'}_n^{(1)}(x)$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

The spherical Hankel derivative is computed in the routine \texttt{sbesselhp}
\begin{equation}
{h'}_n^{(1)}(x) = -h_{n+1}^{(1)}(x) + \dfrac{n}{x}h_n^{(1)}(x)
\end{equation}

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesselhp.m}
}

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Spherical Hankel function derivative: $[xh_n^{(1)}(x)]'$}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

Similar to before, this variant is computed in the routine \texttt{sbesselhp2}
\ea{[xh_n^{(1)}(x)]' &=& h_n^{(1)}(x) + x {h'}_n^{(1)}(x) \\
\ &=&  (1+n)h_n^{(1)}(x) - x h_{n+1}^{(1)}(x)}

This variant is found in the Mie scattering solution for spheres.

{\footnotesize
\VerbatimInput{\code/BesselFunctions/sbesselhp2.m}
}