\section{Homotopy fibers}
An important, but easy, fact about fibrations is that the canonical map $X\to \ast$ from any space $X$ is a fibration
\footnote{Model category theorists get excited about this, because this says that all objects in the associated model structure on
topological spaces is fibrant.}.
This is because the dotted lift in the diagram below can be taken to the map $(t,w)\mapsto f(w)$:
\begin{equation*}
    \xymatrix{
    W\ar[d]\ar[r]^f & X\ar[d]\\
    I\times W\ar@{-->}[ur]\ar[r] & \ast.
}
\end{equation*}
However:
\begin{exercise}
    The inclusion $\ast\hookrightarrow X$ is not always a cofibration;
    if it is, say that $\ast$ is a \emph{nondegenerate basepoint} of $X$.
    Give an example of a compactly generated space $X$ for which this is true.
\end{exercise}
If $\ast$ has a neighborhood in $X$ that contracts to $\ast$,
the inclusion $\ast\hookrightarrow X$ is a cofibration.
Note that if $\ast$ is a nondegenerate basepoint, the canonical map $X^A\xrightarrow{\mathrm{ev}} X$ is a fibration,
where $A$ is a pointed subspace of $X$ (with basepoint given by $\ast$).
The fiber of $\mathrm{ev}$ is exactly the space of pointed maps $A\to X$.

\begin{remark}
    In Example \ref{intervalcofib}, we saw that $\{0,1\}\hookrightarrow I$ is a cofibration;
    this implies that the map $Y^I\to Y\times Y$ (given by $\omega\mapsto (\omega(0),\omega(1))$) is a fibration.
\end{remark}
\subsection{``Fibrant replacements''}
The purpose of this subsection is to provide a proof of the following result, which says that every map can be ``replaced''
(up to homotopy) by a fibration.
\begin{theorem}\label{fibrep}
    For any map $f:X\to Y$, there is a space $T(f)$, along with a fibration $p:T(f) \to Y$ and
    a homotopy equivalence $X \xar{\simeq} T(f)$, such that the following diagram
    commutes:
    \begin{equation*}
	\xymatrix{
	    X\ar[r]^{\simeq}\ar[dr]_f & T(f)\ar[d]^p\\
	    & Y.
	    }
    \end{equation*}
\end{theorem}
\begin{proof}
    Consider the map $Y^I\xrightarrow{\binom{\ev_0}{\ev_1}}Y\times Y$.
    Let $T(f)$ be the pullback of the following diagram:
	\begin{equation*}
	    \xymatrix{
	T(f)\ar[r]\ar[d] & Y^I\ar[d]^{\binom{\ev_0}{\ev_1}}\\
		    X\times Y\ar[r]_{f\times 1} & Y\times Y.
		}
	\end{equation*}
	So, as a set, we can write
	$$T(f)=\{(x,\omega)\in X\times Y^I|f(x) = \omega(0)\}.$$

	Let us check that the canonical map $T(f) \to Y$, given by $(x,\omega) \mapsto \omega(1)$, is a fibration.
	The projection map $\pr: X\times Y\to Y$ is a fibration, so it suffices to show that the map $T(f) \to X\times Y$
	is also a fibration.
	Since fibrations are closed under pullbacks, we are reduced to checking that the map $Y^I\to Y\times Y$ is a fibration;
	but this is exactly saying that the inclusion $\{0,1\} \hookrightarrow I$ is a cofibration, which it is
	(Example \ref{intervalcofib}).

	To prove that $X$ is homotopy equivalent to $T(f)$, we need to produce a map $X\to T(f)$.
	This is equivalent to giving maps $X\to X\times Y$ and $X\to Y^I$ that have compatible images in $Y\times Y$.
	The first map can be chosen to be $X\xrightarrow{\binom{1}{f}}X\times Y$.
	Define the map $X\to Y^I$ by sending $x\in X$ to the constant loop at $f(x)$.
	It is clear that both composites $X\to X\times Y\to Y\times Y$ and $X\to Y^I\to Y\times Y$ are the same;
	this defines a map $X\to T(f)$, denoted $g$.
	As one can easily check, the composite $X\to T(f)\xar{p} Y$ is the map $f:X\to Y$ that we started off with.
	It remains to check that this map $X\xar{g} T(f)$ is a homotopy equivalence.
	We will construct a homotopy inverse to this map.

	The composite $X\to T(f)\to X\times Y\to X$ is the identity, so one candidate for a homotopy inverse to $g$ is
	the composite
	$$T(f)\to X\times Y\xrightarrow{pr_1} X.$$
	To prove that this map is indeed a homotopy inverse to $g$, we need to consider the composite
	$T(f)\to X\xar{g} T(f)$, which sends $(x,\omega)\mapsto x\mapsto (x,c_{f(x)})$ where,
	recall, $c_{f(x)}$ is the constant path at $x$.
	We need to produce a homotopy between this composite and the identity on $T(f)$.

	Let $s\in I$. Given $\omega \in Y^I$, define a new loop $\omega_s$ by $\omega_s(t) = \omega(st)$.
	For instance, $\omega_1 = \omega$, and $\omega_0 = c_{\omega(0)}$ --- so, the loop $\omega_s$ ``sucks in''
	the point $\omega(1)$.
	This is precisely what we need to produce a homotopy between the composite $T(f) \to X \xar{g} T(f)$ and
	$\mathrm{id}_{T(f)}$, since the only constraint on $(x,\omega)\in T(f)$ is on $\omega(0)$.
	The following map provides the desired homotopy equivalence $X \simeq T(f)$.
	\begin{align*}
	    H:I\times T(f)\to & T(f)\\
	    (s,(x,\omega))\mapsto & (x,\omega_s).
	\end{align*}
%If you want to work through the diagram, this is how it looks.
%\begin{equation*}
%    \xymatrix{
%        X\ar@{-->}[dr]\ar[drr]^{x\mapsto c_{f(x)}}\ar[ddr]_{\begin{pmatrix}1 \\ f\end{pmatrix}} & & \\
%    	& T(f)\ar[d]\ar[r]\ar[dr] & Y^I\ar[d]^{\begin{pmatrix}ev_0 \\ ev_1\end{pmatrix}}\\
%    	    X & X\times Y\ar[l]^{pr_1}\ar[d]^{pr_2}\ar[r] & Y\times Y\\
%        & Y
%        }
%\end{equation*}
\end{proof}

\begin{example}[Path-loop fibration]
    This is a silly, but important, example.
    If $X = \ast$, the space $T(f)$ consists of paths $\omega$ in $Y$ such that $\omega(0) = \ast$.
    In other words, $T(f) = Y^I_\ast$; this is called the \emph{(based) path space} of $Y$, and is denoted by $P(Y,\ast)$,
    or simply by $PY$.
    The fiber of the fibration $T(f) = PY \to Y$ consists of paths that begin and end at $\ast$, i.e.,
    loops on $Y$ based at $\ast$.
    This is denoted $\Omega Y$, and is called the \emph{(based) loop space} of $Y$.
    The resulting fibration $PY \to Y$ is called the \emph{path-loop fibration}.
\end{example}

\begin{exercise}[``Cofibrant replacements'']\label{cofibrep}
    In this exercise, you will prove the analogue of Theorem \ref{fibrep} for cofibrations.
    Let $f:X\to Y$ be any map.
    Show that $f$ factors (functorially) as a composite $X \to M \to Y$, where $X\to M$ is a cofibration and $M\to Y$ is a homotopy
    equivalence.
\end{exercise}

\begin{solution}
    Define $Mf$ via the pushout:
    \begin{equation*}
	\xymatrix{
	    X\ar[r]^f \ar[d]_{\mathrm{in}_0} & Y\ar[d]^g\\
	    I\times X\ar[r] & Mf.
	    }
    \end{equation*}
    Define $r:Mf\to Y$ via $r(y) = y$ on $Y$ and $r(x,s) = f(x)$ on $X\times I$.
    Then, clearly, $rg = \mathrm{id}_Y$.
    There is a homotopy $\mathrm{id}_{Mf}\simeq gr$ given by the map $h:Mf\times I\to Mf$,
    defined by the formulae
    $$h(y,t) = y,\text{ and }h((x,s),t) = (x,(1-t)s).$$
    %Essentially, one's pushing the cylinder $X\times I$ down.

    We now have to check that $X\to Mf$ is a cofibration, i.e., that
    $Mf\times I$ retracts onto $Mf\times\{0\}\cup_X (X\times I)$.
    This can be done by ``pushing'' $Y\times I$ to $Y\times\{0\}$ and $X\times I\times I$ down to $X\times I$,
    while fixing $X\times\{0\}$.

    It is easy to see that this factorization is functorial:
    if $f:X\to Y$ is sent to $g:W\to Z$ via $p:X\to W$ and $q:Y\to Z$,
    then $Mf\to Mg$ can be defined as the dotted map in the following diagram
    (which exists, by the universal property of the pushout):
    \begin{equation*}
	\xymatrix{
	    X\ar[rr]^f\ar[dd]_{\mathrm{in}_0}\ar[dr]|p & & Y\ar[dd]|\hole\ar[dr]|q & \\
	    & W\ar[rr]_g\ar[dd] & & Z\ar[dd]\\
	    X\times I\ar[rr]\ar[dr]_{p\times \mathrm{id}} & & Mf\ar@{-->}[dr] & \\
	    & W\times I\ar[rr] & & Mg.
	    }
    \end{equation*}
    \todo{Fix the overlapping arrows here, I don't know how to do this...}
\end{solution}

\subsection{Homotopy fibers}
One way to define the fiber (over a basepoint) of a map $f:X \to Y$ is via the pullback
\begin{equation*}
    \xymatrix{
	f^{-1}(\ast)\ar[r]\ar[d] & X\ar[d]^f\\
	\ast\ar[r] & Y.
    }
\end{equation*}
If $g:W \to X$ is another map such that the composite $W\xar{g} X\xar{f} Y$ is trivial, the map $g$ factors through $f^{-1}(\ast)$.
In homotopy theory, maps are generally not trivial ``on the nose''; instead, we usually have a nullhomotopy of a map.
Nullhomotopies of composite maps do not factor through this ``strict'' fiber; this leads to the notion of a homotopy fiber.
\begin{definition}[Homotopy fiber]
    The \emph{homotopy fiber} of a map $f:X\to Y$ is the pullback:
    \begin{equation*}
	\xymatrix{
	    F(f,\ast)\ar[r]\ar[d] & T(f)\ar[r]^\simeq \ar[d]^p & X\ar[dl]^f\\
	    \ast \ar[r] & Y &
	    }
    \end{equation*}
\end{definition}
As a set, we have
\begin{equation}\label{sethomotopyfiber}
    F(f,\ast) = \{(x,\omega)\in X\times Y^I| f(x) = \omega(0), \omega(1) = \ast\}.
\end{equation}
A nullhomotopic composite $W\to X\xar{f} Y$ factors as $W\to F(f,\ast)\to X\xar{f} Y$.
\begin{warning}
    The ordinary fiber and the homotopy fiber of a map are generally not the same!
    There is a canonical map $p^{-1}(\ast) \to F(p,\ast)$, but it is generally not a homotopy equivalence.
\end{warning}
\begin{prop}\label{strictcomparison}
    Suppose $p:X\to Y$ is a fibration.
    Then the canonical map $p^{-1}(\ast)\to F(p,\ast)$ is a homotopy equivalence.
\end{prop}
You will prove this in a series of exercises.

\begin{exercise}\label{sectionuptohomotopy}
    Prove Proposition \ref{strictcomparison} by working through the following statements.
    \begin{enumerate}
	\item Let $p:E\to B$ be a fibration.
	    Suppose $g:X \to B$ lifts across $p$ up to homotopy, i.e., there exists a map $f:X\to E$ such that
	    $p \circ f \simeq g$.
	    Prove that there exists a map $f^\prime:X\to E$ that is homotopic to $f$, such that $p \circ f^\prime = g$
	    (on the nose).
%	\item Let $p:E_0 \to B_0$ be a fibration, and let $f:B \to B_0$ be a homotopy equivalence.
%	    Prove that the induced map $B\times_{B_0} E_0 \to E_0$ is a homotopy equivalence.
%	    (Warning: this exercise has a lot of technical details! The end of this chapter describes an
%	    alternative\footnote{``Alternative'' in the sense that the proof uses statements
%	    not covered yet in this book.}
%	    solution to this exercise, when $E_0$ and $B\times_{B_0} E_0$ are CW-complexes.)
%	    \todo{Don't forget to do this!}
	\item Show that if $p:E\to B$ and $p^\prime:E^\prime \to B$ are fibrations, and $f:E \to E^\prime$ such that
	    $p^\prime \circ f = p$, the map $f$ is a \emph{fiber homotopy equivalence}: there is a homotopy inverse
	    $g: E^\prime \to E$ such that $g$, and the two homotopies $fg \simeq \mathrm{id}_{E^\prime}$ and
	    $gf \simeq \mathrm{id}_{E}$ are all fiber preserving (e.g., $p\circ g = p^\prime$).
	\item Conclude Proposition \ref{strictcomparison}.
   \end{enumerate}
\end{exercise}

Before we proceed, recall that we constructed the homotopy fiber by replacing $f:X\to Y$ by a fibration.
In doing so, we implicitly made a choice: we could have replaced the map $\ast\to Y$ by a fibration.
Are the resulting pullbacks the same?

By replacing $\ast \to Y$ by a fibration (namely, the path-loop fibration), we end up with the following pullback diagram:
\begin{equation*}
    \xymatrix{
	F^\prime(f,\ast)\ar[r]\ar[d] & P(Y,\ast)\ar[r]^{\simeq}\ar[d] & \ast\ar[dl]\\
	X\ar[r]_f & Y &
    }
\end{equation*}
As a set, we have
$$F^\prime(f,\ast) = \{(x,\omega)\in X\times Y^I\text{ such that }\omega(0) = \ast\text{ and }\omega(1) = f(x)\}.$$
Our description of $F(f,\ast)$ in \eqref{sethomotopyfiber} is almost exactly the same --- except that
the directions of the paths are reversed.
Thus there's a homeomorphism $F^\prime(f,\ast) \simeq F(f,\ast)$ given by reversing directions of paths.
\begin{remark}
    One could also replace both $f:X\to Y$ and $\ast\to Y$ by fibrations, and the resulting pullback is also homeomorphic
    to $F(f,\ast)$. (Prove this, if the statement is not immediate.)
\end{remark}

%Here's what you'll prove for homework.
%\begin{theorem}
%    Suppose you have two fibrations $p$ and $p^\prime$ such that the following diagram commutes, where $f$ is a homotopy equivalence.
%    \begin{equation*}
%	\xymatrix{
%	    E\ar[r]^p\ar[dr]^p & E^\prime\ar[d]^{p^\prime}\\
%	    & B
%	    }
%    \end{equation*}
%    Then $f$ is a fiber homotopy equivalence. That means that it's a homotopy equivalence in $\Top_{/B}$. What this means is that there is a map $g:E^\prime\to E$ over $B$ compatible with the fibrations and homotopies $I\times E\to E$ over $B$ and $I\times E^\prime\to E^\prime$ over $B$. I.e., the following three diagrams commute:
%    \begin{equation*}
%	\xymatrix{
%	    E^\prime\ar[r]^g\ar[dr]^{p^\prime} & E\ar[d]^p\\
%	    & B
%	    }
%    \end{equation*}
%    and
%    \begin{equation*}
%	\xymatrix{
%	    I\times E\ar[r]^{1\sim gf}\ar[dr] & E\ar[d]^p\\
%	    & B
%	    }
%    \end{equation*}
%    and
%    \begin{equation*}
%	\xymatrix{
%	    I\times E^\prime\ar[r]^{1\sim fg}\ar[dr] & E^\prime\ar[d]^{p^\prime}\\
%	    & B
%	    }
%    \end{equation*}
%\end{theorem}
%So we find that for all $b$, $p^{-1}(b)\xrightarrow{\simeq} (p^\prime)^{-1}(b)$. So in particular, the fiber $F(f,\ast)$, i.e., the homotopy fiber, of $T(f)\to B$ and the fiber $f^{-1}(\ast)$ of $f:E\to B$ are homotopy equivalent if $f$ is a fibration.


\section{Change of Variables}\label{sec:ChangeofVariables}

One of the most useful techniques for evaluating integrals is
substitution, both ``$u$-substitu\-tion'' and trigonometric
substitution, in which we change the variable to something more
convenient. As we have seen, sometimes changing from rectangular
coordinates to another coordinate system is helpful, and this too
changes the variables. This is certainly a more complicated change,
since instead of changing one variable for another we change an entire
suite of variables, but as it turns out it is really very similar to
the kinds of change of variables we already know as substitution.\index{integration!change of variables}

\begin{figure}[H]
\hbox to \hsize{\hfill
\begin{tikzpicture}[baseline=0,x=4cm,y=4cm,smooth]
\draw (0,0) -- (1.1,0) node [right] {$x$};
\draw (0,0) -- (0,0.5) ;
\foreach \x in {0,0.5,1} \draw (\x,0) -- (\x,-2pt) node[anchor=north] {$\x$};
\foreach \y in {0,0.1,0.2,0.3,0.4,0.5} \draw (0,\y) -- (-2pt,\y) node[anchor=east] {$\y$};
\draw[fill=red!20,domain=0:1,samples=30] plot (\x,{\x*\x*sqrt(1-\x*\x)}) -- (1,0);
\draw (0.7,0) rectangle (0.8,0.35);
\end{tikzpicture}\hfill
\begin{tikzpicture}[baseline=0,x=4cm,y=4cm,smooth]
\draw (0,0) -- (1.7,0) node [right] {$u$};
\draw (0,0) -- (0,0.5) ;
\foreach \x in {0,0.5,1,1.5} \draw (\x,0) -- (\x,-2pt) node[anchor=north] {$\x$};
\foreach \y in {0,0.1,0.2,0.3,0.4,0.5} \draw (0,\y) -- (-2pt,\y) node[anchor=east] {$\y$};
\draw[fill=red!20,domain=0:pi/2,samples=50] plot (\x,{sin(deg(\x))*sin(deg(\x))*sqrt(1-sin(deg(\x))*sin(deg(\x)))});
\draw (0.775,0) rectangle (0.927,0.35);
\end{tikzpicture}\hfill}
\caption{Single change of variable.}
\label{fig:onechangeofvariable}
\end{figure}

Let's examine the single variable case again, from a slightly
different perspective than we have previously used.
Suppose we start with the problem
\[\int_0^1 x^2\sqrt{1-x^2}\,dx;\]
this computes the area in the left graph of
Figure~\ref{fig:onechangeofvariable}.
We use the substitution $x=\sin u$ to transform the function from
$x^2\sqrt{1-x^2}$ to $\sin^2u\sqrt{1-\sin^2u}$, and we also convert
$dx$ to $\cos u\,du$. Finally, we convert the
limits 0 and 1 to 0 and $\pi/2$.
This transforms the integral:
\[\int_0^1 x^2\sqrt{1-x^2}\,dx=\int_0^{\pi/2}\sin^2u\sqrt{1-\sin^2u} \cos u\,du.\]
We want to notice that there are three different conversions: the
main function, the differential $dx$, and the interval of
integration. The function is converted to
$\sin^2u\sqrt{1-\sin^2u}$, shown in the right-hand graph of
Figure~\ref{fig:onechangeofvariable}. It is evident that the two
curves pictured there have the same $y$-values in the same order, but
the horizontal scale has been changed. Even though the heights are the
same, the two integrals
\[\int_0^1 x^2\sqrt{1-x^2}\,dx\qquad\hbox{and}\qquad
\int_0^{\pi/2}\sin^2u\sqrt{1-\sin^2u}\,du\]
are not the same; clearly the right hand area is larger. One way to
understand the problem is to note that if both areas are approximated
using, say, ten subintervals, that the approximating rectangles on the
right are wider than their counterparts on the left, as indicated. In
the picture, the width of the rectangle on the left is $\Delta x=0.1$,
between $0.7$ and $0.8$.  The rectangle on the right is situated
between the corresponding values $\arcsin(0.7)$ and $\arcsin(0.8)$ so
that $\Delta u=\arcsin(0.8)-\arcsin(0.7)$. To
make the widths match, and the areas therefore the same, we can
multiply $\Delta u$ by a correction factor; in this case the
correction factor is approximately $\cos u=\cos(\arcsin(0.7))$, which
we compute when we convert $dx$ to $\cos u\,du$.

Now let's move to functions of two variables. Suppose we want to
convert an integral
\[\int_{x_0}^{x_1}\int_{y_0}^{y_1} f(x,y)\,dy\,dx\]
to use new variables $u$ and $v$. In the single variable case, there's
typically just one reason to want to change the variable: to make the
function ``nicer'' so that we can find an antiderivative. In the two
variable case, there is a second potential reason: the two-dimensional
region over which we need to integrate is somehow unpleasant, and we
want the region in terms of $u$ and $v$ to be nicer---to be a
rectangle, for example. Ideally, of course, the new function and the
new region will be no worse than the originals, and at least one of
them will be better; this doesn't always pan out.

As before, there are three parts to the conversion: the function
itself must be rewritten in terms of $u$ and $v$, $dy\,dx$ must be
converted to $du\,dv$, and the old region must be converted to the new
region. We will develop the necessary techniques by considering a
particular example, and we will use an example we already know how to
do by other means.

Consider
\[\int_{-1}^1\int_0^{\sqrt{1-x^2}} \sqrt{x^2+y^2}\,dy\,dx.\]
The limits correspond to integrating over the top half of a circular
disk, and we recognize that the function will simplify in polar
coordinates, so we would normally convert to polar coordinates:
\[\int_{0}^\pi\int_0^1 \sqrt{r^2}\;r\,dr\,d\theta={\pi\over3}.\]
But let's instead approach this as a substitution problem, starting
with $x=r\cos\theta$, $y=r\sin\theta$. This pair of equations
describes a function from ``$r$-$\theta$ space'' to ``$x$-$y$ space'',
and because it involves familiar concepts, it is not too hard to
understand what it does. In Figure~\ref{fig:twodimensionaltransformation}
we have indicated geometrically a bit about how this
function behaves. The four dots labeled \emph{a}--\emph{d} in the $r$-$\theta$ plane
correspond to the three dots in the $x$-$y$ plane; dots \emph{a} and
\emph{b} both go to the origin because $r=0$. The horizontal arrow in the
$r$-$\theta$ plane has $r=1$ everywhere and $\theta$ ranges from 0 to
$\pi$, so the corresponding points
$x=r\cos\theta$, $y=r\sin\theta$ start at $(1,0)$ and follow the unit
circle counter-clockwise. Finally, the vertical arrow has
$\theta=\pi/4$ and $r$ ranges from 0 to 1, so it maps to the straight
arrow in the $x$-$y$ plane. Extrapolating from these few examples,
it's not hard to see that every vertical line in the $r$-$\theta$ plane
is transformed to a line through the origin in the $x$-$y$ plane, and
every horizontal line in the $r$-$\theta$ plane
is transformed to a circle with center at the origin in the $x$-$y$
plane. Since we are interested in integrating over the half-disk in
the $x$-$y$ plane, we will integrate over the rectangle
$[0,\pi]\times[0,1]$ in the $r$-$\theta$ plane, because we now see
that the points in this rectangle are sent precisely to the upper half
disk by $x=r\cos\theta$ and $y=r\sin\theta$.

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <2truecm,2truecm>
\setplotarea x from 0 to 3.2, y from 0 to 1.1
\axis left ticks numbered from 0 to 1 by 1 /
\axis bottom  ticks withvalues {$\pi$} / at 3.14 / /
\put {$\theta$} [l] <3pt,0pt> at 3.2 0
\put {$r$} [b] <0pt,3pt> at 0 1.1
\arrow <5pt> [0.17, 0.5] from 0 1 to 3.14 1
\arrow <5pt> [0.17, 0.5] from 0.785 0 to 0.785 1
\multiput {$\bullet$} at 0 0 1.57 0 0 1 1.57 0.5 /
\put {$a$} [t] <0pt,-4pt> at 0 0
\put {$b$} [t] <0pt,-4pt> at 1.57 0
\put {$c$} [tl] <2pt,-2pt> at 0 1
\put {$d$} [tr] <-2pt,-2pt> at 1.57 0.5
\setcoordinatesystem units <2truecm,2truecm> point at -6 0
\setplotarea x from -1.1 to 1.1, y from 0 to 1.1
\axis left shiftedto x=0 /
\axis bottom  ticks numbered from -1 to 1 by 1 /
\put {$x$} [l] <3pt,0pt> at 1.1 0
\put {$y$} [b] <0pt,3pt> at 0 1.1
\multiput {$\bullet$} at 0 0 1 0 0 0.5 /
\put {$a,b$} [br] <-2pt,2pt> at 0 0
\put {$c$} [br] <-2pt,2pt> at 1 0
\put {$d$} [br] <-2pt,2pt> at 0 0.5
\arrow <5pt> [0.17, 0.5] from 0 0 to 0.707 0.707
\circulararc 180 degrees from 1 0 center at 0 0
\multiput {\beginpicture %
\setcoordinatesystem units <0.4truein,0.4truein> point at 0 0
\startrotation by 0 -1 about 0 0 %
\plot -.1 .05 0 0 -.1 -.05 / \endpicture} at -1 0 /
\endpicture}}
\caption{Double change of variable.}
\label{fig:twodimensionaltransformation}
\end{figure}

At this point we are two-thirds done with the task: we know the
$r$-$\theta$ limits of integration, and we can easily convert the
function to the new variables:
\begin{equation}\label{eq:transformedfunction}
\sqrt{x^2+y^2}=\sqrt{r^2\cos^2\theta+r^2\sin^2\theta}=r\sqrt{\cos^2\theta+\sin^2\theta} = r.
\end{equation}
The final, and most difficult, task is to figure out what replaces
$dx\,dy$. (Of course, we actually know the answer, because we are in
effect converting to polar coordinates. What we really want is a
series of steps that gets to that right answer but that will also work
for other substitutions that are not so familiar.)

Let's take a step back and remember how integration arises from
approximation. When we approximate the integral in the $x$-$y$ plane,
we are computing the volumes of tall thin boxes, in this case boxes
that are $\Delta x\times \Delta y\times \sqrt{x^2+y^2}$. We are aiming
to come up with an integral in the $r$-$\theta$ plane that looks like
this:
\begin{equation}\label{eq:targetintegral}
\int_0^\pi\int_0^1 r (?) \,dr\,d\theta.
\end{equation}
What we're missing is exactly the right quantity to replace the ``?''
so that we get the correct answer. Of course, this integral is also
the result of an approximation, in which we add up volumes of boxes
that are $\Delta r\times\Delta \theta\times\hbox{height}$; the problem
is that the height that will give us the correct answer is not simply
$r$. Or put another way, we can think of the correct height as $r$,
but the area of the base $\Delta r\Delta\theta$ as being wrong. The
height $r$ comes from Equation~\ref{eq:transformedfunction}, which
is to say, it is precisely the same as the corresponding height in the
$x$-$y$ version of the integral. The problem is that the area of the
base $\Delta x\times \Delta y$ is not the same as the area of the base
$\Delta r\times\Delta\theta$. We can think of the ``?'' in the integral as a
correction factor that is needed so that
$?\,dr\,d\theta$ = $dx\,dy$.

So let's think about what that little base $\Delta
r\times\Delta\theta$ corresponds to. We know that each bit of
horizontal line in the $r$-$\theta$ plane corresponds to a bit of
circular arc in the $x$-$y$ plane, and each bit of vertical line
in the $r$-$\theta$ plane corresponds to a bit of
``radial line'' in the $x$-$y$ plane. In
Figure~\ref{fig:correspondingareas} we show a typical rectangle in
the $r$-$\theta$ plane and its corresponding area in the $x$-$y$ plane.

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <2truecm,2truecm>
\setplotarea x from 0 to 3.2, y from 0 to 1.1
\axis left ticks numbered from 0 to 1 by 1 /
\axis bottom  ticks withvalues {$\pi$} / at 3.14 / /
\put {$\theta$} [l] <3pt,0pt> at 3.2 0
\put {$r$} [b] <0pt,3pt> at 0 1.1
\put {$\Delta r$} [r] <-3pt,0pt> at 2.094 0.625
\put {$\Delta\theta$} [t] <0pt,-3pt> at 2.356 0.5
\plot 2.094 0.5 2.094 0.75 2.618 0.75 2.618 0.5 2.094 0.5 /
\setcoordinatesystem units <2truecm,2truecm> point at -6 0
\setplotarea x from -1.1 to 1.1, y from 0 to 1.1
\axis left shiftedto x=0 /
\axis bottom  ticks numbered from -1 to 1 by 1 /
\put {$x$} [l] <3pt,0pt> at 1.1 0
\put {$y$} [b] <0pt,3pt> at 0 1.1
%\put {$\Delta r$} [tr] <-2pt,-2pt> at -0.54 0.31
%\put {$r\Delta\theta$} [br] <-2pt,2pt> at -0.53 0.53
\circulararc 30 degrees from -0.25 0.433 center at 0 0
\circulararc 30 degrees from -0.375 0.65 center at 0 0
\plot -0.25 0.433 -0.375 0.65 /
\plot -0.433 0.25 -0.65 0.375 /
\endpicture}}
\caption{Corresponding areas.}
\label{fig:correspondingareas}
\end{figure}

In this case, the region in the $x$-$y$ plane is approximately a
rectangle with dimensions $\Delta r\times r\Delta\theta$, but in
general the corner angles will not be right angles, so the region will
typically be (almost) a parallelogram. We need to compute the area of
this parallelogram. We know
a neat way to do this: compute the length of a certain cross product.
If we can determine an appropriate two vectors we'll be nearly done.

Fortunately, we've really done this before. The sides of the region in
the $x$-$y$ plane are formed by temporarily fixing either $r$ or
$\theta$ and letting the other variable range over a small interval.
In Figure~\ref{fig:correspondingareas}, for example, the upper right
edge of the region is formed by fixing $\theta=2\pi/3$ and letting $r$
run from $0.5$ to $0.75$. In other words, we have a vector function
$\vect{v}(r)=\langle r\cos\theta_0, r\sin\theta_0, 0\rangle$, and we
are interested in a restricted set of values for $r$. A vector tangent
to this path is given by the derivative $\vect{v}'(r)=\langle
\cos\theta_0, \sin\theta_0, 0\rangle$, and a small tangent vector,
with length approximately equal to the side of the region, is $\langle
\cos\theta_0, \sin\theta_0, 0\rangle\,dr$. Likewise, if we fix
$r=r_0=0.5$, we get the vector function $\vect{w}(\theta)=\langle
r_0\cos\theta, r_0\sin\theta, 0\rangle$ with derivative $\vect{w}(\theta)=\langle -r_0\sin\theta, r_0\cos\theta, 0\rangle$ and a
small tangent vector $\langle -r_0\sin\theta_0, r_0\cos\theta_0,
0\rangle\,d\theta$ when $\theta=\theta_0$ (at the corner we're
focusing on). These vectors are shown in
Figure~\ref{fig:smallapproximatingparallelogram}, with the actual
region outlined by a dotted boundary. Of course, since both $\Delta r$
and $\Delta\theta$ are quite large, the parallelogram is not a
particularly good approximation to the true area.

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <3truecm,3truecm>
\setplotarea x from -1.1 to 1.1, y from 0 to 1.1
\axis left shiftedto x=0 /
\axis bottom  ticks numbered from -1 to 1 by 1 /
\put {$x$} [l] <3pt,0pt> at 1.1 0
\put {$y$} [b] <0pt,3pt> at 0 1.1
%\put {$\Delta r$} [tr] <-2pt,-2pt> at -0.54 0.31
%\put {$r\Delta\theta$} [br] <-2pt,2pt> at -0.53 0.53
\arrow <5pt> [0.17, 0.5] from  -0.25 0.433  to  -0.375 0.65
\arrow <5pt> [0.17, 0.5] from -0.25 0.433 to -0.477  0.302
\setdashes
\plot -0.477  0.302 -0.602 0.519 -0.375 0.65 /
\findlength{\circulararc 30 degrees from -0.25 0.433 center at 0 0 }
\setdotsnear <2pt> for <\totalarclength>
\circulararc 30 degrees from -0.25 0.433 center at 0 0
\findlength{\circulararc 30 degrees from -0.375 0.65 center at 0 0 }
\setdotsnear <2pt> for <\totalarclength>
\circulararc 30 degrees from -0.375 0.65 center at 0 0
\setlinear
\plot -0.433 0.25 -0.65 0.375 /
\endpicture}}
\caption{The approximating parallelogram.}
\label{fig:smallapproximatingparallelogram}
\end{figure}

The area of this parallelogram is the length of the cross product:
\begin{align*}
\langle -r_0\sin\theta_0, r_0\cos\theta_0, 0\rangle\,d\theta\times\langle\cos\theta_0, \sin\theta_0, 0\rangle\,dr
&= \left|\begin{matrix}	\vect{i}			&	\vect{j}		&	\vect{k}	\\
						-r_0\sin\theta_0	&	r_0\cos\theta_0	&	0			\\
						\cos\theta_0		&	\sin\theta_0	&	0
		 \end{matrix}\right|\,d\theta\,dr	\\
&=\langle 0,0,-r_0\sin^2\theta_0-r_0\cos^2\theta_0\rangle\,d\theta\,dr	\\
&=\langle 0,0,-r_0\rangle\,d\theta\,dr.
\end{align*}
The length of this vector is $r_0\,dr\,d\theta$. So in general, for
any values of $r$ and $\theta$, the area in the $x$-$y$ plane
corresponding to a small rectangle anchored at $(\theta,r)$ in the
$r$-$\theta$ plane is approximately $r\,dr\,d\theta$. In other words, ``$r$''
replaces the ``?'' in Equation~\ref{eq:targetintegral}.

In general, a substitution will start with equations $x=f(u,v)$ and
$y=g(u,v)$. Again, it will be straightforward to convert the function
being integrated. Converting the limits will require, as above, an
understanding of just how the functions $f$ and $g$ transform the
$u$-$v$ plane into the $x$-$y$ plane. Finally, the small vectors we
need to approximate an area will be
$\langle f_u,g_u,0\rangle\,du$ and $\langle f_v,g_v,0\rangle\,dv$.
The cross product of these is $\langle
0,0,f_ug_v-g_uf_v\rangle\,du\,dv$ with length
$|f_ug_v-g_uf_v|\,du\,dv$. The quantity $|f_ug_v-g_uf_v|$ is usually
denoted
$$\left|{\partial(x,y)\over\partial(u,v)}\right|=|f_ug_v-g_uf_v|$$
and called the \dfont{Jacobian}\index{Jacobian}.
Note that this is the absolute value of the two by two determinant
\[\left|\begin{matrix}	f_u	&	g_u	\\
						f_v	&	g_v
		\end{matrix}\right|,\]
which may be easier to remember. (Confusingly, the matrix, the
determinant of the matrix, and the absolute value of the determinant
are all called the Jacobian by various authors.)

Because there are two things to worry about, namely, the form of the
function and the region of integration, transformations in two (or
more) variables are quite tricky to discover.

\begin{example}{Integral of an Ellipse}{changeofvariables}
Integrate $x^2-xy+y^2$ over the region $x^2-xy+y^2\le 2$.
\end{example}
\begin{solution}
The equation $x^2-xy+y^2= 2$ describes an ellipse as in
Figure~\ref{fig:skewedellipse}; the region of integration is the
interior of the ellipse. We will use the transformation
$x=\sqrt2 u-\sqrt{2/3}v$, $y=\sqrt2 u+\sqrt{2/3}v$.
Substituting into the function itself we get
\[x^2-xy+y^2=2u^2+2v^2.\]
The boundary of the ellipse is $x^2-xy+y^2=2$, so the boundary of
the corresponding region in the $u$-$v$ plane is $2u^2+2v^2=2$
or $u^2+v^2=1$, the unit circle, so this substitution makes the region
of integration simpler.

Next, we compute the Jacobian, using $f=\sqrt2 u-\sqrt{2/3}v$ and
$g=\sqrt2 u+\sqrt{2/3}v$:
\[f_ug_v-g_uf_v=\sqrt2\sqrt{2/3}+\sqrt2\sqrt{2/3}={4\over\sqrt3}.\]
Hence the new integral is
\[\iint_R (2u^2+2v^2){4\over\sqrt3}\,du\,dv,\]
where $R$ is the interior of the unit circle.
This is still not an easy integral, but it is easily transformed to
polar coordinates, and then easily integrated.
\end{solution}

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <1.2truecm,1.2truecm>
\setplotarea x from -2 to 2, y from -2 to 2
\axis left shiftedto x=0 /
\axis bottom shiftedto y=0  /
\put {$x$} [l] <3pt,0pt> at 2 0
\put {$y$} [b] <0pt,3pt> at 0 2
\plot -1.630 -0.729 -1.548 -0.325 -1.467 -0.112 -1.386 0.056 -1.304 0.199
-1.222 0.326 -1.141 0.441 -1.060 0.546 -0.978 0.644 -0.896 0.734
-0.815 0.818 -0.734 0.897 -0.652 0.971 -0.570 1.040 -0.489 1.105
-0.408 1.166 -0.326 1.223 -0.244 1.276 -0.163 1.326 -0.082 1.372
0.000 1.414 0.082 1.453 0.163 1.489 0.244 1.521 0.326 1.549
0.408 1.573 0.489 1.594 0.570 1.610 0.652 1.623 0.734 1.630
0.815 1.633 0.896 1.630 0.978 1.622 1.060 1.606 1.141 1.582
1.222 1.549 1.304 1.503 1.386 1.441 1.467 1.355 1.548 1.223
1.630 0.901 /
\plot -1.630 -0.901 -1.548 -1.223 -1.467 -1.355 -1.386 -1.441 -1.304 -1.503
-1.222 -1.549 -1.141 -1.582 -1.060 -1.606 -0.978 -1.622 -0.896 -1.630
-0.815 -1.633 -0.734 -1.630 -0.652 -1.623 -0.570 -1.610 -0.489 -1.594
-0.408 -1.573 -0.326 -1.549 -0.244 -1.521 -0.163 -1.489 -0.082 -1.453
0.000 -1.414 0.082 -1.372 0.163 -1.326 0.244 -1.276 0.326 -1.223
0.408 -1.166 0.489 -1.105 0.570 -1.040 0.652 -0.971 0.734 -0.897
0.815 -0.818 0.896 -0.734 0.978 -0.644 1.060 -0.546 1.141 -0.441
1.222 -0.326 1.304 -0.199 1.386 -0.056 1.467 0.112 1.548 0.325
1.630 0.729 /
\plot 1.63 0.729 1.633 0.8164965809 1.63 0.901 /
\plot -1.63 -0.729 -1.633 -0.8164965809 -1.63 -0.901 /
\endpicture}}
\caption{$x^2-xy+y^2=2$}
\label{fig:skewedellipse}
\end{figure}

There is a similar change of variables formula for triple integrals,
though it is a bit more difficult to derive.
Suppose we use three substitution functions, $x=f(u,v,w)$,
$y=g(u,v,w)$, and $z=h(u,v,w)$.  The Jacobian determinant is now
\[
  {\partial(x,y,z)\over\partial(u,v,w)} =
  \left|\begin{matrix}	f_u	&	g_u	&	h_u	\\
  						f_v	&	g_v	&	h_v	\\
  						f_w	&	g_w	&	h_w
  		\end{matrix}\right|.
\]
Then the integral is transformed in a similar fashion:
\[
  \iiint_R F(x,y,z) \, dV =
  \iiint_S F(f(u,v,w),g(u,v,w),h(u,v,w))
  \left|{\partial(x,y,z)\over\partial(u,v,w)}\right| \,du\,dv\,dw,
\]
where of course the region $S$ in $uvw$ space corresponds to the
region $R$ in $xyz$ space.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:ChangeofVariables}}

\begin{enumialphparenastyle}

\begin{ex}
Complete Example~\ref{exa:changeofvariables} by
converting to polar coordinates and evaluating the integral.
\begin{sol}
$\ds 4\pi\sqrt3/3$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint xy\,dx\,dy$ over the square
with corners $(0,0)$, $(1,1)$, $(2,0)$, and $(1,-1)$ in two ways:
directly, and using $x=(u+v)/2$, $y=(u-v)/2$.
\begin{sol}
$0$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint x^2+y^2\,dx\,dy$ over the square
with corners $(-1,0)$, $(0,1)$, $(1,0)$, and $(0,-1)$ in two ways:
directly, and using $x=(u+v)/2$, $y=(u-v)/2$.
\begin{sol}
$2/3$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint (x+y)e^{x-y}\,dx\,dy$ over the triangle
with corners $(0,0)$, $(-1,1)$, and $(1,1)$ in two ways:
directly, and using $x=(u+v)/2$, $y=(u-v)/2$.
\begin{sol}
$\ds {e^2-1\over 2e^2}$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint y(x-y)\,dx\,dy$ over the parallelogram
with corners $(0,0)$, $(3,3)$, $(7,3)$, and $(4,0)$ in two ways:
directly, and using $x=u+v$, $y=u$.
\begin{sol}
$36$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint \sqrt{x^2+y^2}\,dx\,dy$ over the
triangle with corners $(0,0)$, $(4,4)$, and $(4,0)$ using $x=u$, $y=uv$.
\begin{sol}
$32(\sqrt2+\ln(1+\sqrt2))/3$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint y\sin(xy)\,dx\,dy$ over the
region bounded by $xy=1$, $xy=4$, $y=1$, and $y=4$ using
$x=u/v$, $y=v$.
\begin{sol}
$3\cos(1)-3\cos(4)$
\end{sol}
\end{ex}

\begin{ex}
Evaluate $\ds\iint \sin(9x^2 + 4y^2)\,dA,$ over the region
 in the first quadrant bounded by the ellipse $9x^2+4y^2 = 1$.
\begin{sol}
$\pi(1-\cos(1))/24$
\end{sol}
\end{ex}

\begin{ex}
Compute the Jacobian for the substitutions
$x=\rho\sin\phi\cos\theta$, $y=\rho\sin\phi\sin\theta$,
$z=\rho\cos\phi$.
\end{ex}

\begin{ex}
Evaluate $\ds\iiint_E\,dV$ where $E$ is the solid
enclosed by the ellipsoid
$${x^2\over a^2} + {y^2\over b^2} + {z^2\over c^2} = 1,$$
using the transformation $x=au$, $y=bv$, and $z=cw$.
\begin{sol}
$(4/3)\pi abc$
\end{sol}
\end{ex}

\end{enumialphparenastyle}


\section*{Ex.3.2}
\subsection*{what is the upper bound}

As $h$ is strongly $c$-universal the keys hash independently, hence

\begin{align*}
P(h(x)=q \land h(y)=r) &= P(h(x)=q \cap h(y)=r) = P(h(x)=q) P(h(y)=r) \\
                       &\leq \frac{c}{m} \frac{c}{m} = \left(\frac{c}{m}\right)^2
\end{align*}

So the upper bound is $c^2/m^2$


\subsection{Bellman equations}

We breakdown the value function into an immediate reward, and the discounted value function of the next state.

This is because the expectation function is linear.

\(v_\pi (s)=R_{s,\pi(s)}+\gamma \sum_{s'}P_{s,\pi(s)}(s')v_\pi (s')\)

We can write this in matrix form.

\(v_pi (s)= r_\pi + \gamma P_\pi v_\pi(s)\)

We can then solve this:

\(v_pi (s)= (I-\gamma P_\pi)^{-1})r_\pi\)

This depends on the starting state.


\section{Optimization Methods}

In order to explain the \emph{convergence} and \emph{efficiency} properties of the following optimization methods, we need to introduce some preliminary definitions about \emph{convexity} and the \emph{L-smoothness} of a function~\cite{boyd2004convex}.

\begin{definition}[Convexity] \label{def:convexity}
	\hfill
	\begin{enumerate}[i]
		\item We say that a function $f: \Re^m \rightarrow \Re$ is convex if:
		$$
			(\lambda x + (1 - \lambda) y ) \leq \lambda f(x) + (1 - \lambda) f(y) \ \forall \ x, y \in \Re^m, \lambda \in [0,1]
		$$
		\item We say that a differentiable function $f: \Re^m \rightarrow \Re$, i.e., $f \in C^1$, is convex if:
		$$
			f(x) \geq f(y) + \langle \nabla f(y), x - y \rangle \ \forall \ x, y \in \Re^m
		$$
		\item We say that a twice differentiable function $f: \Re^m \rightarrow \Re$, i.e., $f \in C^2$ and the Hessian matrix is \emph{symmetric}, is convex iff:
		$$
			\nabla^2 f(x) \succeq 0 \ \forall \ x \in \Re^m
		$$ i.e., the Hessian matix is \emph{positive semidefinite}.
	\end{enumerate}
\end{definition}

\begin{definition}[Strong Convexity] \label{def:strong_convexity}
We say that a function $f: \Re^m \rightarrow \Re$ is $\mu$-strongly convex if the function:
$$
g(x) = f(x) - \frac{\mu}{2} \| x \|^2
$$
is convex for any $\mu > 0$.
If $f$ is differentiable, i.e., $f \in C^1$, this is also equivalent to:
$$
f(y) \geq f(x) + \langle \nabla f(x), y - x \rangle + \frac{\mu}{2} \| y - x \|^2 \ \forall \ x, y \in \Re^m
$$
and, if $f$ is a twice differentiable function, i.e., $f \in C^2$ and the Hessian matrix is \emph{symmetric}, then $f$ is $\mu$-strongly convex iff:
$$
\nabla^2 g(x) \succ 0 \ \forall \ x \in \Re^m
$$
i.e., the Hessian matix is \emph{positive definite}, which is:
$$
\nabla^2 f(x) \succeq \mu \ \forall \ x \in \Re^m
$$
\end{definition}

\begin{definition}[L-smoothness] \label{def:l_smoothness}
We say that a function $f: \Re^m \rightarrow \Re$ is L-smooth, i.e., L-Lipschitz continuous, if it is differentiable and if:
$$
\| \nabla f(x) - \nabla f(y) \| \leq L \| x - y \| \ \forall \ x, y \in \Re^m
$$
\end{definition}

\pagebreak

\subsection{Gradient Descent}

The Gradient Descent algorithm is the simplest \emph{first-order optimization} method that exploits the orthogonality of the gradient wrt the level sets to take a descent direction. In particular, it performs the following iterations:

\begin{algorithm}[H]
	\caption{Gradient Descent}
	\label{alg:gd}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Function{GradientDescent}{$f,\alpha$}
			\State Initialize weight vector $x_0$
			\State $t = 0$
			\While{$not\_convergence$}
				\State $x_{t+1} = x_t - \alpha \nabla f(x_t)$
				\State $t = t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

\begin{theorem}[Gradient Descent convergence for convex functions] \label{thm:cvx_gd_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-smooth convex function. Let $x_*$ be the minimum of $f$ on $\Re^m$. Then the Gradient Descent with step size $\alpha \leq 1/L$ satisfies:
$$
f(x_t) - f(x_*) \leq \frac{\| x_0 - x_* \|^2}{2 \alpha t}
$$
In particular, for $\alpha = 1/L$:
$$
f(x_t) - f(x_*) \leq \frac{L \| x_0 - x_* \|^2}{2 t}
$$
\end{theorem}

\begin{theorem}[Gradient Descent convergence for strongly convex functions] \label{thm:str_cvx_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-smooth, $\mu$ strongly convex function. Let $x_*$ be the minimum of $f$ on $\Re^m$. Then the Gradient Descent with step size $\alpha \leq 1/L$ satisfies:
$$
f(x_t) - f(x_*) \leq (1 - \alpha \mu)^t \| x_0 - x_* \|^2
$$
In particular, for $\alpha = 1/L$:
$$
\begin{aligned}
	f(x_t) - f(x_*) \leq & \bigg(1 - \frac{\mu}{L}\bigg)^t \| x_0 - x_* \|^2 \\
						= & \bigg(1 - \frac{1}{\kappa}\bigg)^t \| x_0 - x_* \|^2
\end{aligned}
$$
where $\kappa = L/\mu$.
\end{theorem}

Gradient Descent is based on full gradients, since at each iteration we compute the average gradient on the whole dataset:
$$
\nabla f(x) = \frac{1}{n} \sum_{i=1}^n \nabla f_i(x)
$$
The downside is that every step is very computationally expensive, $\mathcal{O}(nm)$ per iteration, where $n$ is the number of samples in our dataset and $m$ is the number of dimensions.

Since \emph{Gradient Descent} becomes impractical when dealing with large datasets we introduce a stochastic version, called \emph{Stochastic Gradient Descent}, which does not use the whole set of examples to compute the gradient at every step. By doing so, we can reduce computation all the way down to $\mathcal{O}(m)$ per iteration, instead of $\mathcal{O}(nm)$.

\begin{algorithm}[H]
	\caption{Stochastic Gradient Descent}
	\label{alg:sgd}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Require{Batch size $k$}
		\Function{StochasticGradientDescent}{$f,\alpha,k$}
			\State Initialize weight vector $x_0$
			\State $t \gets 0$
			\While{$not\_convergence$}
				\State Sample $(i_1,\dots,i_k) \sim \mathcal{U}^k(1,\dots,n)$
				\State $\displaystyle x_{t+1} \gets x_t - \alpha \frac{1}{k} \sum_{j=1}^k \nabla f_{i_j}(x_t)$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

Note that in expectation, we converge like GD, since $\displaystyle \mathbb{E}_{i \sim \mathcal{U}(1,\dots,n)}[\nabla f_i(x_t)] = \nabla f(x_t)$, therefore, the expected iterate of SGD converges to the optimum.

SGD’s convergence rate for L-smooth convex functions is $\displaystyle \mathcal{O}\Big(\frac{1}{\sqrt{t}}\Big)$ and $\displaystyle \mathcal{O}\Big(\frac{1}{t}\Big)$ for strongly convex. More iterations are needed to reach the same accuracy as GD, but the iterations are far cheaper.

\subsubsection{Momentum}

To mitigate the pathological zig-zagging of the SGD method we introduce two accelerated methods~\cite{polyak1964some} and~\cite{nesterov1998introductory} that exploits information from the history, i.e., past iterates, to add some inertia, i.e., the momentum, to yield smoother trajectory.

In the Polyak's method~\cite{polyak1964some} the velocity vector $v_t$ is calculated by applying the $\beta$ momentum to the previous $v_{t-1}$ displacement, and subtracting the gradient step to $x_t$.

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.5]{img/momentum}
  	\caption{Polyak's and Nesterov's Momentum}
  	\label{fig:momentum}
\end{figure}

\begin{algorithm}[H]
	\caption{Polyak Accelerated Gradient Descent or or Heavy-Ball method}
	\label{alg:hbg}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Require{Momentum $\beta \in [0,1)$}
		\Function{PolyakAcceleratedGradientDescent}{$f,\alpha,\beta$}
			\State Initialize weight vector $x_1 \gets x_0$ and velocity vector $v_0 \gets 0$
			\State $t \gets 1$
			\While{$not\_convergence$}
				\State $v_t = \beta v_{t-1} + \alpha \nabla f(x_t)$
				\State $x_{t+1} = x_t - v_t$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

Leveraging the idea of momentum introduced by Polyak, Nesterov introduced a slightly altered update rule that has been shown to converge not only for quadratic functions, but for general convex functions. In the Nesterov's method~\cite{nesterov1998introductory}, instead, the velocity vector $v_t$ is calculated by applying the $\beta$ momentum to the previous $v_{t-1}$ displacement, and subtracting the gradient step to $x_t + \beta v_{t-1}$, which is the point where the momentum term leads from $x_t$.

\begin{algorithm}[H]
	\caption{Nesterov Accelerated Gradient Descent}
	\label{alg:nag}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate $\alpha > 0$}
		\Require{Momentum $\beta \in [0,1)$}
		\Function{NesterovAcceleratedGradientDescent}{$f,\alpha,\beta$}
			\State Initialize weight vector $x_1 \gets x_0$ and velocity vector $v_0 \gets 0$
			\State $t \gets 1$
			\While{$not\_convergence$}
				\State $\hat{x}_t \gets x_t + \beta v_{t-1}$
				\State $v_t \gets \beta v_{t-1} + \alpha \nabla f(\hat{x}_t)$
				\State $x_{t+1} \gets x_t - v_t$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

Comparing the algorithm \ref{alg:hbg} with the algorithm \ref{alg:nag}, we can see that Polyak’s method evaluates the gradient before adding momentum, whereas Nesterov’s algorithm evaluates it after applying momentum, which intuitively brings us closer to the minimum $x^*$, as showb in figure \ref{fig:momentum}.

\begin{theorem}[Nesterov Accelerated Gradient Descent convergence for convex functions] \label{thm:cvx_nag_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-smooth convex function. Let $x_*$ be the minimum of $f$ on $\Re^m$. Then the Gradient Descent with step size $\alpha \leq 1/L$ and $\beta_{t+1} = t / (t+3)$ satisfies:
$$
f(x_t) - f(x_*) \leq \frac{2 \| x_0 - x_* \|^2}{\alpha (t+1)^2}
$$
In particular, for $\alpha = 1/L$:
$$
f(x_t) - f(x_*) \leq \frac{2L \| x_0 - x_* \|^2}{(t+1)^2}
$$
\end{theorem}

\begin{theorem}[Nesterov Accelerated Gradient Descent convergence for strongly convex functions] \label{thm:str_cvx_nag_convergence}
Let $f: \Re^m \rightarrow \Re$ be a L-smooth, $\mu$ strongly convex function. Let $x_*$ be the minimum of $f$ on $\Re^m$. Then the Gradient Descent with step size $\alpha \leq 1/L$ and $\beta_t = \displaystyle \frac{1 - \sqrt{\mu / L}}{1 + \sqrt{\mu / L}} = \frac{1-1/\sqrt{\kappa}}{1+1/\sqrt{\kappa}}$ satisfies:
$$
\begin{aligned}
	f(x_t) - f(x_*) \leq & \frac{\| x_0 - x_* \|^2}{\alpha} \Bigg(1 - \sqrt{\frac{\mu}{L}}\Bigg)^t \\
						= & \frac{\| x_0 - x_* \|^2}{\alpha} \Bigg(1 - \frac{1}{\sqrt{\kappa}}\Bigg)^t
\end{aligned}
$$
In particular, for $\alpha = 1/L$:
$$
\begin{aligned}
	f(x_t) - f(x_*) \leq & L \| x_0 - x_* \|^2 \Bigg(1 - \sqrt{\frac{\mu}{L}}\Bigg)^t \\
						= & L \| x_0 - x_* \|^2 \Bigg(1 - \frac{1}{\sqrt{\kappa}}\Bigg)^t
\end{aligned}
$$
where $\kappa = L/\mu$.
\end{theorem}

Nesterov momentum brings the rate of convergence from $\displaystyle \mathcal{O}\Big(\frac{1}{t}\Big)$ to $\displaystyle \mathcal{O}\Big(\frac{1}{t^2}\Big)$ and in the case of smooth and strongly convex functions gives the acceleration that we had with Polyak’s momentum for quadratic functions. This is great, because we get the guarantee for a more general class of functions.

\bigskip

We can write the iteration complexity of these methods, i.e., the smallest $t$ such that we’re within $\epsilon$, for a L-smooth and $\mu$-strongly convex function as $\displaystyle \mathcal{O}\Big(\kappa \log \frac{1}{\epsilon}\Big)$ for the standard GD method, $\displaystyle \mathcal{O}\Big(\sqrt{\kappa}\log \frac{1}{\epsilon}\Big)$ for the Polyak’s method and, finally, $\displaystyle \mathcal{O}\Big(\frac{1}{\sqrt{\epsilon}}\Big)$ for the NAG method to get $\epsilon$-close to global optimum and where $\kappa$, i.e., the \emph{conditioning number}, is defined as $\kappa = L/\mu$ and where $L$ and $\mu$ are also equal to the smallest and the largest eigenvalues $\lambda_{min}$ and $\lambda_{max}$ respectively.

\pagebreak

\subsection{AdaGrad}

Due to the sparsity of the weight vector of the \emph{Lagrangian dual}, i.e., the Lagrange multipliers, we might end up in a situation where some components of the gradient are very small and others large. This, in terms of \emph{conditioning number}, i.e., $\kappa = L/\mu \gg 1$, means that the level sets of $f$ are ellipsoid, i.e., we are dealing with an ill-conditioned problem. So, given a learning rate, a standard gradient descent approach might end up in a situation where it decreases too quickly the small weights or too slowly the large ones.

Another method, that is usually deprecated in ML applications due to its increased computational complexity, is Newton’s method. Newton’s method favors a much faster convergence rate, i.e., number of iterations, at the cost of being more expensive per iteration. For convex problems, the recursion is similar to the gradient descent algorithm:

$$
x_{t+1} = x_t - \alpha H^{-1} \nabla f(x_t)
$$

where $\alpha$ is often close to one (damped-Newton) or one, and $H^{-1}$ denotes the Hessian of $f$ at the current point, i.e., $\nabla^2 f(x_t)$.

The above suggest a general rule in optimization: find any preconditioner, in convex optimization it has to be positive semidefinite, that improves the performance of gradient descent in terms of iterations, but without wasting too much time to compute that precoditioner. The above result into:

$$
x_{t+1} = x_t - \alpha P^{-1} \nabla f(x_t)
$$

where $P$ is the preconditioner. This idea is the basis of the BFGS quasi-Newton method.

The \emph{AdaGrad}~\cite{duchi2011adaptive} algorithm is just a variant of preconditioned gradient descent, where $P$ is selected to be a diagonal preconditioner matrix and is updated using the gradient information, in particular it is the diagonal approximation of the inverse of the square roots of gradient outer products, until the $k$-th iteration. The above lead to the algorithm:

\begin{algorithm}[H]
	\caption{AdaGrad}
	\label{alg:adagrad}
	\begin{algorithmic}
		\Require{Function $f$ to minimize}
		\Require{Learning rate or step size $\alpha > 0$}
		\Require{Offset $\epsilon > 0$ to ensures not divide by 0}
		\Function{AdaGrad}{$f,\alpha,\epsilon$}
			\State Initialize weight vector $x_0$ and the squared accumulated gradients vector $s_t \gets 0$
			\State $t = 1$
			\While {$not\_convergence$}
				\State $g_t \gets \nabla f(x_t)$
				\State $s_t \gets s_{t-1} + g_t^2$
				\State $x_{t+1} \gets x_t - \alpha P_t^{-1} g_t = x_t - \displaystyle \frac{\alpha}{\sqrt{s_t + \epsilon}} \odot g_t \ \text{where} \ P_t \gets diag(s_t + \epsilon)^{1/2}$
				\State $t \gets t + 1$
			\EndWhile
			\State \Return $x_t$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

In practical terms, \emph{AdaGrad} addresses the problem of the sparse optimal by adaptively scaling the learning rate for each dimension with the magnitude of the gradients. Coordinates that routinely correspond to large gradients are scaled down significantly, whereas others with small gradients receive a much more gentle treatment. \emph{AdaGrad}'s convergence rate for L-smooth convex functions is $\displaystyle \mathcal{O}\Big(\frac{1}{\sqrt{t}}\Big)$.

\pagebreak

\subsection{Sequential Minimal Optimization}

The \emph{Sequential Minimal Optimization (SMO)}~\cite{platt1998sequential} method is the most popular approach for solving the SVM QP problem without any extra $Q$ matrix storage required by common QP methods. The advantage of SMO lies in the fact that it performs a series of two-point optimizations since we deal with just one equality constraint, so the Lagrange multipliers can be solved analitically.

\subsubsection{Classification}

At each iteration, SMO chooses two $\alpha_i$ to jointly optimize, let $\alpha_1$ and $\alpha_2$, finds the optimal values for these multipliers and update the SVM to reflect these new values. In order to solve for two Lagrange multipliers, SMO first computes the constraints over these and then solves for the constrained minimum. Since there are only two multipliers, the box-constraints cause the Lagrange multipliers to lie within a box, while the linear equality constraint causes the Lagrange multipliers to lie on a diagonal line inside the box. So, the constrained minimum must lie there as shown in \ref{fig:smo_lagrange_multipliers}.

\begin{figure}[h!]
	\centering
	\includegraphics[scale=0.5]{img/smo_multipliers}
	\caption{SMO for two Lagrange multipliers}
	\label{fig:smo_lagrange_multipliers}
\end{figure}

In case of classification the ends of the diagonal line segment, i.e., the lower and upper bounds, can be espressed as follow if the target $y_1 \ne y_2$:

\begin{equation} \label{eq:smo_svc_bounds_update1}
	\begin{aligned}
		& L = max(0, \alpha_2 - \alpha_1) \\
		& H = min(C, C + \alpha_2 - \alpha_1)
	\end{aligned}
\end{equation}

or, alternatively, if the target $y_1 = y_2$:

\begin{equation} \label{eq:smo_svc_bounds_update2}
	\begin{aligned}
		& L = max(0, \alpha_2 + \alpha_1 - C) \\
		& H = min(C, \alpha_2 + \alpha_1)
	\end{aligned}
\end{equation}

The second derivative of the objective quadratic function along the diagonl line can be expressed as:

\begin{equation} \label{eq:smo_eta}
	\eta = K(x_1, x_1) + K(x_2, x_2) - 2K(x_1, x_2)
\end{equation}

that will be grather than zero if the kernel matrix will be positive definite, so there will be a minimum along the linear equality constraints that will be:

\begin{equation} \label{eq:smo_svc_a2_new}
	\alpha_2^{new} = \alpha_2 + \frac{y_2(E_1 - E_2)}{\eta}
\end{equation}

where $E_i = y_i - y'_i$ is the error on the $i$-th training example and $y'_i$ is the output of the SVC for the same.

Then, the box-constrained minimum is found by clipping the unconstrained minimum to the ends of the line segment:

\begin{equation} \label{eq:smo_svc_a2_new_clipped}
    \alpha_2^{new,clipped} =
        \begin{cases}
            H & \text{if} \ \alpha_2^{new} \geq H \\
            \alpha_2^{new} & \text{if} \ L < \alpha_2^{new} < H \\
            L & \text{if} \ \alpha_2^{new} \leq L \\
        \end{cases}
\end{equation}

Finally, the value of $\alpha_1$ is computed from the new clipped $\alpha_2$ as:

\begin{equation} \label{eq:smo_svc_a1_new}
	\alpha_1^{new} = \alpha_1 + s (\alpha_2 - \alpha_2^{new,clipped})
\end{equation}

where $s = y_1 y_2$.

Since the \emph{Karush-Kuhn-Tucker} conditions are necessary and sufficient conditions for optimality of a positive definite QP problem and the KKT conditions for the classification problem~\eqref{eq:svc_min_wolfe_dual} are:

\begin{equation} \label{eq:svc_smo_kkt}
	\begin{aligned}
		\alpha_i = 0 & \Leftrightarrow y_i y'_i \geq 1 \\
		0 < \alpha_i < C & \Leftrightarrow y_i y'_i = 1 \\
		\alpha_i = C & \Leftrightarrow y_i y'_i \leq 1
	\end{aligned}
\end{equation}

the steps described above will be iterate as long as there will be an example that violates them.

After optimizing $\alpha_1$ and $\alpha_2$, we select the threshold $b$ such that the KKT conditions are satisfied for $x_1$ and $x_2$. If, after optimization, $\alpha_1$ is not at the bounds, i.e., $0 < \alpha_1 < C$, then the following threshold $b_{up}$ is valid, since it forces the SVC to output $y_1$ when the input is $x_1$:

\begin{equation} \label{eq:smo_svc_b1}
	b_{up} = E_1 + y_1 (\alpha_1^{new} - \alpha_1) K(x_1,x_1) + y_2 (\alpha_2^{new,clipped} - \alpha_2) K(x_1,x_2) + b
\end{equation}

similarly, the following threshold $b_{low}$ is valid if $0 < \alpha_2 < C$:

\begin{equation} \label{eq:smo_svc_b2}
	b_{low} = E_2 + y_1 (\alpha_1^{new} - \alpha_1) K(x_1,x_2) + y_2 (\alpha_2^{new,clipped} - \alpha_2) K(x_2,x_2) + b
\end{equation}

If, after optimization, both $0 < \alpha_1 < C$ and $0 < \alpha_2 < C$ then both these thresholds are valid, and they will be equal; else, if both $\alpha_1$ and $\alpha_2$ are at the bounds, i.e., $\alpha_1 = 0$ or $\alpha_1 = C$ and $\alpha_2 = 0$ or $\alpha_2 = C$, then all the thresholds between $b_{up}$ and $b_{low}$ satisfy the KKT conditions, so we choose the threshold to be halfway in between $b_{up}$ and $b_{low}$. This gives the complete equation for $b$:

\begin{equation} \label{eq:smo_svc_b}
	b =
        \begin{cases}
            b_{up} & \text{if} \ 0 < \alpha_1 < C \\
            b_{low} & \text{if} \ 0 < \alpha_2 < C \\
            \displaystyle \frac{b_{up}+b_{low}}{2} & \text{otherwise} \\
        \end{cases}
\end{equation}

\newpage

\begin{breakablealgorithm}
	\caption{Sequential Minimal Optimization for Classification}
	\label{alg:smo_classifier}
	\begin{algorithmic}
		\Require{Training examples matrix $X \in \Re^{n \times m}$}
		\Require{Training target vector $y \in \pm1^n$}
		\Require{Kernel matrix $K \in \Re^{n \times n}$}
		\Require{Regularization parameter $C > 0$}
		\Require{Tolerance value $tol$ for stopping criterion}
		\Function{SMOClassifier}{$X,y,K,C,tol$}
			\State Initialize the Lagrange multipliers vector $\alpha \in \Re^n, \alpha \gets 0$
			\State Initialize the empty set $I0 \gets \{i : 0 < \alpha_i < C\}$
			\State Initialize the set $I1 \gets \{i : y_i = +1, \alpha_i = 0\}$ to contain all the indices of the training examples of class $+1$
			\State Initialize the empty set $I2 \gets \{i : y_i = -1,	 \alpha_i = C\}$
			\State Initialize the empty set $I3 \gets \{i :  y_i = +1,	 \alpha_i = C\}$
			\State Initialize the set $I4 \gets \{i : y_i = -1, \alpha_i = 0\}$ to contain all the indices of the training examples of class $-1$
			\State Initialize $b_{up} \gets -1$
			\State Initialize $b_{low} \gets +1$
			\State Initialize the error cache vector $errors \in \Re^n, errors \gets 0$
			\While {$num\_changed > 0$ \OR $examine\_all = True$}
				\State $num\_changed \gets 0$
				\State $examine\_all \gets True$
				\If {$examine\_all = True$}
					\For {$i \gets 0$ to $n$} \Comment loop over all training examples
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
					\EndFor
				\Else
					\For {$i$ in $I0$} \Comment loop over examples where $\alpha_i$ are not already at their bounds
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
						\If {$b_{up} > b_{low} - 2 tol$} \Comment check if optimality on $I0$ is attained
							\State {$num\_changed \gets 0$}
							\Break
						\EndIf
					\EndFor
				\EndIf
				\If {$examine\_all = True$}
					\State $examine\_all \gets False$
				\ElsIf {$num\_changed = 0$}
					\State $examine\_all \gets True$
				\EndIf
			\EndWhile
			\State Compute $b$ by~\eqref{eq:smo_svc_b}
			\State \Return $\alpha,b$
		\EndFunction
	\end{algorithmic}

	\newpage

	\begin{algorithmic}
		\Require{$i2$-th Lagrange multiplier}
		\Function{ExamineExample}{$i2$}
			\If {$i2$ in $I0$}
				\State $E_2 \gets errors_{i2}$
			\Else
				\State Compute $E_2$
				\State $errors_{i2} \gets E_2$
				\State Update $(b_{low}, i_{low})$ or $(b_{up}, i_{up})$ using $(E_2, i2)$
			\EndIf
			\If {optimality is attained using current $b_{low}$ and $b_{up}$}
				\State \Return 0
			\Else
				\State Find an index $i1$ to do joint optimization with $i2$
				\If {$\Call{TakeStep}{i1,i2}$ = True}
					\State \Return 1
				\Else
					\State \Return 0
				\EndIf
			\EndIf
		\EndFunction
	\end{algorithmic}

	\newpage

	\begin{algorithmic}
		\Require{$i1$-th Lagrange multiplier}
		\Require{$i2$-th Lagrange multiplier}
		\Function{TakeStep}{$i1,i2$}
			\If {$i1 = i2$}
				\State \Return False
			\EndIf
			\State Compute $L$ and $H$ using~\eqref{eq:smo_svc_bounds_update1} or~\eqref{eq:smo_svc_bounds_update2}
			\If {$L = H$}
				\State \Return False
			\EndIf
			\State Compute $\eta$ by~\eqref{eq:smo_eta} \Comment we assume that $\eta > 0$, i.e., the kernel matrix $K$ is positive definite
			\If {$\eta < 0$}
				\State Choose $\alpha_2^{new,clipped}$ between $L$ and $H$ according to the largest value of the objective function at these points
			\Else
				\State Compute $\alpha_2^{new}$ by~\eqref{eq:smo_svc_a2_new}
				\State Compute $\alpha_2^{new,clipped}$ by~\eqref{eq:smo_svc_a2_new_clipped}
			\EndIf
			\If {changes in $\alpha_2^{new,clipped}$ are larger than some eps}
				\State Compute $\alpha_1^{new}$ by~\eqref{eq:smo_svc_a1_new}
				\State Update $\alpha_2^{new,clipped}$ and $\alpha_1^{new}$
				\For {$i$ in $I0$}
					\State Update $errors_i$ using new Lagrange multipliers
				\EndFor
				\State Update $\alpha$ using new Lagrange multipliers
				\State Update $I0, I1, I2, I3$ and $I4$
				\State Update $errors_{i1}$ and $errors_{i2}$
				\For {$i$ in $I0 \cup \{i1,i2\}$}
					\State Compute $(i_{low}, b_{low})$ by $b_{low} = \max\{errors_i : i \in I0 \cup I3 \cup I4\}$
					\State Compute $(i_{up}, b_{up})$ by $b_{up} = \min\{errors_i : i \in I0 \cup I1 \cup I2\}$
				\EndFor
				\State \Return True
			\Else
				\State \Return False
			\EndIf
		\EndFunction
	\end{algorithmic}
\end{breakablealgorithm}

\newpage

\subsubsection{Regression}

In case of regression the bounds and the new multipliers $\alpha_1^{+,new}$ and $\alpha_2^{+,new}$ can be expressed as follows if ($\alpha_1^+ > 0$ or ($\alpha_1^- = 0$ and $ E_1 - E_2 > 0$)) and ($\alpha_2^+ > 0$ or ($\alpha_2^- = 0$ and $ E_1 - E_2 < 0$)):

\begin{equation} \label{eq:smo_svr_bounds_update1}
	\begin{aligned}
		& L = max(0, \gamma - C) \\
		& H = min(C, \gamma)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new1}
	\alpha_2^{+,new} = \alpha_2^+ - \frac{E_1 - E_2}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new1}
	\alpha_1^{+,new} = \alpha_1^+ - (\alpha_2^{+,new,clipped} - \alpha_2^+)
\end{equation}

or, if ($\alpha_1^+ > 0$ or ($\alpha_1^- = 0$ and $ E_1 - E_2 > 2 \epsilon$)) and ($\alpha_2^- > 0$ or ($\alpha_2^+ = 0$ and $ E_1 - E_2 > 2 \epsilon$)):

\begin{equation} \label{eq:smo_svr_bounds_update2}
	\begin{aligned}
		& L = max(0, -\gamma) \\
		& H = min(C, -\gamma + C)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new2}
	\alpha_2^{-,new} = \alpha_2^- + \frac{(E_1 - E_2) - 2 \epsilon}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new2}
	\alpha_1^{+,new} = \alpha_1^+ + (\alpha_2^{-,new,clipped} - \alpha_2^-)
\end{equation}

or, if ($\alpha_1^- > 0$ or ($\alpha_1^+ = 0$ and $ E_1 - E_2 < - 2 \epsilon$)) and ($\alpha_2^+ > 0$ or ($\alpha_2^- = 0$ and $ E_1 - E_2 < - 2 \epsilon$)):

\begin{equation} \label{eq:smo_svr_bounds_update3}
	\begin{aligned}
		& L = max(0, \gamma) \\
		& H = min(C, C + \gamma)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new3}
	\alpha_2^{+,new} = \alpha_2^+ - \frac{(E_1 - E_2) + 2 \epsilon}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new3}
	\alpha_1^{-,new} = \alpha_1^- + (\alpha_2^{+,new,clipped} - \alpha_2^+)
\end{equation}

or, finally, if ($\alpha_1^- > 0$ or ($\alpha_1^+ = 0$ and $ E_1 - E_2 < 0$)) and ($\alpha_2^- > 0$ or ($\alpha_2^+ = 0$ and $ E_1 - E_2 > 0$)):

\begin{equation} \label{eq:smo_svr_bounds_update4}
	\begin{aligned}
		& L = max(0, -\gamma - C) \\
		& H = min(C, -\gamma)
	\end{aligned}
\end{equation}

\begin{equation} \label{eq:smo_svr_a2_new4}
	\alpha_2^{-,new} = \alpha_2^- + \frac{E_1 - E_2}{\eta}
\end{equation}

\begin{equation} \label{eq:smo_svr_a1_new4}
	\alpha_1^{-,new} = \alpha_1^- - (\alpha_2^{-,new,clipped} - \alpha_2^-)
\end{equation}

where $\gamma = \alpha_1^+ - \alpha_1^- + \alpha_2^+ - \alpha_2^-$. Notice that $\eta$ and $\alpha_2^{+,new,clipped}$ or $\alpha_2^{-,new,clipped}$ are identical to~\eqref{eq:smo_eta} and~\eqref{eq:smo_svc_a2_new_clipped} respectively.

The KKT conditions for the regression problem~\eqref{eq:svr_min_wolfe_dual} are:

\begin{equation} \label{eq:svr_smo_kkt}
	\begin{aligned}
		\alpha_i^+ - \alpha_i^- = 0 & \Leftrightarrow | y_i - y'_i | < \epsilon \\
		-C < \alpha_i^+ - \alpha_i^- < C & \Leftrightarrow | y_i - y'_i | = \epsilon \\
		\alpha_i^+ + \alpha_i^- = C & \Leftrightarrow | y_i - y'_i | > \epsilon
	\end{aligned}
\end{equation}

so, the steps described above will be iterate as long as there will be an example that violates them.

In case of regression we select the threshold $b$ as follows:

\begin{equation} \label{eq:smo_svr_b1}
	b_{up} = E_1 + ((\alpha_1^+ - \alpha_1^-) - (\alpha_1^{+,new} - \alpha_1^{-,new})) K(x_1,x_1) + ((\alpha_2^+ - \alpha_2^-) - (\alpha_2^{+,new,clipped} - \alpha_2^{-,new,clipped})) K(x_1,x_2) + b
\end{equation}

\begin{equation} \label{eq:smo_svr_b2}
	b_{low} = E_2 + ((\alpha_1^+ - \alpha_1^-) - (\alpha_1^{+,new} - \alpha_1^{-,new})) K(x_1,x_2) + ((\alpha_2^+ - \alpha_2^-) - (\alpha_2^{+,new,clipped} - \alpha_2^{-,new,clipped})) K(x_2,x_2) + b
\end{equation}

\begin{equation} \label{eq:smo_svr_b}
	b =
        \begin{cases}
            b_{up} & \text{if} \ 0 < \alpha_1^+, \alpha_1^- < C \\
            b_{low} & \text{if} \ 0 < \alpha_2^+, \alpha_2^- < C \\
            \displaystyle \frac{b_{up}+b_{low}}{2} & \text{otherwise} \\
        \end{cases}
\end{equation}

\bigskip
\bigskip

The improvements described in~\cite{keerthi2001improvements, shevade1999improvements} for classification and regression respectively are about the definition of subsets of multipliers to efficiently update them at each iteration by separating the multipliers at the bounds from those who can be further minimized.

\newpage

\begin{breakablealgorithm}
	\caption{Sequential Minimal Optimization for Regression}
	\label{alg:smo_regression}
	\begin{algorithmic}
		\Require{Training examples matrix $X \in \Re^{n \times m}$}
		\Require{Training target vector $y \in \Re^n$}
		\Require{Kernel matrix $K \in \Re^{n \times n}$}
		\Require{Regularization parameter $C > 0$}
		\Require{Epsilon-tube value $\epsilon \geq 0$ within which no penalty is associated in the epsilon-insensitive loss function}
		\Require{Tolerance value $tol$ for stopping criterion}
		\Function{SMORegression}{$X,y,K,C,\epsilon,tol$}
			\State Initialize the Lagrange multipliers vector $\alpha^+ \in \Re^n, \alpha^+ \gets 0$
			\State Initialize the Lagrange multipliers vector $\alpha^- \in \Re^n, \alpha^- \gets 0$
			\State Initialize the empty set $I0 \gets \{i : 0 < \alpha^+_i, \alpha^-_i < C\}$
			\State Initialize the set $I1 \gets \{i : \alpha^+_i = 0, \alpha^-_i = 0\}$ to contain all the indices of the training examples
			\State Initialize the empty set $I2 \gets \{i : \alpha^+_i = 0, \alpha^-_i = C\}$
			\State Initialize the empty set $I3 \gets \{i : \alpha^+_i = C, \alpha^-_i = 0\}$
			\State Initialize $i_{up} \gets 0$ \Comment or any other target index $i_{up}$ from the training examples
			\State Initialize $i_{low} \gets 0$ \Comment or any other target index $i_{low}$ from the training examples
			\State Initialize $b_{up} \gets y_{i_{up}} + \epsilon$
			\State Initialize $b_{low} \gets y_{i_{low}} - \epsilon$
			\State Initialize the error cache vector $errors \in \Re^n, errors \gets 0$
			\While {$num\_changed > 0$ \OR $examine\_all = True$}
				\State $num\_changed \gets 0$
				\State $examine\_all \gets True$
				\If {$examine\_all = True$}
					\For {$i \gets 0$ to $n$} \Comment loop over all training examples
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
					\EndFor
				\Else
					\For {$i$ in $I0$} \Comment loop over examples where $\alpha^+_i$ and $\alpha^-_i$ are not already at their bounds
						\State $num\_changed \gets num\_changed + \Call{ExamineExample}{i}$
						\If {$b_{up} > b_{low} - 2 tol$} \Comment check if optimality on $I0$ is attained
							\State {$num\_changed \gets 0$}
							\Break
						\EndIf
					\EndFor
				\EndIf
				\If {$examine\_all = True$}
					\State $examine\_all \gets False$
				\ElsIf {$num\_changed = 0$}
					\State $examine\_all \gets True$
				\EndIf
			\EndWhile
			\State Compute $b$ by~\eqref{eq:smo_svr_b}
			\State \Return $\alpha^+,\alpha^-,b$
		\EndFunction
	\end{algorithmic}

	\newpage

	\begin{algorithmic}
		\Require{$i1$-th Lagrange multiplier}
		\Require{$i2$-th Lagrange multiplier}
		\Function{TakeStep}{$i1,i2$}
			\If {$i1 = i2$}
				\State \Return False
			\EndIf
			\State $finished = False$
			\While {\NOT $finished$}
				\State Compute $L$ and $H$ using~\eqref{eq:smo_svr_bounds_update1},~\eqref{eq:smo_svr_bounds_update2},~\eqref{eq:smo_svr_bounds_update3} or~\eqref{eq:smo_svr_bounds_update4}
				\If {$L < H$}
					\State Compute $\eta$ by~\eqref{eq:smo_eta} \Comment we assume that $\eta > 0$, i.e., the kernel matrix $K$ is positive definite
					\If {$\eta < 0$}
						\State Choose $\alpha_2^{+,new,clipped}$ or $\alpha_2^{-,new,clipped}$ between $L$ and $H$ according to the largest value of the objective function at these points
					\Else
						\State Compute $\alpha_2^{+,new}$ or $\alpha_2^{-,new}$ using~\eqref{eq:smo_svr_a2_new1},~\eqref{eq:smo_svr_a2_new3} or~\eqref{eq:smo_svr_a2_new2},~\eqref{eq:smo_svr_a2_new4} respectively
						\State Compute $\alpha_2^{+,new,clipped}$ or $\alpha_2^{-,new,clipped}$ by~\eqref{eq:smo_svc_a2_new_clipped}
					\EndIf
					\State Compute $\alpha_1^{+,new}$ or $\alpha_1^{-,new}$ using~\eqref{eq:smo_svr_a1_new1},~\eqref{eq:smo_svr_a1_new2} or~\eqref{eq:smo_svr_a1_new3},~\eqref{eq:smo_svr_a1_new4} respectively
					\If {changes in $\alpha_2^{+,new,clipped}, \alpha_2^{-,new,clipped}, \alpha_1^{+,new}$ or $\alpha_1^{-,new}$ are larger than some eps}
						\State Update $\alpha_2^{+,new,clipped}, \alpha_2^{-,new,clipped}, \alpha_1^{+,new}$ or $\alpha_1^{-,new}$
					\EndIf
				\Else
					\State $finished = True$
				\EndIf
			\EndWhile
			\If {changes in $\alpha_2^{+,new,clipped}, \alpha_2^{-,new,clipped}, \alpha_1^{+,new}$ or $\alpha_1^{-,new}$ are larger than some eps}
				\For {$i$ in $I0$}
					\State Update $errors_i$ using new Lagrange multipliers
				\EndFor
				\State Update $\alpha^+$ and $\alpha^-$ using new Lagrange multipliers
				\State Update $I0, I1, I2$ and $I3$
				\State Update $errors_{i1}$ and $errors_{i2}$
				\For {$i$ in $I0 \cup \{i1,i2\}$}
					\State Compute $(i_{low}, b_{low})$ by $b_{low} = \max\{errors_i : i \in I0 \cup I1 \cup I2\}$
					\State Compute and $(i_{up}, b_{up})$ by $b_{up} = \min\{errors_i : i \in I0 \cup I1 \cup I3\}$
				\EndFor
				\State \Return True
			\Else
				\State \Return False
			\EndIf
		\EndFunction
	\end{algorithmic}
\end{breakablealgorithm}


\chapter{Sequential Data}
\label{chap:Sequential Data}
\section{Hidden Markov Models}
\subsection{representation}
	\textbf{Directed Probabilistic Graphical Models} with \textbf{Latent Variables} to represent \textbf{Hidden Markov Models}.Hidden Markov Models are \textbf{State Space Models}.

	Introduce latent variable $\vec{z}_n$ with 1-of-K coding scheme.
	\textbf{Transition probabilities} or \textbf{transition matrix} $\vec{A}_{jk} = p(z_{nk}=1|z_{n-1,j}=1)$ and probabilities satisfy
$0\leq \vec{A}_{jk}\leq 1$ with $\sum_k{A_{jk}} = 1$.So that matrix $\vec{A}$ has $K(K-1)$ independent parameters.Write the conditional probability explicitly
\begin{align}
p(\vec{z}_n|\vec{z}_{n-1},\vec{A}) = \prod_{k=1}^{K}\prod_{j=1}^{K}\vec{A}_{jk}^{z_{n-1,j}z_{nk}}
\end{align}

The initial latent node does not have a parent node,so the marginal prior distribution is represented by a vector of probabilities $\pi$ with $\pi_k\equiv p(z_{1k}=1)$,so that
\begin{align}
p(\vec{z}_1|\vec{\pi}) = \prod_{k=1}^{K}\pi_k^{z_{1k}}
\end{align}
where $\sum_{k}^{\pi_k}=1$.
The \textbf{emission probabilities} can be represented in the form
\begin{align}
p(\vec{x}_n|\vec{z}_n,\vec{\phi})=\prod_{k=1}^{K}p(\vec{x}_n|\vec{\phi}_k)^{z_{nk}}
\end{align}
For \textbf{homogeneous} models,the joint distribution over both latent and observed variables is given by
\begin{align}\label{eqn:hmm joint distribution}
p(\vec{X},\vec{Z}|\vec{\theta})=p(\vec{z}_1|\vec{\pi})\left[\prod_{n=2}^{N}p(\vec{z}_n|\vec{z}_{n-1},\vec{A}) \right]
\prod_{m=1}^{N}p(\vec{x}_m|\vec{z}_m,\vec{\phi})
\end{align}
where $\vec{\theta} = \left\{\vec{\pi},\vec{A},\vec{\phi} \right\}$ denotes the set of parameters governing the model.

\subsection{evaluation}
The likelihood function is obtained from the joint distribution by marginalizing over the latent variables
\begin{align}
p(\vec{X}|\vec{\theta})=\sum_{\vec{Z}}{p(\vec{X},\vec{Z}|\vec{\theta})}
\end{align}

\subsection{optimization}
\subsubsection{EM algorithm}
Direct maximization of the likelihood function lead to complex expressions with no close-form solutions(not i.i.d),so we turn the \textbf{expectation maximization} (simplified \textbf{variational bayesian}).In the \textbf{E step},take these parameters to find the posterior distribution of the latent variables $p(\vec{\vec{Z}|\vec{X},\vec{\theta}^{\text{old}}})$.Then use this posterior distribution to evaluate the expectation of the logarithm of complete-data likelihood function,as a function of parameters $\vec{\theta}$
\begin{align}
\mathcal{Q}(\vec{\theta},\vec{\theta}^{\text{old}}) =\sum_{\vec{Z}}{p(\vec{Z}|\vec{X},\vec{\theta}^{\text{old}})}\ln p(\vec{X},\vec{Z}|\vec{\theta})
\end{align}
Introduce some notation
\begin{align}
\gamma(\vec{z}_n) &= p(\vec{z}_n|\vec{X},\theta^{\text{old}})
=\mathbb{E}[z_{nk}]=\sum_{\vec{z}_n}{\gamma(\vec{z}_n)} \\
\xi(\vec{z}_{n-1},\vec{z}_n)&=p(\vec{z}_{n-1},\vec{z}_n|\vec{X},\vec{\theta}^{\text{old}})
=\mathbb{E}[{z_{n-1,j}z_{nk}}]=
\sum_{\vec{z}_{n-1},\vec{z}_n}\xi(\vec{z}_{n-1},\vec{z}_n)z_{n-1,j}z_{nk}
\end{align}
Substitute the joint distribution $p(\vec{X},\vec{Z}|\vec{\theta})$\ref{eqn:hmm joint distribution},we obtain
\begin{align}
\mathcal{Q}(\vec{\theta},\vec{\theta}^{\text{old}})
&= \sum{k=K}\gamma(z_1 k)\ln\pi_k+\sum_{n=2}^{N}\sum_{j=1}^{K}\sum_{k=1}^{K}\xi(\vec{z}_{n-1},\vec{z}_n)\ln\vec{A}_{jk}\\
&+ \sum_{n=1}^{N}\sum_{k=1}^{K}\gamma(z_{nk})\ln p(\vec{x}_n|\vec{\phi}_k)
\end{align}
where we have marginalized over the parameters' joint distribution.

In \textbf{M step} we maximize $\mathcal{Q}$ with respect to the parameters,in which we treat $\gamma$ and $\xi$ as constant.

In the case of discrete multinomial observed variables,the conditional distribution of the observations takes the form
\begin{align}
	p(\vec{x}|\vec{z})=\prod_{i=1}^{D}\prod_{k=1}^{K}\mu_{ik}^{x_i z_k}
\end{align}
subject to $\sum_{i=1}^{D}\mu_{ik}=1$.
	Introduce \textbf{Lagrangian multiplier} $\lambda$,the objective is to maximize
	\begin{align}
	\mathcal{L}(\vec{\mu},\lambda) = \sum_{n=1}^{N}\sum_{k=1}^{K}\gamma(z_{nk})\sum_{i=1}^{D}x_{ni}\ln\mu_{ik}+\lambda(\sum_{i=1}^{D}\mu_{ik}-1)
	\end{align}
	Set the derivatives to zero
	\begin{align}
	\because\begin{cases}
	\dfrac{\partial\mathcal{L}}{\partial \mu_{ik}} &= \sum_{n=1}^{N}\gamma(z_{nk})x_{ni}\dfrac{1}{\mu_{ik}} + \lambda = 0\\
	\dfrac{\partial\mathcal{L}}{\partial \lambda} &= \sum_{i=1}^{D}\mu_{ik} -1 = 0
	\end{cases}\\
	\therefore
	\begin{cases}
	\mu_{ik}&=-\dfrac{\sum_{n=1}^{N}\gamma(z_{nk})x_{ni}}{\lambda} \\
	\lambda &= \sum_{i=1}^{D}\gamma(z_{nk})
	\end{cases}\\
	\Longrightarrow \\
	\mu_{ik}&=\dfrac{\sum_{n=1}^{N}\gamma(z_{nk})x_{ni}}{\sum_{n=1}^{N}\gamma(z_{nk})}
	\end{align}
\subsubsection{forward-backward algorithm}
	The quantities $\gamma$ and $\xi$ are posterior distribution of latent variables,and can be obtained using a two-stage message passing algorithm,known as Baum-Welch,a.k.a alpha-beta algorithm.

\subsubsection{scaling factors}
	In the recursion relationship,these probabilities are significantly less than unity,as we work our way forward along the chain,the values of $\alpha$ can go to zero exponentially quickly.

\subsection{predict}

\subsection{title}

\section{Linear Dynamic Systems}
TODO


\subsection{Semantic consequence}

A formula, \(A\), semantically implies another, \(B\), if for every interpretation of \(A\), \(B\) is true.

We show this with:

\(A\vDash B\)

Formula \(B\) is satisfisable if there is some \(A\) where this is true.

For example:
\(A\land B \vDash A\)

Formula \(B\) is a tautology if this is true for any \(A\). We can also write this as \(\vDash B\).

\subsection{Logical equivalence}

If \(A\vDash B\) and \(B\vDash A\) we say that \(A\) and \(B\) are logically equivalent.

This is shown as \(A \Leftrightarrow B\).


\input{../header_function}

%---------- start document ---------- %
 \section{combinatorial -- combinatorial functions}\linkedzero{combinatorial}
%
  \subsection{binomial -- binomial coefficient}\linkedone{combinatorial}{binomial}
   \func{binomial}
   {%
     \hiki{n}{integer},\ %
     \hiki{m}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the binomial coefficient for \param{n} and \param{m}.
   In other words, $\displaystyle{\frac{\param{n} !}{(\param{n} - \param{m}) ! \param{m} !}}$.\\
   \spacing
   % added document
   \negok For convenience, {\tt binomial(n, n+i)} returns \(0\) for positive \(i\), and {\tt binomial(0,0)} returns \(1\).\\
   \spacing
   % input, output document
   \quad \param{n} must be a positive integer and \param{m} must be
   a non-negative integer. \\

  \subsection{combinationIndexGenerator -- iterator for combinations}\linkedone{combinatorial}{combinationIndexGenerator}

   \func{combinationIndexGenerator}{%
     \hiki{n}{integer},\ %
     \hiki{m}{integer}
   }{%
     \out{iterator}
   }\\
   \spacing
   % document of basic document
   \quad Return an iterator which generates indices of \param{m}
   element subsets of \param{n} element set.\\
   \spacing
   \quad The number of generated indices is \linkingone{combinatorial}{binomial}(\param{n}, \param{m}).
   \spacing
   \quad {\tt combination\_index\_generator}\linkedone{combinatorial}{combination\_index\_generator}
   is an alias of {\tt combinationIndexGenerator}.\\

  \subsection{factorial -- factorial}\linkedone{combinatorial}{factorial}
   \func{factorial}{%
     \hiki{n}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return \(\param{n}!\) for non-negative integer \param{n}.\\

  \subsection{permutationGenerator -- iterator for permutation}\linkedone{combinatorial}{permutationGenerator}
  \func{permutationGenerator}{%
    \hiki{n}{integer}
  }{%
    \out{iterator}
  }
   \spacing
   \quad Generate all permutations of \param{n} elements as list iterator.\\
   %
   \spacing
   \quad The number of generated list is \param{n}'s
   \linkingone{combinatorial}{factorial}, so be careful to use
   big \param{n}.\\
   \spacing
   \quad {\tt permutation\_generator}\linkedone{combinatorial}{permutation\_generator}
   is an alias of {\tt permutationGenerator}.\\

  \subsection{fallingfactorial -- the falling factorial}\linkedone{combinatorial}{fallingfactorial}
   \func{fallingfactorial}{%
     \hiki{n}{integer},\ %
     \hiki{m}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the falling factorial; \param{n} to the \param{m} falling,
   i.e. \(n(n-1)\cdots(n-m+1)\).\\

  \subsection{risingfactorial -- the rising factorial}\linkedone{combinatorial}{risingfactorial}
   \func{risingfactorial}{%
     \hiki{n}{integer},\ %
     \hiki{m}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the rising factorial; \param{n} to the \param{m} rising,
   i.e.\, \(n(n+1)\cdots(n+m-1)\).\\

  \subsection{multinomial -- the multinomial coefficient}\linkedone{combinatorial}{multinomial}
   \func{multinomial}{%
     \hiki{n}{integer},\ %
     \hiki{parts}{list}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the multinomial coefficient.\\
   \spacing
   % input, output document
   \quad \param{parts} must be a sequence of natural numbers and the sum of elements in \param{parts} should be equal to \param{n}.\\

  \subsection{bernoulli -- the Bernoulli number}\linkedone{combinatorial}{bernoulli}
   \func{bernoulli}{%
     \hiki{n}{integer}
   }{%
     \out{Rational}
   }\\
   \spacing
   % document of basic document
   \quad Return the \param{n}-th Bernoulli number.\\

  \subsection{catalan -- the Catalan number}\linkedone{combinatorial}{catalan}
   \func{catalan}{%
     \hiki{n}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the \param{n}-th Catalan number.\\

  \subsection{dyck\_word\_generator -- generator for Dyck words}\linkedone{combinatorial}{dyck\_word\_generator}
   \func{dyck\_word\_generator}{%
     \hiki{n}{integer}
     \hikiopt{alphabet}{sequence}{(0, 1)}
   }{%
     \out{iterator}
   }\\
   \spacing
   % document of basic document
   \quad  Generate all Dyck words of length \(2\times\)\param{n} as tuples.\\
   \spacing
   \quad The Dyck words are words on a two character alphabet.
   The number of each character in a word is equal,
   and the number of the second character never exceeds the first
   in any initial parts of the word.\\
   \spacing
   \quad The number of generated words is the \param{n}-th Catalan number.
   (see \linkingone{combinatorial}{catalan})\\
   \spacing
   \quad The alphabet is \(\{0, 1\}\) by default, but you can pass it into the
   optional argument \param{alphabet}.\\

  \subsection{euler -- the Euler number}\linkedone{combinatorial}{euler}
   \func{euler}{%
     \hiki{n}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the \param{n}-th Euler number.\\

  \subsection{bell -- the Bell number}\linkedone{combinatorial}{bell}
   \func{bell}{%
     \hiki{n}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the \param{n}-th Bell number.\\
   \spacing
   % added document
   \quad The Bell number \(b\) is defined by:
   \[b(n) = \sum_{i=0}^{n} S(n, i),\ \]
   where \(S\) denotes Stirling number of the second kind (\linkingone{combinatorial}{stirling2}).\\

  \subsection{stirling1 -- Stirling number of the first kind}\linkedone{combinatorial}{stirling1}
   \func{stirling1}{%
     \hiki{n}{integer},\ %
     \hiki{m}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return Stirling number of the first kind.\\
   \spacing
   % added document
   \quad Let \(s\) denote the Stirling number and \((x)_n\) the falling factorial, then
   \[(x)_n = \sum_{i=0}^{n} s(n,\ i) x^i. \]\\
   \(s\) satisfies the recurrence relation:
   \[s(n,\ m) = s(n-1,\ m-1) - (n-1)s(n-1,\ m)\ .\]\\

  \subsection{stirling2 -- Stirling number of the second kind}\linkedone{combinatorial}{stirling2}
   \func{stirling2}{%
     \hiki{n}{integer},\ %
     \hiki{m}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return Stirling number of the second kind.\\
   \spacing
   % added document
   \quad Let \(S\) denote the Stirling number, \((x)_i\) falling factorial, then:
   \[x^n = \sum_{i=0}^{n} S(n,\ i) (x)_i\]
   \(S\) satisfies:
   \[S(n,\ m) = S(n-1,\ m-1) + m S(n-1,\ m)\]\\

  \subsection{partition\_number -- the number of partitions}\linkedone{combinatorial}{partition\_number}
   \func{partition\_number}{%
     \hiki{n}{integer}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return the number of partitions of \param{n}.\\

  \subsection{partitionGenerator -- iterator for partition}\linkedone{combinatorial}{partitionGenerator}
   \func{partitionGenerator}{%
     \hiki{n}{integer},\ %
     \hikiopt{maxi}{integer}{0}
   }{%
     \out{iterator}
   }\\
   \spacing
   % document of basic document
   \quad Return an iterator which generates partitions of \param{n}.\\
   % input, output document
   \spacing
   \quad If \param{maxi} is given, then summands are limited not to exceed \param{maxi}.\\
   \quad The number of partitions (given by
   \linkingone{combinatorial}{partition\_number}) grows exponentially,
   so be careful to use big \param{n}.\\
   \spacing
   \quad {\tt partition\_generator}\linkedone{combinatorial}{partition\_generator}
   is an alias of {\tt partitionGenerator}.\\

  \subsection{partition\_conjugate -- the conjugate of partition}\linkedone{combinatorial}{partition\_conjugate}
   \func{partition\_conjugate}{%
     \hiki{partition}{tuple}
   }{%
     \out{tuple}
   }\\
   \spacing
   % document of basic document
   \quad Return the conjugate of \param{partition}.\\

\begin{ex}
>>> combinatorial.binomial(5, 2)
10L
>>> combinatorial.factorial(3)
6L
>>> combinatorial.fallingfactorial(7, 3) == 7 * 6 * 5
True
>>> combinatorial.risingfactorial(7, 3) == 7 * 8 * 9
True
>>> combinatorial.multinomial(7, [2, 2, 3])
210L
>>> for idx in combinatorial.combinationIndexGenerator(5, 3):
...     print idx
...
[0, 1, 2]
[0, 1, 3]
[0, 1, 4]
[0, 2, 3]
[0, 2, 4]
[0, 3, 4]
[1, 2, 3]
[1, 2, 4]
[1, 3, 4]
[2, 3, 4]
>>> for word in combinatorial.dyck_word_generator(3, alphabet=("(", ")")):
...     print "".join(word)
...
()()()
()(())
(())()
(()())
((()))
>>> for part in combinatorial.partitionGenerator(5):
...     print part
...
(5,)
(4, 1)
(3, 2)
(3, 1, 1)
(2, 2, 1)
(2, 1, 1, 1)
(1, 1, 1, 1, 1)
>>> combinatorial.partition_number(5)
7
>>> def limited_summands(n, maxi):
...     "partition with limited number of summands"
...     for part in combinatorial.partitionGenerator(n, maxi):
...         yield combinatorial.partition_conjugate(part)
...
>>> for part in limited_summands(5, 3):
...     print part
...
(2, 2, 1)
(3, 1, 1)
(3, 2)
(4, 1)
(5,)
\end{ex}%Don't indent!(indent causes an error.)

\C

%---------- end document ---------- %

\input{../footer}


\section{Introduction}
Our aim for this course is to explore optimisation problems, typically minimising a function $f(x)$ with $f:\mathbb R^n\to\mathbb R$ the objective function subject to the regional constraint $g(x)=b,x\in X$ where $X\subset\mathbb R^n$ defines a regional constraint, and $g:\mathbb R^n\to\mathbb R^m$ defines $m$ functional constraints.\\
A feasible solution to the problem is any $x\in X$ such that $g(x)=b$, so a feasible solution is just one that satisfy the constraint of the problem.
An optimal solution is a feasible solution $x^\star$ such that $f(x^\star)\le f(x)$ for all feasible $x$.
The problem itself is feasible if there exists at least one feasible solution, and it is bounded if $\inf\{f(x):g(x)=b,x\in X\}\in\mathbb R$, which is the main focus of us.\\
Notice that it makes no difference if we consider the maximisation instead of minimisation by putting appropriate minus signs in the definitions.


\section{Rotating Frames of Reference}
Newton's Second Law works only in inertial frames.
A rotating frame of reference (wrt an inertial frame) is clearly non-inertial in general.
So the equation of motion in this frame needs to be modified relative to Newton's Second Law.\\
Let $S$ be an inertial frame and $S'$ another frame that is rotating along $z$-axis in $S$ with angular velocity $\omega=\dot\theta$ where $\theta$ is the angle between $x,y$-axis in $S$ and in $S'$.
Denote the basis vectors of $S$ by $\underline{e_1}=\underline{\hat{x}},\underline{e_2}=\underline{\hat{y}},\underline{e_3}=\underline{\hat{z}}$ and that of $S'$ by $\underline{e_1'}=\underline{\hat{x}}',\underline{e_2'}=\underline{\hat{y}}',\underline{e_3'}=\underline{\hat{z}}'$.
Consider a particle at rest in $S'$ viewed in $S$, then its velocity will be
$$\left( \frac{\mathrm d\underline{r}}{\mathrm dt} \right)_S=\underline{w}\times\underline{r}=\omega\underline{\hat{z}}\times\underline{r}$$
Conventionally we take $\omega>0$ as anticlockwise.
We certainly have some formula that applies to the basis vectors of $S'$, namely
$$\left( \frac{\mathrm d}{\mathrm dt}\underline{e_i'} \right)_S=\underline{\omega}\times\underline{e_i'}$$
So for a time-dependent vector $\underline{a}$ we have
$$\underline{a}(t)=\sum_{i=1}^3a_i'(t)\underline{e_i'}(t)$$
So when we observe in $S'$, the rate of change has
$$\left( \frac{\mathrm d}{\mathrm dt}\underline{a}(t) \right)_{S'}=\sum_{i=1}^3\left( \frac{\mathrm d}{\mathrm dt}a_i'(t) \right)\underline{e_i'}(t)$$
Therefore
\begin{align*}
    \left(\frac{\mathrm d}{\mathrm dt}\underline{a}\right)_S&=\sum_{i=1}^3\frac{\mathrm da_i'}{\mathrm dt}\underline{e_i'}+\sum_{i=1}^3a_i'\left( \frac{\mathrm de_i'}{\mathrm dt} \right)_S\\
    &=\sum_{i=1}^3\frac{\mathrm da_i'}{\mathrm dt}\underline{e_i'}+\sum_{i=1}^3a_i'(\underline{\omega}\times\underline{e_i'})\\
    &=\left( \frac{\mathrm d}{\mathrm dt}\underline{a} \right)_{S'}+\underline{\omega}\times\underline{a}
\end{align*}
Which is the key identity that relates rate of change in one frame to that in the other.
If we apply this to the position vector $\underline{r}$, then
$$\left(\frac{\mathrm d}{\mathrm dt}\underline{r}\right)_S=\left( \frac{\mathrm d}{\mathrm dt}\underline{r} \right)_{S'}+\underline{\omega}\times\underline{r}$$
And applying to velocity,
\begin{align*}
    \left( \frac{\mathrm d^2\underline{r}}{\mathrm dt^2} \right)
    &=\left( \left( \frac{\mathrm d}{\mathrm dt} \right)_{S'}+\underline{\omega}\times \right)\left( \left( \frac{\mathrm d}{\mathrm dt}\underline{r} \right)_{S'}+\underline{\omega}\times\underline{r} \right)\\
    &=\left( \frac{\mathrm d^2\underline{r}}{\mathrm dt^2} \right)_{S'}+2\underline{\omega}\times\left( \frac{\mathrm d\underline{r}}{\mathrm dt} \right)_{S'}+\underline{\dot{\omega}}\times\underline{r}+\underline{\omega}\times(\underline{\omega}\times\underline{r})
\end{align*}
This gives the acceleration.
\subsection{Equation of Motion in a Rotating Frame}
$S$ is inertial, therefore Newton's Laws of Motion applies, hence
$$m\left( \frac{\mathrm d^2\underline{r}}{\mathrm dt^2} \right)_S=\underline{F}$$
Hence we have
$$m\left( \frac{\mathrm d^2\underline{r}}{\mathrm dt^2} \right)_{S'}=\underline{F}-m\left( 2\underline{\omega}\times\left( \frac{\mathrm d\underline{r}}{\mathrm dt} \right)_{S'}+\underline{\dot{\omega}}\times\underline{r}+\underline{\omega}\times(\underline{\omega}\times\underline{r}) \right)$$
The second term is known as the fictitious forces, whcih are needed to explain the motion observed in a non-intertial frame.
We give names to each term in the fictitious forces:\\
Coriolis force: $-2m\underline{\omega}\times(\mathrm d\underline{r}/\mathrm dt)_{S'}$.\\
Euler force: $-m\underline{\dot{\omega}}\times\underline{r}$.\\
Centrifugal force: $-m\underline{\omega}\times(\underline{\omega}\times\underline{r})$.
Sometimes we take $\underline{\omega}$ to be constant, so the Euler force will be zero.
\subsection{Centrifugal Force}
Note that for $\underline{\omega}=\omega\underline{\hat{\omega}}$ with $\underline{\hat{\omega}}$ being unit,
\begin{align*}
    -m\underline{\omega}\times(\underline{\omega}\times\underline{r})&=-m((\underline{\omega}\cdot\underline{r})\omega-|\underline{\omega}|^2\underline{r})\\
    &=m\omega^2(\underline{r}-\underline{\hat{\omega}}(\underline{\hat{\omega}}\cdot\underline{r}))\\
    &=m\omega^2\underline{r}_\perp
\end{align*}
where $\underline{r}_\perp$ is the projection of $\underline{r}$ onto the plane that is perpendicular to $\underline{\omega}$, that is basically the plane of rotation.
So the centrifugal force is directed away from the rotatrion axis and its magnitude is $m\omega^2d$ where $d$ is the distance of the particle to the rotation axis.
Note that
$$|\underline{r}_\perp|^2=|\underline{r}|^2-(\underline{\hat{\omega}}\cdot\underline{r})^2=|\underline{r}\times\underline{\hat\omega}|^2$$
While we also have $\nabla{|\underline{r}_\perp|^2}=2\underline{r}-2\underline{\hat\omega}(\underline{\hat\omega}\cdot\underline{r})=2\underline{r}_\perp$.
Therefore
$$m\omega^2\underline{r}_\perp=\nabla\left( \frac{1}{2}m|\underline{r}\times\underline{\omega}|^2 \right)$$
Therefore the centrifugal force is a potential force.
On a rotating planet, we can combine the centrifugal force with gravitational force to create the notion of an effective gravity $\underline{g}_{\rm eff}=\underline{g}+\omega^2\underline{r}_\perp$.
Consider a point $P$ on the surface of the rotating planet, where the rotation axis is through the poles.
We define a local coordinate at $P$ where $\underline{\hat{z}}$ is the normal pointing outwards, $\underline{\hat{y}}$ is tangent northward, and $\underline{\hat{x}}$ is the tangent eastward.
Assume that the point $P$ is at latitude $\lambda$.
So $\underline{r}=R\underline{\hat{z}}$ where $R$ is the radius of the planet.
Also, as for the angular velocity, $\underline{\omega}=\omega(\underline{\hat{y}}\cos\lambda+\underline{\hat{x}}\sin\lambda)$.
\begin{align*}
    \underline{g}_{\rm eff}&=-g\underline{\hat{z}}+\omega^2R\cos\lambda(\underline{\hat{z}}\cos\lambda-\underline{\hat{y}}\sin\lambda)\\
    &=-(g-\omega^2R\cos^2\lambda)\underline{\hat{z}}-\omega^2R\cos\lambda\sin\lambda\underline{\hat{y}}
\end{align*}
So the angle between $\underline{g}$ and $\underline{g}_{\rm eff}$ would be
$$\alpha=\tan^{-1}\left( \frac{\omega^2R\cos\lambda\sin\lambda}{g-\omega^2R\cos^2\lambda} \right)$$
For earth, $\omega\approx 2\pi/86400$, so upon calculation, we obtain $\alpha\approx 3.5\times 10^{-3}$ which is very small.
\subsection{The Coriolis Force}
The coriolis force
$$-2m\underline{\omega}\times(\mathrm d\underline{r}/\mathrm dt)_{S'}=-2m\underline{\omega}\times\underline{v}$$
is perpendicular to the velocity, so it does not do any work.
This is just like the magnetic force.
We consider te horizontal motion on a rotating planet again.
The velocity is given by $\underline{v}=v_x\underline{\hat{x}}+v_y\underline{\hat{y}}$.
As before in our choice of model we have $\underline{\omega}=\omega(\underline{\hat{y}}\cos\lambda+\underline{\hat{z}}\sin\lambda)$.
Therefore
\begin{align*}
    -2m\underline{\omega}\times\underline{v}&=-2m\omega(\underline{\hat{y}}\cos\lambda+\underline{\hat{z}}\sin\lambda)\times v_x\underline{\hat{x}}+v_y\underline{\hat{y}}\\
    &=2m\omega\sin\lambda(v_y\underline{\hat{x}}-v_x\underline{\hat{y}})+2m\omega\cos\lambda v_x\underline{\hat{z}}
\end{align*}
So by considering tthe sign, the horizontal coriolis force gives a acceleration, which is to the right if we are on the northern hemisphere, and to the left on the southern hemisphere.
In atmosphere, the coriolis force can be balanced by a pressure gradient.
The horizontal motion then gives the difference in the direction of cyclones, which is anticlockwise in northern hemisphere and clockwise in the southern hemisphere.
\begin{example}
    Consider a ball dropped from the top of a tower, we want to know where does it land.
    We have
    $$\underline{\ddot{r}}=\underline{g}-2\underline{\omega}\times\underline{\dot{r}}-\underline{\omega}\times(\underline{\omega}\times\underline{r})=\underline{g}-2\underline{\omega}\times\underline{\dot{r}}+O(\omega^2)$$
    where the rotation is slow (i.e. $\omega^2R/g$ is small).
    Integrate it to get
    $$\underline{\dot{r}}=\underline{g}t-2\underline{\omega}\times (\underline{r}-\underline{r}(0))+O(\omega^2)$$
    We substitute this back to the original equation to get $\underline{\ddot{r}}=\underline{g}-2\underline{\omega}\times(\underline{g}t)+O(\omega^2)$, which solves to
    $$\underline{r}=\underline{r}(0)+\frac{1}{2}\underline{g}t^2-\underline{\omega}\times\underline{g}\frac{t^3}{3}+O(\omega^2)$$
    So if we take $\underline{g}=(0,0,-g),\underline{\omega}=(0,\omega,0)$ and $\underline{r}(0)=(0,0,R+h)$, then
    $$\underline{r}=\left(\frac{1}{3}\omega gt^3,0,R+h-\frac{1}{2}gt^2\right)$$
    So the time to reach the ground would be $t=\sqrt{2h/g}$, then it would travel a horizontal distance of approximately
    $$\frac{1}{3}\omega g\left(\frac{2h}{g}\right)^{3/2}$$
\end{example}
(Foucaul Pendulum)
Consider a pendulum at north pole, then the plane of its oscillation is rotating opposing the direction of rotation of the earth.
At latitude $\lambda$, the angular velocity of plane of rotation is $\omega\sin\lambda$, therefore the period $2\pi/(\omega\sin\lambda)$ which is greater than a day if $\lambda<\pi/2$.


\subsection{Neural networks}
\subsubsection{Fully connected neural network}
A fully connected neural network (FCNN) is such a neural network in which each neuron is connected to every neuron in the previous layer and each connection has its own weight. \par This is a general purpose connection pattern and makes no assumptions about the features in the input data to recognize. This type of network is very expensive in terms of memory (weights) and computations (connections).
\begin{figure}[H]
\centering
\includegraphics[width=250px]{pictures/fcnn.png}
\caption{Fully connected neural network}
\end{figure}

\subsubsection{Convolutional neural network}
Convolutional neural network (CNN) is a type of feed-forward artificial neural network, which means that connections between the neurons do not form a cycle. Information in the network moves only forward so the given signal goes through the neuron once.
\par CNN includes a convolutional layer and usually few other hidden layers. Each neuron is connected only to a small region of the previous layer called receptive field. Receptive fields of different neurons are overlapping with other neuron's fields so that together they are covering the whole input.
\par CNN are used for image processing because while using convolution they can recognize edges of an object on the image.
\begin{figure}[H]
\centering
\includegraphics[width=250px]{pictures/cnn.png}
\caption{Convolutional neural network}
\end{figure}

\subsubsection{Recurrent neural network}
In recurrent neural network (RNN) connections between the neurons form a directed cycle. It means that RNN can use its internal memory to learn  sequences of inputs. The same set of weights is applied recursively to the structure.
\par RNN are used for text or speech recognition.
\begin{figure}[H]
\centering
\includegraphics[width=250px]{pictures/rnn.png}
\caption{Convolutional neural network}
\end{figure}

\subsection{Activation function}
The activation function defines the output of considered neuron with given input. There are many functions that can be used as an activation function in artificial neural networks. The most popular are:
\subsubsection{Linear}
\begin{center}
$ f(x)=ax + b $ \\ $ a,b \in R $ \\~\\
\begin{tikzpicture}
	\begin{axis}
		\addplot[cmhplot,-]{x};
	\end{axis}
\end{tikzpicture}
\end{center}

\subsubsection{Rectified Linear Unit (ReLU)}
\[
f(x)=
\left\{
\begin{array}{ll}
      0 ,& x < 0 \\
      x ,& x\geq 0 \\
\end{array}
\right. \]
\begin{center}
\begin{tikzpicture}
    \begin{axis}[
            xmin=-4,xmax=4,
            ymin=-4,ymax=4,
        ]
        \addplot[cmhplot,-,domain=-4:0]{0};
        \addplot[cmhplot,-,domain=0:4]{x};

    \end{axis}
\end{tikzpicture}
\end{center}

\subsubsection{Sigmoid}
\begin{center}
$f(x) = \frac{1}{1 + e^{-x}}$ \\
\begin{tikzpicture}
    \begin{axis}[
            xmin=-5,xmax=5,
            ymin=-1,ymax=1,
        ]
        \addplot[cmhplot,-]{1/(1 + e^(-x))};
    \end{axis}
\end{tikzpicture}
\end{center}

\subsubsection{TanH}
\begin{center}
$ f(x)=\tanh(x)=\frac{2}{1+e^{-2x}}-1 $ \\
\begin{tikzpicture}
    \begin{axis}[
            xmin=-5,xmax=5,
            ymin=-1,ymax=1,
        ]
        \addplot[cmhplot,-]{2/(1 + e^(-2*x))-1};
    \end{axis}
\end{tikzpicture}
\end{center}

\subsection{Optimization and regularization}
\subsubsection{Dropout}
\subsubsection{Initial weights}
\subsubsection{Learning rate decay}
\subsubsection{Gradient Descent}