\subsection{Section 1}
1. Consider the bahaviour of the function $f(x)=|x|$ at $x=0$.

\subsection{Section 2}
1.
\begin{align*}
    \lim_{h\to0}\frac{\frac{1}{x+h}-\frac{1}{x}}{h}
    =&\lim_{h\to0}\frac{-h}{hx(x+h)}\\
    =&\lim_{h\to0}-\frac{1}{x(x+h)}\\
    =&-\frac{1}{x^2}
\end{align*}
Therefore,
$$f^\prime(x)=\lim_{h\to0}\frac{f(x+h)-f(x)}{h}=-\frac{1}{x^2}$$
2.
\begin{align*}
    \lim_{h\to0}\frac{\sqrt{x+h}-\sqrt{x}}{h}
    =&\lim_{h\to0}\frac{h}{h\sqrt{x}+\sqrt{x+h}}\\
    =&\lim_{h\to0}\frac{1}{\sqrt{x}+\sqrt{x+h}}\\
    =&\frac{1}{2\sqrt{x}}
\end{align*}
Therefore,
$$f^\prime(x)=\lim_{h\to0}\frac{f(x+h)-f(x)}{h}=\frac{1}{2\sqrt{x}}$$
3.
\begin{align*}
    \lim_{h\to0}\frac{e^{ax+ah}-e^{ax}}{h}
    =&\lim_{h\to0}ae^{ax}\frac{e^{ah}-1}{ah}\\
    =&(ae^{ax})(1)\\
    =&ae^{ax}
\end{align*}
Therefore,
$$f^\prime(x)=\lim_{h\to0}\frac{f(x+h)-f(x)}{h}=ae^{ax}$$
4. (a)
\begin{align*}
    \lim_{h\to0}\frac{\sin(x+h)-\sin(x)}{h}
    =&\lim_{h\to0}\frac{2\cos(x+h/2)\sin(h/2)}{h}\\
    =&\lim_{h\to0}\cos(x+h/2)\frac{\sin(h/2)}{h/2}\\
    =&\cos(x)
\end{align*}
Therefore,
$$f^\prime(x)=\lim_{h\to0}\frac{f(x+h)-f(x)}{h}=\cos(x)$$
(b)
\begin{align*}
    \lim_{h\to0}\frac{\cos(x+h)-\cos(x)}{h}
    =&-\lim_{h\to0}\frac{\sin(x-\pi/2+h)-\sin(x-\pi/2)}{h}\\
    =&\left.\frac{\mathrm d\sin(z)}{\mathrm dz}\right|_{z=x-\pi/2}\\
    =&-\cos(x-\pi/2)\\
    =&-\sin(x)
\end{align*}
Therefore,
$$f^\prime(x)=\lim_{h\to0}\frac{f(x+h)-f(x)}{h}=-\sin(x)$$
5.
\begin{align*}
    \lim_{h\to0}\frac{g(x+h)-g(x)}{h}
    =&\lim_{h\to0}\frac{cf(x+h)-cf(x)}{h}\\
    =&c\lim_{h\to0}\frac{f(x+h)-f(x)}{h}\\
    =&cf^\prime(x)
\end{align*}
Therefore,
$$f^\prime(x)=\lim_{h\to0}\frac{g(x+h)-g(x)}{h}=cf^\prime(x)$$
\subsection{Section 3}
1.
$$\frac{\mathrm d}{\mathrm dx}\frac{\sqrt{x}}{e^x}=\frac{\frac{1}{2\sqrt{x}}e^x-\sqrt{x}e^x}{e^{2x}}=\frac{1-2x}{2\sqrt{x}e^x}$$
2. (a)
$$\frac{\mathrm d}{\mathrm dx}x^3=(1)x^2+x(2x)=3x^2$$
$$\frac{\mathrm d}{\mathrm dx}x^4=(1)x^3+x(3x^2)=4x^3$$
(b) Let $P(n)$ be the proposition. When $n=1$, $\mathrm{L.H.S.}=1=\mathrm{R.H.S.}$, so $P(1)$ is true. Now assume that $P(k)$ is true for some positive integer $k$, then when $n=k+1$,
\begin{align*}
    \mathrm{L.H.S.}
    =&\frac{\mathrm d}{\mathrm dx}x^{k+1}\\
    =&\left(\frac{\mathrm d}{\mathrm dx}x\right)x^k+x\left(\frac{\mathrm d}{\mathrm dx}x^k\right)\\
    =&x^k+x(kx^{k-1})\\
    =&(k+1)x^k=\mathrm{R.H.S.}
\end{align*}
So $P(k+1)$ is true. By the principle of mathematical induction, $P(n)$ is true for all positive integer $n$.\\
3. (a)
\begin{align*}
    \lim_{h\to0}\frac{\frac{1}{f(x+h)}-\frac{1}{f(x)}}{h}
    =&-\lim_{h\to0}\frac{1}{f(x)f(x+h)}\frac{f(x+h)-f(x)}{h}\\
    =&-\frac{f^\prime(x)}{f(x)^2}
\end{align*}
(b)
\begin{align*}
    \frac{\mathrm d}{\mathrm dx}\frac{f(x)}{g(x)}
    =&\frac{\mathrm d}{\mathrm dx}f(x)\frac{1}{g(x)}\\
    =&\left(\frac{\mathrm d}{\mathrm dx}f(x)\right)\frac{1}{g(x)}+f(x)\left(\frac{\mathrm d}{\mathrm dx}\frac{1}{g(x)}\right)\\
    =&\frac{f^\prime(x)}{g(x)}-\frac{f(x)g^\prime(x)}{g(x)^2}\\
    =&\frac{f^\prime(x)g(x)-f(x)g^\prime(x)}{g(x)^2}
\end{align*}

\subsection{Section 4}
1. $x^9e^x+9x^8e^x$.\\
2. $5x^4-1$.

\subsection{Section 5}
1. $\frac{e^{\sqrt{x}}}{2\sqrt{x}}$

\subsection{Section 6}
1. $ex^{e-1}-e^x$

\subsection{Section 7}
1. (3)
$$\frac{\mathrm d}{\mathrm dx}\tan(x)=\frac{\mathrm d}{\mathrm dx}\frac{\sin(x)}{\cos(x)}=\frac{\sin^2(x)+\cos^2(x)}{\cos^2(x)}=\frac{1}{\cos^2(x)}=\sec^2(x)$$
(4)
$$\frac{\mathrm d}{\mathrm dx}\sec(x)=\frac{\mathrm d}{\mathrm dx}\frac{1}{\cos(x)}=-\frac{1}{\cos^2(x)}(-\sin(x))=\sec(x)\tan(x)$$
(5)
$$\frac{\mathrm d}{\mathrm dx}\cot(x)=\frac{\mathrm d}{\mathrm dx}\frac{\cos(x)}{\sin(x)}=\frac{-\sin^2(x)-\cos^2(x)}{\sin^2(x)}=-\frac{1}{\sin^2(x)}=-\csc^2(x)$$
(6)
$$\frac{\mathrm d}{\mathrm dx}\csc(x)=\frac{\mathrm d}{\mathrm dx}\frac{1}{\sin(x)}=-\frac{1}{\sin^2(x)}(\cos(x))=-\csc(x)\cot(x)$$
2. (3)
\begin{align*}
    \lim_{h\to0}\frac{\tan(x+h)-\tan(x)}{h}
    =&\lim_{h\to0}\frac{\sin(x+h)\cos(x)-\cos(x+h)\sin(x)}{h\cos(x)\cos(x+h)}\\
    =&\lim_{h\to0}\frac{\sin(h)}{h}\frac{1}{\cos(x)\cos(x+h)}\\
    =&\sec^2(x)
\end{align*}
Therefore,
$$\frac{\mathrm d\tan(x)}{\mathrm dx}=\sec^2(x)$$
(4)
\begin{align*}
    \lim_{h\to0}\frac{\sec(x+h)-\sec(x)}{h}
    =&\lim_{h\to0}\frac{\cos(x)-\cos(x+h)}{h\cos(x)\cos(x+h)}\\
    =&\lim_{h\to0}\frac{-2\sin(x+h/2)\sin(-h/2)}{h\cos(x)\cos(x+h)}\\
    =&\lim_{h\to0}\frac{\sin(x+h/2)}{\cos(x)\cos(x+h)}\frac{\sin(h/2)}{h/2}\\
    =&\frac{\sin(x)}{\cos^2(x)}\\
    =&\sec(x)\tan(x)
\end{align*}
Therefore,
$$\frac{\mathrm d\sec(x)}{\mathrm dx}=\sec(x)\tan(x)$$
(5)
\begin{align*}
    \lim_{h\to0}\frac{\cot(x+h)-\cot(x)}{h}
    =&\lim_{h\to0}\frac{\sin(x)\cos(x+h)-\cos(x)\sin(x+h)}{h\sin(x)\sin(x+h)}\\
    =&\lim_{h\to0}-\frac{\sin(h)}{h}\frac{1}{\sin(x)\sin(x+h)}\\
    =&-\csc^2(x)
\end{align*}
Therefore,
$$\frac{\mathrm d\cot(x)}{\mathrm dx}=-\csc^2(x)$$
(6)
\begin{align*}
    \lim_{h\to0}\frac{\csc(x+h)-\csc(x)}{h}
    =&\lim_{h\to0}\frac{\sin(x)-\sin(x+h)}{h\sin(x)\sin(x+h)}\\
    =&\lim_{h\to0}\frac{2\cos(x+h/2)\sin(-h/2)}{h\sin(x)\sin(x+h)}\\
    =&\lim_{h\to0}-\frac{\cos(x+h/2)}{\sin(x)\sin(x+h)}\frac{\sin(h/2)}{h/2}\\
    =&-\frac{\cos(x)}{\sin^2(x)}\\
    =&-\csc(x)\cot(x)
\end{align*}
Therefore,
$$\frac{\mathrm d\csc(x)}{\mathrm dx}=-\csc(x)\cot(x)$$
3.
$$f^\prime(x)=\cos(x^2\cos(e^x))(2x\cos(e^x)-x^2\sin(e^x)e^x)$$

\subsection{Section 8}
1. We have $x^x=e^{x\ln x}$, so
$$\frac{\mathrm dx^x}{\mathrm dx}=\frac{\mathrm de^{x\ln x}}{\mathrm dx}=e^{x\ln x}(\ln x+x\frac{1}{x})=x^x(\ln x+1)$$
2.
\begin{align*}
    ye^y=&\ln x\\
    (ye^y+e^y)\frac{\mathrm dy}{\mathrm dx}=&\frac{1}{x}\\
    \frac{\mathrm dy}{\mathrm dx}=&\frac{1}{xye^y+xe^y}
\end{align*}
So when $x=1,y=0$, we have
$$\left.\frac{\mathrm dy}{\mathrm dx}\right|_{(1,0)}=1$$

\subsection{Section 9}
1. (a) When $f(x)=\sin(x)$,
$$\mathrm{L.H.S.}=f^{\prime\prime}=\frac{\mathrm d}{\mathrm dx}\cos(x)=-\sin(x)=-f=\mathrm{R.H.S.}$$
When $f(x)=\cos(x)$,
$$\mathrm{L.H.S.}=f^{\prime\prime}=\frac{\mathrm d}{\mathrm dx}-\sin(x)=-\cos(x)=-f=\mathrm{R.H.S.}$$
(b) When $f(x)=A\cos(x)+B\sin(x)$,
\begin{align*}
    \mathrm{L.H.S.}
    =&f^{\prime\prime}\\
    =&\frac{\mathrm d}{\mathrm dx}(-A\sin(x)+B\cos(x))\\
    =&-A\cos(x)-B\sin(x)\\
    =&-f=\mathrm{R.H.S.}
\end{align*}


\section{Dependent function types}
\label{ch:pi}

A fundamental concept in dependent type theory is that of a dependent function. A dependent function is a function of which the type of the output may depend on the input. They are a generalization of ordinary functions, because an ordinary function $f:A\to B$ is a function of which the output $f(x)$ has type $B$ regardless of the value of $x$.

\subsection{Dependent function types}
Consider a section $b$ of a family $B$ over $A$ in context $\Gamma$, i.e.,
\begin{equation*}
  \Gamma,x:A\vdash b(x):B(x).
\end{equation*}
From one point of view, such a section $b$ is an operation, or a program, that takes as input $x:A$ and produces a term $b(x):B(x)$. From a more mathematical point of view we see $b$ as a choice of an element of each $B(x)$. In other words, we may see $b$ as a function that takes $x:A$ to $b(x):B(x)$. Note that the type $B(x)$ of the output is dependent on $x:A$. In this section we postulate rules for the \emph{type} of all such dependent functions: whenever $B$ is a family over $A$ in context $\Gamma$, there is a type
\begin{equation*}
  \prd{x:A}B(x)
\end{equation*}
in context $\Gamma$, consisting of all the dependent functions of which the output at $x:A$ has type $B(x)$. There are four principal rules for $\Pi$-types:
\begin{enumerate}
\item The formation rule, which tells us how we may form dependent function types.
\item The introduction rule, which tells us how to introduce new terms of dependent function types.
\item The elimination rule, which tells us how to use arbitrary terms of dependent function types.
\item The computation rules, which tell us how the introduction and elimination rules interact. These computation rules guarantee that every term of a dependent function type behaves as expected: as a dependent function.
\end{enumerate}
In the cases of the formation rule, the introduction rule, and the elimination rule, we will also provide conversion rules that ensure that all the constructions respect judgmental equality.

\subsubsection{The $\Pi$-formation rule}
\define{Dependent function types}\index{dependent function type}\index{pi-type@{$\Pi$-type}} are formed by the following \define{$\Pi$-formation rule}\index{rule!pi-formation@{$\Pi$-formation}}:
\begin{prooftree}
\AxiomC{$\Gamma,x:A\vdash B(x)~\textrm{type}$}
\RightLabel{$\Pi$.}
\UnaryInfC{$\Gamma\vdash \prd{x:A}B(x)~\mathrm{type}$}
\end{prooftree}
With the following conversion rule we postulate that formation of dependent function types respects judgmental equality:
\begin{prooftree}
\AxiomC{$\Gamma\vdash A\jdeq A'~\mathrm{type}$}
\AxiomC{$\Gamma,x:A\vdash B(x)\jdeq B'(x)~\textrm{type}$}
\RightLabel{$\Pi$-eq.}
\BinaryInfC{$\Gamma\vdash \prd{x:A}B(x)\jdeq\prd{x:A'}B'(x)~\mathrm{type}$}
\end{prooftree}
Furthermore, when $x'$ is a fresh variable, i.e., which does not occur in the context $\Gamma,x:A$, we also postulate that
\begin{prooftree}
\AxiomC{$\Gamma,x:A\vdash B(x)~\textrm{type}$}
\RightLabel{$\Pi$-$x'/x$.}
\UnaryInfC{$\Gamma\vdash \prd{x:A}B(x)\jdeq \prd{x':A}B(x')~\mathrm{type}$}
\end{prooftree}

\subsubsection{The $\Pi$-introduction rule}
The introduction rule for dependent function types is also called the $\lambda$-abstraction rule. Recall that dependent functions are formed from terms $b(x)$ of type $B(x)$ in context $\Gamma,x:A$. Therefore \define{$\lambda$-abstraction rule}\index{lambda-abstraction@{$\lambda$-abstraction}}\index{rule!lambda-abstraction@{$\lambda$-abstraction}} is as follows:
\begin{prooftree}
  \AxiomC{$\Gamma,x:A \vdash b(x) : B(x)$}
  \RightLabel{$\lambda$}
  \UnaryInfC{$\Gamma\vdash \lam{x}b(x) : \prd{x:A}B(x)$}
\end{prooftree}

Just like ordinary mathematicians, we will sometimes write $x\mapsto f(x)$ for a function $f$. The map $n\mapsto n^2$ is an example. The $\lambda$-abstraction is also required to respect judgmental equality. Therefore we postulate that
\begin{prooftree}
  \AxiomC{$\Gamma,x:A \vdash b(x)\jdeq b'(x) : B(x)$}
  \RightLabel{$\lambda$-eq.}
  \UnaryInfC{$\Gamma\vdash \lam{x}b(x)\jdeq \lam{x}b'(x) : \prd{x:A}B(x)$}
\end{prooftree}

\subsubsection{The $\Pi$-elimination rule}

The elimination rule for dependent function types provides us with a way to \emph{use} dependent functions. The way to use a dependent function is to apply it to an argument of the domain type. The $\Pi$-elimination rule is therefore also called the \define{evaluation rule}\index{evaluation}\index{rule!evaluation}. It asserts that given a dependent function $f:\prd{x:A}B(x)$ in context $\Gamma$ we obtain a term $f(x)$ of type $B(x)$ in context $\Gamma,x:A$. More formally:
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:\prd{x:A}B(x)$}
\RightLabel{$ev$}
\UnaryInfC{$\Gamma,x:A\vdash f(x) : B(x)$}
\end{prooftree}
Again we require that evaluation respects judgmental equality:
\begin{prooftree}
  \AxiomC{$\Gamma\vdash f\jdeq f':\prd{x:A}B(x)$}
  \UnaryInfC{$\Gamma,x:A\vdash f(x)\jdeq f'(x):B(x)$}
\end{prooftree}

\subsubsection{The $\Pi$-computation rules}
The computation rules for dependent function types postulate that $\lambda$-abstraction rule and the evaluation rule are mutual inverses. Thus we have two computation rules.

First we postulate the \define{$\beta$-rule}\index{beta-rule@{$\beta$-rule}}\index{rule!beta-rule@{$\beta$-rule}}
\begin{prooftree}
\AxiomC{$\Gamma,x:A \vdash b(x) : B(x)$}
\RightLabel{$\beta$.}
\UnaryInfC{$\Gamma,x:A \vdash (\lambda y.b(y))(x)\jdeq b(x) : B(x)$}
\end{prooftree}
Second, we postulate the \define{$\eta$-rule}\index{eta-rule@{$\eta$-rule}}\index{rule!eta-rule@{$\eta$-rule}}
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:\prd{x:A}B(x)$}
\RightLabel{$\eta$.}
\UnaryInfC{$\Gamma \vdash \lam{x}f(x) \jdeq f : \prd{x:A}B(x)$}
\end{prooftree}
This completes the specification of dependent function types.

\subsection{Ordinary function types}
In the case where both $A$ and $B$ are types in context $\Gamma$, we may first weaken $B$ by $A$, and then apply the formation rule for the dependent function type:
\begin{prooftree}
\AxiomC{$\Gamma\vdash A~\textrm{type}$}
\AxiomC{$\Gamma\vdash B~\textrm{type}$}
\BinaryInfC{$\Gamma,x:A\vdash B~\textrm{type}$}
\UnaryInfC{$\Gamma\vdash \prd{x:A}B~\textrm{type}$}
\end{prooftree}
The result is the type of functions that take an argument of type $A$, and return a term of type $B$. In other words, terms of the type $\prd{x:A}B$ are \emph{ordinary} functions from $A$ to $B$. We write $A\to B$ for the \define{type of functions}\index{function type} from $A$ to $B$. Sometimes we will also write $B^A$ for the type $A\to B$.

We give a brief summary of the rules specifying ordinary function types, omitting the rules that the asserted operations respect judgmental equality. All of these rules can be derived from the corresponding rules for $\Pi$-types.
\begin{prooftree}
\AxiomC{$\Gamma\vdash A~\textrm{type}$}
\AxiomC{$\Gamma\vdash B~\textrm{type}$}
\RightLabel{$\to$\index{arrow-formation@{$\to$-formation}}\index{rule!arrow-formation@{$\to$-formation}}}
\BinaryInfC{$\Gamma\vdash A\to B~\textrm{type}$}
\end{prooftree}%
\begin{center}
\begin{minipage}{.45\textwidth}
\begin{prooftree}
\AxiomC{$\Gamma\vdash B~\textrm{type}$}
\AxiomC{$\Gamma,x:A\vdash b(x):B$}
\RightLabel{$\lambda$\index{lambda-abstraction@{$\lambda$-abstraction}}\index{rule!lambda-abstraction@{$\lambda$-abstraction}}}
\BinaryInfC{$\Gamma\vdash \lam{x}b(x):A\to B$}
\end{prooftree}%
\end{minipage}
\begin{minipage}{.45\textwidth}
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:A\to B$}
\RightLabel{$ev$\index{rule!evaluation}\index{evaluation}}
\UnaryInfC{$\Gamma,x:A\vdash f(x):B$}
\end{prooftree}%
\end{minipage}
\end{center}
\begin{center}
\begin{minipage}{.45\textwidth}
\begin{prooftree}
\AxiomC{$\Gamma\vdash B~\textrm{type}$}
\AxiomC{$\Gamma,x:A\vdash b(x):B$}
\RightLabel{$\beta$\index{rule!beta-rule@{$\beta$-rule}}\index{beta-rule@{$\beta$-rule}}}
\BinaryInfC{$\Gamma,x:A\vdash(\lam{y}b(y))(x)\jdeq b(x):B$}
\end{prooftree}%
\end{minipage}
\begin{minipage}{.45\textwidth}
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:A\to B$}
\RightLabel{$\eta$\index{rule!eta-rule@{$\eta$-rule}}\index{eta-rule@{$\eta$-rule}}}
\UnaryInfC{$\Gamma\vdash\lam{x} f(x)\jdeq f:A\to B$}
\end{prooftree}
\end{minipage}
\end{center}

\begin{comment}
\begin{rmk}
Similar to \cref{rmk:ev_var}, we can derive
\begin{prooftree}
\AxiomC{$\Gamma\vdash A~\mathrm{type}$}
\AxiomC{$\Gamma\vdash B~\mathrm{type}$}
\BinaryInfC{$\Gamma,f:B^A,x:A\vdash f(x):B$}
\end{prooftree}
\end{rmk}
\end{comment}

\subsection{The identity function, composition, and their laws}
\begin{defn}
For any type $A$ in context $\Gamma$, we define the \define{identity function}\index{identity function|textbf} $\idfunc[A]:A\to A$ using the variable rule\index{variable rule}\index{rule!variable rule}:
\begin{prooftree}
\AxiomC{$\Gamma\vdash A~\textrm{type}$}
\UnaryInfC{$\Gamma,x:A\vdash x:A$}
\UnaryInfC{$\Gamma\vdash \idfunc[A]\defeq \lam{x}x:A\to A$}
\end{prooftree}
\end{defn}

A judgment of the form $\Gamma\vdash a\defeq b:A$ should be read as "$b$ is a well-defined term of type $A$ in context $\Gamma$, and we will refer to it as $a$".

\begin{defn}
For any three types $A$, $B$, and $C$ in context $\Gamma$, there is a \define{composition}\index{composition!of functions|textbf} operation
\begin{equation*}
\mathsf{comp}:(B\to C)\to ((A\to B)\to (A\to C)),
\end{equation*}
i.e., we can derive
\begin{prooftree}
\AxiomC{$\Gamma\vdash A~\textrm{type}$}
\AxiomC{$\Gamma\vdash B~\textrm{type}$}
\AxiomC{$\Gamma\vdash C~\textrm{type}$}
\TrinaryInfC{$\Gamma\vdash\mathsf{comp}:(B\to C)\to ((A\to B)\to (A\to C))$}
\end{prooftree}
We will write $g\circ f$ for $\mathsf{comp}(g,f)$.
\end{defn}

\begin{constr}
  The idea of the definition is to define $\mathsf{comp}(g,f)$ to be the function $\lam{x}g(f(x))$. The derivation we use to construct $\mathsf{comp}$ is as follows:
  \begin{prooftree}
    \AxiomC{$\Gamma\vdash A~\mathrm{type}$}
    \AxiomC{$\Gamma\vdash B~\mathrm{type}$}
    \BinaryInfC{$\Gamma,f:B^A,x:A\vdash f(x):B$}
    \UnaryInfC{$\Gamma,g:C^B,f:B^A,x:A\vdash f(x):B$}
    \AxiomC{$\Gamma\vdash B~\mathrm{type}$}
    \AxiomC{$\Gamma\vdash C~\mathrm{type}$}
    \BinaryInfC{$\Gamma,g:C^B,y:B\vdash g(y):C$}
    \UnaryInfC{$\Gamma,g:C^B,f:B^A,y:B\vdash g(y):C$}
    \UnaryInfC{$\Gamma,g:C^B,f:B^A,x:A,y:B\vdash g(y):C$}
    \BinaryInfC{$\Gamma,g:C^B,f:B^A,x:A\vdash g(f(x)) : C$}
    \UnaryInfC{$\Gamma,g:C^B,f:B^A\vdash \lam{x}g(f(x)):C^A$}
    \UnaryInfC{$\Gamma,g:B\to C\vdash \lam{f}{x}g(f(x)):B^A\to C^A$}
    \UnaryInfC{$\Gamma\vdash\mathsf{comp}\defeq \lam{g}{f}{x}g(f(x)):C^B\to (B^A\to C^A)$}
  \end{prooftree}
\end{constr}

\begin{lem}
Composition of functions is associative\index{associativity!of function composition}\index{composition!of functions!associativity}, i.e., we can derive
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:A\to B$}
\AxiomC{$\Gamma\vdash g:B\to C$}
\AxiomC{$\Gamma\vdash h:C\to D$}
\TrinaryInfC{$\Gamma \vdash (h\circ g)\circ f\jdeq h\circ(g\circ f):A\to D$}
\end{prooftree}
\end{lem}

\begin{proof}
  The main idea of the proof is that both $((h\circ g)\circ f)(x)$ and $(h\circ (g\circ f))(x)$ evaluate to $h(g(f(x))$, and therefore $(h\circ g)\circ f$ and $h\circ(g\circ f)$ must be judgmentally equal. This idea is made formal in the following derivation:
  \begin{prooftree}
    \AxiomC{$\Gamma\vdash f:A\to B$}
    \UnaryInfC{$\Gamma,x:A\vdash f(x):B$}
    \AxiomC{$\Gamma\vdash g:B\to C$}
    \UnaryInfC{$\Gamma,y:B\vdash g(y):C$}
    \UnaryInfC{$\Gamma,x:A,y:B\vdash g(y):C$}
    \BinaryInfC{$\Gamma,x:A\vdash g(f(x)):C$}
    \AxiomC{$\Gamma\vdash h:C\to D$}
    \UnaryInfC{$\Gamma,z:C\vdash h(z):D$}
    \UnaryInfC{$\Gamma,x:A,z:C\vdash h(z):D$}
    \BinaryInfC{$\Gamma,x:A\vdash h(g(f(x))):D$}
    \UnaryInfC{$\Gamma,x:A\vdash h(g(f(x)))\jdeq h(g(f(x))):D$}
    \UnaryInfC{$\Gamma,x:A\vdash (h\circ g)(f(x))\jdeq h((g\circ f)(x)):D$}
    \UnaryInfC{$\Gamma,x:A\vdash ((h\circ g)\circ f)(x)\jdeq (h\circ (g \circ f))(x):D$}
    \UnaryInfC{$\Gamma\vdash (h\circ g)\circ f\jdeq h\circ(g\circ f):A\to D$.}
  \end{prooftree}
\end{proof}

\begin{lem}\label{lem:fun_unit}
Composition of functions satisfies the left and right unit laws\index{left unit law|see {unit laws}}\index{right unit law|see {unit laws}}\index{unit laws!of function composition}, i.e., we can derive
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:A\to B$}
\UnaryInfC{$\Gamma\vdash \idfunc[B]\circ f\jdeq f:A\to B$}
\end{prooftree}
and
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:A\to B$}
\UnaryInfC{$\Gamma\vdash f\circ\idfunc[A]\jdeq f:A\to B$}
\end{prooftree}
\end{lem}

\begin{proof}
The derivation for the left unit law is
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:A\to B$}
\UnaryInfC{$\Gamma,x:A\vdash f(x):B$}
\AxiomC{$\Gamma\vdash B~\mathrm{type}$}
\UnaryInfC{$\Gamma,y:B\vdash \idfunc[B](y)\jdeq y:B$}
\UnaryInfC{$\Gamma,x:A,y:B\vdash \idfunc[B](y)\jdeq y:B$}
\BinaryInfC{$\Gamma,x:A\vdash \idfunc[B](f(x))\jdeq f(x):B$}
\UnaryInfC{$\Gamma,x:A\vdash (\idfunc[B]\circ f)(x)\jdeq f(x):B$}
\UnaryInfC{$\Gamma\vdash \idfunc[B]\circ f\jdeq f:A\to B$}
\end{prooftree}
The right unit law is left as \cref{ex:fun_right_unit}.
\end{proof}

\begin{exercises}
\item \label{ex:fun_right_unit}Give a derivation for the right unit law of \cref{lem:fun_unit}.\index{unit laws!for function composition}
\item Show that the rule
\begin{prooftree}
\AxiomC{$\Gamma,x:A \vdash b(x) : B(x)$}
\RightLabel{$\lambda$-$x'/x$}
\UnaryInfC{$\Gamma\vdash \lam{x}b(x)\jdeq \lam{x'}b(x') : \prd{x:A}B(x)$}
\end{prooftree}
is admissible for any variable $x'$ that does not occur in the context $\Gamma,x:A$.
\item
  \begin{subexenum}
  \item Construct the \define{constant function}\index{constant function}\index{function!constant function}
    \begin{prooftree}
      \AxiomC{$\Gamma\vdash A~\textrm{type}$}
      \UnaryInfC{$\Gamma,y:B\vdash \mathsf{const}_y:A\to B$}
    \end{prooftree}
  \item Show that
    \begin{prooftree}
      \AxiomC{$\Gamma\vdash f:A\to B$}
      \UnaryInfC{$\Gamma,z:C\vdash \mathsf{const}_z\circ f\jdeq\mathsf{const}_z : A\to C$}
    \end{prooftree}
  \item Show that
    \begin{prooftree}
      \AxiomC{$\Gamma\vdash A~\textrm{type}$}
      \AxiomC{$\Gamma\vdash g:B\to C$}
      \BinaryInfC{$\Gamma,y:B\vdash g\circ\mathsf{const}_y\jdeq \mathsf{const}_{g(y)}:A\to C$}
    \end{prooftree}
  \end{subexenum}
\item In this exercise we generalize the composition operation of non-dependent function types\index{composition!of dependent functions}:
\begin{subexenum}
\item Define a composition operation for dependent function types
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:\prd{x:A}B(x)$}
\AxiomC{$\Gamma\vdash g:\prd{x:A}{y:B(x)} C(x,y)$}
\BinaryInfC{$\Gamma\vdash g\circ' f:\prd{x:A} C(x,f(x))$}
\end{prooftree}
and show that this operation agrees with ordinary composition when it is specialized to non-dependent function types.
\item Show that composition of dependent functions agrees with ordinary composition of functions:
  \begin{prooftree}
    \AxiomC{$\Gamma\vdash f:A\to B$}
    \AxiomC{$\Gamma\vdash g:B\to C$}
    \BinaryInfC{$\Gamma\vdash (\lam{x}g)\circ' f\jdeq g\circ f:A \to C$}
  \end{prooftree}
\item Show that composition of dependent functions is associative.\index{associativity!of dependent function composition}\index{composition!of dependent functions!associativity}
\item Show that composition of dependent functions satisfies the right unit law\index{unit laws!dependent function composition}:
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:\prd{x:A}B(x)$}
\UnaryInfC{$\Gamma\vdash (\lam{x}f)\circ'\idfunc[A]\jdeq f :\prd{x:A}B(x)$}
\end{prooftree}
\item Show that composition of dependent functions satisfies the left unit law\index{unit laws!dependent function composition}:
\begin{prooftree}
\AxiomC{$\Gamma\vdash f:\prd{x:A}B(x)$}
\UnaryInfC{$\Gamma\vdash (\lam{x}\idfunc[B(x)])\circ' f\jdeq f:\prd{x:A}B(x)$}
\end{prooftree}
\end{subexenum}
\item \label{ex:swap}
\begin{subexenum}
\item Given two types $A$ and $B$ in context $\Gamma$, and a type $C$ in context $\Gamma,x:A,y:B$, define the \define{swap function}\index{function!swap}\index{swap function}
\begin{equation*}
\Gamma\vdash \sigma:\Big(\prd{x:A}{y:B}C(x,y)\Big)\to\Big(\prd{y:B}{x:A}C(x,y)\Big)
\end{equation*}
that swaps the order of the arguments.
\item Show that
\begin{equation*}
\Gamma\vdash \sigma\circ\sigma\jdeq\idfunc:\Big(\prd{x:A}{y:B}C(x,y)\Big)\to \Big(\prd{x:A}{y:B}C(x,y)\Big).
\end{equation*}
\end{subexenum}
\end{exercises}


\section{Basic Calculus}
\begin{definition}
    Let $f:U\to\mathbb R$ where $U\subset\mathbb R$, then if $\forall\epsilon>0,\exists\delta>0$ such that
    $$|x-x_0|<\delta\implies|f(x)-A|<\epsilon$$
    for some $x_0,A\in\mathbb R$, we say
    $$\lim_{x\to x_0}f(x)=A$$
\end{definition}
\begin{theorem}
    Assume that
    $$\lim_{x\to x_0}f(x)=A, \lim_{x\to x_0}f(x)=B$$
    Then the followings hold:
    $$\lim_{x\to x_0}cf(x)=cA$$
    $$\lim_{x\to x_0}f(x)+g(x)=A+B$$
    $$\lim_{x\to x_0}f(x)g(x)=AB$$
    $$B\neq 0\implies\lim_{x\to x_0}f(x)/g(x)=A/B$$
\end{theorem}
\begin{proof}
    Check definitions.
\end{proof}
\begin{definition}[One-sided limits]
    If $\forall\epsilon>0,\exists\delta>0$ such that $0<x-x_0<\delta\implies|f(x)-A|<\epsilon$
    for some $x_0,A\in\mathbb R$, we say
    $$\lim_{x\to x_0^+}f(x)=A$$
    Simialrly, if it is $0<x_0-x<\delta\implies|f(x)-A|<\epsilon$, then
    $$\lim_{x\to x_0^-}f(x)=A$$
\end{definition}
\begin{definition}
    If the limit
    $$\lim_{h\to0}\frac{f(x+h)-f(x)}{h}$$
    exists for all $x$ in a given domain, we say that $f$ is differentiable in this domain and its derivative is
    $$f^\prime(x)=\frac{\mathrm df}{\mathrm dx}=\lim_{h\to0}\frac{f(x+h)-f(x)}{h}$$
\end{definition}
For sufficiently smooth functions, we can differentiate it recursively.
We denote the $n^{th}$ derivative of $f$ as
$$\frac{d^nf}{dx^n}\text{ or }f^{(n)}(x)$$
Several immediate facts are available from here:
\begin{theorem}
    1. The differential operator is linear.\\
    2. (Chain Rule) Suppose both $F$ and $g$ are differentiable and $f(x)=F(g(x))$,
    then $f$ is differentiable and $f^\prime(x)=F^\prime(g(x))g^\prime(x)$.\\
    3. (Leibniz's Rule) Suppose both $u$ and $v$ are differentiable and $f(x)=u(x)v(x)$, then
    $$f^{(n)}(x)=\sum_{k=0}^n\binom{n}{k}u^{(k)}(x)v^{(n-k)}(x)$$
    in particular $f^\prime(x)=u(x)v^\prime(x)+u^\prime(x)v(x)$.
\end{theorem}
\begin{proof}
    Trivial.
\end{proof}


\section{Introduction}

An important area of research in quantum computing is
to determine the problems for which quantum
computers can outperform classical computers.
To avoid technical problems in the comparison of
computers--the largest
of which is that large-scale quantum computers
currently do not exist--we turn to theoretical
analysis to characterize the efficiency of
quantum and classical algorithms.

We implement a tool that places an asymptotic lower bound on runtime by calculating the optimal quantum query complexity of a Boolean function.
The query complexity of a function
is the number of times we must examine the
contents of the input string to determine the correct
output of the function.
Each query of the given function takes a certain
amount of time, therefore the query
complexity places a lower bound on runtime.
The reverse is not true: the runtime
may asymptotically exceed query complexity
if the algorithm performs substantially more
computations than queries.

To give a concrete example of query complexity,
recall the canonical OR function.
The $n$-bit OR
function returns $0$ if there are only $0$'s in the input
string and $1$ if there are any $1$'s in the input string.
In the worst case, a classical algorithm must check that
every single bit of the input is a $0$ to return $0$.
Therefore the classical query complexity of the $n$-bit OR
function is $n$.
However, the quantum algorithm Grover's
search can solve the function OR in $\sqrt{n}$ queries
\cite{grover1996fast}. In the case of OR, we conclude that
the quantum algorithm outperforms the classical algorithm.

For an arbitrary function $f$,
Reichardt presents a semidefinite program (SDP)
whose solution corresponds to the optimal quantum query complexity
of $f$ \cite{reichardt2009span}.
In addition, the solution of the same SDP can
be used to construct a span program that, when run
on a quantum computer, meets the optimal quantum query complexity.

Our contribution is an SDP solver
that finds the optimal quantum query complexity
and query optimal quantum algorithm for given Boolean functions.
While the SDP can be solved for arbitrary functions with finite outputs,
we limit the scope of our work to Boolean functions.

Although there are many SDP solver libraries such as CVXOPT and
SDPA for convex optimization problems,
none easily support solving Reichardt's SDP
\cite{cvxopt, SDPA}.
As a result, we develop an implementation that solves the SDP
and includes optimizations specific to our problem formulation.
We first convert Reichardt's form into the standard SDP form
as described by Boyd \cite{boyd2004convex}, which we verify here.
To solve the SDP, we implement Wen et al.'s alternating direction method (ADM)
algorithm in order to exploit the sparsity of our SDP.
We finally optimize the functions and data structures we use
to speed up our program \cite{adm}.

Our program takes as input a set of bitstrings $D$
and a Boolean function $f: D \rightarrow \{0,1\}$.
By solving Reichardt's SDP problem with
Wen et al.'s ADM algorithm,
we return the optimal quantum query complexity of $f$
and a quantum algorithm that meets this query complexity.
We hope that our solver proves useful
for researchers constructing optimal quantum algorithms.


\subsection{Lines}
\begin{itemize}
	\item Lines through the origin can be represented as $\theta = \tan^{-1}{m}$ where $m$ is the slope of the line.
	\item Lines of the form $x = a$ can be represented as $r = a\sec{\theta}$.
	\item Lines of the form $y = a$ can be represented as $r = a\csc{\theta}$.
	\item All other lines of the form $y=ax+b$ can be represented as $r = \frac{b}{\sin{\theta} - a\cos{\theta}}$. This form also covers the previous 2.
\end{itemize}


\documentclass{ximera}
\input{../preamble}
\title{Exercises: Cumulative}
%%%%%\author{Philip T. Gressman}

\begin{document}
\begin{abstract}
Exercises relating to various topics we have studied.
\end{abstract}
\maketitle

A key skill to practice is deciding which of the several techniques we have learned should be applied to a particular problem. In these exercises and questions, you will be responsible for choosing a method which works for you.

\begin{exercise}
Compute the indefinite integral.
\[ \int \ln x \, dx = \answer{ x \ln x - x} + C\]
(You should choose an answer which equals $-1$ at $x = 1$; if your answer does not satisfy this, add a constant to make it so.)
\begin{hint}
Integrate by parts.
\end{hint}
\begin{hint}
Let $u = \ln x$ and $dv = 1$.
\end{hint}
\end{exercise}

\begin{exercise}
Compute the indefinite integral.
\[ \int \frac{dx}{(x^2+1)^2} = \answer{ \frac{1}{2} \frac{x}{x^2+1} + \frac{1}{2} \arctan x} + C.\]
(You should choose an answer which equals $0$ at $x = 0$; if your answer does not satisfy this, add a constant to make it so.)
\begin{hint}
Partial fractions won't help in this case because the expression is already simplified as much as possible.
\end{hint}
\begin{hint}
Try a trigonometric substitution
\end{hint}
\begin{hint}
Make the substitution $x = \tan \theta$.
\end{hint}
\begin{hint}
You should arrive at the integral $\int \cos^2 \theta \, d \theta$ after the substitution; here you'll need to use a power-reduction formula to continue.
\end{hint}
\end{exercise}

\section*{Sample Quiz Questions}

\begin{question}%%%%%[shelllogibp001]

The region in the plane between the  \(x\)-axis and the graph
\[ y = \frac{1}{2} \ln{x} + 1 \]
 in the range \(\frac{1}{5} \leq x \leq 1\) is revolved around the axis \(x = \frac{1}{10}\). Compute the volume of the resulting solid.
\begin{multiplechoice}
\choice{\(\displaystyle \frac{11}{25} \pi\)}
\choice{\(\displaystyle \frac{14}{25} \pi\)}
\choice[correct]{\(\displaystyle \frac{16}{25} \pi\)}
\choice{\(\displaystyle \frac{19}{25} \pi\)}
\choice{\(\displaystyle \frac{21}{25} \pi\)}
\choice{\(\displaystyle \frac{24}{25} \pi\)}
\end{multiplechoice}
\begin{feedback}
If the variable \(x\) is used for slicing, then slices are parallel to the axis of rotation, which indicates the shell method should be used.
The radius of a shell is \(x - \frac{1}{10}\). The height of a shell is exactly \(\frac{1}{2} \ln{x} + 1\).
The volume of the region is therefore given by
\[ \int_{\frac{1}{5}}^{1} \frac{\pi}{10} \left(10 x - 1\right) \left(\ln{x} + 2\right)\, dx. \]
 To compute the integral, we can use integration by parts. A reasonable strategy is to integrate  \(x - \frac{1}{10}\) and differentiate  \(\ln{x} + 2\).
 This gives the equality
\[ \begin{aligned} \pi \int \left(x - \frac{1}{10}\right) \left(\ln{x} + 2\right)\, dx & = \pi \left(\left(\frac{x^{2}}{2} - \frac{x}{10}\right) \left(\ln{x} + 2\right) - \int \frac{1}{x} \left(\frac{x^{2}}{2} - \frac{x}{10}\right)\, dx\right) \\
 & = - \pi \left(\frac{x^{2}}{4} - \frac{x}{10}\right) + \pi \left(\frac{x^{2}}{2} - \frac{x}{10}\right) \left(\ln{x} + 2\right). \end{aligned} \]
Therefore
\[ \begin{aligned} \pi \int_{\frac{1}{5}}^{1} \left(x - \frac{1}{10}\right) \left(\ln{x} + 2\right)\, dx & = \left. \left[- \pi \left(\frac{x^{2}}{4} - \frac{x}{10}\right) + \pi \left(\frac{x^{2}}{2} - \frac{x}{10}\right) \left(\ln{x} + 2\right) \right] \right|_{\frac{1}{5}}^{1}\\ & = \left(\frac{13}{20} \pi \right) - \left(\frac{\pi}{100} \right) = \frac{16}{25} \pi. \end{aligned} \]
\end{feedback}

\end{question}

\begin{question}%%%%%[COMsinibp001]

Consider the region given by \(2 \pi \leq x \leq \frac{5}{2} \pi\) and \(0 \leq y \leq \sin {x}\).
Compute the \(x\)-coordinate of the centroid (i.e., assuming constant density).
\begin{multiplechoice}
\choice{\(\displaystyle -1 + \frac{5}{2} \pi\)}
\choice[correct]{\(\displaystyle 1 + 2 \pi\)}
\choice{\(\displaystyle \frac{5}{2} \pi\)}
\choice{\(\displaystyle -1 + 3 \pi\)}
\choice{\(\displaystyle 3 \pi\)}
\choice{\(\displaystyle 4 \pi\)}
\end{multiplechoice}
\begin{feedback}
The mass  \(M\) will be given by the integral
\[ \int_{2 \pi}^{\frac{5}{2} \pi} \sin {x}\, dx \]
 One can check that
\[ \begin{aligned} \int_{2 \pi}^{\frac{5}{2} \pi} \sin {x}\, dx & = 1. \end{aligned} \]
To compute the  \(x\)-coordinate of the centroid, we also need to compute the integral
\[ \int_{2 \pi}^{\frac{5}{2} \pi} x \sin {x}\, dx \]
 To compute the integral, we can use integration by parts. A reasonable strategy is to integrate  \(\sin {x}\) and differentiate  \(x\).
 This gives the equality
\[ \begin{aligned} \int x \sin {x}\, dx & = - x \cos {x} - \int \left(- \cos {x}\right)\, dx \\
 & = - x \cos {x} + \sin {x}. \end{aligned} \]
Therefore
\[ \begin{aligned} \int_{2 \pi}^{\frac{5}{2} \pi} x \sin {x}\, dx & = \left. \left[- x \cos {x} + \sin {x} \right] \right|_{2 \pi}^{\frac{5}{2} \pi}\\ & = 1 - \left(- 2 \pi \right) = 1 + 2 \pi. \end{aligned} \]
The corret answer is the ratio of the integrals, i.e.,
\[ \begin{aligned} \overline{x} & = \frac{1 + 2 \pi}{1} = 1 + 2 \pi. \end{aligned} \]
\end{feedback}

\end{question}

\section*{Sample Exam Questions}

\begin{question}%%%%%[2015C.09]

An object moves in such a way that its acceleration at time \(t\) seconds is \((t^2 + 5t + 6)^{-1}\) meters per second squared. If the initial velocity of the object is \(2/3\) meters per second, what is the limit of its velocity as \(t \rightarrow \infty\)?
\begin{multiplechoice}
\choice{\(\displaystyle \ln \frac{3}{2}\) meters per second}
\choice{\(\displaystyle \ln 6\) meters per second}
\choice{\(1\) meters per second}
\choice{\(\displaystyle \ln \frac{4}{9}\) meters per second}
\choice{\(\displaystyle \ln \frac{9}{4}\) meters per second}
\choice[correct]{\(0\) meters per second}
\end{multiplechoice}

\end{question}

\begin{question}%%%%%[2017C.01]

Find the volume of the solid generated by revolving the region bounded above by \(y = \sin x\) and bounded below by \(y = 0\) for \(0 \leq x \leq \pi\) about the line \(x = \pi\).
\begin{multiplechoice}
\choice{\(\pi^2\)}
\choice[correct]{\(2\pi^2\)}
\choice{\(4\pi^2\)}
\choice{\(\displaystyle\frac{\pi^2}{2}\)}
\choice{\(\displaystyle\frac{\pi^2}{4}\)}
\choice{none of these}
\end{multiplechoice}

\end{question}

\begin{question}%%%%%[2017C.05]

Evaluate \(\displaystyle \int_1^2 x \ln (x^2 + 1) dx\).
\begin{multiplechoice}
\choice{\(0\)}
\choice{\(1\)}
\choice{\(\ln 2\)}
\choice{\(\displaystyle \frac{1}{2}\)}
\choice{\(\displaystyle \ln(2) - \frac{1}{2}\)}
\choice[correct]{none of these}
\end{multiplechoice}
\begin{feedback}
This integral can be computed via integration by parts. If we integrate \(x\) and differentiate \(\ln (x^2+1)\), we get
\[\begin{aligned}
\int_1^2 x \ln (x^2+1) dx & = \left. \frac{x^2}{2} \ln (x^2+1) \right|_{1}^2 - \int_1^2 \frac{x^2}{2} \frac{2x}{x^2+1} dx \\
& = 2 \ln 5 - \frac{1}{2} \ln 2 - \int_1^2 \frac{x^3}{x^2+1} dx.
\end{aligned}\]
The latter integral can be simplified using polynomial long division: \(\displaystyle \frac{x^3}{x^2+1} = x - \frac{x}{x^2+1}\).
Therefore
\[\begin{aligned}
\int_1^2 x \ln (x^2+1) dx & = 2 \ln 5 - \frac{1}{2} \ln 2 - \int_1^2 x dx + \int_1^2 \frac{x}{x^2+1} dx \\
& = 2 \ln 5 - \frac{1}{2} \ln 2 - \left. \frac{x^2}{2} \right|_1^2 + \left. \frac{1}{2} \ln (x^2+1) \right|_1^2 \\
& = 2 \ln 5 - \frac{1}{2} \ln 2 - 2 + \frac{1}{2} + \frac{1}{2} \ln 5 - \frac{1}{2} \ln 2 \\
& = \frac{5}{2} \ln 5 - \frac{2}{2} \ln 4 - \frac{3}{2} - \frac{3}{2} = \ln \left( \frac{5^5}{4} \right) - \frac{3}{2}.
\end{aligned}\]
\end{feedback}

\end{question}

\end{document}


\section{Sets}

\subsection{Equivalence relations}

\begin{defn}\label{defn:eq_rel}
Let $R:A\to (A\to\prop)$ be a binary relation valued in the propositions. We say that $R$ is an \define{($0$-)equivalence relation}\index{equivalence relation|textbf}\index{0-equivalence relation|see {equivalence relation}} if $R$ comes equipped with
\begin{align*}
\rho & : \prd{x:A}R(x,x) \\
\sigma & : \prd{x,y:A} R(x,y)\to R(y,x) \\
\tau & : \prd{x,y,z:A} R(x,y)\to (R(y,z)\to R(x,z)).
\end{align*}
Given an equivalence relation $R:A\to (A\to\prop)$, the \define{equivalence class}\index{equivalence class|textbf} $[x]_R$ of $x:A$ is defined to be
\begin{equation*}
[x]_R\defeq R(x).
\end{equation*}
\end{defn}

\begin{defn}
Let $R:A\to (A\to\prop)$ be a $0$-equivalence relation.
We define for any $x,y:A$ a map\index{class_eq@{$\mathsf{class\usc{}eq}$}|textbf}
\begin{equation*}
\mathsf{class\usc{}eq}:R(x,y)\to ([x]_R=[y]_R).
\end{equation*}
\end{defn}

\begin{proof}[Construction.]
Let $r:R(x,y)$. By function extensionality, the identity type $R(x)=R(y)$ is equivalent to the type
\begin{equation*}
\prd{z:A}R(x,z)=R(y,z).
\end{equation*}
Let $z:A$. By the univalence axiom, the type $R(x,z)=R(y,z)$ is equivalent to the type
\begin{equation*}
\eqv{R(x,z)}{R(y,z)}.
\end{equation*}
We have the map $\tau_{y,x,z}(\sigma(r)):R(x,z)\to R(y,z)$. Since this is a map between propositions, we only have to construct a map in the converse direction to show that it is an equivalence. The map in the converse direction is just $\tau_{x,y,z}(r):R(y,z)\to R(x,z)$.
\end{proof}

\begin{thm}\label{thm:equivalence_classes}
Let $R:A\to (A\to\prop)$ be a $0$-equivalence relation.
Then for any $x,y:A$ the map
\begin{equation*}
\mathsf{class\usc{}eq} : R(x,y)\to ([x]_R=[y]_R)
\end{equation*}
is an equivalence.
\end{thm}

\begin{proof}
By the 3-for-2 property of equivalences, it suffices to show that the map
\begin{equation*}
\lam{r}{z}\tau_{y,x,z}(\sigma(r)) : R(x,y)\to \prd{z:A} \eqv{R(x,z)}{R(y,z)}
\end{equation*}
is an equivalence. Since this is a map between propositions, it suffices to construct a map of type
\begin{equation*}
\Big(\prd{z:A} \eqv{R(x,z)}{R(y,z)}\Big)\to R(x,y).
\end{equation*}
This map is simply $\lam{f} \sigma_{y,x}(f_x(\rho(x)))$.
\end{proof}

\begin{rmk}
By \cref{thm:equivalence_classes} we can begin to think of the \emph{quotient}\index{quotient} $A/R$ of a type $A$ by an equivalence relation $R$. Classically, the quotient is described as the set of equivalence classes, and \cref{thm:equivalence_classes} establishes that two equivalence classes $[x]_R$ and $[y]_R$ are equal precisely when $x$ and $y$ are related by $R$.

However, the type $A\to\prop$ may contain many more terms than just the $R$-equivalence classes. Therefore we are facing the task of finding a type theoretic description of the smallest subtype of $A\to\prop$ containing the equivalence classes.
Another to think about this is as the \emph{image}\index{image} of $R$ in $A\to \prop$.
The construction of the (homotopy) image of a map can be done with \emph{higher inductive types}\index{higher inductive type}, which we will do in \cref{chap:image}.
\end{rmk}

The notion of $0$-equivalence relation which we defined in \cref{defn:eq_rel} fits in a hierarchy of `$n$-equivalence relations'\index{n-equivalence relation@{$n$-equivalence relation}}, the study of which is a research topic on its own. However, we already know an example of a relation that should count as an `$\infty$-equivalence relation'\index{infinity-equivalence relation@{$\infty$-equivalence relation}}: the identity type. Analogous to \cref{thm:equivalence_classes}, the following theorem shows that the canonical map
\begin{equation*}
(x=y)\to (\idtypevar{A}(x)=\idtypevar{A}(y))
\end{equation*}
is an equivalence, for any $x,y:A$. In other words, $\idtypevar{A}(x)$ can be thought of as the equivalence class of $x$ with respect to the relation $\idtypevar{A}$.

\begin{thm}
Assuming the univalence axiom on $\UU$, the map
\begin{equation*}
\idtypevar{A}:A\to (A\to\UU)
\end{equation*}
is an embedding, for any type $A:\UU$.\index{identity type!is an embedding|textit}
\end{thm}

\begin{proof}
Let $a:A$. By function extensionality it suffices to show that the canonical map
\begin{equation*}
(a=b)\to \idtypevar{A}(a)\htpy\idtypevar{A}(b)
\end{equation*}
that sends $\refl{a}$ to $\lam{x}\refl{(a=x)}$ is an equivalence for every $b:A$, and by univalence it therefore suffices to show that the canonical map
\begin{equation*}
(a=b)\to \prd{x:A}\eqv{(a=x)}{(b=x)}
\end{equation*}
that sends $\refl{a}$ to $\lam{x}\idfunc[(a=x)]$ is an equivalence for every $b:B$. To do this we employ the type theoretic Yoneda lemma, \cref{thm:yoneda}.

By the type theoretic Yoneda lemma\index{Yoneda lemma} we have an equivalence
\begin{equation*}
\Big(\prd{x:A} (b=x)\to (a=x)\Big)\to (a=b)
\end{equation*}
given by $\lam{f} f(b,\refl{b})$, for every $b:A$. Note that any family of maps $\prd{x:A}(b=x)\to (a=x)$ induces an equivalence of total spaces by \cref{ex:contr_equiv}, since their total spaces are are both contractible by \cref{cor:contr_path}. It follows that we have an equivalence
\begin{equation*}
\varphi_b:\Big(\prd{x:A} \eqv{(b=x)}{(a=x)}\Big)\to (a=b)
\end{equation*}
given by $\lam{f} f(b,\refl{b})$, for every $b:A$.

Note that $\varphi_a(\lam{x}\idfunc[(a=x)])\jdeq\refl{a}$. Therefore it follows by another application of \cref{thm:yoneda} that the unique family of maps
\begin{equation*}
\alpha_b:(a=b)\to \Big(\prd{x:A} \eqv{(b=x)}{(a=x)}\Big)
\end{equation*}
that satisfies $\alpha_a(\refl{a})=\lam{x}\idfunc[(a=x)]$ is a family of sections of $\varphi$.
It follows that $\alpha$ is a family of equivalences. Now the proof is completed by reverting the direction of the family of equivalences in the codomain.
\end{proof}


\section*{Exercises}

\begin{ex}
  For each of the following vector functions $T:\R^n\to\R^n$, show
  that $T$ is a linear transformation and find the corresponding
  matrix $A$ such that $T(\vect{x}) = A\vect{x}$.
  \begin{enumerate}
  \item $T$ multiplies the $j\th$ component of $\vect{v}$ by a
    non-zero number $b$.
  \item $T$ adds $b$ times the
    $j\th$ component of $\vect{v}$ to the $i\th$ component.
  \item $T$ switches the $i\th$ and $j\th$ components.
  \end{enumerate}
  \begin{sol}
    \begin{enumerate}
    \item The matrix of $T$ is the elementary matrix that is like the
      identity matrix, except that the $(j,j)$-entry is $b$.
    \item The matrix of $T$ is the elementary matrix that is like the
      identity matrix, except that the $(i,j)$-entry is $b$.
    \item The matrix of $T$ is the elementary matrix that switches the
      $i\th$ and the $j\th$ rows.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}\label{ex:linear-transformation-inconvenient}
  Assume that $T:\R^n\rightarrow \R^m$ is a linear transformation,
  and that $\vect{u}_1,\ldots,\vect{u}_n$ is a basis of $\R^n$. For
  all $i=1,\ldots,n$, let $\vect{v}_i = T(\vect{u}_i)$. Let $A$ be the
  matrix that has $\vect{u}_1,\ldots,\vect{u}_n$ as its columns, and
  let $B$ be the matrix that has $\vect{v}_1,\ldots,\vect{v}_n$ as its
  columns. Show that $A$ is invertible and the matrix of $T$ is
  $BA^{-1}$.
  \begin{sol}
    Since $\vect{u}_1,\ldots,\vect{u}_n$ is a basis of $\R^n$, we know
    that $A$ is invertible by  Proposition~\ref{prop:invertible-matrices}.
    For each $i=1,\ldots,n$, since $A\vect{e}_i = \vect{u}_i$, we have
    $A^{-1}\vect{u}_i=\vect{e}_i$, and therefore
    \begin{equation*}
      BA^{-1}\vect{u}_i = B\vect{e}_i = \vect{v}_i = T(\vect{u}_i).
    \end{equation*}
    Now let $\vect{w}$ be some arbitrary vector in $\R^n$. Since
    $\vect{u}_1,\ldots,\vect{u}_n$ is a basis, there exists
    $a_1,\ldots,a_n$ such that $\vect{w} =
    a_1\vect{u}_1+\ldots+a_n\vect{u}_n$. Then
    \begin{eqnarray*}
      BA^{-1}\vect{w}
      &=& BA^{-1}(a_1\vect{u}_1+\ldots+a_n\vect{u}_n) \\
      &=& a_1(BA^{-1}\vect{u}_1) + \ldots + a_n(BA^{-1}\vect{u}_n) \\
      &=& a_1T(\vect{u}_1) + \ldots + a_nT(\vect{u}_n) \\
      &=& T(a_1\vect{u}_1+\ldots+a_n\vect{u}_n) ~=~ T(\vect{w}).
    \end{eqnarray*}
    Therefore, $BA^{-1}$ is the matrix of $T$.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose $T$ is a linear transformation such that
  \begin{equation*}
    T\begin{mymatrix}{r} 1 \\ 2 \\ -6 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r} 5 \\ 1 \\ 3 \end{mymatrix},
    \quad
    T\begin{mymatrix}{r} -1 \\ -1 \\ 5 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r} 1 \\ 1 \\ 5 \end{mymatrix},
    \quad\mbox{and}\quad
    T\begin{mymatrix}{r} 0 \\ -1 \\ 2 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r} 5 \\ 3 \\ -2 \end{mymatrix}.
  \end{equation*}
  Find the matrix of $T$. Hint: use
  Exercise~\ref{ex:linear-transformation-inconvenient}.
  \begin{sol}
    \begin{equation*}
      \begin{mymatrix}{rrr}
        5 & 1 & 5 \\
        1 & 1 & 3 \\
        3 & 5 & -2
      \end{mymatrix}
      \begin{mymatrix}{rrr}
        1 & -1 & 0 \\
        2 & -1 & -1 \\
        -6 & 5 & 2
      \end{mymatrix}^{-1}
      = \begin{mymatrix}{rrr}
        5 & 1 & 5 \\
        1 & 1 & 3 \\
        3 & 5 & -2
      \end{mymatrix}
      \begin{mymatrix}{rrr}
        3 & 2 & 1 \\
        2 & 2 & 1 \\
        4 & 1 & 1
      \end{mymatrix}
      = \begin{mymatrix}{rrr}
        37 & 17 & 11 \\
        17 & 7 & 5 \\
        11 & 14 & 6
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Suppose $T$ is a linear transformation such that
  \begin{equation*}
    T\begin{mymatrix}{r} 1 \\ 1 \\ -8 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r} 1 \\ 3 \\ 1 \end{mymatrix},
    \quad
    T\begin{mymatrix}{r} -1 \\ 0 \\ 6 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r} 2 \\ 4 \\ 1 \end{mymatrix},
    \quad\mbox{and}\quad
    T\begin{mymatrix}{r} 0 \\ -1 \\ 3 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r} 6 \\ 1 \\ -1 \end{mymatrix}.
  \end{equation*}
  Find the matrix of $T$. Hint: use
  Exercise~\ref{ex:linear-transformation-inconvenient}.
  \begin{sol}
    \begin{equation*}
      \begin{mymatrix}{rrr}
        1 & 2 & 6 \\
        3 & 4 & 1 \\
        1 & 1 & -1
      \end{mymatrix}
      \begin{mymatrix}{ccc}
        1 & -1 & 0 \\
        1 & 0 & -1 \\
        -8 & 6 & 3
      \end{mymatrix}^{-1}
      = \begin{mymatrix}{rrr}
        1 & 2 & 6 \\
        3 & 4 & 1 \\
        1 & 1 & -1
      \end{mymatrix}
      \begin{mymatrix}{ccc}
        6 & 3 & 1 \\
        5 & 3 & 1 \\
        6 & 2 & 1
      \end{mymatrix}
      = \begin{mymatrix}{ccc}
        52 & 21 & 9 \\
        44 & 23 & 8 \\
        5 & 4 & 1
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the following linear transformations
  $T:\R^3\rightarrow \R^2$.  For each, determine the matrix $A$
  such that $T(\vect{x})=A\vect{x}$.
  \begin{enumerate}
  \item $T\begin{mymatrix}{c}
      x \\
      y \\
      z
    \end{mymatrix} =\begin{mymatrix}{c}
      x+2y+3z \\
      2y-3x+z
    \end{mymatrix}$

  \item $T\begin{mymatrix}{c}
      x \\
      y \\
      z
    \end{mymatrix} =\begin{mymatrix}{c}
      7x+2y+z \\
      3x-11y+2z
    \end{mymatrix}$

  \item $T\begin{mymatrix}{c}
      x \\
      y \\
      z
    \end{mymatrix} =\begin{mymatrix}{c}
      3x+2y+z \\
      x+2y+6z
    \end{mymatrix}$

  \item $T\begin{mymatrix}{c}
      x \\
      y \\
      z
    \end{mymatrix} =\begin{mymatrix}{c}
      2y-5x+z \\
      x+y+z
    \end{mymatrix}$
  \end{enumerate}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Find the matrix for
  $T(\vect{w}) = \proj_{\vect{v}}(\vect{w})$, where
  $\vect{v}=\mat{1, -2, 3}^T$.
  \begin{sol}
    Recall that the desired matrix has $i\th$ column equal to
    $\displaystyle\proj_{\vect{u}}(\vect{e}_i) =
    \frac{\vect{u}\dotprod\vect{e}_i}{\norm{\vect{u}}^2}\vect{u}$.
    Therefore, the matrix is
    \begin{equation*}
      \frac{1}{14}\begin{mymatrix}{rrr}
        1 & -2 & 3 \\
        -2 & 4 & -6 \\
        3 & -6 & 9
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Find the matrix for
  $T(\vect{w}) = \proj_{\vect{v}}(\vect{w})$, where
  $\vect{v}=\mat{1,5,3}^T$.
  \begin{sol}
    \begin{equation*}
      \frac{1}{35}\begin{mymatrix}{rrr}
        1 & 5 & 3 \\
        5 & 25 & 15 \\
        3 & 15 & 9
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}


\chapter{Background}
\label{sec:background}

This chapter presents the supporting concepts and methods that will be used throughout this work.

\section{Submodular Set Functions}

Functions over sets are of great interest and can be applied to diverse settings. In particular, submodular set functions have been utilized in multiple domains, such as document summarization and information gathering \citep{krause14submodular}.

A set function is defined on the powerset $2^{V}$ of a ground set $V$, i.e. $F:2^{V} \rightarrow \mathbb{R}$ and it is submodular if it exhibits a "diminishing returns" property , namely if it satisfies:

\begin{equation}
  \label{eq:submod-condition}
  \forall S,T \subseteq V : S \subseteq T, i \notin T \mid F(S \cup i) - F(S) \geq F(T \cup i) - F(T)
\end{equation}

Intuitively this indicates that adding an element to a smaller set yields a larger gain, i.e. in terms of the set function $F$ value, than adding it to a larger one. This is a natural property in the context of summarization where adding more information to a large summary is less effective than adding it to a smaller one. This property also makes them good candidates for modeling the concept of diversity \citep{tschiatschek16learning}.

Analogous to the use of submodular functions to model diversity, supermodular functions can be used to model complementarity or coherence between elements.

A set function is supermodular if it satisfies the condition in \eqref{eq:submod-condition} with the inequality sign reversed, i.e. a set function $F(S)$ is supermodular iff $-F(S)$ is submodular. Supermodularity is used extensively in economics to model complementarity between strategies in games \citep{amir2005supermodularity}.

Finally, if a function $F$ is both submodular and supermodular, i.e. it satisfies condition \eqref{eq:submod-condition} with equality, then it is said to be a modular function. These are the simplest examples of submodular or supermodular functions and are akin to linear functions but in a discrete domain.

\section{Probabilistic Log-sub/supermodular Models}

Set functions may also express probability distributions, in which case they are known as Probability Set Functions. Formally, a set function $F$ is a probability set function if it satisfies that  $\forall S \subseteq V, P: 2^V \rightarrow \mathbb{R} \mid 0 \leq P(S) \leq 1$ and $\sum_{S \subseteq V} P(S) = 1$ \citep{bruno99sets}.

In this work we are interested in a particular class of probability set functions, namely probabilistic log-submodular and log-supermodular models, which are probabilities $P(S)$ of the form \citep{djolonga14variational}:

\begin{equation}
  P(S) = \frac{1}{Z}\exp(F(S)),
\end{equation}

where $F(S)$ is a submodular or supermodular function, respectively. \citet{djolonga14variational, djolonga15scalable} indicate that these models encompass many practical probabilistic models such as repulsive Ising models, Determinantal Point Processes (DPPs) and binary pairwise Markov Random Fields (MRFs).

The normalization constant $Z$ is also known as the \textit{partition function} and it is necessary to fully determine the normalized model and compute quantities such as marginal probabilities, however its exact computation is known to be \#P-complete in general \citep{jerrum1990}. This makes it impossible in practice to perform such computations except for special cases of the model.

\section{Facility Location Diversity Model (FLID)}

Every modular function can be written as a sum of individual weights for each element $i \in V$ assuming a normalization such that $F(\emptyset) = 0$, i.e.

\begin{equation}
  F(S) = \sum_{i \in S} u(i),
\end{equation}

where $u(i)$ is some function $u: V \rightarrow \mathbb{R}$. For convenience, denote $\mathbf{u}$ as the vector of modular weights where $u_{i} = u(i)$. Therefore any log-modular function can be written as:

\begin{equation}
  \label{eq:modular}
  P(S) \propto \exp{\left(\sum_{i \in S}u_{i}\right)}= \prod_{i \in S} \exp{\left(u_{i}\right)}
\end{equation}

This is the simplest log-submodular probability function and is the modular part for the FLID model proposed by \citet{tschiatschek16learning}. In this model, $\mathbf{u}$ can be thought of as modeling the relevance or utility of each element, for example in the context of spatial summarization this utility could be proportional to the popularity of a location or to how many times it has been photographed.

However, these utilities alone can not capture interactions between the elements in a set. To address this, \citet{tschiatschek16learning} proposed an additional term that models set diversity, its objective is to identify redundant elements and assign lower probabilities to sets that contain them.

This diversity promoting term is based on representing each element $i$ with an $L$-dimensional vector $\mathbf{w}^{b} \in \mathbb{R}^{L}_{\geq 0}$, where each dimension $d$, $1 \leq d \leq L$ captures a concept related to set diversity and $w^{b}_{i,d}$ quantifies the relevance of each element $i$ for each concept $d$. Hereby, define $\mathbf{W}^{b} \in \mathbb{R}^{|V| \times L}_{\geq 0}$ as the matrix where the $i$-th row is the representation $\mathbf{w}^{b}_{i}$ of element $i$.

For each dimension $d$, the diversity of a set $S$ is quantified by $\max_{i \in S}{w^{b}_{i,d}} - \sum_{i \in S}{w^{b}_{i,d}}$. This assigns a negative value for sets that contain more than one element with nonzero weight $w^{b}_{\cdot,d}$ in that dimension \citep{tschiatschek16learning}. Equation \eqref{eq:diversity} shows the complete diversity term, this sums over all $L$ dimensions to account the diversity in each concept.

\begin{equation}
  \label{eq:diversity}
  \mathrm{div}(S) = \sum_{d=1}^{L}\left(\max_{i \in S}{w^{b}_{i, d}} - \sum_{i \in S}{w^{b}_{i,d}}\right)
\end{equation}

Finally, the complete FLID model proposed by \citet{tschiatschek16learning} combines the modular and diversity terms and is given by:

\begin{equation}
  \tag{FLID}
  P(S) = \frac{1}{Z}\exp{\left(\sum_{i \in S}u_{i} + \sum_{d=1}^{L}\left(\max_{i \in S}{w^{b}_{i, d}} - \sum_{i \in S}{w^{b}_{i,d}}\right)\right)}
  \label{eq:flid}
\end{equation}

As mentioned before, the computation of the partition function is generally intractable for log-submodular models. However, for FLID this can be computed in time $\mathcal{O}(|V|^{L+1})$ which is polynomial in the size of the ground set and significantly better than the cost of enumerating the powerset of $V$, i.e. $\mathcal{O}(2^{|V|})$ \citep{tschiatschek16learning}. Nevertheless, it is worth noting that this complexity is exponential in $L$ which means that the partition function computation is only pratical for a limited range of FLID models, namely those with $L \ll |V|$.

\subsection{Example: Two Landmarks}
\label{sec:flid-toy}

In order to better illustrate the model, consider a town with 3 popular locations: A town hall ($h$), a statue ($s$) and a fountain ($f$). Assume that historic data shows that visitors only take photos at either the town hall and the statue, or at the town hall and the fountain. This can be modeled with FLID by introducing a latent dimension representing some concept that is present in both the statue and the fountain but not in the town hall, e.g. "is not a building".

Concretely, let $V = \{h, s, f\}$ and $\mathbf{u} = \left(2, 2, 2\right)^{\intercal}$, indicating that all locations are equally popular. A suitable diversity weight matrix would then be $\mathbf{W}^{b} = \left(0, 20, 20\right)^{\intercal}$. Table \ref{tab:flid-toy-probs} shows the resulting probabilities of the subsets, accurately representing the example scenario.

\begin{table}
  \centering
  \caption{FLID probability distribution for the scenario in Example \ref{sec:flid-toy}.}
  \begin{tabular}{@{}ll@{}}
    \toprule
    $S$ & $P(S)$  \\
    \midrule
    $\{h,s\}, \{h,f\}$ & $\approx 0.41$ \\
    $\{h\}, \{s\}, \{f\}$ & $\approx 0.06$ \\
    $\{\}, \{s,f\}, \{h,s,f\}$ & $\approx 0.00$ \\
    \bottomrule
  \end{tabular}
  \label{tab:flid-toy-probs}
\end{table}

\section{Learning from Data}
\label{sec:learning}

Much of the interest in log-submodular models is concentrated on performing inference for known functions. However, \citet{tschiatschek16learning} explore how to learn such models from data, i.e. estimate the model parameters from observations with an unknown distribution assumed to be of the same family as the models.

\subsection{Learning Log-modular Models}
\label{sec:learning_modular}

A special case of log-submodular models where the parameters can be estimated efficiently using Maximum Likelihood Estimation (MLE) is the log-modular model presented in Equation \eqref{eq:modular}. For this basic model, it is possible to estimate the utilities $\mathbf{u}$ through the maximum likelihood estimator for the marginals $\hat{P}(i \in S)$. These marginals are given by:

\begin{equation}
P(i \in S) = \frac{1}{1 + \exp(-u_{i})}
\end{equation}

And its maximum likelihood estimator for a dataset $\mathcal{D}$ is:

\begin{equation}
\hat{P}(i \in S) = \frac{N(i \in S)}{|\mathcal{D}|},
\end{equation}

where $N(i \in S)$ is the number of sets containing element $i$. Therefore, the utilities can be estimated as:

\begin{equation}
\label{eq:modular-mle}
u_{i} = -\log{\left(\frac{1}{\hat{P}(i \in S)} - 1\right)}
\end{equation}

This model also allows computing the partition function in closed form, i.e.:

\begin{equation}
\label{eq:modular-z}
Z = \prod_{i \in S}(1 + \exp(u_{i}))
\end{equation}

Because these quantities can be computed in closed form, it is possible to efficiently estimate the parameters of a log-modular model and sample from it. This is useful for producing noise samples and constructing baselines for the experiments.

Even though MLE is the commonly used method for the task of parameter estimation from data, it only works efficiently for normalized models. This makes it intractable for the general class of log-submodular models, and also a wide range of FLID models because the computation of $Z$ is exponential on $L$. \citet{Gutmann12NCE} propose an alternative method for parameter estimation in unnormalized models known as Noise Contrastive Estimation (NCE) and this is the method used by \citet{tschiatschek16learning} to learn the FLID model.

\subsection{NCE Learning}

The idea behind NCE is to transform the unsupervised learning task of estimating a probability density from data into a supervised classification task. In order to do this, the observed data $\mathcal{D}$, assumed to be drawn from an unknown distribution $P_{d}$, is compared to an artificially generated set of noise samples $\mathcal{N}$ drawn from a known distribution $P_{n}$ that can be efficiently normalized. The classification task is to maximize the likelihood of correctly discriminating each sample as either observed data or artificial noise.

Formally, denote $\mathcal{A}$ as the complete set of labeled samples, i.e. $\mathcal{A} = \{(S,Y_{s}) : S \in \mathcal{D} \cup \mathcal{N}\}$ where $Y_{s} = 1$ for $S \in \mathcal{D}$ and $Y_{s} = 0$ for $S \in \mathcal{N}$. Additionally, let $\nu$ be the noise-to-data ratio, i.e. $\nu = \nicefrac{|\mathcal{N}|}{|\mathcal{D}|}$.

The goal is to estimate the posterior probabilities $P(Y_{s} = 1 \mid S;\boldsymbol{\theta})$ and $P(Y_{s} = 0 \mid S;\boldsymbol{\theta})$, in order to discriminate noise from data samples. These probabilities are given by equations \eqref{eq:posterior-1} and \eqref{eq:posterior-2}.

\begin{align}
  P(Y_{s} &= 1 \mid S;\boldsymbol{\theta}) = \frac{\hat{P}_{d}(S;\boldsymbol{\theta})}{\hat{P}_{d}(S;\boldsymbol{\theta}) + \nu P_{n}(S)}
  \label{eq:posterior-1} \\
  P(Y_{s} &= 0 \mid S;\boldsymbol{\theta}) = \frac{\nu P_{n}(S)}{\hat{P}_{d}(S;\boldsymbol{\theta}) + \nu P_{n}(S)}
  \label{eq:posterior-2}
\end{align}

It is worth noting that $\hat{P}_{d}$ is used instead of $P_{d}$, because the real density is not known. As indicated by \citet{Gutmann12NCE}, $\hat{P}_{d}$ can be an unnormalized distribution for NCE where the partition function $Z$ is included in the set of parameters $\boldsymbol{\theta}$ as $\hat{Z}$, hence resulting in an approximately normalized distribution after the optimization.

In order to obtain these posterior probabilities, the following conditional log-likelihood objective is maximized \citep{Gutmann12NCE}:

\begin{equation}
\label{eq:log-likelihood}
g(\boldsymbol{\theta}) = \sum_{S \in \mathcal{D}}{\log{P(Y_{s} = 1 \mid S;\boldsymbol{\theta})}} + \sum_{S \in \mathcal{N}}{\log{P(Y_{s} = 0 \mid S;\boldsymbol{\theta})}}
\end{equation}

This maximization can be performed using a gradient-based optimization method such as Stochastic Gradient Descent (SGD).

\subsubsection{Theoretical Considerations}
\label{sec:nce-theoretical}

A couple of theoretical conditions are necessary for obtaining asymptotic convergence with NCE \citep{Gutmann12NCE}:

\begin{enumerate}
  \item The parameterized probability function $\hat{P}_{d}(S;\theta)$ must be of the same family as the real distribution $P_{d}$, i.e. $\exists \boldsymbol{\theta}^{*} \mid \hat{P}_{d}(S;\boldsymbol{\theta}^{*}) = P_{d}$.
  \item The noise distribution $P_{n}$ is nonzero whenever $P_{d}$ is nonzero.
\end{enumerate}

\subsubsection{Practical Considerations}
\label{sec:nce-practical}

Additionally, the following statements must be considered when choosing the noise distribution $P_{n}$ \citep{Gutmann12NCE}:

\begin{enumerate}
  \item A distribution that can be sampled easily.
  \item Noise that is as similar as possible to the data, otherwise the classification problem could be too easy.
  \item A noise sample as large as possible.
\end{enumerate}

\subsection{Learning FLID via NCE and SGD}

Stochastic Gradient Descent was used by \citet{tschiatschek16learning} to learn the FLID model through NCE. SGD is a gradient-based method that has been proven effective in large-scale learning tasks due to its efficiency when the computation time is a limiting factor \citep{Bottou2010, Zhang2004}, hence making it appropriate for the scale of data sourced from the internet, such as public user photos in Flickr.

In each iteration, the gradient $\nabla \log{P(Y_{s} = y \mid S;\boldsymbol{\theta})}$ must be computed. For FLID, the parameter vector $\boldsymbol{\theta}$ is composed by $\mathbf{u}, \mathbf{W}^{b}$ and $\hat{Z}$, and the corresponding gradients are given by,

\begin{align}
  \label{eq:gradient-flid-1}
  \nabla \log{P(Y_{s} = y \mid S;\boldsymbol{\theta})} &= \left(y - \frac{1}{1 + \nu \frac{P_{n}(S)}{\hat{P}_{d}(S;\boldsymbol{\theta})}}\right) \nabla \log{\hat{P}_{d}(S;\boldsymbol{\theta})} \\
  \label{eq:gradient-flid-2}
  \hat{P}_{d}(S;\boldsymbol{\theta}) &= \frac{1}{\hat{Z}}P(S; \mathbf{u}, \mathbf{W}^{b}) \\
  \left(\nabla_{\mathbf{u}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{u}, \mathbf{W}^{b})}\right)_{i} &= \begin{cases}
    1 &  \text{if}\ i \in S \\
    0 & \text{otherwise}
  \end{cases} \\
  \label{eq:gradient-flid-3}
  \left(\nabla_{\mathbf{W}^{b}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{u}, \mathbf{W}^{b})}\right)_{i,d} &= \begin{cases}
    -1 & \text{if}\ i \in S\ \text{and}\ i \neq \argmax_{j \in S}{w^{b}_{j,d}} \\
    0 & \text{otherwise}
  \end{cases} \\
  \label{eq:gradient-flid-4}
  \nabla_{\hat{Z}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{u}, \mathbf{W}^{b})}&= \frac{-1}{\hat{Z}},
\end{align}

where $P(S;\mathbf{u}, \mathbf{W}^{b})$ is the unnormalized \eqref{eq:flid} equation, $\left(\nabla_{\mathbf{u}}\cdot \right)_{i}$ represents the $i$-th entry of the gradient with respect to $\mathbf{u}$ and $\left(\nabla_{\mathbf{W}^{b}}\cdot\right)_{i,d}$ represents the $(i,d)$-th entry of the gradient with respect to $\mathbf{W}^{b}$ \citep{tschiatschek16learning}.

After the gradient is computed, the parameters are updated according to the following equation,

\begin{equation}
  \theta^{t+1} = \theta^{t} + \eta(t) \nabla g(\theta),
\end{equation}

where $\eta(t)$ is the learning rate.

\subsubsection{Computational Complexity}

As described by \citet{tschiatschek16learning}, the running time requirement for calculating the sub-gradient for FLID is $\mathcal{O}(L|S|)$, therefore a complete pass over data and noise samples is $\mathcal{O}(|\mathcal{D}\cup\mathcal{N}|\kappa L)$ where $\kappa = \max_{S \in \mathcal{D}\cup\mathcal{N}}{|S|}$, i.e. the largest subsed in $\mathcal{D} \cup \mathcal{N}$. This shows that learning is efficient as it is only linear on the amount of data and noise.

\subsection{AdaGrad}
\label{sec:adagrad}

A commonly used learning rate function for SGD is $\eta(t) = \eta_{0}t^{-p}$, which offers the best theoretical convergence speed for convex functions under certain conditions but often also in practice \citep{bottou2012stochastic}, however it can lead to poor performance due to slow rates of convergence to the solution \citep{darken1992towards}. By contrast, choosing a larger $\eta_{0}$ may not lead to better results due to the instability in the parameters for small $t$ \citep{Darken1990}.

This behavior was observed during the experiments with real data presented in later chapters, therefore an alternative for the learning rate was sought. In particular, Adaptive Gradient (AdaGrad) was implemented.

AdaGrad was proposed by \citet{Duchi2011adagrad}, the idea of this method is to adapt the learning rate for each parameter in $\boldsymbol{\theta}$ individually in a way that frequently updated parameters have slower learning rates while infrequent parameters have larger ones. The intuition is that an update to a rare parameter is more informative than one to a parameter that is frequently updated.

Concretely, with AdaGrad the update step for each parameter $\gamma$ in $\boldsymbol{\theta}$ is given by,

\begin{equation}
\theta^{\tau + 1}_{\gamma} \leftarrow \theta^{\tau}_{\gamma} - \frac{\eta}{\sqrt{G_{\gamma}}} \nabla g(\theta)^{\tau}_{\gamma},
\end{equation}

where $\mathbf{G}^{\tau}$ is the vector of accumulated gradients at time $\tau$ and each component $G^{\tau}_\gamma$ is given by,

\begin{equation}
G^{\tau}_{\gamma} = \sum_{t=1}^{\tau} \left(\nabla g(\theta)^{t}_{\gamma}\right)^{2},
\label{eq:adagrad-g}
\end{equation}

where $\nabla g(\theta)^{t}_{\gamma}$ denotes the gradient of the $\gamma$ parameter at time step $t$.

It is worth noting that AdaGrad does not incur in an significant increase in running time or memory requirements for the training which makes it inexpensive to include in the implementation of NCE.


\section*{WDA - Wasserstein Discriminant Analysis}

Rémi Flamary, Marco Cuturi, Nicolas Courty, Alain Rakotomamonjy
(Submitted on 29 Aug 2016 (v1), last revised 23 May 2018 (this version, v2))
Wasserstein Discriminant Analysis (WDA) is a new supervised method that can improve classification of high-dimensional data by computing a suitable linear map onto a lower dimensional subspace. Following the blueprint of classical Linear Discriminant Analysis (LDA), WDA selects the projection matrix that maximizes the ratio of two quantities: the dispersion of projected points coming from different classes, divided by the dispersion of projected points coming from the same class. To quantify dispersion, WDA uses regularized Wasserstein distances, rather than cross-variance measures which have been usually considered, notably in LDA. Thanks to the underlying principles of optimal transport, WDA is able to capture both global (at distribution scale) and local (at samples scale) interactions between classes. Regularized Wasserstein distances can be computed using the Sinkhorn matrix scaling algorithm; We show that the optimization of WDA can be tackled using automatic differentiation of Sinkhorn iterations. Numerical experiments show promising results both in terms of prediction and visualization on toy examples and real life datasets such as MNIST and on deep features obtained from a subset of the Caltech dataset.


\section{Iterators} % (fold)
\label{sec:iterators}
\begin{questions}
\titledquestion{Collatz sequence} % (fold)
\label{sub:collatz_sequence}

Write a generator that generates the Collatz sequence with initial value $n$.
Use this to print out the Collatz sequence started at 103

Recall the Collatz sequence problem from last week.
A Collatz sequence is formed as follows:
We start with some number $x_0$, and we find the next number in the sequence by
\[
    x_{i+1} = \begin{cases}
        x_i / 2 & \text{ if $x_i$ is even}\\
        3x_i + 1 & \text{ if $x_i$ is odd}
    \end{cases}
\]
If $x_i = 1$, we stop iterating and have found the full sequence.

% titledquestion collatz_sequence (end)

\titledquestion{Collatz array using Numpy} % (fold)
\label{sub:collatz_array}

Use the Collatz generator you wrote in the previous exercise to generate a vector
with as ellements the Collatz sequence started at 61.

Hint: use the \texttt{np.fromiter} function.

% titledquestion collatz_array (end)

\titledquestion{Prime numbers} % (fold)
\label{sub:prime_numbers}

Write an iterator that iterates over the first $n$ prime numbers.
Use this to print out the first 10,000 primes.

% titledquestion prime_numbers (end)

\end{questions}
% section iterators (end)


\chapter{Limit and Continuity}
%================================================
%::::::::::::::::::::::::::::::::::::::::::::::::

In the sections below, the capitals with blackboard bold font, such as $\mathbb A$, denote the normed vector spaces.

\section{O Notations}
%================================================
%------------------------------------------------

\begin{definition}
	\label{def: little-o}
	Let $f: \mathbb X \to \mathbb Y: \mathbf x \mapsto f(\mathbf x)$ and $g: \mathbb X \to \mathbb S: \mathbf x \mapsto g(\mathbf x)$.

	$f$ is a \textit{little-o of $g$ as $\mathbf x \to \mathbf p$}, denoted
	$$
	f(\mathbf x) = o(g(\mathbf x)) \quad \text{as $\mathbf x \to \mathbf p$},
	$$
	iff for any $\varepsilon \in \mathbb R_{> 0}$, there exists a neighbourhood $N$ of $\mathbf p$, such that for any $\mathbf x \in N$, $\| f(\mathbf x) \|_{\mathbb Y} \le \varepsilon \| g (\mathbf x)\|_{\mathbb S}$; equivalently, that is,
	$$
	\lim_{\mathbf x \to \mathbf p} \frac{\|f(\mathbf x)\|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S}} = 0 \text{ or, equivalently, } \lim_{\mathbf x \to \mathbf p} \frac{f(\mathbf x)}{\| g(\mathbf x) \|_{\mathbb S}} = \mathbf 0_{\mathbb Y}
	$$
\end{definition}

\begin{lemma}
	\label{lm: little-o: zero limit}
	With the condition in Definition \ref{def: little-o}, suppose
	$$
	\lim_{\mathbf x \to \mathbf p} \| g(\mathbf x) \| \in \mathbb R,
	$$

	Then
	$$
	f(\mathbf x) = o(g(\mathbf x)) \quad \text{as $\mathbf x \to \mathbf p$},
	$$
	implies
	$$
	\lim_{\mathbf x \to \mathbf p} f(\mathbf x) = \mathbf 0_{\mathbb Y}.
	$$

	\begin{proof}
		Aiming for a contradiction, suppose there exists $\mathbf r \in \mathbb Y \setminus \{ \mathbf 0_{\mathbb Y} \}$, such that
		$$
		\lim_{\mathbf x \to \mathbf p} f(\mathbf x) = \mathbf r,
		$$
		then, we have
		$$
		\begin{aligned}
			\lim_{\mathbf x \to \mathbf p} \frac{\| f(\mathbf x) \|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S}} = \lim_{\mathbf x \to \mathbf p} \frac{\| \mathbf r \|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S}} > 0.
		\end{aligned}
		$$

		This contradicts the assumption.
	\end{proof}
\end{lemma}

\begin{lemma}
	\label{lm: little-o: negative}
	With the condition in Definition \ref{def: little-o}, $f$ is a little-o of $g$ iff $-f$ is a little-o of $g$.

	\begin{proof}
		$$
		\lim_{\mathbf x \to \mathbf p} \frac{\| - f(\mathbf x) \|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S}} = \lim_{\mathbf x \to \mathbf p} \frac{\| f(\mathbf x) \|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S}} = 0.
		$$
	\end{proof}
\end{lemma}

\begin{lemma}
	\label{lm: little-o: finite sum}
	Let $f_1, f_2: \mathbb X \to \mathbb Y: \mathbf x \mapsto f_1(\mathbf x), f_2(\mathbf x)$, and let $g: \mathbb X \to \mathbb S: \mathbf x \mapsto g(\mathbf x)$.

	If $f_1$ and $f_2$ are both little-o of $g$ as $\mathbf x \to \mathbf p$, i.e.,
	$$
	f_1(\mathbf x) = o_1(g(\mathbf x)) \text{ and } f_2(\mathbf x) = o_2(g(\mathbf x)) \text{ as $\mathbf x \to \mathbf p$},
	$$
	then $f_1 + f_2$ is also a little-o of $g$ as $\mathbf x \to \mathbf p$, i.e.,
	$$
	f_1(\mathbf x) + f_2(\mathbf x) = o_3(g(\mathbf x))
	$$

	\begin{proof}
		By triangle inequality,  we have
		$$
		\begin{aligned}
			\lim_{\mathbf x \to \mathbf p}\frac{\| f_1(\mathbf x) + f_2(\mathbf x) \|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S} } &\le \lim_{\mathbf x \to \mathbf p} \frac{\| f_1(\mathbf x) \|_{\mathbb Y} + \| f_2(\mathbf x) \|_{\mathbb Y}}{\| g(\mathbf x) \|_{\mathbb S}} \\
			&\le 2 \lim_{\mathbf x \to \mathbf p} \frac{\max\{ \| f_1(\mathbf x) \|_{\mathbb Y} + \| f_2(\mathbf x) \|_{\mathbb Y} \}}{\| g(\mathbf x) \|_{\mathbb S}} \\
			&= 0.
		\end{aligned}
		$$

		By Definition \ref{def: little-o}, $f_1 + f_2$ is a little-o of $g$ as $\mathbf x \to \mathbf p$.
	\end{proof}
\end{lemma}

\begin{note}
	In Lemma \ref{lm: little-o: finite sum}, consider $A$ be the set of all mappings being little-o of $g$ as $\mathbf x \to \mathbf p$, then Lemma \ref{lm: little-o: finite sum} tells that $A$ is finitely additive. That is, for any finite $B \subseteq A$,
	$$
	\sum_{o \in B} o(g(\mathbf p)) \in A.
	$$
\end{note}

\chapter{Differentiation}
%================================================
%::::::::::::::::::::::::::::::::::::::::::::::::

\section{Differentiable Mappings}
%================================================
%------------------------------------------------

\begin{definition}
	\label{def: differentiable mappings}

	Let $f: \mathbb X \to \mathbb Y$.

	$f$ is said to be \textit{differentiable at $\mathbf p \in \mathbb X$} iff there exists a linear mapping $\phi: \mathbb X \to \mathbb Y$ such that for any $\mathbf t \in \mathbb X$,
	$$
	f(\mathbf p + \mathbf t) = f(\mathbf p) + \phi(\mathbf t) + o(\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$}.
	$$
\end{definition}

\begin{lemma}
	\label{lm: uniqueness of phi}

	In Definition \ref{def: differentiable mappings}, the linear mapping $\phi$ is unique.

	\begin{proof}
		Suppose there is another linear mapping $\lambda: \mathbb X \to \mathbb Y$, such that for any $\mathbf t \in \mathbb X$,
		$$
		f(\mathbf p + \mathbf t) = f(\mathbf p) + \lambda(\mathbf t) + o_\lambda (\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$},
		$$
		then we have
		$$
		\begin{aligned}
			& \phi(\mathbf{\hat t}) - \lambda(\mathbf{\hat t}) = \lim_{\mathbf t \to \mathbf 0_{\mathbb X}}\frac{\phi(\mathbf t) - \lambda(\mathbf t)}{\| \mathbf t \|_{\mathbb X}} = \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{o(\mathbf t) - o_\lambda(\mathbf t)}{\| \mathbf t\|_{\mathbb X}}
		\end{aligned}
		$$

		By Lemma \ref{lm: little-o: negative}, $-o_{\lambda}(\mathbf t)$ is also a little-o of $\mathbf t$ as $\mathbf t \to \mathbf 0_{\mathbb X}$, thus, by Lemma \ref{lm: little-o: finite sum}, $o(\mathbf t) - o_\lambda(\mathbf t)$ is a little-o of $\mathbf t$ as $\mathbf t \to \mathbf 0_{\mathbb X}$. By Definition \ref{def: little-o},
		$$
		\phi(\mathbf{\hat t}) - \lambda(\mathbf{\hat t}) = \mathbf 0_{\mathbb X}.
		$$

		As $\mathbf t$ is arbitrarily given, $\mathbf{\hat t}$ defines all possible directions in $\mathbb X$. Thus,
		$$
		\phi = \lambda.
		$$
	\end{proof}
\end{lemma}

\begin{lemma}
	With the condition in Definition \ref{def: differentiable mappings}, $f$ is differentiable at $\mathbf p \in \mathbb X$ iff there exists a linear mapping $\phi: \mathbb X \to \mathbb Y$ such that for any $\mathbf t \in \mathbb X$,
	\begin{equation}
		\tag{i}
		\lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{\| f(\mathbf p + \mathbf t) - f(\mathbf p) - \phi(\mathbf t) \|_{\mathbb Y}}{\| \mathbf t \|_{\mathbb X}} = 0.
	\end{equation}
	Equivalently, that is,
	\begin{equation}
		\tag{i'}
		\lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{ f(\mathbf p + \mathbf t) - f(\mathbf p) - \phi(\mathbf t) }{\| \mathbf t \|_{\mathbb X}} = \mathbf 0_{\mathbb Y}.
	\end{equation}

	\begin{proof}
		This can be proved from both sides. Consider the equations in this proposition and in Definition \ref{def: differentiable mappings}. We observe that the equation in Definition \ref{def: differentiable mappings} holds iff
		$$
		\begin{aligned}
			& \frac{f(\mathbf p + \mathbf t) - f(\mathbf p) - \phi(\mathbf t)}{\| \mathbf t \|_{\mathbb X}} = \frac{o(\mathbf t)}{\| \mathbf t \|_{\mathbb X}}\\
			\iff & \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{f(\mathbf p + \mathbf t) - f(\mathbf p) - \phi(\mathbf t)}{\| \mathbf t \|_{\mathbb X}} = \mathbf 0_{\mathbb X} & \text{((i') is proved)} \\
			\iff & \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{\| f(\mathbf p + \mathbf t) - f(\mathbf p) - \phi(\mathbf t)\|_{\mathbb Y}}{\| \mathbf t \|_{\mathbb X}} = 0. & \text{((i) is proved)}
		\end{aligned}
		$$
	\end{proof}
\end{lemma}

\begin{lemma}
	With the condition in Definition \ref{def: differentiable mappings}, if $f$ is differentiable at $\mathbf p$, then $f$ is continuous at $\mathbf p$.

	\begin{proof}
		As $f$ is differentiable at $\mathbf p$, there exists a linear mapping $\phi: \mathbb X \to \mathbb Y$, such that for any $\mathbf t \in \mathbb X$,
		$$
		f(\mathbf p + \mathbf t) = f(\mathbf p) + \phi(\mathbf t) + o(\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb R^m}$}.
		$$

		As
		$$
		\lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \phi(\mathbf t) = \mathbf 0_{\mathbb Y}
		$$
		and, by Lemma \ref{lm: little-o: zero limit},
		$$
		\lim_{\mathbf t \to \mathbf 0_{\mathbb X}}o(\mathbf t) = \mathbf 0_{\mathbb Y},
		$$

		we have
		$$
		\lim_{\mathbf t \to \mathbf 0_{\mathbb X}} f(\mathbf p + \mathbf t) = f(\mathbf p),
		$$
		which implies that $f$ is continuous at $\mathbf p$.
	\end{proof}
\end{lemma}

\begin{lemma}
	\label{lm: differentiable: composed}

	Let $f: \mathbb X \to \mathbb Y$ and let $g: \mathbb Y \to \mathbb S$. If $f$ is differentiable at a point $\mathbf p \in \mathbb X$, and $g$ is differentiable at $f(\mathbf x)$, then $g \circ f$ is differentiable at $\mathbf p$.

	\begin{proof}
		As $g$ is differentiable at $f(\mathbf p)$, there exists $\lambda: \mathbb Y \to \mathbb S$ such that for any $\mathbf s \in \mathbb Y$ with $f(\mathbf p) + \mathbf s \in f[\mathbb X]$,
		$$
		g(f(\mathbf p) + \mathbf s) = g(f(\mathbf p)) + \lambda(\mathbf s) + o(\mathbf s) \quad \text{as $\mathbf s \to \mathbf 0_{\mathbb Y}$}.
		$$

		As $f$ is differentiable at $\mathbf p$, $f$ is continuous at $\mathbf p$, thus, there exists $\mathbf t \in \mathbb X$, such that $\displaystyle\lim_{\mathbf t \to \mathbf 0_{\mathbb X}} f(\mathbf p + \mathbf t) = f(\mathbf p) + \mathbf s$. Since $f$ is differentiable at $\mathbf p$, there exists a linear mapping $\phi: \mathbb X \to \mathbb Y$, such that
		$$
		f(\mathbf p + \mathbf t) = f(\mathbf p) + \phi(\mathbf t) + o_1 (\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$}.
		$$

		Then we have
		$$
		g(f(\mathbf p + \mathbf t)) = g(f(\mathbf t)) + \lambda(\Delta f) + o(\Delta f) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$},
		$$
		where
		$$
		\Delta f = f(\mathbf p + \mathbf t) - f(\mathbf p) = \phi(\mathbf t) + o_1(\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$}.
		$$

		First, find $\lambda(\Delta f)$. As $\lambda$ is linear,
		$$
		\begin{aligned}
			\lambda(\Delta f) &= \lambda(\phi(\mathbf t) + o_1(\mathbf t))
			= \lambda(\phi(\mathbf t)) + \lambda(o_1 (\mathbf t)) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$}.
		\end{aligned}
		$$
		As $\lambda$ is linear, $\lambda \circ \phi$ is also linear, and $\lambda(o_1(\mathbf t))$ is a little-o of $\mathbf t$, i.e., $o_2(\mathbf t) = \lambda(o_1(\mathbf t))$ as $\mathbf t \to \mathbf 0_{\mathbb X}$, for
		$$
		\begin{aligned}
			\lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{\lambda(o_1(\mathbf t))}{\| \mathbf t \|_{\mathbb X}} &= \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \lambda\left( \frac{ o_1 (\mathbf t) }{ \| \mathbf t \|_{\mathbb X} } \right) \\
			&= \lambda(\mathbf 0_{\mathbb Y}) \\
			&= \mathbf 0_{\mathbb S}.
		\end{aligned}
		$$

		Let $\gamma = \lambda \circ \phi$ for convenience.

		Then, find $o(\Delta f)$.
		$$
		\begin{aligned}
			\mathbf 0_{\mathbb S} &= \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{o(\phi(\mathbf t) + o_1(\mathbf t))}{\| \phi(\mathbf t) + o_1(\mathbf t) \|_{\mathbb Y}}
			&\text{(Definition \ref{def: little-o})} \\
			&= \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{ o(\phi(\mathbf t) + o_1(\mathbf t)) \| \mathbf t \|_{\mathbb X}^{-1} }{ \| \phi(\mathbf t) + o_1(\mathbf t) \|_{\mathbb Y} \| \mathbf t \|_{\mathbb X}^{-1} } \\
			&= \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{o(\phi(\mathbf t) + o_1(\mathbf t))}{\| \mathbf t \|_{\mathbb X} \| \phi(\mathbf{\hat t})\| _{\mathbb Y}}
			&\text{(as $\phi$ is linear)} \\
			&= \| \phi(\mathbf{\hat t}) \|_{\mathbb Y}^{-1} \lim_{\mathbf t \to \mathbf 0_{\mathbb X}} \frac{o(\phi(\mathbf t) - o_1(\mathbf t))}{\| \mathbf t \|_{\mathbb X}}.
		\end{aligned}
		$$

		Thus, $o(\phi(\mathbf t) - o_1(\mathbf t)) = o_3(\mathbf t)$ as $\mathbf t \to \mathbf 0_{\mathbb X}$.

		Now, we have
		$$
		g(f(\mathbf p + \mathbf t)) = g(f(\mathbf t)) + \gamma(\mathbf t) + o_2(\mathbf t) + o_3(\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$}.
		$$

		By Lemma \ref{lm: little-o: finite sum},
		$$
		o_2(\mathbf t) + o_3(\mathbf t) = o_4(\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$}.
		$$

		Finally, we have
		$$
		g(f(\mathbf p + \mathbf t)) = g(f(\mathbf t)) + \gamma(\mathbf t) + o_4(\mathbf t) \quad \text{as $\mathbf t \to \mathbf 0_{\mathbb X}$},
		$$
		which implies $g \circ f$ is differentiable at $\mathbf p$.
	\end{proof}
\end{lemma}

\section{Directional Derivatives}
%================================================
%------------------------------------------------

\begin{definition}
	\label{def: directional derivatives}

	Let $f: \mathbb X \to \mathbb Y$, and let $\mathbf u \in \mathbb X \setminus \{ \mathbf 0_{\mathbb X} \}$.

	The \textit{$\mathbf u$-directional derived mapping}
\end{definition}

---

---

---

---

---

---

---

---

---

---

---

---

%


\chapter{Depth-First Search}

One important operation in a graph is search.  A useful kind of search
is Depth-First, in which we begin as some start node, marking the node
visited, then we recurse on the neighbours one by one until no more
unvisited neighbours exist.

During a visit, we label each node with a number before visiting its
children and another number after visiting its children.  This number
(called the clock) starts at 1, and is incremented every time a node
receives a label.  The number a node $v$ receives before its children
are visited is called its pre-number ($pre(v)$), and the number that
node receives after its children are visited is called its post-number
($post(v)$).

\begin{theorem}[Parenthesis Theorem]

For nodes $u,v$, the interval between $pre(u)$ and $post(u)$ and the
same interval for $v$ are either:

\begin{itemize}

\item Entirely disjoint
\item The interval of $u$ is completely within the interval of $v$
\item The interval of $v$ is completely within the interval of $u$

\end{itemize}

The name comes from the fact that this is just like properly nested
parentheses.

\end{theorem}

Running a DFS on a graph reveals a tree structure where the start node
is the root and neighbours are parent and child depending on which was
visited first.  There are a few interesting classifications of edges
in a DFS tree:

\begin{enumerate}

\item A \emph{Tree Edge} goes from a parent node to a child node.

\item A \emph{Forward Edge} goes from ancestor to descendant (but not
  parent to child)

\item A \emph{Back Edge} goes from descendant to ancestor

\item A \emph{Cross Edge} goes to a non-ancestor non-descendant

\end{enumerate}

\begin{theorem}
A digraph has a cycle if and only if a DFS reveals a back edge.
\end{theorem}

\begin{proof}

First let us assume there is a back edge.  By definition this is a
descendant linking back to an ancestor, which has already been
visited.  This gives us a cycle.

Next let us assume there is a cycle.  A DFS will visit each node in
the cycle, and as soon as the last edge in the cycle is visited it
will link a descendant to ancestor which is the definition of a back
edge.

\end{proof}

\begin{theorem}

After running a DFS on a directed acyclic graph (DAG), each edge leads
to a vertex with a lower post number.

\end{theorem}

The proof of this theorem is left up to the reader.


\chapter{Algorithms for One-Dimensional Optimization}
\label{sec:algOneDimOpt}
\section{Interval Division Algorithms}
\lab{sec:IntDivAlg}
Interval division algorithm can be used to minimize a function $f \colon \Re \to \Re$,
(i.e., the function depends on one independent parameter only,)
over a user-specified interval.
The algorithms do not require derivatives and
they require only one function evaluation per interval division,
except for the initialization.

First, we explain a master algorithm for the interval division algorithms.
The master algorithm is used to implement two commonly used interval division algorithms:
The Golden Section search and the Fibonacci Division.

\begin{figure}
\centering
\epsfig{file=img/int_div.eps, bb=65 345 245 535}
\caption{Interval division.}
\label{fig:intDivGen}
\end{figure}

% --------
\subsection{General Interval Division}
We now describe the ideas behind the interval division methods.
For given $x_0, x_3 \in \Re$, with $x_0 < x_3$,
let $\mathbf X \triangleq [x_0, x_3]$.
Suppose we want to minimize $f(\cdot)$ on $\mathbf X$,
and suppose that $f \colon \Re \rightarrow \Re$ has a unique minimizer
$x^* \in \mathbf X$.
For some $s \in (0, 1)$, let
\begin{eqnarray}
   x_1 & \triangleq & x_0 + s \, (x_3 - x_0), \\
   x_2 & \triangleq & x_1 + s \, (x_3 - x_1).
\end{eqnarray}
If $f(x_1) \le f(x_2)$, then $x^* \in  [x_0, \, x_2]$.
Hence, we can eliminate the interval $(x_2, \, x_3]$
and restrict our search to $[x_0, \, x_2]$.
Similarly, if $f(x_1) > f(x_2)$, then $x^* \in [x_1, \, x_3]$
and we can eliminate $[x_0, \, x_1)$.
Thus, we reduced the initial interval to a new interval that contains the minimizer $x^*$.\\

Let $i \in \Na$ be the iteration number.
We want to nest the sequence of intervals
\begin{equation}
[x_{0,(i+1)}, \, x_{3,(i+1)}] \subset [x_{0,i}, \, x_{3,i}], \qquad i \in \{0, \, 1, \, 2, \, \ldots \},
\end{equation}
such that we have to evaluate $f\depd$ in each step at one new point only.
To do so, we assign the new bounds of the interval such that either
$[x_{0,(i+1)}, \, x_{3,(i+1)}] = [x_{0,i}, \, x_{2,i}]$, or $[x_{0, (i+1)}, \, x_{3, (i+1)}] = [x_{1,i}, \, x_{3,i}]$,
depending on which interval has to be eliminated.
By doing so, we have to evaluate only one new point in the interval.
It remains to decide where to locate the new point.
The Golden Section and Fibonacci Division differ in this decision.

% --------
\subsection{Golden Section Interval Division}
Suppose we have three points $x_0 < x_1 < x_3$ in $\mathbf X \subset \Re$
such that for some $q \in (0, 1)$, to be determined later,
\begin{subequations}
\begin{equation}
   \frac{ | x_0 - x_1 | }{ | x_0 - x_3 |  } = q.
   \label{eq:golSecQDef}
\end{equation}
Hence,
\begin{equation}
   \frac{ | x_1 - x_3 | }{ | x_0 - x_3 |  } = 1-q.
\end{equation}
\end{subequations}

Suppose that $x_2$ is located somewhere between $x_1$ and $x_3$ and define
the ratio
\begin{equation}
  w \triangleq \frac{ | x_1 - x_2 | }{ | x_0 - x_3 |  }.
\end{equation}
Depending on which interval is eliminated, the interval in the next iteration step will
either be of length
$(q+w) \, | x_0 - x_3|$,
or $(1-q)  \, | x_0 - x_3|$.
We select the location of $x_2$ such that the two intervals are of the same length.
Hence,
\begin{subequations}
\begin{equation}
   q + w = 1 - q.
  \label{eq:golSecqw}
\end{equation}
Now, we determine the fraction $q$.
Since we apply the process of interval division recursively, we know by scale similarity that
\begin{equation}
   \frac{ w }{ 1 - q } = q.
   \label{eq:golSecq}
\end{equation}
\end{subequations}
\begin{subequations}
Combining (\ref{eq:golSecqw}) and (\ref{eq:golSecq}) leads to
\begin{equation}
   q^2 - 3 q + 1 = 0,
\end{equation}
with solutions
\begin{equation}
   q_{1,2} = \frac{3 \pm \sqrt{5}}{2}.
\end{equation}
Since $q < 1$ by (\ref{eq:golSecQDef}), the solution of interest is
\begin{equation}
   q = \frac{3 - \sqrt{5}}{2} \approx 0.382.
\end{equation}
\end{subequations}

The fractional distances $q \approx 0.382$ and $1-q \approx 0.618$ correspond to the so-called \emph{Golden Section}, which gives this algorithm its name.\\

Note that the interval is reduced in each step by the fraction $1-q$, i.e., we have \emph{linear convergence}. In the $m$-th iteration, we have
\begin{eqnarray}
   | x_{ 0, \, m} - x_{ 2, \, m} | & = &
   | x_{ 1, \, m} - x_{ 3, \, m} | =
   | x_{ 0, \, (m+1)} - x_{ 3, \, (m+1)} | \nonumber \\
 & = &
   (1-q)^{m+1} \,    | x_{ 0, \, 0} - x_{ 3, \, 0} |.
\end{eqnarray}
Hence, the required number of iterations, $m$, to reduce the initial interval of uncertainty $|x_{0, \,0} - x_{3, \, 0} |$ to at least a fraction $r$, defined as
\begin{equation}
   r \triangleq \frac{ | x_{ 0, \, m} - x_{ 2, \, m} | }
                { | x_{ 0, \, 0} - x_{ 3, \, 0} | }
    = \frac{ | x_{ 1, \, m} - x_{ 3, \, m} | }
                { | x_{ 0, \, 0} - x_{ 3, \, 0} | },
  \label{eq:golSecDefR}
\end{equation}
is given by
\begin{equation}
  m = \frac{\ln r}{ \ln (1-q)} - 1.
\end{equation}

% =======================
\subsection{Fibonacci Division}
Another way to divide an interval such that we need one function evaluation per iteration can be constructed as follows: Given an initial interval $[x_{0, \, i}, x_{3, \, i}]$ , $i=0$, we divide it into three segments symmetrically around its midpoint. Let $d_{1,\, i} < d_{2,\, i} < d_{3,\, i}$ denote the distance of the segment endpoints, measured from $x_{0, \,i}$. Then we have by symmetry $d_{3,\, i}=d_{1,\, i} + d_{2,\, i}$. By the bracket elimination procedure explained above, we know that we are eliminating a segment of length $d_{1,\, i}$. Therefore, our new interval is of length $d_{3,\, (i+1)}= d_{2,\, i}$. By symmetry we also have $d_{3,\, (i+1)}= d_{1,\, (i+1)} + d_{2,\, (i+1)}$. Hence, if we construct our segment length such that $d_{3,\, (i+1)} = d_{1,\, (i+1)} + d_{2,\, (i+1)}= d_{2,\, i}$ we can reuse one known point. Such a construction can be done by using \emph{Fibonacci} numbers, which are defined recursively by
\begin{subequations}
\begin{eqnarray}
  F_0 & \triangleq & F_1 \triangleq 1, \\
  F_i & \triangleq & F_{i-1} + F_{i-2}, \qquad i \in \{2, \, 3, \, \ldots \}.
\end{eqnarray}
\end{subequations}
The first few numbers of the Fibonacci sequence are $\{1, \, 1, \,  2, \, 3, \, 5, \, 8, \, 13, \, 21, \, \ldots \}$. The length of the intervals $d_{1,\, i}$ and $d_{2, \, i}$, respectively, are then given by
\begin{equation}
   d_{1, \, i} = \frac{F_{m-i} }{ F_{m-i+2} }, \quad
  d_{2, \, i} = \frac{F_{m-i+1} }{ F_{m-i+2} }, \qquad
  i \in \{0, \, 1, \, \ldots \, , \, m \},
\end{equation}
where $m > 0 $ describes how many iterations will be done. Note that $m$ must be known prior to the first interval division.
Hence, the algorithm must be stopped after $m$ iterations.\\

The reduction of the length of the uncertainty interval per iteration is given by
\begin{equation}
  \frac{ d_{3, \, (i+1)} }{  d_{3, \, i}  } =
  \frac{ d_{2, \, i} }{  d_{1, \, i} + d_{2, \, i}  } =
   \frac{  \frac{ F_{m-i+1} }{F_{m-i+2}}  }
     {  \frac{F_{m-i}}{F_{m-i+2}} +  \frac{F_{m-i+1}}{F_{m-i+2}}   } =
        \frac{ F_{m-i+1}   }{ F_{m-i+2}   }.
\end{equation}
After $m$ iterations, we have
\begin{eqnarray}
  \frac{d_{3, \, m}}{d_{3, \, 0}} & = &
  \frac{ d_{ 3, \, m}  }{ d_{ 3, \, (m-1)}  } \,
  \frac{ d_{ 3, \, (m-1)}  }{ d_{ 3, \, (m-2)}  } \,
   \ldots \,
   \frac{ d_{ 3, \, 2}  }{ d_{ 3, \, 1}  } \,
   \frac{ d_{ 3, \, 1}  }{ d_{ 3, \, 0}  } \nonumber \\
 & = &
  \frac{F_{2} }{ F_{3} } \,
  \frac{F_{3} }{ F_{4} } \, \ldots \,
  \frac{F_{m} }{ F_{m+1} } \,
  \frac{F_{m+1} }{ F_{m+2} }
=
  \frac{2}{F_{m+2}}.
\end{eqnarray}
The required number of iterations $m$ to reduce the initial interval $d_{3, \, 0}$
to at least a fraction $r$, defined by (\ref{eq:golSecDefR}), can again be obtained by expansion from
\begin{eqnarray}
  r & = &
  \frac{d_{2, \, m}}{d_{3, \, 0}} =
  \frac{ d_{ 3, \, (m+1)}  }{ d_{ 3, \, 0}  } =
  \frac{ d_{ 3, \, (m+1)}  }{ d_{ 3, \, m}  } \,
  \frac{ d_{ 3, \, m}  }{ d_{ 3, \, (m-1)}  } \,
   \ldots \,
   \frac{ d_{ 3, \, 2}  }{ d_{ 3, \, 1}  } \,
   \frac{ d_{ 3, \, 1}  }{ d_{ 3, \, 0}  } \, \nonumber \\
 & = &
  \frac{F_{1} }{ F_{2} } \,
  \frac{F_{2} }{ F_{3} } \, \ldots \,
  \frac{F_{m} }{ F_{m+1} } \,
  \frac{F_{m+1} }{ F_{m+2} }
=
  \frac{1}{F_{m+2}}.
\end{eqnarray}
Hence, $m$ is given by
\begin{equation}
  m = \argmin_{m \in \Na} \left\{ m \ | \ r \ge \frac{1}{F_{m+2}} \right\}.
\end{equation}

% -----------------
\subsection{Comparison of Efficiency}
The Golden Section is more efficient than the Fibonacci Division. Comparing the reduction of the interval of uncertainty, $| x_{0, \, m} - x_{3, \, m}|$, in the limiting case for $m \rightarrow \infty$, we obtain
\begin{equation}
   \lim_{m \rightarrow \infty} \frac{ | x_{0, \, m} - x_{3, \, m}  |_{GS} }
  { | x_{0, \, m} - x_{3, \, m}  |_{F} }
= \lim_{m \rightarrow \infty} \frac{F_{m+2}}{2} \, (1-q)^m = 0.95.
\end{equation}

% --------------------

\subsection{Master Algorithm for Interval Division}
The following master algorithm explains the steps of
the interval division algorithm.\\

\noindent
\begin{minipage}[b]{\textwidth}
\begin{algorithm}
[Model Interval Division Algorithm]
~\\
{\em
\begin{tabularx}{\headwidth}{m{2cm}l}
\multicolumn{2}{l}{\hspace{\textwidth}~} \\ \\[-8ex]\\
\hline \\[-2ex]
 \textbf{Data}:
 & $x_0$, $x_3$. \\
 & Procedure that returns $r_i$, defined as\\
 & $r_i \triangleq |x_{0, \, i} - x_{2, \, i} | / |x_{0, \, 0} - x_{3, \, 0}|$. \\
\textbf{Step 0:}
 & {\it Initialize }\\
  & $\Delta x = x_3 - x_0$,\\
  &$ x_2 = x_0 + r_1 \, \Delta x$,\\
  & $x_1 = x_0 + r_2 \, \Delta x$,\\
  & $f_1 = f(x_1)$, $ f_2 = f(x_2)$, and\\
  & $i = 2$.\\
\textbf{Step 1:} & {\it Iterate.}\\
  & Replace $i$ by $i + 1$.\\
  & If  $(f_2 < f_1)$\\
    &  \hspace{1cm}  Set  $x_0 = x_1$, $x_1 = x_2$,\\
    &   \hspace{1cm}   $f_1 = f_2$,\\
    &   \hspace{1cm}   $x_2 = x_3 - r_i \, \Delta x$, and\\
    &    \hspace{1cm}  $f_2 = f(x_2)$.\\
   & else\\
   &    \hspace{1cm} Set $x_3 = x_2$, $x_2 = x_1$,\\
   &    \hspace{1cm}  $f_2 = f_1$,\\
   &    \hspace{1cm}  $x_1 = x_0 + r_i \,  \Delta x$,\\
   &    \hspace{1cm}  $f_1 = f(x_1)$.\\
   \textbf{Step 2:} & Stop or go to Step 1.\\
   \hline \\
\end{tabularx}
}
\lab{al:ModOneDim}
\end{algorithm}
\end{minipage}
% ---------------------

\subsection{Keywords}
For the Golden Section and the Fibonacci Division algorithm, the command file (see page~\pageref{par:comFil}) must contain only one continuous parameter.\\

To invoke the Golden Section or the Fibonacci Division algorithm, the \texttt{Algorithm} Section of the GenOpt command file must have following form:
\begin{lstlisting}
Algorithm{
   Main              = GoldenSection | Fibonacci;
  [AbsDiffFunction   = Double;  |   // 0 < AbsDiffFunction
   IntervalReduction = Double;  ]   // 0 < IntervalReduction
}
\end{lstlisting}
\pagebreak[2]
\noindent The keywords have the following meaning
\begin{codedescription}
\item[Main]
The name of the main algorithm.
\end{codedescription}
The following two keywords are optional. If none of them is specified, then the algorithm stops after \texttt{MaxIte} function evaluations (i.e., after \texttt{MaxIte}$-2$ iterations), where \texttt{MaxIte} is specified in the section \texttt{OptimizationSettings}. If both of them are specified, an error occurs.
\begin{codedescription}
\item[AbsDiffFunction]
The absolute difference defined as
\begin{equation}
\Delta f \triangleq | \min \{ f(x_0),\, f(x_3) \} - \min \{ f(x_1), \, f(x_2) \}|.
\end{equation}
If $\Delta f$ is lower than \texttt{AbsDiffFunction}, the search stops successfully.\\
\underline{Note:} Since the maximum number of interval reductions must be known for the initialization of the Fibonacci algorithm, this keyword can be used only for the Golden Section algorithm. It must not be specified for the Fibonacci algorithm.
\item[IntervalReduction]
The required maximum fraction, $r$, of the end interval length relative to the initial interval length (see equation~\eqref{eq:golSecDefR}).
\end{codedescription}