% Written by Anders Sjoqvist and Ulf Lundstrom, 2009
% The main sources are: tinyKACTL, Beta and Wikipedia

\chapter{Mathematics}

\section{Equations}

\[\begin{aligned}ax+by=e\\cx+dy=f\end{aligned}
\Rightarrow
\begin{aligned}x=\dfrac{ed-bf}{ad-bc}\\y=\dfrac{af-ec}{ad-bc}\end{aligned}\]

In general, given an equation $Ax = b$, the solution to a variable $x_i$ is given by
\[x_i = \frac{\det A_i'}{\det A} \]
where $A_i'$ is $A$ with the $i$'th column replaced by $b$.

% \section{Recurrences}
% If $a_n = c_1 a_{n-1} + \dots + c_k a_{n-k}$, and $r_1, \dots, r_k$ are distinct roots of $x^k + c_1 x^{k-1} + \dots + c_k$, there are $d_1, \dots, d_k$ s.t.
% \[a_n = d_1r_1^n + \dots + d_kr_k^n. \]
% Non-distinct roots $r$ become polynomial factors, e.g. $a_n = (d_1n + d_2)r^n$.

\section{Trigonometry}
\begin{align*}
\sin(v+w)&{}=\sin v\cos w+\cos v\sin w\\
\cos(v+w)&{}=\cos v\cos w-\sin v\sin w\\
\tan(v+w)&{}=\dfrac{\tan v+\tan w}{1-\tan v\tan w}\\
\sin v+\sin w&{}=2\sin\dfrac{v+w}{2}\cos\dfrac{v-w}{2}\\
\cos v+\cos w&{}=2\cos\dfrac{v+w}{2}\cos\dfrac{v-w}{2}
\end{align*}
\begin{align*}
	a\cos x+b\sin x&=r\cos(x-\phi)\\
	a\sin x+b\cos x&=r\sin(x+\phi)
\end{align*}
where $r=\sqrt{a^2+b^2}, \phi=\operatorname{atan2}(b,a)$.

\section{Geometry}

\subsection{Triangles}
Side lengths: $a,b,c$\\
Semiperimeter: $s=\dfrac{a+b+c}{2}$\\
Area: $A=\sqrt{s(s-a)(s-b)(s-c)}$\\
Circumradius: $R=\dfrac{abc}{4A}$\\
Inradius: $r=\dfrac{A}{p}$\\
Length of median (divides triangle into two equal-area triangles): $m_a=\tfrac{1}{2}\sqrt{2b^2+2c^2-a^2}$\\
Length of bisector (divides angles in two): $s_a=\sqrt{bc\left[1-\left(\dfrac{a}{b+c}\right)^2\right]}$\\
Law of sines: $\dfrac{\sin\alpha}{a}=\dfrac{\sin\beta}{b}=\dfrac{\sin\gamma}{c}=\dfrac{1}{2R}$\\
Law of cosines: $a^2=b^2+c^2-2bc\cos\alpha$\\
Law of tangents: $\dfrac{a+b}{a-b}=\dfrac{\tan\dfrac{\alpha+\beta}{2}}{\tan\dfrac{\alpha-\beta}{2}}$\\

\subsection{Quadrilaterals}
With side lengths $a,b,c,d$, diagonals $e, f$, diagonals angle $\theta$, area $A$ and
magic flux $F=b^2+d^2-a^2-c^2$:

\[ 4A = 2ef \cdot \sin\theta = F\tan\theta = \sqrt{4e^2f^2-F^2} \]

 For cyclic quadrilaterals the sum of opposite angles is $180^\circ$,
$ef = ac + bd$, and $A = \sqrt{(s-a)(s-b)(s-c)(s-d)}$.

\subsection{Spherical coordinates}
\centerline{\includegraphics[width=25mm]{content/math/sphericalCoordinates}}
\[\begin{array}{cc}
x = r\sin\theta\cos\phi & r = \sqrt{x^2+y^2+z^2}\\
y = r\sin\theta\sin\phi & \theta = \textrm{acos}(z/\sqrt{x^2+y^2+z^2})\\
z = r\cos\theta & \phi = \textrm{atan2}(y,x)
\end{array}\]

\section{Derivatives/Integrals}
\begin{align*}
	\dfrac{d}{dx}\arcsin x = \dfrac{1}{\sqrt{1-x^2}} &&& \dfrac{d}{dx}\arccos x = -\dfrac{1}{\sqrt{1-x^2}} \\
	\dfrac{d}{dx}\tan x = 1+\tan^2 x &&& \dfrac{d}{dx}\arctan x = \dfrac{1}{1+x^2} \\
	\int\tan ax = -\dfrac{\ln|\cos ax|}{a} &&& \int x\sin ax = \dfrac{\sin ax-ax \cos ax}{a^2} \\
	\int e^{-x^2} = \frac{\sqrt \pi}{2} \text{erf}(x) &&& \int xe^{ax}dx = \frac{e^{ax}}{a^2}(ax-1)
\end{align*}

Integration by parts:
\[\int_a^bf(x)g(x)dx = [F(x)g(x)]_a^b-\int_a^bF(x)g'(x)dx\]

\section{Sums/Series}

\begin{comment}
\[ c^a + c^{a+1} + \dots + c^{b} = \frac{c^{b+1} - c^a}{c-1}, c \neq 1 \]
\begin{align*}
	1 + 2 + 3 + \dots + n &= \frac{n(n+1)}{2} \\
	1^2 + 2^2 + 3^2 + \dots + n^2 &= \frac{n(n+1)(2n+1)}{6} \\
	1^3 + 2^3 + 3^3 + \dots + n^3 &= \frac{n^2(n+1)^2}{4} \\
	1^4 + 2^4 + 3^4 + \dots + n^4 &= \frac{n(n+1)(2n+1)(3n^2+3n - 1)}{30}
\end{align*}
\end{comment}

% $$1^4 + 2^4 + 3^4 + \dots + n^4= \frac{n(n+1)(2n+1)(3n^2+3n - 1)}{30}$$
% $$e^x = 1+x+\frac{x^2}{2!}+\frac{x^3}{3!}+\dots,\,(-\infty<x<\infty)$$
$$\ln(1+x) = x-\frac{x^2}{2}+\frac{x^3}{3}-\frac{x^4}{4}+\dots,\,(-1<x\leq1)$$
$$\sqrt{1+x} = 1+\frac{x}{2}-\frac{x^2}{8}+\frac{2x^3}{32}-\frac{5x^4}{128}+\dots,\,(-1\leq x\leq1)$$
$$\sin x = x-\frac{x^3}{3!}+\frac{x^5}{5!}-\frac{x^7}{7!}+\dots,\,(-\infty<x<\infty)$$
$$\cos x = 1-\frac{x^2}{2!}+\frac{x^4}{4!}-\frac{x^6}{6!}+\dots,\,(-\infty<x<\infty)$$
% $$e^{ix}=\cos x+i\sin x$$

\section{Probability theory}
Let $X$ be a discrete random variable with probability $p_X(x)$ of assuming the value $x$. It will then have an expected value (mean) $\mu=\mathbb{E}(X)=\sum_xxp_X(x)$ and variance $\sigma^2=V(X)=\mathbb{E}(X^2)-(\mathbb{E}(X))^2=\sum_x(x-\mathbb{E}(X))^2p_X(x)$ where $\sigma$ is the standard deviation.
\[\mathbb{E}(aX+bY) = a\mathbb{E}(X)+b\mathbb{E}(Y)\]
If $X,Y$ are independent, \[V(aX+bY) = a^2V(X)+b^2V(Y).\]
% Expectation is linear
% If $X$ is instead continuous it will have a probability density function $f_X(x)$ and the sums above will instead be integrals with $p_X(x)$ replaced by $f_X(x)$.

\subsection{Discrete distributions}

% \subsubsection{Binomial distribution}
% \# of successes in $n$ independent yes/no experiments, each which yields success with probability $p$ is $\textrm{Bin}(n,p),\,n=1,2,\dots,\, 0\leq p\leq1$.
% \[p(k)=\binom{n}{k}p^k(1-p)^{n-k}\]
% \[\mu = np,\,\sigma^2=np(1-p)\]
% $\textrm{Bin}(n,p)\approx \textrm{Po}(np)$ for small $p$.

\subsubsection{First success distribution}
The number of trials needed to get the first success in independent yes/no experiments, each wich yields success with probability $p$ is $\textrm{Fs}(p),\,0\leq p\leq1$.
\[p(k)=p(1-p)^{k-1},\,k=1,2,\dots\]
\[\mu = \frac1p,\,\sigma^2=\frac{1-p}{p^2}\]

\subsubsection{Poisson distribution}
The number of events occurring in a fixed period of time $t$ if these events occur with a known average rate $\kappa$ and independently of the time since the last event is $\textrm{Po}(\lambda),\,\lambda=t\kappa$.
\[p(k)=e^{-\lambda}\frac{\lambda^k}{k!}, k=0,1,2,\dots\]
\[\mu=\lambda,\,\sigma^2=\lambda\]
$\textrm{Bin}(n,p)\approx \textrm{Po}(np)$ for small $p$ (binomial distribution with $n$ coin flips, each of which is heads with probability $p$).

\subsection{Continuous distributions}

% \subsubsection{Uniform distribution}
% If the probability density function is constant between $a$ and $b$ and 0 elsewhere it is $\textrm{U}(a,b),\,a<b$.
% \[f(x) = \left\{
% \begin{array}{cl}
% \frac{1}{b-a} & a<x<b\\
% 0 & \textrm{otherwise}
% \end{array}\right.\]
% \[\mu=\frac{a+b}{2},\,\sigma^2=\frac{(b-a)^2}{12}\]

\subsubsection{Exponential distribution}
The time between events in a Poisson process is $\textrm{Exp}(\lambda),\,\lambda>0$.
\[f(x) = \left\{
\begin{array}{cl}
\lambda e^{-\lambda x} & x\geq0\\
0 & x<0
\end{array}\right.\]
\[\mu=\frac{1}{\lambda},\,\sigma^2=\frac{1}{\lambda^2}\]

\subsubsection{Normal distribution}
Most real random values with mean $\mu$ and variance $\sigma^2$ are well described by $\mathcal{N}(\mu,\sigma^2),\,\sigma>0$.
\[ f(x) = \frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(x-\mu)^2}{2\sigma^2}} \]
% If $X_1 \sim \mathcal{N}(\mu_1,\sigma_1^2)$ and $X_2 \sim \mathcal{N}(\mu_2,\sigma_2^2)$ then
% \[ aX_1 + bX_2 + c \sim \mathcal{N}(\mu_1+\mu_2+c,a^2\sigma_1^2+b^2\sigma_2^2) \]

\section{Markov chains}
A \emph{Markov chain} is a discrete random process with the property that the next state depends only on the current state.
Let $X_1,X_2,\ldots$ be a sequence of random variables generated by the Markov process.
Then there is a transition matrix $\mathbf{P} = (p_{ij})$, with $p_{ij} = \Pr(X_n = i | X_{n-1} = j)$,
and $\mathbf{p}^{(n)} = \mathbf P^n \mathbf p^{(0)}$ is the probability distribution for $X_n$ (i.e., $p^{(n)}_i = \Pr(X_n = i)$),
where $\mathbf{p}^{(0)}$ is the initial distribution.

% \subsubsection{Stationary distribution}
$\mathbf{\pi}$ is a stationary distribution if $\mathbf{\pi} = \mathbf{\pi P}$.
If the Markov chain is \emph{irreducible} (it is possible to get to any state from any state),
then $\pi_i = \frac{1}{\mathbb{E}(T_i)}$ where $\mathbb{E}(T_i)$  is the expected time between two visits in state $i$.
$\pi_j/\pi_i$ is the expected number of visits in state $j$ between two visits in state $i$.

For a connected, undirected and non-bipartite graph, where the transition probability is uniform among all neighbors, $\pi_i$ is proportional to node $i$'s degree. \textbf{(IMPORTANT)} % https://codeforces.com/gym/101081/problem/B

% \subsubsection{Ergodicity}
A Markov chain is \emph{ergodic} if the asymptotic distribution is independent of the initial distribution.
A finite Markov chain is ergodic iff it is irreducible and \emph{aperiodic} (i.e., the gcd of cycle lengths is 1).
$\lim_{k\rightarrow\infty}\mathbf{P}^k = \mathbf{1}\pi$.

\begin{comment}
% \subsubsection{Absorption}
A Markov chain is an A-chain if the states can be partitioned into two sets $\mathbf{A}$ and $\mathbf{G}$, such that all states in $\mathbf{A}$ are absorbing ($p_{ii}=1$), and all states in $\mathbf{G}$ lead to an absorbing state in $\mathbf{A}$.
The probability for absorption in state $i\in\mathbf{A}$, when the initial state is $j$, is $a_{ij} = p_{ij}+\sum_{k\in\mathbf{G}} a_{ik}p_{kj}$.
The expected time until absorption, when the initial state is $i$, is $t_i = 1+\sum_{k\in\mathbf{G}}p_{ki}t_k$.
\end{comment}


\section*{Exercises}

\begin{ex}
  Let $z=2+7i$ and let $w=3-8i$. Compute $z + w$, $z - 2w$, $zw$, and $\frac{w}{z}$.
  \begin{sol}
    $z + w = 5-i$, $z - 2w = -4 + 23i$, $zw = 62+5i$, and
    $\displaystyle\frac{w}{z} = -\frac{50}{53}-\frac{37}{53}i$.
  \end{sol}
\end{ex}

\begin{ex}
  Let $z = 1 - 4i$. Compute $\conjugate{z}$, $z^{-1}$, and $\abs{z}$.
\end{ex}

\begin{ex}
  Let $z = 3+5i$ and $w = 2-i$. Compute $\conjugate{zw}$, $\abs{zw}$,
  and $z^{-1}w$.
\end{ex}

\begin{ex}
  Use the properties of complex numbers to prove that if $z$ is a
  complex number, then there exists a complex number $w$ with
  $\abs{w}=1$ and $wz=\abs{z}$.
  \begin{sol}
    If $z=0$, let $w=1$. If $z\neq 0$, let
    $w =\displaystyle\frac{\abs{z}}{z}$. Note that
    $\displaystyle wz = \frac{\abs{z}}{z}z = \abs{z}$ and
    $\displaystyle \abs{w} = \frac{\abs{z}}{\abs{z}} = 1$.
  \end{sol}
\end{ex}

\begin{ex}
  I claim that $1=-1$. Here is why.
  \begin{equation*}
    -1=i\,^2=\sqrt{-1}\sqrt{-1}=\sqrt{(-1)^2}=\sqrt{1}=1.
  \end{equation*}
  What is wrong with this argument?
  \begin{sol}
    The problem is that there is no single $\sqrt{-1}$. In the complex
    numbers, $-1$ has two square roots, namely $i$ and $-i$. Since
    each complex number has two square roots, and generally neither of
    them is positive or even real, the notation $\sqrt{z}$ does not
    have a fixed meaning in the complex numbers. Therefore, the
    equation $\sqrt{z}\sqrt{w} = \sqrt{zw}$ cannot be used. At best,
    we could maybe say $\sqrt{z}\sqrt{w} = \pm\sqrt{zw}$.
  \end{sol}
\end{ex}


\lab{One-dimensional Optimization}{One-dimensional Optimization}
\objective{Most mathematical optimization problems involve estimating the minimizer(s) of a scalar-valued function.
Many algorithms for optimizing functions with a high-dimensional domain depend on routines for optimizing functions of a single variable.
There are many techniques for optimization in one dimension, each with varying degrees of precision and speed.
In this lab, we implement the golden section search method, Newton's method, and the secant method, then apply them to the backtracking problem.
% Some of these techniques also generalize to optimization in higher dimensions.
}

\section*{Golden Section Search} % --------------------------------------------

A function $f:[a,b]\rightarrow\mathbb{R}$ satisfies the \emph{unimodal property} if it has exactly one local minimum and is monotonic on either side of the minimizer.
In other words, $f$ decreases from $a$ to its minimizer $x^*$, then increases up to $b$ (see Figure \ref{fig:1dopt-golden-unimodal}).
% Unimodal functions are the ``easiest'' kinds of functions to optimize, but there are several strategies for doing so.
The \emph{golden section search} method optimizes a unimodal function $f$ by iteratively defining smaller and smaller intervals containing the unique minimizer $x^*$.
This approach is especially useful if the function's derivative does not exist, is unknown, or is very costly to compute.

By definition, the minimizer $x^*$ of $f$ must lie in the interval $[a,b]$.
To shrink the interval around $x^*$, we test the following strategically chosen points.
\begin{equation*}
\tilde{a} = b - \frac{b - a}{\varphi}\qquad\qquad
\tilde{b} = a + \frac{b - a}{\varphi}
% \label{eq:1dopt-golden-testpts}
\end{equation*}
Here $\varphi = \frac{1 + \sqrt{5}}{2}$ is the \emph{golden ratio}.
At each step of the search, $[a,b]$ is refined to either $[a,\tilde{b}]$ or $[\tilde{a}, b]$, called the \emph{golden sections}, depending on the following criteria.
 % \footnote{Note that this is the same as $\tilde{a} = a + \rho(b-a)$ and $\tilde{b} = a + (1-\rho)(b-a)$ where $\rho = \frac{3-\sqrt{5}}{2}$.}
% These points define evenly sized intervals $[\tilde{a}, b]$ and $[a, \tilde{b}]$, called \emph{golden sections} since the ratio of the lengths of $[a,\tilde{a}]$ to $[\tilde{a},b]$ is the same as the ratio of the lengths of $[\tilde{a}, b]$ and $[a, b]$.
% Additionally, the property of the golden ratio that  $\frac{1}{\varphi^2} = 1 - \frac{1}{\varphi}$ leads to a smaller amount of computations needed.

If $f(\tilde{a}) < f(\tilde{b})$, then since $f$ is unimodal, it must be increasing in a neighborhood of $\tilde{b}$.
The unimodal property also guarantees that $f$ must be increasing on $[\tilde{b}, b]$ as well, so $x^* \in [a, \tilde{b}]$ and we set $b = \tilde{b}$.
By similar reasoning, if $f(\tilde{a}) > f(\tilde{b})$, then $x^* \in [\tilde{a}, b]$ and we set $a = \tilde{a}$.
If, however, $f(\tilde{a}) = f(\tilde{b})$, then the unimodality of $f$ does not guarantee anything about where the minimizer lies.
Assuming either $x^*\in [a, \tilde{b}]$ or $x^* \in [\tilde{a}, b]$ allows the iteration to continue, but the method is no longer guaranteed to converge to the local minimum.

At each iteration, the length of the search interval is divided by $\varphi$.
The method therefore converges linearly, which is somewhat slow.
However, the idea is simple and each step is computationally inexpensive.

\begin{figure}[H] % Golden section search for unimodal optimization.
    \centering
    \includegraphics[width=.7\textwidth]{figures/golden_section.pdf}
    \caption{The unimodal $f:[a,b]\rightarrow\mathbb{R}$ can be minimized with a golden section search.
    For the first iteration, $f(\tilde{a}) < f(\tilde{b})$, so $x^* \in [a, \tilde{b}]$.
    New values of $\tilde{a}$ and $\tilde{b}$ are then calculated from this new, smaller interval.}
    \label{fig:1dopt-golden-unimodal}
\end{figure}

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{golden\_section}{$f$,\ $a$,\ $b$,\ \li{tol},\ \li{maxiter}}
   \State $x_0 \gets (a + b)/2$
        \Comment{Set the initial minimizer approximation as the interval midpoint.}
   \State $\varphi = (1+\sqrt{5})/2$
        % \Comment{Set $\varphi$.}
   \For{$i=1,2,\dots,$\ \li{maxiter}}
        \Comment Iterate only \li{maxiter} times at most.
        \State $c \gets (b - a)/\varphi$
        \State $\tilde{a} \gets b - c$
            % \Comment{Set $\tilde{a}$.}
        \State $\tilde{b} \gets a + c$
            % \Comment{Set $\tilde{b}$. }
        \If{$f(\tilde{a}) \leq f(\tilde{b})$}
            \Comment{Get new boundaries for the search interval.}
            \State $b \gets \tilde{b}$
         \Else{}
            \State $a \gets \tilde{a}$
                \EndIf
        \State $x_1 \gets (a + b)/2$
            \Comment{Set the minimizer approximation as the interval midpoint.}
        \If{$|x_0-x_1| <$ \li{tol}}
              \State \texttt{break}
                \Comment{Stop iterating if the approximation stops changing enough.}
       \EndIf
        \State $x_0 \gets x_1$
   \EndFor
   \State \pseudoli{return} $x_1$
\EndProcedure
\end{algorithmic}
\caption{The Golden Section Search}
\label{Alg:Golden-Section-Search}
\end{algorithm}

\begin{problem} % Implement golden search.
Write a function that accepts a function $f:\mathbb{R}\rightarrow\mathbb{R}$, interval limits $a$ and $b$, a stopping tolerance \li{tol}, and a maximum number of iterations \li{maxiter}.
Use Algorithm \ref{Alg:Golden-Section-Search} to implement the golden section search.
Return the approximate minimizer $x^*$, whether or not the algorithm converged (\li{true} or \li{false}), and the number of iterations computed.

Test your function by minimizing $f(x) = e^x - 4x$ on the interval $[0, 3]$, then plotting the function and the computed minimizer together.
Also compare your results to SciPy's golden section search, \li{scipy.optimize.golden()}.

\begin{lstlisting}
>>> from scipy import optimize as opt
>>> import numpy as np

>>> f = lambda x : np.exp(x) - 4*x
>>> opt.golden(f, brack=(0,3), tol=.001)
1.3862578679031485              # ln(4) is the minimizer.
\end{lstlisting}
\label{prob:golden-section-search}
\end{problem}

\section*{Newton's Method} % ==================================================

\emph{Newton's method} is an important root-finding algorithm that can also be used for optimization.
Given $f:\mathbb{R}\rightarrow\mathbb{R}$ and a good initial guess $x_0$, the sequence $(x_k)_{k=1}^\infty$ generated by the recursive rule
\[
x_{k+1} = x_k - \frac{f(x_k)}{f'(x_k)}
\]
converges to a point $\bar{x}$ satisfying $f(\bar{x}) = 0$.
The first-order necessary conditions from elementary calculus state that if $f$ is differentiable, then its derivative evaluates to zero at each of its local minima and maxima.
Therefore using Newton's method to find the zeros of $f'$ is a way to identify potential minima or maxima of $f$.
Specifically, starting with an initial guess $x_0$, set
\begin{equation}
x_{k+1} = x_k - \frac{f'(x_k)}{f''(x_k)}
\label{eq:1dopt-newton}
\end{equation}
and iterate until $|x_k - x_{k-1}|$ is satisfactorily small.
Note that this procedure does not use the actual function $f$ at all, but it requires many evaluations of its first and second derivatives.
As a result, Newton's method converges in few iterations, but it can be computationally expensive.

Each step of \eqref{eq:1dopt-newton} can be thought of approximating the objective function $f$ by a quadratic function $q$ and finding its unique extrema.
That is, we first approximate $f$ with its second-degree Taylor polynomial centered at $x_k$.
\[
q(x) = f(x_k) + f'(x_k) (x - x_k) + \frac{1}{2} f''(x_k) (x - x_k)^2
\]
This quadratic function satisfies $q(x_k) = f(x_k)$ and matches $f$ fairly well close to $x_k$.
Thus the optimizer of $q$ is a reasonable guess for an optimizer of $f$.
To compute that optimizer, solve $q'(x) = 0$.
\[
0 = q'(x) = f'(x_k) + f''(x_k)(x - x_k)
\qquad\Longrightarrow\qquad
x = x_k - \frac{f'(x_k)}{f''(x_k)}
\]
This agrees with \eqref{eq:1dopt-newton} using $x_{k+1}$ for $x$.
See Figure \ref{fig:1dopt-quadratic-newton}.

\begin{figure}[H]
\centering
\includegraphics[width = .7 \textwidth]{figures/quad_approx.pdf}
\caption{A quadratic approximation of $f$ at $x_k$.
The minimizer $x_{k+1}$ of $q$ is close to the minimizer of $f$.
}
\label{fig:1dopt-quadratic-newton}
\end{figure}

Newton's method for optimization works well to locate minima when $f''(x) > 0$ on the entire domain.
However, it may fail to converge to a minimizer if $f''(x) \le 0$ for some portion of the domain.
If $f$ is not unimodal, the initial guess $x_0$ must be sufficiently close to a local minimizer $x^*$ in order to converge.

\begin{problem} % Newton's method for optimization
Let $f:\mathbb{R}\rightarrow\mathbb{R}$.
Write a function that accepts $f'$, $f''$, a starting point $x_0$, a stopping tolerance \li{tol}, and a maximum number of iterations \li{maxiter}.
Implement Newton's method using \eqref{eq:1dopt-newton} to locate a local optimizer.
Return the approximate optimizer, whether or not the algorithm converged, and the number of iterations computed.

Test your function by minimizing $f(x) = x^2 + \sin(5x)$ with an initial guess of $x_0 = 0$.
Compare your results to \li{scipy.optimize.newton()}, which implements the root-finding version of Newton's method.
\begin{lstlisting}
>>> df = lambda x : 2*x + 5*np.cos(5*x)
>>> d2f = lambda x : 2 - 25*np.sin(5*x)
>>> opt.newton(df, x0=0, fprime=d2f, tol=1e-10, maxiter=500)
-1.4473142236328096
\end{lstlisting}
Note that other initial guesses can yield different minima for this function.
\end{problem}

\subsection*{The Secant Method} % ---------------------------------------------

The second derivative of an objective function is not always known or may be prohibitively expensive to evaluate.
The \emph{secant method} solves this problem by numerically approximating the second derivative with a difference quotient.
\[
f''(x) \approx \frac{f'(x + h) - f'(x)}{h}
\]
Selecting $x = x_k$ and $h = x_{k-1} - x_k$ gives the following approximation.
\begin{equation}
f''(x_k) \approx \frac{f'(x_k + x_{k-1} - x_k) - f'(x_k)}{x_{k-1} - x_k}
= \frac{f(x_k) - f'(x_{k-1})}{x_k - x_{k-1}}
\label{eq:1dopt-secant-approx}
\end{equation}
% The \emph{Secant method} approximates it using secant lines, or in other words, the rate of change between points.
Inserting \eqref{eq:1dopt-secant-approx} into \eqref{eq:1dopt-newton} results in the complete secant method formula.
\begin{equation}
x_{k+1}
= x_k - \frac{x_k - x_{k-1}}{f'(x_k) - f'(x_{k-1})}f'(x_k)
= \frac{x_{k-1}f'(x_k) - x_{k}f'(x_{k-1})}{f'(x_k) - f'(x_{k-1})}
\label{eq:1dopt-secant-method}
\end{equation}
Notice that this recurrence relation requires two previous points (both $x_{k}$ and $x_{k-1}$) to calculate the next estimate.
This method converges superlinearly---slower than Newton's method, but faster than the golden section search---with convergence criteria similar to Newton's method.

\begin{problem} % Implement secant method.
Write a function that accepts a first derivative $f'$, starting points $x_0$ and $x_1$, a stopping tolerance \li{tol}, and a maximum of iterations \li{maxiter}.
Use \eqref{eq:1dopt-secant-method} to implement the Secant method.
Try to make as few computations as possible by only computing $f'(x_k)$ once for each $k$.
Return the minimizer approximation, whether or not the algorithm converged, and the number of iterations computed.

Test your code with the function $f(x) = x^2 + \sin(x) + \sin(10x)$ and with initial guesses of $x_0 = 0$ and $x_1 = -1$.
Plot your answer with the graph of the function.
Also compare your results to \li{scipy.optimize.newton()}; without providing the \li{fprime} argument, this function uses the secant method.
% Note that this function is highly sensitive to the starting point, which is why it is not as helpful to compare your function with SciPy's method.
However, it still only takes in one initial condition, so it may converge to a different local minimum than your function.

\begin{lstlisting}
>>> df = lambda x: 2*x + np.cos(x) + 10*np.cos(10*x)
>>> opt.newton(df, x0=0, tol=1e-10, maxiter=500)
-3.2149595174761636
\end{lstlisting}
\end{problem}

\section*{Descent Methods} % ==================================================

Consider now a function $f:\mathbb{R}^n\rightarrow\mathbb{R}$.
% Minimizing $f$ is a \emph{high-dimensional optimization problem} because the domain of $f$ is $\mathbb{R}^n$ instead of $\mathbb{R}$.
\emph{Descent methods}, also called \emph{line search methods}, are optimization algorithms that create a convergent sequence $(x_k)_{k=1}^\infty$ by the following rule.
\begin{equation}
\x_{k+1} = \x_k + \alpha_k \textbf{p}_k
\end{equation}
Here $\alpha_k \in \mathbb{R}$ is called the \emph{step size} and $\textbf{p}_k \in \mathbb{R}^n$ is called the \emph{search direction}.
The choice of $\textbf{p}_k$ is usually what distinguishes an algorithm;
in the one-dimensional case ($n = 1$), $p_k = f'(x_k)/f''(x_k)$ results in Newton's method, and using the approximation in \eqref{eq:1dopt-secant-approx} results in the secant method.

To be effective, a descent method must also use a good step size $\alpha_k$.
If $\alpha_k$  is too large, the method may repeatedly overstep the minimum; if $\alpha_k$ is too small, the method may converge extremely slowly.
See Figure \ref{fig:1dopt-overstep}.

\begin{figure}[H]
    \centering
    \includegraphics[width=.7\textwidth]{figures/large_alpha.pdf}
    \caption{If the step size $\alpha_k$ is too large, a descent method may repeatedly overstep the minimizer.}
    \label{fig:1dopt-overstep}
\end{figure}

Given a search direction $\textbf{p}_k$, the best step size $\alpha_k$ minimizes the function $\phi_k(\alpha) = f(\x_k + \alpha\textbf{p}_k)$.
Since $f$ is scalar-valued, $\phi_k:\mathbb{R}\rightarrow\mathbb{R}$, so any of the optimization methods discussed previously can be used to minimize $\phi_k$.
However, computing the best $\alpha_k$ at every iteration is not always practical.
 % especially since computing a good $\mathbf{p}_k$ is generally more critical.
Instead, some methods use a cheap routine to compute a step size that may not be optimal, but which is good enough.
% These methods do not seek to minimize $\phi_k(\alpha)$, but instead seek to sufficiently decrease it.
The most common approach is to find an $\alpha_k$ that satisfies the \emph{Wolfe conditions}:
\begin{align}
&f(\textbf{x}_k + \alpha_k \textbf{p}_k) \leq f(\textbf{x}_k) + c_1\alpha_k Df(\textbf{x}_k)\trp \textbf{p}_k
\label{eq:1dopt-wolfe-armijo}
\\
& - Df(\textbf{x}_k + \alpha_k \textbf{p}_k)\trp \textbf{p}_k \leq -c_2 Df(\textbf{x}_k)\trp \textbf{p}_k
\label{eq:1dopt-wolfe-curvature}
\end{align}
where $0 < c1 < c2 < 1$ (for the  best results, choose $c1 << c2$).
The condition \eqref{eq:1dopt-wolfe-armijo} is also called the \emph{Armijo rule} and ensures that the step decreases $f$.
However, this condition is not enough on its own.
By Taylor's theorem,
\[
f(\textbf{x}_k + \alpha_k \textbf{p}_k) = f(\textbf{x}_k) + \alpha_k Df(\textbf{x}_k)\trp \textbf{p}_k + \mathcal{O}(\alpha_k^2).
\]
Thus, a very small $\alpha_k$ will always satisfy \eqref{eq:1dopt-wolfe-armijo} since $Df(\textbf{x}_k)\trp\textbf{p}_k < 0$ (as $\textbf{p}_k$ is a descent direction).
The condition \eqref{eq:1dopt-wolfe-curvature}, called the \emph{curvature condition}, ensures that the $\alpha_k$ is large enough for the algorithm to make significant progress.

It is possible to find an $\alpha_k$ that satisfies the Wolfe conditions, but that is far from the minimizer of $\phi_k(\alpha)$.
The \emph{strong Wolfe conditions} modify \eqref{eq:1dopt-wolfe-curvature} to ensure that $\alpha_k$ is near the minimizer.
\[
| Df(\textbf{x}_k + \alpha_k \textbf{p}_k)\trp \textbf{p}_k| \leq c_2| Df(\textbf{x}_k)\trp \textbf{p}_k|
\]
The \emph{Armijo--Goldstein conditions} provide another alternative to \eqref{eq:1dopt-wolfe-curvature}:
\[
f(\textbf{x}_k) + (1 - c)\alpha_k Df(\textbf{x}_k)\trp \textbf{p}_k \leq f(\textbf{x}_k + \alpha_k\textbf{p}_k) \leq f(\textbf{x}_k) + c\alpha_k Df(\textbf{x}_k)\trp\textbf{p}_k,
\]
where $0 < c < 1$.
These conditions are very similar to the Wolfe conditions (the right inequality is \eqref{eq:1dopt-wolfe-armijo}), but they do not require the calculation of the directional derivative $ Df(\textbf{x}_k + \alpha_k \textbf{p}_k)\trp\textbf{p}_k$.
% They perform as well as the Wolfe conditions in most situations, but are not well-matched for quasi-Newton methods with positive definite Hessians. % ?

\subsubsection*{Backtracking}

A \emph{backtracking line search} is a simple strategy for choosing an acceptable step size $\alpha_k$: start with an fairly large initial step size $\alpha$, then repeatedly scale it down by a factor $\rho$ until the desired conditions are satisfied.
The following algorithm only requires $\alpha$ to satisfy \eqref{eq:1dopt-wolfe-armijo}.
This is usually sufficient, but if it finds $\alpha$'s that are too small, the algorithm can be modified to satisfy \eqref{eq:1dopt-wolfe-curvature} or one of its variants.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{backtracking}{$f$,\ $Df$,\ $\x_k$,\ $\mathbf{p}_k$,\ $\alpha$,\ $\rho$,\ $c$}
    \State \texttt{Dfp} $\gets Df(\x_k)\trp\mathbf{p}_k$
        \Comment{Compute these values only once.}
    \State \texttt{fx} $\gets f(\x_k)$
    \While{$\big(f(\x_k + \alpha\mathbf{p}_k) >$ \texttt{fx} $+\ c\alpha$\texttt{Dfp}$\big)$}
        \State $\alpha \gets \rho\alpha$
    \EndWhile
    \Return $\alpha$
\EndProcedure
\end{algorithmic}
\caption{Backtracking using the Armijo Rule}
\label{Alg:opt1d-backtracking}
\end{algorithm}

\begin{problem}
Write a function that accepts a function $f:\mathbb{R}^n\rightarrow\mathbb{R}$, its derivative $Df:\mathbb{R}^n\rightarrow\mathbb{R}^n$, an approximate minimizer $\x_k$, a search direction $\textbf{p}_k$, an initial step length $\alpha$, and parameters $\rho$ and $c$.
Implement the backtracking method of Algorithm \ref{Alg:opt1d-backtracking}.
Return the computed step size.

The functions $f$ and $Df$ should both accept 1-D NumPy arrays of length $n$.
For example, if $f(x,y,z) = x^2 + y^2 + z^2$, then $f$ and $Df$ could be defined as follows.
\begin{lstlisting}
>>> f = lambda x: x[0]**2 + x[1]**2 + x[2]**2
>>> Df = lambda x: np.array([2*x[0], 2*x[1], 2*x[2]])
\end{lstlisting}

SciPy's \li{scipy.optimize.linesearch.scalar_search_armijo()} finds an acceptable step size using the Armijo rule.
It may not give the exact answer as your implementation since it decreases $\alpha$ differently, but the answers should be similar.
\begin{lstlisting}
>>> from scipy.optimize import linesearch
>>> from autograd import numpy as anp
>>> from autograd import grad

# Get a step size for f(x,y,z) = x^2 + y^2 + z^2.
>>> f = lambda x: x[0]**2 + x[1]**2 + x[2]**2
>>> x = anp.array([150., .03, 40.])         # Current minimizer guesss.
>>> p = anp.array([-.5, -100., -4.5])       # Current search direction.
>>> phi = lambda alpha: f(x + alpha*p)      # Define phi(alpha).
>>> dphi = grad(phi)
>>> alpha, _ = linesearch.scalar_search_armijo(phi, phi(0.), dphi(0.))
\end{lstlisting}
\end{problem}

\begin{comment} % This is a little too confusing and needs revision if it's going to be in print.

\newpage

\section*{Additional Material} % ==============================================

\subsection*{Golden Search Derivations} % -------------------------------------

The ratio of the lengths of $[a, \tilde{a}]$ and $[\tilde{a}, b]$ is the same as the ratio between the lengths of $[\tilde{a}, b]$ and $[a, b]$ as follows:
\begin{align*}
\frac{\tilde{a}-a}{b-\tilde{a}} &= \frac{(b-a)(1-\frac{1}{\varphi})}{b-b+\frac{b-a}{\varphi}} \\
&= \varphi(1-\frac{1}{\varphi}) \\
\frac{b-\tilde{a}}{b-a} &= \frac{1}{\varphi}
\end{align*}
As one of the properties of the golden ratio states that $\frac{1}{\varphi^2} = 1 - \frac{1}{\varphi}$, they are equal.

Chosing the test points according to the golden ratio saves on computations.
For example, consider the case where $f(\tilde{a}) > f(\tilde{b})$ and label $a_0 = a$ and $a_1 = \tilde{a}$.
Thus, $x^* \in [\tilde{a}, b]$ and for the next iteration
\begin{align*}
\tilde{a_1} &= b - \frac{b - a_1}{\varphi} \\
&= b - \frac{b - b + \frac{b - a_0}{\varphi}}{\varphi}\\
&= b - \frac{b - a_0}{\varphi^2}\\
&= a_0 + \frac{b - a_0}{\varphi}\\
&= \tilde{b}
\end{align*}
Therefore, the next $\tilde{a}$ is the previous $\tilde{b}$.
Similarly, if $f(\tilde{a}) < f(\tilde{b})$, then the next $\tilde{b}$ will be the previous $\tilde{a}$.
\end{comment}


\subsection{G{\"o}del’s completeness theorem}

\subsubsection{Completeness of first-order logic}

We previously showed that zero-order logic was complete. What about first-order logic?

G{\"o}del’s’ completeness theorem says that for first order logic, a theory can include all tautologies, the first category.

If the completeness theorem is true and a formula is not in the theory, then the formula is either refutable or satisfiable under some, but not all interpretations.

That is, either the theory will contain \(\theta \), \(\neg \theta \), or \(\theta \) will be satisfiable in some but not all interpretations, and neither will be in the in theory.

To prove this we look for a proof that every formula is either refutable or true under some structure. So for an arbitrary formula \(\theta \) we want to show it is either refutable or satisfiable under some interpretation.

\subsubsection{Part 1: Converting the form of the formula}

Remove free variables, functions

Note that if this is true, all valid formulae of the form below are provable:

\(\neg \theta \)

This means that there is no interpretation where the following is true:

\(\theta \)

Conversely if \(\neg \theta \) is not in the theory, then \(\theta \) must be true under some interpretation.

That is, if all valid formulae are provable, then all

Reformulating the question:

This is the most basic form of the completeness theorem. We immediately restate it in a form more convenient for our purposes:

Theorem 2. Every formula \(\theta \) is either refutable or satisfiable in some structure.

"\(\theta \)is refutable" means by definition "\(\neg \theta \) is provable".

\subsubsection{Decidability}

Given a formula, can we find out if can be derived from the axioms? We can follow a process for doing so which would inform us if the formula was or was not a theorem. Alternative, the process could carry on forever.

If the process never carries on forever the system is decidable: there is a finite process to determine whether the formula is in or out. If the process halts for true formulas, but can carry on forever for false formulas, the system is semi-decidable. If the process takes a long time, we do not know if it is looping infinitely, or approaching its halt point.

Intuitively, use of axioms can make an existing formula shorter or longer, so finding all short formulas can require going forwards and backwards an infinite number of times.


\section{Gaussian Mixture Model}%
\label{sec:l2hmc_gmm}
%
The Gaussian Mixture Model (GMM) is a notoriously difficult example for
traditional HMC to sample accurately due to the existence of multiple modes.
%
In particular, HMC cannot mix between modes that are reasonably separated
without recourse to additional tricks.
%
This is due, in part, to the fact that HMC cannot easily traverse the
low-density zones which exist between modes.

In the most general case, we consider a target distribution described by a
mixture of $M > 1$ components in
$\mathbb{R}^{D}$ for $D \geq 1$:
%
\begin{equation}
    p(\mathbf{x}) \equiv \sum_{m=1}^{M} p(m) p(\mathbf{x}|m) \equiv
        \sum_{m=1}^{M} \pi_m p(\mathbf{x}|m) \quad \forall \,\,\mathbf{x} \in
        \mathbb{R}^{D}
    \label{eq:gmm_model}
\end{equation}
%
where $\sum_{m=1}^{M} \pi_m = 1$, $\pi_M \in (0, 1)$ $\forall m = 1, \ldots, M$
and each component distribution is a normal probability distribution in
$\mathbb{R}^{D}$.
%
So $\mathbf{x}|m \sim \mathcal{N}(\bm{\mu}_m, \bm{\Sigma}_m)$, where
$\bm{\mu}_m \equiv \mathbb{E}_{p{(\mathbf{x}|m)}}\left\{\mathbf{x}\right\}$ and
$\mathbf{\Sigma}_m \equiv \mathbb{E}_{p{(\mathbf{x}|m)}}{\left\{{(\mathbf{x} -
\bm{\mu}_m)}{(\mathbf{x} - \bm{\mu}_m)}^{T}\right\}} > 0$ are the mean vector
and covariance matrix, respectively, of component $m$.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Example}
%
Consider a simple 2D case consisting of two Gaussians
%
\begin{equation}
    \mathbf{x} \sim \pi_1 \,\mathcal{N}(\bm{\mu}_1, \bm{\Sigma}_1) +
        \pi_2\, \mathcal{N}(\bm{\mu}_2, \bm{\Sigma}_2)
    \label{eq:log_likelihood_example}
\end{equation}
%
with $\pi_1 = \pi_2 = 0.5$, $\bm{\mu}_1 = (-2, 0)$, $\bm{\mu}_2 = (2, 0)$ and
%
\begin{equation}
    \bm{\Sigma}_1 = \bm{\Sigma}_2 =
        \begin{bmatrix}
            0.1    & 0 \\
            0       & 0.1
        \end{bmatrix}
    \label{eq:covariance_matrix}
\end{equation}
%
The results of trajectories generated using both traditional HMC and the L2HMC
algorithm can be seen in Fig.~\ref{fig:gmm_trajectories}.
a
Note that traditional HMC performs poorly and is unable to mix between the two
modes, whereas L2HMC is able to correctly sample from the target distribution
without getting stuck in either of the individual modes.

%
The L2HMC sampler was trained using simulated annealing using the schedule
shown in Eq~\ref{eq:gmm_annealing} with a starting temperature of $T = 10$, for
$5,000$ training steps.
%
By starting with a high temperature, the chain is able to move between both
modes (`tunnel') successfully.
%
Once it has learned this, we can lower the temperature back to $T = 1$ and
recover the initial distribution while preserving information about tunneling
in the networks ``memory''.
%
\begin{equation}
  T(n) = {\left(T_{i} - T_{f}\right)} \cdot {\left(1 -
  \frac{n}{N_{\mathrm{train}}}\right)} + T_{f}
\label{eq:gmm_annealing}
\end{equation}
%
\begin{figure}[htbp]
    \centering
    \includegraphics[width=\textwidth]{gmm_figures/iso_gmm_chains1}
    \caption{Comparison of trajectories generated using L2HMC (top), and
        traditional HMC with $\eps = 0.25$ (middle) and $\eps = 0.5$ (bottom).
        Note that L2HMC is able to successfully mix between modes, whereas HMC
        is not.}%
\label{fig:gmm_trajectories}
\end{figure}
%
\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.98\textwidth]{gmm_figures/gmm_acl}
    \caption{Autocorrelation vs.\ gradient evaluations (i.e.\ MD steps). Note
    that L2HMC (blue) has a significantly reduced autocorrelation after the
  same number of gradient evaluations when compared to either of the two HMC
trajectories}%
\label{fig:gmm_autocorrelation}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%


\chapter{Completions}
	\setlist[enumerate,1,2]{leftmargin=1.8cm}
	In this chapter we are intended to do \textit{elementary calculus} with a heavy taste of algebra instead of analysis. A lot of concepts in analysis are stolen by algebraists with different generalisations.
	\section{Absolute value}
		Let $K$ be a field. An \textbf{absolute value} on $K$ is a function $K \to \mathbb{R}$, $x \mapsto |x|_v$ satisfying the following three properties:
		\begin{enumerate}[start=1,label={\bfseries AV \arabic*}]
			\item We have $|x|_v \ge 0$ and $=0$ if and only if $x = 0$.
			\item For all $x,y \in K$ we have $|xy|_v = |x|_v|y|_v$.
			\item $|x+y|_v \le |x|_v + |y|_v$. \label{av3}
		\end{enumerate}

		If instead of \ref{av3} the absolute value satisfies the stronger condition
		\begin{enumerate}[start=4,label={\bfseries AV \arabic*}]
			\item $|x+y|_v \le \max(|x|_v,|y|_v)$
		\end{enumerate}
		then we shall say that it is a \textbf{valuation} or that it is non-archimedean. in particular, if the absolute value $|\cdot|$ satisfy $|x|=1$ for all $x \ne 0$, then we say $|\cdot|$ is \textbf{trivial}. We will exclude this case from now on.


\subsection{Describing events}

In vector space \(\mathbb{R}^n\).

\(\mathbf{q} \in \mathbb{R}^n\)

\subsection{Describing the path of a particle}

Also known as a worldline.

Index to \(t\)

\(\mathbf{q} (t)\)

\subsection{Describing the velocity of a particle}

\(v=\dfrac{\delta \mathbf{q}}{\delta t}\)

\subsection{Describing the acceleration of a particle}

\(a=\dfrac{\delta v}{\delta t}\)

\(a=\dfrac{\delta^2 \mathbf{q}}{\delta t^2}\)


\documentclass{article}
\usepackage{amsmath}
\usepackage{amsfonts}

\author{a1trl9}
\title{Machine Learning Review}
\date{}

\setlength\parindent{0pt}
\counterwithin*{equation}{section}

\newcommand{\V}[1]{\boldsymbol{#1}}
\newcommand{\TM}{\bigtriangledown}

\begin{document}
\maketitle
\section{Regression}
\subsection{Linear Regression}
\subsection{Logistic Regression}
\subsubsection{Why Logistic Regression is a Linear Classifier}

Noting:
\begin{equation}
P(y_i=1)=\frac{1}{exp(-\V{w'}\V{x_i})+1}
\end{equation}

If we set \(p=0.5\) as the threshold, then:
\begin{equation}
\left\{
\begin{aligned}
&\hat{y}=1 \quad if \quad \V{w'}\V{x}>0\\
&\hat{y}=0 \quad if \quad \V{w'}\V{x}\leq0
\end{aligned}
\right.
\end{equation}

It yields that if all training data are linearly separable, logistic regression
may lead to over-fitting. Intuitively, when \(\V{w}\) subjects to all data are
classified correctly, scaling \(\V{w}\) up can always increase likelihood and when
\(\V{w} \to \infty\), \(p(y_i)=1\) for all data points.

\subsubsection{The convexity of Negative Log-likelihood Function}

Since:
\begin{equation}
\left\{
\begin{aligned}
&P(y_i=1)=\frac{1}{exp(-\V{w'}\V{x_i})+1}\\
&P(y_i=0)=\frac{1}{exp(\V{w'}\V{x_i})+1}
\end{aligned}
\right.
\end{equation}
Where \(\V{w}\) is the coefficient vector while \(\V{x_i}\) is the feature vector.
Assuming the dimension is \(m\).

If we transform the label set from \(\{1, 0\}\) to \(\{1, -1\}\), then
\begin{equation}
p(y_i)=\frac{1}{exp(-y_i\V{w'}\V{x_i})+1}
\end{equation}

Let \(z=-y_i\V{w'}\V{x_i}\), then the negative log-likelihood function is:
\begin{equation}
-L=-\sum_{i}^{N}log[\frac{1}{exp(z)+1}]
\end{equation}

Here we assume \(L\) is \textbf{twice continuously differentiable}.

Let \(g(z)=\frac{1}{exp(z)+1}\), then:
\begin{equation}
g'(z)=-\frac{exp(z)}{[exp(z)+1]^2}=-g(z)[1-g(z)]
\end{equation}

Therefore:
\begin{equation}
\begin{aligned}
\frac{d(-L)}{dw_j}&=-\sum_{i}^N\frac{1}{g(z)}g(z)[1-g(z)]y_ix_{ij}\\
&=-\sum_{i}^Ny_ix_{ij}[1-g(z)]
\end{aligned}
\end{equation}

And:
\begin{equation}
\begin{aligned}
\frac{d(-L)}{dw_jdw_k}&=\sum_{i}^Ny_i^2x_{ij}x_{ik}g(z)[1-g(z)]\\
&=\sum_{i}^Nx_{ij}x_{ik}g(z)[1-g(z)]
\end{aligned}
\end{equation}

Let \(\V{a}\) be any m-dimension vector. And \(\TM^2\) is the Hessian matrix
of \(-L\). Then:

\begin{equation}
\begin{aligned}
\V{a'}\TM^2\V{a}&=\sum_{i}^N\sum_{j}^m\sum_{k}^ma_ja_kx_{ij}x_{ik}g(z)[1-g(z)]\\
&=\sum_{i}^N{\{\sqrt{g(z)[1-g(z)]}\V{a'}\V{x_i}\}^2}\\
&\geq 0
\end{aligned}
\end{equation}

So \(\TM^2\) is a positive semi-definite matrix, and \(-L\) is Therefore
a convex function.

\vspace{2mm}
\textbf{Further}
\begin{itemize}
\item It can be proved, that for any identifiable model from exponential family,
the negative log-likelihood function is strictly convex and therefore
the maximum likelihood estimate is unique \textbf{if it exists}. When model
is not identifiable (happens when variables are not independent), only
convex holds.
\item When \(x_1, x_2, \cdots, x_N\) are not linearly independent, it is possible
that \(\V{a'}\TM\V{a}=0\), in which case, \(-L\) is not strictly convex and
the minimum might not be unique.
\item If we add L2 regularizer, then \(-L\) will be strictly convex.
\end{itemize}

\subsubsection{*Convexity \& Concavity and Hessian Matrix}

Suppose \(f: V \rightarrow \mathbb{R} \) is a twice differentiable function. \(V\) is
a convex set on \(\mathbb{R}^n\). If \(f\) is not convex, then \(\exists (\V{x_1},
\V{x_2}) \in V, \exists t \in (0, 1) \), \(f((1-t)\V{x_1} + t\V{x_2}) > (1-t)f(\V{x_1})+tf(\V{x_2})\).
Let \(\theta: (0, 1) \rightarrow R: \theta(k)=f((1-k)\V{x_1}+k\V{x_2})\). Then
\begin{equation}
\theta(t)=f((1-t)\V{x_1}+t\V{x_2}) > (1-t)f(\V{x_1})+tf(\V{x_2})=(1-t)\theta(0)+t\theta(1)\
\end{equation}

Therefore:
\begin{equation}
\left\{
\begin{aligned}
&\theta(t)-\theta(0)>t(\theta(1)-\theta(0))\\
&\theta(1)-\theta(t)<(1-t)(\theta(1)-\theta(0))
\end{aligned}
\right.
\end{equation}

According to Mean Value Theorem, \(\exists t_1 \in (0, t), t_2 \in (t, 1)\),
subject to \(\frac{\theta}{dt}(t_1)t=\theta(t)-\theta(0),
\frac{\theta}{dt}(t_2)(1-t)=\theta(1)-\theta(t)\). So:

\begin{equation}
\left\{
\begin{aligned}
&\frac{\theta}{dt}(t_1)>\theta(1)-\theta(0)\\
&\frac{\theta}{dt}(t_2)<\theta(1)-\theta(0)
\end{aligned}
\right.
\end{equation}

It further yields:

\begin{equation}
\frac{\theta}{dt}(t_1)>\frac{\theta}{dt}(t_2)
\end{equation}

Apply Mean Value Theorem again:
\begin{equation}
\frac{\theta}{dt}(t_2)-\frac{\theta}{dt}(t_1)=(t_2-t_1)\frac{d^2\theta}{dt^2}(t_3)
\end{equation}

Where \(t_3 \in (t_1, t_2)\). And obviously \(\frac{d^2\theta}{dt^2}(t_3)<0\).

As we know \(\frac{d^2\theta}{dt^2}=(\V{x_2}-\V{x_1})'d^2f(\V{x_2}-\V{x_1})\),
where \(d^f\) is the Hessian matrix of \(f\). When it is positive semi-definite,
\(\V{x_2}-\V{x_1})'d^2f(\V{x_2}-\V{x_1})\geq 0\), which is contradict to
\(\frac{d^2\theta}{dt^2}(t_3)<0\).

Therefore, when the Hessian matrix of \(f\) is positive semi-definite, \(f\) is
convex. Similarly, if the Hessian matrix of \(f\) is negative semi-definite,
\(f\) is concave.

\section{Regularisation}
\subsection{Transformation for Linear Regression}
Without regularisation:
\begin{equation}
L=||\mathbf{X}\V{w}-\V{y}||^2
\end{equation}

With regularisation:
\begin{equation}
L=||\mathbf{X}\V{w}-\V{y}||^2+\lambda||\V{w}||^2
\end{equation}

Where \(\lambda\) controls the strength of regularisation. Now we try to solve:
\begin{equation}
\begin{aligned}
&2\mathbf{X}'(\mathbf{X}\V{w}-\V{y})+2\lambda \V{w}=\V{0}\\
&\implies (\mathbf{X}'\mathbf{X}+\lambda \mathbf{I})\V{w}=\mathbf{X}'\V{y}\\
&\implies \V{w}=(\mathbf{X}'\mathbf{X}+\lambda \mathbf{I})'\mathbf{X}'\V{y}
\end{aligned}
\end{equation}

As we know \(\mathbf{A}=\mathbf{X}'\mathbf{X}\) is a symmetric matrix and for any non-zero vector \(V{v}\)
(same dimension as \(\mathbf{A}\)):
\begin{equation}
\V{v'}\mathbf{A}\V{v}=\V{v'}\mathbf{X}'\mathbf{X}\V{v}=(\mathbf{X}\V{v})'\mathbf{X}\V{v}\geq 0
\end{equation}

Then \(\mathbf{A}\) is a positive semi-definite matrix. Besides, consider
any eigenvector / eigenvalue pair \(\V{p}, t\) for \(\mathbf{A}\):
\begin{equation}
\begin{split}
(\mathbf{A}+\lambda\mathbf{I})\V{p}=\mathbf{A}\V{p}+\lambda\V{P}
=t\V{p}+\lambda\V{p}=(t+\lambda)\V{p}
\end{split}
\end{equation}

Therefore, for all eigenvalues \(t_1, t_2, \cdots, t_n\) for \(\mathbf{A}\),
\(t_1+\lambda, t_2+\lambda, \cdots, t_n+\lambda\) is eigenvalues for
\(\mathbf{A}+\lambda\mathbf{I}\). Since \(t_1, t_2, \cdots, t_n\) have all been
non-negative as \(\mathbf{A}\) is positive semi-definite, and \(\lambda>0\), \(t_1+\lambda, t_2+\lambda, \cdots, t_n+\lambda\)
are all positive. As the multiple of \(n\) eigenvalues is equal to the
determinant of \(\mathbf{A}\), \(det(\mathbf{A})\neq 0\) and \(rank(\mathbf{A})=n\),
\(\mathbf{X}'\mathbf{X}+\lambda\mathbf{I}\) must be invertable.

\subsection{Transformation for Logistic Regression}

Without regularisation (ignoring intercept):
\begin{equation}
\begin{aligned}
&L=-\sum_{i}^my_ilog(\sigma(\V{x}_i\V{w}))+(1-y_i)log(1-\sigma(\V{x}_i\V{w}))\\
&\implies
\frac{dL}{\V{w}}=-\sum_{i}^m(y_i(1-\sigma(\V{x}_i))-(1-y_i)\sigma(\V{x}_i\V{w})))\V{x}'_i\\
&=\sum_{i}^m(\sigma(\V{x}_i\V{w})-y_i)\V{x}_i'\\
\end{aligned}
\end{equation}

With regularisation:
\begin{equation}
\begin{aligned}
&\frac{dL}{\V{w}}=-\sum_{i}^m(y_i(1-\sigma(\V{x}_i))-(1-y_i)\sigma(\V{x}_i\V{w})))\V{x}'_i+\lambda\V{w}\\
&=\sum_{i}^m(\sigma(\V{x}_i\V{w})-y_i)\V{x}_i'+\lambda\V{w}\\
\end{aligned}
\end{equation}

\section{Perceptron}
\subsection{Perceptron Convergence}
Assuming the training data (\(N\) points) are linearly separable, in which case,
\(\exists \V{v}: ||\V{v}||=1\), subjects to \(y_i\V{v'}\V{x_i}>\gamma\), for
\(i=1, 2, \cdots N\).
Noting at each gradient step, the update is in the form of:

\begin{equation}
\V{w}^{t+1}=\V{w}^t+\eta y_j\V{x_j}
\end{equation}

Where \(<y_j, \V{x_j}>\) is the pair of predicted class \& selected misclassified point.

As we know:
\begin{equation}
||y_k\eta\V{x_k} +\V{w}^{k-1}||^2=||y_k\eta\V{x_k}||^2+||\V{w}^{k-1}||^2+y_k\eta\V{x_k}\cdot \V{w}^{k-1}
\end{equation}

Moreover, \(y_k\V{x}\V{w}^{k-1}\) is always negative since \(\V{x_k}\) is a
misclassified point. Therefore:

\begin{equation}
\begin{aligned}
||\V{w}^k||^2&=||y_k\eta\V{x_k} +\V{w}^{k-1}||^2<||y_k\eta \V{x_k}||^2+||\V{w}^{k-1}||^2\\
&\leq (\eta R)^2 + ||\V{w}^{k-1}||^2
\end{aligned}
\end{equation}

Iteratively, we get:

\begin{equation}
||\V{w}^k||^2<||\V{w}^0||^2+k(\eta R)^2
\end{equation}

Meanwhile:
\begin{equation}
\begin{aligned}
\sum_{i=1}^k|\V{v'}\cdot (\V{w}^i-\V{w}^{i-1})|&=\V{v'}\cdot\sum_{i=1}^k(\V{w}^i-\V{w}^{i-1})\\
&=\V{v'}\cdot (\V{w}^k-\V{w}^0)\\
&\leq ||\V{v'}||||\V{w}^k-\V{w}^0||\\
&=||\V{w}^k-\V{w}^0||\\
&= \sqrt{||\V{w}^k||^2 + ||\V{w}^0||^2 - \V{w}^k \cdot \V{w}^0}\\
&\leq \sqrt{||\V{w}^k||^2+||\V{w}^0||^2+||\V{w}^k||||\V{w}^0||}
\end{aligned}
\end{equation}

In addition, since:
\begin{equation}
\begin{aligned}
\sum_{i=1}^k|\V{v'}\cdot (\V{w}^i-\V{w}^{i-1})|&=\sum_{i=1}^k|\V{v'}\cdot (y_i\eta\V{x_i}')|\\
&\geq ky
\end{aligned}
\end{equation}

We finally get:
\begin{equation}
\begin{aligned}
k\gamma &\leq \sum_{i=1}^k|\V{v'}\cdot (\V{w}^i-\V{w}^{i-1})|\\
&\leq \sqrt{||\V{w}^k||^2+||\V{w}^0||^2+||\V{w}^k||||\V{w}^0||}\\
&\leq\sqrt{||\V{w}^0||^2+k(\eta R)^2+||\V{w}^0||^2+||\V{w}^0||\sqrt{||\V{w}^0||^2+k(\eta R)^2}}
\end{aligned}
\end{equation}

Here, we treat \(\V{w}^0\) as constant and therefore:

\begin{equation}
k\gamma < \sqrt{c_1}\sqrt{k(\eta R)^2}
\end{equation}

So \(k<O((\frac{\eta R}{\gamma})^2)\)

\section{Neural Network}
\subsection{Local Quadratic Approximation}
Consider the Taylor expansion of the error function \(E(\V{w})\) around
a point \(\hat{\V{w}}\) in the weight space:
\begin{equation}
E(\V{w})=E(\hat{\V{w}})+(\V{w}-\hat{\V{w}})'\V{b}
+\frac{1}{2}\hat{\V{w}}'\V{H}\hat{\V{w}}
\end{equation}

Where \(\V{H}\) is the Hessian matrix at \(\hat{\V{w}}\).

First, let \(V\) be the n-dimension linear space for the linear
transformation \(H\). As \(\V{H}\) is symmetric, we can construct \(n\) orthonormal eigenvectors.

\vspace{2mm}
\textbf{Proof}

Let \(\lambda_1, \lambda_2\) be two eigenvalues of \(\V{H}\), so we have:
\begin{equation}
\begin{split}
&\V{H}\V{p_1}=\lambda_1\V{p_1}\\
&\V{H}\V{p_2}=\lambda_1\V{p_2}\\
\end{split}
\end{equation}

So:
\begin{equation}
\begin{split}
(\lambda_1\V{p_1})'\V{p_2}=(\V{H}\V{p_1})'\V{p_2}
=\V{p_1}'\V{H}'\V{p_2}
\end{split}
\end{equation}

As \(\V{H}\) is symmetric, \(\V{H}'=\V{H}\):
\begin{equation}
\lambda_1\V{p_1}\V{p_2}=
\V{p_1}'\V{H}'\V{p_2}=\V{p_1}'\V{H}\V{p_2}=\V{p_1}'\lambda_2\V{p_2}
=\lambda_2\V{p_1}\V{p_2}
\end{equation}

Now consider two cases:

First,
\(\lambda_1, \lambda_2\) are distinct eigenvalues, in this case
\(\V{p_1}, \V{p_2}\) are orthogonal.

\vspace{2mm}
Second, if \(\lambda_1, \lambda_2\) are the same eigenvectors. Then
build a pair of orthogonal eigenvectors \(\alpha_1\V{p_1} + \beta_1\V{p_1}\)
and \(\alpha_2\V{p_1}+\beta_2\V{p_2}\) in the eigenvector space
\(\{\V{a}|\V{H}\V{a}=\lambda_1\V{a}\}\). Since \(\V{H}\) is diagonalizable,
direct sum of eigenvector spaces equals to \(V\), the linear representation of \(\alpha_1\V{p_1} + \beta_1\V{p_1}\)
and \(\alpha_2\V{p_1}+\beta_2\V{p_2}\) are unique and therefore they are
linearly independent
with eigenvectors paired with eigenvalues \(\lambda\neq \lambda_1\). Combining
all these eigenvectors could finally construct an \(n\times n\) orthogonal eigenvector matrix

\vspace{2mm}
For both cases, we could then normalize the eigenvector matrix.

We can use these \(n\) orthonormal eigenvectors as a basis in the
linear space \(V\). And present \(\hat{\V{w}}\) as
\(\sum_{i=1}^n\alpha_i\V{p}_i\) where \(\V{p}_i\) is the eigenvector.
Now:
\begin{equation}
\begin{split}
\hat{\V{w}}'\V{H}\hat{\V{w}}
&=(\sum_{i=1}^n\alpha_i\V{p}_i)'\V{H}(\sum_{i=1}^n\alpha_i\V{p}_i)\\
&=(\sum_{i=1}^n\alpha_i\V{p}_i)'\sum_{i=1}^n
\alpha_i\V{H}\V{p_i}\\
&=(\sum_{i=1}^n\alpha_i\V{p}_i)'\sum_{i=1}^n\alpha_i\lambda_i\V{p_i}\\
&=\sum_{i=1}^n\sum_{j=1}^n\alpha_i\alpha_j\lambda_j\V{p}_i'\V{p}_j\\
&=\sum_{i=1}^n\alpha_i^2\lambda_i\V||{p}_i||^2\\
&=\sum_{i=1}^n\lambda_i\alpha_i^2
\end{split}
\end{equation}

\subsection{Numerical Solution -- Finite Difference}
\subsubsection{Why central difference is more precise?}
For one-side difference, with Taylor expansion:
\begin{equation}
f(x+\epsilon)=f(x)+\epsilon f'(x)+o(\epsilon)
\end{equation}
So:
\begin{equation}
\frac{f(x+\epsilon)-f(x)}{\epsilon}=f'(x)+o(\epsilon)
\end{equation}

For central difference, with Taylor expansion:
\begin{equation}
\begin{split}
&f(x+\epsilon)=f(x)+\epsilon f'(x)+\epsilon^2\frac{f''(x)}{2!}
+o(\epsilon^2)\\
&f(x-\epsilon)=f(x)-\epsilon f'(x)+\epsilon^2\frac{f''(x)}{2!}
+o(\epsilon^2)
\end{split}
\end{equation}
Therefore:
\begin{equation}
\begin{split}
&f(x+\epsilon)-f(x-\epsilon)=2\epsilon f'(x)+o(\epsilon^2)\\
&\implies \frac{f(x+\epsilon)-f(x-\epsilon)}{2\epsilon}=f'(x)+o(\epsilon^2)
\end{split}
\end{equation}

\end{document}


% !TeX spellcheck = en_US
\subsection{Direct encoding}
As the name suggests, the direct encoding assign for each CSP variable $x$ and for each possible value from its domain $i$ a new propositional variable $x_i$ on the SAT side. The boolean variable is then True if and only if the original variable is assigned the value $i$.  To ensure consistent logical assignment for each propositional variable, we need to extend the SAT problem to include clauses that avoid inconsistent results such as assigning a CSP variable two values at the same time or no value at all (at-least-once and at-most-once clauses) \cite{petke2011order}.

\subsubsection{Example}
Consider the following instance of the map coloring problem:\\
\begin{figure}[H]
	\centering
	\includegraphics[width=0.5\linewidth]{assets/map_coloring_unsolved}
	\captionsetup{justification=centering,margin=2cm}
	\caption{Unsolved instance of the map coloring problem}
	\label{fig:map_coloring_unsolved}
\end{figure}
The map consists of four regions $A,B,C,D$ sharing boarder as shown in the figure \ref{fig:map_coloring_unsolved}. The purpose of this problem is to color each region of the map so that no two adjacent regions have the same color. The variables in this case are the regions themselves $\{A,B,C,D\}$ and each of them has the domain $\{r,g,b\}$ which stands for the colors red, green and blue respectively. To fully define the CSP we also need to define the constraints' set which comprise two types of constraints:
\begin{itemize}
	\item unary constraints such as $A\neq r$ since $A$ is already adjacent to a red colored region
	\item binary constraints such as $A\neq B, A\neq C,\dots$ i.e. no two adjacent regions share the same color
\end{itemize}
Hereafter, the unary constraints are avoided by restricting the domain of $A$ and $D$ to not include red; on the other hand, it should be noted that usually all the variables are given the same domain to simplify the problem.

Encoding this problem directly will result in the following SAT instance:
a new propositional variable is associated with each value that can be assigned to a CSP variable. So we get the variables: $\{ A_g, A_b, B_r, B_g,\\ B_b, \dots\}$. To ensure that each CSP variable will be assigned at least one color, we have to include the following at-least-once (ALO) clauses: $\{ A_g \vee A_b, B_g \vee B_r \vee B_b, \dots \}$. Similarly, we must ensure that no CSP variable is assigned multiple colors at the same time, which can be expressed as these at-most-once (AMO) clauses: $\{ A_g \oplus A_b, B_g \oplus B_r, B_g \oplus B_b, B_r \oplus B_b, \dots \} $, where $\oplus$ stands for the logical exclusive or operation $a \oplus b \equiv (a \wedge \neg b) \vee (\neg a \wedge b) \equiv (a \vee b) \wedge \neg (a \wedge b)$.
\begin{figure}[H]
	\centering
	\includegraphics[width=0.75\linewidth]{assets/alo_amo}
	\captionsetup{justification=centering,margin=2cm}
	\caption{Both types of clauses in direct encoded SAT}
	\label{fig:alo_amo}
\end{figure}
ALO and AMO clauses will enforce each CSP variable to be true only for one value from its domain, hence the name exactly-once (EO) clauses.

We still need to encode the binary constraints from CSP into some sort of clauses in SAT. This can be done easily by ensuring that no two CSP variables such as $C$ and $B$ share the same color, which can be translated into the following clauses:
$$
	\neg (C_r \wedge B_r) \quad, \neg (C_g \wedge B_g) \quad, \neg (C_b \wedge B_b)
$$
The same principle applies for the reset.
%\textcolor{red}{TODO: add a word about the similar support encoding (since both are sparse encodings)}

The direct encoding can be categorized under a general type called the sparse encoding. Notice that the direct encoding encodes conflict points (so called conflict or no-good) assignments such as $A$ and $B$ can't share the same color at the same time $\neg (A_c \wedge B_c) \equiv \neg A_c \vee \neg B_c$. It is also possible to encode the allowed (so called supports) assignments, e.g. $\neg A_c \vee B_c \vee C_c \vee D_c$. This type of clauses is the basis for the other type of sparse encoding called accordingly the support encoding \cite{petke2011order}.

\subsubsection{Proposition}
Solving the CSP problem directly with FC compared to the direct encoded SAT instance of the same CSP problem solved with DPLL will always result in similar search tree structure. i.e. FC on the original CSP problem and DPLL on the direct encoded SAT will explore the same number of branches. This proposition assumes equivalent branching heuristics for DPLL and FC.

\subsubsection{Proof idea}\label{subsec:direct_encoding_proof}
Considering the generated search tree, our version of DPLL comprise only two rules: the one literal and the branching rule. This means at each node in the tree we either have to branch or the given variable must be assigned a truth value of either False or True.

On the FC side branching is equivalent to its counterpart in DPLL. The second case in DPLL (unit propagation by the one literal rule) is also equivalent to applying forward checking by enforcing arc consistency and reducing the domain of each CSP variable to one value only.

Similarly, we can trace the whole structure of both trees and process with the same logic on each propositional variable. By induction, we can prove that each algorithm explore the same number of branches \cite{walsh2000sat}.

\subsubsection{Example}
Compare the following search trees of FC applied to the previous map coloring problem and DPLL applied to direct encoded instance of the same previous map-coloring problem:
\begin{figure}[H]
	\centering
	\includegraphics[width=1\linewidth]{assets/direct_fc_vs_dpll}
	\captionsetup{justification=centering,margin=2cm}
	\caption{Search trees of FC and DPLL}
	\label{fig:direct_fc_vs_dpll}
\end{figure}
The arrows in node level on the FC side indicates enforcing arc consistency and the results are show directly underneath on the next line.


%!TEX root =  ../main.tex

\objective{Compose and cecompose sums of power functions}

\subsection{Curvilinear Asymptotes}
We said last section that a rational function where the degree of the numerator is greater than
the degree of the denominator will have an asymptote defined by the quotient of the two,
ignoring the remainder.  Let us see an example.

$$
f(x)=\frac{(x+1)(x-2)(x-5)(x+5)}{(x-1)(x+2)}
$$

We can see that the first and last terms of the top will be $x^4 \dots +50$.  The bottom
multiplies out to$x^2+x-2$.  The first two and the last two pairs in the numerator are easy
to multiply: $(x^2-x-2)(x^2-25)$.  While a bit tortuous, multiply two trinomials is certainly
the easiest way to find the numerator, and it is $x^4-x^3-27x^2+25x+50$.  We can now find
the curvilinear asymptote.

\polylongdiv{x^4-x^3-27x^2+25x+50}{x^2+x-2}

The asymptote is a parabola!  It opens upward, has a $y$-intercept of -23 and a centerline
at $x=1$.

\begin{figure}
\begin{centering}
\includegraphics[width=\textwidth]{\chapdir/pics/curvilinear}
\caption{A function with a parabolic asymptote}
\end{centering}
\end{figure}

\subsection{Partial Fraction Decomposition}
Calculus is not concerned what the function does everywhere: we have the function itself
for that.  Calculus is satisfied local behavior, what a function does nearer and near to
certain places.  The curved asymptote we just found is more and more right, the larger
(positive or negative) a number we plug into it.  We know how to find lines that behave
live the function almost any other point on the graph: tangent lines with a slope from
the derivative.  For example, at (5,0), the behavior can be modeled with
$y-0 = \frac{47}{5}(x-5)$.  But what about around the asymptotes, at 1 and -2?  That is
where we bring back the ``remainder'' from the polynomial long division.

$\frac{44x+4}{x^2+x-2}$ is the non-asymptote part of the quotient, which we know is
composed of $(x-1)(x+2)$ in the denominator.  Is there some way to rip it apart, into
two fractions, one with a denominator of (x-1) and another with (x+2)?  Let us suppose
there is, and that each fraction has a simple constant in the numerator.  That would
mean we are hypothesizing

$$
\frac{A}{x-1} + \frac{B}{x+2} = \frac{44x+4}{x^2+x-2}
$$

where $A$ and $B$ are plain numbers.  We can begin to figure out what they are by
``clearing the fraction'', multiplying by $x^2+x-2$ on both sides.  By factoring and canceling,
we get

$$
A(x+2) + B(x-1) = 44x + 4
$$

If we distribute and group for like terms, we get

$$
(A+B)x + (2A-B) = 44x + 4
$$

Now we already said $A$ and $B$ are numbers, so if we have $A+B x$'s on the left,
then $A+B$ must equal 44, and $2A-B$ must equal 4.  We can either add these two
equations, or use a matrix and find that $A=16$ and $B=28$.  This means $\frac{16}{x-1}
+ \frac{28}{x+2} = \frac{44x+4}{x^2+x-2}$.

Additionally, we have decomposed the fraction into its partial fraction components,
a fancy way of saying we made it easy to integrate.  This assume that you know that
$\int \frac{1}{x} = \ln{|x|}$, which we will not prove until chapter 8.

In the vicinity of $x=1$, $\frac{16}{x-1}$ is a good model, and in the vicinity of $x=-2$,
$\frac{28}{x-2}$ is a good model of $f(x)$.

\subsection{Sums of Power Functions}
Finally, everything we have learned up until this point is also good for sums of
power function that do not obey the definition of polynomials and rational functions.
For example, suppose you wanted to make a doorway modeled by the equation
$h(x) = \sqrt[3]{(x+4)^2}+\sqrt[3]{(x-4)^2}$ from -4 to 4.  If you had to program a
machine lathe to make the curved top, what would its slope be at every point
on the interval?

We will need both the integral and the derivative of the function, but fortunately
it is the Power Rule all around.  $h'(x) = \frac{2}{3\sqrt[3]{x+4}}+\frac{2}{3\sqrt[3]{x-4}}$
and $H(x) = \frac{3}{5}\sqrt[3]{(x+4)^5}+\frac{3}{5}\sqrt[3]{(x-4)^5}+C$


\section{Use of List comprehensions}

\begin{lstlisting}[language=Haskell]
toBeImplemented = undefined

-- the type for values
type Value = Double
-- the type for matrices
type Matrix = [[Value]]

-- We represent matrices as lists of lists, where each inner list represents a row
-- (rather than a column) of the matrix.
-- Example:
-- [[a11, a12, a13],
--  [a21, a22, a23]]
-- Indexing, as often done in matrix computations, starts with 1 rather than 0 (beg Dijkstra's forgiveness).

-- We assume as precondition for most of the following functions that all inner lists
-- have the same length (if there are any). Here is a function that might be used to check this condition.

exaIsMat =
  isMat [[1, 2], [3, 4]] &&
  isMat [[1, 2]] &&
  isMat [] &&
  isMat [[], []] &&
  not (isMat [[1, 2], []])

isMat :: Matrix -> Bool
isMat [] = True
isMat (row1 : rows) =
  and [length row == lengthRow1 | row <- rows]
  where lengthRow1 = length row1

-- Sometimes we use square matrices. Here is a function that might be used to check whether a list of lists represents a square matrix.
exaIsSquareMat =
  isSquareMat [[1, 2], [3, 4]] &&
  isSquareMat [[1]] &&
  isSquareMat [] &&
  not (isSquareMat [[]]) &&
  not (isSquareMat [[1, 2, 3], [4, 5, 6]])

isSquareMat = isSquareMatV2

isSquareMatV1 :: Matrix -> Bool
isSquareMatV1 [] = True
isSquareMatV1 mat@(row1 : _) = -- example of an as-pattern
  isMat mat && length mat == length row1

isSquareMatV2 :: Matrix -> Bool
isSquareMatV2 [] = True
isSquareMatV2 mat =
  isMat mat && length mat == length (head mat)

-- zeroMat generates an m * n zero matrix, that is, a matrix with all entries 0.
exaZeroMat =
  zeroMat 2 3 == [[0, 0, 0], [0, 0, 0]]

zeroMat :: Int -> Int -> Matrix
zeroMat a b = [[0 | _ <- [1..b]] | _ <- [1..a]]

-- Check whether a list of lists represents a zero matrix or not.
isZeroMatV1 :: Matrix -> Bool
isZeroMatV1 mat =
  isMat mat && and [a == 0 | row <- mat, a <- row]

isZeroMatV2 :: Matrix -> Bool
isZeroMatV2 mat =
  isMat mat && and [a == 0 | a <- concat mat]

-- unitMat generates an n * n unit matrix, that is, a square matrix in which all entries are zero, except of the elements on the main diagonal, which are 1.
exaUnitMat =
  unitMatV1 3 == [[1, 0, 0], [0, 1, 0], [0, 0, 1]] &&
  unitMatV2 3 == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]

unitMat = unitMatV1

unitMatV1 :: Int -> Matrix
unitMatV1 n = [unitRow n x | x <- [1..n]]
  where unitRow a b = [if y == b then 1 else 0 | y <- [1..a]]

unitMatV2 :: Int -> Matrix
unitMatV2 n = [unitRow i | i <- [1 .. n]]
  where
    unitRow i = [0 | _ <- [1 .. i-1]] ++ 1 : [0 | _ <- [i+1 .. n]]

-- Check whether two matrices are equal of not.
equalMat :: Matrix -> Matrix -> Bool
equalMat = (==)

-- Check whether a list of lists represents a unit matrix or not.
isUnitMat :: Matrix -> Bool
isUnitMat mat = mat == unitMat (length mat)

-- neg negates a matrix, that is,
-- negates all corresponding components of it.
exaNeg =
  neg [[1, -2], [0, 4]] == [[-1, 2], [0, -4]]

neg :: Matrix -> Matrix
neg mat = [[-a | a <- row] | row <- mat]

-- plusMat adds two matrices, that is, adds all corresponding components of them. precondition: mat1 and mat2 have the same size
exaPlusMat =
  unit3 `plusMat` (neg unit3) == zeroMat 3 3
    where unit3 = unitMat 3

plusMat = plusMatV1

plusMatV1 :: Matrix -> Matrix -> Matrix
mat1 `plusMatV1` mat2 =
    [row1 `plusRow` row2 | (row1, row2) <- zip mat1 mat2]
  where
    row1 `plusRow` row2 = [a1 + a2 | (a1, a2) <- zip row1 row2]

plusMatV2 :: Matrix -> Matrix -> Matrix
mat1 `plusMatV2` mat2 = [zipWith (+) row1 row2 | (row1, row2) <- zip mat1 mat2]

-- Reads the element in row i and column j of matrix mat.
-- precondition: i and j are in range of mat
readMat :: Matrix -> Int -> Int -> Value
readMat mat i j = (mat !! i) !! j

-- updateMat 'updates' matrix mat.
-- The updated matrix is the same as mat, except of
-- position row i column j, where the new value is val.
-- Note: there is no imperative update; the function returns a new matrix.
-- precondition: i and j are in range of mat
exaUpdateMat =
  updateMat (unitMat 3) 3 1 5 == [[1,0,0],[0,1,0],[5,0,1]]

updateMat :: Matrix -> Int -> Int -> Value -> Matrix
updateMat mat i j val =
  [if i' == i then updateRow row j else row | (i', row) <- zip [1 ..] mat]
    where
      updateRow row j =
        [if j' == j then val else val' | (j', val') <- zip [1 ..] row]

\end{lstlisting}

\clearpage


\documentclass{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{amsthm}

\newtheorem{theorem}{Theorem}[section]
\newtheorem*{problem}{Problem}

\begin{document}

\title{Some functional equations}
\author{Dave Neary}

\maketitle

\section{2001 Dutch IMO qualification}

Suppose for all $x,y \in \mathbb{R}$ that we have:
\[ f(x + y) = f(x) + f(y) + xy \]
and $f(4) = 10$. Find $f(2001)$.

Setting $x=y=2$:
\[ f(4) = 10 = 2f(2) + 4 \implies f(2) = 3\]

Setting $x=y=1$:
\[ f(2) = 3 = 2f(r12) + 1 \implies f(1) = 1\]

Setting $y=0$:
\[ f(x) = f(x) + f(0) \implies f(0) = 0 \]

Finally, setting $y=1$:
\[ f(x+1) = f(x) + f(1) + x \]

This is already enough for us to calculate $f(n)$ for all positive integers:
\begin{align}
	f(n) - f(n-1) & = n \\
	& \vdots \\
	f(1)-f(0) &= 1
\end{align}
Adding LHS and RHS we get:
\[ f(n) = \sum_{i=1}^{n} i = \frac{1}{2}(n)(n+1) \]

and $f(2001) = 2003001$.

We can easily extend to show that the formula applies to negative integers:
\[f(n + (-n)) f(0) = 0 = f(n) + f(-n) - n^2 \]
\[ \implies f(-n) = n^2 - \frac{1}{2}(n)(n+1) = n^2 - \frac{n^2}{2} - \frac{n}{2} \]
\[ \implies f(-n) = \frac{1}{2}(-n)(-n+1) \]

The question remains, however: what class of functions satisfies the functional relationship in
general, outside of integers?

Since $f(nx) = f((n-1)x) + f(x) + (n-1)x^2$, we can show by simple iteration that:
\[ f(nx) = nf(x) + x^2((n-1)+(n-2)+\cdots + 1) = nf(x) + \frac{(n)(n-1)}{2} x^2 \]

From above, we know that $f(n) = \frac{1}{2}(n)(n+1)$ for $n\in \mathbb{Z}$, so
for $x=\frac{1}{m}, n=m, m \in \mathbb{Z}/{0}$:
\begin{align}
	f(1) &= f(m\times\frac{1}{m}) \\
	     &= mf(\frac{1}{m}) + \frac{m^2-m}{2m^2} \\
	     &= 1 \\
	f(\frac{1}{m}) &= \frac{1}{m}(1- \frac{1}{2}(1-\frac{1}{m})) \\
	               &= \frac{1}{2}(\frac{1}{m})(\frac{1}{m} + 1)
\end{align}

We can now generalize to all rational numbers:
\begin{align}
	f(\frac{n}{m}) &= nf(\frac{1}{m}) + \frac{n^2-n}{2m^2} \\
	& = \frac{n}{2}(\frac{1}{m})(\frac{1}{m} + 1) + \frac{1}{2}((\frac{n}{m})^2-\frac{n}{m^2}) \\
	& = \frac{1}{2}(\frac{n}{m})(\frac{n}{m} + 1)
\end{align}

But does this mean that $f(x) = \frac{1}{2}(x)(x+1)$ for all $x\in \mathbb{R}$? Bizarrely enough,
the answer is no. However, if $f(x)$ is defined to be continuous, then this is true. To see this,
let's consider an arbitrary irrational number $x$. We can define a sequence of rational numbers
$\{a_n\}$ with $10^nx > 10^n a_n > 10^n x - 1$ for all $n$ by defining
$a_n = \frac{1}{10^n}\lfloor 10^nx \rfloor$. This set of rational numbers will converge to $x$, and the
set $\{f(a_n)\} = \{\frac{1}{2}(a_n)(a_n+1)\}$ also converges to $f(x) = \frac{1}{2}(a_n)(a_n+1)$
if $f$ is continuous.

However, if $f$ is not continuous, we can construct infinitely many $f(x)$ which satisfy the
conditions. For any irrational number, we can arbitrarily define $f(x) = c$, and then
\[ f(qx) &= qc + \frac{1}{2}(q)(q-1)(x^2) \]

And since $qx$ will never be a rational number, this is linearly independent to the rational
numbers we already have above.

\end{document}


\section*{Sparse Coding}
\subsection*{Orthogonal Basis}
Transform: For $\mathbf{x}$ and orthog. mat. $\mathbf{U}$ compute $\mathbf{z} = \mathbf{U}^\top \mathbf{x} $. For compression, can drop small values: $ \mathbf{\hat{x}} = \mathbf{U\hat{z}}$, $\hat{z}_i = z_i$ if $ \lvert z_i \rvert > \epsilon$ else 0.
Pros: fast inverse; preserves energy.
or $\mathbf{x}$ and orthog. mat. $\mathbf{U}$ compute $\mathbf{z} = \mathbf{U}^\top \mathbf{x} $ else 0.
Reconstr. Error $\|\mathbf{x}-\mathbf{\hat{x}}\|^2 = \sum_{d\notin\sigma}\langle\mathbf{x},\mathbf{u}_d\rangle ^2$ for a subset of basis $\sigma$.\\
\textbf{Haar Wavelets:} scaling function $\phi(x)=[1,1,1,1]$, mother $W(x)=[1,1,-1,-1]$, dilated $W(2x)=[1,-1,0,0]$, translated $W(2x-1)=[0,0,1,-1]$. Do not forget to normalize.\\
\textbf{Comparison to Fourier basis:} local (not global) support, good for localized (not $\sin$ like, repeating) signals.
\textbf{PCA basis:} data-dependent, but optimal for given $\Sigma$. $\mathbf{\hat x} = \mathbf{U}_K \mathbf{z}_{[1:K]} $
\subsection*{Overcomplete Dictionaries}
Use more atoms than dimensions, then choose the best representation. (e.g.Gabor wavelets use Fourier like features in a localized Gaussian window). \\ Linear dependency measure:  \textbf{Coherence}\\
\begin{inparaitem}[\color{gray}\textbullet]
	\item $m(\mathbf{U}) = \max_{i,j:\, i \neq j} | \mathbf{u}_i^\top \mathbf{u}_j |$
	\item $m(\mathbf{B}) = 0$ if $\mathbf{B}$ orth. mat.
	\item $m([\mathbf{B}, \mathbf{u}]) \geq \frac{1}{\sqrt{D}}$ if atom $\mathbf{u}$ is added to $\mathbf{B}$
\end{inparaitem}\\
\textbf{Signal Reconstruction:} orthonormal: $\mathbf{x} = \mathbf{Uz}$, spanning basis (linearly independent): $\mathbf{x} = (\mathbf{U}^T)^{-1}\mathbf{z}$ (can be ill-conditioned), overcomplete: solve $\mathbf{z}^* \in \argmin_\mathbf{z} ||\mathbf{z}||_0$ s.t. $\mathbf{x}=\mathbf{Uz}$. (NP hard). Can convexify with $L_1$ norm or greedy approx.: \\
\textbf{Matching Pursuit (MP)}
\begin{inparaenum}[\color{gray}1.]
	\item init: $\hat z \leftarrow 0, r \leftarrow x$
	\item while $\|\mathbf{z}\|_0 < K$ do
	\item select atom with smallest angle $j^* = \argmax_j |\langle \mathbf{u}_j, \mathbf{r} \rangle|$
	\item update coefficients: $\hat z\leftarrow \hat z + \langle \mathbf{u}_{i^\star}, \mathbf{r} \rangle \mathbf{u_{j^*}}$
	\item update residual: $\mathbf{r} \leftarrow \mathbf{r} - \langle \mathbf{u}_{i^\star}, \mathbf{r} \rangle \mathbf{u}_{i^\star}$.
\end{inparaenum}
\textbf{Exact recovery} when: $K<1/2( 1+1/m(\mathbf{U}))$ \\
\textbf{Instance when MP never exactly match x}: Idea: if we always just half residual and start with a pos. number, we will never arrive at $0$. Start with (0,1). u1 = (1,0) u2 = ($\sqrt 2$/2, $\sqrt 2$/2), u3 = ($\sqrt 3$/2, 1/2). \\
\textbf{Instance where MP does not have best solution:} u1 = (1,0), u2 = (0,1), u3 = (sqrt(2)/2, sqrt(2)/2). x = (2,1), will have largest correlation with u3, but residual then cannot be expressed with only u1 or u2, so we will use all 3 vectors instead of only u1 \& u2.
\subsection*{Compressive Sensing}
Aquire set $\mathbf{y}$ of $M$ linear combinations of signal, then reconstruct from it. $y_k = \langle \mathbf{w}_k, \mathbf{x} \rangle, k=1,...,M $
$\mathbf{y}=\mathbf{Wx}=\mathbf{WUz} =: \Theta\mathbf{z}$ with $\Theta = \mathbf{WU} \in \mathbb{R}^{M\times D}$.
Any orthonormal basis $\mathbf{U}$ can obtain a stable reconstr. for any $K$-sparse compressible signal if: \begin{inparaitem}[\color{gray}\textbullet]
\item $\mathbf{W}$ is Gaussian random projection, i.e. $w_{ij}\sim \mathcal{N}(0,\frac{1}{D})$
\item $M\geq cK \log \frac{D}{K}$(some constant $c$).
\end{inparaitem}
Reconstruct as before: $\mathbf{z}^\star \in \argmin_{\mathbf{z}}\|\mathbf{z}\|_0$, s.t. $\mathbf{y} = \Theta\mathbf{z}$


\section{Improper Integrals}
Now that we've developed the tools to deal with limits as they approach infinity and the possible indeterminate forms that may arise, we can apply these ideas to integrals, allowing us to have $-\infty$ and $\infty$ as limits of integration.
We call these integrals with $\pm\infty$ as limits of integration, and functions that become $\pm\infty$ somewhere within the interval we're integrating on improper integrals.

\subsection{Infinite Integration Limits}
\begin{definition}
	If $f$ is continuous on $[a,\infty)$, then
	\begin{equation*}
		\int_{a}^{\infty}{f(x)\d{x}} = \lim_{b\to\infty}{\int_{a}^{b}{f(x)\d{x}}}.
	\end{equation*}
	If $f$ is continuous on $(-\infty,b]$, then
	\begin{equation*}
		\int_{\infty}^{b}{f(x)\d{x}} = \lim_{a\to-\infty}{\int_{a}^{b}{f(x)\d{x}}}.
	\end{equation*}
	If $f$ is continuous on $(-\infty,\infty)$, then
	\begin{equation*}
		\int_{-\infty}^{\infty}{f(x)\d{x}} = \int_{-\infty}^{c}{f(x)\d{x}} + \int_{c}^{\infty}{f(x)\d{x}}
	\end{equation*}
	for any real constant $c$.
	If these limits exist, then the integral converges and has a value.
	Otherwise, the integral diverges and does not have a value.
\end{definition}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{2}^{\infty}{\frac{3}{x^2-x}\d{x}}.
	\end{equation*}
\end{example}
\begin{answer}
	Applying the definition,
	\begin{align*}
		\int_{2}^{\infty}{\frac{3}{x^2-x}\d{x}} &= \lim_{b\to\infty}{\int_{2}^{b}{\frac{3}{x^2-x}\d{x}}} \\
		&= \lim_{b\to\infty}{3\int_{2}^{b}{\left(\frac{1}{x-1}-\frac{1}{x}\right)\d{x}}} \\
		&= \lim_{b\to\infty}{3\ln{\bigg\lvert\frac{x-1}{x}\bigg\rvert}\Biggr\rvert_{2}^{b}} \\
		&= \lim_{b\to\infty}{3\ln{\bigg\lvert\frac{b-1}{b}\bigg\rvert}} - 3\ln{\bigg\lvert\frac{1}{2}\bigg\rvert} \\
		&= 0 + 3\ln{2} \\
		&= 3\ln{2}.
	\end{align*}
\end{answer}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{\sqrt[4]{x}}}.
	\end{equation*}
\end{example}
\begin{answer}
	Applying the definition,
	\begin{align*}
		\int_{1}^{\infty}{\frac{\d{x}}{\sqrt[4]{x}}} &= \lim_{b\to\infty}{\int_{1}^{b}{\frac{\d{x}}{\sqrt[4]{x}}}} \\
		&= \lim_{b\to\infty}{\frac{4}{3}\sqrt[4]{x^3}\biggr\rvert_{1}^{b}} \\
		&= \lim_{b\to\infty}{\frac{4}{3}\sqrt[4]{b^3}} - \frac{4}{3}\sqrt[4]{1^3} \\
		&= \infty - \frac{4}{3} \\
		&= \text{diverges}.
	\end{align*}
\end{answer}

\subsection{Infinite Discontinuities}
An infinite discontinuity occurs when a function takes on a value of $\pm\infty$ on the interval we're integrating on.
In this case, we'll need to split the integral into pieces, evaluating the limit as we approach this infinite discontinuity from both sides.
\begin{definition}
	If $f$ is continuous on $(a,b]$, then
	\begin{equation*}
		\int_{a}^{b}{f(x)\d{x}} = \lim_{c\to a^+}{\int_{c}^{b}{f(x)\d{x}}}.
	\end{equation*}
	If $f$ is continuous on $[a,b)$, then
	\begin{equation*}
		\int_{a}^{b}{f(x)\d{x}} = \lim_{c\to b^-}{\int_{a}^{c}{f(x)\d{x}}}.
	\end{equation*}
	If $f$ is continuous on $[a,c) \cup (c,b]$, then
	\begin{equation*}
		\int_{a}^{b}{f(x)\d{x}} = \int_{a}^{c}{f(x)\d{x}} + \int_{c}^{b}{f(x)\d{x}}.
	\end{equation*}
	If these limits exist, then the integral converges and has a value.
	Otherwise, the integral diverges and does not have a value.
\end{definition}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{0}^{1}{\frac{\d{x}}{x^2}}.
	\end{equation*}
\end{example}
\begin{answer}
	We see that we have an infinite discontinuity at $x=0$.
	Applying the definition,
	\begin{align*}
		\int_{0}^{1}{\frac{\d{x}}{x^2}} &= \lim_{c\to 0^+}{\int_{c}^{1}{\frac{\d{x}}{x^2}}} \\
		&= \lim_{c\to 0^+}{\frac{-1}{x}\biggr\rvert_{c}^{1}} \\
		&= \frac{-1}{1} + \lim_{c\to 0^+}{\frac{1}{c}} \\
		&= -1 + \infty \\
		&= \text{diverges}.
	\end{align*}
\end{answer}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{0}^{1}{\frac{\d{x}}{x^{1/2}}}.
	\end{equation*}
\end{example}
\begin{answer}
	We see that we have an infinite discontinuity at $x=0$.
	Applying the definition,
	\begin{align*}
		\int_{0}^{1}{\frac{\d{x}}{x^{1/2}}} &= \lim_{c\to 0^+}{\int_{c}^{1}{\frac{\d{x}}{x^{1/2}}}} \\
		&= \lim_{c\to 0^+}{2x^{1/2}\biggr\rvert_{c}^{1}} \\
		&= 2 - \lim_{c\to 0^+}{2c^{1/2}} \\
		&= 2 - 0 \\
		&= 2.
	\end{align*}
\end{answer}

\subsection{Convergence Tests}
\subsubsection{P-Test}
\begin{lemma}
	The following integral will converge when $p > 1$ and diverge if $0 < p \leq 1$.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{x^p}}.
	\end{equation*}
\end{lemma}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{x}}.
	\end{equation*}
\end{example}
\begin{answer}
	We have an integral where $p=1$.
	So, by the P-Test, the integral diverges.
\end{answer}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{x^{1.001}}}.
	\end{equation*}
\end{example}
\begin{answer}
	We have an integral where $p=1.001$.
	So, by the P-Test, the integral converges.
	\begin{align*}
		\int_{1}^{\infty}{\frac{\d{x}}{x^{1.001}}} &= \lim_{b\to\infty}{\int_{1}^{b}{\frac{\d{x}}{x^{1.001}}}} \\
		&= \lim_{b\to\infty}{-1000x^{-0.001}\biggr\rvert_{1}^{b}} \\
		&= \lim_{b\to\infty}{-1000b^{-0.001}} + 1000(1)^{-0.001} \\
		&= 0 + 1000 \\
		&= 1000.
	\end{align*}
\end{answer}

\subsubsection{Direct Comparison Test}
\begin{lemma}
	Let $f$ and $g$ be continuous on $[a,\infty)$ with $0 \leq f(x) \leq g(x)$ for all $x \geq a$.
	\begin{align*}
		\int_{a}^{\infty}{f(x)\d{x}} &\text{ converges if } \int_{a}^{\infty}{g(x)\d{x}} \text{ converges.} \\
		\int_{a}^{\infty}{g(x)\d{x}} &\text{ diverges if } \int_{a}^{\infty}{f(x)\d{x}} \text{ diverges.}
	\end{align*}
\end{lemma}

That is, if a larger function converges, then so will a smaller funtion; if a smaller function diverges, then so will a larger function. \\

The hardest part of the Direct Comparison Test is deciding what function you should compare to.
A general rule is to pick a function that is similar to, but simpler than then given function.

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{x^2-0.1}}.
	\end{equation*}
\end{example}
\begin{answer}
	If the 0.1 wasn't inside the square root, the function would simplify to $1/x$.
	Since the 0.1 is subtracted, the denominator is smaller than $1/x$.
	So, $1/x$ is a function that is smaller on $[1,\infty)$, meaning if it diverges, then so will the original function.
	We know by the P-Test that the integral of $1/x$ from 1 to $\infty$ will diverge, so the original function also diverges.
\end{answer}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{e^{-x^2}\d{x}}.
	\end{equation*}
\end{example}
\begin{answer}
	Since the exponent is negative, a smaller exponent would mean a larger value.
	So, $e^{-x}$ is a larger function on $[1,\infty)$.
	\begin{equation*}
		\int_{1}^{\infty}{e^{-x}\d{x}} = \frac{1}{e},
	\end{equation*}
	meaning it converges, so the original function also converges.
\end{answer}

\subsubsection{Limit Comparison Test}
\begin{lemma}
	If positive functions $f$ and $g$ are continuous on $[a,\infty)$ and
	\begin{equation*}
		\lim_{x\to\infty}{\frac{f(x)}{g(x)}}
	\end{equation*}
	converges to a positive real number, then
	\begin{equation*}
		\int_{a}^{\infty}{f(x)\d{x}} \text { and } \int_{a}^{\infty}{g(x)\d{x}}
	\end{equation*}
	both converge or both diverge.
\end{lemma}

Many functions to which you can apply the Limit Comparison Test you can also apply the Direct Comparison Test.
The practical use of the limit comparison test is to take an uglier function, that may be tedious to integrate and compare it to a function that is easy to determine whether it diverges using something like the P-Test.
A common strategy, especially for rational functions, is to look at their end behavior model.

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{1+x^2}}.
	\end{equation*}
\end{example}
\begin{answer}
	Although you might recognize this as the derivative of $\arctan$, let's continue with the Direct Comparison Test.
	This function looks very similar to $1/x^2$, which we know by the P-Test will converge on $[1,\infty)$.
	\begin{equation*}
		\lim_{x\to\infty}{\frac{\frac{1}{x^2}}{\frac{1}{1+x^2}}} = \lim_{x\to\infty}{\frac{1+x^2}{x^2}} = 1.
	\end{equation*}
	Since 1 is a positive real constant and the integral of $1/x^2$ converges, then the original integral also converges by the Limit Comparison Test.
\end{answer}

\begin{example}
	Evaluate the following integral or state that it diverges.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{3x+6}{1-5x+7x^2}\d{x}}.
	\end{equation*}
\end{example}
\begin{answer}
	Looking at this rational function, we see a degree 1 polynomial in the numerator and a degree 2 polynomial in the denominator.
	So, we'd expect this rational function to have the same end behavior model as $1/x$, which we know by the P-Test diverges.
	\begin{equation*}
		\lim_{x\to\infty}{\frac{\frac{3x+6}{1-5x+7x^2}}{\frac{1}{x}}} = \lim_{x\to\infty}{\frac{3x^2+6x}{7x^2-5x+1}} = \frac{3}{7}.
	\end{equation*}
	Since 3/7 is a positive real constant and the integral of $1/x$ diverges, the the original integral also diverges by the Limit Comparison Test.
\end{answer}


\subsection{Circles}
\noindent
A circle of radius $R$ centered at the origin can be represented as $r = R$.\\
Circles not centered at the origin require using the transformation equations.
\begin{equation*}
	(x-a)^2 + (y-b)^2 = R^2 \implies (r\cos{\theta}-a)^2 + (r\sin{\theta}-b)^2 = R^2
\end{equation*}
\begin{equation*}
	r^2\cos^{2}{\theta} + r^2\sin^{2}{\theta} - 2ra\cos{\theta} - 2rb\sin{\theta} + a^2 + b^2 = R^2
\end{equation*}
\begin{equation*}
	r^2 - 2r(a\cos{\theta} + b\sin{\theta}) - R^2 + a^2 + b^2 = 0
\end{equation*}
\begin{equation*}
	r = (a\cos{\theta} + b\sin{\theta}) \pm \sqrt{R^2 - a^2 - b^2 + (a\cos{\theta} + b\sin{\theta})^2}
\end{equation*}