\section*{Exercises}

\begin{ex}
  For each matrix $A$, determine whether the formula
  $\iprod{\vect{u},\vect{v}} = \vect{u}^T A\vect{v}$ determines an
  inner product on $\R^2$.
  \begin{equation*}
    (a)~ A = \begin{mymatrix}{cc} 1 & 0 \\ 0 & 2 \end{mymatrix},\quad
    (b)~ A = \begin{mymatrix}{cc} 3 & 1 \\ 1 & 3 \end{mymatrix},\quad
    (c)~ A = \begin{mymatrix}{cc} 1 & 1 \\ 0 & 2 \end{mymatrix},\quad
    (d)~ A = \begin{mymatrix}{cc} 1 & 0 \\ 0 & -1 \end{mymatrix},\quad
    (e)~ A = \begin{mymatrix}{cc} 1 & 1 \\ 1 & 1 \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    \begin{enumerate}
    \item Yes, it is an inner product. Since $A=A^T$, symmetry and
      linearity follow as in Example~\ref{exa:rn-with-inner-product}.
      For the positive definite property, note that
      $\iprod{\vect{u},\vect{u}} = u_1^2 + 2u_2^2\geq 0$, and equality
      holds if and only if $u_1,u_2=0$.
    \item Yes, it is an inner product. Since $A=A^T$, symmetry and
      linearity follow as in Example~\ref{exa:rn-with-inner-product}.
      For the positive definite property, we have
      $\iprod{\vect{u},\vect{u}} = 3u_1^2 + 2u_1u^2 + 3u_2^2 =
      (u_1+u_2)^2 + 2u_1^2 + 2u_2^2 \geq 0$. Equality holds if and
      only if $u_1+u_2$, $u_1$, and $u_2$ are all zero, which is the
      case if and only if $\vect{u}=0$.
    \item No, it is not symmetric. For example,
      $\iprod{
        \begin{mymatrix}{c}1\\0\end{mymatrix},
        \begin{mymatrix}{c}0\\1\end{mymatrix}
      } = 1$
      but
      $\iprod{
        \begin{mymatrix}{c}0\\1\end{mymatrix},
        \begin{mymatrix}{c}1\\0\end{mymatrix}
      } = 0$.
    \item No, it is not positive definite. For example,
      $\iprod{
        \begin{mymatrix}{c}0\\1\end{mymatrix},
        \begin{mymatrix}{c}0\\1\end{mymatrix}
      } = -1$.
    \item No, it is not positive definite. For example,
      $\iprod{
        \begin{mymatrix}{c}1\\-1\end{mymatrix},
        \begin{mymatrix}{c}1\\-1\end{mymatrix}
      } = 0$
      although $\begin{mymatrix}{c}1\\-1\end{mymatrix}\neq \vect{0}$.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the inner product space $C[0,1]$ as in
  Example~\ref{exa:continuous-interval}. Compute the following inner
  products:
  \begin{equation*}
    (a)~ \iprod{1,x}, \quad
    (b)~ \iprod{x,x^2}, \quad
    (c)~ \iprod{1+x,2+x^2}.
  \end{equation*}
  \begin{sol}
    \begin{enumerate}
    \item $\iprod{1,x} = \int_{0}^{1} 1\cdot x\,dx
      = \int_{0}^{1} x\,dx = \frac{1}{2}$.
    \item $\iprod{x,x^2} = \int_{0}^{1} x\cdot x^2\,dx
      = \int_{0}^{1} x^3\,dx = \frac{1}{4}$.
    \item $\iprod{1+x,2+x^2}
      = \int_{0}^{1} (1+x)(2+x^2)\,dx
      = \int_{0}^{1} 2 + x^2 + 2x + x^3\,dx
      = \frac{43}{12}.$
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the inner product space $C[0,1]$ as in
  Example~\ref{exa:continuous-interval}. Compute the following norms:
  \begin{equation*}
    (a)~ \norm{1}, \quad
    (b)~ \norm{x}, \quad
    (c)~ \norm{x^2+1}.
  \end{equation*}
  \begin{sol}
    \begin{enumerate}
    \item $\norm{1}^2 = \iprod{1,1} = \int_{0}^{1} 1\,dx
      = 1$, therefore $\norm{1}=1$.
    \item $\norm{x}^2 = \iprod{x,x} = \int_{0}^{1} x^2\,dx
      = \frac{1}{3}$, therefore $\norm{x} = \frac{1}{\sqrt{3}}$.
    \item $\norm{x^2+1}^2 = \iprod{x^2+1,x^2+1}
      = \int_{0}^{1} x^4 + 2x^2 + 1\,dx
      = \frac{1}{5} + \frac{2}{3} + 1 = \frac{28}{15}$, therefore
      $\norm{x^2+1} = \sqrt{\frac{28}{15}}$.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  For $\vect{u},\vect{v}$ vectors in $\R^3$, define the product
  $\vect{u}\ast \vect{v} = u_1v_1+2u_2v_2+3u_3v_3$. Show
  that
  \begin{equation*}
    \abs{\vect{u}\ast \vect{v}} \leq (\vect{u}\ast \vect{u})^{1/2}
    (\vect{v}\ast \vect{v})^{1/2}.
  \end{equation*}
  Hint: first show that the operation
  $\iprod{\vect{u},\vect{v}}=\vect{u}\ast\vect{v}$ is an inner product
  on $\R^3$, then use Proposition~\ref{prop:inner-product-and-norm}.
  \begin{sol}
    The operation $\iprod{\vect{u},\vect{v}}=\vect{u}\ast\vect{v}$ is
    an inner product on $\R^3$. Symmetry and linearity are
    straightforward to check, as is the positive definite
    property. Therefore, the claimed inequality holds by
    Proposition~\ref{prop:cauchy-schwarz-inequality}.
  \end{sol}
\end{ex}

\begin{ex}
  In $C[-1,1]$, find (a) the angle between $x$ and $x^2$, (b) the
  angle between $x$ and $x^3$.
  \begin{sol}
    \begin{enumerate}
    \item We have $\iprod{x,x} = \int_{-1}^1 x^2\,dx = \frac{2}{3}$,
      $\iprod{x^2,x^2} = \int_{-1}^1 x^4\,dx = \frac{2}{5}$, and
      $\iprod{x, x^2} = \int_{-1}^1 x^3\,dx = 0$.  Therefore
      \begin{equation*}
        \cos\theta
        ~=~ \frac{\iprod{x,x^2}}{\norm{x}\norm{x^2}}
        ~=~ 0.
      \end{equation*}
      Therefore, the angle $\theta$ is $\pi/2$ radians, or $90$
      degrees. In other words, $x$ and $x^2$ are orthogonal in
      $C[-1,1]$.
    \item We have $\iprod{x,x} = \int_{-1}^1 x^2\,dx = \frac{2}{3}$,
      $\iprod{x^3,x^3} = \int_{-1}^1 x^6\,dx = \frac{2}{7}$, and
      $\iprod{x, x^3} = \int_{-1}^1 x^4\,dx = \frac{2}{5}$.  Therefore
      \begin{equation*}
        \cos\theta
        ~=~ \frac{\iprod{x,x^3}}{\norm{x}\norm{x^3}}
        ~=~ \frac{\frac{2}{5}}{\sqrt{\frac{2}{3}}\sqrt{\frac{2}{7}}}
        ~=~ \frac{\sqrt{21}}{5}.
      \end{equation*}
      The angle $\theta$ is $\cos^{-1}(\frac{\sqrt{21}}{5})$, which is
      approximately $0.4115$ radians or $23.58$ degrees.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}\label{ex:hilbert-space}.
  In this exercise, we will work out the details of
  Example~\ref{exa:hilbert-space}. We must show that $\Hilb_{\R}$ is a
  vector space. We will do this by showing that it is a subspace of
  $\Seq_{\R}$. Further, we must show that the inner product is
  well-defined. This requires some knowledge of convergent series from
  calculus.
  \begin{enumerate}
  \item Assume $a=(a_0,a_1,\ldots)$ and $b=(b_0,b_1,\ldots)$ are
    square summable sequences. Show that the series
    $a_0b_0 + a_1b_1 + a_2b_2 + \ldots$ converges to a real
    number. Hint: consider the series
    $\abs{a_0b_0} + \abs{a_1b_2} + \ldots$ and use the Cauchy-Schwarz
    inequality and the absolute convergence test.
  \item Using the result of part (a), show that $\Hilb_{\R}$ is a
    subspace of $\Seq_{\R}$.
  \item Show that the operation $\iprod{a,b} = a_0b_0 + a_1b_1 +
    a_2b_2 + \ldots$ is an inner product on $\Hilb_{\R}$.
  \end{enumerate}
  \begin{sol}
    (a) By assumption, $a$ and $b$ are square summable. Let
    $N=a_0^2 + a_1^2 + \ldots$ and $M=b_0^2 + b_1^2 + \ldots$.
    By the Cauchy-Schwarz inequality, for all $n$, we have
    \begin{equation*}
      \abs{a_0}\abs{b_0} + \ldots + \abs{a_n}\abs{b_n}
      ~\leq~ \sqrt{\abs{a_0}^2 + \ldots + \abs{a_n}^2} \sqrt{\abs{b_0}^2 + \ldots + \abs{b_n}^2}
      ~\leq~ \sqrt{N}\sqrt{M}.
    \end{equation*}
    Therefore the series $\abs{a_0b_0} + \abs{a_1b_1} + \ldots$ is
    bounded. By the absolute convergence test from calculus, it
    follows that the series $a_0b_0 + a_1b_1 + \ldots$ converges.

    (b) It is clear that the zero sequence is square summable, and
    also that a scalar multiple of a square summable sequence is
    square summable. Hence $\Hilb_{\R}$ contains the zero vector and
    is closed under scalar multiplication. To show that it is closed
    under addition, assume $a,b\in\Hilb_{\R}$, and let $c=a+b$. We
    must show that $c$ is square summable. But
    \begin{equation*}
      c_0^2 + c_1^2 + \ldots
      ~=~
      (a_0+b_0)^2 + (a_1+b_1)^2 + \ldots
      ~=~ (a_0^2 + a_1^2 + \ldots) + (b_0^2 + b_1^2 + \ldots)
      + (2a_0b_0 + 2a_1b_1 + \ldots).
    \end{equation*}
    The series $a_0^2 + a_1^2 + \ldots$ and $b_0^2 + b_1^2 + \ldots$
    converge by assumption, and the series $2a_0b_0 + 2a_1b_1 +
    \ldots$ converges by part (a). It follows that $\Hilb_{\R}$ is
    closed under addition.

    (c) Symmetry and linearity follow straightforwardly from
    properties of convergent series. For example,
    \begin{eqnarray*}
      \iprod{a,kb+\ell c}
      &=& a_0(kb_0+\ell c_0) + a_1(kb_1+\ell c_1) + \ldots \\
      &=& k(a_0b_0 + a_1b_1 + \ldots) + \ell(a_0c_0 + a_1c_1 + \ldots) \\
      &=& k\iprod{a,b} + \ell\iprod{a,c}.
    \end{eqnarray*}
    As for the positive definite property, note that
    \begin{equation*}
      \iprod{a,a}
      ~=~ a_0^2 + a_1^2 + \ldots
      ~\geq~ 0.
    \end{equation*}
    Moreover, since all terms in the series are $\geq 0$, it follows
    that $\iprod{a,a}=0$ if and only if $a_i=0$ for all $i$.
  \end{sol}
\end{ex}


\chapter{Floating point numbers}\label{appA:fp-numbers}
Computers must represent real numbers with a finite amount of bits. In order to do this, they use a fixed amount of bits for the significand and the exponent, and one bit for sign. A comprehensive introduction to floating point numbers and arithmetic can be found in many textbooks, for instance \cite{overton-num-analysis-book}; in this appendix we only recall concepts related to the thesis subjects.

Normal values are represented as a significand (or coefficient) $c$, determined by $t$ bits, an exponent $q$, by $e$ bits, and the sign. In this thesis, we interpret the significand $c$ as the positive integer number whose binary representation is the sequence of $t$ bits multiplied by $2^{-t+1}$: this ensures that, if the binary representation of $c$ begins with a $1$ (as it is always the case for normal values), its value is always in the interval $[1, 2)$. We interpret the exponent $q$ as the signed integer whose binary representation is the sequence of $e$ bits, so that its range is $[-2^e, 2^e - 1]$.
Special values as $0$, $-0$, $+\infty$ and $-\infty$ are defined by the standard as well and represented with special combination of bits, but we don't care about their exact representation.

The significand $c$ of a floating point number can be interpreted as the binary representation of a non integer value with the decimal point right after the first digit, that is always $1$. For instance, if $t = 4$, the bit sequence $1010$ is interpreted as the binary number $1.010$, that is $1.25$ in decimal. With this interpretation in mind, the first bit of the significand has value $2^0$ while the last $2^{1-t}$.
This means for instance that $1$ is represented as $c = 1.00 \dots 00$ and $q = 0$, that the smallest floating point number greater than $1$ is $c = 1.00 \dots 01$ and $q = 0$, that is $1 + 2^{1-t}$. The greatest floating point number smaller than $1$ is instead represented as $1.11 \dots 11$ and $q = -1$, and is $1 - 2^{-t}$.

Many real numbers can't be represented as floating points with $t + e$ bits, hence the need to approximate them. Without dwelling into details, we denote with $\fp(x)$ the floating point approximation of the real number $x$, a common choice being rounding to nearest. The choice of representable numbers guarantee that, independently of $x$
\[
\left\lvert \frac{\fp(x) - x}{x} \right\rvert < \macheps
\]
where $\macheps$ is called machine precision, and we're guaranteed that $\macheps \le 2^{1-t}$, the value of the least significant bit. The actual value of $\macheps$ depends on the kind of approximation performed. Moreover, all approximation techniques ensures that $\fp$ is monotone, that is if $x \le y$ then $\fp(x) \le fp(y)$. Of course, it is not strictly monotone: even if $x < y$ it can be the case that $\fp(x) = \fp(y)$.
Moreover, floating point approximation $\fp$ preserves powers of $2$:
\[
\fp(x \cdot 2^q) = \fp(x) \cdot 2^q
\]

An arithmetic operation on a pair of floating point numbers may not be a floating point number. To solve this issue, the standard requires a floating point version of each operation, that we denote with a circle around the standard operator symbol, whose output is always a floating point number. The definition of those operations require that they are performed on their inputs as if in infinite precision, and then the result is approximated to a floating point number. In formula, if $\times$ is a generic binary operation, its floating point approximation $\otimes$ is defined as
\[
x \otimes y = \fp(x \times y)
\]
Some of the properties of operations are kept by their floating point counterparts, while other are not. For instance, associativity is often lost because of approximation. On the other hand, if $\times$ is commutative, also $\otimes$ is because
\[
x \otimes y = \fp(x \times y) = \fp (y \times x) = y \otimes x
\]

\section{Multiplications}
In this section we enunciate and prove some properties of floating point multiplications. We denote by $\cdot$ the exact product and by $\odot$ its approximation.

Since $\cdot$ is commutative, as observed above also $\odot$ is.
By monotonicity of $\cdot$ and $\fp$, for any two floating point numbers $x \le y$ and any positive floating point number $z > 0$ we have
\[
x \odot z = \fp(x \cdot z) \le \fp(y \cdot z) = y \odot z
\]
Since $\fp$ preserves powers of $2$ we have that, for any two floating point numbers $x = c \cdot 2^q$ and $y = c' \cdot 2^{q'}$, their floating point product can be computed as
\[
x \odot y = \fp((c \cdot 2^q) \cdot (c' \cdot 2^{q'})) = \fp(c \cdot c') \cdot 2^{q + q'} = (c \odot c') \cdot 2^{q + q'}
\]

We now present some results on floating point multiplications that aren't classic from numerical analysis, so we also prove them.
\begin{prop}\label{appA:th:mult-no-fixpoints}
	Let $y$ be a floating point number different from $1$ and $0$. Then, if $t \ge 2$ and $\fp$ is rounding to nearest, for any floating point number $x \neq 0$ we have
	\[
	x \odot y \neq x
	\]
\end{prop}
\begin{proof}
	If $y < 0$ the thesis clearly hold because $x$ and $x \odot y$ have different signs. So assume $y > 0$.
	Let $x = c \cdot 2^q$, assuming without loss of generality that $x > 0$, and let us distinguish four cases on $y = c_0 \cdot 2^{q_0}$.

	If $q_0 \ge 1$, since $c_0 \ge 1$ we have
	\begin{align*}
		x \odot y &= (c \odot c_0) \cdot 2^{q + q_0} \\
		&\ge (c \odot 1) \cdot 2^{q + 1} \\
		&= c \cdot 2^{q} \cdot 2 = 2 x > x
	\end{align*}

	If $q_0 \le -2$, since $c_0 < 2$ we have
	\begin{align*}
		x \odot y &\le (c \odot 2) \cdot 2^{q + q_0} \\
		&\le (c \odot 1) \cdot 2^{q + q_0 + 1} \\
		&= c \cdot 2^{q} \cdot 2^{q_0 + 1} \le x / 2 < x
	\end{align*}

	If $q_0 = 0$, since $y \neq 1$, it must be the case that $c \neq 1$ too, hence $c \ge 1 + 2^{1-t}$. From this it follows
	\begin{align*}
		x \odot y &= (c \odot c_0) \cdot 2^q \\
		&\ge (c + c \cdot 2^{1-t}) \cdot 2^q
	\end{align*}
	Since $c \ge 1$ we have that $c \cdot 2^{-t+1} \ge 2^{1-t}$ and hence this is an increase in one of the bits of the machine representation of $c \cdot c_0$, thus $c \odot c_0 \neq c$ because they differ in at least one bit.
	For instance, assume $t = 4$, $c = 1.011$ and $c_0 = 1.001$ (that is its smallest possible value). Then the product is
	\begin{center}
		\begin{tabular}{c@{\;}c@{\,}c@{\,}c@{\,}c@{\,}c@{\,}c@{\,}c}
			& & & & 1. & 0 & 1 & 1 \\
			$\times$ & & & & 1. & 0 & 0 & 1 \\
			\hline
			& 0. & 0 & 0 & 1 & 0 & 1 & 1 \\
			$+$ & 1. & 0 & 1 & 1 & 0 & 0 & 0 \\
			\hline
			& 1. & 1 & 0 & 0 & 0 & 0 & 1
		\end{tabular}
	\end{center}
	so the result is represented as $1.100$, that is different from $c = 1.011$.

	Lastly, if $q_0 = -1$ it must be the case that $y = c_0 \cdot 2^{-1} \le 1 - 2^{-t}$, that is $c_0 \le 2 - 2^{1-t}$. Suppose without loss of generality that $x = c \cdot 2^0$, if that is not the case just scale all computations by $2^q$.
	If $c = 1$ then $x \odot y = c \odot (2 - 2^{1-t}) \cdot 2^{-1} = (2 - 2^{1-t}) \cdot 2^{-1} < 1 = c = x$. If $c > 1$ then
	\begin{align*}
		x \cdot y &= c \cdot c_0 \cdot 2^{-1} \\
		&\le c \cdot (2 - 2^{1-t}) \cdot 2^{-1} \\
		&= (2 c - 2^{1-t} c) \cdot 2^{-1} \\
		&< (2 c - 2^{1-t}) \cdot 2^{-1} = c - 2^{-t}
	\end{align*}
	Since $2^{-t}$ is exactly half the value of the least significant bit of $c$, this means $c - 2^{-t}$ is exactly in the middle between $c$ and $c - 2^{1-t}$, that is the greatest floating point smaller than $c$. Note that this is the case only because $c > 1$, because it implies $c \ge 1 + 2^{1-t}$, hence $c - 2^{1-t} \ge 1$.
	\begin{center}
	\begin{tikzpicture}
		\tikzset{func/.style = {->,> = latex'}}

		% number line
		\draw[func] (-6, 0) to (6, 0);

		% c-2^{-t}, c-2^{1-t} and c
		\draw (0, 0.2) -- (0, -0.2);
		\node[label=below:$c-2^{-t}$] at (0, 0){};
		\draw (3, 0.2) -- (3, -0.2);
		\node[label=below:$c$] at (3, 0){};
		\draw (-3, 0.2) -- (-3, -0.2);
		\node[label=below:$c-2^{1-t}$] at (-3, 0){};

		% c \cdot c_0
		\fill[red] (-1, 0) circle (2pt);
		\node[label=above:$x \cdot y$] at (-1, 0){};
	\end{tikzpicture}
	\end{center}
	As shown in the line number, given that $x \cdot y = c \cdot c_0 \cdot 2^{-1} < c - 2^{-t}$, if $\fp$ rounds to the nearest floating point we have that $c - 2^{1-t}$ is closer to $x \cdot y$ than $c$, hence
	\[
	x \odot y = \fp(x \cdot y) \neq c = x
	\]
\end{proof}

We remark that multiplication by a floating point is not injective. For instance, if $t = 4$, both products
\begin{center}
	\parbox{0.45\textwidth}{\centering
		\begin{tabular}{c@{\;}c@{\;}c@{\,}c@{\,}c@{\,}c@{\,}c@{\,}c@{\,}c}
			& & & & & 1. & 1 & 1 & 0 \\
			$\times$ & & & & & 1. & 0 & 1 & 0 \\
			\hline
			& & 0. & 0 & 1 & 1 & 1 & 0 & 0 \\
			$+$ & & 1. & 1 & 1 & 0 & 0 & 0 & 0 \\
			\hline
			& 1 & 0. & 0 & 0 & 1 & 1 & 0 & 0
	\end{tabular}}
	\parbox{0.45\textwidth}{\centering
		\begin{tabular}{c@{\;}c@{\;}c@{\,}c@{\,}c@{\,}c@{\,}c@{\,}c@{\,}c}
			& & & & & 1. & 1 & 1 & 1 \\
			$\times$ & & & & & 1. & 0 & 1 & 0 \\
			\hline
			& & 0. & 0 & 1 & 1 & 1 & 1 & 0\\
			$+$ & & 1. & 1 & 1 & 1 & 0 & 0 & 0 \\
			\hline
			& 1 & 0. & 0 & 1 & 0 & 1 & 1 & 0
	\end{tabular}}
\end{center}
are approximated by the same floating-point number $1.001 \cdot 2^1$ (assuming rounding to nearest) but their first arguments differ by the last bit.

\begin{prop}\label{appA:th:fp-inverse}
	If $t > 1$ and we assume a rounding to nearest, every floating point number $y = c \cdot 2^q$ such that $1 \le c < 3/2$ and $q$ is different from both the maximum and the minimum exponents has an inverse floating point, that is a floating point $y'$ such that $y \odot y' = 1$
\end{prop}
\begin{proof}
	If $c = 1$ there is such $y'$, that is $1 \cdot 2^{-q}$. This is a floating point number since $q \neq -2^e$, hence $-q \in [-2^e, 2^e - 1]$. So assume $c \neq 1$. This in particular entails $c \ge 1 + 2^{1-t}$.

	Consider $d = c^{-1}$, and the two floating point numbers obtained rounding it up and down to $t$ binary digits. Since this number is between $2/3$ and
	\[
	\frac{1}{1 + 2^{1-t}} < 1 - 2^{-t}
	\]
	where the inequality holds whenever $2^{-t} < 1/2$, hence when $t > 1$, these two floating point numbers are in the range $[1/2, 1 - 2^{-t}]$, that means that they have exponent $-1$. Said $\overline{d}$ and $\underline{d}$ to be respectively the rounding up and down of $d = c^{-1}$, since they have both exponent $-1$ and their difference is exactly one least significant bit, we have
	\[
	\overline{d} - \underline{d} = 2^{1-t} \cdot 2^{-1} = 2^{-t}
	\]
	By monotonicity of multiplication
	\[
	c \odot \overline{d} = \fp(c \cdot \overline{d}) \ge \fp(c \cdot d) = \fp(1) = 1
	\]
	and analogously $c \odot \underline{d} \le 1$.
	Suppose by way of contradiction that $c \odot \overline{d} > 1$ and $c \odot \underline{d} < 1$. This implies that
	\[
	c \odot \overline{d} \ge 1 + 2^{1-t}
	\]
	that, by the rounding to nearest approximation hypothesis, implies
	\[
	c \cdot \overline{d} \ge \frac{1 + (1 + 2^{1-t})}{2} = 1 + 2^{-t}
	\]
	Analogously, $c \odot \underline{d} < 1$ implies
	\[
	c \cdot \underline{d} \le \frac{1 + (1 - 2^{-t})}{2} = 1 - 2^{-t-1}
	\]
	This means
	\[
	c \cdot \overline{d} - c \cdot \underline{d} \ge 1 + 2^{-t} - 1 + 2^{-t-1} = 3/2 \cdot 2^{-t}
	\]
	But we also know
	\[
	c \cdot \overline{d} - c \cdot \underline{d} = c \cdot (\overline{d} - \underline{d}) = c \cdot 2^{-t}
	\]
	that yields the desired contradiction because $c < 3/2$ and
	\[
	c \cdot 2^{-t} \ge 3 / 2 \cdot 2^{-t}
	\]
	To show the proof graphically, we marked on the number line $1$, the two floating point nearest to $1$, the middle points of the two segments connecting these to $1$ and the two products $c \cdot \underline{d}$ and $c \cdot \overline{d}$.
	\begin{center}
	\begin{tikzpicture}
		\tikzset{func/.style = {->,> = latex'}}

		% number line
		\draw[func] (-6, 0) to (6, 0);

		% 1, 1- and 1+
		\draw (0, 0.2) -- (0, -0.2);
		\node[label=below:$1$] (1) at (0, 0){};
		\draw (4, 0.2) -- (4, -0.2);
		\node[label=below:$1+2^{1-t}$] (1p) at (4, 0){};
		\draw (-2, 0.2) -- (-2, -0.2);
		\node[label=below:$1+2^{-t}$] (1m) at (-2, 0){};

		% middle points
		\draw (-1, 0.15) -- (-1, -0.15);
		\draw (2, 0.15) -- (2, -0.15);

		% c \cdot d^ and d_
		\fill[red] (-1.2, 0) circle (2pt);
		\node[label=above:$c \cdot \underline{d}$] at (-1.2, 0){};
		\fill[red] (2.2, 0) circle (2pt);
		\node[label=above:$c \cdot \overline{d}$] at (2.2, 0){};
	\end{tikzpicture}
	\end{center}
	In order to have that $c \odot \overline{d} \ge 1 + 2^{1-t}$ we need the real result $c \cdot \overline{d}$ to the right of the middle point of the segment. Analogously, to have $c \odot \underline{d}$ we need $c \cdot \underline{d}$ to the left of the other middle point. However, the distance between the two can't be more than that between the two middle points, that is exactly $3/2 \cdot 2^{-t} \le c \cdot 2^{-t}$, thus yielding the contradiction.
\end{proof}

\begin{prop}\label{appA:th:interval-number-elem}
	If $t \ge 3$, given a floating point number $w$, the interval
	\[
	\left[ (1 - 2 \macheps) w, (1 + 2 \macheps) w \right]
	\]
	contains at most $17$ floating point numbers.
\end{prop}
\begin{proof}
	Let $w = c \cdot 2^q$. Since $t \ge 3$ we have $1 + 2 \macheps < 2$ and $1 - 2 \macheps \ge 1/2$, hence all floating point numbers in that interval have a distance from $w$ of at most a factor of $2$. This implies that their exponent is at least $q - 1$, hence their least significant bit has value at least $2^{1-t} \cdot 2^{q-1}$. Using that $\macheps \le 2^{1-t}$ and $c < 2$ we get
	\[
	2^{1-t} \cdot 2^{q-1} \le \macheps \cdot 2^{q-1} < \macheps \cdot c / 2 \cdot 2^{q} \cdot 2^{-1} = \frac{1}{4} w \cdot \macheps
	\]
	Since the length of the interval is $4\macheps \cdot w$ and two floating point in it have distance at least $\macheps \cdot w /4$, there are at most $17$ of them in it.
\end{proof}


\subsection{Injective, bijective and surjective functions}

\subsubsection{Injective functions}

\(f(a)=f(b)\rightarrow a=b\)

\subsubsection{Surjective functions}

All points in codomain have at least one matching point in domain

Mapping info, details

\subsubsection{Bijective}

Both injective and surjective

\subsubsection{Other}

Identity function

The identity function maps a term to itself.

Idempotent

An idempotent function is a function which does not change the term if the function is used more than once. An example is multiplying by \(0\).

\subsubsection{Inverse functions}

An inverse function of a function is one which maps back onto the original value.

\(g(x)\) is an inverse function of \(f(x)\) if

\(g(f(x))=x\)

\subsubsection{Properties of binary functions}

Binary functions can be written as:

\(f(a,b)=a\oplus b\)

A function is commutative if:

\(x\oplus y = y\oplus x\)

A function is associative if:

\((x\oplus y)\oplus z = x\oplus (y\oplus z)\)

A function \(\otimes \) is left distributive over \(\oplus \) if:

\(x\otimes (y\oplus z)=(x\otimes y) \oplus (x\otimes z)\)

Alternatively, function \(\otimes \) is right distributive over \(\oplus \) if:

\((x\oplus y)\otimes z=(x\otimes z) \oplus (y\oplus z)\)

A function is distributive over another function if it both left and right distributive over it.


% Created 2020-01-29 Wed 15:17
% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\date{}
\title{Universal Constructions}
\hypersetup{
 pdfauthor={Vaibhav Pujari},
 pdftitle={Universal Constructions},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.3)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Terminal objects}
\label{sec:orgfe6dc0d}
Objects that have a unique morphism coming into them from every other object in the category

\begin{equation}
\forall c \in \zeta \exists! c \to T
\end{equation}

If there are more than one such objects, they are isomorphic

\subsection{Example}
\label{sec:orgc179554}
In a category of sets, the terminal object is a singleton set, because there can be only one unique function from any set to a singleton set.

\subsection{Code}
\label{sec:org9dd47c1}
\begin{verbatim}
data Terminal = MkTerm
bang :: a -> Terminal
bang _ = MkTerm
\end{verbatim}
In haskell, the object \texttt{()} is a terminal object

\section{Initial object}
\label{sec:org3548e7d}
Objects that have a unique morphism going out to every other object in the category

\begin{equation}
\forall c \in \zeta \exists! I \to c
\end{equation}

If there are more than one such objects, they are isomorphic

\subsection{Example}
\label{sec:org802f106}
In a category of sets, the initial object is a null set, because there can be
only one unique function from a null set to any other set.

\subsection{Code}
\label{sec:org9ea2a9b}
\begin{verbatim}
data Void
absurd :: Void -> a
absurd x = absurd x
\end{verbatim}

\section{Products}
\label{sec:org99e3acd}
A product of two objects \(a\) and \(b\) is an object \(a \times b\) that has projections \(\pi_1\) and
\(\pi_2\) to \(a\) and \(b\) respectively. These projections factorize any other
projections that exist from other objects \(c\) (potential products) to \(a\) and \(b\)

\subsection{Example}
\label{sec:org639505e}
A pair/tuple is a product

\subsection{Code}
\label{sec:org5e9fdb6}
\begin{verbatim}
data Pair a b = MkPair a b
fst :: Pair a b -> a
fst (MkPair a _) = a
snd :: Pair a b -> b
snd (MkPair _ b) = b

-- For potential products c
tuple :: (c->a, c->b) -> c -> Pair a b
tuple f g = \c -> MkPair (f c) (g c)

untuple :: (c -> Pair a b) -> (c->a, c->b)
untuple h = (fst . h, snd . h)

-- Bifunctor
-- Code easily obtained by visualizing that Pair a' b' is
-- a product of a' and b', and thinking of Pair a b as a potential
-- product c. So what we are basically doing is finding a mapping
-- from potential product to actual product
bimap :: (a->a', b->b') -> Pair a b -> Pair a' b'
bimap f g = tuple (f . fst) (g . snd)
\end{verbatim}

\section{Coproducts}
\label{sec:orga821aab}
If \(c,d \in Ob(\zeta)\), then the coproduct of \(c\) and \(d\) consists of:
\begin{enumerate}
\item An object \(c \oplus d \in Ob(\zeta)\)
\item A morphism \(Left: c \to c \oplus d\)
\item A morphism \(Right: d \to c \oplus d\)
\end{enumerate}
satisfying the property that \(\forall X \in Ob(\zeta)\), and morphisms \(L: c \to
X\), \(R: d \to X\) there exists a unique morphism \(c \oplus d \to X\), and the
diagram commutes.

\subsection{Code}
\label{sec:org4033af4}
\begin{verbatim}
data Either a b = Left a | Right b
-- Left :: a -> Either a b
-- Right :: b -> Either a b

either :: (c -> x, d -> x) -> Either c d -> x
either (f, g) (Left c) = f c
either (f, g) (Right d) = g d

-- Bifunctor
-- Code can be easily obtained by drawing a diagram
bimap :: (c->c', d->d') -> Either c d -> Either c' d'
bimap (f, g) = either (Left . f) (Right . g)
\end{verbatim}

\section{Exponentials}
\label{sec:org938c44b}
Given a function from a product \((a, b)\) to \(c\), we can always get a function
from \(a\) to a function \(f: b \to c\). A function \(f: b \to c\) is an exponential
object \(c^b\)

\subsection{Code}
\label{sec:org5c6e419}
\begin{verbatim}
curry :: ((a, b) -> c) -> a -> b -> c
curry f = \a -> (\b -> f (a,b))

-- And the counterpart
uncurry :: (a -> (b -> c) -> ((a, b) -> c)
uncurry f = \(a,b) -> (f a) b
\end{verbatim}
\end{document}


\section{Preliminaries}

\subsection{Complexity of counting}

\begin{frame}
  \begin{definition}[The complexity class $\# \mathsf{P}$]
    A function $f \colon \{0,1\}^{*} \to \mathbb{N}$ is in $\# \mathsf{P}$ if there is a
    nondeterministic polynomial-time Turing machine $M_f$ such that, for each input $w$, $M_f$
    has exactly $f(w)$ accepting paths.
  \end{definition}
  \begin{example}[$\#SAT$]
    \textbf{Instance:} A boolean formula $F$ in conjunctive normal form. \textit{Example:}
    $F \equiv (x_1 \lor \neg x_2 \lor \neg x_3) \wedge (x_1 \lor x_3) \wedge (\neg x_2 \lor x_4)$.

    \textbf{Output:} The number of satisfying assignments of $F$. \\
  \end{example}

  \begin{definition}[$\# P$-hardness]
    A computational problem $C$ is $\# \mathsf{P}$-hard if, for any $f \in \# \mathsf{P}$, the problem ``evaluating $f$'' is Turing reducible to $C$.

    %A function $f$ is $\# \mathsf{P}$-complete if  $f \in \#P$ and the existence of a polynomial-time algorithm that computes $f$ implies the existence of such an algorithm for any other function in $\# \mathsf{P}$.
  \end{definition}
\end{frame}

\begin{frame}
	  \begin{definition}[$\# P$-completeness]
	  	A function $f \colon \{0,1\}^{*} \to \mathbb{N}$ is $\# \mathsf{P}$-complete if $f \in \# \mathsf{P}$ and evaluating $f$ is $\# \mathsf{P}$-hard.
	  \end{definition}

    \begin{enumerate}
    	\item {\color{TurkishRose} \textbf{Cook-Levin theorem:}} $\#SAT$ is $\# \mathsf{P}$-complete.
    	\item There are $\# \mathsf{P}$-complete problems whose decision version is trivial.
    \end{enumerate}

  \begin{example}[Counting independent sets is $\# \mathsf{P}$-complete]
  	\begin{minipage}{0.4\textwidth}
    \textbf{Instance:} A graph $G$. \\
    \textit{Example: }
  	\end{minipage}
  	\begin{minipage}{0.4\textwidth}
      \begin{tikzpicture}
      \begin{scope}[every node/.style={circle,thick,draw}]
      \node (A) at (0,0) {A}; \node (B) at (1,0.5) {B}; \node (C) at (2,0) {C}; \node (D) at (3.5,0) {D};
      \end{scope}

      \begin{scope}[>={Stealth[black]},
      every edge/.style={draw=black,very thick}]
      \path [-] (A) edge node {} (B); \path [-] (A) edge node {} (C); \path [-] (B) edge
      node {} (C); \path [-] (C) edge
      node {} (D);
      \end{scope}
      \end{tikzpicture}
  	\end{minipage}

     \vspace{3mm}
    \textbf{Output:} The number of independent sets of $G$.\\
  \end{example}
\end{frame}

\subsection{Partition functions}

\begin{frame}

  \begin{definition}[Partition function]
    Given a family $\mathcal{F}_n$ of subsets of the set $\{1, \ldots, n\}$, we define the partition
    function of $\mathcal{F}_n$ as the polynomial
    \begin{equation*}
      P_{\mathcal{F}_n}(x_1, \ldots, x_n) = \sum_{S \in \mathcal{F}_n} \prod_{j \in S} x_j.
    \end{equation*}
    \vspace*{-2mm}
  \end{definition}

  \begin{itemize}
  \item Hard to compute: enumerating $\mathcal{F}_n$ is usually not feasible.

%    \begin{itemize}
%    \item     \normalsize
%     the family $\mathcal{F}$ is exponentially large on $n$;
%    \item     \normalsize
%     enumerating $\mathcal{F}$ is believed to not be feasible in polynomial-time.
%    \end{itemize}
  \item Many partition functions arise in statistical mechanics.
  \end{itemize}

  \begin{example}[The independent sets polynomial]
    \vspace*{-1mm}
    Let $G$ be a graph. The independent sets polynomial of $G$ is
    \begin{equation*}
      Z(G; y) = \sum_{I \text{ independent set of } G} y^{|I|}.
    \end{equation*}
    \vspace*{-1mm}
  \end{example}

\end{frame}


\section{Singular Isothermal Sphere (SIS) Model}

% lensing_ipy1/Gravtational lensing
Singular Isothermal Sphere

$\rho = \rho_0r^{-2}$

Surface density:

$\Sigma(\xi) =\displaystyle\int^\infty_{-\infty} \mathrm{d}r_3 \rho\sqrt{\xi^2+r_3^2}$

$=\rho_0 \displaystyle\int^\infty_{-\infty} \dfrac{\mathrm{d}r_3}{(\xi^2+r_3^2)}$

$=\dfrac{\pi\rho_0}{\xi}$

$\rightarrow \kappa(\theta) = \dfrac{\Sigma(D_L\theta)}{\Sigma_{cr}}=\dfrac{\pi\rho_0}{\Sigma_{cr}\xi}=\dfrac{\pi\rho_0}{\Sigma_{cr}D_L \theta}$

Now using:

$\alpha(\theta) =  \dfrac{2}{|\vec\theta|}\displaystyle\int^\theta_{\theta'=0} \kappa(\theta')\theta' \mathrm{d}\theta'$

$\rightarrow \alpha(\theta) = \dfrac{2}{|\vec\theta|}\displaystyle\int^\theta_{\theta'=0}  \dfrac{\pi\rho_0}{\Sigma_{cr}D_L \theta'}\theta' \mathrm{d}\theta'$

$=\dfrac{2\pi\rho_0}{\Sigma_{cr}D_L \theta} \left[\theta'\right]^\theta_{\theta'=0}$

$=\dfrac{2\pi\rho_0}{\Sigma_{cr}D_L}$

i.e. a constant bend angle


\subsection{Family-Wise Error Rate (FWER)}

What is the chance of making at least one false positive result?

Number of tests: \(m\)

Number of false positive results: \(V\)

\(FWER=P(V>0)\)

\subsection{False Discovery Rate (FDR)}

The proportion of false discoveries is:

\(Q=\frac{V}{V+S}\)

Where:
\(V\) is the number of false positives

\(S\) is the number of true positives

The FRD is \(E[Q]\).

\subsection{The Bonferroni correction}

We change the significance level.

reject if \(p\le \frac{\alpha }{m}\)

If \(m=1\) this is the standard test.


\subsection{Sufficient statistics}

We can make estimates of a population parameter using statistics from the same.

A statistic is sufficient if it contains all the information needed to estimate the parameter.

We can describe the role of a parameter as:

\(P(x|\theta, t)\)

\(t\) is a sufficient statistic for \(\theta \) if:

\(P(x|t)=P(x|\theta, t)\)


\section{Siamese Network}\label{sec:Siamese}

\textit{Few-shot learning} encompasses classification tasks where there is a large number of classes but very limited labeled data per class --- as few as a single example in the case of ``one-shot learning.''  Few-shot learning is particularly common in computer vision spaces such as facial recognition~\cite{Guo:2017} and fingerprint identification~\cite{Marasco:2016}.

First proposed by Koch\etal~\cite{Koch:2015}, the \textit{Siamese neural network} is specifically targeted towards these few-shot, vision-based learning tasks.  Formally, the network is a function, $\SiamFunc:\xDomain\mapsto\mathbb{R}^m$, where ${m\in\mathbb{Z}_{{+}}}$ is a hyperparameter. Consider two examples, $\xBase,\xBase'\in\xDomain$, with labels $y$ and $y'$, and distance metric ${\delta:\mathbb{R}^m\times\mathbb{R}^{m}\mapsto\mathbb{R}_{{\geq}0}}$ satisfying the identity, symmetry, and triangle-inequality properties.  The \textbf{fundamental intuition} of Siamese networks is that if ${y=y'}$, then $\siamDist{\SiamFunc}{\xBase}{\xBase'}$ should be \textit{small}.  Otherwise if ${y\ne y}'$, $\siamDist{\SiamFunc}{\xBase}{\xBase'}$ should be \textit{large}.  The relative definitions of ``small'' and ``large'' vary by application and as a function of~$m$.

Siamese networks are trained to satisfy this fundamental intuition through a unique \textit{triplet loss function} --- whose name derives from the three training examples required for each loss calculation.  Define $\exA$ as the \underline{a}nchor example used as the reference for comparison.  \underline{P}ositive example,~$\exP$, has the same label as~$\exA$ while \underline{n}egative example,~$\exN$, has a different label than both $\exA$~and~$\exP$.

Eq.~\eqref{eq:Loss:Triplet} defines the triplet loss formally.  Hyperparameter ${\alpha\in\mathbb{R}_{{>}0}}$ enforces the relative distinctions of ``small'' and ``large'' mentioned earlier.   The loss is minimized by either reducing the distance between the mapped representation of examples in the same class --- i.e.,~$\siamDist{\SiamFunc}{\exA}{\exP}$ --- or by increasing the distance between the mapped representation of instances with different labels, i.e.,~$\siamDist{\SiamFunc}{\exA}{\exN}$.

\begin{equation}\label{eq:Loss:Triplet}
  \lTrip = \max\Big\{ \siamDist{\SiamFunc}{\exA}{\exP} - \siamDist{\SiamFunc}{\exA}{\exN} + \alpha, 0 \Big\}
\end{equation}


\chapter{Timestepping}
We now briefly discuss how to solve initial value problems. For more on this see Bradie~\cite[Chap. 7]{Bra06}. A slightly longer but still quick introduction to these ideas can also be found in Boyce and DiPrima~\cite{BoyDip10}.
%%%%%%%
%Section
%%%%%%%
\section{Forward Euler}
In order to compute solutions to differential equations on computers efficiently, it is convenient to do our calculations at a finite number of specified points and then interpolate between these points. For many calculations it is convenient to use a grid whose points are equally distant from each other.

For the rest of the section $h$ will be our step size, which is assumed to be constant.  When solving an ODE or PDE, the choice of $h$ isn't selected at random, but rather requires some intuition and/or theoretical analysis. We are going to start with the forward Euler method which is the most basic numerical method. Let us first denote the time at the $n$th time-step by $t^n$ and the computed solution at the $n^{th}$ time-step by $y^n$, where $y^n \equiv y(t=t^n)$. The step size $h$ in terms of $t$ is defined as $h=t^{n+1}-t^n$. Lets first start with a basic ODE with initial conditions, in which $f(t,y)$ is some arbitrary function and $y(t)$ is our solution,
\begin{eqnarray}\label{eq:ode}
\ \frac{dy}{dt} = f(t,y) \qquad y(t^0)=y^0.
\end{eqnarray}
The differential equation can be approximated by finite differences,
\begin{eqnarray}
\frac{y^{n+1}-y^n}{h} = f(t^n,y^n).
\end{eqnarray}
Now all we have to do is solve for $y^{n+1}$ algebraically,
\begin{eqnarray}\label{eq:odeFE}
\ y^{n+1}=y^n+h f(t^n,y^n) \qquad \text{(Forward Euler/Explicit method)}
\end{eqnarray}
If we wanted to calculate $\frac{dy}{dt}$ at time $t^0$, then we could generate an approximation for the value at time $t^{n+1}$ using \eqref{eq:odeFE} by first finding $y(t^0)$ and using it to compute $y^{n+1}$. We then repeat this process until the final time is reached.

%Subsection
\subsection{An Example Computation}
Let us consider the ODE in eq.\ \eqref{eq:ode} with $f(t,y)=y$ and initial conditions $y(t^0)=1$ where $t^0=0$. Two numerical solutions are computed using the forward Euler method with $h=1$ and $h=.1$
\begin{figure}
\begin{center}
\includegraphics[scale=0.6]{./Timestepping/Forward_Euler.jpg}
\caption{A numerical solution to the ODE in eq.\ \eqref{eq:ode} with $f(t,y)=y$ demonstrating the accuracy of the Forward Euler method for different choices of timestep.} \label{fig:FEexample}
\end{center}
\end{figure}

It should be no surprise that a smaller step size like $h=.1$ compared to $h=1$ will be more accurate. Looking at the line for $h=1$, you can see that $y(t)$ is calculated at only 4 points then straight lines interpolate between each point. This is obviously not very accurate, but gives a rough idea of what the function looks like. The solution for $h=.1$ might require 10 times more steps to be taken, but it is clearly more accurate. Forward Euler is an example of a first-order method and approximates the exact solution using the first two terms in the Taylor expansion\footnote{The derivation of the Taylor expansion can be found in most books on calculus.}
\begin{eqnarray}
y(t^n+h)=y(t^n)+h \left.\frac{dy}{dt}\right|_{t^n}+\text{O}(h^2),
\end{eqnarray}
where terms of higher order than O$(h^2)$ are omitted in the approximate solution. Substituting this into eq.\ \eqref{eq:odeFE} we get that
\begin{align*}
&{} y^n +h \left.\frac{dy}{dt}\right|_{t^n}+\text{O}(h^2) = y^n +hf(t^n,y^n)
\end{align*}
after cancelling terms and dividing by $h$, we get that
\begin{align*}
&{} \left.\frac{dy}{dt}\right|_{t^n}+\text{O}(h) = f(t^n,y^n),
\end{align*}
from which it is clear that the accuracy of the method changes linearly with the step size, and hence it is first-order accurate.
%%%%%%%
%Section
%%%%%%%
\section{Backwards Euler}
A variation of forward Euler can be obtained by approximating a derivative by using a backward difference quotient. Using eq.\ \eqref{eq:ode} and applying
\begin{eqnarray}
\frac{y^{n}-y^{n-1}}{h}&\approx&f(t^n,y^n) \\
y^{n}&=&y^{n-1}+h f(t^n,y^n).
\end{eqnarray}
Stepping the index up from $n$ to $n+1$ we obtain,
\begin{eqnarray}
y^{n+1}=y^n+h f(t^{n+1},y^{n+1})\qquad \text{(Backwards Euler/Implicit method)}
\end{eqnarray}
Notice how $y^{n+1}$ is not written explicitly like it was in the forward Euler method. This equation instead implicitly defines $y^{n+1}$ and must be solved  to determine the value of $y^{n+1}$. How difficult this is depends entirely on the complexity of the function $f$. For example, if $f$ is just $y^2$, then the quadratic formula could be used, but many nonlinear PDEs require other methods. Some of these methods will be introduced later.
%%%%%%%
%Section
%%%%%%%
\section{Crank-Nicolson}
By taking an average of the forward and backward Euler methods, we can find the Crank-Nicolson method:
\begin{eqnarray}
\frac{y^{n+1}-y^{n}}{h}&=&\frac{1}{2}f(t^{n+1},y^{n+1})+\frac{1}{2}f(t^n,y^n)
\end{eqnarray}
Rearranging we obtain,
\begin{eqnarray}
y^{n+1}=y^n+\frac{h}{2}\left[ f(t^{n+1},y^{n+1})+f(t^n,y^n) \right] \qquad \text{(Crank-Nicolson)}
\end{eqnarray}
Notice again how $y^{n+1}$ is not written explicitly like it was in forward Euler. This equation instead implicitly defines $y^{n+1}$ and so the equation must be solved algebraically to obtain $y^{n+1}$.
%%%%%%%
%Section
%%%%%%%
\section{Stability of Forward Euler, Backward Euler and Crank-Nicolson}
Let's look at the following ODE
\begin{eqnarray}
\frac{dy}{dt}= -\lambda y(t) \label{eq:ODEstability}
\end{eqnarray}
where $\lambda$ is a constant and $y(t^0)=1$ where $t^0=0$. Lets numerically solve this ODE using the forward Euler, backward Euler and Crank-Nicolson time-stepping schemes. The results are as follows
\begin{eqnarray}
y^{n+1}=y^n-\lambda h y^n \qquad \text{(Forward Euler)}\\
y^{n+1} = \frac{y^n}{(1+\lambda h)} \qquad \text{(Backward Euler)} \\
y^{n+1} = y^n\left(\frac{2-\lambda h}{2+\lambda h}\right) \qquad \text{(Crank-Nicolson)}
\end{eqnarray}
and the exact solution is given by
\begin{eqnarray}
y(t)= e^{-\lambda t} \qquad \text{(Exact solution)}
\end{eqnarray}

\begin{figure}[h]
\begin{center}
\includegraphics[scale=.4]{./Timestepping/unstable_forward_Euler.jpg}
\caption{A numerical solution to the ODE in eq.\ \eqref{eq:ODEstability} with $\lambda=20$ and with a timestep of $h=0.1$ demonstrating the instability of the Forward Euler method and the stability of the Backward Euler and Crank Nicolson methods.} \label{fig:ODEstability}
\end{center}
\end{figure}
Figure \ref{fig:ODEstability} above shows how both methods converge to the solution, but the forward Euler solution is unstable for the chosen timestep. Listing \ref{lst:MatlabInstability} is a Matlab program where you can play around with the value of $\lambda$ to see how, for a fixed timestep, this changes the stability of the method.
\lstinputlisting[style=matlab_style,language=Matlab,label=lst:MatlabInstability,caption={A Matlab program to demonstrate instability of different timestepping methods.}]{./Timestepping/Programs/Simple_ODE_Example_of_Unstable_FE.m}

%%%%%%%
%Section
%%%%%%%
\section{Stability and Accuracy of Forward Euler, Backward Euler and Crank-Nicolson Time Stepping Schemes for $y'=-\lambda y$}

The examples discussed show that numerical stability is an important consideration when finding approximate solutions to differential equations on computers. Numerical stability requires a careful choice of numerical method and timestep for each numerical solution to a differential equation. We now try to understand these observations so that we have some guidelines to design numerical methods that are stable. The numerical solution to an initial value problem with a bounded solution is {\bf stable} if the numerical solution can be bounded by functions which are independent of the step size. There are two methods which are typically used to understand stability. The first method is linearized stability, which involves calculating eigenvalues of a linear system to see if small perturbations grow or decay. A second method is to calculate an energy like quantity associated with the differential equation and check whether this remains bounded.

We shall assume that $\lambda\geq0$ so that the exact solution to the ODE does not grow without bound. The forward Euler method gives us
\begin{align*}
&{} \frac{y^{n+1}-y^n}{h}=-\lambda y^n
\\&{} y^{n+1}=(1-\lambda h)y^n
\\&{} \Rightarrow \lvert y^{n+1} \rvert \geq \lvert (1-\lambda h) \rvert \lvert y^n \rvert \quad \text{ if } \lvert (1-\lambda h) \rvert > 1
\\&{} \Rightarrow \lvert y^{n+1} \rvert \leq \lvert (1-\lambda h) \rvert \lvert y^n \rvert \quad \text{ if } \lvert (1-\lambda h) \rvert <1.
\end{align*}

We can do a similar calculation for backward Euler to get
\begin{align*}
&{} \frac{y^{n+1}-y^n}{h}=-\lambda y^{n+1}
\\&{} y^{n+1}=\frac{y^n}{1+\lambda h}
\\&{} \Rightarrow \lvert y^{n+1} \rvert \leq \left\lvert \frac{y^n}{1+\lambda h} \right\rvert \leq \lvert y^n \rvert \quad \text{ since } \left\lvert \frac{1}{1+\lambda h} \right\rvert <1.
\end{align*}
Thus, the backward Euler method is unconditionally stable, whereas the forward Euler method is not. We leave the analysis of the Crank-Nicolson method as an exercise.

A second method, often used to show stability for partial differential equations is to look for an energy like quantity and show that this bounds the solution and prevents it from becoming too positive or too negative. Usually, the quantity is chosen to be non negative, then all one needs to do is deduce there is an upper bound. We sketch how this is done for an ordinary differential equation so that we can use the same ideas when looking at partial differential equations. Recall that the forward Euler algorithm is given by
$$\frac{y^{n+1}-y^n}{h}=-\lambda y^n.$$
Multiplying this by $y^{n+1}$ we find that
\begin{align*}
&{} (y^{n+1})^2=(1-h\lambda)y^ny^{n+1}.
\end{align*}
Now to obtain a bound on $\lvert y^{n+1}\rvert$ in terms of $\lvert y^n \rvert$, we use the following fact
$$(a-b)^2\geq0\Rightarrow a^2+b^2\geq2ab\Rightarrow\frac{(y^{n+1})^2+(y^n)^2}{2}\geq y^ny^{n+1}.$$
Hence a sufficient condition for stability if
$$(1-h\lambda)>0$$
is that
\begin{align*}
&{} (y^{n+1})^2\leq(1-h\lambda)\frac{(y^{n+1})^2+(y^n)^2}{2}
\\&{} (y^{n+1})^2\frac{1+h\lambda}{2}\leq\frac{1-h\lambda}{2}(y^n)^2
\\&{} (y^{n+1})^2\leq\frac{1-h\lambda}{1+h\lambda}(y^n)^2,
\end{align*}
thus if $1-h\lambda>0$, then $0<\frac{1-h\lambda}{1+h\lambda}<1$ and so we have stability, we again see that the algorithm is stable provided the timestep is small enough. There are many situations for which $\lambda$ is large and so the timestep, $h$ needs to be very small. In such a situation, the forward Euler method can be very slow on a computer.

Stability is not the only requirement for a numerical method to approximate the solution to an initial value problem. We also want to show that as the timestep is made smaller, the numerical approximation becomes better. For the forward Euler method we have that
\begin{align*}
  &{} \frac{y^{n+h}-y^{n}}{h}=-\lambda y^n
\end{align*}
now if
\begin{align*}
&{}y^n=y(t) \\
&{}y^{n+1}=y(t+h)
\end{align*}
then\footnote{We will use big `Oh' to mean that there exists a constant so that if $f~O(h)$, then for $h\rightarrow0$, we have that $\left|\frac{f}{h}\right|<C$, where $C$ is some constant.}
\begin{align*}
\\&y^{n+1}= y(t)+h\frac{\mathrm{d}y}{\mathrm{d}t} + O(h^2)
\end{align*}
so
\begin{align*}
\frac{y^{n+1}-y^n}{h}+\lambda y^n &{} = \frac{y(t+h)-y(t)}{h} +\lambda y(t)
\\&{} =\frac{\mathrm{d}y}{\mathrm{d}t} +O(h)+\lambda y(t)
\\&{} = O(h).
\end{align*}
We can do a similar calculation to show that the Crank-Nicolson method is second-order. In this case however, we use Taylor expansions around $y(t+h/2)$.
\begin{align*}
&{}\frac{y^{n+1}-y^n}{h}=-\lambda \frac{ y^{n+1} + y^n}{2}
\end{align*}
so
\begin{align*}
&{} y^{n+1} =y(t+h)=y(t+h/2)+(h/2)\frac{\mathrm{d}y}{\mathrm{d}t} +(h/2)^2\frac{1}{2}\frac{\mathrm{d}^2y}{\mathrm{d}t^2} + O(h^3)
\\&{} y^{n} =y(t)=y(t+h/2)-(h/2)\frac{\mathrm{d}y}{\mathrm{d}t} +(h/2)^2\frac{1}{2}\frac{\mathrm{d}^2y}{\mathrm{d}t^2} + O(h^3)
\end{align*}
hence
\begin{align*}
\frac{y^{n+1}-y^n}{h}+\lambda \frac{ y^{n+1} + y^n}{2} &{} =\frac{\mathrm{d}y}{\mathrm{d}t} + O(h^2) +\lambda \left[y(t+h/2)+O(h^2) \right]
\\&{} = O(h^2).
\end{align*}
Thus this is a second-order method.

%%%%%%%
%Section
%%%%%%%
\section{Exercises}
\begin{enumerate}
\item[1)] Determine the real values of $\lambda$ and timestep $h$ for which the implicit midpoint rule is stable for the ODE
\begin{equation*}
\frac{\mathrm{d}y}{\mathrm{d}t}=-\lambda y
\end{equation*}
Sketch the stable region in a graph of $\lambda$ against timestep $h$.
\item[2)] Show that the backward Euler method is a first-order method.
\end{enumerate}


\section{Numerical Quadrature}
Define
$$
\Sigma_{n, M}(\mathbb{P}_t^d)=\{ \sum_{i=1}^n a_i\sigma_k(\omega_i\cdot x + b_i):  \omega_i\in \mathbb{S}^{d-1},\ |b_i|\le 2\}.
$$
\begin{lemma}
It holds that $\Sigma_{n, M}(\mathbb{P}_t^d)\subset W^{k,\infty}(\Omega)$ and
$$
\|v\|_{k,\infty}\le M,\quad \forall v\in \Sigma_{n, M}(\mathbb{P}_t^d),
$$
where $M=\|u\|_{\mathcal{K}_1(\mathbb{P}_t^d)}$.
\end{lemma}

For any $n>0$, there exists $\displaystyle \omega_i\in \mathbb{S}^{d-1}$, $\displaystyle |b_i|\le 2, \sum_{i=1}^n |a_i|\le M$ such that
$$
u_n(x)= \sum_{i=1}^n a_i\sigma_k(\omega_i\cdot x + b_i)
$$
satisfies
$$
\|u-u_n\|_a\lesssim n^{-\frac12-\frac{2k+1}{2d}}\|u\|_{\mathcal{K}_1(\mathbb{P}_k^d)}.
$$
For a bounded domain $\Omega$,
the activation function $\sigma_k\in W^{k,\infty}(\Omega)$.
Suppose quadrature points $x_i$  and weights $w_i$ satisfies
$$
\int_\Omega g(x)dx - |\Omega|\sum_{i=1}^{N} w_ig(x_i)=0, \quad \forall g(x)\in P_{r} (\Omega)
$$
with $N=\frac{(k-1)d}{2}$, and
$$
\left |\int_\Omega g(x)dx - |\Omega|\sum_{i=1}^N w_ig(x_i)\right |\lesssim N^{-\frac{r+1}{d}}\|g\|_{r,\infty}.
$$
Recall $J(v)$ in \eqref{auv} and \eqref{Jv} with $m=1$ and consider
$$
J(u)=\min J(v),\quad J_N(u_N)=\min J_N(v_N),
$$
where
$$
J_N(v_N)=|\Omega |\sum_{i=1}^Nw_i \left(\frac12 |\nabla v_N(x_i)|^2 - f(x_i)v_N(x_i)\right).
$$
For any conforming $v$,
\begin{align}
|J_N(v) - J(v)|&\le N^{-\frac{k-1}{d}}\|v\|_{k,\infty}.
\end{align}
Since $\Omega$ is bounded, $\displaystyle \omega_i\in \mathbb{S}^{d-1}$, $|b_i|\le 2, \sum_{i=1}^n |a_i|\le M$,
$$
\|u_N\|_{k,\infty}\lesssim 1.
$$
Since $J_N(u_N)\le J_N(v_N)$,
\begin{align*}
\frac12 \|u_N-u\|_a^2
=&J(u_N)- J(u)
\\
\le &J(u_N)- J_N(u_N) + J_N(v_N) - J(v_N)+ J(v_N)- J(u)
\\
\lesssim &N^{-\frac{k-1}{d}}  + \|v_N-u\|_a^2
\lesssim N^{-\frac{k-1}{d}} +n^{-\frac12-\frac{2k+1}{2d}}.
\end{align*}
Choose
$ N = \mathcal{O}(n^{2+\frac{d+2}{2(k-1)}})$, then
\begin{align*}
\|u_N-u\|_a^2
\lesssim & n^{-\frac12-\frac{2k+1}{d}}.
\end{align*}


\lab{Metropolis Algorithm}{Metropolis Algorithm}
\objective{Understand the basic principles of the Metropolis algorithm and apply these ideas to the
Ising Model.}

\section*{The Metropolis Algorithm}
Sampling from a given probability distribution is an important task in applications throughout the sciences.
When these distributions are complicated, as is often the case when modeling the world, direct sampling methods
can become difficult, as they might involve computing high-dimensional integrals.
The Metropolis algorithm is an effective method to sample from many distributions, requiring only that we
be able to evaluate the probability density function up to a constant of proportionality. In particular,
the Metropolis algorithm does not require us to compute difficult high-dimensional integrals that are found,
for example, in the denominator of Bayesian posterior distributions.

Like Gibbs sampling, the Metropolis algorithm is a MCMC sampling method, and generates a sequence of random
variables forming a Markov Chain whose invariant distribution is equal to the distribution from which we wish
to sample. Suppose that $h : \mathbb{R}^n \rightarrow \mathbb{R}$ is the probability density function of distribution,
and suppose that $f(\theta) = ch(\theta)$ for some nonzero constant $c$ (in practice, we assume that $f$ is an easy
function to evaluate, while $h$ is difficult). Let $Q : \mathbb{R}^n \times \mathbb{R}^n \rightarrow \mathbb{R}$ be
a symmetric \emph{proposal function}
(so that $Q(\cdot, y)$ is a probability density function for all $y \in \mathbb{R}^n$
 and $Q(x,y) = Q(y,x)$ for all $x,y \in \mathbb{R}^n$) and let
 $A : \mathbb{R}^n \times \mathbb{R}^n \rightarrow \mathbb{R}$ be an \emph{acceptance function} defined by
\[
A(x,y) = \min\left(1, \frac{f(x)}{f(y)}\right).
\]
We combine these functions to sample from the aforementioned Markov Chain by following Algorithm \ref{alg:metropolis}.
\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{Metropolis Algorithm}{}
    \State \textrm{Choose initial point } $x_0$.
    \For{$t=1,2,\ldots$}
        \State \textrm{Draw } $x' \sim Q(\cdot, x_{t-1})$
        \State \textrm{Draw } $a \sim \text{unif}(0,1)$
        \If{$a \leq A(x',x_{t-1})$}
            \State $x_t = x'$
        \Else
            \State $x_t = x_{t-1}$
        \EndIf
    \EndFor
    \State \textrm{Return } $x_1,x_2,x_3,\ldots$
\EndProcedure
\end{algorithmic}
\caption{Metropolis Algorithm}
\label{alg:metropolis}
\end{algorithm}
The Metropolis algorithm can be interpreted as follows:
given our current state $y$, we propose a new state according to the distribution $Q(\cdot, y)$. We then accept or reject it according to $A$, and continue the process. So long as $Q$ defines an irreducible, non-null recurrent, and aperiodic Markov chain, we will have a Markov chain whose unique invariant distribution will have density $h$. Furthermore, given any initial state, the chain will converge to this invariant distribution.
Note that for numerical reasons, it is often wise to make calculations of the acceptance functions in log space:
\[
\log A(x,y) = \min(0, \log f(x) - \log f(y)).
\]

Let's apply the Metropolis algorithm to a simple Bayesian analysis.
Consider the problem of computing the posterior distribution over the mean $\mu$ and variance $\sigma^2$
of a normal distribution for which we have $N$ data points $y_1,\ldots,y_N$. For concreteness,
we use the data in {\tt examscores.csv} and we assume
the prior distributions
\begin{align*}
\mu &\sim N(\mu_0=80,\sigma_0^2=16)\\
\sigma^2 &\sim IG(\alpha=3,\beta=50).
\end{align*}
In this situation, we wish to sample from posterior distribution
\[
p(\mu,\sigma^2 \,|\,y_1,\ldots,y_N) = \frac{p(\mu)p(\sigma^2)\prod_{i=1}^N N(y_i \, | \, \mu, \sigma^2)}
{\int_{-\infty}^\infty\int_{0}^\infty p(\mu)p(\sigma^2)\prod_{i=1}^N N(y_i \, | \, \mu, \sigma^2)\,d\sigma^2d\mu}
\]
but we can conveniently calculate only the numerator of the expression on the right. Since the denominator
is simply a constant with respect to $\mu$ and $\sigma^2$, the numerator can serve as the function $f$
in the Metropolis algorithm. We choose our proposal function to be based on a bivariate Normal distribution:
\[
Q(x,y) = N(x\, | \, y, sI),
\]
where $I$ is the $2\times 2$ identity matrix and $s$ is some positive scalar.
Let's create these functions in Python:
\begin{lstlisting}
import numpy as np
from math import sqrt, exp, log
import scipy.stats as st
from matplotlib import pyplot as plt
from scipy.stats import gaussian_kde

# load in the data
scores = np.loadtxt('examscores')

# initialize the hyperparameters
alpha=3
beta=50
mu0 = 80
sig20 = 16

# initialize the prior distributions
muprior=st.norm(loc=mu0, scale=sqrt(sig20))
sig2prior = st.invgamma(alpha,scale=beta)

# define the proposal function
def proposal(y, s):
    return st.multivariate_normal.rvs(mean=y, cov=s*np.eye(len(y)))

# define the log of the proportional density
def propLogDensity(x):
    return muprior.logpdf(x[0])+sig2prior.logpdf(x[1])+st.norm.logpdf(scores,loc=x[0],scale=sqrt(x[1])).sum()
\end{lstlisting}
\begin{figure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=\textwidth]{mu_traces.pdf}
	\end{subfigure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=.75\textwidth]{mu_kernel.pdf}
	\end{subfigure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=\textwidth]{sig_traces.pdf}
	\end{subfigure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=.75\textwidth]{sig_kernel.pdf}
	\end{subfigure}
\caption{Metropolis samples and KDEs for the marginal posterior distribution of $\mu$ (top row) and $\sigma^2$ (bottom row).}
\label{fig:metropolis_results}
\end{figure}
\begin{figure}
\centering
\includegraphics[width=.75\textwidth]{logprobs.pdf}
\caption{Log densities of the first 500 Metropolis samples.}
\label{fig:logprobs}
\end{figure}
We are now ready to code up the Metropolis algorithm using these functions.
We will keep track of the samples generated by the algorithm, along with the proportional log densities of the
samples and the proportion of proposed samples that were accepted. Study the implementation below to make sure
you understand the process:
\begin{lstlisting}
def metropolis(x0, s, n_samples):
    """
    Use the Metropolis algorithm to sample from posterior.

    Parameters
    ----------
    x0 : ndarray of shape (2,)
        The first entry is mu, the second entry is sigma2
    s : float > 0
        The standard deviation parameter for the proposal function
    n_samples : int
        The number of samples to generate

    Returns
    -------
    draws : ndarray of shape (n_samples, 2)
        The MCMC samples
    logprobs : ndarray of shape (n_samples)
        The log density of the samples
    accept_rate : float
        The proportion of proposed samples that were accepted
    """
    accept_counter = 0
    draws = np.empty((n_samples,2))
    logprob = np.empty(n_samples)
    x = x0.copy()
    for i in xrange(n_samples):
        xprime = proposal(x,s)
        u = np.random.rand(1)[0]
        if log(u) <= propLogDensity(xprime) - propLogDensity(x):
            accept_counter += 1
            x = xprime
        draws[i] = x
        logprob[i] = propLogDensity(x)
    return draws, logprob, accept_counter/float(n_samples)
\end{lstlisting}
Now let's sample from the posterior. We will choose an initial guess of $\mu=40$ and $\sigma^2=10$,
and we will set $s = 20$. We draw $10000$ samples as follows:
\begin{lstlisting}
>>> draws, lprobs, rate = metropolis(np.array([40, 10], dtype=float), 20., 10000)
>>> print "Acceptance Rate:", r
Acceptance Rate: 0.3531
\end{lstlisting}

We can evaluate the quality of our results by plotting the log probabilities, the $\mu$ samples, the $\sigma^2$ samples,
and kernel density estimators for the marginal posterior distributions of $\mu$ and $\sigma^2$. The code
below will accomplish these tasks:
\begin{lstlisting}
>>> # plot the first 500 log probs
>>> plt.plot(lprobs[:500])
>>> plt.show()
>>> # plot the mu samples
>>> plt.plot(draws[:,0])
>>> plt.show()
>>> # plot the sigma2 samples
>>> plt.plot(draws[:,1])
>>> plt.show()
>>> # build and plot KDE for posterior mu
>>> mu_kernel = gaussian_kde(draws[50:,0])
>>> x_min = min(draws[50:,0]) - 1
>>> x_max = max(draws[50:,0]) + 1
>>> x = np.arange(x_min, x_max, step=0.1)
>>> plt.plot(x,mu_kernel(x))
>>> plt.show()
>>> # build and plot KDE for posterior sigma2
>>> sig_kernel = gaussian_kde(draws[50:,1])
>>> x_min = 20
>>> x_max = 200
>>> x = np.arange(x_min, x_max, step=0.1)
>>> plt.plot(x,sig_kernel(x))
>>> plt.show()
\end{lstlisting}
Your results should be close to those given in Figures \ref{fig:metropolis_results} and \ref{fig:logprobs}.

\begin{comment}
We will use the Metropolis algorithm to obtain samples from a multivariate normal distribution to demonstrate this process.
Suppose also that we desire to obtain samples from a multivariate normal distribution with arbitrary covariance matrix $\Sigma$, and that this is difficult (obviously we can do this directly in Python, but this is merely a tutorial to see how the Metropolis algorithm works). Suppose further that we are able to easily compute the ratio of the density of this distribution at two points $\mathbf{x}$ and $\mathbf{y}$ of length $K$, i.e.
\begin{align*}
\frac{N(\mathbf{x} \; ; \; \mu, \Sigma)}{N(\mathbf{y} \; ; \; \mu, \Sigma)} & = \frac{\frac{1}{(2\pi)^{K/2}|\Sigma|^{1/2}} e^{-\frac{1}{2}(\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}}{\frac{1}{(2\pi)^{K/2}|\Sigma|^{1/2}} e^{-\frac{1}{2}(\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}} \\
& = \frac{e^{-\frac{1}{2}(\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}}{e^{-\frac{1}{2}(\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}} \\
& = e^{-\frac{1}{2}\left((\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu) - (\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{y} - \mu)\right)}
\end{align*}

\begin{problem} \label{problem1}
Write an acceptance function that computes
\begin{equation*}
p = \min \{1, e^{-\frac{1}{2}\left((\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu) - (\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{y} - \mu)\right)}\}
\end{equation*}
given $\mathbf{x}, \mathbf{y}, \mu,$ and $\Sigma$, and then draws from a Bernoulli distribution with parameter $p$. It should return a $1$ if it accepts the new state, and a $0$ if it rejects it.
\end{problem}

Specifically, we will try to sample from the distribution centered at the origin, with covariance matrix
\begin{equation*}
\Sigma = \left[ \begin{array}{cc} 12 & 4 \\ 4 & 16 \end{array} \right]
\end{equation*}

\begin{lstlisting}
>>> mu = np.zeros(2)
>>> sigma = np.array([[12., 10.], [10., 16.]])
\end{lstlisting}

We will let $Q(\mathbf{x} | \mathbf{y}) = N(\mathbf{x} \; ; \; \mathbf{y}, I)$ be our proposal distribution, given that we are currently in state $\mathbf{y}$, i.e. we propose a new state by drawing from the multivariate normal distribution centered at $\mathbf{y}$ with identity covariance. We then accept according to our acceptance probability, computed in Problem \ref{problem1}.

\begin{problem}
Write a function that accepts a current state, the mean and covariance from the distribution we desire to sample from, and returns the next state. We should propose according to $Q$ described above, and accept according to the function in Problem \ref{problem1}.
\end{problem}

We now have a way to sample a new state from an old state. As we've stated before, this method creates a Markov chain that \emph{converges} to the desired distribution; at the beginning, however, if our initial guess is highly unlikely for the desired distribution, it may take a while before we get there. We would like to measure our progress.

\begin{problem}
Write a function that computes the log of the multivariate normal density of a point $\mathbf{x}$ given a mean $\mu$ and covariance matrix $\Sigma$. Be intelligent about how you implement this, that is, do not simply compute the multivariate normal density and then take the log of it, as this may lead to numerical issues. The whole purpose of looking at the multivariate log is to make this more stable.
\end{problem}

We will finally put everything together.

\begin{problem}
Write a function that accepts an initial point $\mathbf{x}$, a mean $\mu$ and covariance $\Sigma$ for the desired sampling distribution, and which performs the Metropolis algorithm for a number of iterations, $n\_samples$. Save each sample $\mathbf{x}$ as produced by the algorithm. Also compute the log of the multivariate normal density of each point, and return both the samples and the logprobs.
\end{problem}

We would like to see how long it takes for our algorithm to converge to the right distribution. We can do this by plotting the log-probs returned by our function. Here we use an initial state $\mathbf{x} = \left[\begin{array}{cc} 100 & 100 \end{array}\right]$.

\begin{figure}[h]
\centering
\includegraphics[width=\textwidth]{logprobs.pdf}
\caption{Log probabilities of our samples.}
\end{figure}

From this we can see that after between $300$ and $500$ iterations, we had converged to the correct distribution. We can visualize the path of our sampler by plotting the samples themselves:
\begin{figure}[h]
\centering
\includegraphics[width=\textwidth]{samples.pdf}
\caption{Samples from the Metropolis algorithm.}
\end{figure}

\begin{problem}
Using $\mu$ and $\Sigma$ as defined previously and using an initial state $\mathbf{x} = \left[ \begin{array}{cc} 1000 & -1000 \end{array} \right]$ run your Metropolis sampler for $10000$ iterations. Plot the log probs as well as the samples. How long did it take to converge?
\end{problem}
\end{comment}
\section*{The Ising Model}
In statistical mechanics, the Ising model describes how atoms interact in ferromagnetic material. More specifically, we assume that there is some lattice $\Lambda$ of sites. We say $i \sim j$ if $i$ and $j$ are adjacent sites. With each site $i$ in our lattice is an associated \emph{spin} $\sigma_{i} \in \{\pm 1\}$. A \emph{state} in our Ising model is a particular spin configuration $\sigma = (\sigma_{j})_{j \in \Lambda}$. If $L = |\Lambda|$, then there are $2^{L}$ possible states in our model. If $L$ is large, the state space becomes huge, which is why MCMC sampling methods (in particular the Metropolis algorithm) are so useful in calculating model estimations.

With any spin configuration $\sigma$, there is an associated energy
\[
H(\sigma) = -J \sum_{i \sim j} \sigma_{i} \sigma_{j}
\]
 where $J > 0$ for ferromagnetic materials, and $J < 0$ for antiferromagnetic materials. Throughout this lab, we will assume $J = 1$, leaving the energy equation to be $H(\sigma) = -\sum_{i \sim j} \sigma_{i}\sigma_{j}$ where the interaction from each pair is added only once.

We will consider a lattice that is a $100 \times 100$ square grid. The adjacent sites for a given site are those directly above, below, to the left, and right of the site, so to speak. For sites on the edge of the grid, we assume it wraps around. In other words, a site at the furthest left of the grid is adjacent to the corresponding site on the furthest right. Thus, a single spin configuration can be represented as a $100 \times 100$ array, with entries from $\{\pm 1\}$.

\begin{problem}
Write a function that initializes a spin configuration for an $n \times n$ lattice. It should return an $n \times n$ array, each entry of which is either $1$ or $-1$, chosen randomly. Test this for the grid described above, and plot the spin configuration using \li{matplotlib.pyplot.imshow}. It should look fairly random, as in Figure \ref{fig:random_spin}.
\end{problem}

\begin{figure}
\centering
\includegraphics[width=.5\textwidth]{init.pdf}
\caption{Spin configuration from random initialization.}
\label{fig:random_spin}
\end{figure}

\begin{problem}
Write a function that computes the energy of a spin configuration with an $n \times n$ lattice described above. Make sure that you do not double count site pair interactions!
\end{problem}

Different spin configurations occur with different probabilities, depending on the energy of the spin configuration and $\beta > 0$, a quantity inversely proportional to the temperature. More specifically, for a given $\beta$, we have
\begin{equation*}
\mathbb{P}_{\beta}(\sigma) = \frac{e^{-\beta H(\sigma)}}{Z_{\beta}}
\end{equation*}
where $Z_{\beta} = \sum_{\sigma} e^{-\beta H(\sigma)}$. Because for our particular lattice there are $2^{100 \cdot 100} = 2^{10000}$ possible spin configurations, computing this sum is unfeasible. However, the numerator is quite simple, provided we can efficiently compute the energy $H(\sigma)$ of a spin configuration. Thus the ratio of the probability densities of two spin configurations is simple:
\begin{align*}
\frac{\mathbb{P}_{\beta}(\sigma^{*})}{\mathbb{P}_{\beta}(\sigma)} & = \frac{e^{-\beta H(\sigma^{*})}}{e^{-\beta H(\sigma)}} \\
& = e^{\beta (H(\sigma) - H(\sigma^{*}))}
\end{align*}

The simplicity of this ratio should lead us to think that a Metropolis algorithm might be appropriate by which to sample from the spin configuration probability distribution, in which case our acceptance probability would be
\begin{equation*}
A(\sigma^{*}, \sigma) = \begin{cases} 1 & \mbox{if } H(\sigma^{*}) < H(\sigma) \\ e^{\beta (H(\sigma) - H(\sigma^{*}))} & \mbox{ otherwise.} \end{cases}
\end{equation*}

By choosing our transition matrix $Q$ cleverly, we can also make it easy to compute the energy for any proposed spin configuration. We restrict our possible proposals to only those spin configurations in which we have flipped the spin at exactly one lattice site, i.e. we choose a lattice site $i$ and flip its spin. Thus, there are only $L$ possible proposal spin configurations $\sigma^{*}$ given $\sigma$, each being proposed with probability $\frac{1}{L}$, and such that $\sigma_{j}^{*} = \sigma_{j}$ for all $j \neq i$, and $\sigma_{i}^{*} = - \sigma_{i}$. Note that we would never actually write out this matrix (it would be $2^{10000} \times 2^{10000}$!!!). Computing the proposed sites energy is simple: if the spin flip site is $i$, then we have $H(\sigma^{*}) = H(\sigma) + 2\sum_{j: j \sim i} \sigma_{i}\sigma_{j}$.

\begin{problem}
Write a function that proposes a new spin configuration given the current spin configuration on an $n \times n$ lattice described above. This function simply needs to return a pair of indices $(i,j)$, chosen with probability $\frac{1}{n^{2}}$.
\end{problem}

\begin{problem}
Write a function that computes the energy of a proposed spin configuration, given the current spin configuration, its energy, and the proposed spin flip site indices.
\end{problem}

\begin{problem}
Write a function that accepts or rejects a proposed spin configuration, given the current configuration. It should accept the current energy, proposed energy, and $\beta$, and return a boolean.
\end{problem}

To track the convergence of the markov chain, we would like to look at the probabilities of each sample at each time. However, this would require us to compute the denominator $Z_{\beta}$, which as we explained previously is generally the reason we have to use a Metropolis algorithm to begin with.
We can get away with examining only $-\beta H(\sigma)$. We should see this increase as the algorithm proceeds, and converge once we are sampling from the correct distribution. Note that we don't expect these values to converge
to a specific value, but rather a restricted range of values.

\begin{problem}
Write a function that initializes a spin configuration for an $n \times n$ lattice as done previously, and then performs the Metropolis algorithm, choosing new spin configurations and accepting or rejecting them. It should burn in first, and then iterate $n\_samples$ times, keeping every $100^{\text{th}}$ sample (this is to prevent memory failure) and all of the above values $-\beta H(\sigma)$ (keep the values even for the burn in period). It should also accept $\beta$ as an argument, allowing us to effectively adjust the temperature for the model.
\end{problem}

\begin{problem}
Test your Metropolis sampler on a $100 \times 100$ grid, with $200000$ iterations, with $n\_samples$ large enough to get $50$ samples, testing with $\beta = 1$ and then with $\beta = 0.2$. Plot the proportional log probabilities, and also plot a late sample from each test using \li{matplotlib.pyplot.imshow}. How does the ferromagnetic material behave differently with differing temperatures? Recall that $\beta$ is an inverse function of temperature. You should see more structure with lower temperature, as illustrated in Figures \ref{fig:config1} and \ref{fig:config2}.
\end{problem}

\begin{figure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=\textwidth]{beta1_logprobs.pdf}
		\caption{Proportional log probs when $\beta = 1$.}
		\label{fig:lprobs1}
	\end{subfigure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=.75\textwidth]{beta1.pdf}
		\caption{Spin configuration sample when $\beta = 1$.}
		\label{fig:config1}
	\end{subfigure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=\textwidth]{beta0_2_logprobs.pdf}
		\caption{Proportional log probs when $\beta = 0.2$.}
		\label{fig:lprobs2}
	\end{subfigure}
	\begin{subfigure}[b]{.49\textwidth}
		\centering
		\includegraphics[width=.75\textwidth]{beta0_2.pdf}
		\caption{Spin configuration sample when $\beta = 0.2$.}
		\label{fig:config2}
	\end{subfigure}
\end{figure}


\section{Convex Functions}
We take a break from calculus of variations and go back to calculus in $\mathbb R^n$ for a moment.
There is a class of functions whose stationary points are easy to classify.
\begin{definition}
    A set $S\subset\mathbb R^n$ is convex if for any $\underline{x},\underline{y}\in S$ and $t\in[0,1]$, we have $(1-t)\underline{x}+t\underline{y}\in S$.
\end{definition}
\begin{definition}
    A graph of a function $f:S\to\mathbb R$ where $S\subset\mathbb R^n$ is a surface described by $z-f(\underline{x})=0$, which is a surface in $S\times\mathbb R$.\\
    A chord of $f$ is a line segment in $\mathbb R^n\times\mathbb R$ joining two points on the graph of $f$.
\end{definition}
\begin{definition}
    Let $S\subset\mathbb R^n$ and $f:S\to\mathbb R$ be a function.
    We say $f$ is convex if $S$ is convex and for any $\underline{x},\underline{y}\in S$ and $t\in[0,1]$ we have
    $$f((1-t)\underline{x}+t\underline{y})\le (1-t)f(\underline{x})+tf(\underline{y})$$
\end{definition}
Loosely speaking, a function is convex iff its domain is convex and every chord of it is above (or on) it, where the notion of ``above'' is in the sense of the last component of $\mathbb R^n\times\mathbb R$ in our above definition.
\begin{remark}
    We can define concave functions correspondingly by saying a function $f$ is concave iff $-f$ is convex.\\
    A function $f$ is strictly convex if the $\le$ in the definition is replaced by $<$ given $t\in (0,1)$.
    It is obvious that $f$ is strictly convex only if it is convex.
    This is due to the observation that the definition does not really change if we replace $t\in[0,1]$ by $t\in(0,1)$.
\end{remark}
\begin{example}
    The function $f:\mathbb R\to\mathbb R$ by $x\mapsto x^2$ is convex as $\mathbb R$ is convex and for any $x,y\in\mathbb R,t\in (0,1)$,
    \begin{align*}
        f((1-t)x+ty)-(1-t)f(x)-tf(y)&=((1-t)x+ty)^2-(1-t)x^2-ty^2\\
        &=-(1-t)t(x-y)^2\\
        &<0
    \end{align*}
    Hence $f$ is strictly convex.
\end{example}
\begin{example}
    Consider the function $f:\mathbb R\setminus\{0\}\to\mathbb R$ by $x\mapsto x^{-1}$.
    It is not convex as its domain is not, but its restriction on $\mathbb R_{>0}$ is (strictly) convex.
\end{example}
\subsection{Conditions for Convexity}
There are quite a few tests for convexity of a function $f$ (mostly with some properties).
\begin{proposition}
    Suppose $f:S\to\mathbb R$ is differentiable for some convex $S$, then $f$ is convex iff for any $\underline{x},\underline{y}\in S$,
    $$f(\underline{y})\ge f(\underline{x})+(\underline{y}-\underline{x})\cdot\nabla f(\underline{x})$$
\end{proposition}
\begin{proof}
    If the inequality is true, then by applying it twice,
    $$\begin{cases}
        f(\underline{x})\ge f(\underline{z})+(\underline{x}-\underline{z})\cdot\nabla f(\underline{z})\\
        f(\underline{y})\ge f(\underline{z})+(\underline{y}-\underline{z})\cdot\nabla f(\underline{z})
    \end{cases}$$
    So for $t\in (0,1)$, we set $\underline{z}=(1-t)\underline{x}+t\underline{y}\in S$, then the result follows by adding $1-t$ times the first inequality to $t$ times the second.\\
    Conversely, set $h:[0,1]\to\mathbb R$ by
    $$h(t)=(1-t)f(\underline{x})+tf(\underline{y})-f((1-t)\underline{x}+t\underline{y})\ge 0$$
    by convexity.
    It is also differentiable in $[0,1]$ as $f$ is.
    Now
    $$-f(\underline{x})+f(\underline{y})-(\underline{y}-\underline{x})\cdot\nabla f(\underline{x})=h^\prime(0)\ge 0$$
    as $h(0)=0$ and $h\ge 0$.
\end{proof}
\begin{corollary}
    If $f$ is convex and has a stationary point at $\underline{x}$, then $\underline{x}$ is a global minimum of $f$.
\end{corollary}
\begin{proof}
    Immediate from the preceding proposition.
\end{proof}
\begin{proposition}
    Let $S$ be convex and $f:S\to\mathbb R$.
    If $(\nabla f(\underline{y})-\nabla f(\underline{x}))\cdot (\underline{y}-\underline{x})\ge 0$ for any $\underline{x},\underline{y}\in S$, then $f$ is convex.
\end{proposition}
Note that if $S\subset\mathbb R$, then it is just saying that $f^\prime$ is monotonically increasing
\begin{proof}
    Exercise.
\end{proof}
\begin{proposition}
    Assume that $f$ is twice differentiable, then $f$ is convex iff $H$ is always nonnegative definite where $H$ is the Hessian of $f$, i.e.
    $$H_{ij}=\frac{\partial^2f}{\partial x_i\partial x_j}$$
\end{proposition}
An easy extension of this asserts the strict convexity of $f$ given that $H$ is positive definite.
We will only show the forward direction.
\begin{proof}
    If $f$ is convex, then by taking $\underline{y}=\underline{x}+\underline{h}$ in the preceding proposition we have
    $$\underline{h}\cdot(\nabla f(\underline{x}+\underline{h})-\nabla f(\underline{x}))\ge 0$$
    for any $\underline{h}\in S-\underline{x}$.
    So for small $|\underline{h}|$,
    $$\frac{\partial f}{\partial x_i}(\underline{x}+\underline{h})=\frac{\partial f}{\partial x_i}(\underline{x})+h_jH_{ij}(\underline{x})+O(|\underline{h}|^2)$$
    where the summation is implied.
    Hence
    $$h_ih_jH_{ij}+O(|\underline{h}|^2)\ge 0$$
    by taking $|\underline{h}|$ small, we obtain the nonnegative definiteness of $H(\underline{x})$.
\end{proof}
\begin{example}
    Take $f(x,y)=1/(xy)$ defined on the domain $(\mathbb R_{>0})^2$.
    Then
    $$H=\frac{1}{xy}\begin{pmatrix}
        2/x^2&1/(xy)\\
        1/(xy)&2/y^2
    \end{pmatrix}$$
    which has positive determinant and trace, hence its eigenvalues are all positive, therefore $H$ is positive definite hence strictly convex.
\end{example}
\subsection{The Legendre Transform}
\begin{definition}
    Let $S\subset\mathbb R^n$ and $f:S\to\mathbb R$ be a function.
    Its Legendre transform is defined to be
    $$f^\star(\underline{p})=\sup_{\underline{x}\in S}(\underline{p}\cdot\underline{x}-f(\underline{x}))$$
    provided that it exists and is always finite.
\end{definition}
\begin{example}
    Take $S=\mathbb R$, then the Legendre transform is simply the maximum (signed) distance between the line $z=f(x)$ and $z=px$, given that it exists.\\
    Take for example $f(x)=ax^2$ where $a>0$ (not that if $a<0$ then the Legendre transform does not exist anywhere), then
    $$f^\star(p)=\sup_{x\in\mathbb R}(px-ax^2)=\frac{p^2}{4a}$$
    by some calculation.
    Now, by taking $a\mapsto 1/(4a)$ we obtain $(f^\star)^\star(s)=as^2$, hence $(f^\star)^\star=f$.
\end{example}
It turns out to be always true that $(f^\star)^\star=f$ (in a domain where both of them are defined) if $f$ is convex.
\begin{proposition}
    $f^\star$ is convex in any convex subset $T$ of its domain.
\end{proposition}
\begin{proof}
    By definition, for any $\underline{p},\underline{q}\in T$,
    \begin{align*}
        f^\star((1-t)\underline{p}+t\underline{q})&=\sup_{\underline{x}}((1-t)\underline{p}\cdot\underline{x}+t\underline{q}\cdot\underline{x}-f(\underline{x}))\\
        &=\sup_{\underline{x}}((1-t)(\underline{p}\cdot\underline{x}-f(\underline{x}))+t(\underline{q}\cdot\underline{x}-f(\underline{x})))\\
        &\le (1-t)f^{\star}(\underline{p})+tf^\star(\underline{q})
    \end{align*}
    As desired.
\end{proof}
In practice, if $f$ is convex and differentiable, then if its Legendre transform at $\underline{p}$ exists we can (given certain conditions on the domain and/or the function) find it by the equation
$$\nabla(\underline{p}\cdot\underline{x}-f(\underline{x}))=\underline{0}\implies \underline{p}=\nabla f$$
If $f$ is strictly convex, then there is an unique inversion $\underline{x}=\underline{x}(\underline{p})$, so $f^\star(p)=\underline{p}\cdot\underline{x}(\underline{p})-f(\underline{x}(\underline{p}))$.
\subsection{Applications to Thermodynamics}
A many-particle system (like gas) are hard to study directly from the first principles (like Newton's Laws), hence are often analysed by looking at microscopic variables like the pressure $P$, volume $V$, temperature $T$ or entropy $S$ (which measures the global disorder of the system) or internal energy $u(S,V)$.
\begin{definition}
    The Hermholtz free energy is defined by
    $$F(T,V)=\inf_{S}(u(S,V)-TS)$$
\end{definition}
Now we can rewrite this in terms of Legendre transform, that is
$$F(T,V)=\inf_{S}(u(S,V)-TS)=-\sup_{S}(TS-u(S,V))=-u^\star(T,V)$$
Note that the transformation of $u$ here is with respect to $T$ with $V$ fixed as a parameter.
Consequently,
$$\left.\frac{\partial}{\partial S}(TS-u(S,V))\right|_{T,V}=0\implies T=\left.\frac{\partial u}{\partial S}\right|_V$$
There are other examples where we can make use of the Legendre transform too, for example:
\begin{definition}
    The enthalpy of the system is described by
    $$H(S,P)=\inf_{V}(u(S,V)+pV)$$
\end{definition}
Then $H(S,P)=-u^\star(-P,S)$ where the Legendre transform is again taken with $S$ fixed.\\
Like these, Legendre transforms can be a way to swap from $(S,V)$ dependence to dependence of other variables.


\section{Introduction}
The purpose of this paper is to study the relationships between ordered spaces and their induced topology. In particular, we want to study the p
\subsection{Topological spaces}
Using standard notation as in \cite{Bour04}, a topological space is a set $X$ and a subset $\tau \subset \mathcal{P}(X) = 2^X$ fulfulling the following axioms:
\begin{enumerate}
\item $X$ itself and the empty set $\emptyset$ are elements in $\tau$, and
\item $\tau$ is closed under finite union and intersection.
\end{enumerate}
Speaking in the manner of abstract algebra, $\tau$ is a monoid wrt. intersection $\cap$ and union $\cup$ with neutral elements $X$ and $\emptyset$, respectively. Associativity is rather obvious and uniquness of the neutral elements follows immediately from uniqueness of $X$ itself and the empty set.
\subsubsection{Open and closed topological spaces}
Given a family of subsets $\mathcal{U} \subset \tau$, we call $\mathcal{U}$ a family of open subsets (and its elements the open subsets of $X$/in $\tau$)if
$$\bigcup_{U \in \mathcal{U}'} U \in \tau,\ \forall \mathcal{U}' \subset \mathcal{U}.$$
In other words, the topology $\tau(\mathcal{U})$, generated by a family of open sets $\mathcal{U}$, is closed under arbitrary union. $\tau(\mathcal{U})$ is sometimes referred to as an open topology.\\
\indent Given a family of subsets $\mathcal{F} \subset \tau$, we call $\mathcal{F}$ a family of closed subsets
if
$$\bigcap_{F \in \mathcal{F}'} F \in \tau,\ \forall \mathcal{F}' \subset \mathcal{F}.$$
In other words, the topology $\tau(\mathcal{F})$, generated by a family of closed sets $\mathcal{F}$, is closed under arbitrary intersection. $\tau(\mathcal{F})$ is sometimes referred to as a closed topology.\\
\paragraph{Exercise} Let us discuss some examples:
\begin{enumerate}
\item Given $X = \{0, 1, 2, 3\}$ which of the following families of subsets is a topology
\begin{enumerate}
\item $\tau = \left\{\emptyset, \{0, 1\}, \{0, 1, 2\}, \{1, 2, 3\}, X\right\}$.
\item $\tau = \left\{\emptyset, \{0, 1\}, \{0, 1, 2\}, \{0, 1, 3\}, X\right\}$.
\item $\tau = \left\{\emptyset, \{2\}, \{3\}, \{2, 3\}, X\right\}$.
\item $\tau = \left\{\emptyset, \{0, 1\}, \{2, 3\}, X\right\}$.
\item $\tau = \left\{\emptyset, \{1, 2\}, \{2, 3\}, \{1, 2, 3\}, X\right\}$.
\item $\tau = \left\{\emptyset, \{1, 3\}, \{2, 3\}, \{1, 2, 3\}, X\right\}$.
\item $\tau = \left\{\emptyset, \{1, 2\}, \{0, 3\}, \{1, 2, 3\}, X\right\}$.
\end{enumerate}
\item Compute the topology $\tau(\beta)$ generated by the following families $\beta$ of subsets of $X$ as above:
\begin{enumerate}
\item $\beta = \left\{\{0\}, \{2\}, \{1, 3\}\right\}$.
\item $\beta = \left\{\{1\}, \{2\}, \{1, 3\}\right\}$.
\end{enumerate}
\end{enumerate}
\paragraph{Complement}
The complement of any given subset $X' \subset X$ for some set $X$ - denoted by $X'^{c}$ - is simply:
$$X'^c := \{x \in X : x \notin X'\}.$$
The complement of an open set $U \in \tau$ is closed in $\tau$ and, conversely, the complement of a closed set is, indeed, open. Therefore, the topology of the complement of an open topology is closed and, conversely, the topology of the complement of a closed topology is open.\\
\indent More interestingly, the subset $\emptyset, X \in \tau$ are closed and open at the same time (as $X$ can be seen as the union of all its open sets, whatever they are, and $\emptyset$ is the intersection of all closed sets). Thus, $X^c = \emptyset$ is open and $\emptyset^c = X$ is closed. We call these sets clopen subsets in $\tau$. A clopen topology $\tau$ is generated by clopen sets. Furthermore, if all elements in $\tau$ are clopen, we can show that $\tau$ is a subtopology of $\mathcal{P}(X)$.
\subsection{Ordered sets}
Given a non empty set $X$, a relation is a subset $R \subset X \times X$. We say a pair $(x,y) \in X^2$ holds $R$ if $(x,y) \in R$ or in a manner of notation: $x R y$. An equivalence relation $\Delta \subset X^2$ is a relation, fulfilling the following axioms
\begin{description}
\item[Reflexivity] $x \Delta x$, for all $x \in X$,
\item[Symmetry] $x \Delta y \Leftrightarrow y \Delta x$, for all $x, y \in X$ and
\item[Transitivity] $x \Delta y$ and $y \Delta z$ implies $x \Delta z$, for all $x, y, z \in X$.
\end{description}


\section{Constant Coefficients}
\begin{definition}
	The general form of an nth order linear equation is
	\begin{equation*}
		a_n(x)y^{(n)} + a_{n-1}y^{(n-1)} + \ldots + a_1(x)y' + a_0y = b(x).
	\end{equation*}
	If each $a_i(x)$ is a constant, then the equation has constant coefficients.
\end{definition}

\noindent
We already know how to solve linear first order differential equations using an integrating factor, but let's see if we can develop a method that can solve any order linear, homogeneous differential equation with constant coefficients.

\begin{example}
	Let's try to solve the following equation by guessing and checking likely solutions.
	\begin{equation*}
		y'' - 3y' + 2y = 0
	\end{equation*}
\end{example}
\noindent
Exponentials seem like good guesses. Let's try an exponential of the form $y = Ce^{rx}$ first.
\begin{equation*}
	\left(Ce^{rx}\right)'' - 3\left(Ce^{rx}\right)' + 2\left(Ce^{rx}\right) =  0
\end{equation*}
\begin{equation*}
	Cr^2e^{rx} - 3rCe^{rx} + 2Ce^{rx} = Ce^{rx}\left(r^2 - 3r + 2\right) = 0.
\end{equation*}
Since $Ce^{rx} \neq 0$ unless $C = 0$, we only need to solve the quadratic.
Note that the coefficients of the quadratic are the same as the coefficients in the original differential equation.
\begin{equation*}
	r^2 - 3r + 2 = 0 \implies r = 1 \text{, } 2.
\end{equation*}
So, our two fundamental solutions are
\begin{equation*}
	\begin{cases}
		y = C_1e^{x} \\
		y = C_2e^{2x}
	\end{cases}.
\end{equation*}
Since these two fundamental solutions are linearly independent, the general solution is the sum of the fundamental solutions.
\begin{equation*}
	y = C_1e^{x} + C_2e^{2x}
\end{equation*}

% Auxillary Equation
\input{./higherOrder/constCoeffs/auxillaryEquation.tex}