\subsection{Binary classification}

Classification models are a type of regression model, where \(y\) is discrete rather than continuous.

So we want to find a mapping from a vector \(X\) to probabilities across discrete \(y\) values.

A classifier takes \(X\) and returns a vector.

For a classifier we have \(K\) classes.

\subsection{Classification}

Confusion matrix. true positve, false positive, false negative, true negative

Can use this to get

Accuracy: percentage correct

Precision: percentage of positive predictions which are correct

Recall (sensitivity): percentage of poitive cases that were predicted as positive

Specificity: percentage of negative cases preicated as negative


\section{Calculus}

\begin{definition}[Adiabatic]
    adiabatic wall between two thermodynamic systems does not allow heat or
    matter to pass across it.

\end{definition}

\begin{definition}[A priori estimate]
    An estimate for a size of a solution or it's deriviates of a PDE.\
    The estimate is made before the solution is known to exists.

\end{definition}

\begin{definition}[Affine]
    $
    f(x_{1}, x_{2}, \dots, x_{n}) =
    a_{1}x_{1}, a_{2}x_{2}, \dots a_{n}x_{n}
    $
\end{definition}

\begin{definition}[Analytic function]
    A function given by~\nameref{convergence}~\nameref{powerseries}.

    Any polynomial, exponential and trigonometric function is analytic.
    Functions that are not differentiable at given points are not analytic.

\end{definition}

\begin{definition}[arcsin]
    \begin{align*}
        \sin{y} &= x \\
        \arcsin{x} &= \sin^{-1}{x} = y
    \end{align*}

    Properties:
    \begin{itemize}
        \item $\arcsin{x} = \frac{\pi}{2} - \arccos{x} = 90º - \arccos{x}$
        \item $\cos{(\arcsin{x})} = \sin{(\arccos{x})} = \sqrt{1-x^{2}}$
        \item $ x = -1 \rightarrow \arcsin{x} = -1\times\frac{\pi}{2}$
        \item $ x = 1 \rightarrow \arcsin{x} = \frac{\pi}{2}$
        \item $ x = 0 \rightarrow \arcsin{x} = 0$
    \end{itemize}
\end{definition}

\begin{definition}[arccos]\label{arccos}
    Properties:
    \begin{itemize}
        \item $ x = -1 \rightarrow \arccos{x} = \pi$
        \item $ x = 1 \rightarrow \arccos{x} = 0$
        \item $ x = 0 \rightarrow \arccos{x} = \frac{\pi}{2}$
    \end{itemize}
\end{definition}

\begin{definition}[Arc length]
    Length of a curve when straightened out.
\end{definition}
\begin{definition}[arccos]
    While cosine shows the relation between lengths in a triangle,
    arccos gives the angle.
\end{definition}

\begin{definition}[Arithmetic-geometric mean inequality]\label{arigeo}
    $
    \newline {(\prod\limits_{i = 1}^{k} a_{i})}^{1/k}
    \leq {1 \over k} \sum\limits_{i = 1}^{k} a_{i}
    $
\end{definition}

\begin{definition}[Azimuth]
     is an angular measurement in a spherical coordinate system. The vector
     from an observer (origin) to a point of interest is projected
     perpendicularly onto a reference plane; the angle between the projected
     vector and a reference vector on the reference plane is called the
     azimuth.

     \includegraphics[scale=0.3]{azimuth.png}

\end{definition}

\begin{definition}[Bijection]
    \begin{align}
        S,R \text{\ are sets} \\
        \forall{i \in S}, \exists!{f(i) \in R} \wedge \\
        \forall{i \in R}, \exists!{f(i) \in S} \\
    \end{align}
\end{definition}

\begin{definition}[Binomial]\label{binomial}
    A polynomial which is the sum of two terms.
\end{definition}

\begin{definition}[Boundary value problem]
    A differential equation with additional restrains. A solution to a boundary
    value problem is a solution to the differential equation which also
    satisfies the boundary conditions.

\end{definition}

\begin{definition}[Cauchy sequence]
    a sequence whose elements become arbitrarily close to each other as the
    sequence progresses.
    Think of \textit{two} curves that approximate each other.

\end{definition}

\begin{definition}[Cardinal number, ]
    The size of a set.
\end{definition}

\begin{definition}[Closure]
    The closure of a set is the points in the set including the limit points.
\end{definition}

\begin{definition}[Convergence]\label{convergence}
    A series is convergent is the sum of it's elements approaches a given
    number.
\end{definition}

\begin{definition}[Convolution]
    Dictionary: a coil or twist, especially one of many.\newline
    Informal: an expression of how a shape from one function is modified by
        the other.

    Can also be used to smoothen a discontinous a function (making it continous
    on the given range). We need to normalize our $g(x - \tau)$ so that we do
    not continously increase the $f(x)$

    A convolution in time domain is equal to multiplication in frequency domain.

    Convolution is a common way to combine wavelet to recreate signals.
\end{definition}

\begin{definition}[Continuous]
In mathematics, a continuous function is a function for which
``small'' changes in the input result in ``small'' changes in the output.

As an example, consider the function $h(t)$, which describes the height of a
growing flower at time t. This function is continuous. By contrast, if $M(t)$
denotes the amount of money in a bank account at time t, then the function
jumps whenever money is deposited or withdrawn, so the function $M(t)$ is
discontinuous.

\end{definition}

\begin{definition}[Continuous variables]
    There are three types of continuous variables:
    \begin{center}
    \begin{description}
        \item[Nominal] Categorized within groups: box 1, 2, 3 or 4.
        \item[Dichotomous] Boolean, yes or no.
        \item[Ordinal] A nominal variable, just that different groups give
            different values. E.g.\ to like something on a scale of 1 to 10
            gives you a ordinal range
    \end{description}
\end{center}
\end{definition}

\begin{definition}[Conjugate]
    is a~\nameref{binomial} formed by negating the second term of
    a~\nameref{binomial}. The conjugate of $x + y$ is $x − y$.

    A functions' conjugate is often denoted $\overline{f(x)}$.

\end{definition}

\begin{definition}[Concave]
    Let $f$ be a function defined on the interval $[x_{1}, x_{2}]$.
    This function is concave according to the definition if, for every pair of
    numbers a and b with $x_{1} \leq a \leq x_{2}$ and $x_{1} \leq b \leq
    x_{2}$, the line segment from $(a,  f (a))$ to $(b,  f (b))$
    lies on or below the function.

    \begin{itemize}
        \item The sine function is concave on the interval $[0, \pi]$
        \item Concave if every line segment joining two point is never above
              the graph
        \item Concave functions has $f\prime\prime(x) \leq 0$ in a given
              interval
    \end{itemize}
\end{definition}

\begin{definition}[Congruence]
    Similar shape and growth, just different scalar / rotation.
\end{definition}

\begin{definition}[Damping]
    is an influence within or upon an oscillatory system that has the effect of
    reducing, restricting or preventing its oscillations.
    Underdamped systems will start bouncy and then stop.

\end{definition}

\begin{definition}[Differential operator]
    An operator to do differentiation. This is mainly to abstract
    differenentiation.

\end{definition}

\begin{definition}[Discrete Fourier Transform]\label{dft}
    converts a finite list of equally spaced samples of a function into the
    list of coefficients of a finite combination of complex sinusoids, ordered
    by their frequencies, that has those same sample values.

\end{definition}

\begin{definition}[Discrete Laplace transform]
    An integral transfrom, which in 2d uses the kernel
    % \begin{bmatrix}
    %     0 & 1 & 0 \\
    %     1 & -4 & 1 \\
    %     0 & 1 & 0
    % \end{bmatrix}
\end{definition}

\begin{definition}[Discrete Sine Transform]
    Similar to~\nameref{dft}, but uses only a real matrix.

\end{definition}

\begin{definition}[Discretization]
    concerns the process of transferring
    continuous models and equations into discrete counterparts.
    AKA\ smoothening curves to avoid jumps.

\end{definition}

\begin{definition}[Displacement]
    The shortest distance from source $s$ to $t$ (like air-distance).

\end{definition}

\begin{definition}[Divergence]\label{divergence}
    measures the magnitude of a~\nameref{vectorfield}'s source or sink at a
    given point, in terms of a signed scalar.

    E.g.\ air can be thought of to have a point $s \text{ and } t$, where they
    push out hot and cool air, respectively. From these points, you can create
    a~\nameref{vectorfield} that shows how air spreads from $s \text{ and }
    t$. The divergence measures the collected value from each of
    these~\nameref{vectorfield}s.

    The operator for divergence is \verb|{div}|, represented by nabla.

    Given vector field $F = Ui, Vj, Wk$:
    \begin{align}
            div \textbf{F} = \nabla \cdot F =
            \frac{\partial{U}}{\partial{x}}+
            \frac{\partial{V}}{\partial{y}} +
            \frac{\partial{W}}{\partial{z}}
    \end{align}

\end{definition}

\begin{definition}[Elementary function]
    In mathematics, an elementary function is a function of one variable built
    from a finite number of exponentials, logarithms, constants, and $nth$ roots
    through composition and combinations using the four elementary operations
    $(+ – × ÷)$.

\end{definition}

\begin{definition}[Even function]\label{evenfunc}
    Kind of like a mirror (90 degrees) around an axis;
    \begin{align}
        f(x) &= f(-x) \\
    \end{align}
    See also~\nameref{oddfunc}
\end{definition}

\begin{definition}[Expansion function]
    A function, with a series, to express a function. The series will in
    most cases be an approximation to the original function.
    Signal functions are often modeled using expansion functions.
\end{definition}

\begin{definition}[Eulers formula]
    $e^{ix} = \cos{x} + i\sin{x}$
\end{definition}

\begin{definition}[Explicit methods]\label{explicitmethod}
    In numerical analysis, an explicit method calculates the state of the system
    using only earlier history. See also~\nameref{implicitmethod}.
\end{definition}

\begin{definition}[Extreme point]
    A point furthest away from something.
\end{definition}

\begin{definition}[Field]
    A physcial quantity that has a value for each point in space and time.
\end{definition}

\begin{definition}[Finite difference]
    The difference in a function for $f(x + a) - f(x + b)$.

    There are three types:
    \begin{description}
        \item[Forward] is of the form $\bigtriangleup{f}(x) = f(x + h) - f(x)$
        \item[Backward] is of the form $\nabla{f}(x) = f(x) - f(x - h)$
        \item[Centered] is of the form $\delta{f}(x) = f(x + \frac{1}{2}h) - f(x - \frac{1}{2}h)$
    \end{description}
\end{definition}

\begin{definition}[Finite element method]
    (FEM) is a numerical technique for finding approximate solutions to
    boundary value problems for differential equations. It uses variational
    methods (the calculus of variations) to minimize an error function and
    produce a stable solution.  Analogous to the idea that connecting many tiny
    straight lines can approximate a larger circle, FEM encompasses all the
    methods for connecting many simple element equations over many small
    subdomains, named finite elements, to approximate a more complex equation
    over a larger domain.

\end{definition}

\begin{definition}[Filter bank]
    To take a signal and expose it to a series of filters that separate the
    input signal into different channel. One example is sound equalizers:
    you can adjust different parts of the signal.
\end{definition}

\begin{definition}[Fourier transform]
    Map one function onto another function expressed in terms of a
    series of sine and cosine terms, that when joined together form
    a translated domain.

    Fourier transforms have the property that they can cover all the frequencies
    from the original input, but the mapped frequency loses the time domain:
    one can not determine at what time what frequency ``stuff''  was sent.

\end{definition}

\begin{definition}[Fourier Analysis]
    the study of the way general functions may be represented or approximated
    by sums of simpler trigonometric functions

\end{definition}

\begin{definition}[Flow]
    Motion of particles in a given set.
\end{definition}

\begin{definition}[Galerkin methods]
    Methods to convert continous operator problems to discrete ones.
\end{definition}

\begin{definition}[Generalized function]
   A distribution without steps, i.e.\ it is continious.
\end{definition}

\begin{definition}[Group]
    A set and an operator, such that if you take two elements from the set,
    apply the operator to them, then you will ``receive'' an element that
    could also be a part of the set (closure). Also it must satisfy for
    $\forall{a,b} \in S \rightarrow, (a \cdot b) \cdot c = a \cdot (b \cdot c)$ (associativity),
    \dots also indentity and inverses.
    Example: integers and ``+''.

    Groups have symmetry.
\end{definition}

\begin{definition}[Harmonic numbers]
    $H_{n} = 1 + \frac{1}{2} + \frac{1}{3} + \cdots + {1}{n} =
    {\sum\limits_{k = 1}^{k}} \frac{1}{k} \simeq \ln n$
\end{definition}

\begin{definition}[Heavyside Step Function]
    $$
    H(x) = \left\{
            \begin{array}{l l}
                0 & \text{for} x < 0 \\
                \frac{1}{2} & x = 0 \\
                1 & \text{for} x > 0 \\
            \end{array}
        \right.
    $$
    (it looks just like you think\dots)
\end{definition}

\begin{definition}[Heisenberg's uncertanty principle]
    kjkj
\end{definition}

\begin{definition}[Homogenous]
    If a function $f$ is multiplied by a sclar $k$, then the result
    is a number multiplied by the scalar raised to some power $a$.
\end{definition}

\begin{definition}[Hyperbolic]
    Two curves that kind of face each other like bananas.
\end{definition}

\begin{definition}[Identity function]
    $\forall{x}, f (x) = x$
\end{definition}

\begin{definition}[Implicit method]\label{implicitmethod}
    In numerical analysis, an implicit method calculates the state of the system
    using both the current state and future ones. E.g.\ Gauss-seidel method
    See also~\nameref{explicitmethod}.
\end{definition}

\begin{definition}[Idempotence]
    An operation that can be applied multiple times without changing the
    the result. E.g.\ $f (f (x)) = f (x) $
\end{definition}

\begin{definition}[Idenpendent variable]
    A variable that may be given without considering the value of other
    variables. E.g.\ for $y = 4x + 2$, x is indepdent (can be chosen freely)
    whereas $y$ is not: it depends on the value of $x$.
\end{definition}

\begin{definition}[Invariant]
    A sentence that is believed to be true throughout some timespan.
\end{definition}

\begin{definition}[Integral]
    The ``reverse'' to a derivate.

    \begin{align}
        \int_{a}^{b}x^{n} dx = \frac{x^{n+1}}{n+1} + C \\
    \end{align}

    A function is said to be \textit{locally integrable} if it's integral is
    finite in a given domain.

    \textbf{Integration by parts:} $\int{u \times v dx} = u\int{v dx} - \int{u\prime{(v dx)}}dx $

    Some identities:
    \begin{longtable}{|l|l|}
        $\int_{a}^{b}\cos{x} $ & $\sin{x} + C $ \\
    $\int_{a}^{b}\sin{x}$ & $-1 \times \cos{x} + C $\\
    $e^{x}$ & $e^{x} + C $
    \end{longtable}

\end{definition}

\begin{definition}[Integral transform]\label{inttrans}
    Input a function, and output another (using integrals).
    To do the transformation, one typically usues a \textit{kernel function} (e.g.\
    using a stencil in a grid).

    Usually, this is done to find a domain or function that is easier to compute
    on.

    After completing the computations, one performs an inverse integral transform
    to translate back to the original domain.
\end{definition}

\begin{definition}[Interpolating polynomial]
    Given a set of points, define a function $f$ that intersects these points.
\end{definition}

\begin{definition}[Kronecker delta]
    A function for two variables, that returns 1 if the variables are equal,
    and zero otherwise.
    $$
    \delta_{i,j} = \begin{array}{ll}
        0 & if i \neq j \\
        1 & if i = j
        \end{array}
    $$
\end{definition}

\begin{definition}[Law of cosines]
    \begin{align*}
        v \cdot u = |v||u|\cos{\theta} \\
    \end{align*}
    Note that if $v, u$ are unit vectors, their lengths are both 1.
    We can then rewrite the expression as
    \begin{align*}
        v \cdot u &= \cos{\theta} \\
        \arccos{(v \cdot u)} &= \theta
    \end{align*}
\end{definition}

\begin{definition}[Laplace operator]
    Transform a function of $f$ of $t$ to a function $f$ of $s$

    A differential operator given by the~\nameref{divergence} of a gradient
    function in euclidian space.
\end{definition}

\begin{definition}[Laplacian Matrix]
    sometimes called admittance matrix, Kirchhoff matrix or discrete Laplacian,
    is a matrix representation of a graph.

\end{definition}

\begin{definition}[Lattice]
    In mathematics, especially in geometry and group theory, a lattice in
    $\mathbf{R}^n$ is a discrete subgroup of $\mathbf{R}^n$ which spans the real
    vector space $\mathbf{R}^n$. Every lattice in $\mathbf{R}^n$ can be generated
    from a basis for the vector space by forming all linear combinations with
    integer coefficients.

\end{definition}

\begin{definition}[Line integral]
    An integral where the function to be integrated is along a curve.
    The function is usually a~\nameref{vectorfield} or~\nameref{scalarfield}.

\end{definition}

\begin{definition}[Locus]
    a set of points whose location satisfies or is determined by one or more
    specified conditions

\end{definition}

\begin{definition}[Merit function]
    A value that says something about how good we have acheived a goal.
\end{definition}

\begin{definition}[Moment]
    TBD.\
\end{definition}

\begin{definition}[Numerical analysis]
    The study of algorithms that use numerical approximation (as opposed to
    general symbolic manipulations) for the problems of mathematical analysis.

    Given a space $V$, find a finite subspace of $V$, such that you can
    calculate on it\dots

    \includegraphics[scale=1.0]{fem.png}
    A function in  with zero values at the endpoints (blue), and a piecewise
    linear approximation (red)

\end{definition}

\begin{definition}[Odd function]\label{oddfunc}
A function with the same values when you rotate 180 degrees around\dotso{}
    \begin{align}
        - f(x) &= f(-x) \\
        f(x) + f(-x) &= 0
    \end{align}
    See also~\nameref{evenfunc}
\end{definition}

\begin{definition}[Pathological]
    Something that is counter-intuitive. The opposite is \textbf{well-behaved}.

\end{definition}

\begin{definition}[Parabola]
    A U-shaped, 2d, symmetrical curve.
\end{definition}

\begin{definition}[Parameterization]
    Represent a curve as a function.
    \begin{align}
        x = \cos{t} \\
        y = \sin{t}
    \end{align}
    is the parametric representation of a unit circle.
\end{definition}

\begin{definition}[Partial derivative]
    Given a function $f$ with multiple parameters, a partial derivative is a
    derivative with respect to one of those variables.

    Partial derivatives are often denoted by $\partial$.

    To use partial derivation, you often assume that the other variables are
    constants. Otherwise, there is an infinite number of tangent lines at
    any point, so you will have to have some range on the tangents.

\end{definition}

\begin{definition}[Partial differential equation]
    A set of variables, and equations that show how they are all linked
    together.
\end{definition}

\begin{definition}[Pertubation theory]\label{pertubation}
    The ``art'' of finding an approximate solution to a problem by starting
    with an exact solution to an easier problem.
\end{definition}

\begin{definition}[Power series]\label{powerseries}
    \begin{align}
        f(x) = \sum\limits_{n=0}^{\infty}{a_{n}{(x-c)}^{n}}
    \end{align}
\end{definition}

\begin{definition}[Rectangle function]
    $$
    \Pi(x) = \left\{
            \begin{array}{l l}
                0 & \text{for} x > \frac{1}{2} \\
                \frac{1}{2} & \text{for} x = \frac{1}{2} \\
                1 & \text{for} x < \frac{1}{2} \\
            \end{array}
        \right.
    $$
\end{definition}

\begin{definition}[Reference value]
    The optimal/starting value. Note that when calculating differences,
    a reference value implies that order matters, i.e.\ $ x - y \neq y - x$.

\end{definition}

\begin{definition}[Residual]
    In numerical analysis, this is the error from a result.
    E.g.\ in integration we have something-something + $C$, where C is the
    the residual.

    It can (perhaps betterly) be stated that residuals is the error that occurs
    when approximating a function.

\end{definition}

\begin{definition}[Riemanns sum]\label{riemannsum}
    Sum from an integral. Divide the area under/over a curve into rectangles
    or trapezoids, and sum together their area. The smaller the shapes, the
    better.

    It is important to note that~\nameref{riemannsum} is an approximation -
    one can not perfectly reconstruct the area under a curve.
\end{definition}

\begin{definition}[Risk function]
    Gives the expected value from a lossy function (compare two results, diff).
\end{definition}

\begin{definition}[Sine wave]
or sinusoid is a mathematical curve that describes a smooth repetitive
oscillation.

Its most basic form as a function of time $(t)$ is:

$y(t) = A\sin(2 \pi f t + \varphi) = A\sin(\omega t + \varphi)$
where:
\begin{description}
    \item[A], the amplitude, is the peak deviation of the function from zero.
    \item[f], the ordinary frequency, is the number of oscillations (cycles)
        that occur each second of time.
    \item[ω] = 2πf, the angular frequency, is the rate of change of the
    function argument in units of radians per second
    \item[$\varphi$], the phase, specifies (in radians) where in its cycle the
        oscillation is at t = 0. I.e.\ the ``horizontal shift'' from a regular
        sine wave(????).
    If you e.g.\ represent two functions, $f(x) = \sin{x}$ and $g(x) = \sin{x} + 3$
    , the difference, i.e.\ phase shift, is 3. Note that these two functions
    have the same amplitude and frequency.
\end{description}

\end{definition}

\begin{definition}[Standard form]
    To write a number as a power of 10.
\end{definition}

\begin{definition}[Stiffness matrix]
    A system of linear equations that must be solved to get an approximate
    solution to a differential equation.

    To do this, consider Poission problem:
\end{definition}

\begin{definition}[Taylor series]
    A representation of a function as an infinite sum of terms that are
    calculated from the derivatives at a point in the function.
    The formula is:
    \begin{align}
        \sum\limits_{n=0}^{\infty}{\frac{f^{(n)}(a)}{n!}{(x-a)}^{n}}
    \end{align}
\end{definition}

\begin{definition}[Shift invariant system]
    A discrete form of~\nameref{TIS} - the timesteps are discretized to follow
    a lattice spacing.
\end{definition}

\begin{definition}[Stationary point]
    A place in which the derivative is zero.
\end{definition}

\begin{definition}[Symmetric function]
    A function where the ordering of the parameters does not matter.
    \begin{align}
        f(x,y) &= x + y \\
        f(y,x) &\equiv f(x,y)
    \end{align}
\end{definition}

\begin{definition}[Time invariant system]\label{TIS}
    A function with an output that does not depend on the output:
    \begin{align}
        x(t) &= y(t) \\
        x(t + \delta) &= y(t + \delta)
    \end{align}

    One example is the derivative function of polynomials (and others?) -
    no matter what timestep you choose - the value if the same.

    The ``antonym'' would be a time variant system.
\end{definition}

\begin{definition}[Translation invariant system]
    A system such that after a translation from A to T, any operator
    applied to A yields the same results in T.
\end{definition}

\begin{definition}[Unity]
    The number one. ``Unit functions'', ``unit circles'' all have the number 1.
\end{definition}

\begin{definition}[Univariate]
    An equation with only one variable.
\end{definition}

\begin{definition}[Vector Field]\label{vectorfield}
    An assigment of direction for a given set of points in an euclidian space.
    E.g.\ select every (10n, 10n) pixels in an image and get their derivative.
\end{definition}

\begin{definition}[Weak formulation]
    Transfer a concept from linear algebra to solve equations in other field.

    Equations that have weak formulations are not required to be correct for
    all input (:: weak solutions, PDE might not have derivates but you make pretend)

\end{definition}

\begin{definition}[Window function]
    A function that is zero outside of a given domain.
\end{definition}


\section{Approximation results for various activation functions}
\input{6DL/BarronSpace}
%\input{6DL/FourierRepresentation}
%\input{6DL/DoubleFourier}
%\input{6DL/DoubleFourier2}
\input{6DL/BsplineApprox}
%\input{6DL/ReLUFourierSimple}
%\input{6DL/ReLUFourier}
\input{6DL/PeriodicActivation}
\endinput
 Since $f(x)$ is real-valued, it implies that, for $x, x_B\in B$
 \begin{equation}
 \label{key}
 \begin{aligned}
 f(x)-f(x_B)
 &={\rm Re}\int_{\mathbb{R}^d}
 (e^{i\omega\cdot x}-e^{i\omega\cdot x_B})
 \hat{f}(\omega)d\omega \\
 &={\rm Re}\int_{\mathbb{R}^d}
 (e^{i\omega\cdot x}-e^{i\omega\cdot x_B})
 e^{i\beta
 	(\omega)}|\hat{f}(\omega)|d\omega \\
 &=\int_{\mathbb{R}^d}(\cos(\omega\cdot
 x+\beta(\omega))-\cos(\omega\cdot x_B+\beta(\omega)))|\hat{f}(\omega)|d\omega \\
 &=\int_{\mathbb{R}^d}(\cos(\omega\cdot(x-x_B)+\beta_B(\omega))-\cos(\beta_B(\omega)))|\hat{f}(\omega)|d\omega
 \end{aligned}
 \end{equation}
 where
 $$
 \beta_B(\omega)=\omega\cdot x_B+\beta(\omega),\quad|\omega|_B:=\sup\limits_{x\in B}|\omega\cdot(x-x_B)|
 $$
 \begin{lemma}
 	\begin{equation}
 	\label{eq:2}
 	f(x)-f(x_B)=\int_{\mathbb{R}^d}k(x,\omega)d\omega
 	\end{equation}
 	\begin{equation}
 	\label{eq:1}
 	k(x, \omega)=\cos(\omega\cdot(x-x_B)+\beta_B(\omega))-\cos(\beta_B(\omega)))|\hat{f}(\omega)|
 	\end{equation}
 	satisfying
 	$$
 	|D^\alpha k(x, \omega)|\lesssim|\omega|^{|\alpha|}|\hat{f}(\omega)|
 	$$
 \end{lemma}

 For simplicity, we take $x_B=0$, we then have
 \begin{equation}
 \label{eq:2}
 f(x)-f(0)=\int_{\mathbb{R}^d}k(x,\omega)d\omega
 \end{equation}
 \begin{equation}
 \label{eq:1}
 k(x, \omega)=(\cos(\omega\cdot x+\beta(\omega))-\cos(\beta(\omega)))|\hat{f}(\omega)|
 \end{equation}

\endinput

Using Lemma~\ref{lem:sample}, we only need to find a function
$\rho(\theta)$ so the following will give a good bound:
\begin{equation}
\label{f-sigma}
\int_{\mathbb R^{d+1}}
\int_{\Omega}|\sigma(a^{-1}\omega\cdot  x+b)|^2dx \frac{|\hat{f}(\omega)|^2}{\rho(\theta)}d\theta
\end{equation}

We note that
\begin{equation}
\label{eq:4}
D^\alpha  k(x,\theta)=
\frac{(a^{-1}\omega)^\alpha}{2\pi\hat{\sigma}(a)}
\sigma^{(|\alpha|)}\left(a^{-1}\omega\cdot
x+b\right) |\hat{f}(\omega)|\chi(\omega, b)
\end{equation}
Thus
$$
|D^\alpha  k(x,\theta)|\le C_a h(\omega, b)(1+|\omega|)^m |\hat{f}(\omega)|
$$
where
$$
|\sigma^{(|\alpha|)}(a^{-1}\omega\cdot x+b)|\le h(\omega, b)
$$


\documentclass[11pt]{amsart}
\usepackage{geometry}                % See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   % ... or a4paper or a5paper or ...
%\geometry{landscape}                % Activate for for rotated page geometry
%\usepackage[parfill]{parskip}    % Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{epstopdf}
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}

\usepackage{amsthm}
\newtheorem{theorem}{Theorem}

\title{Rotational Symmetry}
\author{Arthur Ryman}
\date{\today}                                           % Activate to display a given date or no date

\begin{document}
\maketitle
\section{Introduction}

I recently needed to generate a random sample from the uniform distribution on the sphere $S^n$ where $n$ was very large.
My strategy was to generate a rotationally symmetric sample of non-zero vectors $v$ in some region of $\mathbb{R}^{n+1}$
and then normalize those to get unit vectors $u = v/\|v\|$ on $S^n$.
I first attempted to generate a random sample from the uniform distribution on the cube $[-1,1]^n$ and then take the subset of those
vectors $v$ such that $0 < \|v\| \leq 1$.
However, it turns out that the volume enclosed by $S^n$ becomes a very small fraction of the volume of $[-1,1]^n$ as $n$ gets large.
The probability that $\|v\| \leq 1$ goes to zero, so this approach fails.

I then remembered the well-known fact that the distribution of $n$ independent and identically  distributed (IID) standard normal variates is rotationally symmetric.
This symmetry seems miraculous and in fact only occurs for the normal distribution.
This article gives a proof of that fact.

\section{Rotational Symmetry of $2$ IID Variates}

Let the usual polar coordinate system on the plane be:
$$
\begin{aligned}
x &= r \cos \theta \\
y &= r \sin \theta
\end{aligned}
$$

Note that:
$$
\begin{aligned}
\frac{\partial}{\partial\theta}	&= \frac{\partial x}{\partial\theta} \frac{\partial}{\partial x} + \frac{\partial y}{\partial\theta} \frac{\partial}{\partial y} \\
						&= -r \sin\theta \frac{\partial}{\partial x} + r \cos\theta \frac{\partial}{\partial y} \\
						&= -y\frac{\partial}{\partial x} + x\frac{\partial}{\partial y}
\end{aligned}
$$

A function $F$ on the plane is rotationally symmetric when it is independent of $\theta$, i.e.:
$$
\frac{\partial F}{\partial \theta} = 0
$$
or, in terms of $x$ and $y$:
$$
-y\frac{\partial F}{\partial x} + x\frac{\partial F}{\partial y} = 0
$$
Therefore, a rotationally symmetric function must satisfy the following linear partial differential equation:
$$
x\frac{\partial F}{\partial y} = y\frac{\partial F}{\partial x}
$$

The probability density function $f$ of a normal distribution with mean zero and variance $\sigma^2$ is
$$
f(x) = Ce^{-\frac{x^2}{2\sigma^2}}
$$
where the normalization factor is $C=1/\sqrt{2\pi}$.

\begin{theorem}
The joint distribution of two IID random variates is rotationally symmetric if and only the random variates are normally distributed with mean zero.
\end{theorem}

\begin{proof}

First prove that the distribution of two IID normal variates with mean zero is rotationally symmetric.
The probability density function $F$ of the joint distribution is:
$$
\begin{aligned}
F(x,y)	&= f(x)f(y) \\
		&= Ce^{-\frac{x^2}{2\sigma^2}}Ce^{-\frac{y^2}{2\sigma^2}} \\
		&= C^2e^{-\frac{x^2}{2\sigma^2}-\frac{y^2}{2\sigma^2}} \\
		&= C^2 e^{-\frac{x^2+y^2}{2\sigma^2}} \\
		&= C^2 e^{-\frac{r^2}{2\sigma^2}}
\end{aligned}
$$

which does not depend on $\theta$.
Therefore the joint distribution is rotationally symmetric.

Now prove that if the joint distribution of two IID variates is rotationally symmetric then the variates are normally distributed with mean zero.
Let $g$ be the probability density function of any random variate such that the joint distribution
$$G(x,y) = g(x)g(y)$$
is rotationally symmetric.
The partial derivatives of $G$ are:
$$
\begin{aligned}
\frac{\partial G}{\partial x} &= g'(x)g(y) \\
\frac{\partial G}{\partial y} &= g(x)g'(y)
\end{aligned}
$$
The condition for rotational symmetry is:
$$
x\frac{\partial G}{\partial y} = y\frac{\partial G}{\partial x}
$$
Substituting in the expressions for the partial derivatives of $G$ we have:
$$
x g(x)g'(y) = y g'(x)g(y)
$$
Dividing both sides by $x y g(x) g(y)$ we have:
$$
\frac{g'(x)}{x g(x)} = \frac{g'(y)}{y g(y)}
$$
The LHS depends only on $x$ and the RHS depends only on $y$.
But $x$ and $y$ can vary independently.
Therefore each side of the equation must be equal to some constant, say $\alpha$:
$$
\frac{g'(x)}{x g(x)} = \alpha
$$
Using the identity $(\ln g)'(x) = g'(x)/g(x)$ we can rewrite the preceeding equation as:
$$
(\ln g)'(x) = \alpha x
$$
Integrating, we get:
$$
\ln g(x) = \frac{\alpha}{2} x^2 + \beta
$$
where $\beta$ is a constant of integration.
Exponentiating, we get:
$$
\begin{aligned}
g(x) 	&= e^{\frac{\alpha}{2} x^2 + \beta} \\
	&= B e^{\frac{\alpha}{2} x^2}
\end{aligned}
$$
where $B = e^\beta$.
Since $g$ is a probability density function we must have
$$
\lim_{|x|\rightarrow\infty}g(x) = 0
$$
so $\alpha$ must be negative.

Therefore setting
$$
\begin{aligned}
\alpha 	&= -\frac{1}{\sigma^2} \\
B 		&= C
\end{aligned}
$$
we see that $g$ is the probability density function of a normal distribution with mean zero and variance $\sigma^2$.
\end{proof}

\section{Rotational Symmetry of $n$ IID Variates}

\begin{theorem}
The joint distribution of $n$ IID random variates is rotationally symmetric if and only the random variates are normally distributed with mean zero.
\end{theorem}

\begin{proof}
Clearly, the joint distribution of $n$ IID normal variates with mean zero is rotationally symmetric.

To prove the converse it suffices to consider rotations in the plane defined by any pair of coordinates.
The preceding theorem implies that the variates are normal with mean zero.
\end{proof}

\end{document}


\section{NFA with $\epsilon$-transitions}

We shall now introduce another extension to NFA, called \eNFA, which
is a NFA whose labels can be the empty string, noted~\(\epsilon\).
The interpretation of this new kind of transition, called
\(\epsilon\)-transition, is that the current state changes by
following this transition \emph{without reading any input}. This is
sometimes referred as a \emph{spontaneous transition}. The rationale
is that \(\epsilon a = a \epsilon = a\), so recognising~\(\epsilon a\)
or~\(a\epsilon\) is the same as recognising~\(a\). In other words, we
do not need to read something more than~\(a\) as input.

For example, the \fig~\vref{fig:enfa_num} specifies signed natural and
decimal numbers by means of the \eNFA.
\begin{figure}
\centering
\includegraphics[bb=49 660 288 758]{enfa_num}
\caption{Signed natural and decimal numbers\label{fig:enfa_num}}
\end{figure}
This is not the simplest \eNFA we can imagine for these numbers, but
note the utility of the \(\epsilon\)-transition between~\(q_0\)
and~\(q_1\). In case of lexical analysers, \eNFA{s} enable the
separate design of a NFA for each token, then create an initial
(respectively, final) state connected to all their initial
(respectively, final) states with an \(\epsilon\)-transition.

For instance, for keywords \term{fun} and \term{function} and
identifiers, we have
\begin{center}
\includegraphics[bb=48 630 421 732,scale=0.88]{enfa_kwd_id}
\end{center}
In lexical analysis, once we have a single \eNFA, we can
\begin{itemize}

  \item either remove all the \(\epsilon\)-transitions and either create
  a NFA and then maybe a DFA, or create directly a DFA;

  \item or use a formal definition of \eNFA that directly leads to a
  recognition algorithm, just as we did for DFAs and NFAs.

\end{itemize}
Both methods assume that it is always possible to create an equivalent
NFA, hence a DFA, from a given \eNFA.  In other words, \emph{DFA, NFA
  and \eNFA have the same expressive power.}

The first method constructs explicitly the NFA and maybe the DFA,
while the second does not, at the possible cost of more computations
at run-time.

Before entering into the details, we need to define formally an \eNFA,
as suggested by the second method. The only difference between an NFA
and an \eNFA is that the transition function~\(\delta_E\) takes as
second argument an element in \(\Sigma \cup \{\epsilon\}\), with
\(\epsilon \not\in \Sigma\), instead of~\(\Sigma\) --~but the alphabet
still remains~\(\Sigma\).

\subsection*{$\epsilon$-closure}

We need now a function called \emph{\(\eClose\)}, which takes an
\eNFA~\(\mathcal{E}\), a state~\(q\) of~\(\mathcal{E}\) and returns
all the states which are accessible in~\(\mathcal{E}\) from~\(q\) with
label~\(\epsilon\). The idea is to achieve a \emph{depth-first
  traversal} of~\(\mathcal{E}\), starting from~\(q\) and following
only \(\epsilon\)-transitions. Let us call \(\eDFS\)
(`\(\epsilon\)-Depth-First-Search') the function such that \(\eDFS(q,
Q)\) is the set of states reachable from~\(q\) following
\(\epsilon\)-transitions and which is not included in~\(Q\),
\emph{\(Q\) being interpreted as the set of states already visited in
  the traversal}. The set \(Q\) ensures the termination of the
algorithm even in presence of cycles in the automaton. Therefore, let
\begin{equation*}
\eClose (q) = \eDFS (q, \varnothing), \quad \text{if} \; q \in Q_E,
\end{equation*}
where the \eNFA is \(\mathcal{E} = (Q_E, \Sigma, \delta_E, q_0,
F_E)\). Now we define \(\eDFS\) as follows:
\begin{align}
   \eDFS (q, Q)
&= \varnothing,
& \text{if} \; q \in Q; \label{eDFS_1}\\
  \eDFS (q, Q)
&= \{q\} \quad \cup \bigcup_{p \in \delta_E(q,\epsilon)}{\eDFS (p, Q
    \cup \{q\})},
& \text{if} \; q \not\in Q. \label{eDFS_2}
\end{align}
The \eNFA in \fig~\vref{fig:enfa_num} leads to the following
\(\epsilon\)-closures:
\begin{align*}
    \eClose (q_0) &= \{q_0, q_1\}\\
    \eClose (q_1) &= \{q_1\}\\
    \eClose (q_2) &= \{q_2\}\\
    \eClose (q_3) &= \{q_3, q_5\}\\
    \eClose (q_4) &= \{q_4, q_3, q_5\}
    \eClose (q_5) &= \{q_5\}.
\end{align*}
Consider, as a more difficult example, the following
\eNFA~\(\mathcal{E}\):
\begin{center}
\includegraphics[bb=48 677 236 735]{enfa_eg}
\end{center}
\begin{align*}
\eClose (q_0)
  &= \eDFS (q_0, \varnothing), \quad \text{since} \; q_0 \in Q_E\\
  &= \{q_0\} \cup \eDFS (q_1, \{q_0\}) \cup \eDFS (q_4, \{q_0\})
     \quad \text{by eq.~\ref{eDFS_2}}\\
  &= \{q_0\} \cup \biggl( \{q_1\} \cup
     \bigcup_{p \in \delta_E(q_1,\epsilon)}{\eDFS (p, \{q_0, q_1\})}\biggr)
     \quad \text{by eq.~\ref{eDFS_2}}\\
  &\mathrel{\phantom{=}} \phantom{\{q_0\}}{}
     \cup \biggl( \{q_4\} \cup
     \bigcup_{p \in \delta_E(q_4,\epsilon)}{\eDFS (p, \{q_0, q_4\})} \biggr)
     \quad \text{by eq.~\ref{eDFS_2}}\\
  &= \{q_0\} \cup \biggl( \{q_1\} \cup
     \bigcup_{p \in \{q_2\}}{\eDFS (p, \{q_0, q_1\})}\biggr)\\
  &\mathrel{\phantom{=}} \phantom{\{q_0\}}{}
     \cup \biggl( \{q_4\} \cup \bigcup_{p \in \varnothing}{\eDFS (p,
     \{q_0, q_4\})} \biggr)\\
  &= \{q_0\} \cup (\{q_1\} \cup \eDFS (q_2, \{q_0, q_1\}))
     \cup (\{q_4\} \cup \varnothing)\\
  &= \{q_0, q_1, q_4\} \cup \eDFS (q_2, \{q_0, q_1\})\\
  &= \{q_0, q_1, q_4\} \cup \biggl(\{q_2\} \cup \bigcup_{p \in
     \delta_E(q_2,\epsilon)}{\eDFS (p, \{q_0, q_1, q_2\})}\biggr)\\
  &= \{q_0, q_1, q_4\} \cup \biggl(\{q_2\} \cup \bigcup_{p \in
     \{q_1, q_3\}}{\eDFS (p, \{q_0, q_1, q_2\})}\biggr)\\
  &= \{q_0, q_1, q_2, q_4\} \cup \eDFS (q_1, \{q_0, q_1, q_2\})\\
  &\mathrel{\phantom{=}} \phantom{\{q_0,q_2,q_2,q_4\}} \cup \eDFS
     (q_3, \{q_0, q_1, q_2\})\\
  &= \{q_0, q_1, q_2, q_4\} \cup \varnothing \qquad\quad
     \text{by eq.~\ref{eDFS_1}, since} \; q_1 \in \{q_0, q_1, q_2\}\\
  &\mathrel{\phantom{=}} \;\,
     \cup \biggl(\{q_3\} \cup
     \bigcup_{p \in \delta_E(q_3, \epsilon)}{\eDFS (p, \{q_0, q_1, q_2, q_3\})}\biggr)
     \quad \text{by eq.~\ref{eDFS_2}}\\
  &= \{q_0, q_1, q_2, q_3, q_4\} \cup \bigcup_{p \in
     \varnothing}{\eDFS (p, \{q_0, q_1, q_2, q_3\})}\\
  &= \{q_0, q_1, q_2, q_3, q_4\}.
\end{align*}
It is useful to extend \(\eClose\) to sets of states, not just
states. Let us note \(\eCloseSet\) this extension, which we can easily
define as
\begin{equation*}
\eCloseSet (Q) = \bigcup_{q \in Q}{\eClose (q)},
\end{equation*}
for any subset \(Q \subseteq Q_E\) where the \eNFA is \(\mathcal{E} =
(Q_E, \Sigma, \delta_E, q_E, F_E)\).

\subsection*{Optimisation}
\label{enfa_closure}

Let us compute the \(\epsilon\)-closure of~\(q_0\) in the following
\eNFA~\(\mathcal{E}\):
\begin{center}
\includegraphics[bb=65 618 315 721]{enfa_closure}
\end{center}
where the sub-\eNFA~\(\mathcal{E'}\) contains only
\(\epsilon\)-transitions and all its~\(Q'\) states are accessible
from~\(q_3\).
\begin{align*}
   \eClose (q_0)
&= \eDFS (q_0, \varnothing)\\
&= \{q_0\} \cup \eDFS (q_1, \{q_0\}) \cup \eDFS (q_2, \{q_0\})\\
&= \{q_0\} \cup (\{q_1\} \cup \eDFS (q3, \{q_0, q_1\}))\\
&\mathrel{\phantom{=}} \phantom{\{q_0\}}{} \cup (\{q_2\} \cup \eDFS
   (q_3, \{q_0, q_2\}))\\
&= \{q_0, q_1, q_2\} \cup \eDFS (q3, \{q_0, q_1\}) \cup \eDFS
   (q_3, \{q_0, q_2\})\\
&= \{q_0, q_1, q_2, q_3,\} \cup (\{q_3\} \cup Q') \cup (\{q_3\} \cup Q')\\
&= \{q_0, q_1, q_2, q_3,\} \cup Q'.
\end{align*}
We compute \(\{q_3\} \cup Q'\) twice, that is, we traverse
twice~\(q_3\) and all the states of~\(\mathcal{E'}\), which can be
inefficient if~\(Q'\) is large. The way to avoid repeating traversals
is to change the definitions of~\(\eClose\) and
\(\eCloseSet\). Dually, we need a new definition of \(\eDFS\) and
create a function \(\eDFSset\) which is similar to \(\eDFS\), except
that it applies to set of states instead of one state:
\begin{align*}
   \eClose (q)
&= \eDFS (q, \varnothing),
&& \text{if} \; q \in Q_E;\\
   \eCloseSet (Q)
&= \eDFSset (Q, \varnothing),
&& \text{if} \; Q \subseteq Q_E.
\end{align*}
We interpret~\(Q'\) in \(\eDFS (q, Q')\) and \(\eDFSset (Q, Q')\) as
the set of states that have already been visited in the depth-first
search. Variables~\(q\) and~\(Q\) denote, respectively, a state and a
set of states that have to be explored. In the first definition we
computed the \emph{new reachable states}, whilst, in the new one, we
compute the \emph{currently reached states}. Then let us redefine
\(\eDFS\) this way:
\begin{align*}
  \eDFS (q, Q')
&= Q',
&& \text{if} \; q \in Q'; \tag{1'}\\
  \eDFS (q, Q')
&= \eDFSset (\delta_E (q, \epsilon), Q' \cup \{q\}),
&& \text{if} \; q \not\in Q'. \tag{2'}
\end{align*}
Contrast with the first definition
\begin{align*}
   \eDFS (q, Q')
&= \varnothing,
& \text{if} \; q \in Q'; \tag{1} \label{empty}\\
  \eDFS (q, Q')
&= \{q\} \quad \cup \bigcup_{p \in \delta_E(q,\epsilon)}{\eDFS (p, Q'
    \cup \{q\})},
& \text{if} \; q \not\in Q'. \tag{2} \label{parallel}
\end{align*}
Hence, in~\eqref{empty} we return \(\varnothing\) because there is no
new state, that is, none not already in \(Q'\), whereas in (1') we
return~\(Q'\) itself. The new definition of \(\eDFSset\) is not more
difficult than the first one:
\begin{align}
   \eDFSset (\varnothing, Q')
&= Q',\label{eDFSset:1}\\
   \eDFSset (\{q\} \cup Q, Q')
&= \eDFSset (Q, \eDFS (q, Q')),
& \text{if} \; q \not\in Q. \label{eDFSset:2}
\end{align}
Notice that the definitions of \(\eDFS\) and \(\eDFSset\) are mutually
recursive. In~\eqref{parallel} we traverse states in parallel
(consider the union operator), starting from each element in
\(\delta_E (q, \epsilon)\), whereas in~(2') and~\eqref{eDFSset:2}, we
traverse them sequentially so we can use the information collected
(currently reached states) in the previous searches.

Coming back to our example \vpageref{enfa_closure}, we find
\begin{align*}
   \eClose (q_0)
&= \eDFS (q_0, \varnothing)
&& q_0 \in Q_E\\
&= \eDFSset (\{q_1, q_2\}, \{q_0\})
&& \text{by eq.~(2')}\\
&= \eDFSset (\{q_2\}, \eDFS (q_1, \{q_0\}))
&& \text{by eq.~(4)}\\
&= \eDFSset (\{q_2\}, \eDFSset (\{q_3\}, \{q_0,
   q_1\}))
&& \text{by eq.~(2')}\\
&= \eDFSset (\{q_2\}, \eDFSset (\varnothing, \eDFS (q_3, \{q_0, q_1\})))
&& \text{by eq.~(4)}\\
&= \eDFSset (\{q_2\}, \eDFS (q_3, \{q_0, q_1\}))
&& \text{by eq.~(3)}\\
&= \eDFSset (\{q_2\}, \{q_0, q_1, q_3\} \cup Q')\\
&= \eDFSset (\varnothing, \eDFS (q_2, \{q_0, q_1, q_3\} \cup Q'))
&& \text{by eq.~(4)}\\
&= \eDFS (q_2, \{q_0, q_1, q_3\} \cup Q')
&& \text{by eq.~(3)}\\
&= \eDFSset (\{q_3\}, \{q_0, q_1, q_2, q_3\} \cup Q')
&& \text{by eq.~(2')}\\
&= \eDFSset (\varnothing, \eDFS (q_3, \{q_0, q_1, q_2, q_3\} \cup Q'))
&& \text{by eq.~(4)}\\
&= \eDFS (q_3, \{q_0, q_1, q_2, q_3\} \cup Q')
&& \text{by eq.~(3)}\\
&= \{q_0, q_1, q_2, q_3\} \cup Q'
&& \text{by eq.~(1')}\\
\end{align*}
The important thing here is that we did not compute (traverse) several
times~\(Q'\). Note that some equations can be used in a different
order and~\(q\) can be chosen arbitrarily in equation~(4), but the
result is always the same.

\subsection*{Extended transition functions}

The \(\epsilon\)-closure allows to explain how a \eNFA recognises or
rejects a given input. Let \(\mathcal{E} = (Q_E, \Sigma, \delta_E,
q_0, F_E)\). We want \(\hat{\delta}_E (q, w)\) be the set of states
reachable from~\(q\) along a path whose labels, when concatenated, for
the string~\(w\). The difference here with NFAs is that
several~\(\epsilon\) can be present along this path, despite not
contributing to \(w\).  For all state \(q \in Q_E\), let
\begin{align*}
   \hat{\delta}_E (q, \epsilon)
&= \eClose (q),\\
   \hat{\delta}_E (q, wa)
&= \eCloseSet \Biggl(\bigcup_{p \in \hat{\delta}_E (q, w)}{\delta_N (p,
     a)}\Biggr),
&& \text{for all} \; a \in \Sigma, w \in \Sigma^{*}.
\end{align*}
This definition is based on the regular identity \(wa =
((w\epsilon^{*})a)\epsilon^{*}\).

As an illustration, let us consider again the \eNFA in
\fig~\vref{fig:enfa_num} and compute the states reached on the input
\verb+5.6+:
\begin{align*}
   \hat{\delta}_E (q_0, \epsilon)
&= \eClose (q_0) = \{q_0, q_1\};\\
   \hat{\delta}_E (q_0, \mathtt{5})
&= \eCloseSet \Biggl( \bigcup_{p \in \hat{\delta}_E (q_0,
   \epsilon)}{\delta_N (p, \mathtt{5})} \Biggr)\\
&= \eCloseSet (\delta_N (q_0, \mathtt{5}) \cup \delta_N (q_1,
   \mathtt{5})) = \eCloseSet (\varnothing \cup \{q_1, q_4\})\\
&= \{q_1, q_3, q_4, q_5\};\\
   \hat{\delta}_E (q_0, \mathtt{5.})
&= \eCloseSet \Biggl( \bigcup_{p \in \hat{\delta}_N (q_0,
   \mathtt{5})}{\delta_N (p, \mathtt{.})} \Biggr)\\
&= \eCloseSet (\delta_N (q_1, \mathtt{.}) \cup \delta_N (q_3,
   \mathtt{.}) \cup \delta_N (q_4, \mathtt{.}) \cup \delta_N (q_5,
   \mathtt{.}))\\
   \hat{\delta}_E (q_0, \mathtt{5.})
&= \eCloseSet (\{q_2\} \cup \varnothing \cup \varnothing \cup
   \varnothing)
 = \{q_2\};\\
   \hat{\delta}_N (q_0, \mathtt{5.6})
&= \eCloseSet \Biggl( \bigcup_{p \in \hat{\delta}_E (q_0,
     \mathtt{5.})}{\delta_N (p, \mathtt{6})} \Biggr)\\
&= \eCloseSet (\delta_N (q_2, \mathtt{6}))
= \eCloseSet (\{q_3\})
= \{q_3, q_5\} \ni q_5.
\end{align*}
Since~\(q_5\) is a final state, the string \verb+5.6+ is recognised as
a number.

\subsection*{Subset construction for \eNFA{}s}

Let us present now how to construct a DFA from a \eNFA such that both
recognise the same language. The method is a variation of the subset
construction we presented for NFA: we must take into account the
states reachable through \(\epsilon\)-transitions, with help of
\(\epsilon\)-closures. Let us assume that \(\mathcal{E} = (Q, \Sigma,
\delta, q_0, F)\) is an \eNFA. Let us define as follows the equivalent
DFA \(\mathcal{D} = (Q_D, \Sigma, \delta_D, q_D, F_D)\).
\begin{enumerate}

  \item \(Q_D\) is the set of subsets of \(Q_E\). More precisely, all
    accessible states of \(\mathcal{D}\) are \(\epsilon\)-closed
    subsets of \(Q_E\), that is to say, sets \(Q \subseteq Q_E\) such
    that \(Q = \eClose(Q)\);

  \item \(q_D = \eClose (q_0)\), in other words, we get the start
    state of~\(\mathcal{D}\) by \(\epsilon\)-closing the set made of
    only the start state of~\(\mathcal{E}\);

  \item \(F_D\) is those sets of states that contain at least one
    final state of \(\mathcal{E}\), that is to say, \(F_D = \{Q \,
    \lvert \, Q \in Q_D \; \text{and} \; Q \cap F_E \neq
    \varnothing\}\);

  \item For all \(a \in \Sigma\) and \(Q \in Q_D\), let \(\delta_D (Q,
    a) = \eCloseSet \bigl(\bigcup_{q \in Q}{\delta_E (q,
      a)}\bigr)\).

\end{enumerate}
Let us consider again the \eNFA in \fig~\vref{fig:enfa_num}. Its
transition table is
\begin{equation*}
\begin{array}{r@{}l||c|c|c|c|c}
\multicolumn{2}{c||}{\mathcal{E}} & + & - & 0, \ldots, 9 & \mathtt{.} &
\epsilon\\
\hhline{==::=====}
\rightarrow & q_0 & \{q_1\} & \{q_1\} & \varnothing & \varnothing
                  & \{q_1\}\\
            & q_1 & \varnothing  & \varnothing & \{q_1, q_4\}
                  & \{q_2\} & \varnothing\\
            & q_2 & \varnothing  & \varnothing & \{q_3\} & \varnothing
                  & \varnothing\\
            & q_3 & \varnothing & \varnothing & \{q_3\} & \varnothing
                  & \{q_5\}\\
            & q_4 & \varnothing & \varnothing & \varnothing
                  & \varnothing & \{q_3\}\\
         \# & q_5 & \varnothing & \varnothing & \varnothing
                  & \varnothing & \varnothing
\end{array}
\end{equation*}
By applying the subset construction to this \eNFA, we get the table
\begin{equation*}
\begin{array}{r@{}l||c|c|c|c}
\multicolumn{2}{c||}{\mathcal{D}} & + & - & 0, \ldots, 9 & \mathtt{.}\\
\hhline{==::====}
\rightarrow
   & \{q_0,q_1\}
             & \{q_1\} & \{q_1\} & \{q_1, q_3, q_4, q_5\} & \{q_2\}\\
   & \{q_1\} & \varnothing  & \varnothing & \{q_1, q_3, q_4, q_5\}
             & \{q_2\}\\
\# & \{q_1, q_3, q_4, q_5\}
             & \varnothing  & \varnothing & \{q_1, q_3, q_4, q_5\}
             & \{q_2\}\\
   & \{q_2\} & \varnothing & \varnothing & \{q_3, q_5\} & \varnothing\\
\# & \{q_3, q_5\}
             & \varnothing & \varnothing & \{q_3, q_5\} & \varnothing
\end{array}
\end{equation*}
Let us rename the states of~\(\mathcal{D}\) and get rid of the empty
sets:
\begin{equation*}
\begin{array}{r@{}l||c|c|c|c}
\multicolumn{2}{c||}{\mathcal{D}} & + & - & 0, \ldots, 9 & \mathtt{.}\\
\hhline{==::====}
\rightarrow
   & A & B & B & C & D\\
   & B &   &   & C & D\\
\# & C &   &   & C & D\\
   & D &   &   & E &\\
\# & E &   &   & E &
\end{array}
\end{equation*}
The transition diagram of~\(\mathcal{D}\) is shown in
\fig~\vref{fig:nfa_num}.
\begin{figure}[H]
\centering
\includegraphics[bb=49 610 208 758]{dfa_from_enfa_num}
\caption{Determinisation of the \eNFA in \fig~\vref{fig:enfa_num}
\label{fig:nfa_num}}
\end{figure}


\section*{Problem 2 Solution}

\begin{enumerate}[a)]

\item

The steady-state, continuous energy diffusion equation is
$$ -\nabla D(r,E) \nabla \phi(r,E) + \Sigma_t(r,E) \phi(r,E) = \int_0^{\infty} \Sigma_s(r,E' \rightarrow E)\phi(r,E')\,dE' + \chi(E) \int_0^{\infty} \nu(E') \Sigma_f(r,E')\phi(r,E')\,dE' .$$
As a reminder, the terms are defined as:
\begin{align*}
\textbf{Loss terms (left side)} \qquad\qquad& \\
-\nabla D(r,E) \nabla \phi(r,E)&:\text{ the neutrons with energy }E\text{ streaming from location }r\text{ to any other location} \\
\Sigma_t(r,E) \phi(r,E)&:\text{ the total interaction rate of neutrons with energy }E\text{ at location }r  \\
\textbf{Gain terms (right side)} \qquad\qquad& \\
\int_0^{\infty} \Sigma_s(r,E' \rightarrow E)\phi(r,E')\,dE'&:\text{ the scattering rate of neutrons with any energy }E'\text{ into energy }E\text{ at location }r \\
\chi(E) \int_0^{\infty} \nu(E') \Sigma_f(r,E')\phi(r,E')\,dE'&:\text{ the fission rate of neutrons with any energy }E'\text{ at location }r; \\
& \quad\chi(E)\text{ is the fraction of neutrons produced by fission with energy }E
\end{align*}
It is given that $D$ and $\nu$ are constants, and we are told that the reactor is homogeneous so none of the cross sections depend on $r$. Our diffusion equation simplifies down to
$$\boxed{ -D \lap \phi(r,E) + \Sigma_t(E) \phi(r,E) = \int_0^{\infty} \Sigma_s(E' \rightarrow E)\phi(r,E')\,dE' + \chi(E) \int_0^{\infty} \nu \Sigma_f(E')\phi(r,E')\,dE' }.$$

\item

We are asked to derive the three-group multigroup equation, so we need to find the collective behavior in each group. We can do this by taking the integral of the continuous equation over the energy range corresponding to each group. This process will allow us to take the continuous equation which is valid for all energies (and which may not be able to be solved analytically), and turn it into a discretized form that can be solved either by hand, or more likely, with a computer.

Say we take the integral over some energy group, $g$, spanning energies $E_{g-1}$ to $E_g$ (for the lowest energy group, $E_{g-1} = 0$, and for the highest energy group, $E_g = \infty$). Our equation becomes
\begin{align*}
\int_{E_{g-1}}^{E_g} -D \lap \phi(r,E)\,dE + \int_{E_{g-1}}^{E_g} \Sigma_t(E) \phi(r,E)\,dE = \int_{E_{g-1}}^{E_g} \int_0^{\infty} \Sigma_s(E' &\rightarrow E)\phi(r,E')\,dE'\,dE \\
&+ \int_{E_{g-1}}^{E_g} \chi(E) \int_0^{\infty} \nu \Sigma_f(E')\phi(r,E')\,dE'\,dE .
\end{align*}
We can similarly separate our integrals over $dE'$ on the right side into three intervals, one for each group. We will express our total integral, from $E' = 0$ to $E'=\infty$, as the sum of the integrals over these three intervals, each ranging from $E_{g'-1}$ to $E_{g'}$.
\begin{align*}
\int_{E_{g-1}}^{E_g} -D \lap \phi(r,E)\,dE + \int_{E_{g-1}}^{E_g} \Sigma_t(E) \phi(r,E)\,dE = &\int_{E_{g-1}}^{E_g} \sum_{g'=1}^3 \int_{E_{g'-1}}^{E_{g'}} \Sigma_s(E' \rightarrow E)\phi(r,E')\,dE'\,dE \\
 &+ \int_{E_{g-1}}^{E_g} \chi(E) \sum_{g'=1}^3 \int_{E_{g'-1}}^{E_{g'}} \nu \Sigma_f(E')\phi(r,E')\,dE'\,dE
\end{align*}
Where neither $D$ nor the Laplacian depend on $E$, we can remove them from the integral in the first term. And, since the only factor in the fission term depending on $E$ is $\chi(E)$, the fraction of fission-born neutrons produced into energy $E$, we can separate it as it's own integral.
$$ \chi_g = \int_{E_{g-1}}^{E_g} \chi(E)\,dE $$
$\chi_g$ is just the fraction of fission-born neutrons produced in group $g$, the sum of the fraction of fission-born neutrons created in all energies from $E_{g-1}$ to $E_g$. Our diffusion equation simplifies to
\begin{align*}
-D \lap \int_{E_{g-1}}^{E_g} \phi(r,E)\,dE + \int_{E_{g-1}}^{E_g} \Sigma_t(E) \phi(r,E)\,dE = \int_{E_{g-1}}^{E_g} \sum_{g'=1}^3 \int_{E_{g'-1}}^{E_{g'}} \Sigma_s(E' &\rightarrow E)\phi(r,E')\,dE'\,dE \\
 &+ \chi_g \nu \sum_{g'=1}^3 \int_{E_{g'-1}}^{E_{g'}} \Sigma_f(E')\phi(r,E')\,dE' .
\end{align*}
Here we recognize that the total flux in group $g$ is given by
$$ \phi_g(r) = \int_{E_{g-1}}^{E_g} \phi(r,E)\,dE ,$$
and that average cross sections for some energy group are equal to the flux-weighted average of the reaction rate over that group's energy range.
$$ \Sigma_g = \frac{\int_{E_{g-1}}^{E_g} \Sigma(E)\phi(r,E)\,dE}{\int_{E_{g-1}}^{E_g} \phi(r,E)\,dE} = \frac{\int_{E_{g-1}}^{E_g} \Sigma(E)\phi(r,E)\,dE}{\phi_g(r)} $$
Multiplying both sides by $\phi_g$ gives
$$ \Sigma_g \phi_g = \int_{E_{g-1}}^{E_g} \Sigma(E)\phi(E)\,dE ,$$
and so we can simplify all the flux and reaction rate integrals in the diffusion equation.
$$ -D \lap \phi_g(r) + \Sigma_{t,g} \phi_g(r) = \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow g}\phi_{g'}(r) + \chi_g \nu \sum_{g'=1}^3 \Sigma_{f,g'}\phi_{g'}(r) $$
Remember, this is actually a set of three equations, one for each group (1, 2, and 3).
\begin{align*}
-D \lap \phi_1(r) + \Sigma_{t,1} \phi_1(r) &= \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow 1}\phi_{g'}(r) + \chi_1 \nu \sum_{g'=1}^3 \Sigma_{f,g'}\phi_{g'}(r) \\
-D \lap \phi_2(r) + \Sigma_{t,2} \phi_2(r) &= \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow 2}\phi_{g'}(r) + \chi_2 \nu \sum_{g'=1}^3 \Sigma_{f,g'}\phi_{g'}(r) \\
-D \lap \phi_3(r) + \Sigma_{t,3} \phi_3(r) &= \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow 3}\phi_{g'}(r) + \chi_3 \nu \sum_{g'=1}^3 \Sigma_{f,g'}\phi_{g'}(r)
\end{align*}

Now we begin to impose the assumptions of the system. First, fission only produces neutrons in group 1, so $\chi_2 = 0$ and $\chi_3 = 0$. Also fission is only induced in group 3, so $\Sigma_{f,1} = 0$ and $\Sigma_{f,2} = 0$. Our set of equations is now:
\begin{align*}
-D \lap \phi_1(r) + \Sigma_{t,1} \phi_1(r) &= \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow 1}\phi_{g'}(r) + \chi_1 \nu \Sigma_{f,3}\phi_{3}(r) \\
-D \lap \phi_2(r) + \Sigma_{t,2} \phi_2(r) &= \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow 2}\phi_{g'}(r) \\
-D \lap \phi_3(r) + \Sigma_{t,3} \phi_3(r) &= \sum_{g'=1}^3 \Sigma_{s,g' \rightarrow 3}\phi_{g'}(r)
\end{align*}
There is no upscattering---group 2 can only inscatter (scatter back into group 2) or scatter into group 3; group 3 can only inscatter. The groups are directly coupled, so group 1 can only scatter into group 2. These assumptions mean $\Sigma_{s,1 \rightarrow 3} = 0$, $\Sigma_{s,2 \rightarrow 1} = 0$, $\Sigma_{s,3 \rightarrow 1} = 0$, and $\Sigma_{s,3 \rightarrow 2} = 0$, and our equations (which we now right out explicitly) simplify further:
\begin{align*}
-D \lap \phi_1(r) + \Sigma_{t,1} \phi_1(r) &= \Sigma_{s,1 \rightarrow 1}\phi_1(r) + \chi_1 \nu \Sigma_{f,3}\phi_{3}(r) \\
-D \lap \phi_2(r) + \Sigma_{t,2} \phi_2(r) &= \Sigma_{s,1 \rightarrow 2}\phi_1(r) +\Sigma_{s,2 \rightarrow 2}\phi_2(r) \\
-D \lap \phi_3(r) + \Sigma_{t,3} \phi_3(r) &= \Sigma_{s,2 \rightarrow 3}\phi_{2}(r) + \Sigma_{s,3 \rightarrow 3}\phi_{3}(r).\\
\end{align*}

\item

To form a matrix equation, we will start by writing the multigroup equations suggestively, with everything but fission terms on the left and with our flux terms grouped together:
\begin{align*}
\left(-D \lap + \Sigma_{t,1} - \Sigma_{s,1 \rightarrow 1}\right)&\phi_1(r) && && &&= \chi_1 \nu \Sigma_{f,3}\phi_{3}(r) \\
-\Sigma_{s,1 \rightarrow 2}&\phi_1(r) &+ \left(-D \lap + \Sigma_{t,2} - \Sigma_{s,2 \rightarrow 2}\right)&\phi_2(r) && &&= 0 \\
&&- \Sigma_{s,2 \rightarrow 3}&\phi_{2}(r) &+ \left(-D \lap + \Sigma_{t,3} - \Sigma_{s,3 \rightarrow 3}\right)&\phi_{3}(r) &&= 0
\end{align*}
In this form, we can more easily see that each term multiplying a flux could be an entry in a
matrix, with the matrix equation being given by
$$\begin{bmatrix}
-D \lap + \Sigma_{t,1} - \Sigma_{s,1 \rightarrow 1} & 0 & 0 \\
-\Sigma_{s,1 \rightarrow 2} & -D \lap + \Sigma_{t,2} - \Sigma_{s,2 \rightarrow 2} & 0 & \\
0 & - \Sigma_{s,2 \rightarrow 3} & -D \lap + \Sigma_{t,3} - \Sigma_{s,3 \rightarrow 3}
\end{bmatrix}\begin{bmatrix}
\phi_1 \\
\phi_2 \\
\phi_3
\end{bmatrix} = \begin{bmatrix}
0 & 0 & \chi_1 \nu \Sigma_{f,3} \\
0 & 0 & 0 \\
0 & 0 & 0
\end{bmatrix}\begin{bmatrix}
\phi_1 \\
\phi_2 \\
\phi_3
\end{bmatrix}$$
\end{enumerate}


% !TEX root = main.tex

%-------------------------------------------------
\section{Random variables}\label{sec:rvs}

Let $(\Omega,\prob)$ be a probability space associated with some random experiment. Random experiments with numerical outcomes lend themselves to mathematical analysis; for more abstract sample spaces we introduce the notion of \emph{random variables}, which map sample spaces to the real numbers. Random variables are typically represented by uppercase letters.

\begin{definition}
A \emph{random variable} is a function which maps a sample space $\Omega$ to the real numbers,
\[
\begin{array}{rccl}
	X:	& \Omega	& \to		& \R \\
		& \omega	& \mapsto	& X(\omega)
\end{array}
\]
\end{definition}

We are often not directly interested in the outcome of a random experiment, but rather in some consequence of the outcome: a gambler might be more interested in his losses than in the outcomes of the individual games which led to them. Random variables can be used to pick out particular features of an experiment that are of interest.

\begin{example}
A fair coin is tossed until a head is observed. Suppose we win \pounds 3 if the coin is tossed an odd number of times and lose \pounds 5 if the coin is tossed an even number of times. The sample space is the countably infinite set $\Omega=\{H,TH,TTH,TTTH,\ldots\}$. We win if event $A=\{H,TTH, TTTTH,\ldots\}$ occurs and lose if event $A^c=\{TH, TTTH, TTTTH,\ldots\}$ occurs. Because the coin is fair,
\[\begin{array}{lll}
\prob(A) 	& = \frac{1}{2} + \frac{1}{8} + \frac{1}{32} + \ldots	& = \frac{2}{3}, \\
\prob(A^c) 	& = \frac{1}{4} + \frac{1}{16} + \frac{1}{64} + \ldots 	& =  \frac{1}{3}.
\end{array}\]
Our situation can be represented more concisely by the random variable $X:\Omega\to\R$ defined by
\[
X(\omega) =
  \begin{cases}
   +3 & \text{if } \omega\in \{H, TTH, TTTTH, \ldots\}, \\
   -5 & \text{if } \omega\in \{TH, TTTH, TTTTTH, \ldots\}.
  \end{cases}
\]
From here it follows that $\prob(X=3)=2/3$ and $\prob(X=-5)=1/3$.
\end{example}

%-----------------------------
\subsection{Distributions}\label{ss:dist}

Let $B$ be a subset of $\R$. We use the notation $\{X\in B\}$ as shortand for the event $\{\omega: X(\omega)\in B\}$ and $\prob(X\in B)$ as shorthand for the probability of this event. Thus $\{X\in B\}$ is the event consisting of those outcomes that are mapped by $X$ into $B$, and $\prob(X\in B)$ is therefore the probability that $X$ takes a value in the set $B$.

\begin{definition}\label{def:distribution}
The \emph{distribution} of $X$ is the function $\prob_X$ defined on subsets of $\R$ by
\[
\prob_X(B) = \prob(X\in B)
\]
\end{definition}

\begin{theorem}\label{th:distribution}
$\prob_X$ is a probability measure on subsets of $\R$.
\begin{proof}
First we check that $\prob_X(\R) = 1$:
\[
\prob_X(\R) = \prob(X\in\R) = \prob\big(\big\{\omega:X(\omega)\in\R\big\}\big) = \prob(\Omega) = 1.
\]
Next we show that $\prob_X$ is countably additive. Let $B_1,B_2,\ldots$ be a sequence of pairwise disjoint subsets of $\R$. Then
\begin{align*}
\prob_X\big(\textstyle\bigcup_{i=1}^{\infty} B_i\big)
	& = \prob\big(\big\{\omega : X(\omega)\in \textstyle\bigcup_{i=1}^{\infty} B_i\big\}\big) \\
	& = \prob\big(\textstyle\bigcup_{i=1}^{\infty} \{\omega : X(\omega)\in B_i\}\big) \\
	& = \sum_{i=1}^{\infty} \prob\big(\{\omega : X(\omega)\in B_i\}\big) \quad\text{because the $B_i$ are disjoint,} \\
	& = \sum_{i=1}^{\infty} \prob_X(B_i),
\end{align*}
as required.
\end{proof}
\end{theorem}

%-----------------------------
\subsection{Indicator variables}
Every event can be represented by its \emph{indicator} variable, which \textit{indicates} whether or not the event occurs. Indicator variables provide an explicit link between random events and random variables.

\begin{definition}\label{def:indicator}
The \emph{indicator variable} of an event $A$ is the function $I_A:\Omega\to\R$ defined by
\[
I_A(\omega) =
  \begin{cases}
   1 & \text{if } \omega\in A, \\
   0 & \text{if } \omega\notin A.
  \end{cases}
\]
\end{definition}

\begin{exercise}
Let $A$ and $B$ be any two events. Show that $I_{A^c} = 1 - I_A$, $I_{A\cap B} = I_A I_B$ and $I_{A\cup B} = I_A + I_B - I_{A\cap B}$. Note that two functions are equal if and only if they are equal at every point.
\end{exercise}

%-----------------------------
\section{CDFs}\label{sec:cdfs}

A probability distribution $\prob_X$ is uniquely determined by the values it takes on intervals of the form $(-\infty, x\,]$, and hence by its \emph{cumulative distribution function} (CDF).

%\begin{definition}
%The CDF of a random variable $X:\Omega\to\R$ is the function
%\[
%\begin{array}{cccl}
%F:	& \R	& \longrightarrow	& [0,1] \\
%	& x 			& \mapsto			& \prob(X\leq x).
%\end{array}
%\]
%\end{definition}

\begin{definition}
The CDF of a random variable $X$ is the function $F:\R\to[0,1]$ given by
\[
F(x) =  \prob(X\leq x).
\]
\end{definition}

%\begin{remark}
%In the probability space $(\R,\prob_X)$, the measure of the interval $(a,b\,]$ is
%\[
%\prob_X\big[(a,b\,]\big] = \prob(a < X\leq b) = \prob(X\leq b)-\prob(X\leq a)
%= F(b) - F(a).
%\]
%Compare this to the usual measure of its length, $\mathbb{L}\big[(a,b\,]\big] = b - a$.
%\end{remark}

%-----------------------------
%\subsection{Properties of CDFs}

We can use the properties of probability measures to derive some properties of CDFs.

\begin{theorem}[Properties of CDFs]\label{thm:props_cdfs}
Let $F:\R\to[0,1]$ be a CDF. Then
\ben
\it $F$ is an increasing function,
\it $F(x)\to 0$ as $x\to-\infty$,
\it $F(x)\to 1$ as $x\to+\infty$, and
\it $F(x+h)\to F(x)$ as $h\downarrow 0$ (right continuity).
\een
\end{theorem}

\begin{proof}
Let $X:\Omega\to\R$ be a random variable and let $F$ be its CDF.
\ben
\it % (i): increasing
To show that $F$ is increasing, let $x < y$ and consider the events $A = \{X\leq x\}$ and $B = \{X\leq x'\}$, i.e.
\[
A	= \{\omega: X(\omega)\leq x\}
\quad\text{and}\quad
B	= \{\omega: X(\omega)\leq x'\}.
\]
By construction, $F(x)=\prob(A)$ and $F(x')=\prob(B)$, and because $x < x'$ we have $A\subseteq B$. By the monotonicity of probability measures, $\prob(A)\leq \prob(B)$ or equivalently $F(x) \leq F(x')$, so $F$ is an increasing function.

\it % (ii): F(x) -> 0 as x -> -\infty
To show that $F(x)\to 0$ as $x\to-\infty$, let $B_n = \{X\leq -n\}$ for $n=1,2,\ldots$. Then $F(-n)= \prob(B_n)$, and $B_1,B_2,\ldots$ is a decreasing sequence ($B_{n+1}\subseteq B_n$), with
\[
\bigcap_{n=1}^{\infty} B_n = \emptyset.
\]
(This is because for any $x$, there exists an $n$ such that $x\notin (-\infty,-n]$.) By the continuity of probability measures,
\[
\lim_{n\to\infty}F(-n) = \lim_{n\to\infty}\prob(B_n) = \prob\left(\bigcap_{n=1}^n B_n\right) = \prob(\emptyset) = 0.
\]
Because $F(x)$ is an increasing function, we conclude that $\lim_{x\to-\infty}F(x)=0$.

\it % (iii): F(x) -> 1 as x -> \infty
To show that $F(x)\to 1$ as $x\to\infty$, let $A_n = \{X\leq n\}$ for $n=1,2,\ldots$. Then $F(n)= \prob(A_n)$, and $A_1,A_2,\ldots$ is an increasing sequence ($A_n\subseteq A_{n+1}$), with
\[
\bigcup_{n=1}^{\infty} A_n = \Omega.
\]
(This is because for any $x$, there exists an $n$ such that $x\in (-\infty,n]$).  By the continuity of probability measures,
\[
\lim_{n\to\infty}F(n) = \lim_{n\to\infty}\prob(A_n) = \prob\left(\bigcup_{n=1}^{\infty} A_n\right) = \prob(\Omega) = 1,
\]
Because $F(x)$ is an increasing function, we conclude that $\lim_{x\to\infty}F(x)=1$.

\it % (iv): right continuity
To show that $F(x)$ is right-continuous, let $B_n = \{X\leq x+1/n\}$ for $n=1,2,\ldots$. Then $F\left(x+1/n\right) = \prob(B_n)$, and $B_1,B_2,\ldots$ is a decreasing sequence ($B_{n+1}\subseteq B_n$), with \[
\bigcap_{n=1}^{\infty} B_n = (-\infty,x].
\]
By the continuity of probability measures,
\[
F(x) = \prob\left(\bigcap_{n=1}^{\infty} B_n\right) = \lim_{n\to\infty} \prob(B_n) = \lim_{n\to\infty}F\left(x+\frac{1}{n}\right).
\]
Because $F(x)$ is an increasing function, we conclude that $\lim_{h\downarrow 0}F(x+h)=F(x)$.
\een
\end{proof}

\begin{exercise}
%\begin{questions}
%\question % GS 2.1.4
It can be shown that any function $F:\R\to[0,1]$ which has the four properties listed in Theorem~\ref{thm:props_cdfs} is a CDF. With this in mind, show that if $F$ and $G$ are CDFs and $0<\lambda<1$ is a constant, then $\lambda F + (1-\lambda)G$ is also a CDF.
\begin{answer}
Let $H(x) = \lambda F(x) + (1-\lambda)G(x)$.
\ben
\it if $x < x'$ then
\[
H(x) = \lambda F(x) + (1-\lambda)G(x) \leq \lambda F(x') + (1-\lambda)G(x') = H(x').
\]
\it
\[
\lim_{x\to-\infty} H(x)
	= \lim_{x\to-\infty}\left[\lambda F(x)+(1-\lambda)G(x)\right]
	=  \lambda\lim_{x\to-\infty} F(x) + (1-\lambda)\lim_{x\to-\infty} G(x)
	= 0.
\]
\it
\[
\lim_{x\to\infty} H(x)
	= \lim_{x\to\infty}\left[\lambda F(x)+(1-\lambda)G(x)\right]
	=  \lambda\lim_{x\to\infty} F(x) + (1-\lambda)\lim_{x\to\infty} G(x)
	= \lambda + (1-\lambda)
	= 1.
\]
\it
\begin{align*}
\lim_{\epsilon\downarrow 0} H(x+\epsilon)
	& = \lim_{\epsilon\downarrow 0}\left[\lambda F(x+\epsilon)+(1-\lambda)G(x+\epsilon)\right] \\
	& = \lambda\lim_{\epsilon\downarrow 0} F(x+\epsilon) + (1-\lambda)\lim_{\epsilon\downarrow 0} G(x+\epsilon) \\
	& = \lambda F(x) + (1-\lambda)G(x)
	= H(x).
\end{align*}
\een
Thus $H$ satisfies the four properties of Theorem~\ref{thm:props_cdfs}, and is therefore a CDF.
\end{answer}

%\question % GS 2.2.3
%Let $X_1,X_2,\ldots,X_n$ be independent and identically distributed random variables, and let $F$ denote their common CDF. If $F$ is unknown, find a way of estimating $F$ by considering the indicator variables of the events $\{X_i\leq x\}$.
%\begin{answer}
%Let $X$ be a random variable with same CDF, and let $I_i(x)$ the indicator variable of the event $\{X_i\leq x\}$. Then
%\[
%\prob(X\leq x) \approx \frac{1}{n}\sum_{i=1}^{n} I_i(x),
%\]
%which is the proportion of observations that are at most equal to $x$. This is called the \emph{empirical CDF} of $X$.
%\end{answer}
%\end{questions}
\end{exercise}


\chapter{Background}\label{ch2}

In this chapter, the foundations of the thesis are elaborated. We formally define methods, concepts and tools used. Starting with the description of the 3D Morphable Model (3DMM) \cite{Blanz:1999:MMS:311535.311556, Romdhani3DM} that we utilize, followed by an explanation of a novel fully probabilistic  method
to interpret a single face image with the 3DMM\cite{Schoenborn2017}. As well as, the important characteristics of the depth cameras.

\section{3D Morphable Model}
The 3DMM is a fully parametric generative face model constructed from 200 high quality face scans. The model contains probabilistic PCA (PPCA) models for shape, color and expression \cite{8373814, EGGER2017115}.  In a traditional PCA-based settings the shape S, color C and facial expression E models are constructed through the parameter set $\theta=\{\theta_S, \theta_C, \theta_E\}$ as follows:
\begin{equation}
    \begin{split}
    S(\theta) &= \mu_S + U_S D_S \theta_S + \mu_E + U_E D_E \theta_E \\
    C(\theta) &= \mu_C + U_C D_C
    \end{split}
\end{equation}
Where $\mu$ denotes the mean of the corresponding model, $U$ is a matrix that contains the principal components, and $D$ denotes a diagonal matrix consisting of the variances along the principal directions. The expression model here, is modeled as a deformation of the neutral (mean) face shape. In a probabilistic setting the shape and color models are transformed into a distribution of shape $P(S \mid \theta)$ and color $P(C \mid \theta)$ components with the additive Gaussian noise\cite{ALBRECHT2013959, EGGER2017115}:

\begin{equation}
    \begin{split}
    P(S \mid \theta) &= \mathcal{N}(S \mid \mu_S + U_S D_S \theta_S + \mu_E + U_E D_E \theta_E, \sigma_S^2 I) \\
    P(C \mid \theta) &= \mathcal{N}(C \mid \mu_C + U_C D_C, \sigma_C^2 I)
    \end{split}
\end{equation}

The parameter set $\theta$ follow a standard normal distribution in latent vector space\cite{EGGER2017115} which is also defined outside the linear span of 200 scans:

\begin{equation}
    P(\theta) = \mathcal{N}(\theta \mid 0, I)
\end{equation}
To generate realistic 3DMM instances (Figure \ref{f2.1}) and render synthetic face images $\mathcal{I}$, parameter set $\theta$ contains an additional camera $\theta_P$ and illumination $\theta_L$ parameters that are modeled separately from the other three.

\begin{figure}
    \centering
    \captionsetup{labelformat=empty}
    \begin{minipage}{.32\textwidth}
        \centering
        \includegraphics[width=\textwidth]{Figures/Pictures/rs1_t.png}
        \caption*{Sample}
      \end{minipage}
    \begin{minipage}{.32\textwidth}
      \centering
      \includegraphics[width=\textwidth]{Figures/Pictures/mean_t.png}
      \caption*{Mean}
    \end{minipage}
    \begin{minipage}{.32\textwidth}
        \centering
        \includegraphics[width=\textwidth]{Figures/Pictures/rs2_expr_t.png}
        \caption*{Sample with expression}
    \end{minipage}
      \captionsetup{labelformat=default}
      \caption{3DMM face model instances.}
      \label{f2.1}
\end{figure}

This allows us to synthesize the real world appearance of the face. By tuning all the mentioned parameters, we can approximate the appearance of the synthetic face to the target face. Besides parameter space, the model is also able to take care of the image rendering process $\Re$ which renders an image $\mathcal{I}(\theta)$ based on given parameters through
\begin{equation}
    \mathcal{I}(\theta) = \Re(\mathcal{M}(\theta_S, \theta_C, \theta_E); \theta_P, \theta_L).
\end{equation}
The 3DMM model also has a set of modified model versions that are either restricted to a certain region or are down-scaled, low-resolution versions\footnote{Down-scaled versions usually have a lesser number of points and triangles hence are less flexible, but they perform the mesh operations faster} of the original model. The model restricted to a certain region is important for domain specific applications that only utilize specific parts of the model. For example, we use the model (Figure \ref{f2.2}) from \cite{Schoenborn2017} which only covers the face region and discards other parts like ears, neck, and part of the forehead.

\begin{figure}
    \centering
    \begin{minipage}{.32\textwidth}
        \centering
        \includegraphics[width=\textwidth]{Figures/Pictures/sideA_t.png}
      \end{minipage}
      \begin{minipage}{.33\textwidth}
          \centering
          \includegraphics[width=\textwidth]{Figures/Pictures/face_bfm_close_t.png}
    \end{minipage}
    \begin{minipage}{.32\textwidth}
        \centering
        \includegraphics[width=\textwidth]{Figures/Pictures/sideB_t.png}
    \end{minipage}
    \caption{Restricted face model displayed over the full model(dark)}
    \label{f2.2}
\end{figure}

Beneficially to our application, this restriction also ignores parts of the model that are too complex, noisy or low quality, for example ears, that could potentially cause some problems during the fitting.
\section{Fitting Pipeline}
Having the flexible Parametric Appearance Model (PAM) as a basis of the fitting framework allows us to model a variety of real world scenes with challenging illumination conditions. In this section, we explain how standard fitting pipeline and its augmented variant utilizes Markov Chain Monte Carlo sampling to perform approximate inference.\bigskip

\subsection{Markov Chain Monte Carlo (MCMC)}\label{s2.2.1}
To generate parametric face instances, the Analysis-by-Synthesis method employs a probabilistic image analysis using Bayesian inference. Formally, the fitting pipeline is a procedure that for any given face image returns a model fit which best corresponds to the face in the input image. It is important to note that commonly applications aim to obtain a single solution as a result, however, this is not the case for fitting pipeline. Instead of aiming for a single solution the fitting pipeline produces a posterior distribution of possible solutions. Since the true posterior distribution is unknown, the fitting process needs to approximate it. To perform approximate inference in this setting the Metropolis-Hastings (MH) algorithm is used. The algorithm is a method of Markov Chain Monte Carlo sampling and it is especially useful when sampling from distributions from which direct sampling is not feasible. The algorithm is an iterative process that draws random samples $\theta'$ from a proposal distribution $Q(\theta'\mid\theta)$ that are being evaluated based on their likelihood value. In the figure bellow single step of MH algorithm is shown.\bigskip

\begin{algorithm}[H]
    \SetAlgoLined
     $\bullet$ Initialize sample $\theta \sim Q(\theta)$\;
     $\bullet$ Generate the next sample with current sample $\theta$:\\
     {\addtolength{\leftskip}{5mm}
     1. Propose a sample $\theta'$ based on current sample: $\theta'\sim Q(\theta'\mid\theta)$\;
     2. With probability: $\alpha = min\Big\lbrace\frac{P(\theta')}{P(\theta)}\frac{Q(\theta\mid\theta')}{Q(\theta'\mid\theta)}, 1\Big\rbrace$ accept or reject samples\;

      3. If sample $\theta'$ is accepted set $\theta'$ to be a new state $\theta$, otherwise keep $\theta$ unchanged\;
     }
     \caption{Metropolis-Hastings algorithm}
     \label{a1}
\end{algorithm}\bigskip

The two principal parts of the algorithm are the proposal distribution $Q$ from which samples are drawn and the likelihood estimation $P(\theta)$. Let us describe both of these key components in detail.  \bigskip

The proposal distribution should be simple enough to draw random samples from it. In the settings of the fitting pipeline commonly used proposal distribution is a simple Gaussian random walk proposal distribution:

\begin{equation}
   Q(\theta'\mid\theta) = \mathcal{N}(\theta'\mid\theta, \sigma^2 I_d).
   \label{2.5}
\end{equation}

Where $d$ stands for the dimension and $\sigma$ controls the step size the algorithm makes at each sampling iteration. In the fitting pipeline we usually have more than one proposal distributions for each 3DMM parameter update $\theta = \{\theta_S, \theta_C, \theta_E\, \theta_P, \theta_L\}$. Therefore, in practice, $Q(\theta'\mid\theta)$ consists of multiple proposal distributions $Q_i$ combined into one mixture proposal distribution:

\begin{equation}
    Q(\theta' | \theta) = \sum_i c_i Q_i(\theta'|\theta), \sum_i c_i = 1.
    \label{eq2.3}
\end{equation}

Where $\theta$ is a current sample, $\theta'$ is a newly proposed sample drawn from a proposal distribution $Q_i$, and $c_i$ coefficient controls how often samples are being drawn from the specific $Q_i$ distribution\cite{Schoenborn2017}.\bigskip

The second key part of the MH algorithm is a likelihood estimation term $P(\theta)$ commonly written as $P(\theta\mid\mathcal{I})$. It determines a posterior belief of the current sample parameter set $\theta$ conditioned on the target image $\mathcal{I}$. Computing the posterior directly is not possible, therefore, we employ the classical Bayes' theorem\footnote{Bayes' theorem — \url{https://en.wikipedia.org/wiki/Bayes'\_theorem}} to approximate it based on the prior knowledge $P(\theta)$ and an image likelihood $P(\mathcal{I}\mid\theta)$:

\begin{equation}
    P(\theta \mid \mathcal{I}) = \frac{P(\theta)P(\mathcal{I}\mid \theta)}{\int P(\mathcal{I}\mid \theta)P(\theta)d\theta}
\end{equation}

In this setting finding a single solution then becomes a problem of a maximum-a-posteriori (MAP) inference, which is finding the parameters with the highest posterior probability\cite{10.1007/978-3-642-40602-7_11}. Since we are only interested in the ratio of two different likelihoods $P(\theta\mid\mathcal{I})$ and $P(\theta'\mid\mathcal{I})$ the normalization term can be ignored, and we can rewrite the above formulation as:

\begin{equation}
    P(\theta | \mathcal{I}) \propto P(\theta)P(\mathcal{I} | \theta)
    \label{eq2.5}
\end{equation}

The prior probability $P(\theta)$ of the model estimate $\theta$ is usually defined as a normal distribution:
\begin{equation}
    \label{eq3}
    P(\theta) = \mathcal N(\theta | 0, \mathcal I).
\end{equation}

The only missing part of the Equation \ref{eq2.5} than is an image likelihood $P(\mathcal{I}\mid\theta)$ which can be reformulated as $P(\mathcal{I}\mid\theta) = \mathcal{L}(\theta;\mathcal{I})$ with equation \ref{eq2.5} transforming to:

\begin{equation}
    \label{eq2.5}
    P(\theta | \tilde{\mathcal I}) \propto \mathcal L(\theta;\tilde{\mathcal I})P(\theta)
\end{equation}
An image $\mathcal{I}$ that appears in likelihood term means that during the fitting target image is treated as an observation of the 3DMM model instance.
Depending on the phase the fitting pipeline is in, a combination of likelihood functions is used. We briefly discuss the most important likelihood functions used in the fitting pipeline alongside with the phase they are used in.

\subsection{Landmark Fitting}
During the starting phase of the fitting, it is important to have good pose estimation parameters since the pipeline is crucially dependent on it. If the pose is wrong, the rest of the model parameters cannot be inferred reliably. To deal with this problem, the standard fitting pipeline relies on the landmark fitting phase. During this phase fitting only modifies 3DMMs $\theta_P$ and $\theta_S$ parameters. It does so by evaluating a set of landmark point locations $x_i$ detected (by detection algorithm or manually) onto the target image and projected to the model instance using computer graphics. The projection method uses a pinhole camera model to align model instance to the target face by applying a rotation and translation parameters and then render individual landmarks onto the image plane. By treating those points as an observation of corresponding model points, the algorithm evaluates them using isotropic Gaussian likelihood function:

\begin{equation}
\mathcal L(\theta; x_1, x_2,\dots, x_N) = \prod_{i=1}^{N} \mathcal N(x_i\mid y_i(\theta), \sigma^2_{LM} I_2) \text{\cite{Schoenborn2017}}.
\label{lmeval}
\end{equation}

Where $x_i$ are observed landmark positions and $y_i(\theta)$ are model landmark positions based on $\theta$. Proposed samples with new camera and shape parameters are getting accepted or rejected based on its likelihood value according to the procedure discussed earlier in Algorithm \ref{a1}. Pose parameters that are part of $\theta_P$ are a combination of Euler rotation angles \textit{\{yaw, nick, roll\}}, translation vector $\vec{t} = \{t_x, t_y, t_z\}$ with $t_z$ indicating the distance from the camera, and scaling parameter to control the focal length of the camera. The proposal distribution (Equation
\ref{eq2.3}) for landmark fitting consists of a mixture distribution over shape and camera (pose) parameter update proposals. As discussed previously, shape parameters $\theta_S$ are PCA coefficients of the low-rank expansion of the Gaussian Process\cite{8010438} model that are proposed by a weak isotropic Gaussian perturbation proposal\cite{Schoenborn2017}.  The described process adjusts the pose and at the same time makes initial shape correction. Landmark fitting is usually a very fast process since the only few landmark points (usually less than 10) are getting evaluated at a time.

\subsection{Color Fitting}

After the landmark fitting phase the pose of the model instance usually fits well with the target face in the image and the model instance is ready to be utilized for color, illumination and optionally expression proposals as well as shape update proposals. Thus, during the color fitting phase the mixture proposal $Q(\theta'\mid\theta)$ will be a combination of all the above mentioned parameter proposals with various step size and drawing probability as of Equation \ref{eq2.3}. To evaluate proposed samples in this phase the standard fitting pipeline together with landmark evaluation (Equation \ref{lmeval}) relies on independent pixel evaluator likelihood which distinguishes foreground and background pixels\cite{Schonborn:2015:BMG:2798342.2798359} and is formulated as follows:

\begin{equation}
    \mathcal L \left (\theta; \tilde{\mathcal I} \right )
= \prod_{i \in \mathcal F} \mathcal N \left( \tilde{\mathcal I}_i \,\middle |\, \mathcal I_i(\theta), \sigma^2 I_3 \right ) \prod_{i \in \mathcal B} \mathcal L_{\text{BG}}\left ( \tilde{\mathcal I}_i \right ).
\label{eq2.7}
\end{equation}

Where the first product evaluates foreground pixels $i\in\mathcal F$ and second product evaluates background pixels $i\in\mathcal B$. The motivation behind incorporating background pixels into this likelihood and not ignoring them is that, when they are ignored, their likelihood value is assumed to be 1, which is not necessarily true in most cases. \bigskip

So called propose-and-verify flow of the initial architecture of the Metropolis-Hastings algorithm used by \cite{Schoenborn2017, Schoenborn2014} is shown in Figure \ref{f2.3}. As we have mentioned previously, the architecture only uses color image and 2D landmarks as an input.

\begin{figure}
    \centering
    \includegraphics[width=0.85\textwidth]{Figures/flow1.PNG}
    \caption{The architecture of the MH algorithm used by \cite{Schoenborn2017} taken from \cite{betschard2016}}.
    \label{f2.3}
\end{figure}

An augmented version of this flow by \cite{betschard2016} can be seen in Figure \ref{f2.4}, where authors introduced additional depth image and 3D landmark input sources, with respective evaluators. \bigskip

\begin{figure}
    \centering
    \includegraphics[width=0.85\textwidth]{Figures/flow2.PNG}
    \caption{Augmented MH architecture proposed by \cite{betschard2016}, changes made in the architecture are in bold.}
    \label{f2.4}
\end{figure}

We introduce an alternative way of dealing with this architecture. Instead of using just a depth image (only z-Buffer) we construct a triangle mesh from a point cloud obtained using the depth camera. Details of this process are described in Section \ref{s3.3.3}. We were also forced to find a work-around way to obtain 3D landmarks (Section \ref{s3.3.2}), since they are not provided by the camera SDK anymore. We split the standard fitting pipeline into two sub-fitting pipelines, one that deals with the shape and pose parameter estimation hereafter referred to as \textit{shape fitting module} and the other with color, illumination, and expression with slight shape and pose parameter updates hereafter referred as \textit{color fitting module}. Both of these modules are described in Section \ref{s3.4}.

\section{Depth Camera}

Inferring the exact size of an object when analyzing color images is a challenging task, there is no effective way to reliably recover this information based on color intensities. The depth camera helps us to resolve this issue by providing a distance measure for each pixel (if it is available). There are a few different ways distance is obtained by the depth cameras. One of the algorithms commonly used (and the one our camera uses) to calculate distance is the depth from stereo algorithm better known as \textbf{Stereoscopic Vision}\footnote{\url{https://en.wikipedia.org/wiki/Computer_stereo_vision}} which is a realization of a natural Binocular vision. The basic idea behind Stereoscopic vision is to estimate the distance from the camera to each point by calculating disparities between two parallel view-ports\cite{serg, DBLP:journals/corr/KeselmanWGB17} (Figure \ref{f2.5}). View-port parallelism is achieved with the traditional Image Rectification\footnote{\url{https://en.wikipedia.org/wiki/Image_rectification}} approach, which transforms images onto a common virtual image plane and makes sure that two view-port points are matching.

\begin{figure}
    \centering
    \includegraphics[width=0.55\textwidth]{Figures/Pictures/stereo-ssd-1.png}
    \caption{Depth from stereo. Visualizing two separate view-ports that are being used to calculate disparities. Source: Intel}
    \label{f2.5}
\end{figure}

\subsection{De-projection}
Pixel-to-Point and Point-to-Pixel de-projection is a common use-case in computer vision. When needed this feature offers pixel to point mapping and vice-versa by relying on cameras' depth information and intrinsic parameters. Successful de-projection is based on the traditional pinhole camera model \cite{Hartley:2003:MVG:861369} shown in Figure \ref{f2.6}.

\begin{figure}[h]
    \centering
    \includegraphics[width=\textwidth]{Figures/pcm.png}
    \caption{Pinhole camera model. $\mathcal F_c$ is the center of the camera, (X, Y, Z) are coordinates of a 3D point in the world coordinate system, ($u$,$v$) are coordinates of a respective point projection in pixels, ($c_x, c_y$) indicates principal points usually located at the image center.  Source: \url{https://docs.opencv.org}}
    \label{f2.6}
\end{figure}

The problem of point to pixel projection is formulated as follows. Given a 3D point coordinates $P_{3D}(X, Y, Z)$ with camera intrinsic parameters\footnote{\url{https://github.com/IntelRealSense/librealsense/blob/master/include/librealsense2/h/rs\_types.h\#L55}} containing ($width$, $height$, $ppx$, $ppy$, $f_x$, $f_y$), calculate the respective pixel coordinates $P(u, v)$ with no distortion introduced. Where in camera parameters, $width$ and $height$ are image dimensions in pixels, $ppx$ and $ppy$ are the horizontal and vertical coordinate of the principal point of the image given as an offset from the left and top edge respectively, and the $f_x$ and $f_y$ are the focal lengths of the image as a multiple of pixel width and height. The focal length is usually the distance from the center of the camera to the image plane also known as the focal plane. It is shown as gray square in Figure \ref{f2.6}.  Once we have all these variables, then the $u$ and $v$ pixel coordinates of $P(u, v)$ are calculated as follows:

\begin{equation}
    \begin{split}
    &x' = \frac{X}{Z}\\
    &y' = \frac{Y}{Z}\\
    &u = x' \cdot f_x + ppx\\
    &v = y' \cdot f_y + ppy\\
    \end{split}
\end{equation}

On the other hand, the problem of de-projecting pixel coordinates into the 3D coordinate space to obtain a 3D point is formulated as follows. Given a pixel coordinates $P(u, v)$ and depth information $d = Z$ alongside the camera parameter set mentioned previously with no distortion introduced, compute the corresponding $P_{3D}(X, Y, Z)$ point in the 3D space. Then $P_{3D}(X, Y, Z)$ coordinates are being calculated as follows:

\begin{equation}
    \begin{split}
    &x' = \frac{u - ppx}{fx}\\
    &y' = \frac{v - ppy}{fy}\\
    &X = d \cdot x'\\
    &Y = d \cdot y'\\
    &Z = d
    \end{split}
\end{equation}

Both techniques we have discussed are already implemented in the camera SDK for us, therefore, throughout the project, we make use of those default methods provided by the SDK.


%% NOTE: This section must follow the sections on Orthogonality

\section{Application: Raising a symmetric matrix to a high power}

We already have seen how to use matrix diagonalization to compute
powers of matrices. This requires computing eigenvalues of the matrix
$A$, and finding an invertible matrix of eigenvectors $P$ such that
$P^{-1}AP$ is diagonal. In this section we will see that if the matrix
$A$ is symmetric (see
Definition~\ref{def:symmetric-and-antisymmetric}), then we can
actually find such a matrix $P$ that is an orthogonal matrix of
eigenvectors. Thus $P^{-1}$ is simply its transpose $P^T$, and $P^TAP$
is diagonal. When this happens we say that $A$ is \textbf{orthogonally
  diagonalizable}%
\index{matrix!orthogonally diagonalizable}.

In fact this happens if and only if $A$ is a symmetric matrix as shown
in the following important theorem.

\begin{theorem}{Principal axis theorem}{principal-axis}
  The following conditions are equivalent for an $n\times n$-matrix
  $A$:
  \begin{enumerate}
  \item $A$ is symmetric.
  \item $A$ has an orthonormal set of eigenvectors.
  \item $A$ is  orthogonally diagonalizable.
  \end{enumerate}
\end{theorem}

\begin{proof}
  The complete proof is beyond this course, but to give an idea assume
  that $A$ has an orthonormal set of eigenvectors, and let $P$ consist
  of these eigenvectors as columns. Then $P^{-1}=P^T$, and $P^TAP=D$ a
  diagonal matrix. But then $A=PDP^T$, and
  \begin{equation*}
    A^T=(PDP^T)^T = (P^T)^TD^TP^T=PDP^T=A,
  \end{equation*}
  so $A$ is symmetric.

  Now given a symmetric matrix $A$, one shows that eigenvectors
  corresponding to different eigenvalues are always orthogonal. So it
  suffices to apply the Gram-Schmidt process on the set of basic
  eigenvectors of each eigenvalue to obtain an orthonormal set of
  eigenvectors.
\end{proof}

We demonstrate this in the following example.

\begin{example}{Orthogonal diagonalization of a symmetric matrix}{orthogonal-diagonalization}
  Let $\def\arraystretch{1.2}A=\begin{mymatrix}{rrr}
    1 & 0 & 0 \\
    0 & \frac{3}{2} & \frac{1}{2} \\
    0 & \frac{1}{2} & \frac{3}{2}
  \end{mymatrix}$.  Find an orthogonal matrix $P$ such that $P^{T}AP$
  is a diagonal matrix.
\end{example}

\begin{solution}
  In this case, verify that the eigenvalues are 2 and 1. First we will
  find an eigenvector for the eigenvalue $2$. This involves row
  reducing the following augmented matrix:
  \begin{equation*}
    \def\arraystretch{1.2}
    \begin{mymatrix}{ccc|c}
      2 - 1 & 0 & 0 & 0 \\
      0 & 2-\frac{3}{2} & -\frac{1}{2} & 0 \\
      0 & -\frac{1}{2} & 2-\frac{3}{2} & 0
    \end{mymatrix}.
  \end{equation*}
  The {\rref} is
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      1 & 0 & 0 & 0 \\
      0 & 1 & -1 & 0 \\
      0 & 0 & 0 & 0
    \end{mymatrix},
  \end{equation*}
  and so an eigenvector is
  \begin{equation*}
    \begin{mymatrix}{c}
      0 \\
      1 \\
      1
    \end{mymatrix}.
  \end{equation*}
  Finally to obtain an eigenvector of length one (unit eigenvector) we
  simply divide this vector by its length to yield:
  \begin{equation*}
    \def\arraystretch{1.2}
    \begin{mymatrix}{c}
      0 \\
      \frac{1}{\sqrt{2}} \\
      \frac{1}{\sqrt{2}}
    \end{mymatrix}.
  \end{equation*}
  Next consider the case of the eigenvalue $1$. To obtain basic
  eigenvectors, the matrix which needs to be row reduced in this case
  is
  \begin{equation*}
    \def\arraystretch{1.2}
    \begin{mymatrix}{ccc|c}
      1-1 & 0 & 0 & 0 \\
      0 & 1-\frac{3}{2} & -\frac{1}{2} & 0 \\
      0 & -\frac{1}{2} & 1-\frac{3}{2} & 0
    \end{mymatrix}.
  \end{equation*}
  The {\rref} is
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      0 & 1 & 1 & 0 \\
      0 & 0 & 0 & 0 \\
      0 & 0 & 0 & 0
    \end{mymatrix}.
  \end{equation*}
  Therefore, the eigenvectors are of the form
  \begin{equation*}
    \begin{mymatrix}{c}
      s \\
      -t \\
      t
    \end{mymatrix}.
  \end{equation*}
  Note that all these vectors are automatically orthogonal to
  eigenvectors corresponding to the first eigenvalue. This follows
  from the fact that $A$ is symmetric, as mentioned earlier.
  We obtain basic eigenvectors
  \begin{equation*}
    \begin{mymatrix}{r}
      1 \\
      0 \\
      0
    \end{mymatrix} \text{ and }\begin{mymatrix}{r}
      0 \\
      -1 \\
      1
    \end{mymatrix}.
  \end{equation*}
  Since they are themselves orthogonal (by luck here) we do not need
  to use the Gram-Schmidt process and instead simply normalize these
  vectors to obtain
  \begin{equation*}
    \def\arraystretch{1.2}
    \begin{mymatrix}{r}
      1 \\
      0 \\
      0
    \end{mymatrix} \text{ and }\begin{mymatrix}{c}
      0 \\
      -\frac{1}{\sqrt{2}} \\
      \frac{1}{\sqrt{2}}
    \end{mymatrix}.
  \end{equation*}
  An orthogonal matrix $P$ to orthogonally diagonalize $A$ is then
  obtained by letting these basic vectors be the columns.
  \begin{equation*}
    \def\arraystretch{1.3}
    P= \begin{mymatrix}{ccc}
      0 & 1 & 0 \\
      -\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
      \frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
    \end{mymatrix}.
  \end{equation*}
  We verify this works. $P^{T}AP$ is of the form
  \begin{equation*}
    \def\arraystretch{1.3}
    \begin{mymatrix}{ccc}
      0 & -\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
      \\
      1 & 0 & 0 \\
      0 & \frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
    \end{mymatrix} \begin{mymatrix}{ccc}
      1 & 0 & 0 \\
      0 & \frac{3}{2} & \frac{1}{2} \\
      0 & \frac{1}{2} & \frac{3}{2}
    \end{mymatrix} \begin{mymatrix}{ccc}
      0 & 1 & 0 \\
      -\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
      \frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    =\allowbreak \begin{mymatrix}{ccc}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
      0 & 0 & 2
    \end{mymatrix},
  \end{equation*}
  which is the desired diagonal matrix.
\end{solution}

We can now apply this technique to efficiently compute high powers of
a symmetric matrix.

\begin{example}{Powers of a symmetric matrix}{powers-symmetric-matrix}
  Let $\def\arraystretch{1.2}
  A=\begin{mymatrix}{rrr}
    1 & 0 & 0 \\
    0 & \frac{3}{2} & \frac{1}{2} \\
    0 & \frac{1}{2} & \frac{3}{2}
  \end{mymatrix}$.
  Compute $A^7$.
\end{example}

\begin{solution}
  We found in Example~\ref{exa:orthogonal-diagonalization} that
  $P^TAP=D$ is diagonal, where
  \begin{equation*}
    \def\arraystretch{1.3}
    P= \begin{mymatrix}{ccc}
      0 & 1 & 0 \\
      -\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
      \frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
    \end{mymatrix} \text{ and }
    D = \begin{mymatrix}{ccc}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
      0 & 0 & 2
    \end{mymatrix}.
  \end{equation*}
  Thus $A=PDP^T$ and $A^7=PDP^T \; PDP^T \; \cdots PDP^T = PD^7P^T$
  which gives:
  \begin{equation*}
    \def\arraystretch{1.3}
    \begin{array}{rr}
      A^7 & =
            \begin{mymatrix}{ccc}
              0 & 1 & 0 \\
              -\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
              \frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
            \end{mymatrix}
            \begin{mymatrix}{ccc}
              1 & 0 & 0 \\
              0 & 1 & 0 \\
              0 & 0 & 2
            \end{mymatrix} ^7
            \begin{mymatrix}{ccc}
              0 & -\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}\\
              1 & 0 & 0 \\
              0 & \frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
            \end{mymatrix}  \\\\[-2ex]
          & =
            \begin{mymatrix}{ccc}
              0 & 1 & 0 \\
              -\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
              \frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
            \end{mymatrix}
            \begin{mymatrix}{ccc}
              1 & 0 & 0 \\
              0 & 1 & 0 \\
              0 & 0 & 2^7
            \end{mymatrix}
            \begin{mymatrix}{ccc}
              0 & -\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}\\
              1 & 0 & 0 \\
              0 & \frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
            \end{mymatrix}  \\\\[-2ex]
          & =
            \begin{mymatrix}{ccc}
              0 & 1 & 0 \\
              -\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
              \frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
            \end{mymatrix}
            \begin{mymatrix}{ccc}
              0 & -\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}\\
              1 & 0 & 0 \\
              0 & \frac{2^7}{\sqrt{2}} & \frac{2^7}{\sqrt{2}}
            \end{mymatrix}  \\\\[-2ex]
          & =
            \begin{mymatrix}{ccc}
              1 & 0 & 0 \\
              0 & \frac{2^7+1}{2} & \frac{2^7-1}{2}\\
              0 & \frac{2^7-1}{2} & \frac{2^7+1}{2}
            \end{mymatrix}.
    \end{array}
  \end{equation*}
\end{solution}