\subsection{Infinite plate with a circular hole}
\paragraph{}
In this example, an infinite plate with a traction free hole under uniaxial tension $(\sigma = \SI{1}{\newton \per \square \meter} )$
along x-axis (see Fig.~\ref{iso_fig:circular_hole_geo_bc}) is considered.
%
    \begin{figure}[h!]
        \centering
        \scalebox{0.5}{
            \includegraphics{isogeometric_sbfem/images/circular_hole_geo_bc.eps}
        }
        \caption{ Infinite plate with a circular hole: geometry and boundary conditions}
        \label{iso_fig:circular_hole_geo_bc}
    \end{figure}
%
The exact solution of the stresses in polar coordinate $(r,\theta)$ is given by \citep{Sukumar2001}:
    \begin{subequations}
        \begin{align}
            \sigma_{x}(r,\theta) &= 1 - \frac{a^2}{r^2} \left(
                \frac{3}{2} \cos2\theta +
                \cos4\theta
            \right) +
            \frac{3a^4}{2r^4} \cos4\theta\\
            \sigma_{y}(r,\theta) &= -\frac{a^2}{r^2} \left(
                \frac{1}{2} \cos2\theta -
                \cos4\theta
            \right) -
            \frac{3a^4}{2r^4} \cos4\theta\\
            \gamma_{xy}(r,\theta) &= -\frac{a^2}{r^2} \left(
                \frac{1}{2} \sin2\theta +
                \sin4\theta
            \right) -
            \frac{3a^4}{2r^4} \sin4\theta
        \end{align}
        \label{iso_eq:ex_chole_stress_sol}
    \end{subequations}
%
where $a$ is the radius of the hole.
Owing to symmetry, only one quarter of the plate is modeled.
% Fig. 10 shows a typical control net used for the study.
The material properties are: Young’s modulus $E = \SI{100}{\newton \per \square \meter}$ and Poisson’s ratio $\nu = 0.3$.
The closed form displacement in Cartesian coordinate is given as
    \begin{subequations}
        \begin{align}
            u_x(r,\theta) = \frac{R}{8G} \left[
                \frac{r}{R} (\kappa + 1) \cos\theta +
                2 \frac{R}{r} \left(
                    (1+ \kappa ) \cos\theta +
                    \cos3\theta
                \right) -
                2 \frac{R^3}{r^3}\cos3\theta
            \right] \\
            u_y(r,\theta) = \frac{R}{8G} \left[
                \frac{r}{R} (\kappa -3) \sin\theta +
                2 \frac{R}{r} \left(
                    (1 - \kappa) \sin\theta +
                    \sin3\theta
                \right) -
                2 \frac{R^3}{r^3}\sin3\theta
            \right]
        \end{align}
    \label{iso_eq:ex_chole_disp_sol}
    \end{subequations}
%
where $G$ is the shear modulus and $\kappa$ (Kolosov constant) is defined as
    \begin{equation}
        \kappa = \left\{
            \begin{aligned}
                3-4 \nu \\
                \frac{3- \nu }{1+ \nu }
            \end{aligned}
        \right.
    \label{iso_eq:kolosov_constant}
    \end{equation}
%
In this example, analytical tractions in Eq.~\ref{iso_eq:ex_chole_stress_sol} are applied on the boundary.
The left and bottom boundaries are constrained with a roller boundary condition. $u_x=0$ where $y=0$ and $u_y=0$ where $x=0$.

\paragraph{}
% result analysis
The convergence rate in terms of the displacement norm is shown in Fig.~\ref{iso_fig:circular_hole_convergence}
It is observed that the error decreases as the order of the shape functions is increased.
Another unique feature of the proposed method is that the method allows different type or order of shape functions to be employed for different segments of the boundary.
For example, in Fig.~\ref{iso_fig:circular_hole_mesh}, the arc A–B and the line segment B–C can be represented by different basis functions, i.e., the arc can be represented by NURBS and the line segment can be represented by conventional Lagrange basis functions.
Fig.~\ref{iso_fig:circular_hole_basis} shows the plot of the NURBS basis and Lagrange basis functions.
It can be seen that at point B, the shape function is continuous.
In order to assess the behavior, we represent the arc with quadratic NURBS and the straight lines with conventional Lagrange shape functions.
Since the NURBS are interpolatory at the ends, the compatibility requirement is automatically satisfied (see Fig.~\ref{iso_fig:circular_hole_shape_function_convergence}).
% The convergence of the relative error in the L 2 and H 1 is shown in Fig. 13.
The results from both the approaches converge with mesh refinement

    \begin{figure}
        \begin{subfigure}[b]{0.5\linewidth}
            \centering
            \scalebox{1}{
                \input{isogeometric_sbfem/images/circular_hole_mesh_1.tikz}
            }
        \end{subfigure}
        \begin{subfigure}[b]{0.5\linewidth}
            \centering
            \scalebox{1}{
                \input{isogeometric_sbfem/images/circular_hole_mesh_2.tikz}
            }
        \end{subfigure}
        \caption[Plate with circular hole: mesh]{
            Plate with circular hole: control net for two different discretizations.
            Control points (
                \tikz\draw[blue,fill=blue] (0,0) circle (.7ex);
            ), Boundary lines (
                \tikz[baseline=-0.5ex]\draw[blue,thick] (0,0) -- (0.5,0);
            ), Control polygon (
                \begin{tikzpicture}[baseline=-0.5ex]
                    \draw[blue,thick,dashed] (0,0) -- (1,0);
                    \draw[blue,fill=blue] (0.5,0) circle (.7ex);
                \end{tikzpicture}
            ),
            Scaling center (
                \tikz[baseline=-0.5ex]\draw[blue] (0,0) circle (0.7ex);
            ).
        }
    \label{iso_fig:circular_hole_mesh}
    \end{figure}

    \begin{figure}
        \begin{subfigure}[b]{1\linewidth}
            \centering
            \scalebox{0.7}{
                \input{isogeometric_sbfem/images/circular_hole_displacement_convergence.tikz}
            }
            \label{iso_fig:circular_hole_displacement_convergence}
            \caption{the relative error in displacement norm $(L^2)$}
        \end{subfigure}

        \begin{subfigure}[b]{1\linewidth}
            \centering
            \scalebox{0.7}{
                \input{isogeometric_sbfem/images/circular_hole_energy_convergence.tikz}
            }
            \label{iso_fig:circular_hole_energy_convergence}
            \caption{the relative error in the energy norm}
        \end{subfigure}
    \caption{Bending of thick cantilever beam: Convergence results}
    \label{iso_fig:circular_hole_convergence}
    \end{figure}

    \begin{figure}
        \centering
        \scalebox{0.5}{
            \includegraphics{isogeometric_sbfem/images/circular_hole_basis.png}
        }
        \caption[NURBS basis functions and Lagrange basis functions]{NURBS basis functions and Lagrange basis functions. It can be seen that at Point $B$, the shape functions are continuous}
        \label{iso_fig:circular_hole_basis}
    \end{figure}

    \begin{figure}
        \centering
        \scalebox{0.5}{
            \includegraphics{isogeometric_sbfem/images/circular_hole_shape_function_convergence.png}
        }
        \caption[Infinite plate with a circular hole: Convergence results]{
            Infinite plate with a circular hole: Convergence results for the relative error in the displacement norm ($L^2$) and
                the relative error in the energy norm. In this case, the arc is represented with NURBS and Lagrange shape functions.
        }
        \label{iso_fig:circular_hole_shape_function_convergence}
    \end{figure}


\section*{Exercises}

\begin{ex}
  Find the cofactor matrix and the adjugate of each of the following
  matrices.
  \begin{equation*}
    A =
    \begin{mymatrix}{rr}
      1 & 0 \\
      3 & 2 \\
    \end{mymatrix},
    \quad
    B =
    \begin{mymatrix}{rrr}
      1  & 0  & 1  \\
      -1 & 1  & -2 \\
      2  & -1 & 3  \\
    \end{mymatrix},
    \quad
    C =
    \begin{mymatrix}{rrr}
      1 &  1 & 3 \\
      2 &  3 & 1 \\
      1 & -1 & 1 \\
    \end{mymatrix}.
  \end{equation*}
\end{ex}

\begin{ex}
  For each of the following matrices, determine whether it is
  invertible by checking whether the determinant is non-zero. If the
  determinant is non-zero, use the adjugate formula to find the
  inverse.
  \begin{equation*}
    A =
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      0 & 2 & 1 \\
      3 & 1 & 0 \\
    \end{mymatrix},
    \quad
    B =
    \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 2 & 1 \\
      3 & 1 & 1 \\
    \end{mymatrix},
    \quad
    C =
    \begin{mymatrix}{rrr}
      1 & 3 & 3 \\
      2 & 4 & 1 \\
      0 & 1 & 1 \\
    \end{mymatrix},
    \quad
    D =
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      0 & 2 & 1 \\
      2 & 6 & 7 \\
    \end{mymatrix},
    \quad
    E =
    \begin{mymatrix}{rrr}
      1 & 0 & 3 \\
      1 & 0 & 1 \\
      3 & 1 & 0 \\
    \end{mymatrix}.
  \end{equation*}

  \begin{sol}
    \begin{enumerate}
    \item $\det(A) = -13$ and so $A$ is invertible. This inverse is
      \begin{eqnarray*}
        \frac{1}{-13}\begin{mymatrix}{rrr}
          \begin{absmatrix}{cc}
            2 & 1 \\
            1 & 0 \\
          \end{absmatrix} & -\begin{absmatrix}{cc}
            0 & 1 \\
            3 & 0 \\
          \end{absmatrix} & \begin{absmatrix}{cc}
            0 & 2 \\
            3 & 1 \\
          \end{absmatrix} \\\\[-1ex]
          -\begin{absmatrix}{cc}
            2 & 3 \\
            1 & 0 \\
          \end{absmatrix} & \begin{absmatrix}{cc}
            1 & 3 \\
            3 & 0 \\
          \end{absmatrix} & -\begin{absmatrix}{cc}
            1 & 2 \\
            3 & 1 \\
          \end{absmatrix} \\\\[-1ex]
          \begin{absmatrix}{cc}
            2 & 3 \\
            2 & 1 \\
          \end{absmatrix} & -\begin{absmatrix}{cc}
            1 & 3 \\
            0 & 1 \\
          \end{absmatrix} & \begin{absmatrix}{cc}
            1 & 2 \\
            0 & 2 \\
          \end{absmatrix}\end{mymatrix}^T &=&\frac{1}{-13}\begin{mymatrix}{rrr}
          -1 & 3  & -6 \\
          3  & -9 & 5  \\
          -4 & -1 & 2  \\
        \end{mymatrix}^T
        \\
              &=& \def\arraystretch{1.3}
                  \begin{mymatrix}{rrr}
                    \frac{1}{13} & -\frac{3}{13} & \frac{4}{13} \\
                    -\frac{3}{13} & \frac{9}{13} & \frac{1}{13} \\
                    \frac{6}{13} & -\frac{5}{13} & -\frac{2}{13} \\
                  \end{mymatrix}.
      \end{eqnarray*}
    \item $\det(B) = 7$, so $B$ is invertible. The inverse is
      $\displaystyle
      \def\arraystretch{1.3}
      \frac{1}{7}
      \begin{mymatrix}{rrr}
        1 & 3 & -6 \\
        -2 & 1 & 5 \\
        2 & -1 & 2 \\
      \end{mymatrix}^T = \begin{mymatrix}{rrr}
        \frac{1}{7} & -\frac{2}{7} & \frac{2}{7} \\
        \frac{3}{7} & \frac{1}{7} & -\frac{1}{7} \\
        -\frac{6}{7} & \frac{5}{7} & \frac{2}{7} \\
      \end{mymatrix}.$
    \item $\det(C) = 3$, so $C$ is invertible. The inverse is
      $\def\arraystretch{1.3}
      \begin{mymatrix}{rrr}
        1 & 0 & -3 \\
        -\frac{2}{3} & \frac{1}{3} & \frac{5}{3} \\
        \frac{2}{3} & -\frac{1}{3} & -\frac{2}{3} \\
      \end{mymatrix}$.
    \item $\det(D) = 0$, so $D$ is not invertible.
    \item $\det(E) = 2$, and so $E$ is invertible. The inverse is
      $\def\arraystretch{1.3}
      \begin{mymatrix}{rrr}
        -\frac{1}{2} & \frac{3}{2} & 0 \\
        \frac{3}{2} & -\frac{9}{2} & 1 \\
        \frac{1}{2} & -\frac{1}{2} & 0 \\
      \end{mymatrix}$.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Determine whether each of the following matrices is invertible. If
  so, use the adjugate formula to find the inverse. If the inverse
  does not exist, explain why.
  \begin{equation*}
    A = \begin{mymatrix}{rr}
      1 & 1 \\
      1 & 2 \\
    \end{mymatrix},
    \quad
    B = \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      0 & 2 & 1 \\
      4 & 1 & 1 \\
    \end{mymatrix},
    \quad
    C = \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      2 & 3 & 0 \\
      0 & 1 & 2 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    \begin{enumerate}
    \item $\det(A) = 1$, so $A$ is invertible. The inverse is
      $A^{-1} = \begin{mymatrix}{rr}
        2 & -1 \\
        -1 & 1 \\
      \end{mymatrix}$.
    \item $\det(B) = -15$, so $B$ is invertible. The inverse is
      $\displaystyle B^{-1} = \frac{1}{15}\begin{mymatrix}{rrr}
        -1 & -1 &  4 \\
        -4 & 11 &  1 \\
        8  & -7 & -2 \\
      \end{mymatrix}$.
    \item $\det(C) = 0$, so $C$ is not invertible.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Use the adjugate formula to find the inverse of the matrix
  \begin{equation*}
    A=\begin{mymatrix}{rrr}
      3 & 0 & 3 \\
      -1 & 2 & -3 \\
      -5 & 4 & -3
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    We have
    \begin{equation*}
      \det(A)=36,
      \quad
      \cof(A)
      ~=~ \begin{mymatrix}{rrr}
        6  & 12 & 6   \\
        12 & 6  & -12 \\
        -6 & 6  & 6   \\
      \end{mymatrix},
      \quad
      \adj(A)
      ~=~ \begin{mymatrix}{rrr}
        6  &  12 & -6 \\
        12 &   6 &  6 \\
        6  & -12 &  6 \\
      \end{mymatrix},
    \end{equation*}
    and therefore
    \begin{equation*}
      \def\arraystretch{1.4}
      A^{-1}
      ~=~
      \frac{1}{36}
      \begin{mymatrix}{rrr}
        6  &  12 & -6 \\
        12 &   6 &  6 \\
        6  & -12 &  6 \\
      \end{mymatrix}
      ~=~
      \begin{mymatrix}{rrr}
        \frac{1}{6}  &  \frac{1}{3} & -\frac{1}{6} \\
        \frac{1}{3} &   \frac{1}{6} &  \frac{1}{6} \\
        \frac{1}{6}  & -\frac{1}{3} &  \frac{1}{6} \\
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Use the adjugate formula to find the inverse of the matrix
  \begin{equation*}
    A=\begin{mymatrix}{rrr}
      1 &  1 & 0 \\
      3 &  1 & 2 \\
      2 & -2 & 5 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    We have
    \begin{equation*}
      \det(A)
      ~=~ 2,
      \quad
      \cof(A)
      ~=~ \begin{mymatrix}{rrr}
        -9 & 11 &  8  \\
        -2 &  2 &  2  \\
        5  & -5 & -4 \\
      \end{mymatrix},
      \quad
      \adj(A)
      ~=~ \begin{mymatrix}{rrr}
        -9 & -2 &  5 \\
        11 &  2 & -5 \\
        8  &  2 & -4 \\
      \end{mymatrix},
    \end{equation*}
    and therefore
    \begin{equation*}
      \def\arraystretch{1.4}
      A^{-1}
      ~=~
      \frac{1}{2}
      \begin{mymatrix}{rrr}
        -9 & -2 &  5 \\
        11 &  2 & -5 \\
        8  &  2 & -4 \\
      \end{mymatrix}
      ~=~
      \begin{mymatrix}{rrr}
        -\frac{9}{2} & -1 & \frac{5}{2} \\
        \frac{11}{2} &  1 & -\frac{5}{2} \\
        4  &  1 & -2 \\
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the matrix
  \begin{equation*}
    A =
    \begin{mymatrix}{ccc}
      1 & 0 & 0 \\
      0 & \cos t & -\sin t \\
      0 & \sin t & \cos t \\
    \end{mymatrix}.
  \end{equation*}
  Does there exist a value of $t$ for which this matrix fails to be
  invertible? Explain.
  \begin{sol}
    No. It has non-zero determinant $\det(A)=\cos^2 t+\sin^2 t = 1$ for
    all $t$, so it is invertible for all $t$.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the matrix
  \begin{equation*}
    A =
    \begin{mymatrix}{rrr}
      1 & t & t^2 \\
      0 & 1 & 2t \\
      t & 0 & 2
    \end{mymatrix}.
  \end{equation*}
  Does there exist a value of $t$ for which this matrix fails to be
  invertible? Explain.
  \begin{sol}
    $\det(A) = \begin{absmatrix}{ccc}
      1 & t & t^2 \\
      0 & 1 & 2t \\
      t & 0 & 2
    \end{absmatrix} = t^3+2$,
    and so $A$ has no inverse when $t=-\sqrt[3]{2}$.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the matrix
  \begin{equation*}
    A =
    \begin{mymatrix}{ccc}
      e^{t} & \cosh t & \sinh t \\
      e^{t} & \sinh t & \cosh t \\
      e^{t} & \cosh t & \sinh t
    \end{mymatrix}.
  \end{equation*}
  Does there exist a value of $t$ for which this matrix fails to be
  invertible? Explain.
  \begin{sol}
    Since the matrix $A$ has two identical rows, we have $\det(A)=0$
    for all $t$. So this matrix is non-invertible for all $t$.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the matrix
  \begin{equation*}
    A =
    \begin{mymatrix}{ccc}
      e^{t} & e^{-t}\cos t & e^{-t}\sin t \\
      e^{t} & -e^{-t}\cos t-e^{-t}\sin t & -e^{-t}\sin t+e^{-t}\cos t \\
      e^{t} & 2e^{-t}\sin t & -2e^{-t}\cos t
    \end{mymatrix}.
  \end{equation*}
  Does there exist a value of $t$ for which this matrix fails to be
  invertible? Explain.
  \begin{sol}
    \begin{equation*}
      \det \begin{mymatrix}{ccc}
        e^{t} & e^{-t}\cos t & e^{-t}\sin t \\
        e^{t} & -e^{-t}\cos t-e^{-t}\sin t & -e^{-t}\sin t+e^{-t}\cos t \\
        e^{t} & 2e^{-t}\sin t & -2e^{-t}\cos t%
      \end{mymatrix} = 5e^{-t} \neq 0
    \end{equation*}
    and so this matrix is always invertible.
  \end{sol}
\end{ex}

\begin{ex}
  Use the adjugate formula to find the inverse of the matrix
  \begin{equation*}
    A=\begin{mymatrix}{ccc}
      e^{t} & 0 & 0 \\
      0 & \cos t & \sin t \\
      0 & \cos t-\sin t & \cos t+\sin t
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    \begin{equation*}
      \det \begin{mymatrix}{ccc}
        e^{t} & 0 & 0 \\
        0 & \cos t & \sin t \\
        0 & \cos t-\sin t & \cos t+\sin t
      \end{mymatrix} = e^{t}.
    \end{equation*}
    Hence the inverse is
    \begin{equation*}
      e^{-t}\begin{mymatrix}{ccc}
        1 & 0 & 0 \\
        0 & e^{t}\cos t+e^{t}\sin t & -e^t(\cos t-\sin t) \\
        0 & -e^{t}\sin t & e^{t}\cos t
      \end{mymatrix}^T \\
      ~=~ \begin{mymatrix}{ccc}
        e^{-t} & 0 & 0 \\
        0 & \cos t+\sin t  & -\sin t \\
        0 & \sin t-\cos t  & \cos t
      \end{mymatrix}.
    \end{equation*}
    \end{sol}
\end{ex}

\begin{ex}
  Find the inverse, if it exists, of the matrix
  \begin{equation*}
    A =
    \begin{mymatrix}{ccc}
      e^{t} & \cos t & \sin t \\
      e^{t} & -\sin t & \cos t \\
      e^{t} & -\cos t & -\sin t
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    \begin{equation*}
      \def\arraystretch{1.3}
      \begin{mymatrix}{ccc}
        e^{t} & \cos t & \sin t \\
        e^{t} & -\sin t & \cos t \\
        e^{t} & -\cos t & -\sin t
      \end{mymatrix}^{-1} \\
      ~=~\begin{mymatrix}{ccc}
        \frac{1}{2}e^{-t} & 0 & \frac{1}{2}e^{-t} \\
        \frac{1}{2}\cos t+\frac{1}{2}\sin t & -\sin t & \frac{1}{2}\sin t-\frac{1}{2}
        \cos t \\
        \frac{1}{2}\sin t-\frac{1}{2}\cos t & \cos t & -\frac{1}{2}\cos t-\frac{1}{2}
        \sin t
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}


\section{Logarithms}
These rules apply to all positive numbers $a$ and $c$ and all $n \neq 0$. $b$ and $d$ denote the logarithm base and is above 1.
\begin{center}
    \begin{longtable}{lr}
        $\log_b(a \cdot c) = \log_b(a) + \log_b(c)$
        &
        $\log_b\left(\frac{a}{c}\right) = \log_b(a) - \log_b(c)$
        \\
        $\log_b\left(a^n\right) = n \cdot \log_b(a)$
        &
        $\log_b\left(\sqrt[n]{a}\right) = \frac{\log_b(a)}{n}$
        \\
        $b^{\log_b(a)} = a$
        &
        $\log_b{(b^a)} = a$
        \\
        $\log_b(b) = 1$
        &
        $\log_b(1) = 0$
        \\
        $\log_b(a) = \frac{\log_d(a)}{\log_d(b)}$
    \end{longtable}
\end{center}


\section{Random Walks}
\begin{definition}
    A random (or stochastic) process is a sequence of random variables.
\end{definition}
\begin{definition}
    A random walk is a stochastic process $(X_n)_{n\in\mathbb N}$ such that $X_0=x,X_n=x+Y_1+\cdots +Y_n$ where $x\in\mathbb R$ and $Y_i$ are i.i.d. random variables.
\end{definition}
Now we focus on the case where $Y_i$ is $1$ with probability $p$ and $-1$ with probability $1-p$ and $x$ is an integer.
We think of $X_n$ as the fortune of a gambler, and he gets it go up by $1$ with probability $p$ and lose with probability $q=1-p$.
His objective is to get to $a\in\mathbb Z$ before he get to $0$.
We say this is the event $A$.
Let $h_x=\mathbb P(A|X_0=x)$, then $h_x=ph_{x+1}+(1-p)h_{x-1},p_a=1,p_0=0$.
For $p=1/2=1-p$, we get $h_x=x/a$ which is perhaps not surprising.\\
For general $p$, by trying solutions of the form $\lambda^x$, we get $p\lambda^2-\lambda+q=0$, which has solutions $\lambda=1,q/p$, and plugging in the boundary conditions yield
$$h_x=\frac{(q/p)^x-1}{(q/p)^a-1}$$
Now we want to estimate the expected time to absorption, that is the smallest $T$ with $X_T\in\{0,a\}$.
We write $\tau_x$ for such $T$ with $X_0=x$.
we have $\tau_x=1+p\tau_{x+1}+q\tau_{x-1},0<x<a,\tau_0=\tau_a=0$.
For $p=1/2=1-p$, we get the solution $\tau_x=x(a-x)$.
For general $p$, we get the solution
$$\tau_x=\frac{1}{q-p}x-\frac{a}{q-p}\frac{(q/p)^x-1}{(q/p)^a-1}$$


\chapter{Code Snippets}
\renewcommand\thesection{\thechapter.\arabic{section}}

\section{Mass-spring system (Section \ref{sec:massSpringSystem})}\label{app:massSpringCode}

\setlstMAT
\begin{lstlisting}
%% Initialise variables
fs = 44100;             % sample rate [Hz]
k = 1 / fs;             % time step [s]
lengthSound = fs;       % length of the simulation (1 second) [samples]

f0 = 440;               % fundamental frequency [Hz]
omega0 = 2 * pi * f0;   % angular (fundamental) frequency [Hz]
M = 1;                  % mass [kg]
K = omega0^2 * M;       % spring constant [N/m]

%% initial conditions (u0 = 1, d/dt u0 = 0)
u = 1;
uPrev = 1;

% initialise output vector
out = zeros(lengthSound, 1);

%% Simulation loop
for n = 1:lengthSound

    % Update equation Eq. %*\eqrefMatlab[eq:massSpringUpdate] *)
    uNext = (2 - K * k^2 / M) * u - uPrev;

    out(n) = u;

    % Update system states
    uPrev = u;
    u = uNext;
end
\end{lstlisting}

\section{1D wave equation (Section \ref{sec:1DWave})}
\label{app:1DWave}
\begin{lstlisting}
%% Initialise variables
fs = 44100;         % Sample rate [Hz]
k = 1 / fs;         % Time step [s]
lengthSound = fs;   % Length of the simulation (1 second) [samples]

c = 300;            % Wave speed [m/s]
L = 1;              % Length [m]
h = c * k;          % Grid spacing [m] (from CFL condition)
N = floor(L/h);     % Number of intervals between grid points
h = L / N;          % Recalculation of grid spacing based on integer N

lambdaSq = c^2 * k^2 / h^2; % Courant number squared

% Boundary conditions ([D]irichlet or [N]eumann)
bcLeft = "D";
bcRight = "D";

%% Initialise state vectors (one more grid point than the number of intervals)
uNext = zeros(N+1, 1);
u = zeros(N+1, 1);

%% Initial conditions (raised cosine)
loc = round(0.8 * N);       % Center location
halfWidth = round(N/10);    % Half-width of raised cosine
width = 2 * halfWidth;      % Full width
rcX = 0:width;              % x-locations for raised cosine

rc = 0.5 - 0.5 * cos(2 * pi * rcX / width); % raised cosine
u(loc-halfWidth : loc+halfWidth) = rc; % initialise current state

% Set initial velocity to zero
uPrev = u;

% Range of calculation
range = 2:N;

% Output location
outLoc = round(0.3 * N);

%% Simulation loop
for n = 1:lengthSound

    % Update equation Eq. %*\eqrefMatlab[eq:1DwaveUpdate]*)
    uNext(range) = (2 - 2 * lambdaSq) * u(range) ...
        + lambdaSq * (u(range+1) + u(range-1)) - uPrev(range);

    % boundary updates Eq. %*\eqrefMatlab[eq:1DWaveLeftBound]*)
    if bcLeft == "N"
        uNext(1) = (2 - 2 * lambdaSq) * u(1) - uPrev(1) ...
        + 2 * lambdaSq * u(2);
    end

    % Eq. %*\eqrefMatlab[eq:1DWaveRightBound]*)
    if bcRight == "N"
        uNext(N+1) = (2 - 2 * lambdaSq) * u(N+1) - uPrev(N+1) ...
        + 2 * lambdaSq * u(N);
    end

    out(n) = u(outLoc);

    % Update system states
    uPrev = u;
    u = uNext;
end
\end{lstlisting}

\section{2D wave equation (Section \ref{sec:2Dwave})}
\label{app:2DWave}
\begin{lstlisting}

%% Initialise variables
fs = 44100;         % Sample rate [Hz]
k = 1 / fs;         % Time step [s]
lengthSound = fs;   % Length of the simulation (1 second) [samples]

rho = 7850;                     % Material density [kg/m^3]
H =  0.0005;                    % Thickness [m]
T = 1000000;                    % Tension per unit length [N/m]
c = sqrt(T / (rho * H));        % Wave speed [m/s]

Lx = 1;                         % Length in x direction [m]
Ly = 2;                         % Length in y direction [m]

h = sqrt(2) * c * k;            % Grid spacing [m]
Nx = floor(Lx/h);               % Number of intervals in x direction
Ny = floor(Ly/h);               % Number of intervals in y direction
h = min(Lx/Nx, Ly/Ny);          % Recalculation of grid spacing

lambdaSq = c^2 * k^2 / h^2;     % Courant number squared
h = max(Lx/Nx, Ly/Ny);          % Recalculation of grid spacing

%% Create scheme matrices with Dirichlet boundary conditions
Nxu = Nx - 1;
Nyu = Ny - 1;
Dxx = toeplitz([-2, 1, zeros(1, Nxu-2)]);
Dyy = toeplitz([-2, 1, zeros(1, Nyu-2)]);

% Kronecker sum
D = kron(speye(Nxu), Dyy) + kron(Dxx, speye(Nyu));
D = D / h^2;

% Total number of grid points
Nu = Nxu * Nyu;

%% Initialise state vectors (one more grid point than the number of intervals)
uNext = zeros(Nu, 1);
u = zeros(Nu, 1);

%% Initial conditions (2D raised cosine)
halfWidth = floor(min(Nx, Ny) / 5);
width = 2 * halfWidth + 1;
xLoc = floor(0.3 * Nx);
yLoc = floor(0.6 * Ny);
xRange = xLoc-halfWidth : xLoc+halfWidth;
yRange = yLoc-halfWidth : yLoc+halfWidth;

rcMat = zeros(Nyu, Nxu);
rcMat(yRange, xRange) = hann(width) * hann(width)';

% initialise current state
u = reshape(rcMat, Nu, 1);

% Set initial velocity to zero
uPrev = u;

% Output location
xOut = 0.45;
yOut = 0.25;
outLoc = round((xOut + yOut * Nyu) * Nxu);
out = zeros(lengthSound, 1);

%% Simulation loop
for n = 1:lengthSound

    %% Update equation Eq. %*\eqrefMatlab[eq:matrixUpdate2Dwave] *)
    uNext = (2 * eye(Nu) + c^2 * k^2 * D) * u - uPrev;

    % Update system states
    uPrev = u;
    u = uNext;

end
\end{lstlisting}


\chapter{Number theory}

\section{Modular arithmetic}
	\kactlimport{ModularArithmetic.h}
	\kactlimport{ModInverse.h}
	\kactlimport{ModPow.h}
	\kactlimport{ModSum.h}
	\kactlimport{ModMulLL.h}
	\kactlimport{ModSqrt.h}

\section{Primality}
	\kactlimport{eratosthenes.h}
	\kactlimport{MillerRabin.h}
	\kactlimport{factor.h}

\section{Divisibility}
	\kactlimport{euclid.h}
	\kactlimport{Euclid.java}

	%\subsection{Bézout's identity}
	%For $a \neq $, $b \neq 0$, then $d=gcd(a,b)$ is the smallest positive integer for which there are integer solutions to
	%$$ax+by=d$$
	%If $(x,y)$ is one solution, then all solutions are given by
	%$$\left(x+\frac{kb}{\gcd(a,b)}, y-\frac{ka}{\gcd(a,b)}\right), \quad k\in\mathbb{Z}$$

	%\kactlimport{phiFunction.h}

\section{Fractions}
	\kactlimport{ContinuedFractions.h}
	\kactlimport{FracBinarySearch.h}

\section{Chinese remainder theorem}
	\kactlimport{chinese.h}

\section{Pythagorean Triples}
 The Pythagorean triples are uniquely generated by
 \[ a=k\cdot (m^{2}-n^{2}),\ \,b=k\cdot (2mn),\ \,c=k\cdot (m^{2}+n^{2}), \]
 with $m > n > 0$, $k > 0$, $m \bot n$, and either $m$ or $n$ even.

\section{Primes}
	$p=962592769$ is such that $2^{21} \mid p-1$, which may be useful. For hashing
	use 970592641 (31-bit number), 31443539979727 (45-bit), 3006703054056749
	(52-bit). There are 78498 primes less than 1\,000\,000.

	Primitive roots exist modulo any prime power $p^a$, except for $p = 2, a > 2$, and there are $\phi(\phi(p^a))$ many.
	For $p = 2, a > 2$, the group $\mathbb Z_{2^a}^\times$ is instead isomorphic to $\mathbb Z_2 \times \mathbb Z_{2^{a-2}}$.

\section{Estimates}
	$\sum_{d|n} d = O(n \log \log n)$.

	The number of divisors of $n$ is at most around 100 for $n < 5e4$, 500 for $n < 1e7$, 2000 for $n < 1e10$, 200\,000 for $n < 1e19$.


%             %
%%           %%
%%% CHAPTER %%%
%%           %%
%             %

\chapter{Mathematics -- Matrices}

	\label{cha::maths:matrix}

Engineers and scientists use many matrices to describe physical phenomena.
Hence the frequently asked question: \enquote{how can I write a matrix with \LaTeX{}}?
This chapter answer the question.

\section{Packages for matrices}

	As for the mathematics basics (cf. \cref{cha::maths:basics}), the \emph{mathtools} package, which is an upgrade of the well-known and widely used \emph{amsmath} package, provides everything that is necessary to write matrix computations.

	In addition, the \emph{physics} package provides very useful commands to facilitate the creation of specific matrices such as an identity matrix or a zero matrix.

	So the following lines should be written in the preamble of the document:
\begin{lstlisting}[language={[LaTeX]TeX}]
\usepackage{mathtools}
\usepackage{physics}
\end{lstlisting}

\section{Writing matrices}

	Matrices can be generated thanks to the \texttt{matrix} environment which must be used inside a mathematical equation environment.
	The simplest \LaTeX{} code which generates a matrix is
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{equation*}
	\begin{matrix}
		x_{11} & x_{12} \\
		x_{21} & x_{22}
	\end{matrix}
\end{equation*}
\end{lstlisting}
	and results in
	\begin{equation*}
		\begin{matrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{matrix}
	\end{equation*}

	There are several variants of \texttt{matrix} which produce different delimiters surrounding the matrix.
	They are presented here below, with the corresponding environment's name on top of each matrix:
	\begin{align*}
		\text{\texttt{matrix}} && \text{\texttt{pmatrix}} && \text{\texttt{bmatrix}} && \text{\texttt{Bmatrix}} \\
		%
		\begin{matrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{matrix}
		&&
		\begin{pmatrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{pmatrix}
		&&
		\begin{bmatrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{bmatrix}
		&&
		\begin{Bmatrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{Bmatrix} \\
		%
		%
		&& \text{\texttt{vmatrix}} && \text{\texttt{Vmatrix}} && \\
		%
		&&
		\begin{vmatrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{vmatrix}
		&&
		\begin{Vmatrix}
			x_{11} & x_{12} \\
			x_{21} & x_{22}
		\end{Vmatrix}
		&&
	\end{align*}

	The \emph{mathtools} package offers starred versions of the \texttt{matrix} environments which allow to pass an optional argument to specify the alignment within the matrix's columns.
	A \LaTeX{} example is shown here below:
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{align*}
	\begin{pmatrix}
		2  & -3 \\
		42 & 0
	\end{pmatrix}	&& \text{VS} &&
	\begin{pmatrix*}[r]
		2  & -3 \\
		42 & 0
	\end{pmatrix*}
\end{align*}.
\end{lstlisting}
	Observe the difference in alignment between \texttt{pmatrix} (left) and \texttt{pmatrix*} (right):
	\begin{align*}
		\begin{pmatrix}
		2  & -3 \\
		42 & 0
		\end{pmatrix}	&& \text{VS} &&
		\begin{pmatrix*}[r]
		2  & -3 \\
		42 & 0
		\end{pmatrix*}.
	\end{align*}

\section{Facilities for specific matrices}

	The \emph{physics} package proposes many commands that make the \LaTeX{}'s life of scientists and engineers easier.
	Among them, there are commands aiming the easy generation of specific matrices.

	\subsection{Matrices with specific patterns}

		In mathematics, there are specific matrices that are commonly used.
		These matrices are specific because they matches a pattern that is immediately recognised.
		The most known are the zero matrix, the identity matrix and the diagonal matrix.

		The \emph{physics} package provides commands to automatically generate the desired pattern for an $n \times m$ matrix, $n$ and $m$ being adapted according to the arguments passed to the commands.
		All commands must be placed inside a \texttt{matrix}-like environment, itself place within an \texttt{equation}-like environment.

		\subsubsection{Zero matrix}

			The zero matrix is generated with the \texttt{zeromatrix} command, or the shorter form \texttt{zmat}.
			The first argument is the dimension $n$, second argument is the dimension $m$ of the $n \times m$ matrix.
			If $m$ is omitted, the command generates a square zero matrix.

			A comparison of the equivalent \LaTeX{} codes
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{align*}
	% 3x3 matrix
	\begin{pmatrix} % based on physics
		\zmat{3}
	\end{pmatrix}	&&
	\begin{pmatrix} % regular way
		0 & 0 & 0 \\
		0 & 0 & 0 \\
		0 & 0 & 0
	\end{pmatrix}	&& \text{and} &&
	% 3x5 matrix
	\begin{pmatrix} % based on physics
		\zmat{3}{5}
	\end{pmatrix}	&&
	\begin{pmatrix} % regular way
		0 & 0 & 0 & 0 & 0 \\
		0 & 0 & 0 & 0 & 0 \\
		0 & 0 & 0 & 0 & 0
	\end{pmatrix}
\end{align*}
\end{lstlisting}
			generating
			\begin{align*}
				% 3x3 matrix
				\begin{pmatrix} % based on physics
					\zmat{3}
				\end{pmatrix}	&&
				\begin{pmatrix} % regular way
					0 & 0 & 0 \\
					0 & 0 & 0 \\
					0 & 0 & 0
				\end{pmatrix}	&& \text{and} &&
				% 3x5 matrix
				\begin{pmatrix} % based on physics
					\zmat{3}{5}
				\end{pmatrix}	&&
				\begin{pmatrix} % regular way
					0 & 0 & 0 & 0 & 0 \\
					0 & 0 & 0 & 0 & 0 \\
					0 & 0 & 0 & 0 & 0
				\end{pmatrix}.
			\end{align*}
			shows that using the \emph{physics} package avoids a fastidious copy-paste work.

		\subsubsection{Identity matrix}

			The identity matrix is generated with the \texttt{identitymatrix} command, or the shorter form \texttt{imat}.
			The argument is the dimension $n$ of the square matrix.

			Comparing the \enquote{regular} \LaTeX{} with the code based on \emph{physics}
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{equation*}
	\begin{pmatrix} % based on physics
		\imat{3}
	\end{pmatrix}	\quad
	\begin{pmatrix} % regular way
		1 & 0 & 0 \\
		0 & 1 & 0 \\
		0 & 0 & 1
	\end{pmatrix}
\end{equation*}
\end{lstlisting}
			while both result in the same display
			\begin{equation*}
				\begin{pmatrix} % based on physics
					\imat{3}
				\end{pmatrix}	\quad
				\begin{pmatrix} % regular way
					1 & 0 & 0 \\
					0 & 1 & 0 \\
					0 & 0 & 1
				\end{pmatrix}
			\end{equation*}
			highligthts the time saved in this case, especially because the copy-paste is not as efficient as with the zero matrix.

		\subsubsection{Diagonal matrix}

			A diagonal matrix is generated with the \texttt{diagonalmatrix} command, or the shorter form \texttt{dmat}.
			The argument is a list of comma-separated values, the first one being $x_{11}$, the second one being $x_{22}$, and so forth.
			By default, only the values on the diagonal are printed and the rest is filled with white spaces.
			An optional argument allows to fill the spaces.

			For instance, the following code
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{align*}
	% based on physics
	\begin{pmatrix}
		\dmat{a,b,c}
	\end{pmatrix}	&&
	\begin{pmatrix}
		\dmat{1,2,3}
	\end{pmatrix}	&&
	\begin{pmatrix}
		\dmat[0]{1,2,3}
	\end{pmatrix}	\\
	% regular way
	\begin{pmatrix}
		a &   &   \\
		  & b &   \\
		  &   & c
	\end{pmatrix}	&&
	\begin{pmatrix}
		1 &   &   \\
		  & 2 &   \\
		  &   & 3
	\end{pmatrix}	&&
	\begin{pmatrix}
		1 & 0 & 0 \\
		0 & 2 & 0 \\
		0 & 0 & 3
	\end{pmatrix}
\end{align*}
\end{lstlisting}
			generates
			\begin{align*}
				% based on physics
				\begin{pmatrix}
					\dmat{a,b,c}
				\end{pmatrix}	&&
				\begin{pmatrix}
					\dmat{1,2,3}
				\end{pmatrix}	&&
				\begin{pmatrix}
					\dmat[0]{1,2,3}
				\end{pmatrix},	\\
				% regular way
				\begin{pmatrix}
					a &   &   \\
					  & b &   \\
					  &   & c
				\end{pmatrix}	&&
				\begin{pmatrix}
					1 &   &   \\
					  & 2 &   \\
					  &   & 3
				\end{pmatrix}	&&
				\begin{pmatrix}
					1 & 0 & 0 \\
					0 & 2 & 0 \\
					0 & 0 & 3
				\end{pmatrix}.
			\end{align*}
			The above example also shows that literal values can be used as well as numerical values.

			Similarly to the diagonal matrix, an anti-diagonal matrix can be created with the \texttt{antidiagonalmatrix} command, or the shorter form \texttt{admat}.

		\subsubsection{Generalised pattern}

			An $n \times m$ matrix can be filled with a specific element thanks to the \texttt{xmatrix} command, or the shorter form \texttt{xmat}.
			A starred version adds automatic indices.

			As an example, the
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{align*}
	% based on physics
	\begin{pmatrix}
		\xmat{1}{2}{3}
	\end{pmatrix}	&&
	\begin{pmatrix}
		\xmat*{x}{3}{3}
	\end{pmatrix}	&&
	\begin{pmatrix}
		\xmat*{a}{3}{1}
	\end{pmatrix}	&&
	\begin{pmatrix}
		\xmat*{\alpha}{1}{3}
	\end{pmatrix}	\\
	% regular way
	\begin{pmatrix}
		1 & 1 & 1 \\
		1 & 1 & 1
	\end{pmatrix}	&&
	\begin{pmatrix}
		x_{11} & x_{12} & x_{13} \\
		x_{21} & x_{22} & x_{23} \\
		x_{31} & x_{32} & x_{33}
	\end{pmatrix}	&&
	\begin{pmatrix}
		a_{1} \\
		a_{2} \\
		a_{3}
	\end{pmatrix}	&&
	\begin{pmatrix}
		\alpha_{1} & \alpha_{2} & \alpha_{3}
	\end{pmatrix}
\end{align*}
\end{lstlisting}
			\LaTeX{} codes produces
			\begin{align*}
				% based on physics
				\begin{pmatrix}
					\xmat{1}{2}{3}
				\end{pmatrix}	&&
				\begin{pmatrix}
					\xmat*{x}{3}{3}
				\end{pmatrix}	&&
				\begin{pmatrix}
					\xmat*{a}{3}{1}
				\end{pmatrix}	&&
				\begin{pmatrix}
					\xmat*{\alpha}{1}{3}
				\end{pmatrix},	\\
				% regular way
				\begin{pmatrix}
					1 & 1 & 1 \\
					1 & 1 & 1
				\end{pmatrix}	&&
				\begin{pmatrix}
					x_{11} & x_{12} & x_{13} \\
					x_{21} & x_{22} & x_{23} \\
					x_{31} & x_{32} & x_{33}
				\end{pmatrix}	&&
				\begin{pmatrix}
					a_{1} \\
					a_{2} \\
					a_{3}
				\end{pmatrix}	&&
				\begin{pmatrix}
					\alpha_{1} & \alpha_{2} & \alpha_{3}
				\end{pmatrix}.
			\end{align*}
			As observed, the automatic indexing takes the number of lines and columns into account.
			Automatic indexing is especially useful for general formulae and demonstrations.
			Finally, like for the diagonal matrix, literal (including Greek letters) values can be used.

			At this point, the author believes that the reader is convinced by the power of the \emph{physics} package to create matrices which match a specific pattern.

	\subsection{Combinations of patterns}

		Even though the aforementioned commands are usually sufficient for most cases, it is sometimes required to combine patterns together to obtain the desired result.
		The \emph{physics} allow to do so \footnote{This section can be considered as intended for \enquote{advanced} users. However, the author thinks that separate it from the rest of the explanations about matrices is non-sense.}.

		A first possibility consists in creating the pattern and then adding the other elements around.
		The
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{equation*}
	\begin{pmatrix}
		\imat{2} \\
		a & b
	\end{pmatrix}	\quad
	\begin{pmatrix}
		a & b & c \\
		\zmat{2}{3}
	\end{pmatrix}
\end{equation*}
\end{lstlisting}
		example code outputs
		\begin{equation*}
			\begin{pmatrix}
				\imat{2} \\
				a & b
			\end{pmatrix}	\quad
			\begin{pmatrix}
				a & b & c \\
				\zmat{2}{3}
			\end{pmatrix}.
		\end{equation*}

		However, if an element is added on the right or on the left of a pattern, the result is not the one expected, as depicted by the following code:
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{equation*}
	\begin{pmatrix}
		\imat{2} & a \\
		         & b
	\end{pmatrix}	\quad
	\begin{pmatrix}
		         & a \\
		\imat{2} & b
	\end{pmatrix}	\quad
	\begin{pmatrix}
		a & \\
		b & \zmat{2}{3}
	\end{pmatrix}.
\end{equation*}
\end{lstlisting}
		\begin{equation*}
			\begin{pmatrix}
				\imat{2} & a \\
				         & b
			\end{pmatrix}	\quad
			\begin{pmatrix}
						 & a \\
				\imat{2} & b
			\end{pmatrix}	\quad
			\begin{pmatrix}
				a & \\
				b & \zmat{2}{3}
			\end{pmatrix}.
		\end{equation*}

		To overcome this issue, the \texttt{matrixquantity} command, or the shorter form \texttt{mqty}, can be used.
		It creates a matrix as a single element.

		Here follows an example that is explained below:
\begin{lstlisting}[language={[LaTeX]TeX}]
\begin{equation*}
	\begin{pmatrix}
		\mqty{\imat{2}} & \mqty{a\\b} \\
		\mqty{c & d}    & e
	\end{pmatrix}
\end{equation*}
\end{lstlisting}
		\begin{equation*}
			\begin{pmatrix}
				\mqty{\imat{2}} & \mqty{a\\b} \\
				\mqty{c & d}    & e
			\end{pmatrix}.
		\end{equation*}
		Firstly, the identity matrix is created and embedded in \texttt{mqty} to be considered as a single element.
		Secondly, we can add a $2 \times 1$ vector on the right but he must also be encapsulated in \texttt{mqty} as the identity matrix is a single element.
		At this step, \LaTeX{} sees a $1 \times 2$ matrix, even though the displayed result is a $2 \times 3$ matrix.
		Hence $c$ and $d$ that are embedded in \texttt{mqty} whereas $e$ is a \enquote{regular} single element.

% *** End of chapter ***


\section{Introduction to Kernels}
In this section, we present the basic definitions and concepts concerning kernels. We begin with the most basic definition of a pre-kernel.
\begin{definition}
Let $V$ be a Hilbert space, and $\Omega\subset V$. A \textbf{pre-kernel} is a continuous function $k:\Omega\times \Omega \rightarrow \mathbb{R}$ which is symmetric, i.e. for which
\begin{equation}
 k(x,y) = k(y,x).
\end{equation}

\end{definition}
Below are some common examples:
\begin{itemize}
 \item Radial-basis functions $k(x,y) = g(\|x-y\|_V)$, where $g:[0,\infty)\rightarrow \mathbb{R}$ is a given function. Some simple examples are:
 \begin{itemize}
 \item Identity $g(t) = t$
 \item Exponential $g(t) = e^{-ct}$
 \item Gaussian $g(t) = e^{-ct^2}$
 \end{itemize}
 \item Shift-invariant kernels $k(x,y) = \phi(x-y)$, where $\phi(x) = \phi(-x)$ is an even function.
 \item Functions of the inner product, i.e. $k(x,y) = g(\langle x, y\rangle_V)$.
\end{itemize}

\subsection{Interpolation}
Given a collection of datapoints $\xi_1,...,\xi_n\in \Omega$, we form the prekernel space
\begin{equation}
 I_k(\{\xi_1,...,\xi_n\}) = \text{span}\{k(\cdot, \xi_i),~i=1,...,n\}.
\end{equation}
This is a linear space of real-valued functions on $\Omega$, spanned by the kernels `centered' at each of the datapoints $\xi_1,...,\xi_n$.

Given function values $f_i = f(\xi_i)$, a major question is whether we can interpolate the values $f_i$ in $I_k(\{\xi_1,...,\xi_n\})$, i.e. whether we can find coefficients $a_k$ such that
\begin{equation}
 \sum_{j=1}^n a_jk(x_i,x_j) = f_i
\end{equation}
for $i=1,...,n$. This of course means that the function
$$f_n(x) = \sum_{j=1}^n a_jk(x,x_j)\in I_k(\{\xi_1,...,\xi_n\})$$
matches $f$ at the datapoints $\xi_i$.

Evidently, this will always be possible as long as the matrix $(k(\xi_i,\xi_j))^n_{i,j = 1}$ is invertible. This leads us to the following two definitions.
\begin{definition}
 A pre-kernel $k$ is called an \textbf{invertible kernel} if the matrix $(k(\xi_i,\xi_j))^n_{i,j = 1}$ is non-singular for any $n$ and $\xi_1,...,\xi_n\in \Omega$.
\end{definition}
\begin{definition}
 A pre-kernel $k$ is called a \textbf{positive-definite kernel} or just a \textbf{kernel} if the matrix $(k(\xi_i,\xi_j))^n_{i,j = 1}$ is positive definite for any $n$ and $\xi_1,...,\xi_n\in \Omega$.
\end{definition}

In either of these cases, the interpolation problem can always we uniquely solved.

\subsection{Shift-invariant Kernels on $\mathbb{R}^d$}
We now turn to the question of how to determine whether a pre-kernel is invertible or positive-definite. We first consider the case of shift invariant kernels $k(x,y) = \phi(x-y)$ on $\Omega = \mathbb{R}^n$. In this case we determine exactly when a kernel is positive definite in terms of the Fourier transform of $\phi$. We begin with a lemma (where we aren't being completely rigorous).

\begin{lemma}
 A kernel is positive definite iff the bilinear form
 \begin{equation}
  K(f,g) = \int_\Omega\int_\Omega f(x)k(x,y)g(y)dxdy
 \end{equation}
 is positive definite on $L^2(\Omega)$. Another way of saying this is that the operator $K:L^2(\Omega)\rightarrow L^2(\Omega)$ defined by
 \begin{equation}
  K(g)(x) = \int_\Omega k(x,y)g(y)dx
 \end{equation}
 is positive definite.

\end{lemma}
Here we have brushed over the technical issue that this map may not be well-defined on $L^2(\Omega)$ if $\Omega$ is not compact. Technically, this only holds if the map $K$ above is well-defined on $L^2(\Omega)$.

Since the kernel $k(x,y) = \phi(x-y)$ on $\Omega = \mathbb{R}^d$ is shift invariant, the above map $K$ is a convolution $K(g) = \phi*g$. Thus it is diagonalized by the Fourier transform, in particular $\widehat{K(g)} = \hat{\phi}\hat{g}$. This leads us to the following theorem.
\begin{theorem}
 Suppose that $\phi\in L^1(\mathbb{R}^d)$. Then the shift invariant kernel $k(x,y) = \phi(x-y)$ is positive definite iff $\hat{\phi} > 0$.
\end{theorem}
\begin{proof}
 If $\phi\in L^1$, then the map $K:L^2(\mathbb{R}^d)\rightarrow L^2(\mathbb{R}^d)$,
 \begin{equation}
  K(g) = \int_{\mathbb{R}^d} \phi(x-y)g(y)dx = \phi*g
 \end{equation}
 is well-defined by Young's inequality. Moreover the Fourier transform $\hat{\phi}\in L^\infty(\mathbb{R}^d)$. This means that the equation
 \begin{equation}
  \widehat{K(g)} = \hat{\phi}\hat{g}
 \end{equation}
 holds rigorously. Thus we see that $K$ is positive definite (and thus we have a positive definite kernel) iff $\hat{\phi} > 0$.
\end{proof}
Next we give an important corollary, which forms the building block of the radial basis function theory.
\begin{corollary}\label{gaussian-corollary}
 The Gaussian kernel $k_c(x,y) = e^{-c\|x-y\|^2}$ is positive definite for any $c > 0$.
\end{corollary}
\begin{proof}
 This follows since the Gaussian is integrable and it's Fourier transform is another Gaussian (and thus is positive).
\end{proof}

\subsection{Radial Basis Functions}
We now consider the case of kernels defined via radial basis functions, i.e. $k(x,y) = g(\|x-y\|_V)$, for which useful sufficient conditions can be derived. We first consider when we can say that a radial basis kernel is positive definite.
We begin with a definition and classical result.

\begin{definition}
 A smooth function $f:\mathbb{R}_{\geq 0}\rightarrow \mathbb{R}$ is called \textbf{completely monotonic} if $(-1)^nf^{(n)}(x) \geq 0$ for all $x > 0$.
\end{definition}

We have the following characterization of completely monotonic functions.
\begin{theorem}[Bernstein-Widder]
 $f$ is completely monotonic iff there exists a positive measure $\mu$ on $[0,\infty)$ such that
 \begin{equation}
  f(x) = \int_0^\infty e^{-cx} d\mu(c).
 \end{equation}
 In other words, $f$ is completely monotonic iff it lies in the cone generated by the exponentials $e^{-cx}$ for $c \geq 0$.
\end{theorem}

This gives us the following fundamental result.

\begin{theorem}
 If $f$ is completely monotonic and not constant, then the radial basis function $k(x,y) = g(\|x-y\|_V) = f(\|x-y\|_V^2)$ for $g(x) = f(x^2)$ is positive definite.
\end{theorem}
\begin{proof}
 Note first that since the number of points $\xi_1,...,\xi_n$ is always finite, these datapoints lie in a finite dimensional subspace and we can assume without loss of generality that $\Omega = \mathbb{R}^d$ for some finite $d$.

 Now we use the Bernstein-Widder theorem to write our kernel $k(x,y)$ as a positive combination of Gaussian kernels, as follows
 \begin{equation}
  k(x,y) = f(\|x-y\|^2) = \int_0^\infty e^{-c\|x-y\|^2} d\mu(c).
 \end{equation}
 Now since $f$ is non-constant, $\mu$ is not entirely supported at $0$. Thus $k$ is a positive combination of Gaussians, with at least some weight on $c > 0$. Since the Gaussians are all positive semi-definite (and positive definite for $c > 0$, by corollary \ref{gaussian-corollary}), we see that $k$ must be positive-definite as well.

\end{proof}

Let us now give some examples of completely monotonic functions and the associated kernels to which this theorem applies.
\begin{itemize}
 \item Gaussian: $f(x) = e^{-cx}$, $k(x,y) = e^{-c\|x-y\|^2}$.
 \item Multi-quadric: $f(x) = (x^2 + r^2)^{-\frac{1}{2}}$, $k(x,y) = (x^2 + r^2)^{-1}$
\end{itemize}

\section{Positive-Definite Kernels and Feature Maps}
In this section we consider positive definite kernels, their associated reproducting kernel Hilbert spaces, and their feature maps. We begin with the definition of a feature map.
\begin{definition}
 A feature map for a kernel $k$ is a Hilbert space $V$ together with a map $\Phi:\Omega\rightarrow V$ such that
 \begin{equation}
  k(x,y) = \langle\Phi(x), \Phi(y)\rangle_V.
 \end{equation}

\end{definition}
It is clear that if a kernel has a feature map, then it must be positive semi-definite.
\begin{lemma}
 If a kernel $k$ has a feature map, then it must be positive semi-definite.
\end{lemma}
\begin{proof}
 Note that the matrix $(k(x_i,x_j))_{i,j=1}^n$ is the Gram matrix of the vectors $\Phi(x_1),...,\Phi(x_n)$ which is positive semi-definite.
\end{proof}

Next we address the converse question, i.e. whether a positive semi-definite kernel always has a feature map. For simplicity we consider the case of positive definite kernel $k$.

\begin{theorem}
 Suppose that $k$ is a positive definite kernel. Then there exists a feature map $\Phi$ for the kernel $k$.
\end{theorem}
\begin{proof}
 The proof proceeds by constructing the \textit{reproducing kernel Hilbert space} $H_k$ corresponding to $k$ and explicitly exhibiting a feature map $\Phi:\Omega\rightarrow H_k$.

 We define $H_k$ as follows. Let $S_k = \text{span}\{k(\cdot,x):~x\in\Omega\} \subset C(\Omega)$ be the vector space spanned by the kernels centered at each of the points $x\in\Omega$. Define a norm on this space by extending the inner product
 \begin{equation}
  \langle k(\cdot,x), k(\cdot,y)\rangle_{H_k} = k(x,y)
 \end{equation}
 to all finite linear combinations of the kernels. Since $k$ is positive definite, this corresponds to a positive definite norm on $S_k$. Taking the completion of $S_k$ with respect to this norm we obtain the reproducing kernel Hilbert space $H_k$.

 It is now easy to check (essentially follows by definition) that the map $\Phi:\Omega\rightarrow H_k$ given by
 \begin{equation}
  \Phi(x) = k(\cdot,x)\in H_k
 \end{equation}
 is a feature map.

\end{proof}
We collect the following definition from the previous proof.
\begin{definition}
 Let $k$ be a positive definite kernel. The space $H_k$ in the above proof is called the \textbf{reproducing kernel Hilbert space} associated to $k$.
\end{definition}

We conclude with a lemma showing that $H_k\subset C(\Omega)$ under mild conditions on $k$.

\begin{lemma}
 Suppose that $k(x,x) < C$ is bounded for all $x\in \Omega$. Then $H_k\subset C(\Omega)$ and we have the bound
 \begin{equation}
  \|f\|_{L^\infty} \leq \sqrt{C}\|f\|_{H_k}.
 \end{equation}
 for all $f\in H_k$.

\end{lemma}
\begin{proof}
 Since $H_k$ is the completion of a subspace of $C(\Omega)$ with respect to the $H_k$-norm, $\|f\|_{L^\infty} \leq C\|f\|_{H_k}$ implies that $H_k\subset C(\Omega)$. To this end, we see that since
 \begin{equation}
  \langle k(\cdot, x), k(\cdot, y)\rangle_{H_k} = k(y,x)
 \end{equation}
 holds for every $y\in \Omega$ and $k(\cdot, x)$ is a Riesz basis for $H_k$, we have
 \begin{equation}
  \langle f, k(\cdot, y)\rangle_{H_k} = f(y)
 \end{equation}
 holds for every $f\in H_k$. Thus,
 \begin{equation}
  |f(y)| = |\langle f, k(\cdot, y)\rangle_{H_k}| \leq \|k(\cdot, y)\|_{H_k}\|f\|_{H_k}.
 \end{equation}
 Finally,
 \begin{equation}
  \langle k(\cdot, y),k(\cdot, y)\rangle_{H_k} = k(y,y),
 \end{equation}
 so that $\|k(\cdot, y)\|_{H_k} \leq \sqrt{k(y,y)} \leq \sqrt{C}$ as desired.

\end{proof}

\section{Reproducing Kernel Hilbert Spaces}
From the above discussion, a natural question to ask is which Hilbert spaces can arise as reproducing kernel Hilbert spaces for a given kernel $k$.

A complete answer is given by the following theorem.

\begin{theorem}
 A Hilbert space $H$ of functions on $\Omega$ is a reproducing Hilbert space for some kernel $k$ iff the evaluation maps
 \begin{equation}
  \delta_x(f) = f(x),
 \end{equation}
 are continuous with respect to $\|\cdot\|_H$ for all $x\in \Omega$.

\end{theorem}
\begin{proof}
 The proof follows from the Riesz representation theorem. Since the evaluation maps $\delta_x$ are continuous linear functionals, there exist elements $k(\cdot, x)\in H$ such that $\delta_x(f) = f(x) = \langle f, k(\cdot x)\rangle_H$. These functions $k$ are exactly the kernel of the reproducing Hilbert space.
\end{proof}

Note for instance that this theorem tells us that Sobolev spaces of sufficient regularity are reproducing kernel Hilbert spaces.

\section{Mercer's Theorem}
In this section we give Mercer's construction. When $\Omega$ is compact, this gives a useful feature map which helps elucidate the approximation properties of the kernel.

Suppose that $\Omega\subset \mathbb{R}^d$ is compact and $k$ is positive definite. Then the map $T_k:L^2(\Omega)\rightarrow L^2(\Omega)$ given by
\begin{equation}
 T_k(f)(x) = \int_\Omega k(x,y)f(y)dy
\end{equation}
is a positive definite, symmetric, compact operator on $L^2(\Omega)$. Thus it can be diagonalized and has a discrete set of eigenvalues $\lambda_i\rightarrow 0$ and corresponding orthonormal eigenfunctions $\phi_i(x)$. Additionally the kernel can be represented as
\begin{equation}
 k(x,y) = \sum_{i=1}^\infty \lambda_i\phi_i(x)\phi_i(y).
\end{equation}

This provides the corresponding feature map
\begin{equation}
 \Phi(x) = \begin{pmatrix}
            \sqrt{\lambda_1}\phi_1(x) \\
            \sqrt{\lambda_2}\phi_2(x) \\
            \vdots
           \end{pmatrix}\in \ell^2(\mathbb{N}).
\end{equation}

\section{Universality}
Now we come the question of univerality, i.e. when is the reproducing kernel Hilbert space corresponding to a kernel $k$ dense in the space of continuous functions.

The important lemma is the following.
\begin{lemma}
 Let $k$ be a kernel with feature map $\Phi:\Omega\rightarrow V$. Then $k$ is a universal kernel iff
 \begin{equation}
  \overline{\text{span}}(\{\langle \Phi(x), v\rangle_V:~v\in V\}) = C(\Omega).
 \end{equation}
 I.e. if the feature map is dense in $C(\Omega)$.
\end{lemma}

\subsection{Shift-invariant Kernels}

\subsection{Neural Network Kernels}
In this section we consider a deep neural network randomly with randomly initialized weights (appropriately scaled). It turns out the the random function represented by such a network is drawn from a Gaussian process with covariance given by a certain kernel $k$.

Consider a neural network $f(x,\Theta)$ with input $x\in \mathbb{R}^{d_0}$ and output $f(x,\Theta)\in \mathbb{R}^{d_n}$. Here $f$ is defined recursively by
\begin{equation}
\begin{split}
 f_0(x) &= x \\
 f_i(x) &= \sigma(W_if_{i-1}(x) + b_i) \\
 f(x,\Theta) &= W_nf_{n-1}(x) + b_n,
 \end{split}
\end{equation}
where $\Theta = W_i\in \mathbb{R}^{d_i\times d_{i-1}}, b_i\in \mathbb{R}^{d_i}$ are parameters and $\sigma$ is an activation functions.

We now consider the random initialization. Suppose that each entry of the weights $W_i$ are chosen at random with variance $\frac{c_{i-1}}{d_{i-1}}$ and the bias $b$ is initialized at $0$. This gives us a random function $f(x,\Theta_0)$ drawn from a distribution on the space of functions $f:\mathbb{R}^{d_0}\rightarrow \mathbb{R}^{d_n}$. Our goal is to analyze this distribution.

Before we consider the general case, let us consider shallow linear networks, i.e. $\sigma_i = \text{id}$ and $n=2$. In this case $\Theta = W_1, W_2, b_1, b_2$ and
\begin{equation}
 f(x,\Theta_0) = W_2W_1x,
\end{equation}
where $W_1\in \mathbb{R}^{d_1\times d_0}$ is chosen uniformly at random with variance $\frac{c_0}{d_0}$ and $W_2\in \mathbb{R}^{d_2\times d_1}$ independently with variance $\frac{c_1}{d_1}$. Now let $x,y\in \mathbb{R}^{d_0}$ and consider the joint distribution of $f(x,\Theta_0)\in \mathbb{R}^{d_2}$ and $f(y,\Theta_0)\in \mathbb{R}^{d_2}$. We begin by calculating the covariance of the values at $x$ and $y$
\begin{equation}
 \mathbb{E}(f(x,\Theta_0)f(y,\Theta_0)^T) = \mathbb{E}_{W_2}(W_2\mathbb{E}_{W_1}(W_1xy^TW_1^T)W_2^T).
\end{equation}
We now easily calculate
\begin{equation}
 \mathbb{E}_{W_1}(W_1xy^TW_1^T) = \frac{c_0(x^Ty)}{d_0}I_{d_1},
\end{equation}
and so, since $\mathbb{E}_{W_2}(W_2W_2^T) = c_1I_{d_2}$, we get
\begin{equation}
 \mathbb{E}(f(x,\Theta_0)f(y,\Theta_0)^T) = \frac{c_0c_1(x^Ty)}{d_0}I_{d_2}.
\end{equation}
One can also easily calculate that this is the same covariance which would be obtained if the hidden layer were removed or if more hidden layers were added. One can also show that as $d_1\rightarrow \infty$, the random function $f(x,\Theta_0)$ converges in distribution to a Gaussian process with the given convariance
\begin{equation}
 K(x,y) = \frac{c_0c_1(x^Ty)}{d_0}I_{d_2}.
\end{equation}

Adding in an activation function $\sigma$ simply changes the covariance, which now becomes
\begin{equation}
 \mathbb{E}(f(x,\Theta_0)f(y,\Theta_0)^T) = \mathbb{E}_{W_2}(W_2\mathbb{E}_{W_1}(\sigma(W_1x)\sigma(y^TW_1^T))W_2^T),
\end{equation}
which generally won't have a closed form expression.

Note that if $\sigma$ is a positively homogeneous function such as the ReLU, then the relative scaling of the $W_i$ doesn't matter, i.e. only the product of their variance is important. However, for non-ReLU activation functions it may be important to give the correct scale for each $W_i$ and the correct initialization schemes are potentially not known. We now summarize this discussion in the following result.
\begin{theorem}
 In the limit $d_1,...,d_{n-1}\rightarrow \infty$, the randomly initialized function $f(x,\Theta)$ converges in distribution to a Gaussian process on $\mathbb{R}^{d_0}$ with covariance
 \begin{equation}
  K(x,y) = \lim_{d_1,...,d_{n-1}\rightarrow \infty} \mathbb{E}(f(x,\Theta_0)f(y,\Theta_0)^T).
 \end{equation}
 This (multidimensional) kernel $K$ cannot in general be computed explicitly except in the linear case, in which case it is given by
 \begin{equation}
  K(x,y) = \left(\prod_{i=0}^{n-1} c_i\right)\frac{x^Ty}{d_0}I_{d_n}.
 \end{equation}

\end{theorem}
Finally, we note that for positively homogenous $\sigma$ (i.e. a piecewise linear function with discontinuous derivative only at $0$), we can explicitly calculate the diagonal of the kernel $K$. We get
\begin{equation}
 K(x,x) = \left(\frac{s_l^2 + s_r^2}{2}\right)^{n-1}\left(\prod_{i=0}^{n-1} c_i\right)\frac{\|x\|^2}{d_0}I_{d_n},
\end{equation}
where $s_l$ and $s_r$ are the slopes to the left and right of $0$, respectively. Choosing the $c_i$ appropriately so that this quantity is constant independent of the depth $n$ is exactly He's initialization scheme.

\subsection{Neural Tangent Kernel}
We now proceed to describe the neural tangent kernel, which describes the behavior of the function $f(x,\Theta_t)$ under a gradient flow on the parameters $\Theta$ with loss function $L(f)$. Specifically, we consider the gradient flow dynamics
\begin{equation}
 \frac{d}{dt}\Theta_t = -D_\Theta\nabla_\Theta L(f(\cdot, \Theta_t)),
\end{equation}
where $D_\Theta$ is a diagonal preconditioner with diagonal weight $\frac{1}{d_{i-1}}$ for the weights in $W_i$ and weight $1$ for the biases.

We consider the network $f(\cdot, \Theta_t)$ as a curve in the space of functions $f:\mathbb{R}^{d_0}\rightarrow \mathbb{R}^{d_n}$. The neural tangent kernel (NTK) describes this curve in the infinite width limit ($d_1,...,d_{n-1}\rightarrow \infty$).

We begin with an abstract description of the idea. The object we want to calculate is $\frac{d}{dt}f(x,\Theta_t)\in C(\mathbb{R}^{d_0}\rightarrow \mathbb{R}^{d_n})$ (here $C(\mathbb{R}^{d_0}\rightarrow \mathbb{R}^{d_n})$ is the space of continuous functions from $\mathbb{R}^{d_0}$ to $\mathbb{R}^{d_n}$). We want to relate this to the loss function derivative $\partial_fL\in C^\prime(\mathbb{R}^{d_0}\rightarrow \mathbb{R}^{d_n})$, which is the space of $\mathbb{R}^{d_n}$ valued measures on $\mathbb{R}^{d_0}$. The chain rule implies that
\begin{equation}
 \left(\frac{d}{dt}f(\cdot,\Theta_t)\right)(y) = -\int_{\mathbb{R}^{d_0}} \sum_{\theta\in \Theta}D_\theta \left(\frac{d}{d\theta} f(y, \Theta_t)\right)
 \left(\frac{d}{d\theta} f(x, \Theta_t)\right)^T \partial_fL(x) dx.
\end{equation}
Here the neural tangent kernel at $\Theta_t$ is defined by
\begin{equation}
 K^{NT}_t(x,y) = \sum_{\theta\in \Theta} D_\theta\left(\frac{d}{d\theta} f(y, \Theta_t)\right)
 \left(\frac{d}{d\theta} f(x, \Theta_t)\right)^T,
\end{equation}
where $D_\theta$ denotes the diagonal elements of the preconditioner $D_\Theta$.

At this point we proceed by considering the case of shallow linear networks and calculate the above quantity. We obtain, using Einstein notation,
\begin{equation}
 f(x, \Theta_t)_i = w^2_{ik}w^1_{kj}x_j + w^2_{ik}b^1_k + b^2_i.
\end{equation}
This enables us to calculate
\begin{equation}
\begin{split}
 K^{NT}_t(x,y) = &\left(\sum_{k=1}^{d_1} \frac{1}{d_1}(w^1_{kj}x_j)(w^1_{kl}y_l) + \frac{b^1_k}{d_1} + 1\right)I_{d_n} + \left(\frac{1}{d_0}x_jy_j + 1\right) w^2_{ik}w^2_{lk} \\
 = &~\left(\frac{1}{d_1}(W_1x)^T(W_1y) + b_1^T\frac{1}{d_1} + 1\right)I_{d_n} + \left(\frac{1}{d_0}x^Ty + 1\right) W_2W_2^T.
 \end{split}
\end{equation}

There are now two key observations made. The first is that as $d_1\rightarrow \infty$, the initial neural tangent kernel converges almost surely (under the previously considered initialization) to
\begin{equation}
\begin{split}
 K^{NT}_0(x,y) \rightarrow &\left(\frac{c_0}{d_0}x^Ty + 1\right)I_{d_n} + \left(\frac{1}{d_0}x^Ty + 1\right) c_1I_{d_n} \\
 &= \left(\frac{1}{d_0}\left(c_0 + c_1\right)x^Ty + c_1 + 1\right)I_{d_n}
 \end{split}
\end{equation}

The second, much more subtle observation is that as $d_1\rightarrow \infty$, the kernel at time $t$ converges almost surely to the same thing
\begin{equation}
 K^{NT}_t(x,y) \rightarrow K^{NT}_0(x,y) \rightarrow \left(\frac{1}{d_0}\left(c_0 + c_1\right)x^Ty + c_1 + 1\right)I_{d_n}.
\end{equation}

This means that in the infinite width limit the gradient flow in function space is the gradient flow of the functional $L$ with respect to the inner product (on the function space) defined by the neural tangent kernel (which is asymptotically constant)
\begin{equation}
 \langle f, g\rangle_{NT} = \int_{\mathbb{R}^n}\int_{\mathbb{R}^n} f(x)K^{NT}_0(x,y)g(y) dx dy.
\end{equation}
In other words, the flow in function space is given by
\begin{equation}
 \left(\frac{d}{dt}f(\cdot,\Theta_t)\right)(y) = -\int_{\mathbb{R}^{d_0}} \left(\frac{1}{d_0}\left(c_0 + c_1\right)x^Ty + c_1 + 1\right)I_{d_n}\partial_fL(x) dx.
\end{equation}
Notice that for the linear networks, the neural tangent kernel is positive semi-definite and not positive definite. This corresponds to the fact that the linear networks can only represent linear functions, a very small subspace of all continuous functions.

The neural tangent kernel observation is that essentially the same analysis can be carried out for arbitrarily deep neural networks with more complicated activation functions. In this case the neural tangent kernel $K^{NT}_0(x,y)$ is much more complicated and can't be given in closed form. Nevertheless, it is shown in the original paper that the kernel is positive definite as long as $\sigma$ is not a polynomial.

\includepdf[pages=-,pagecommand={},width=\textwidth]{6DL/HandWrittenNotes/KernelNotes.pdf}
\includepdf[pages=-,pagecommand={},width=\textwidth]{6DL/HandWrittenNotes/Kernels2.pdf}


%\chapter{Connection of 1D CNN and DNN}
\section{1D CNN as DNN}
First let us consider a CNN model for 1D date i.e. $x \in \mathbb{R}^d$ as:
$$
(v\ast x)_i = \sum_{k=1}^d w_{i-k+1}x_k, \quad \forall i = 1:d+s,
$$
with
$$
v \in \mathbb{R}^s.
$$
This lead to a $(d+s)\times d$ Toeplitz type convolutional matrix $T^v$ defined by
$$
(T^v)_{i,j} = v_{i-k+1}, \quad i=1:d+s, k = 1:d.
$$

This kind of convolution can be think as ``padding" for not only data but also kernel.
So, this will lead to an increasing of dimensional for output.

Here, we consider ``restriction-activation'' functions as only activation functions,
and alway only have one channel, so we have the CNN model  as:
\begin{equation}\label{CNN_iteration_vector}
f^j(x,\Theta^j) = v^j \ast( g \circ f^{j-1})(x,\Theta^{j-1}) + b^j \in \mathbb{R}^{d + (j+1) \times s }
\end{equation}
where
\begin{equation}
\Theta^j=(\Theta^{j-1},v^j), \quad \Theta^0=(v^0, b^0)
\end{equation}
with
\begin{equation}
f^0(x)=v^0 \ast x + b^0,
\end{equation}
and
\begin{equation}\label{CNN_finallayer}
f(x; \Theta) = f^J.
\end{equation}

Then we say that, use CNN to approximate function is in the class:
$$
\mathcal{H} = \{ \sum_{i= 1}^{d_J} c_i f^J_i(x): c \in \mathbb{R}^{d_J}\},
$$
here $d_j = d + (j+1)\times s$, is the dimension of $f^j$.

Now let us consider, we have a one hidden layer DNN model as:
$$
f_{DNN} = \sum_{i=1}^m \alpha_k g( w_i x + b_k),
$$
then we can construct
$$
W = [w_1, w_2, \cdots, w_m], \in \mathbb{R}^{1\times(md)},
$$
then we have the next decomposition as
$$
W\ast x = w^{\hat J-1} \cdots w^1\ast w^0 \ast x,
$$
for
$$
\hat J \le \frac{md}{s-1} +2.
$$

Now, we have a new CNN model as:
\begin{equation}\label{CNN_iteration_vector}
f^j(x,\Theta^j) = v^j \ast \circ f^{j-1}(x,\Theta^{j-1}) + b^j \in \mathbb{R}^{d + (j+1) \times s }
\end{equation}
and
$$
f^J(x,\Theta^J) = w^J \ast ( g\circ f^{J-1})(x,\Theta^{J-1}) \in \mathbb{R}^{d + (J+1) \times s },
$$
where
\begin{equation}
\Theta^j=(\Theta^{j-1},v^j), \quad \Theta^0=(v^0, b^0)
\end{equation}
with
\begin{equation}
f^0(x)=v^0 \ast x + b^0,
\end{equation}
and
\begin{equation}\label{CNN_finallayer}
f(x; \Theta) = f^J.
\end{equation}
Then we can choose $w^j$ as delta i.e identity map for $j \ge \hat J$, and for the last model
$$
f_{CNN} = \sum_{i= 1}^{d_J} c_i f^J_i(x),
$$
we can have
$$
f_{CNN} = f_{DNN},
$$
by choosing $c$ coefficients and $b^j$ carefully. [As homework.]
So all approximation properties for one hidden layer DNN can be reconstructed for CNN.

\begin{theorem}
	For one dimensional kernel, we have the decomposition as
	$$
	W\ast x = w^{\hat J-1} \cdots w^1\ast w^0 \ast x.
	$$
\end{theorem}
\begin{proof}
	This proof is in the literature of Daubechies 1992.  Let us define the symbol $\tilde w$ as a polynomial on $\mathbb{C}$ by
	$$
	\tilde w (z) = \sum_{i=1}^s w_{i}z^{i-1}.
	$$
	The most important result is that
	$$
	\tilde w(z) \tilde v(z) = \widetilde{(w\ast v)}(z).
	$$
	Then we know that for any polynomial with real coefficients it can be decomposed as
	$$
	\tilde W(z) = W_M \Pi_{k=1}^K\{z^2 - 2\alpha_kz + (\alpha_k^2 + \beta_k^2)\} \Pi_{2K+1}^{M-1}(z - \alpha_k).
	$$
	This means that, for $s \le 2$, we can combine the above decomposition with every $\tilde w^j(z)$ is a $s-1$-degree of polynomial ans
	$$
	\tilde W(z) = \tilde w^{\hat J-1}(z)\cdots \tilde w^{1}(z)\tilde w^{0}(z).
	$$
\end{proof}


\section{Group Actions}
\begin{definition}
    Let $X$ be a set and let $\operatorname{Sym}X$ be the group of all bijections $X\to X$ under composition.
\end{definition}
\begin{definition}
    A group $G$ is a permutation group if it is a subgroup of $\operatorname{Sym}X$ for some set $X$.
    We say it is a permutation group of degree $n$ if $X$ is finite and $|X|=n$.
\end{definition}
\begin{example}
    1. $S_n=\operatorname{Sym}\{1,2,\ldots,n\}$ is a permutation group of degree $n$.
    So is $A_n$.\\
    2. The group $D_{2n}$ is a permutation group of degree $n$ by thinking about the way it transforms the vertices of a regular $n$-gon.
\end{example}
\begin{definition}
    Let $G$ be a group and $X$ be a set.
    An action of $G$ on $X$ is a function $\star:G\times X\to X$ satisfying:\\
    1. $\forall x\in X,1\star x=x$.\\
    2, $\forall g,h\in G,x\in X,(gh)\star x=g\star (h\star x)$.
\end{definition}
\begin{proposition}
    An action $\star:G\times X\to X$ is equivalent to a homomorphism $\phi:G\to\operatorname{Sym}X$.
\end{proposition}
\begin{proof}
    Take $\phi(g)(x)=g\star x$.
\end{proof}
\begin{definition}
    We say such a homomorphism $\phi$ a permutation representation of $G$.
\end{definition}
In particular, if $\phi$ is injective, then $G$ is isomorphic to a subgroup of $\operatorname{Sym}X$.
\begin{definition}
    Let $\star:G\times X\to X$ be a group action, then the orbit of an element $x\in X$ is defined by
    $$\operatorname{Orb}_G(x)=\{g\star x:g\in G\}$$
    The stabiliser of it is defined by
    $$G_x=\{g\in G:g\star x=x\}$$
\end{definition}
\begin{theorem}
    Let $\star:G\times X\to X$ be a group action, then for any $x\in X$ there is a bijection $\operatorname{Orb}_G(x)\to G/G_x$.
\end{theorem}
\begin{proof}
    Take $g\star x\mapsto gG_x$.
\end{proof}
\begin{corollary}
    If $G$ is finite, then $|\operatorname{Orb}_G(x)||G_x|=|G|$.
\end{corollary}
\begin{proof}
    Follows directly.
\end{proof}
\begin{remark}
    1. $\ker\phi=\bigcap_{x\in X}G_x$ is called the kernel of the action.\\
    2. The orbits partition $X$.
    If there is only one orbit, then we say the action is transitive.\\
    3. The stabilisers of $x,y$ in the same orbit are conjugate subgroups of each other.
    That is, $G_{g\star x}=gG_xg^{-1}$.
\end{remark}
\begin{example}
    1. Given a group $G$, it can act on itself by left multiplication.
    This is called the left regular action.
    The kernel of the action is obviously just the identity.
    So the induced permutation representation makes $G$ isomorphic to a subgroup of $\operatorname{Sym}G$.
    In particular, if $|G|=n$, then $G$ is isomorphic to a subgroup of the symmetric group $S_n$.
    This is known as Cayley's Theorem.\\
    2. Consider a group $G$ and a subgroup $H\le G$, then $G$ can act on $G/H$ by left multiplication.
    This action is transitive.
    Also the stabiliser of $xH$ is $xHx^{-1}$, so the kernel of the action becomes $\bigcap_{x\in X}xHx^{-1}$, the largest normal subgroup of $G$ contained in $H$.\\
    3. Let $G$ acts on itself by conjugation, so $g\star x=gxg^{-1}$.
    In this case, the orbit containing $x$ is called the conjugacy class containing $x$, $\operatorname{ccl}_G(x)$, and the stabiliser of $x$ is called the centraliser $C_G(x)$ of $x$.
    The kernel $Z(G)$ of the action is called the centre of $G$.
    Note that $G$ can also act by conjugation on any of its normal subgroup.\\
    4. Let $X$ be the set of subgroups of the group $G$, then $G$ acts on $X$ by conjugation: $g\star H=gHg^{-1}$.
    So the stabiliser of $H$ is called the normalizer $N_G(H)$ of $H$.
    It is the largest subgroup of $G$ to contain $H$ as a normal subgroup.
\end{example}
\begin{theorem}
    Let $G$ be a nonabelian simple group, and $H<G$ has index $n>1$, then $n\ge 5$ and $G$ is isomorphic to a subgroup of $A_n$.
\end{theorem}
\begin{proof}
    Consider the action of $G$ acting on the set of left cosets of $H$ by left multiplication.
    This gives a homomorphism $\phi:G\to S_n$.
    But $\ker\phi$ is normal in $G$.
    It is obviously not the entire group, so $\phi$ has to be injective, so $G$ is isomorphic to a subgroup of $S_n$.
    We know that $A_n\lhd S_n$, so $A_n\cap G\lhd G$, so either $G\subset A_n$ or $G\cap A_n=1$ since $G$ is simple.
    If $G\cap A_n=1$, we have $G\cong G/(G\cap A_n)\cong GA_n/A_n\le S_n/A_n\cong C_2$ by Second Isomorphism Theorem, contradiction.\\
    Therefore $G\le A_n$.
    Finally if $n\le 4$, $A_n$ does not have a nonabelian simple subgroup by simple exhaustion.
    So $n\ge 5$.
\end{proof}
\begin{example}
    Consider the group of rotational symmetries of the icosahedron with $20$ faces, $12$ vertices and $30$ edges with each face an equilateral triangle.
    We have the following table
    \begin{center}
        \begin{tabular}{c|c}
            order&number\\
            \hline
            $1$&$41$\\
            $2$&$15$\\
            $3$&$20$\\
            $5$&$24$\\
            \hline
            total&$60$
        \end{tabular}
    \end{center}
    Note that $G$ acts on the set of vertices transitively, so $|G|=12\times 5=60$, hence these are all.\\
    The elements of order $2$ are all conjugates, same for elements of order $3$.
    The elements of order $5$ splits into two conjugacy classes, each of size $12$.\\
    If $H\lhd G$, then it must be a union of conjugacy classes including the identity one, but they must have a sum that divides $60$.
    One can check that it can happen iff $H$ is trivial.
    Thus $G$ is simple.\\
    We want to show that $G$ is isomorphic to $A_5$.
    We claim that the set of subgroups $H$ of order $4$ removing the identity partitions the $16$ elements of order at most $2$.
    Note that we must have $H\cong C_2\times C_2$ since $G$ does not have element of order $4$, so each such subgroups contains $3$ involutions.
    If $g\in G$ has order $2$, then $g\in C_G(g)$, so $|C_G(g)|=|G|/|\operatorname{ccl}_G(g)|=4$, so every involution is contained in a subgroup of order $4$.\\
    Suppose $1\neq g\in H\cap K$ where $H,K$ are subgroups of order $4$.
    But the centralizer of $g$ has order $4$ and contains both $H$ and $K$ since $H,K$ are abelian, but they all have size $4$, hence $H=K$.\\
    Let $G$ act on the $5$ subgroups of order $4$ by conjugation.
    This gives a homomorphism $\phi:G\to\operatorname{Sym}(X)\cong S_5$, but since $G$ is simple, $\phi$ is injective.
    So $G\le S_5$.
    But again $G\cap A_5$ can only be $A_5$ by exactly the same trick as the proof of the preceding theorem.
    Since $|G|=|A_5|=60$, we have $G=A_5$.
 \end{example}


\subsection{Inference rules  for first-order logic}

\subsubsection{Existential instantiation}

If \(P\) is true for a specific input, then there exists an input for \(P\) where \(P\) is true.

\(P(r)\Rightarrow \exists xP(x)\)

\subsubsection{Existential generalisation}

\(\exists xP(x)\Rightarrow P(r)\)

Where \(r\) is a new symbol.

\subsubsection{Universal instantiation}

If \(P\) is true for all values of \(x\), then \(P\) is true for any input to \(P\).

\(\forall xP(x)\Rightarrow P(a/x)\)

Where \(a / x\) represents substituting \(a\) for \(x\) within \(P\).

\subsubsection{Universal generalisation}

If there is a derivation for \(P(x)\), then there is a derivation for \(\forall x P(x)\).

\(\vdash P(x)\Rightarrow \vdash \forall x P(x)\)


\subsection{Bayesian CART}

\subsubsection{Our prior}

Call the collective parameters of the tree \(\Theta =(T, k, r)\) and \(\theta \).

Collectively our prior is defined by \(P(\Theta )\) and \(P(\theta )\)

\subsubsection{Bayes' theorem}

We want to know the posterior given our data \(X\).

\(P(\Theta | X)=\dfrac{P(X|\Theta )P(\Theta )}{P(X)}\)

\(P(\Theta | X)\propto P(X|\Theta )P(\Theta )\)

\subsubsection{Expanded posterior}

We know explore \(P(X|\Theta )\)

\(P(X|\Theta )=\int P(X| \theta , \Theta)P(\theta)d\theta \)

This means our posterior is:

\(P(\Theta | X)\propto P(\Theta )\int P(X| \theta , \Theta)P(\theta)d\theta \)

\subsubsection{Estimation}

This can be estimated with MCMC.


\chapter{DT systems as linear constant coefficient difference equations}

A \emph{difference equation} is a relation among combinations of two DT functions and shifted versions of them. Similar to differential equations where the solution is a CT function, the solution to a difference equation is a DT function. For example:
\[
y[n+1] + \frac{1}{2}y[n] = x[n]
\]
is a first order, linear, constant-coefficient difference equation. Given $x[n]$ the solution is a function $y[n]$. We can view this as a representation of a DT system, where $x[n]$ is the input signal and $y[n]$ is the output.

There is a parallel theory to differential equations for solving difference equations. However in this lecture we will focus specifically on the iterative solution of linear, constant-coefficient difference equations and the case when the input is a delta function, as this is all we need for this course.

\section{Definition of linear constant coefficient difference equation}

A \emph{linear}, \emph{constant-coefficient}, difference equation (LCCDE) comes in one of two forms.

\begin{itemize}
  \item Delay form.
  \[
  \sum\limits_{k = 0}^N a_k y[n-k] = \sum\limits_{k = 0}^M b_k x[n-k]
  \]
  or
  \[
  a_0y[n] + a_1y[n-1] + \cdots a_N y[n-N] = b_0 x[n] + \cdots b_Mx[n-M]
  \]

\item Advance form. Let $n\rightarrow n+N$, then the delay form becomes
  \[
  \sum\limits_{k = 0}^N a_k y[n+N-k] = \sum\limits_{k = 0}^M b_k x[n+N-k]
  \]
  or
  \[
  a_0y[n+N] + a_1y[n+N-1] + \cdots a_N y[n] = b_0 x[n+N] + \cdots b_Mx[n+N-M]
  \]
\end{itemize}

The {\it order} of the system is given by $N$. The delay and advance forms are equivalent because the equation holds for any $n$, and we can move back and forth between them as needed by a constant index-shift.

\begin{example}[$N=2$, $M=1$]
  The delay form is
  \[
  a_0y[n] + a_1 y[n-1] + a_2 y[n-2] = b_0 x[n] + b_1 x[n-1]
  \]
  Replacing $n \rightarrow n+2$, the advance form is
  \[
  a_0 y[n+2] + a_1 y[n+1] + a_2 y[n] = b+0 x[n+2] + b_1 x[n+1]
  \]
  $\blacksquare$
\end{example}

It will be convenient to define the operator $E^m$ as shifting a DT function by positive $m$, i.e. $E^m x[n] = x[n+m]$, and the operator $D^m$ as shifting a DT function by negative $m$, i.e. $D^m x[n] = x[n-m]$. These are called the advance and delay operators respectively. Then, the advance form of the difference equation using this operator notation is
\[
a_0y[n+N] + a_1y[n+N-1] + \cdots a_N y[n] = b_0 x[n+N] + \cdots b_Mx[n+N-M]
\]
\[
a_0 E^Ny + a_1E^{N-1}y + \cdots a_N y = b_0 E^{N}x + \cdots b_M E^{N-M}x
\]
Factoring out the advance operators gives
\[
\underbrace{\left(a_0E^N + a_1E^{N-1} + \cdots a_N\right)}_{Q(E)} y = \underbrace{\left(b_M E^{N} + \cdots b_M E^{N-M}\right)}_{P(E)} x
\]
or
\[
Q(E)y[n] = P(E)x[n]
\]

Similarly, the delay form of the difference equation using this operator notation is
\[
a_0y[n] + a_1y[n-1] + \cdots a_N y[n-N] = b_0 x[n] + \cdots b_Mx[n-M]
\]
\[
a_0y[n] + a_1 Dy + \cdots a_N D^N y = b_0 x + \cdots b_MD^M x
\]
Note: The DT delay operator $D$ is similar, but \emph{not} identical to the derivative operator $D$ in CT.

\begin{example}
  Consider the difference equation
  \[
  2y[n+1] + 5y[n] + 2y[n-1] = 2x[n+1]
  \]
  The advance form would be:
  \[
  2y[n+2] + 5y[n+1] + 2y[n] = 2x[n+2]
  \]
  or using the advance operator
  \[
  \left(2E^2 + 5E + 2\right)y = 2E^2x
  \]
  with $Q(E) = 2E^2 + 5E + 2$ and $P(E) = 2E^2$.\\[1em]
  The delay form would be:
  \[
  2y[n] + 5y[n-1] + 2y[n-2] = 2x[n]
  \]
  or using the delay operator
  \[
  \left(2D^2 + 5D + 2\right)y = 2x
  \]
$\blacksquare$
\end{example}

\section{Iterative solution of LCCDEs}

Difference equations are different (pun!) from differential equations in that they can be solved by manually running the equation forward using previous values of the output and current and previous values of the input, given some initial conditions. This is called an \emph{iterative} solution for this reason.

To perform an iterative solution we need the difference equation in delay form
\[
a_0y[n] + a_1y[n-1] + \cdots a_N y[n-N] = b_0 x[n] + \cdots b_Mx[n-M]
\]
We then solve for the current output $y[n]$
\[
y[n] =  - \left(\frac{a_1}{a_0}y[n-1] + \cdots \frac{a_N}{a_0} y[n-N]\right) + \frac{b_0}{a_0} x[n] + \cdots \frac{b_M}{a_0}x[n-M]
\]

Now lets examine what this expression says in words. To compute the current output $y[n]$ we need the value of the \emph{previous} $N-1$ outputs, the value of the \emph{current} input $x[n]$ and $M-1$ \emph{previous} inputs (and the coefficients). Then we can compute the next output $y[n+1]$ by adding the previous computation result for $y[n]$ to our list of things to remember, and forgetting one previous value of $y$. This can continue as long as we like.

\begin{example}
  Consider the first-order difference equation
  \[
  y[n+1] + y[n] = x[n+1]
  \]
  where $y[-1] = 1$ and $x[n] = u[n]$. We first convert this to delay form
  \[
  y[n] = -y[n-1] + x[n]\; .
  \]
  Then we can compute $y[0]$ as
  \[
  y[0] = -y[-1] + x[0] = -1 + 1 = 0
  \]
  and continuing
  \begin{align*}
  y[1] &= -y[0] + x[1] = 0 + 1 = 1\\
  y[2] &= -y[1] + x[2] = -1 + 1 = 0\\
  y[3] &= -y[2] + x[3] = 0 + 1 = 1\\
  \mbox{etc.}
  \end{align*}
  We can see that this will continue to give the alternating sequence $1,0,1,0,1,\cdots$.
$\blacksquare$
\end{example}

\section{Solution of the homogeneous LCCDE}

Note the iterative solution does not give us (directly) and analytical expression for the output at arbitrary $n$. We have to start at the initial conditions and compute our way up to $n$. We now consider an analytical solution when the input is zero, the solution to the \emph{homogeneous} difference equation
\[
Q(E)\, y = a_0y[n+N] + a_1y[n+N-1] + \cdots a_N y[n] = 0 \; .
\]
given $N$ sequential auxiliary conditions on $y$.

Similar to differential equations, the homogeneous solution depends on the roots of the characteristic equation $Q(E)=0$ whose roots are either real or occur in complex conjugate pairs. Let $\lambda_i$ be the i-th root of $Q(E) = 0$, then the solution is of the form
\[
y[n] = \sum\limits_{i=1}^N C_i \lambda_i^{n}
\]
where the parameters $C_i$ are determined from the auxiliary conditions.

For a real system (when the coefficients of the difference equation are real) and when the roots are complex $\lambda_{1,2} = |\lambda|e^{\pm j\beta}$, it is cleaner to assume a form for those terms as
\[
y[n] = C |\lambda|^n\cos(\beta n + \theta)
\]
for constants $C$ and $\theta$.

\begin{example}[First-Order]
  Find the solution to the first-order homogeneous LCCDE
  \[
  y[n+1] + \frac{1}{2}y[n] = 0 \mbox{ with } y[0] = 5 \; .
  \]
  Note $Q(E) = E + \frac{1}{2}$ has a single root $\lambda_1 = -\frac{1}{2}$. Thus the solution is of the form
  \[
  y[n] = C\left( -\frac{1}{2}\right)^n
  \]
  where the parameter $C$ is found using
  \[
  y[0] = C = 5
  \]
  to give the final solution
  \[
  y[n] = 5\left( -\frac{1}{2}\right)^n
  \]
  $\blacksquare$
\end{example}

\begin{example}[Second-Order, Complex Roots]
  Find the solution to the second-order homogeneous LCCDE
  \[
  y[n+2] + y[n+1] + \frac{1}{2}y[n] = 0 \mbox{ with } y[0] = 1 \mbox{ and } y[1] = 0\; .
  \]
  Note $Q(E) = E^2 + E + \frac{1}{2}$ has a pair of complex roots $\lambda_{1,2} = -\frac{1}{2} \pm j\frac{1}{2}$. Thus the solution is of the form
  \[
  y[n] = C \left|\frac{1}{\sqrt{2}}\right|^n\cos\left(\frac{3\pi}{4} n + \theta\right)
  \]
  where the parameters are found using
  \[
  y[0] = C\cos\left(\theta\right) = 1
  \]
  \[
  y[1] = C\frac{1}{\sqrt{2}}\cos\left(\frac{3\pi}{4} + \theta\right) = 0
  \]
  This is true when
  \[
  C = -\sqrt{2} \mbox{ and } \theta = -\frac{\pi}{4} + 2\pi m
  \]
  or
  \[
  C = -\sqrt{2} \mbox{ and } \theta = \frac{3\pi}{4} + 2\pi m
  \]
  for any $m\in \mathbb{Z}$ since $\cos$ is periodic in $2\pi$. A final solution is then
  \[
  y[n] = -\sqrt{2} \left|\frac{1}{\sqrt{2}}\right|^n\cos\left(\frac{3\pi}{4} n - \frac{\pi}{4}\right)
  \]
  $\blacksquare$
\end{example}

\section{Impulse response from LCCDE}

Today our goal is to find the solution to $Q(E)y=P(E)x$ when $x[n] = \delta[n]$ assuming $y[n] = 0$ for $n < 0$, giving the \emph{impulse response} $y[n] = h[n]$. We skip the derivation here and just give a procedure.

\textbf{Step 1:} Let $y_h$ be the homogeneous solution to $Q(E)y_h=0$ for $n > N$.

\textbf{Step 2:} Assume a form for $h[n]$ given by
\[
h[n] = \frac{b_N}{a_N}\delta[n] + y_h[n]u[n]
\]

\textbf{Step 3:} Using the iterative procedure above find the $N$ auxiliary conditions we need by,

\begin{itemize}
\item first, rewrite the equation in delay form and solving for $y[n]$,
\item then let $x[n] = \delta[n]$ and manually compute $h[0]$ assuming $h[n] = 0$ for $n < 0$,
\item repeating the previous step for $h[1]$, continuing up to $h[N-1]$.
\end{itemize}

\textbf{Step 4:} Using the auxillary conditions in step 3, solve for the constants in the solution $h[n]$ from step 2.

\begin{example}

  Find the impulse response of the system given by
  \[
  y[n+2] -\frac{1}{4}y[n+1] -\frac{1}{8}y[n]= 2x[n+1]
  \]

  For step 1 we solve the equation
  \[
  y_h[n+2] -\frac{1}{4}y_h[n+1] -\frac{1}{8}y_h[n] = 0
  \]
  which is of the form
  \[
  y_h[n] = C_1 \left( -\frac{1}{4}\right)^n + C_2 \left( \frac{1}{2}\right)^n
  \]
  since the roots of $Q(E) = E^2 - \frac{1}{4}E - \frac{1}{8}$ are $-\frac{1}{4}$ and $\frac{1}{2}$.

  For step 3, we find the auxiliary conditions needed to find $C_1$ and $C_2$ by rewriting the original equation in delay form and solving for $y[0]$ and $y[1]$ when $x[n] = \delta[n]$.
  \[
  y[n] = \frac{1}{4}y[n-1] + \frac{1}{8}y[n-2] + 2x[n-1]
  \]
  Let $x[n] = \delta[n]$ and manually compute $y[0]$ assuming $y[n] = 0$ for $n < 0$
  \[
  y[0] = \frac{1}{4}\underbrace{y[0-1]}_{0} + \frac{1}{8}\underbrace{y[0-2]}_{0} + 2\underbrace{\delta[0-1]}_{0} = 0
  \]
  Repeat for $y[1]$
  \[
  y[1] = \frac{1}{4}\underbrace{y[1-1]}_{0} + \frac{1}{8}\underbrace{y[1-2]}_{0} + 2\underbrace{\delta[1-1]}_{1} = 2
  \]
  Now we find the constants using step 4
  \[
  h[0] = C_1  + C_2  = 0
  \]
  \[
  h[1] = C_1 \left( -\frac{1}{4}\right) + C_2 \left( \frac{1}{2}\right) = 2
  \]
  which gives $C_1 = -\frac{8}{3}$ and $C_2 = \frac{8}{3}$. Thus the final impulse response is
  \[
  h[n] = \frac{b_N}{a_N}\delta[n] + y_h[n]u[n] = -\frac{8}{3}\left( -\frac{1}{4}\right)^nu[n] + \frac{8}{3}\left( \frac{1}{2}\right)^n u[n]
  \]
  since $b_N = 0$.
$\blacksquare$
\end{example}

  Note we can confirm our closed-form result in the previous example, for a few values of $n$, by iteratively solving the difference equation
  \[
    h[0] = \frac{1}{4}\underbrace{h[0-1]}_{0} + \frac{1}{8}\underbrace{h[0-2]}_{0} + 2\underbrace{\delta[0-1]}_{0} = 0
  \]
  \[
      h[1] = \frac{1}{4}\underbrace{h[1-1]}_{0} + \frac{1}{8}\underbrace{h[1-2]}_{0} + 2\underbrace{\delta[1-1]}_{1} = 2
    \]
    \[
      h[2] = \frac{1}{4}\underbrace{h[2-1]}_{2} + \frac{1}{8}\underbrace{h[2-2]}_{0} + 2\underbrace{\delta[2-1]}_{0} = \frac{1}{2}
    \]
    \[
      h[3] = \frac{1}{4}\underbrace{h[3-1]}_{\frac{1}{2}} + \frac{1}{8}\underbrace{h[3-2]}_{2} + 2\underbrace{\delta[2-1]}_{0} = \frac{3}{8}
    \]
    and comparing to our closed-form solution a the same values of $n$
    \[
    h[0] = -\frac{8}{3} + \frac{8}{3} = 0
    \]
    \[
    h[1] = -\frac{8}{3}\left( -\frac{1}{4}\right) + \frac{8}{3}\left( \frac{1}{2}\right) = 2
    \]
    \[
    h[2] = -\frac{8}{3}\left( -\frac{1}{4}\right)^2 + \frac{8}{3}\left( \frac{1}{2}\right)^2 = \frac{1}{2}
    \]
    \[
    h[3] = -\frac{8}{3}\left( -\frac{1}{4}\right)^3 + \frac{8}{3}\left( \frac{1}{2}\right)^3 = \frac{3}{8}
    \]

\begin{example}
  Find the impulse response of the system given by
  \[
  y[n+1] - \frac{1}{2}y[n] = x[n+1] + x[n]
  \]

  In step 1 we note the solution to $Q(E)y[n] = 0$ is of the form
  \[
  y_h[n] = C\left( \frac{1}{2}\right)^n
  \]
  From step 2 we note $b_N = 1$ and $a_N = -\frac{1}{2}$, so that
  \[
  h[n] = -2\delta[n]  +  C\left( \frac{1}{2}\right)^n\, u[n]
  \]
  In step 3 we manually find $h[0]$
  \begin{align*}
    y[n] &= \frac{1}{2}y[n-1] + x[n] + x[n-1]\\
    h[n] &= \frac{1}{2}y[n-1] + \delta[n] + \delta[n-1]\\
    h[0] &= 0 + 1 + 0 = 1
  \end{align*}
  And in step 4 we solve for $C$
  \[
  h[0] = -2  +  C = 1 \mbox{ implies } C = 3
  \]
  to give
  \[
  h[n] = -2\delta[n]  +  3\left( \frac{1}{2}\right)^n\, u[n]
  \]
  $\blacksquare$
\end{example}


\section{Multivariable Functions}
\noindent
Multivariable functions take several values as an input and return a single value as an output.\\
For example, $ z =x^2 + y^2$ takes $\mathbb{R}^2 \to \mathrm{R}$.\\
Although we can only graph and fully visualize up to $\mathbb{R}^2 \to \mathbb{R}$ as a surface, we can imagine a multivariable function with 3 inputs ($\mathbb{R}^3 \to \mathbb{R}$) as a heatmap in 3D space. However, most of the mathematics we will discuss applies to functions with any number of inputs.\\

\noindent
The domain of a multivariable function $f: \mathbb{R}^n \to \mathbb{R}$ is the largest set of points on which $f$ is defined.\\
For example, if $f(x,y) = \ln{\left(9-x^2-y^2\right)}$, the domain of $f(x,y)$ is $\left\{ (x,y) | x^2+y^2<9 \right\}$.