\section{Top-down parsing and parser generators}
\subsection*{LPDA}
\begin{tikzpicture}[shorten >=1pt, node distance=2cm, auto,on grid]
	\node[state] (q1) {$q_1$};
	\node[state] (q2) [right=of q1] {$q_2$};
	\node[state] (q1x) [below=of q1] {$q_1,x$};
	\node[state] (q2a) [below=of q2] {$q_2,a$};
	\node[state] (q2b) [below=of q2a] {$q_2,b$};
	\node[state] (q2p) [below=of q2b] {$q_2,\epsilon$};
	\node[state] (q1e) [right=of q2] {$q_1$};
	\node[state] (q2e) [below=of q1e] {$q_2$};
	\node[state] (q1y) [below=of q2e] {$q_1,y$};
	\node[state] (q2y) [below=of q1y] {$q_2,y$};
	\path[->]
	(q1) edge node [above,midway,sloped] {$x:x,X/\gamma$} (q2)
	(q1x) edge node [above,midway,sloped] {$x,X/\gamma$} (q2a)
	(q1x) edge node [above,midway,sloped] {$x,X/\gamma$} (q2b)
	(q1x) edge node [below,midway,sloped] {$x,X/\gamma$} (q2p)
	(q1e) edge node [left] {$y:\epsilon,X/\gamma$} (q2e)
	(q1y) edge node [left] {$\epsilon,X/\gamma$} (q2y);
\end{tikzpicture}
\subsection*{Prediction}
\textbf{k-look-ahead PDA}: A k-LPDA is a tuple $(Q, \Sigma, \Gamma, \delta, q_0, Z_0, F)$ different: $\delta$ : $Q \times (\Sigma \cup \{\epsilon\}) \times \Gamma \times \Sigma^{\leqslant k} \rightarrow 2^{(q\times \Gamma^*)}$.\\
\textbf{Configuration change LPDA}
The LPDA can move from $(q,auv,X\beta)$ to $(Q',uv,\alpha\beta)$ if there is $(q',\alpha) \in \delta (q,a,X,au)$. $X \in \Gamma$; $a \in \Sigma \cup \{\epsilon\}$; $u \in \Sigma^{\leqslant k-1}$; $v \in \Sigma^*$; and if $|auv| \geqslant k$ then $|au| = k$, else $v = \epsilon$.\\
\subsection*{First \& Follow}
\textbf{First:} $First^k(\alpha)$ = $\{w \in T^*: \alpha \Rightarrow^* wx$ $\wedge$ $(|w| = k$ or $|w| < k \wedge x = \epsilon)\}$; \textbf{Follow:} $Follow^k(\alpha)$ $= \{ w \in T^* |$ $\exists \beta, \gamma: S \Rightarrow^* \beta\alpha\gamma$ $\wedge w \in First^k(\gamma)\}$. \textbf{First/First:} Intersecting First sets. \textbf{First/Follow:} First and Follow sets of variable intersect. \textbf{Solutions:} Left factoring First/First; Substitution First/Follow, introduces First/First; Left recursion removal.
\subsection*{LL(k) grammar}
A CFG is LL(K) iff for all pairs of derivations:\\
$S \Rightarrow* wA\gamma \Rightarrow w\alpha_1\gamma \Rightarrow^* wx_1$\\
$S \Rightarrow* wA\gamma \Rightarrow w\alpha_2\gamma \Rightarrow^* wx_2$\\
with $w,x_1,x_2\in T^*$, $A \in V$, and $\gamma \in (V \cup T)^*$, if $First^k(x_1) = First^k(x_2)$ then $\alpha_1 = \alpha_2$\\
\textbf{Strongly LL(K):} iff for all pairs of rules $A \rightarrow \alpha_1$ and $A \rightarrow \alpha_2$ with $\alpha_1 \neq \alpha_2$: $First^k(\alpha_1\cdot Follow^k(A))$$First^k(\alpha_2\cdot Follow^k(A))$$=\phi$. \textit{every strong LL(k) grammar is LL(k)}. There is a strict hierarchy.
		\subsection*{Recursive-descent parsers}
		Instead of pushing sentential forms into a stack, we can make recursive
		calls to functions handling nonterminals. Consider, e.g., a grammar with rules $S \rightarrow Ab$ and $S \rightarrow Bc$\ldots
		\subsection*{Attribute grammar}
		\textbf{S-attributed} no inherited attributes. \textbf{L-attributed} they allow the attributes to be evaluated in 1 DFS Left-to-right traversal of the derivation tree; $S-att \subseteq L-att$\\
		\textbf{L-attribute practice:} An attribute grammar is L-attributed if for all rules $A \rightarrow X_1...X_n$, for all inhertited attributes $\alpha$ of $X_j$ we have that the corresponding semantic rule depends only on: Attributes of $A \rightarrow X_1...X_n$ and inherited attributes of A.


%

\documentclass[twoside]{article}
\setlength{\oddsidemargin}{0.25 in}
\setlength{\evensidemargin}{-0.25 in}
\setlength{\topmargin}{-0.6 in}
\setlength{\textwidth}{6.5 in}
\setlength{\textheight}{8.5 in}
\setlength{\headsep}{0.75 in}
\setlength{\parindent}{0 in}
\setlength{\parskip}{0.1 in}

%
% ADD PACKAGES here:
%

\usepackage{amsmath,amsfonts,graphicx}

\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}

%
% The following macro is used to generate the header.
%
\newcommand{\lecture}[4]{
   \pagestyle{myheadings}
   \thispagestyle{plain}
   \newpage
   \setcounter{lecnum}{#1}
   \setcounter{page}{1}
   \noindent
   \begin{center}
   \framebox{
      \vbox{\vspace{2mm}
    \hbox to 6.28in { {\bf EE402 - Discrete Time Systems
	\hfill Spring 2018} }
       \vspace{4mm}
       \hbox to 6.28in { {\Large \hfill Lecture #1 \hfill} }
       \vspace{2mm}
       \hbox to 6.28in { {\it Lecturer: #2 \hfill } }
      \vspace{2mm}}
   }
   \end{center}
   \markboth{Lecture #1}{Lecture #1}

   \vspace*{4mm}
}

\renewcommand{\cite}[1]{[#1]}
\def\beginrefs{\begin{list}%
        {[\arabic{equation}]}{\usecounter{equation}
         \setlength{\leftmargin}{2.0truecm}\setlength{\labelsep}{0.4truecm}%
         \setlength{\labelwidth}{1.6truecm}}}
\def\endrefs{\end{list}}
\def\bibentry#1{\item[\hbox{[#1]}]}

\newcommand{\fig}[3]{
			\vspace{#2}
			\begin{center}
			Figure \thelecnum.#1:~#3
			\end{center}
	}

% Use these for theorems, lemmas, proofs, etc.
\newtheorem{theorem}{Theorem}[lecnum]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newenvironment{proof}{{\bf Proof:}}{\hfill\rule{2mm}{2mm}}

% **** IF YOU WANT TO DEFINE ADDITIONAL MACROS FOR YOURSELF, PUT THEM HERE:

\begin{document}

% Lecture Details
\lecture{13}{Asst. Prof. M. Mert Ankarali}

\par

\section*{Matrix Exponential, $e^{At}$}

Let's first review the matrix exponential, $e^{A t}$.
Let $t \in \mathbb{R}$ and $A \in \mathbb{R}^{nxn}$, then $e^{At}$ defined as
%
\begin{align*}
       e^{A t} &:= I + A t + \frac{1}{2 !} A^2 t^2 + \frac{1}{3 !} A^3
                 t^3 + \cdots
\\
	e^{A t} &:= \sum\limits_{k=0}^{\infty} \frac{t^k}{k!} A^k
\end{align*}
%
which converges for all  $t \in \mathbb{R}$ and $A \in
\mathbb{R}^{nxn}$.

Now let's review some properties

\begin{itemize}

 \item \textbf{Claim:}
%
\begin{align*}
  \frac{d}{d t} e^{A t} &= A e^{A t} = e^{A t} A
\end{align*}

\textbf{Proof:}
%
\begin{align*}
 	\frac{d}{dt} e^{A t} &= \frac{d}{dt} \left( \sum\limits_{k=0}^{\infty} \frac{t^{k}}{k!} A^k \right) =
	\sum\limits_{k=0}^{\infty} k \frac{t^{k-1}}{k!} A^k = \sum\limits_{k=1}^{\infty} \frac{t^{k-1}}{(k-1)!} A^k \\
	&= \sum\limits_{n=0}^{\infty} \frac{t^{n}}{n!} A^{n+1} = A \sum\limits_{n=0}^{\infty} \frac{t^{n}}{n!} A^{n}   = \left( \sum\limits_{n=0}^{\infty} \frac{t^{n}}{n!} A^{n} \right) A
	\\
	\frac{d}{dt} e^{A t}  &= A e^{A t} = e^{A t} A
\end{align*}

\item \textbf{Claim:} Let $t_1 , t_2 \in \mathrm{R}$ then

\begin{align*}
  e^{A t_1} e^{A t_2} = e^{A t_2} e^{A t_1} = e^{A \left( t_1 + t_2 \right)}
\end{align*}

\textbf{Proof:}
%
\begin{align*}
	e^{A t_1} e^{A t_2} &= \left(\sum\limits_{k=0}^{\infty} \frac{t_1^k}{k!} A^k \right) \left(\sum\limits_{j=0}^{\infty} \frac{t_2^j}{j!} A^j \right)
	= \sum\limits_{k=0}^{\infty} \sum\limits_{j=0}^{\infty} \frac{t_1^k}{k!} \frac{t_2^j}{j!} A^{k+j}
\end{align*}
%
Let $n = k + j$ and $j = n - k$, then
%
\begin{align*}
	e^{A t_1} e^{A t_2} &= \sum\limits_{k=0}^{\infty} \sum\limits_{n=k}^{\infty} \frac{t_1^k \ t_2^{n-k}}{k! (n-k)!} A^n
	= \sum\limits_{k=0}^{\infty} \sum\limits_{n=k}^{\infty} \frac{t_1^k \ t_2^{n-k}}{n!} \frac{n!}{k! (n-k)!} A^n
    &= \sum\limits_{k=0}^{\infty} \sum\limits_{n=k}^{\infty} \frac{t_1^k \ t_2^{n-k}}{n!}
    \left( \begin{array}{c} n \\ k \end{array} \right) A^n
\end{align*}
%
Since for $n<k,  \left( \begin{array}{c} n \\ k \end{array} \right) = 0$,
%
\begin{align*}
	e^{A t_1} e^{A t_2} &= \sum\limits_{k=0}^{\infty} \sum\limits_{n=0}^{\infty} \frac{t_1^k \ t_2^{n-k}}{n!}
    \left( \begin{array}{c} n \\ k \end{array} \right) A^n
    = \sum\limits_{n=0}^{\infty} \frac{A^n}{n!} \sum\limits_{k=0}^{\infty} t_1^k \ t_2^{n-k}
    \left( \begin{array}{c} n \\ k \end{array} \right)
\end{align*}
%
Using \textit{binomial} theorem we find
%
\begin{align*}
	e^{A t_1} e^{A t_2} &= \sum\limits_{n=0}^{\infty} \frac{A^n}{n!} (t_1 + t_2)^n \\
	e^{A t_1} e^{A t_2} &= e^{A (t_1 + t_2)}
\end{align*}

Now let $t_1 = t$ and $t_2 = -t$, then we have
%
\begin{align*}
	e^{A t} e^{-A t} &= e^{A (t - t)} = I \quad \rightarrow \quad
                           \left( e^{A t} \right)^{-1} = e^{-A t}
\end{align*}

\item \textbf{Claim:} Let $A , B \in \mathrm{R}^{n \times n}$ and $A B
  = B A$, then

\begin{align*}
	e^{A t} e^{B t} &= e^{B t} e^{A t} &= e^{(A + B) t}
\end{align*}

\textbf{Proof:} Mini Project 6

\par

Note that if $A B \neq B A$ then

\begin{align*}
	e^{A t} e^{B t} \neq e^{(A + B) t)}
\end{align*}

\item \textbf{Claim:} Let $P \in \mathrm{R}^{n \times n}$ and
 $\mathrm{det}(P) \neq 0$, then

\begin{align*}
	e^{\left(P^{-1} A P \right) t} = P^{-1} e^{A t} P
\end{align*}

\textbf{Proof:} Mini Project 6

\end{itemize}

\section*{Solution of CT State-Space Equations}

CT state-space representation has the form
%
\begin{align*}
  \dot{x}(t) &= A x(t) + B u(t)
\\
 \dot{y}(t) &= C x(t) + D u(t)
\end{align*}
%
where $\mathrm{Let} \ x(t) \in \mathbb{R}^n \ , \ y(t) \in \mathbb{R}^p \ ,\  u(t) \in
  \mathbb{R}^q$

First consider the homogeneous solution, i.e. $u(t) = 0$ and $x(0) =
x_0$.
%
\begin{align*}
  \dot{x}(t) &= A x(t) \quad , \quad x(0) = x_0
\\
 \dot{y}(t) &= C x(t)
\end{align*}
%
Let's test if $x(t) = e^{A t} x_0$ is a solution of the homogeneous
equation
%
%
\begin{align*}
  x(0) &= e^{A 0} x_0 = x_0
  \\
  \dot{x}(t) - A x(t) &= \left( A e^{At} \right) x_0 - A e^{At} x_0 = 0
\end{align*}
%

Now let's compute the forced response. First let's analyze the
following derivative
%
\begin{align*}
\frac{d}{dt} \left[ e^{-A t} x(t) \right] = e^{-A t} \dot{x}(t) - e^{-A t} A
  x(t) = e^{-A t} \left[ \dot{x}(t) - A x(t) \right]
\end{align*}
%
Now using this relation let's solve the state-space equations
%
\begin{align*}
  \dot{x}(t) &= A x(t) + B u(t)
\\
  \dot{x}(t) - A x(t) &=  B u(t)
\\
  e^{-A t}  \left[ \dot{x}(t) - A x(t) \right] &=  e^{-A t} B u(t)
\\
  \frac{d}{dt} \left[ e^{-A t} x(t) \right] &=  e^{-A t} B u(t)
\\
  e^{-A t} x(t) &= x(0) + \int\limits_{0}^{t} e^{-A \tau} B u(\tau) d
                  \tau
\\
x(t) &= e^{-A t} x_0 + \int\limits_{0}^{t} e^{A ( t - \tau ) } B u(\tau) d
                  \tau
\end{align*}
%
Thus the solution of a system in state-space form can be written as
%
\begin{align*}
  x(t) &= e^{-A t} x_0 + \int\limits_{0}^{t} e^{A ( t - \tau ) } B u(\tau) d
                  \tau
\\
  y(t) &= C e^{-A t} x_0 + \int\limits_{0}^{t} C e^{A ( t - \tau ) } B u(\tau) d
                  \tau + D u(t)
\end{align*}
%

The function $\Psi(t) = e^{A t}$ is called the state-transition matrix
of the system.

\textbf{Example:} Let's assume that system is a SISO system and
$u(t) = \delta({t})$ (unit-impulse function) and $x_0 = 0$, compute
the impulse response of the system, i.e. $y(t) = h(t)$,

\begin{align*}
  h(t) &= \int\limits_{0}^{t} C e^{A ( t - \tau ) } B \delta(\tau) d
                  \tau + D \delta(t)
\\
   &= C e^{A t } B + D \delta(t)
\end{align*}

\subsection*{S-Domain Solution of CT State-Space Equations}

First take the Laplace transform of state evaluation equation
%
\begin{align*}
\mathcal{L} \left[ \dot{x}(t) \right] &= \mathcal{L} \left[  A x(t) + B u(t) \right]
\\
s X(s) - x(0) &= A X(s) + B U(s)
\\
[s I - A]  X(s) &= x(0) + B U(s)
\\
X(s) &= [s I - A]^{-1} x_0 + [s I - A]^{-1} B U(s)
\\
Y(s) &= C [s I - A]^{-1} x_0 + \left[ C [s I - A]^{-1} B + D\right] U(s)
\end{align*}
%
If we relate time and s-domain solutions we obtain
%
\begin{align*}
  e^{A t} &= \mathcal{L}^{-1} \left[ [s I - A]^{-1} \right]
\\
  h(t) &= \mathcal{L}^{-1} \left[ C [s I - A]^{-1} B + D \right]
\end{align*}

\section*{Discretization of CT State-Space Equations}

Consider the CT system with the given state-space representation
%
%
\begin{align*}
  \dot{x}(t) &= A x(t) + B u(t)
\\
  y(t) &= C x(t) + D u(t)
\end{align*}
%
and suppose that the input is piece-wise constant over intervals of
length $T$. That is
%
\begin{align*}
  u(t) = u[k] \quad , \quad t \in ( kT \, \ (k+1) T)
\end{align*}
%
i.e. input of the system is the output of a ZOH operator. Let's derive the
the DT state-space equations with respect to the sampled-state
$x[k] = x(kT)$.

Let's start with the state evolution equation
%
\begin{align*}
  x(t) &= e^{A t} x(0) + \int\limits_{0}^{t} e^{A ( t - \tau ) } B
         u(\tau) d \tau
\end{align*}
%
It is obvious that due to time-invariant the initial time of the
equation above can be generalized as
%
\begin{align*}
  x(t) &= e^{A (t - t_0)} x(t_0) + \int\limits_{t_0}^{t} e^{A ( t - \tau ) } B
         u(\tau) d \tau \quad , \quad t > t_0
\end{align*}
%
Now let $t_0 = k T$ and $t = (k+1) T$,
%
\begin{align*}
  x( (k+1) T ) &= e^{A T} x(k T) + \int\limits_{kT}^{(k+1)T} e^{A ( (k+1)T - \tau ) } B
         u(\tau) d \tau
\end{align*}
%
Since the $u(t) = u[k]$ in this time interval
%
\begin{align*}
  x[k+1] &= e^{A T} x[k] + \int\limits_{kT}^{(k+1)T} e^{A ( (k+1)T -
           \tau ) } B d \tau \ u[k]
\end{align*}
%
Let $\lambda = (k+1)T -\tau $, then
%
\begin{align*}
  x[k+1] &= e^{A T} x[k] - \int\limits_{T}^{0} e^{A \lambda } B d \lambda \ u[k]
\\
&= \left[ e^{A T} \right] x[k] + \left[\left( \int\limits_{0}^{T}
  e^{A \lambda } d \lambda \right) B \right] u[k]
\end{align*}
%
Given that
%
\begin{align*}
  x[k+1] &= G x[k] + H u[k]
\end{align*}
%
$G$ and $H$ matrices can be extracted as
%
\begin{align*}
  G &= e^{A T}
\\
 H &= \left( \int\limits_{0}^{T}
  e^{A \lambda } d \lambda \right) B
\end{align*}
%
\textbf{Claim:} If $A$ is invertable then we also have
%
\begin{align*}
 H &= A^{-1} \left( e^{AT} - I \right) B = \left( e^{AT} - I \right)
     A^{-1}  B
\end{align*}
%
\textbf{Claim:} Mini Project 6

Now let's consider the output equation
%
%
\begin{align*}
  y(t) &= C x(t) + D x(t)
         \\
 y(k T) &= C x( k T) + D x( k T)
\\
 y[k] &= C x[k] + D x[k]
\end{align*}
%
It can be seen that output equation matrices are not affected from
the discretization.

\textbf{Example:} Consider the following CT-plant transfer function.
%
\begin{align*}
  \frac{Y(s)}{U(s)} = \frac{1}{s} + \frac{1}{s+\mathrm{ln}(2)}
\end{align*}
%
Find a CT state-space representation for this system

\textbf{Solution:}

\begin{align*}
 \dot{x}(t) &= \left[ \begin{array}{cc} 0 & 0 \\ 0 & -
         \mathrm{ln}(2) \end{array} \right] x(t)
    + \left[ \begin{array}{c} 1 \\ 1 \end{array} \right] u(t)
 \\
  y(t) &= \left[ \begin{array}{cc} 1 & 1 \end{array} \right] x(t)
\end{align*}

Compute the state-transition matrix

\textbf{Solution:}

\begin{align*}
 e^{A t} &= \left[
\begin{array}{cc}
                     e^{0 t} & 0 \\ 0 & e^{-\mathrm{ln}(2) t }
\end{array} \right]
\\
 &= \left[
\begin{array}{cc}
             1 & 0 \\ 0 & 0.5^t
\end{array} \right]
\end{align*}

Discretize the CT State-Space equation under zero hold operation
and ideal sampling of the defined state variables, with $T = 1 s$.

\textbf{Solution:}
\begin{align*}
  G &=  e^{A T} = \left[
\begin{array}{cc}
             1 & 0 \\ 0 & 0.5
\end{array} \right]
\\
H &=  \left( \int\limits_{0}^{T}
  e^{A \lambda } d \lambda \right) B
= \left( \int\limits_{0}^{1}
\left[
\begin{array}{cc}
             1 & 0 \\ 0 & 0.5^{\lambda}
\end{array} \right]
  d \lambda \right) \left[ \begin{array}{c} 1 \\ 1 \end{array} \right]
=
\int\limits_{0}^{1}
\left[
\begin{array}{c}
             1  \\ 0.5^{\lambda}
\end{array} \right]  d \lambda
\\
H &= \left[ \begin{array}{c}
             1  \\ 0.721
\end{array} \right]
\end{align*}
Full DT state-space formulation takes the form
%
\begin{align*}
 x[k+1] &= \left[ \begin{array}{cc} 1 & 0 \\ 0 & 0.5 \end{array} \right] x[k]
    + \left[ \begin{array}{c} 1 \\ 0.721 \end{array} \right] u[k]
 \\
  y[k] &= \left[ \begin{array}{cc} 1 & 1 \end{array} \right] x[k]
\end{align*}

Compute the DT pulse transfer function $Y(z)/U(z)$
%
 \textbf{Solution:}

\begin{align*}
 \frac{Y(z)}{R(z)} &= C \left[ z I - G \right]^{-1} H
\\
&= \left[ \begin{array}{cc} 1 & 1 \end{array} \right]
\left[ z I - G \right]^{-1}
\left[ \begin{array}{c} 1 \\ 0.721 \end{array} \right]
\\
&= \left[ \begin{array}{cc} 1 & 1 \end{array} \right]
\left[
\begin{array}{cc}
             z-1 & 0 \\ 0 & z-0.5
\end{array}
\right]^{-1}
\left[ \begin{array}{c} 1 \\ 0.721 \end{array} \right]
\\
&= \left[ \begin{array}{cc} 1 & 1 \end{array} \right]
\left[
\begin{array}{cc}
            \frac{1}{z-1} & 0 \\ 0 & \frac{1}{z-0.5}
\end{array}
\right]
\left[ \begin{array}{c} 1 \\ 0.721 \end{array} \right]
\\
&= \frac{1}{z-1} + \frac{0.721}{z-0.5}
\end{align*}

Now discretize $Y(s)/U(s)$ directly under ZOH operation
%

\textbf{Solution:}
%
\begin{align*}
 \frac{Y(z)}{U(z)} &= \mathcal{Z} \left[ \frac{1 - e^{-s}}{s}
                     \frac{Y(s)}{U(s)} \right] = \frac{1}{z-1} + \frac{0.721}{z-0.5}
\end{align*}

% **** This ENDS THE EXAMPLES. DON'T DELETE THE FOLLOWING LINE:
\end{document}


\documentclass{article}

\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{graphicx}
\usepackage[outdir=./]{epstopdf}
\usepackage[square,numbers,comma]{natbib}
\usepackage{url}
\usepackage{subcaption}
\usepackage{hyperref}
\usepackage{hyphenat}
\usepackage{pdfpages}
\usepackage{MnSymbol} %udots
\usepackage{afterpage} %\afterpage{\clearpage}
\usepackage{siunitx} %for measurements
\usepackage[figurename=Fig.]{caption}
\usepackage[numbib]{tocbibind} %number references and figure title
\usepackage{authblk}

\setcounter{tocdepth}{2}
\setcounter{secnumdepth}{2}

\let\footruleskip\undefined
\usepackage{fancyhdr}
\pagestyle{plain}

\DeclareMathOperator{\variance}{\mathbb{V}ar}
\DeclareMathOperator{\normal}{N}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator{\expectation}{\mathbb{E}}

\newcommand{\addNumber}[1]{\protect\input{#1}\unskip}
\newcommand{\inputNumber}[1]{\protect\input{#1}\unskip}

%to number list of figures
\renewcommand{\listoffigures}{\begingroup
\tocsection
\tocfile{\listfigurename}{lof}
\endgroup}

\title{Detection of defects in additive manufacturing from a single x-ray projection using the empirical null filter}
\date{2019}
\author[1]{S.E.~Lo}
\author[2]{J.M.~Warnett}
\author[2]{G.J.~Gibbons}
\author[2]{M.A.~Williams}
\author[3]{T.E.~Nichols}
\author[1,*]{J.A.~Brettschneider}
\affil[1]{Department of Statistics, University of Warwick, Coventry, CV4 7AL, UK}
\affil[2]{Warwick Manufacturing Group, University of Warwick, Coventry, CV4 7AL, UK}
\affil[3]{Big Data Institute, Old Road Campus, University of Oxford, Oxford, OX3 7LF, UK}
\affil[*]{Corresponding author: julia.brettschneider@warwick.ac.uk, +442476574812}

\DeclareSIUnit\pixel{px}
\DeclareSIUnit\adu{ADU}

\begin{document}\sloppy

\begin{titlepage}
\maketitle

\begin{abstract}
\noindent
\textbf{BACKGROUND}: X-ray computed tomography can be used for defect detection in additive manufacturing (AM). Typically, x-ray projections of an object are taken at a number of angles that are then reconstructed to create a 3D representation of the object inclusive of internal features.
\newline
\textbf{OBJECTIVE}: It was investigated whether it is possible to detect defects based on a single angular projection, avoiding the time-consuming reconstruction process.
\newline
\textbf{METHODS}: An AM sample was manufactured with voids to see if they can be detected. The x-ray projection was compared with a simulation of that projection under the face of uncertainty by treating each pixel as a hypothesis test. To overcome the imperfections of the simulation, the empirical null filter was used to cater for the model misspecification so that sensible inference was achieved.
\newline
\textbf{RESULTS}: Voids with diameters in the order of millimetres were detectable. Simulation studies have shown that statistical properties such as the false discovery rate were preserved.
\newline
\textbf{CONCLUSIONS}: Defect detection in projection space is possible. This work is a contribution towards real-time quality control in AM. The main bottleneck is the multiple evaluations of functions required for robust estimation in the empirical null filter. Speeding up could be possible by implementing the filter on a GPU or exploring further robust estimators.
\newline
\newline
\emph{Keywords:} Additive manufacturing, 3D printing, projection space, hypothesis testing, empirical null, local normalisation, image filtering, voids, defects, quality control
\end{abstract}

\end{titlepage}

\section{Introduction}

Additive manufacturing (AM) is an emerging technology that overcomes many traditional methods such as computer numerical control machining (CNC). In AM, the product is manufactured layer by layer to produce complicated internal and external geometries \citep{ngo2018additive, wong2012review}. One common example of AM technology is fused deposition modelling (FDM) \citep{crump1991fused, crump1992apparatus, stratasys2019what}. A jetting head, or nozzle, deposit the molten material onto a platform or on top of the previous layer. The material is usually plastic in the form of a thin filament. It is heated to just above its melting points, typically \SI{1}{\degreeCelsius} \citep{crump1992apparatus}, so that it cools down within \SI{0.1}{\second} \citep{kruth1991material}. The platform moves, and controlled by a computer, in the $x-y$ axis, or left to right and front to back, to produce a layer. The jetting head can move in the $z$ axis, or up and down, to manufacture the next layer. Powder-based AM such as selective laser sintering \citep{3d2019our, deckard1989method, dtm1990the} can manufacture metal products.

There is a need for product inspection, in particular, quality assessment of the internal structures. The properties of materials produced by AM are not yet fully understood. For example, voids can form between layers which can impact the mechanical properties of the product \citep{ngo2018additive, wang20173d}. In power-based AM technologies, gas can become trapped during the manufacturing process forming gas pores in the manufactured object \citep{tammas2015xct, thijs2010study}. Low wetting ability of the melt pool can cause balling, a phenomenon where spherical or ellipsoidal particles form on the surface of the AM part. They are formed by sintered powder with insufficient contact on the existing layer \citep{gu2009balling, li2012balling}.

Non-destructive testing, such as x-ray computed tomography (XCT), can be used to assess the quality of the product \citep{kruth2011computed, sun2012overview}. The object is held by foam on a turntable and placed between an x-ray source and an x-ray detector. X-ray projections are taken at a series of angles. Typically the x-ray is a cone beam. The x-ray projections are then used to reconstruct the object in 3D \citep{brooks1976principles, feldkamp1984practical, smith1990cone}.

In AM, XCT is commonly used for the investigation of pores \citep{thompson2016x}. Pores can be classified as defects if the pores are larger than some volume threshold. This threshold controls the probability of the detection of defects \citep{amrhein2014characterization, gandossi2010probability}. The porosity can be estimated by dividing the volume of all of the pores by the volume of solid material in the XCT scan \citep{taud2005porosity}. More accurate methods of measuring the porosity can be done using Archimedes' method \citep{spierings2011comparison}.

An advantage of XCT is that the location of the pores can be visualised. Studies have shown a link between porosity and stress concentration \citep{carlton2016damage, leuders2015fatigue, siddique2015computed}. The location of the pores is a good predictor of fatigue strength \citep{leuders2015fatigue}. In addition to pores, the surface deviation can be measured by aligning the reconstruction with the computer-aided design (CAD) model and measuring any discrepancies \citep{kim2016inspection, lee2015compliance, villarraga2015assessing}.

A disadvantage of XCT is that it involves the acquisition and reconstruction of thousands of projections of the sample. XCT is not an instantaneous process so progress bars are usually featured in XCT marketing such as \emph{Nikon}'s inline quality control \citep{nikon2015inline}. Projection acquisition can take between 15 minutes to a number of hours \citep{warnett2016towards} but more projections would improve the accuracy of the reconstruction \citep{kruth2011computed}.

Instead of reconstructing the object, the analysis can be done on a single x-ray projection itself by comparing it to a simulated projection. It was demonstrated that detection of defects is possible in projection space by comparing two simulated projections with each other, one with defects and the other without \citep{brierley2018optimized}. The simulated projections are made in software called \emph{aRTist} \citep{bellon2012radiographic, bellon2007artist, jaenisch2008artist}. It is however inevitable that information is lost from the jump from reconstruction space to projection space, for example, a CT scan has superior performance in the diagnostic of pneumonia compared to a chest radiograph \citep{hayden2009chest}.

This paper investigates whether defects can be detected from a single x-ray projection. To validate our method, we designed an AM test sample with purposely designed voids to see if they can be detected. The methods section is split into four parts. In Section \ref{subsection:appratus}, a CAD model of the test sample was designed and manufactured using AM. Replicate x-ray projections were obtained of the test sample as well as a simulated projection, without the voids, using \emph{aRTist}. Section \ref{subsection:empiricalnullfilter} explains how the obtained and simulated projections were compared to detect these voids under the face of uncertainty. The uncertainty was modelled using the replicated projections. To overcome the imperfections of the simulation, the empirical null filter was used to cater for the model misspecification so that sensible inference was achieved. The implementation of the empirical null filter is discussed in Section \ref{subsection:computational} and tested on small simulated projections in Section \ref{subsection:simulation}. The results from the simulation and the experiment are shown in the results section. The conclusion section summarises the study along with a discussion on further work.

\section{Methods}

\subsection{Apparatus and Data Collection}
\label{subsection:appratus}

An ABS plastic cuboid (\SI{40}{\milli\metre} $\times$ \SI{40}{\milli\metre} $\times$ \SI{60}{\milli\metre}) with voids was purposefully manufactured to see if they can be detected. The CAD model of the test sample is shown in Fig.~\ref{fig:inference_testObject}. It was manufactured using a \emph{Fortus 400mc (Stratasys, US)} 3D printer. Six voids of each diameter, \SI{2.4}{\milli\metre}, \SI{1.2}{\milli\metre}, \SI{0.6}{\milli\metre} and \SI{0.3}{\milli\metre}, were designed in the CAD model. Voids with diameters \SI{2.4}{\milli\metre} and \SI{0.6}{\milli\metre} were regularly arranged, the other ones were arranged irregularly. The precision of the manufacturing was in the order of $\pm\SI{0.1}{\milli\metre}$ \citep{hanseen2013fortus}.

X-ray projections were obtained using the \emph{Nikon XT H LC 225/320} x-ray CT scanner \emph{(Nikon Metrology, UK)}. The \SI{225}{\kilo\volt} source was used with a $\SI{2000}{\pixel}\times\SI{2000}{\pixel}$ \emph{Perkin Elmer} detector consisting of \SI{200}{\micro\metre} pixels, separated by a distance of \SI{1870}{\milli\metre}. The sample was placed on the manipulator, positioned to obtain a $5.7\times$ magnification resulting in an equivalent pixel size of \SI{35}{\micro\metre}. A voltage of \SI{80}{\kilo\volt} was used with a power of \SI{20}{\watt} resulting in a spot size of \SI{20}{\micro\metre} such that penumbra effects \citep{kueh2016modelling} were negligible. 20 replicate greyscale projections were acquired using an exposure time of \SI{500}{\milli\second} and the following powers: \SI{0}{\kilo\volt}, \SI{5}{\kilo\volt}, \SI{10}{\kilo\volt}, \SI{15}{\kilo\volt} and \SI{20}{\kilo\volt}. A shading correction \citep{seibert1998flat} was applied to each projection of the test sample using the greyscale projections to remove intensity influences and variations in pixels of the detector.

19 randomly selected projections, of the test sample, were held out. The remaining projection was used to compare with a simulated projection, obtained using \emph{aRTist} \citep{bellon2012radiographic, bellon2007artist, jaenisch2008artist}. \emph{aRTist} can simulate projections of the sample given the specifications of the CT apparatus, such as the x-ray source and the x-ray detector, and the
CAD of the sample \citep{bellon2011simulation, deresch2012simulating}. \emph{aRTist} was used to obtain a simulation projection as if the voids were not there. Numerical methods were used to align the simulated projection to the obtained projection. The same shading correction was applied to the simulated projection, but instead using a flat black image and simulated greyscale projections.

\subsection{The Empirical Null Filter}
\label{subsection:empiricalnullfilter}

We propose a statistical method to compare a single projection with a simulation of that projection to look for any significant differences which may indicate a defect. It is built on two-tailed multiple hypotheses testing \citep{fisher1970statistical, neyman1933on, pearson1900on} controlling the false discovery rate \citep{benjamini2010discovering, benjamini1995controlling}, conducting an individual test for each pixel. A disagreement too big, relative to the anticipated variance of the projection, will classify that pixel as a positive result. Beforehand, the empirical null \citep{efron2004large} is used to locally normalise \citep{sage2018local, sage2003teaching} the test statistics. This is necessary because the simulation of the projection did not perfectly reproduce the obtained projection, leading to incorrect inference.

Fig.~\ref{fig:flowchart} shows a flowchart which summarises the process of the experiment and defect detection. The CAD model was used as a blueprint for the manufacture of the test sample. Projections of the test sample and the simulated projection were obtained and compared under the face of uncertainty. The uncertainty was modelled using the replicated projections. The test statistics from the comparison were filtered using the empirical null filter and converted to $p$-values. A threshold was used for the $p$-values to classify pixels as positive for defects or not.

For each position $(x,y)$, a test statistic was calculated
\begin{equation}
  Z_{x,y} =
  \dfrac{
    \text{projection}_{x,y} - \emph{aRTist}_{x,y}
  }
  {
    \sqrt{{s}^2\left[\emph{aRTist}_{x,y}\right]}
  }
\end{equation}
where ${s}^2\left[\emph{aRTist}_{x,y}\right]$ is the predicted grey value variance of pixel $(x,y)$ in the projection using \emph{aRTist} as a predictor. The variance model ${s}^2\left[\emph{aRTist}_{x,y}\right]$ has a linear relationship between the variance and the grey value \citep{ma2012varaince, yang2010noise}. A gamma distributed generalised linear model \citep{mccullagh1984generalized, nelder1972generalized} was used with an identity link. The fitting of the model was done using the held out replicated projections of the sample (see Section \ref{subsection:appratus}). These replicated projections were manually segmented so that only pixels which represent the test sample were used. This provided variance-mean data, where each pixel is a data point, which was used to fit the variance model. The variance was predicted using the grey value in the simulated projection as the predictor variable.

It was assumed that the grey values were Normally distributed, therefore the test statistics should be approximately Normal as well. Let the test statistic of the pixel at position $(x,y)$ be
\begin{equation}
Z_{x,y}\sim\normal(\mu_{x,y},\widehat{\sigma}_{0,x,y}^2)
\end{equation}
for $x=1,2,\cdots,W$ and $y=1,2,\cdots,H$. $\mu_{0,x,y}$ and $\sigma_{0,x,y}$ are the null mean and null standard deviation at position $(x,y)$. Their estimates, $\widehat{\mu}_{0,x,y}$ and $\widehat{\sigma}_{0,x,y}$ respectively, are called the empirical null mean and empirical null standard deviation. Define the null hypotheses
\begin{equation}
  H_{0,x,y}:\mu_{x,y}=\widehat{\mu}_{0,x,y}
\end{equation}
and the alternative hypotheses
\begin{equation}
  H_{1,x,y}:\mu_{x,y}\neq\widehat{\mu}_{0,x,y} \ .
\end{equation}
The empirical null filter aims to obtain the estimators $\widehat{\mu}
_{0,x,y}$ and $\widehat{\sigma}_{0,x,y}$ for all $x$ and $y$. This is based on the empirical null \citep{efron2004large} where hypothesis testing is done by comparing a test statistic with the empirical distribution of the majority of the test statistics rather than a specified null distribution.

To obtain the estimates, a circular kernel $C_r(x,y)$ with radius $r$ was centred at $(x,y)$. All the pixels captured by the circular kernel and the region of interest (ROI) were pooled together to obtain a kernel density estimate \citep{friedman2001elements, parzen1962on}
\begin{equation}
\widehat{p}_{Z_{x,y}}(z) =
\frac{1}{nh}
  \sum_{i,j\in K_{x,y}}\phi\left(
    \dfrac{z_{i,j}-z}{h}
  \right)
\end{equation}
where $K_{x,y} = C_r(x,y) \cap \text{ROI}$, $n$ is the number of pixels captured by $K_{x,y}$, $\phi(z)$ is the standard Normal density function and $h$ is the bandwidth. The bandwidth was chosen such that
\begin{equation}
  h = (0.9n^{-1/5}\ + 0.16) \times \text{min}\left(s_{x,y},\text{IQR}_{x,y}/1.34\right)
  \label{eq:inference_ourruleofthumb}
\end{equation}
where $s_{x,y}$ and $\text{IQR}_{x,y}$ are the sample standard deviation and sample interquartile range of the test statistics in $K_{x,y}$, similar to Silverman's rule of thumb \citep{sheather2004density, silverman1986density}. The estimates were obtained by solving
\begin{equation}
\widehat\mu_{0,x,y} = \argmax\widehat{p}_{Z_{x,y}}(z)
\end{equation}
using the Newton-Raphson method and
\begin{equation}
  \widehat{\sigma}_{0,x,y} = \left[
    \left.
      -\dfrac{\partial^2}{\partial z^2}\ln\widehat{p}_{Z_{x,y}}(z)
    \right|_{z=\widehat{\mu}_{0,x,y}}
  \right]^{-1/2}
\end{equation}
\citep{efron2004large}. The test statistics were normalised using
\begin{equation}
  T_{x,y} =
  \dfrac{
    Z_{x,y}-\widehat{\mu}_{0,x,y}
  }
  {
    \widehat{\sigma}_{0,x,y}
  }
\end{equation}
and converted to $p$-values
\begin{equation}
  p_{x,y} = 2(1-\Phi(|t_{x,y}|)) \ .
\end{equation}

Pixels are classified as positive for defects if its $p$-value is less than some threshold $\alpha$. It controls the per comparison error rate (PCER) \citep{benjamini1995controlling} which is the average proportion of false positives out of all tests or
\begin{equation}
\text{PCER}=\dfrac{1}{N}\expectation\left[\text{number of false positives}\right]
\end{equation}
where $N$ is the number of pixels to test. A typical choice is $\alpha=0.05$ \citep{wasserstein2019moving}. This method of thresholding is flawed because of the possibility of obtaining at least one false positive increases as the number of tests increases \citep{shaffer1995multiple}. For example, suppose none of the pixels represents a defect, then on average, 5\% of these pixels are tested positive and falsely so. This is troublesome and costly because false positives suggest a defect in areas where there are not any defects.

The false discovery rate (FDR) is the average proportion of positives which are false or
\begin{equation}
\text{FDR}=\expectation\left[\dfrac{\text{number of false positives}}{\text{number of positives}}\right]
\end{equation}
where the ratio is defined to be zero when the number of positives is zero. The Benjamini-Hochberg (BH) \citep{benjamini1995controlling} procedure was used to select the threshold such that the FDR is less than 5\%. This allows control for false positives relative to the number of positives. The BH procedure is done by ordering the $p$-values such that $p_{(1)}\leqslant p_{(2)}\leqslant \cdots \leqslant p_{(N)}$ and then using the threshold
\begin{equation}
  \alpha = \frac{0.05 k}{N}
\end{equation}
where
\begin{equation}
  k\text{ is the largest }i\text{ for which }p_{(i)}\leqslant\frac{i}{N}\alpha
  \ .
\end{equation}

\subsection{Computational Implementation}
\label{subsection:computational}

The empirical null filter was implemented using \emph{ImageJ} \citep{abramoff2004image, perez2013image, schindelin2012fiji, schneider2012nih} by modifying the existing class \texttt{RankFilters}. The class contains implementations of filters, such as the mean filter and the median filter, using a circular kernel and multiple threads. Each thread filters a row in parallel.

Line filtering was done from left to right. At the start on the far left, an initial value of the median of all the test statistics in the kernel was used. For the following pixel to the right, the empirical null mean of the neighbouring left pixel was used as the initial point. This was chosen as it was assumed the empirical null mean would vary slowly and smoothly spatially.

When filtering a pixel, the Newton-Raphson method was used to solve for $\widehat{\mu}_{0,x,y}=z$
\begin{equation}
  \dfrac{\partial}{\partial z}\ln\widehat{p}_{Z_{x,y}}(z) = 0
\end{equation}
by using the iterative procedure
\begin{equation}
  z^{(r+1)} =
  z^{(r)}
  -\dfrac{
    \left.
      \dfrac{
        \partial
      }
      {
        \partial z
      }
      \ln\widehat{p}_{Z_{x,y}}(z)
    \right|_{z = z^{(r)}}
  }
  {
    \left.
      \dfrac{
        \partial^2
      }
      {
        \partial z^2
      }
      \ln\widehat{p}_{Z_{x,y}}(z)
    \right|_{z = z^{(r)}}
  }
\end{equation}
where $z^{(0)}$ is the initial value. A convergence criteria was met when either 10 update steps were taken or when
\begin{equation}
  \log\left[\left|
    \left.
    \dfrac{
      \partial
    }
    {
      \partial z
    }
  \ln\widehat{p}_{x,y}(z)
  \right|_{z=z^{(r)}}
  \right|\right]
  <-5
\end{equation}
at the current step. This was chosen arbitrary to speed up the algorithm without losing too much accuracy. The solution is valid if
\begin{equation}
  \left.
    \dfrac{
      \partial^2
    }
    {
      \partial z^2
    }
    \ln\widehat{p}_{Z_{x,y}}(z)
  \right|_{z=\widehat{\mu}_{0,x,y}}
  < 0 \ .
\end{equation}
Further initial values were generated by sampling from $\normal(z^{(0)}, s_{x,y}^2)$. This was done multiple times until 3 valid solutions were obtained. The best solution, the one with the largest $\ln\widehat{p}_{Z_{x,y}}\left(\widehat{\mu}_{0,x,y}\right)$ out of all the different initial values, was used as the final answer.

To satisfy the condition that the empirical null mean is slowly and smoothly varying, the x-ray projection of the test sample was manually segmented further as shown in Fig.~\ref{fig:inference_segmentFurther}. It was segmented using the edges of the projection of the sample. The empirical null filter was then used on each segment, or ROI, independently, ignoring any pixels outside the ROI the filter was working on. The resulting filtered segments were stitched together to form the resulting filtered image prior to any hypothesis testing.

\subsection{Simulation of Defect Detection}
\label{subsection:simulation}

We tested if the empirical null filter can detect simulated defects from a $\SI{256}{\pixel}\times \SI{256}{\pixel}$ contaminated image. Statistically speaking, a defect is a pixel which has a value not distributed under the null distribution, but instead a alternative distribution. For example suppose $Z_{x,y}|H_{0,x,y}\sim\normal(0,1)$, then a defect would have a value distributed as $Z_{x,y}|H_{1,x,y}\sim\normal(\mu_{1},1)$ where $\mu_1\neq0$. A $\SI{30}{\pixel}\times \SI{30}{\pixel}$ square defect was simulated. A kernel can capture the entire defect if its radius is large enough.

Contamination was simulated by a linear transform of the test statistics $Z_{x,y}$. For example in this experiment, the image was multiplied by 2 and a gradient was added. The null and alternative distributions are
\begin{align}
  Z_{x,y}|H_{0,x,y}&\sim\normal(\mu_{0,x,y},2^2)
  \\
  Z_{x,y}|H_{1,x,y}&\sim\normal(\mu_{0,x,y}+2\mu_1,2^2)
\end{align}
respectively where $\mu_{0,x,y}=0.01(x-x_0)+0.01(y-y_0)$ and $(x_0,y_0)$ is the centre of the image. A simulated contaminated defected image was made by sampling $Z_{x,y}|H_{0,x,y}$ in areas with no defects and $Z_{x,y}|H_{1,x,y}$ with defects. An example of a contaminated square defected image is shown in Fig.~\ref{fig:inference_defectSquare2Example}.

The contamination will cause the test statistics to have larger variance and larger values at the two corners where $\mu_{0,x,y}$ is large. This will cause inflation in false positives.

It was suggested that, as a rule of thumb, the proportion of non-null statistics captured by the kernel should not be larger than $0.1$ to satisfy some assumptions made for the empirical null \citep{efron2004large, schwartzman2008empirical}. In this simulated example, it is about $r>\SI{54}{\pixel}$. A kernel too small would treat the defects as null, making them difficult to detect.

The receiver operating characteristic (ROC) curve \citep{cook2007use, friedman2001elements, green1966signal, hanley1982meaning, metz1978basic} is a parametric plot, displaying the true positive rate (sensitivity) against the false positive rate ($1-\text{specificity}$) for varying thresholds. The area under the ROC curve is a commonly used statistic to quantify the performance of the test \citep{friedman2001elements}. Interpretations of the area do exist \citep{hanley1982meaning, metz1978basic} and discussed thoroughly \citep{cook2007use}.

A simulation study was conducted where a simulated contaminated defected image was filtered for various kernel radiuses for $\mu_1=3$. Hypothesis testing was done on the uncontaminated defected image and filtered contaminated defected image. The area under the ROC curve (AUC) and statistical errors were recorded. This was repeated 100 times by simulating another image and repeating the measurement. To form a baseline to compare with, the results of the uncontaminated defected image were all pooled together to obtain the empirical distribution of the AUC and statistical errors without contamination.

\section{Results}

In the simulation study, defect detection was conducted on a filtered contaminated defected image to investigate the statistical properties of the method. The aim of the filter was to remove the contamination so that sensible inference was achieved. The results from the simulation is shown Fig.~\ref{fig:inference_Experiment_DefectRadiusSquare}.

The area under the ROC curve (AUC) increased with kernel radius which highlighted that a filtered image with a suitable radius kernel can perform almost as well as though there was no contamination. The rule of thumb (see Section \ref{subsection:simulation}) suggest a kernel radius of $r>\SI{54}{\pixel}$ which achieved optimal AUC thus justifies the use of the rule of thumb.

It was observed that there is a kernel radius which minimised the type 2 error. A large kernel radius helped preserve false discovery rate (FDR) control after filtering in these examples but a kernel radius too large can lose statistical power. Keeping the FDR consistent before and after filtering is desirable so it is controlled at a specified level after filtering. For large enough kernel radius, the FDR was controlled at and fluctuates around 5\%.

The running times for the empirical null filter increased with kernel radius. Compared to typical filters, say the mean and variance filter, the empirical null filter is a number of magnitudes slower. The main bottleneck is the evaluation of the density estimate, each step in the Newton-Raphson required the evaluation of $\pi r^2$ data points.

The resulting inference from the test sample is shown in Fig.~\ref{fig:inference_subroiAbsFilterDeg120_inference_radius4}. The larger defects were successfully highlighted by the hypothesis test. The empirical null mean changed face to face, resulting in a clear boundary set by the edges. This suggests that the contamination or inaccuracy from \emph{aRTist} was face dependent.

\section{Conclusion}

Voids in the order of millimetres were detectable which shows potential in defect detection in projection space. This work is a contribution towards real-time quality control in AM.

The empirical null filter catered for inaccuracies made by \emph{aRTist}. This demonstrated that the filter can adjust the null hypotheses according to the data to make a sensible inference. In the experiments, it was found that the FDR level is preserved after filtering.

False positives do occur but this is unavoidable because tests were done at the 5\% FDR level. Typically in the experiments, false positives were isolated single pixels and probably occurred due to random chance. Clusters of positive pixels should raise suspicion. One could create a binary image, assigning a Boolean value whether that pixel was tested positive or not. A binary image filter, such as erode followed by a dilate, can be used to remove isolated positive pixels to emphasise the cluster of positive pixels.

For good results, the faces of the test sample were segmented. This was necessary because the empirical null filter assumes that the null parameters varied spatially smoothly and slowly. Manual segmentation was easy to do because a cuboid has 6 faces. However, segmentation of faces cannot be generalised well to AM products with complicated geometry and curved surfaces.

In the experiment, the angle was chosen to distinguish the designed voids. For general AM products, finding a suitable singular projection angle can be difficult. This problem can be tackled by conducting defect detection on a small number of different angular projections independently to pick up defects hidden behind another.

For a filter to take more than a minute is unacceptable when used on the production line when typical filters can take a fraction of a second. There are a few strategies to accelerate the filter. The empirical null filter may be accelerated using GPUs \citep{eklund2013medical, hwu2011gpu, yang2008parallel}, however efforts to implement the filter in \emph{CUDA} and \emph{C++} is fruitless if there may exist a faster method. Instead, accuracy may be sacrificed for speed by only estimating the null parameters for a number of regularly spaced pixels, the remaining pixels then do estimation through interpolation. The problem is that the assumption of a slow varying null parameter is required much more because of the use of interpolation, this may cause bigger problems due to the face to face transition observed in the experiments. It was found that if a bad solution was found for a point, then that solution would spread its bad solution to neighbour pixels due to the interpolation.

The main bottleneck is the evaluation of the density estimate, each step in the Newton-Raphson required the evaluation of each pixel in the kernel. Perhaps there may be faster methods for density estimation such as fitting a smoothing spline on the histogram \citep{efron2004large}, however, that would require tuning the histogram bins as well as the tuning parameters for the spline. On the other hand, there are methods which use the histogram count and it was found that the estimation of the null parameters was insensitive to the histogram binning \citep{schwartzman2008empirical}.

Estimation of the null parameters is essentially robust statistics, estimating the parameters of the null distribution without being affected by non-null statistics. Potential faster methods than the empirical null could exist in literature \citep{hampel1986robust, huber2009robust, jewson2018principles, maronna2006robust, rousseeuw1987robust}. The EM algorithm \citep{bishop2006pattern, dempster1977maximum} could be used to fit a mixture of Gaussians to identify the null distribution and estimate its parameters. However, the power of hypothesis testing and the empirical null comes from the fact that the alternative distribution does not need to be specified.

\section{Appendix}
Code which reproduces this paper can be found on the \emph{GitHub} page \url{https://github.com/shermanlo77/oxwasp_insideout}.

\section{Acknowledgements}
This work is funded by EPSRC grant numbers EP/K031066/1 (Inside-out: Statistical methods for Computed Tomography validation of complex structures in Additive Layer Manufacturing) and EP/L016710/1 (EPSRC and MRC Centre for Doctoral Training in Next Generation Statistical Science: The Oxford-Warwick Statistics Programme). We would like to thank Wilfred S.~Kendall for his role as principal investigator.

\settocbibname{References}
\bibliographystyle{acm}
\bibliography{../bib}

\setlofname{Figure Captions}
\listoffigures
\clearpage
\section{Figures}

\begin{figure}[h]
  \centering
  \includegraphics[width=0.49\textwidth]{../figures/inference/TestObjectBW.png}
  \caption{The CAD model of the test sample with voids. Six voids of each diameter, \SI{2.4}{\milli\metre}, \SI{1.2}{\milli\metre}, \SI{0.6}{\milli\metre} and \SI{0.3}{\milli\metre}, were designed in the CAD model. Voids with diameters \SI{2.4}{\milli\metre} and \SI{0.6}{\milli\metre} were regularly arranged, the other ones were arranged irregularly. The scale shown is approximate.}
  \label{fig:inference_testObject}
\end{figure}

\clearpage

\begin{figure}
  \centering
  \includegraphics[width=0.7\textwidth]{../figures/flowchart.pdf}
  \caption{Flowchart showing the process of obtaining and comparing a projection of the test sample with the simulated projection.}
  \label{fig:flowchart}
\end{figure}

\begin{figure}
  \centering
  \includegraphics[width=0.49\textwidth]{../figures/inference/DefectExampleSquare40_imageContaminatedBW.eps}
  \caption{A $\SI{256}{\pixel}\times \SI{256}{\pixel}$ contaminated Gaussian image with a $\SI{30}{\pixel} \times \SI{30}{\pixel}$ square defect. The null and alternative distributions are $Z_{x,y}|H_{0,x,y}\sim\normal(\mu_{0,x,y},2^2)$ and $Z_{x,y}|H_{1,x,y}\sim\normal(\mu_{0,x,y}+6,2^2)$ respectively where $\mu_{0,x,y}=0.01(x-x_0)+0.01(y-y_0)$ and $(x_0,y_0)$ is the centre of the image.}
  \label{fig:inference_defectSquare2Example}
\end{figure}

\clearpage

\begin{figure}
  \centering
  \includegraphics[width=0.49\textwidth]{../figures/inference/segmentBW.eps}
  \caption{The $z$ image was segmented further into 7 segments shown by the dashed lines.}
  \label{fig:inference_segmentFurther}
\end{figure}

\clearpage

\begin{figure}
  \centering
    \centerline{
      \begin{subfigure}{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectRadiusSquare_rocBW.eps}
        \caption{AUC}
      \end{subfigure}
      \begin{subfigure}{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectRadiusSquare_type1BW.eps}
        \caption{Type 1 error}
      \end{subfigure}
    }
    \centerline{
      \begin{subfigure}{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectRadiusSquare_type2BW.eps}
        \caption{Type 2 error}
      \end{subfigure}
      \begin{subfigure}{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectRadiusSquare_fdrBW.eps}
        \caption{FDR}
      \end{subfigure}
    }
  \caption{Area under the ROC curve (AUC) and statistical errors obtained when conducting a hypothesis test on a $\SI{256}{\pixel} \times \SI{256}{\pixel}$ filtered contaminated Gaussian image with a $\SI{30}{\pixel}\times \SI{30}{\pixel}$ square defect for Various kernel radiuses. The dashed lines show the resulting 95\% empirical confidence interval when the hypothesis test was done on the uncontaminated image with a defect. The null and alternative distributions are $Z_{x,y}|H_{0,x,y}\sim\normal(\mu_{0,x,y},2^2)$ and $Z_{x,y}|H_{1,x,y}\sim\normal(\mu_{0,x,y}+6,2^2)$ respectively where $\mu_{0,x,y}=0.01(x-x_0)+0.01(y-y_0)$ and $(x_0,y_0)$ is the centre of the image. The test was conducted at the 5\% FDR level. The box plots summarise the 100 repeated measurements of the simulation.}
  \label{fig:inference_Experiment_DefectRadiusSquare}
\end{figure}

\clearpage

\begin{figure}
  \centering
    \centerline{
      \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectDetectSubRoiAbsFilterDeg120_radius4_sigBW.eps}
        \caption{X-ray projection with positive pixels}
      \end{subfigure}
      \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectDetectSubRoiAbsFilterDeg120_radius4_logpBW.eps}
        \caption{$-\log p$}
      \end{subfigure}
    }
    \centerline{
      \begin{subfigure}{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectDetectSubRoiAbsFilterDeg120_radius4_nullMeanBW.eps}
        \caption{Empirical null mean}
      \end{subfigure}
      \begin{subfigure}{0.49\textwidth}
        \includegraphics[width=\textwidth]{../figures/inference/DefectDetectSubRoiAbsFilterDeg120_radius4_nullStdBW.eps}
        \caption{Empirical null std}
      \end{subfigure}
    }
  \caption{Resulting inference when using the empirical null filter, on each segment independently, with radius $r=\SI{130}{\pixel}$. a) X-ray projection with highlighted white positive pixels at the 5\% FDR level. b) $p$-values on the log scale. c) and d) Empirical null mean and standard deviation.}
  \label{fig:inference_subroiAbsFilterDeg120_inference_radius4}
\end{figure}

\end{document}


\section{Tricks}

``Every object in the problem's universe is important, and should be
considered when approaching the problem.''

\begin{itemize}
    \item permutation type problem
    \item do there exist...
    \item proving identities
    \item Dunno
    \item divisibility by primes and prime divisors stuff
\end{itemize}

\begin{enumerate}
    \item Add. Everything. Up.
    \item Infinitude of primes:
        \begin{enumerate}
            \item Eulerian infinitude trick
            \item For large enough numbers, there is a larger prime divisor
            \item Assuming contradiction, if there are any number co-prime to
                the product of the primes, then that must be $ 1 $.
        \end{enumerate}
\end{enumerate}

\Faka\subsection{Digit Sum or Product}

When dealing with the sum of the digits or the product of them, to find the
construction it is very important to consider $ 0 $ and $ 1 $'s in the number.

\Faka\subsection{Diophantine Equations}

\begin{enumerate}
    \item finding some solutions
    \item trying modular cases
    \item making some variables depended on other variables
    \item putting constrains on variables which would make the problems easier
    \item if there are infinitely many solutions, can you find a construction?
    \item factorize (this is BIG)
    \item In these problems, investigation, induction, recursion,
        constructions etc. are essentials
\end{enumerate}

\Faka\subsection{Sequences}

\Faka\subsection{NT Functions}

\Faka\subsection{Construction Problems}

\Faka\subsection{Sets satisfying certain properties}

\Faka\subsection{Other Small Techniques to Remember}

\begin{enumerate}
    \item $ a-b $ stays invariant upon addition, just as $ \frac{a}{b} $ stays
        invariant upon multiplication.
\end{enumerate}

\newpage\section{Lemmas}

\lem{Maybe not useful at all}{
    Let $p\geq 5$ be a prime number. Prove that if
    $p\mid a^2+ab+b^2$, then \[p^3\mid {a+b}^p-a^p-b^p\]
}

\lem{Non-zero digits in base $ b $}{
    If $b\geq 2$ and $b^n-1\mid a$ then there exist at least $n$ non-zero
    digits in the representation of $a$ in base $b$
}

\theo{https://en.wikipedia.org/wiki/Coin_problem}{Frobenius Coin Problem, extended Chicken McNugget}{
    Given a bunch of coprime integers, one can write all integers after a
    certain limit as a linear combination of these integers.\\

    Formally, for any integers $a_1, a_2 \dots a_n$ such that $\gcd(a_1,
    a_2\dots a_n)=1$, there exists positive integers $m$ such that for any
    integer $M\geq m$, there are non-negative integers $b_1, b_2 \dots b_n$
    such that \[\sum_{i=1}^{n}b_ia_i=M\]
}

\begin{Remark}
    If $n=2$ then $m=a_1a_2-a_1-a_2$.\\

    If $n\geq 2$, then there doesn't exist an explicit formula, but if
    $\{a_i\}$ are in arithmetic progression, ($a_i=a_1+(i-1)d$) then
    \[m=\floor*{\frac{a-2}{n-1}}a + (d-1)(a-1) - 1\]
\end{Remark}

\thmbox{https://en.wikipedia.org/wiki/Beatty_sequence}{Beatty's Theorem or
    Rayleigh Theorem}{
    If $a, b$ are two irrational numbers such that $\dfrac{1}{a}+\dfrac{1}{b}=1$,
    then the two sets $\left\{\floor*{ia}\right\}$ and
    $\left\{\floor*{ib}\right\}$, where $i$ are the positive integers, form a
    partition of the set of natural numbers.
}

\rem{
    Express the two conditions in numeric terms assuming the contrary.
}

\proof{
    \textbf{No integer belongs to both sets:} \\

    Suppose there exists some $ j, k,
    m $ such that $ j = \floor{ka} = \floor{mb} $. Then the inequalities will
    hold:
    \[j\le ka < j+1 \text{ and } j\le mb < j+1\]
    Which leads to
    \[j<ka<j+1 \text{ and } j<mb<j+1\]
    From there, it follows that
    \[\frac{j}{a}<k<\frac{j+1}{a} \text{ and } \frac{j}{b}<m<\frac{j+1}{b}\]
    Adding them gives
    \[j<k+m<j+1\]
    Which is false.\\[1em]

    \textbf{Every integer belongs to one set:} \\

    Suppose there is are integers $
    j, k, m $ such that $ \floor{ka} <j <\floor{ka+a} $ and $ \floor{mb} <j
    <\floor{mb+b} $
    Which can be written as
    \[ka<j<ka+a-1 \text{ and } mb<j<mb+b-1\]
    It follows from here that,
    \[k<\frac{j}{a}<k+1-\frac{1}{a} \text{ and } m<\frac{j}{b}<m+1-\frac{1}{b}\]
    Adding them gives us
    \[k+m<j<k+m+1\]
    Which, again, is absurd. So the theorem is proved.
}

\lem{}{
    Let $x,y$ be co-prime. Then
    \[\gcd(z,xy) = \gcd(z,x)\gcd(z,y) = \gcd(z \bmod x,x)\gcd(z \bmod y,y)\]
    \[\implies\gcd(r(a,b),xy))=\gcd(a,x)\gcd(b,y)\]

    (here $r(a,b)$ denotes the smallest integer that satisfies $r(a,b) \equiv
    a \bmod x,\ r(a,b) \equiv b \bmod y$)
}

\lem{NT version of Erdos Theorem}{
    Let $0 < a_1 < a_2 < · · · < a_{(mn+1)}$ be $mn + 1$ integers. Prove that you
    can select either $m + 1$ of them no one of which divides any other, or $n +
    1$ of them each dividing the following one.
}

\theo{https://math.stackexchange.com/questions/1019538/primes-dividing-a-polynomial}{Prime
    divisors of an integer polynomial}{
    If $ P(x)\in \Z[x] $, then the set of primes, $ P = \{ p: p\mid P(x) \} $ is infinite.
}

\newpage
\subsection{Modular Arithmatic Theorems and Useful Results}

\theo{https://en.wikipedia.org/wiki/Wolstenholme's_theorem}{Wolstenholme's Theorem}{
    For all prime $ p $ the following relation is true:
    \[ p^2 \ \bigg|\ 1 + \frac{1}{2} + \frac{1}{3} \dots \frac{1}{p-1} =
    \sum_{i=1}^{p-1} \frac{1}{i} \]
}

\coro{
    \[ p \ \bigg|\ 1 + \frac{1}{2^2} + \frac{1}{3^2} \dots \frac{1}{(p-1)^2} =
    \sum_{i=1}^{p-1} \frac{1}{i^2} \]
}

\coro{
    If $ p > 3 $ is a prime, then
    \[\binom{2p}{p} \equiv 2\ (mod\ p^3)\]
    \[\binom{2p-1}{p-1} \equiv 1\ (mod\ p^3)\]
}

\lem{}{
    \[ \frac{1}{(p-i)!} \equiv (-1)^i (i-1)!\ (mod\ p) \]
    \[ \binom{p-1}{k} \equiv (-1)^k\ (mod\ p) \]
}

\prob{}{}{E}{
    \[\binom{p^{n+1}}{p} \equiv p^n\ (mod\ p^{2n+3})\]
}


\section{Conclusions}\label{sec:conclusion}

We proposed a novel Bayesian preference learning approach
for modelling both the preferences of individuals
and the overall consensus of a crowd.
Our model learns the latent utilities of items from pairwise comparisons
using a combination of Gaussian processes and Bayesian matrix factorisation
to capture differences in  opinion.
We introduce a stochastic variational inference (SVI) method, that,
unlike previous work, can scale to arbitrarily large datasets,
since its time and memory complexity do not grow with the dataset size.
Our experiments confirm the method's scalability and
 show that jointly modelling the consensus and personal
preferences can improve predictions of both.
Our approach performs competitively
against less scalable alternatives
%despite the approximations required for SVI.
and improves on
the previous state of the art
for predicting argument convincingness from crowdsourced data~\citep{simpson2018finding}.
%far better than previous
%Gaussian process preference learning methods
%without harming
%performance on individual utility prediction and significantly improving performance
%on consensus learning.

Future work will investigate learning inducing point locations and
optimising length-scale hyperparameters by maximising
the variational lower bound, $\mathcal L$,
 as part of the variational inference method.
Another important direction will be to generalise the likelihood from pairwise comparisons
to comparisons involving more than two items~\citep{pan2018stagewise}
or best--worst scaling~\citep{kiritchenko2017best}
to provide scalable Bayesian methods for other forms of comparative preference data.
%and investigate the possibility of integrating deep generative models
%for learning feature representations from input data.
%the models can readily be adapted to regression or classification tasks by swapping out the preference likelihood, resulting in
%different values for $\bs G$ and $\bs H$.
% Preference elicitation using information theoretic methods.


% Created 2016-01-20 Wed 11:06
\documentclass{scrartcl}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{fixltx2e}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{float}
\usepackage{wrapfig}
\usepackage{soul}
\usepackage{textcomp}
\usepackage{marvosym}
\usepackage{wasysym}
\usepackage{latexsym}
\usepackage{amssymb}
\usepackage{hyperref}
\tolerance=1000
\usepackage{khpreamble}
\usepackage{subfigure}
\providecommand{\alert}[1]{\textbf{#1}}

\title{Computerized control - homework 1}
\author{Kjartan Halvorsen}
\date{Due 2016-01-22}
\hypersetup{
  pdfkeywords={},
  pdfsubject={},
  pdfcreator={Emacs Org-mode version 7.9.3f}}

\begin{document}

\maketitle

\section{Exercise}
\label{sec-1}
\subsection{Block-diagram calculation}
\label{sec-1-1}

   The block-diagram below shows a so-called \emph{two-degrees-of-freedom} feedback control system. Calculate the transfer function from each of the signals  $u_c$ (command signal), $d$ (disturbance signal) and $n$ (measurement noise) to the system output $y$.

   \begin{center}
   \includegraphics[width=0.6\linewidth]{2dof-block-complete}
   \end{center}
\subsection{Solution to a first-order system}
\label{sec-1-2}

   Consider the first-order system
   \[ \dot{x}(t) = -x(t) + u(t) \]

\begin{enumerate}
\item Write the solution to the system for the input signal
      \begin{displaymath}
        u(t) = \begin{cases} 1, & 0 \le t \le 1\\ 0, & \text{otherwise}.
      \end{displaymath}
      The initial value is $x(t) = 0$.
\item Sketch the solution $x(t)$ for $0\le t \le 4$, (or generate the plot on the computer).
\item On Blackboard you can find a simulink-file with an implementation of the system. Use this to verify your solution. Include a screen-dump of the simulation output in your report.
\end{enumerate}
\section{Solution}
\label{sec-2}
\subsection{Block-diagram}
\label{sec-2-1}

   One alternative is to use \href{https://en.wikipedia.org/wiki/Mason%2527s_gain_formula}{Mason's rule}. Note that the denominator of the closed-loop transfer functions will be
   \[ 1 + GF_b. \] The numerators are given by the direkt path from the respective input to the output, so we get
   \begin{align*}
   \frac{Y}{U_c} &= \frac{GF_f}{1+GF_b}\\
   \frac{Y}{D} &= \frac{1}{1+GF_b}\\
   \frac{Y}{N} &= -\frac{GF_b}{1+GF_b}
   \end{align*}

   The other alternative is to compute directly in the given block-diagram. We get
   \[ Y = D + G\big(F_fU_c - F_b(N+Y)\big). \]
   Moving terms with $Y$ on the left hand side we get
   \[ Y+GF_bY = D + GF_fU_c - GF_bN. \]
   Deviding by \(1+GF_b\) on both sides gives
   \[ Y = \frac{1}{1+GF_b}D + \frac{GF_f}{1+GF_b}U_c - \frac{GF_b}{1+GF_b}N. \]
\subsection{First-order system}
\label{sec-2-2}

\begin{enumerate}
\item The system can be solved by direct integration of the differential equation, or using the Laplace transform.

      Using Laplace, we write the input as the sum of two step signals, of which the second is delayed. The solution is then given by summing the solution to each of the two step signals. Write the input as
      \[ u(t) = u_1(t) + u_2(t), \]
      with
      \begin{align*}
          u_1(t) &= \begin{cases} 1, t \ge 0 \\ 0, & \text{otherwise} \end{cases}\\
          U_1(s) &= \frac{1}{s}.\\
          u_2(t) &= -u_1(t-1) = \begin{cases} -1, t \ge 1 \\ 0, & \text{otherwise} \end{cases}\\
          U_2(s) &= -\mexp{-s}U_1(s) = \frac{\mexp{-s}}{s}
      \end{align*}
      Solving using the Laplace transform we get
      \[ sX = -X + U_1 + U_2\]
      which gives
      \begin{align*}
        X &= \frac{1}{s+1}\left( \frac{1}{s} - \frac{\mexp{-s}{s} \right)\\
          &= \frac{1}{(s+1)s} - \mexp{-s} \frac{1}{(s+1)s},
      \end{align*}
      The first term is a unit step-response, and the second term is also a unit step response, but delayed and with negative sign. The step-response for the system is given by
      \[ x_1(t) = h(t)(1-\mexp{-t}), \]
      where $h(t)$ is the Heaviside step-function. So the solution is
      \[ x(t) = x_1(t) + x_2(t) = x_1(t) - x_1(t-1) = h(t)(1 -\mexp{-t}) - h(t-1)\big(1 - \mexp{-(t-1)}\big). \]

      We can also solve the differential equation by integration using the solution
      \[ x(t) = x(0)\mexp{-t} + \int_0^t \mexp{-(t-\tau)} u(\tau) d\tau \]
      Since \(u(t)\) is equal to 1 for \(0\le t< 1\) and zero elsewhere, the integration is quite simple. For $t<1$ we get (since \(x(0) = 0\))
      \[ x(t) = \int_0^t \mexp{-t+\tau}d\tau
              = \int_{-t}^{0} \mexp{\tau'} d\tau' = \left[ \mexp{\tau'} \right]_{-t}^{0} = 1-\mexp{-t}. \]
      And for $t>1$ we get
      \[ x(t) = \int_0^1 \mexp{-t+\tau}d\tau
              = \int_{-t}^{-t+1} \mexp{\tau'} d\tau' = \left[ \mexp{\tau'} \right]_{-t}^{-t+1} = \mexp{-t+1}-\mexp{-t}. \]
      So,
      \[ x(t) = \begin{cases} 0, & t \le 0\\
                              1-\mexp{-t}, & 0<t\le 1\\
                              \mexp{-t+1}-\mexp{-t}, & t>1 \end{cases} \]
\item Plot of the solution
      \begin{center}
      \includegraphics{first-order-response-bb}
      \end{center}
\item Simulink-model with simulation
      \begin{center}
      \includegraphics[width=0.5\linewidth]{first-order-simulink}
      \end{center}
\end{enumerate}

\end{document}