If you haven't downloaded and unzipped \href{https://libaoj.in/courses/2021f/MATH3341/zip/Math.3341.zip}{\texttt{Math.3341.zip}}. Download and unzip it under \verb|H:| (H Drive if you are working on the Remote Lab). Change the current working directory by typing \verb|cd H:\Math.3341\Math.3341.Lab.02| in the Command Window, and type \verb|edit lab_02_script| in the Command Window to edit \verb|lab_02_script.m|.

%---------------------------------------------
\section{1-D Array: Vector}
%---------------------------------------------
\begin{enumerate}[(a)]
  \item Define two evenly spaced row vectors: \verb|vec1| of which elements start from $1$ to $9$ with $9$ entries and \verb|vec2| that is between $18$ and $1$ with step size $-2$ using \verb|linspace| and \verb|colon|, respectively. Then store lengths of \verb|vec1| and \verb|vec2| to \verb|vec1Length| and \verb|vec2Length|.
  \item Calculate the following products:
    \begin{itemize}
      \item Store the product of all elements of \verb|vec1| to \verb|vec1Product|.
      \item Calculate the elementwise product of \verb|vec1| and \verb|vec2|, then assign the result to \verb|vecProduct|.
      \item Calculate the dot product of \verb|vec1| and \verb|vec2| in three ways using \verb|dot|, matrix multiplication, and \verb|sum| with elementwise multiplication. Store the results to \verb|dotProduct1|, \verb|dotProduct2|, and \verb|dotProduct3|.
    \end{itemize}
\end{enumerate}

%---------------------------------------------
\section{2-D Array: Matrix}
%---------------------------------------------
\begin{enumerate}[(a)]
  \item Verify that column sums, row sums, and main diagonal sum of a magic square matrix are equal. Define \verb|mat1| to be a $3 \times 3$ magic square matrix. Calculate the column sums \verb|mat1ColSum|, row sums \verb|mat1RowSum|, main diagonal sum \verb|mat1DiagSum| using \verb|sum|.
  \item Create a $3 \times 3$ matrix \verb|mat2| $= \begin{bmatrix} 1 & 2 & 3 \\ 4 & 5 & 6 \\ 7 & 8 & 9 \end{bmatrix}$ by reshaping \verb|vec1| using \verb|reshape| and \verb|transpose|. Then calculate \verb|matProduct1| which is the product of \verb|mat1| and \verb|mat2| using matrix multiplication, and \verb|matProduct2| using elementwise multiplication. Observe the difference between these two multiplications.
  \item Stack \verb|matProduct1| and \verb|matProduct2|, then assign it to \verb|mat3| of which the size is $6 \times 3$. Compute the sum \verb|sumAll| of all elements of \verb|mat3|, column minimums \verb|mat3ColMin|, and row maximums \verb|mat3RowMax| of \verb|mat3|. Find the row indices \verb|rowIndex| and column indices \verb|colIndex| of entries that are \emph{no greater than} $20$ in \verb|mat3|.
  \item Add three new columns to \verb|mat3|:
    \begin{itemize}
      \item Extract the second column of \verb|mat3| and assign it as the fourth column of \verb|mat3|.
      \item Append the vector \verb|int8(rand(6, 1)) * 255)| to \verb|mat3| by specifying the column index to be \verb|end + 1|.
      \item Use bracket to append the vector \verb|colon(0,36,200)| to \verb|mat3| as the last column.
      \item Check the size of \verb|mat3| and store it to \verb|mat3Size|.
    \end{itemize}
\end{enumerate}

%---------------------------------------------
\section{Array: Char Array vs. String Array}
%---------------------------------------------
\begin{enumerate}[(a)]
  \item Define \verb|helloChar|, \verb|worldChar|, \verb|helloString|, \verb|worldString| to be \verb|'hello '|, \verb|'world'|, \verb|"hello "|, \verb|"world"|, respectively. Concatenate the above strings as below:
    \begin{itemize}
      \item Use bracket to concatenate \verb|helloChar| and \verb|worldChar| horizontally and assign it to \verb|helloWorldChar1|. Repeat this to \verb|helloString| and \verb|worldString|, then store the result to \verb|helloWorldString1|.
      \item Now use \verb|strcat| instead of bracket (use \verb|help strcat| to check out the syntax) to repeat the above. Store the results to \verb|helloWorldChar2| and \verb|helloWorldString2|.
    \end{itemize}
\item Use \verb|class| and \verb|length| to determine the types and lengths of \verb|helloWorldChar1|, \verb|helloWorldChar2|, \verb|helloWorldString1|, \verb|helloWorldString2|. Create variables with suffix \verb|Class| or \verb|Length| to store the corresponding results. For example, the length of \verb|helloWorldChar1| should be stored as \verb|helloWorldChar1Length|.
\end{enumerate}

%---------------------------------------------
\section{Application: Image Processing}
%---------------------------------------------
Next we will process Figure \ref{fig:a} to produce Figure \ref{fig:b} in MATLAB. For this portion, suppress the output if it is more than 10 lines by putting \verb|;| at the end of the statement.
\begin{figure}[!hbtp]
  \centering
  \begin{subfigure}[b]{0.25\textwidth}
    \centering
    \includegraphics[width=80pt]{../pics/UW_gray.png}
    \caption{\texttt{UW_gray.png}}
    \label{fig:a}
  \end{subfigure}
  \begin{subfigure}[b]{0.25\textwidth}
    \centering
    \includegraphics[width=80pt]{../pics/UW_gray_new.png}
    \caption{\texttt{UW_gray_new.png}}
    \label{fig:b}
  \end{subfigure}
  \caption{UWyo Logo}
\end{figure}

\begin{enumerate}[(a)]
  \item Read \verb|UW_gray.png| and store it to \verb|uwGray| using \verb|imread|. Check the size of \verb|uwGray| and store the size to \verb|uwGraySize|. Find the maximal eigenvalue of the matrix \verb|double(uwGray)| and store it to \verb|maxEigenvalue|.
  \item Crop the Steamboat out of the logo by extracting a submatrix from \verb|uwGray| which starts from row 1 to row 650 and from column 171 to column 650, and save it to \verb|steamboatLeft|. Flip \verb|steamboatLeft| and store it to \verb|steamboatRight|. Then create \verb|steamboat| by concatenating matrix \verb|steamboatLeft| and \verb|steamboatRight|.
  \item Extracting the name \verb|uwName| from \verb|uwGray| which starts from row 651 to row 960 (with all columns). Stack \verb|steamboat| and \verb|uwName| to create \verb|uwGrayNew|. Check the size of \verb|uwGrayNew| which should be $960 \times 960$. If it is correct, write \verb|uwGrayNew| to a file named \verb|UW_gray_new.png| using \verb|imwrite|. List all the variables in the workspace using \verb|whos|.
\end{enumerate}

% %---------------------------------------------
% \section{Defining Variables}
% %---------------------------------------------
% \begin{enumerate}[(a)]
%     \item Define the variable \verb|old_sin_pi = sin(pi)|, then define \verb`pi = 1.25`, and compute \verb`new_sin_pi = sin(pi)`. Compare \verb|old_sin_pi| and \verb|new_sin_pi|.
%     \item Now define \verb`sin = 2.1`. Use \verb`who` and then \verb`whos` to display the list of your currently used variables. Then, evaluate \verb`sin(pi)` in the \verb|try-catch| block.
%     \item Assign \verb`5 + i`, \verb`6 + j` to \verb`a`, \verb`b`, respectively. Then perform summation to \verb`a` and \verb`b` and assign the result to \verb`c`.
% \end{enumerate}
% %---------------------------------------------
% \section{Arrays: Vectors \& Matrices}
% %---------------------------------------------
% \begin{enumerate}[(a)]
%     \item Use \verb`clear` command to clear the variab:sp les in the Workspace.
%     \item Use \verb`linspace` to create a vector $x$ with 10 entries ranging from $0$ to $2\pi$, then assign the transpose of \verb`x` to the variable named \verb`x_transpose` using either \verb`'` or \verb`transpose`. Then use the function \verb`length` to find the length to \verb`x_transpose`.
%     \item Use either \verb`:` or \verb`colon` to create a column vector \verb`v`, of which the range is from 2 to 25 with step size 2. Then use \verb`reshape` to change \verb`v` to a $3 \times 4$ matrix \verb`V`. Find the size of \verb`V` using \verb`size`.
%     \item Define the following two matrices,
%         $$
%         A =
%         \begin{bmatrix}
%             1 & 2 \\
%             3 & 4 \\
%         \end{bmatrix}, \quad
%         B =
%         \begin{bmatrix}
%             5 & 6 \\
%             7 & 8 \\
%         \end{bmatrix}.
%         $$
%         Then store the result of matrix multiplication \verb`A * B` to \verb`C` and the result of element-wise multiplication \verb`A .* B` to \verb`D`. Observe the difference between these two operations.
%     \item Create a $5 \times 5$ magic square matrix \verb`M`. Extract the submatrix of \verb`M` (from row $2$ to row $4$ and from column $3$ to column $5$), and store the submatrix to \verb`M_submatrix`. Then create a vector \verb`M_last_col` by extracting the last column of \verb|M|.
%     \item Create a $4 \times 4$ identity matrix \verb`I` using \verb`eye`, a $6 \times 3$ all-one matrix \verb`N` using \verb`ones`, a $3 \times 4$ all-zero matrix \verb`O` using \verb`zeros`.
% \end{enumerate}

In the Command Window enter the command \verb`diary('lab_02_output.txt')`, run the script file \verb|lab_02_script.m|, then type \verb`diary off` to store the output to \verb`lab_02_output.txt`. Then upload the script file \verb|lab_02_script.m|, output file \verb|lab_02_output.txt|, and \verb|UW_gray_new.png| to the folder \verb|src| on Overleaf. Next open \verb|body.tex| under the folder \verb|LaTeX|. In the last section of the report, you will reproduce the following using \LaTeX{}. Recompile, and submit the generated \verb|.pdf| file to WyoCourses.
%
\newpage

\section{Basics of \LaTeX{}}
\label{sec:bol}
\subsection{Sine functions}
For given $x \in [0, 2\pi]$ with step size $\pi/12$, we can obtain the evaluations of \eqref{eq:sine} at $x$ (see Table \ref{tab:sine}), and the corresponding plot (see Figure \ref{fig:sine}).

\begin{equation}
  \label{eq:sine}
  \begin{cases}
    y_1 = \sin(x/2) \\
    y_2 = \sin(x)   \\
    y_3 = \sin(2x)
  \end{cases}
\end{equation}
\begin{table}[!hbtp]
\centering
\caption{Sine functions}
\label{tab:sine}
\begin{tabular}{ccrr}
\toprule
        $x$ & $\sin(x/2)$ &   $\sin(x)$ &  $\sin(2x)$ \\
\midrule
$0$      & $0$          & $0$  & $0$ \\
$\pi/2$  & $\sqrt{2}/2$ & $1$  & $0$ \\
$\pi$    & $1$          & $0$  & $0$ \\
$3\pi/2$ & $\sqrt{2}/2$ & $-1$ & $0$ \\
$2\pi$   & $0$          & $0$  & $0$ \\
\bottomrule
\end{tabular}
\end{table}
\begin{figure}[!hbtp]
  \centering
  \includegraphics[width=0.3\textheight]{./fig/sine.pdf}
  \caption{Sine functions}
  \label{fig:sine}
\end{figure}

\subsection{Goldbach's Conjecture}
Pursuing this type of analysis more carefully, Hardy and Littlewood in 1923 conjectured (as part of their famous \textsl{Hardy–Littlewood prime tuple conjecture}) that for any fixed $c \geq 2$, the number of representations of a large integer $n$ as the sum of $c$ primes $n = p_1 + \cdots + p_{c}$ with $p_1 \leq \cdots \leq p_c$ should be asymptotically equal to
\begin{equation}
    \label{eq:hardy}
    \left( \prod_{p} \frac{p \gamma_{c,p} (n)}{(p - 1)^c}\right) \int_{2 \leq x_1 \leq \cdots \leq x_c: x_1 + \cdots + x_c = n} \frac{d x_1 \cdots d x_{c - 1}}{\ln{x_1} \cdots \ln{x_c}},
\end{equation}
where the product is over all primes $p$, and $\gamma_{c, p}(n)$ is the number of solutions to the equation $n = q_1 + \cdots + q_c \mod p$ in modular arithmetic, subject to the constraints $q_1, \ldots, q_c \ne 0 \mod p$. This formula \eqref{eq:hardy} has been rigorously proven to be asymptotically valid for $c \geq 3$ from the work of Vinogradov, but is till only a conjecture when $c = 2$. In the latter case, the above formula simplifies to $0$ when $n$ is odd, and to
$$
2 \Pi_2 \left( \prod_{p|n; p \geq 3} \frac{p - 1}{p - 2} \right) \int_{2}^{n} \frac{dx}{(\ln{x})^2} \approx 2 \Pi_2 \left( \prod_{p|n; p \geq 3} \frac{p - 1}{p - 2} \right) \frac{n}{(\ln{n})^2},
$$
when $n$ is even, where $\Pi_2$ is Hardy-Littlewood's twin prime constant
$$
\Pi_2 := \prod_{p \geq 3} \left( 1 - \frac{1}{(p - 1)^2} \right) = 0.6601618158\ldots
$$
This sometimes known as the \textsf{extended Goldbach conjecture}.

\emph{Reference}: \href{https://en.wikipedia.org/wiki/Goldbach's_conjecture}{Goldbach's conjecture}.


\section{Problem Statement}
\label{sec:problem}

The goal of our algorithm is to detect errors in a relational, structured dataset~$D$ by letting the user progressively label attribute values as erroneous or correct.
We denote $D = \{t_{1}, t_{2}, ..., t_{N}\}$ as a dataset of size~$N$, where each $t_{i}$ represents a tuple.
Let $A = \{a_{1}, a_{2}, ..., a_{M}\}$ be the schema with attributes~$a_{j}$. Then, $D[i, j]$ represents the cell value of the attribute~$a_{j}$ in the tuple~$t_{i}$.
We denote $D_{C}$ as the cleaned version of $D$ representing the ground truth.
Thus, we define an error as any cell value~$D[i, j]$ that deviates from its ground truth value~$D_{C}[i, j]$.
This error definition is common in error detection research \cite{rekatsinas2017holoclean,abedjan2016detecting}.

Our goal is to formulate the problem of error detection as a classification task.
Therefore, the first problem is to identify the right feature representation $\rho$, which maps each cell~$D[i, j]$ to a numerical feature vector.
Using this feature representation $\rho$, for each column~$D[:,j]$, we train a model $\phi_{j}$ that learns to classify whether a cell value~$D[i, j]$ is erroneous or correct.
The second problem that we want to solve is, given a user-defined labeling budget~$L$, to choose the optimal training set for each column~$D[:,j]$ to maximize the $F_{1}$-score on the whole dataset~$D$. The error detection $F_{1}$-score is defined as

\begin{equation} \label{equation:problem2}
	F_1 = 2 \times (P \times R) /(P + R),
\end{equation}

where the precision~(P) is the fraction of cells that are correctly detected as errors and the recall~(R) is the fraction of the actual errors that are discovered.

To address the two mentioned problems, we have to design a fitting sampling strategy and identify the appropriate set of features. Our approach does not require additional user-defined configuration, such as user-defined rules, patterns nor model-specific, manual hyperparameter tuning.


\section{Fundamentals}

 \subsection{Exercise 1}
 $x + z = x = x + z' \implies z = z'$.

 \subsection{Exercise 2}
 $0x + x = (0 + 1)x = x$.

 \subsection{Exercise 3}
 Coefficients can be represented as row vectors.

 \subsection{Exercise 4}
 Function can be represented as row vector by letting $a_i = f(s_i)$ for each $s_i \in S$.

 \subsection{Exercise 5}
 Follows from exercises 3 and 4.

 \subsection{Exercise 6}
 $y_1 + z_1 + y_2 + z_2 = (y_1 + y_2) + (z_1 + z_2)$ and $k(y_1 + z_1) = ky_1 + kz_1$.

 \subsection{Exercise 7}
 $a \in Y \cap Z \implies ka \in Y, ka \in Z \implies ka \in Y \cap Z$.

 \subsection{Exercise 8}
 $k0 = 0, 0 + 0 =0$.

 \subsection{Exercise 9}
 If $S$ contains $x_i$ then it must contain  $kx_i$.

 \subsection{Exercise 10}
 If $x_i = 0$, $k_i$ can be anything.

 \subsection{Exercise 11}
 $x = \sum_{i=1}^{m} \sum_{j=1}^{\text{dim}Y_i} y_j^{(i)}$.

 \subsection{Exercise 12}
 Complete basis for $W$ to  $U$ and  $V$. Use  $W$ basis vectors and additional  $U$ and $V$ basis vectors to get
 $\text{dim}X = \text{dim}U - \text{dim}W + \text{dim}V - \text{dim}W + \text{dim}W$.

 \subsection{Exercise 13}
 Send $i^{\text{th}}$ basis vector to $e_i$, where $e_i$ is vector of all zeroes except a one in the  $i^{\text{th}}$ place.
 Can permute mapping to get different isomorphisms.

 \subsection{Exercise 14}
 $x_1 - x_2 + x_2 - x_3 = x_1 - x_3$.

 \subsection{Exercise 15}
 $x' = x + z_x, y' = y + z_y \implies x' + y' = x + y + (z_x + z_y)$.

 \subsection{Exercise 16}
 $x \in X_1 \bigoplus X_2 \implies x = (x_1, x_2) = (x_1, 0) + (0, x_2)$.

 \subsection{Exercise 17}
 Construct a basis for $X$ from  $Y$:
 $y_1, ..., y_j, x_{j+1}, ..., x_n$.
 \\
 Then $X / Y = \text{span}\{x_{j+1}, ..., x_n\}$.


\chapter{Dirac Delta Function}
In mathematical models of physical systems we often come across functions that have finite or infinite discontinuities (Potential barriers, Impulse functions). Even though they dont belongs to the generalm definition of functions we can represent them as generalised function or distributions. The most common among them are the step function and the dirac delta function.
\section{The Step Function}
Let's start with the definition of the unit step function, $\theta(x)$ :
$$
\theta(x)=\left\{\begin{array}{ll}
0 & \text { for } x<0 \\
1 & \text { for } x>0
\end{array}\right.
$$
We do not define $\theta(x)$ at $x=0$. Rather, at $x=0$ we think of it as in transition between 0 and 1 .The function is called the unit step function because it takes a unit step at $x=0$. It is sometimes called the \textbf{Heaviside function}. The graph of $\theta(x)$ is simple.
It is obvious that $\theta(x)$ has a finite jump at $x=0$. It is sometimes convenient to define $\theta(0)$ to be the average value $\frac{1}{2}$, but this is not always necessary.
\begin{align*}
\text{The sum } \ \theta(x)+\theta(-x)&=1\\
\text{The difference}\ \theta(x)-\theta(-x)&=\varepsilon(x)
\end{align*}
Where, $\varepsilon(x)$ is the signum function.
\begin{equation}
\varepsilon(x)=\left\{\begin{array}{rr}
+1 & \text { for } x>0 \\
-1 & \text { for } x<0 \\
0 & \text { for } x=0
\end{array}\right.
\end{equation} The function $\varepsilon(x)$ looks like the limit of a tanh (or hyperbolic tangent) function as the 'kink' in the function becomes more and more steep, i.e., as the slope at the origin tends to infinity, as shown in Figure.\ref{Dirac delta} In fact, we could define $\varepsilon(x)$ as the limit of a continous sequence of functions $\tanh(\frac{x}{\varepsilon(x)})$.
\begin{figure}[H]
	\centering
	\includegraphics[height=4cm,width=5cm]{unit step function}
	\caption{The function of $ \tanh{(x/\epsilon)} $}
	\label{Dirac delta}
\end{figure}
\section{Dirac Delta Function}
\subsection{Kronecker delta $\delta$}
let us consider a sequence $\left(a_{1}, a_{2}, \ldots\right)=\left\{a_{j} \mid j=1,2, \ldots\right\} .$ How do we select a particular member $a_{i}$ from the sequence? We do so by summing over all members of the sequence with a selector called the \textbf{Kronecker delta}, denoted by $\delta_{i j}$ and defined as
$\delta_{i j} \stackrel{\text { def. }}{=}\left\{\begin{array}{ll}1 & \text { if } i=j \\ 0 & \text { if } i \neq j\end{array}\right.$
It follows immediately that
\begin{align*}
\sum_{j} \delta_{i j} a_{j}&=a_{i}\\
\sum_{j} \delta_{i j}&=a_{i}\qquad \text{For each value of } \ i\\
\delta_{i j}&=\delta_{j i} \qquad \text{Symmetry property. }
\end{align*}
Now if  we have a continuos function, we must replace the summation over  $j$ by an integration over $x$. The role of the specified index $i$ is played by the specified point $a$. The analog of the Kronecker delta is written like a function, retaining the same symbol $\delta$ for it. So we seek a `function' $\delta\left(x-a\right)$ such that
\begin{equation}
\int_{-\infty}^{\infty}  \delta\left(x-a\right) f(x) d x=f\left(a\right)
\end{equation}
Exactly as in the discrete case of the Kronecker delta, we impose the normalization and symmetry properties,
\begin{equation}
\int_{-\infty}^{\infty} \delta\left(x-a\right) d x=1 \quad \text { and } \quad \delta\left(x-a\right)=\delta\left(a-x\right)
\end{equation}
 $\delta\left(x-a\right)$ is more like the kernel of an integral
 \begin{equation}
 \int_{-\infty}^{\infty} \delta\left(x-a\right) d x=0 \quad \text { if } \quad x \neq a \quad \text{Or the limit of integration excludes the point `$a$'}
 \end{equation}
 \section{Various representations of Delta function}
 The delta function may be approximated by the sequences of functions,
 $$
 \begin{array}{l}
 \delta_{n}(x)=\left\{\begin{array}{ll}
 0, & x<-\frac{1}{2 n} \\
 n, & -\frac{1}{2 n}<x<\frac{1}{2 n} \\
 0, & x>\frac{1}{2 n}
 \end{array}\right. \\\\
 \delta_{n}(x)=\frac{n}{\sqrt{\pi}} \exp \left(-n^{2} x^{2}\right) \\\\
 \delta_{n}(x)=\frac{n}{\pi} \cdot \frac{1}{1+n^{2} x^{2}} \\\\
 \delta_{n}(x)=\frac{\sin n x}{\pi x}=\frac{1}{2 \pi} \int_{-n}^{n} e^{i x t} d t
 \end{array}
 $$
 These approximations have varying degrees of usefulness.\\
 \begin{minipage}{0.45\textwidth}
 	\begin{figure}[H]
 		\centering
 		\includegraphics[height=3cm,width=3cm]{Dirac delta 1}
 	\end{figure}
 \begin{figure}[H]
 	\centering
 	\includegraphics[height=3cm,width=3cm]{Dirac delta 2}
 \end{figure}
 \end{minipage}
\begin{minipage}{0.45\textwidth}
	\begin{figure}[H]
		\centering
		\includegraphics[height=3cm,width=3cm]{Dirac delta 3}
	\end{figure}
	\begin{figure}[H]
		\centering
		\includegraphics[height=3cm,width=3cm]{Dirac delta 4}
	\end{figure}
\end{minipage}

 \subsection{Rectangular Function}
 Let us consider a rectangular fuunction defined as,
 \begin{equation}
R_{\varepsilon} =\left\{\begin{array}{ll} \frac{1}{2\varepsilon} & a-\varepsilon < x< a+\varepsilon \\ 0 & \text{Otherwise }\end{array}\right.
 \end{equation}
 We can consider  this as a rectangular window of width $2\varepsilon$ and height $\frac{1}{2\varepsilon}$ so that the area is unity. Let us plot the function as shown in the figure below,
 \begin{figure}[H]
 	\centering
 	\includegraphics[height=4cm,width=6cm]{Rectangular}
 	\caption{Rectangular function}
 	\label{Rectangular function}
 \end{figure}
When the window is centred at the choosen point $x_{0}$, the integral of $f(x)$ multiplied by this window function is,
\begin{align*}
\frac{1}{2 \varepsilon} \int_{a-\varepsilon}^{a+\varepsilon} dx f(x)
\intertext{if we take the limit,$\varepsilon \rightarrow 0$ The window becomes vanishingly small and it's height become arbitrarily large such that the area under the curve remains the same.}
\intertext{Then in the limit, $\varepsilon \rightarrow 0 $}
\lim_{\varepsilon \to 0} R_{\varepsilon} = \delta(x-a)
\end{align*}
An explicit form of the dirac delta function make sense only when it occures in an integral like,
\begin{equation}
\int_{-\infty}^{\infty} \delta(x-a)f(x)dx \label{Dirac delta 001 }
\end{equation}
when it acts on ordinary function like $f(x)$ and an integration over $x$ is carried out . Then the value of \ref{Dirac delta 001 }
\begin{align}
\int_{-\infty}^{\infty} \delta(x-a)f(x)dx &=\int_{a-\varepsilon}^{a+\varepsilon} \delta(x-a)f(x)dx
\intertext{in the range ${a-\varepsilon} , {a+\varepsilon}$ as $\varepsilon \rightarrow 0$ the function is almost constant }
\int_{-\infty}^{\infty} \delta(x-a)f(x)dx &=f(a) \lim_{\varepsilon \to 0} \int_{a-\varepsilon}^{a+\varepsilon} R_{\varepsilon} dx\\
&=f(a)
\end{align}
\section{Integral Representations for the Delta Function}
Integral transforms, such as the Fourier integral
\begin{equation}
F(\omega)=\int_{-\infty}^{\infty} f(t) \exp (i \omega t) d t
\end{equation}
Lead to the corresponding integral representations of Dirac's delta function. For example, take
\begin{equation}
\delta_{n}(t-x)=\frac{\sin n(t-x)}{\pi(t-x)}=\frac{1}{2 \pi} \int_{-n}^{n} \exp (i \omega(t-x)) d \omega \label{Dirac delta 007}
\end{equation}
Using Equation \ref{Dirac delta 007}. We have
$$
f(x)=\lim _{n \rightarrow \infty} \int_{-\infty}^{\infty} f(t) \delta_{n}(t-x) d t,
$$
where $\delta_{n}(t-x)$ is the sequence in Eq. (1.192) defining the distribution $\delta(t-x)$. Note that Eq. (1.193a) assumes that $f(t)$ is continuous at $t=x$. If we substitute Eq. (1.192) into Eq. (1.193a) we obtain
$$
f(x)=\lim _{n \rightarrow \infty} \frac{1}{2 \pi} \int_{-\infty}^{\infty} f(t) \int_{-n}^{n} \exp (i \omega(t-x)) d \omega d t .
$$
Interchanging the order of integration and then taking the limit as $n \rightarrow \infty$, we have the Fourier integral theorem, Eq. (15.20).
With the understanding that it belongs under an integral sign, as in Eq. (1.193a), the identification
\section{The derivative of the $\delta$ function}
Consider a bell shaped symmetric function peaked around $x=x_0$ and parametrized by $\varepsilon$ as $\varepsilon\rightarrow0$ the function resembles a dirac delta and the derivative starts to oscillate between positive and negative values around $x=x_0$. So the derivative $\delta^\prime(n-n_0)$ is more singular than $8(x-x_0)$
\begin{align*}
\text{Finally }\quad&\frac{d^n}{dx^n}\delta(n)\text{ is odd $=n$ odd}\\
\delta^\prime(-x)&=-\delta^0(x)\text{ even $=n$ even}\\
\text{and}\quad \delta^\prime (x) &=\frac{-\delta(x)}{x}\\
\text{with} &\int\limits_{-\propto}^{+\propto}\delta^\prime (x)f(x)dx=-f^\prime(a)
\end{align*}
Proof: $\int\limits_{-\propto}^{+\propto}\delta^\prime (x-a)f(x)dx=f(x)\int\limits_{-\propto}^{+\propto}\delta^\prime (x-a)-\int\limits_{-\propto}^{+\propto}(f^\prime (x)\delta^\prime(x-a)dx)dx$\\\\
$=0-\int f^\prime(x)\delta(x-a)=-f^\prime(a),$proved
\section{The occurance of $\delta$-function in physical problems}
\begin{enumerate}
	\item The dirac delta function models the density of a point source. Could be change density mass density etc.\\
	eg:3$\delta(x-x_0)$$\rightarrow$ a charge $3c$ placed at $x=x_0$
	\item Suppose $P(x)dx$ represent the probability distribution for age group of students than if we have a part probability say $N$ student has exactly age $x_g$ then $P(x)dx+N\delta(x-x_g)$
	\item An impulse function in 3-D\\
	\begin{align*}
	\delta^3(r)&=\delta(x-x_0)\delta(y-y_0)\delta(z-z_0)\\
	&=\frac{1}{(2\pi)^3}\int\limits_{+\propto}^{-\propto}dk_1\int\limits_{+\propto}^{-\propto}dk_2\int\limits_{+\propto}^{-\propto}dk_3 e^{ikx_1+k_2y+k_3z}\\
	\delta^3(r)&=\frac{1}{(2\pi)^3}\int d^3ke^{ik\cdot r}
	\end{align*}
\end{enumerate}
The dimension of $\delta(x)$will be the inverse $x$ \\
if $x$ has a dimenssion $L$\\
Then $ \left[ \delta(x)\right]\left[ L^{-1}\right]  $

CSIR- Mathematical Physics/chapter/Dirac delta function.tex


\chapter{Necessary theory}
To continue with the NHPP model, some more theory have to be introduced. This theory includes likelihood function and transformation of variables.
\section{Likelihood and estimators}
\label{chap:like}
In this section likelihood and its estimators are introduced.
To summarize data the likelihood function can be used. The theory presented here is from \cite{casella2002statistical}. The likelihood function is defined in the following definiton.
\begin{defn}
Let $f(\boldsymbol{x}|\Theta)$ denote the joint pdf or pmf of the sample $\boldsymbol{X} = (X_1,...,X_n)$. Then, given that $\boldsymbol{X} = \boldsymbol{x}$ is observed, the function $\Theta$ is defined by
\begin{equation}
L(\Theta | \boldsymbol{x}) = f(\boldsymbol{x}|\Theta)
\label{eq:like}
\end{equation}
is called the likelihood function.
\end{defn}
By using this function one can find estimators for $\Theta$. These estimators are the ones that maximizes the likelihood function. Furthermore these estimators are most likely. Hence the they are given the name maximum likelihood estimators. The formal definition is as follows
\begin{defn} % side 316
For each sample point $\boldsymbol{x}$, let $\hat{\Theta}(\boldsymbol{x})$ be parameter value at which $L(\Theta | \boldsymbol{x})$ attains its maximum as a function of $\Theta$, with $\boldsymbol{x}$ held fixed. A maximum likelihood estimator(MLE) of the parameter $\Theta$ based on a sample $\boldsymbol{X}$ is $\hat{\Theta}(\boldsymbol{X})$.
\end{defn}
The maximum of the likelihood can be found analytically or numerically. Analytically it is recommended to take the log of the likelihood function. This for easier differentiation. Hence it is recommended to solve
\begin{equation*}
\frac{\partial \log L(\Theta | \boldsymbol{x})}{\partial \Theta_i} = 0, \quad i = 1,...,k.
\end{equation*}
This will give the MLE of $\Theta$.

\section{Transformation of variables}
\label{sec:transvar}
For this section we will show tranformation of variables from a distribution to another distribution. From \cite{casella2002statistical} we have a theorem for tranformation of variables. The theorem in one dimension is given as follows.
\begin{theorem}
\label{th:transvar}
Let $X$ have pdf $f_X (x)$, let $Y=g(X)$, where $g$ is a monotone function. Let $\mathcal{X} = \{x: f_X (x) > 0\}$ and $\mathcal{Y} = \{ y: y = g(x)$ for some $x \in \mathcal{X} \}$. Suppose that $f_x(x)$ is continuous on $\mathcal{X}$ and that $g^{-1}(y)$ has a continuous derivative on $\mathcal{Y}$. Then the pdf of $Y$ is given by
\begin{equation*}
f_Y (y) =
\begin{cases}
f_X(g^{-1}(y)) \left| \frac{d}{dy} g^{-1}(y) \right| & y \in \mathcal{Y} \\
0 & \text{otherwise.}
\end{cases}
\end{equation*}
\end{theorem}
For multivariate the pdf of $Y$ is given by
\begin{equation*}
f_Y (y) =
\begin{cases}
f_X(g^{-1}(y)) \left| J(g^{-1}(y)) \right| & y \in \mathcal{Y} \\
0 & \text{otherwise,}
\end{cases}
\end{equation*}
where $J$ is the Jacobian matrix. For the multivariate the requirements for $g$ is that the inverse exists and is unique.


\subsection{Group Actions}

Let $G$ be a group acting on a topological space $X$ via $G\times X\to X$,
$(g,x)\mapsto g\cdot x=g(x)$ such that
\begin{itemize}
\item each map $x\mapsto g\cdot x$ is continuous (as a fn $X\to X$)
\item $e\cdot x=x$
\item $g\cdot(h\cdot x)=(gh)\cdot x$ $\forall g,h\in G,x\in X,e=\text{id}_{G}$.
\end{itemize}
then $x\mapsto g\cdot x=g(x)$ is a homeomorphism.
\begin{defn}
The \emph{orbit} of a point $x$ is $G\cdot x=\{g\cdot x\mid g\in G\}$.
The \emph{orbit space} $X\backslash G$ (or $X/G$) is $X/\sim$ where
$x\sim y\Leftrightarrow y=g\cdot x$ for some $g\in G$.
\end{defn}

\begin{defn}
The free group $F(S)$ generated by $S$ is the set of all reduced
words in $S$ with multiplication defined by juxtaposition followed
by reduction.\end{defn}
\begin{prop}
If $G$ is a group generated by a subset $S$, and $F=F(S)$, then
$G$ is a quotient of $F$.\end{prop}
\begin{proof}
The inclusion $f:S\hookrightarrow G$ extends to a homomorphism $\bar{f}:F(S)\to G$.
Then $S=\{\text{set of generators}\}$ so $\bar{f}$ is onto. The
First Isomorphism Theorem then says $G\cong F/N$, $N=\text{ker}(\bar{f})=\text{normal subgroup of F}.$\end{proof}
\begin{defn}
A group $G$ is defined by a set of generators $S$ and a set of relators
$R$. $G\cong F(S)/N(R)$ where $N(R)=\{\text{smallest normal subgroup containing \ensuremath{R}}\}=\text{"normal closure of \ensuremath{R}"}.$
We write $G=\langle S\mid R\rangle$. A group is \emph{finitely presented}
if both sets are finite.\end{defn}
\begin{prop}
Let $G=\langle S\mid R\rangle$, $H$ any group, and $f:S\to H$ any
function. Then $f$ extends to a homomorphism $\tilde{f}:G\to H$
iff $f(x_{i})^{\epsilon_{1}}\cdots f(x_{n})^{\epsilon_{n}}=1$ for
every relator $x_{1}^{\epsilon_{1}}\cdots x_{n}^{\epsilon_{n}}$.
\end{prop}
\textit{\emph{Constrstructions}}

Direct product: $G_{1}\times G_{2}=\langle S_{1},S_{2}\mid R_{1},R_{2},[x,y]=1\forall x\in S_{1},\forall y\in S_{2}\rangle.$

Free product: $G_{1}*G_{2}=\langle S_{1},S_{2}\mid R_{1},R_{2}\rangle$

and Abelianisation: $G^{ab}=\langle S\mid R,[xy]=1\forall x,y\in S\rangle$

Note: Every finitely-generated Abelian group is isomorphic to a direct
product of cyclic groups $A\cong\mathbb{Z}^{n}\oplus\mathbb{Z}_{d_{1}}\oplus\cdots\oplus\mathbb{Z}_{d_{k}}.$
\begin{lem}
(Lebesgue number lemma) Let $(X,d)$ be a compact metrix space. Given
any open cover $\{U_{\alpha}\}$ of $X$, $\exists\delta>0$, the
Lebesgue number, such that any subset of $X$ with $\text{diam}<\delta$
is contained in one of the $U_{\alpha}$.
\end{lem}
We can use this to help prove Seifert-van Kampen.
\begin{thm}
(Seifert-van Kampen Theorem)

Let $X=U_{1}\cup U_{2}$ with $U_{1},U_{2},U_{0}=U_{1}\cap U_{2}$
are all open, non-empty and path connected. Choose a basepoint $x_{0}\in U_{0}$
for all fundamental groups that we will be looking at. We have the
inclusion maps

\begin{align*}
\xymatrix{ & U_{1}\ar[dr]_{j_{1}}\\
U_{0}\ar[ur]^{i_{1}}\ar[dr]_{i_{2}} &  & X\\
 & U_{2}\ar[ur]_{j_{2}}
}
\end{align*}

which induces the homomorphisms

\begin{align*}
\xymatrix{ & \pi_{1}(U_{1})\ar[dr]_{j_{1*}}\\
\pi_{1}(U_{0})\ar[ur]^{i_{1*}}\ar[dr]_{i_{2*}} &  & \pi_{1}(X)\\
 & \pi_{1}(U_{2})\ar[ur]_{j_{2*}}
}
\end{align*}

then, $\pi_{1}(X)$ is obtained from the free product $\pi_{1}(U_{1})*\pi_{1}(U_{2})$
by adding the ``amalgamation'' relations:

\begin{align*}
(i_{1*})(u) & =(i_{2*})(u)\forall u\in\pi_{1}(U_{0}).
\end{align*}

More explicitly: If $\pi_{1}(U_{1})=\langle S_{1}\mid R_{1}\rangle$,
$\pi_{1}(U_{2})=\langle S_{2}\mid R_{2}\rangle$, $\pi_{1}(U_{0})=\langle S_{0}\mid R_{0}\rangle$
then $\pi_{1}(X)=\langle S_{1},S_{2}\mid R_{1,}R_{2}(i_{1*})(u)=(i_{2*})(u)\forall u\in S_{0}\rangle$.

More precisely, the homomorphisms $(j_{1*}):\pi_{1}(U_{1})\to\pi_{1}(X),(j_{2*}):\pi_{1}(U_{2})\to\pi_{1}(X)$
extend to homomorphism $\pi_{1}(U_{1})*\pi_{1}(U_{2})\to\pi_{1}(X)$
with kernal generated by $\{(i_{1*})(u)(i_{2*})(u)^{-1}\mid u\in\pi_{1}(U_{0})\}$.
\end{thm}


\subsection{Lagrange polynomials}\label{subsec:lagrange_polynomials}

\begin{definition}\label{def:omega_polynomial}
  Given distinct elements \( x_0, \ldots, x_n \) of the field \( \BbbK \), we form the polynomial
  \begin{equation*}
    \omega(X) \coloneqq \prod_{k=0}^n (X - x_j).
  \end{equation*}
\end{definition}

\begin{proposition}\label{thm:omega_polynomial_derivative}
  For the polynomial \( \omega \) from \fullref{def:omega_polynomial}, for \( k = 0, \ldots, n \) we have
  \begin{equation*}
    \omega'(x_j) = \prod_{\substack{j = 0 \\ j \neq k}}^n (x_j - x_k),
  \end{equation*}
  where \( \omega' \) is the algebraic \hyperref[def:algebraic_derivative]{derivative} of \( \omega \).
\end{proposition}
\begin{proof}
  Fix \( k \in \{ 0, \ldots, n \} \) and denote
  \begin{equation*}
    q(X) \coloneqq \prod_{\substack{j = 0 \\ j \neq k}}^n (X - x_j).
  \end{equation*}

  Then
  \begin{equation*}
    \omega(X) = (X - x_k) q(X)
  \end{equation*}
  so
  \begin{equation*}
    \omega'(X) = [q(X) + X q'(X)] - x_k q'(X) = q(X) + (X - x_k) q'(X).
  \end{equation*}

  So for \( x_k \) we have
  \begin{equation*}
    \omega'(x_k) = q(x_k) = \prod_{\substack{j = 0 \\ j \neq k}}^n (x_k - x_j).
  \end{equation*}
\end{proof}

\begin{theorem}[Lagrange interpolation]\label{thm:lagrange_interpolation}
  Let \( x_0, x_1, \ldots, x_n \) be pairwise distinct elements of \( \BbbK \) and \( y_0, y_1, \ldots, y_n \) be arbitrary elements of \( \BbbK \). Then there exists a unique \hyperref[def:polynomial]{polynomial} \( L_n(X) \in \pi_n \) (where \( \pi_n \) is the \hyperref[def:polynomial_free_module]{\( n \)-dimensional polynomial vector space}) such that
  \begin{equation}\label{eq:thm:lagrange_interpolation/condition}
    L_n(x_k) = y_k, k = 0, 1, \ldots, n.
  \end{equation}
\end{theorem}
\begin{proof}
  We will first show uniqueness. Let \( p, q \in \pi_n \) both satisfy \fullref{eq:thm:lagrange_interpolation/condition}. Their difference \( p - q \) is a polynomial of degree at most \( n \) that has \( n + 1 \) roots. By \fullref{thm:integral_domain_polynomial_root_limit}, \( p - q = 0 \). This proves uniqueness.

  We construct the polynomial explicitly. Define the Lagrange polynomial
  \begin{equation*}
    L_n(X) = \sum_{m=0}^n y_m \prod_{\substack{j = 0 \\ j \neq m}}^n \frac {(X - x_j)} {(x_m - x_j)}.
  \end{equation*}

  For \( k = 0, 1, \ldots, n \) we have
  \begin{equation*}
    L_n(x_k) = y_k \underbrace{\prod_{\substack{j = 0 \\ j \neq k}}^n \frac {(x_k - x_j)} {(x_k - x_j)}}_{=1} + \sum_{\substack{m = 0 \\ m \neq k}}^n y_m \overbrace{\frac{(x_k - x_m)}{(x_k - x_m)}}^{=0} \prod_{\substack{j = 0 \\ j \neq k \\ j \neq m}}^n \frac {(x_k - x_j)} {(x_m - x_j)} = y_k.
  \end{equation*}

  Therefore, \( L_n \) satisfies \eqref{eq:thm:lagrange_interpolation/condition}, which proves existence.
\end{proof}


\chapter{Positions in 2D}
\label{chapter:positions}

Positions in 2D have an x- and a y- coordinate, just as a point in a graph. In Esenthel the zero point is the middle of the screen. Positive values on the X-axis are to the right, negative values to the left. Positive values on the Y-axis are in the upper half of the screen, negative values on the lower half.

\begin{figure}[h]
\centering
\includegraphics[width=0.7\linewidth]{images/2Dpositions.png}
\caption[]{2D coordinates.}
\label{fig:pos2D}
\end{figure}

The class \eeClass{Vec2} is used to represent a 2D coordinate. There are several ways to set the x and y values:

\begin{code}
Vec2 pos;
pos.x =  0.1;       // set the x value to  0.1
pos.y = -0.3;       // set the y value to -0.3
pos   =  0.5;       // both x and y are set to 0.5
pos.set(0.1, -0.3); // use the function set(float x, float y)
                    // to assign both x and y
\end{code}

\begin{exercise}
Where would these positions be located on the screen?
\end{exercise}

\video{https://www.youtube.com/embed/fbmTzkUnv3E}

\section{Show a position on the screen}

Although the class \eeClass{Vec2} is mainly used to calculate positions, it is also possible to show it on the screen. This can be done with the function \eeFunc{draw(Color)}. The argument should contain the color in which the coordinate must be drawn.

\begin{code}
Vec2 p1(0.2, 0.4); // Create a point p1, assigning x and y
                   // with the constructor.
Vec2 p2;           // Create a point p2.

void InitPre()
{
   EE_INIT();
}

bool Init()
{
   p2.set(-0.2, -0.5); // assign values to p2 with the set function
   return true;
}

void Shut() {}

bool Update()
{
   if(Kb.bp(KB_ESC)) return false;

   return true;
}

void Draw()
{
   D.clear(BLACK); // Clear the screen
   p1.draw(RED  ); // draw p1 in red
   p2.draw(BLUE ); // draw p2 in blue
   Vec2(0 ,0).draw(GREEN); // create a temporary object and draw
	                         // it in green
}

\end{code}

\begin{exercise}
Write this code in the editor. Don't copy/paste it: you won't learn anything from copying code. Make sure it runs without error. If not, compare your code with this example. Learning to understand errors is very important. Try to understand what went wrong and why.
\end{exercise}

\section{Math}
You can do math with \eeClass{Vec2}, just as you can with plain numbers. It is possible to assign x and y values directly, but you can also do math with the class itself. In this case the operator will be applied to x as well as y:

\begin{code}
Vec2 p1(0.1,  0.3);
Vec2 p2(0.3, -0.1);
Vec2 p3 = p1 + p2;  // x: 0.4 , y: 0.2
p3 -= 0.1;          // x: 0.3 , y: 0.1
p3 *= 2;            // x: 0.6 , y: 0.2
p3 = p1 / 2.f;      // x: 0.05, y: 0.15
\end{code}

\section{Points to remember}
The position \eeClass{Vec2(0,0)} always stands for the middle of the screen. But the borders are not always clear. Not every computer screen has the same size, but you probably want to know the border values are. For example, you want to draw something in the right corner of the screen. For this reason, Esenthel provides a few functions in the object \texttt{D} (display). You can request the width of the screen with \eeFunc{D.w()} and the height of the screen with the function \eeFunc{D.h()}. This makes it very easy to calculate the next points:

\begin{code}
Vec2 middle(0,0);
Vec2 left(-D.w(), 0);
Vec2 right(D.w(), 0);
Vec2 rightUpperCorner(D.w(), D.h());
Vec2 leftUpperCorner(-D.w(), D.h());
\end{code}

If you'd like to draw a point at a distance of 0.1 from the left upper corner, you could do that like this:

\begin{code}
Vec2(-D.w() + 0.1, D.h() - 0.1).draw(PINK);
\end{code}

\section{Summary}
\begin{itemize}
\item Points in 2D have an x- and y-coordinate. In Esenthel you will use the class \eeClass{Vec2} to store a point.
\item The class \eeClass{Vec2} has a function \eeFunc{draw(Color)} to draw a point on the screen.
\item You can do math with \eeClass{Vec2}, just as with an ordinary number.
\item \eeClass{Vec2(0,0)} will always be the middle of the screen.
\item The borders of the screen can be calculated with \eeClass{D.w()} and \eeClass{D.h()}.
\end{itemize}

\begin{exercise}
Create an application which draws these points:
\begin{enumerate}
	\item A white point in the middle of the screen.
	\item A red point on 0.1 units away from the left border of the screen.
	\item A blue point on 0.2 units from the upper right corner.
	\item A yellow dot on 0.35 units from the lower border of the screen, on 2/3 of the total screen width.
\end{enumerate}
\end{exercise}

\video{https://www.youtube.com/embed/a2k420oTySU}


\section*{Neural Networks}
\textbf{Activation:} ReLU: $\max(0,x)$ \\
$\tanh(x)=\frac{e^x-e^{-x}}{e^x+e^{-x}}$, $\tanh'(x) = 1 - \tanh^2(x)$ \\
sigmoid $s(x)= \frac{1}{1+e^{-x}},s^{'}(x)=s(x)(1-s(x))$\\
\textbf{Output}: linear regression $\mathbf{y} = \mathbf{W}^L\mathbf{x}^{L-1}$, \\binary (logistic) $y_1 = \frac{1}{1 + \exp(-\mathbf{w}^T \mathbf{x}^{L-1})}$, \\multiclass (soft-max) $y_k = \frac{\exp( \mathbf{w}_k^T\mathbf{x}^{L-1})}{\sum_{m=1}^{K}{\exp(\mathbf{w}^T\mathbf{x}^{L-1})}}$.\\
\textbf{Loss function:} $l(y, \hat{y})$: squared loss $\frac{1}{2}(y - \hat{y})^2$, \\
cross-entropy loss $-y \log \hat{y} - (1-y)\log(1-\hat{y})$\\
$\text{Conv}_{n,m}^{k\times k}(\mathbf{x};\mathbf{w})=\sigma \left( b + \sum_{i=-k}^k \sum_{j=-k}^k w_{i,j}x_{n+i,m+j} \right) $\\
\textbf{CNN:} weight sharing ($<<$ param), shift invar. filters

\subsection*{Backpropagation}
$J_{ij} = \frac{\partial \mathbf{x}_i^{out}}{\partial \mathbf{x}_j^{in}} = w_{ij}\cdot\sigma'(\mathbf{w}_i^\top\mathbf{x}^{in})$. Across multiple layers: $\frac{\partial\mathbf{x}^{(l)}}{\partial\mathbf{x}^{(l-n)}} = \mathbf{J}^{(l)}\cdot\frac{\partial\mathbf{x}^{(l-1)}}{\partial\mathbf{x}^{(l-n)}}=\mathbf{J}^{(l)}\cdot\mathbf{J}^{(l-1)}\cdots\mathbf{J}^{(l-n+1)}$ and then back prop. $ \nabla_{\mathbf{x}^{(l)}}^\top\ell=\nabla_{\mathbf{y}}^\top\ell\cdot\mathbf{J}^{(L)}\cdots\mathbf{J}^{(l+1)}$\\
$\frac{\partial l}{\partial w_{ij}^{(l)}} = \frac{\partial l}{\partial x_i^{(l)}}\frac{\partial x_i^{(l)}}{\partial w_{ij}^{(l)}}$, $\frac{\partial x_i^{l}}{\partial w_{ij}^{l}} = \sigma'([\mathbf{w}_i^{(l)}]^T \mathbf{x}^{(l-1)})\cdot x_j^{(l-1)}$


\mychapter{13}{Lesson 13} %181109

\section{Number theory}

\begin{theorem}[Fermat's last theorem]
    \[
        \forall x, y, z \in \integer, n > 2 \implies x^n + y^n \neq z^n
    \]
\end{theorem}

\begin{lemma}
    \[
        \forall a \in \integer_n : \gcd(a, n) > 1 \implies a \notin \integer_n^\times
    \]
\end{lemma}

\begin{proof}
    Assume there exists $b \in \integer_n$ such that $ab \equiv_n 1$. Then, there exists a quotient for the division $by$ between a and b with remainder 1. Observe that $\gcd(a, n)$ divides $ab + qn$, which is equal to 1. It entails that $\gcd(a, n) = 1$, which is a contradiction.
\end{proof}

\begin{lemma}
    \[
        \forall a, b \in \nonneg : a \geq b \neq 0 \implies \gcd(a, b) = \gcd(b, a \mod b)
    \]
\end{lemma}

\begin{proof}
    Not given.
\end{proof}

\begin{theorem}
    Given two integers $a$ and $b$, their greatest common divisor can be computed efficiently with respect to their lengths. Additionally, two other numbers u and v can be efficiently computed in order to satisfy B\'ezout's identity: $\gcd(a, b) = au + bv$
\end{theorem}

\begin{proof}
    Hint: Use previous lemma recursively; we stop at $r_{t + 1} = 0$ for some $t$, thus:
    \[
        \gcd(a, b) = \dots = \gcd(r_t, r_{t + 1}) = r_t
    \]
\end{proof}

\begin{claim}
    $r_{i + 2} \leq r_i/2 \forall 0 \leq i \leq t - 2 \implies \#steps = \lambda - 1 \text{ if } |b| \in \binary^\lambda$
\end{claim}

The hypothesis is a natural consequence of the repeated modulo operations.

\begin{observation}[Exponentiation mod n: Square and multiply]
    Let $b \in \binary^l$, where by writing $b_i$ we denote b's i-th bit. Then:
    \[
        a^b \equiv_n a^{\sum_{i = 0}^l 2^ib_i} \equiv_n \prod_{i = 0}^l a^{2^ib_i}
    \]
\end{observation}

%AP190913 - Unproven, can't find it anywhere
\begin{theorem}
    The number of primes lesser than or equal to $x$ is a number greater than or equal to $\frac{x}{3\log_2 x}$
\end{theorem}

There exist many algorithms that solve the problem of primality testing, with time complexity polynomial in the length of their numerical representation; of most relevance are those of Miller-Rabin, which is probabilistic, but consistently used in practice, and the completely deterministic Agrawal-Kayal-Saxena (\textsc{aks}), which has a much greater polynomial rank, and has been deemed impractical for most uses.

What remained as a conjecture is the intractability of determining the factors of a $\lambda$-bit composite number, which is widely known as the \emph{factorization hardness}; this in turn would imply that integer multiplication of two $\lambda$-bit primes is a \owf.

\begin{definition}
    Given a group $G$, its order is the least $i$ such that $a^i \equiv_n 1$
\end{definition}

\begin{corollary}
    \[
        \forall a \in \integer_m^\times \implies a^{\phi(n)} \equiv_n 1 \wedge a^b \equiv_n a^{b \mod \phi(n)} \wedge a^{p - 1} \equiv_p 1
    \]
\end{corollary}

\begin{proof}
    $(\integer_n^\times, \cdot)$ is a group with $\phi(n)$ elements. Take $\left< a \right> = \{a^0, \dots, a^{d + 1}\}$, where $d$ is the order. We must have $\phi(n) = kd$ for some $k$. Therefore, $a^{\phi(n)} = a^{dk} \equiv_n 1$.

    Also, $a^b \equiv_n a^{b\phi(n) + b \mod \phi(n)} \equiv_n 1 \cdot 2^{b \mod \phi(n)}$
\end{proof}

\begin{theorem}
    Let G, H be two groups such that $H < G$, meaning the order of H divides the order of G
\end{theorem}

\section{Standard model assumptions}

We now turn our attention to some conjectures that form the basis for most of the cryptographic schemes that will follow. We will start from the weakest, and go up to the strongest. For all our purposes, let $\groupgen(1^\lambda)$ be a ``group generator'' with security parameter $\lambda$, and let a random sample $(G, g, q) \pickUAR \groupgen(1^\lambda)$ be a triplet composed of the group itself $G$, one of its generators $g$, and its order $q$.

% CDH -> DDH -> DL
\subsubsection{Discrete logarithm}

Given $g$ and $g^x$ in a $\lambda$-bit group, there is no efficient algorithm for computing $y$ such that $g^y = g^x$ without knowing $x$ beforehand:
\[
    \forall \adversary \in \ppt \implies \Pr[\cryptog{dl}(\lambda) = 1] \in \negl(\lambda)
\]

\begin{cryptogame}
    {dlass}
    {$\cryptog{dl}(\lambda)$}
    {dl}

    \receive{\shortstack[l]{
        %AP190913: Actually, the adversary knows this!
        $(G, g, q) \pickUAR \groupgen(1^\lambda)$ \\
        $x \pickUAR \integer_q$
    }}{$y = g^x$}{}

    \send{}{$x'$}{\textsc{Output 1 iff} $y^{x'} \equiv_G y$}

\end{cryptogame}

This means that the \dl{} for a generic group G yields a \owf, whereas in a multiplicative group $\integer_p^\times$, we obtain a \owp.

\subsubsection{Computational Diffie-Hellman}

%AP190913: Probably it is not necessary to compute it directly, we may only need a congruent value
Given a group $G$ and two elements in it $g^x$, $g^y$, it is impractical to compute $g^{xy}$ without knowing $x$ or $y$.

\begin{cryptogame}
    {cdhass}
    {$\cryptog{cdh}(\lambda)$}
    {cdh}

    \receive{\shortstack[l]{
        %AP190913: Actually, the adversary knows this!
        $(G, g, q) \pickUAR \groupgen(1^\lambda)$ \\
        $x, y \pickUAR \integer_q$
    }}{$(g^x, g^y)$}{}

    \send{}{$h$}{\textsc{Output 1 iff} $h = g^{xy}$}

\end{cryptogame}

\subsubsection{Decisional Diffie-Hellman}

Given a group $G$ and three elements in it $g^x$, $g^y$, $g^z$, it is impractical to distinguish $g^{xy}$ from $g^z$ by only knowing $g^x$ and $g^y$, along with the originating triad $(G, g, q))$.

\begin{cryptogame}
    {ddhass}
    {$\cryptog{ddh}(\lambda)$}
    {ddh}

    \cseqdelay

    \receive{\shortstack[l]{
        %AP190913: Actually, the adversary knows this!
        $(G, g, q) \pickUAR \groupgen(1^\lambda)$ \\
        $x, y, z_1 \pickUAR \integer_q$ \\
        $z_0 = xy \mod q$ \\
        $b \pickUAR \binary$
    }}{$(g^x, g^y, g^{z_b})$}{}

    \cseqdelay

    \send{}{$b'$}{\textsc{Output 1 iff} $b' = b$}

\end{cryptogame}

All these assumptions helped in constructing the \emph{Diffie-Hellman key exchange} protocol, which is a way to establish a \ske{} channel from an unsafe channel, with any adversary unable to efficiently break the channel's secrecy. Do note that authentication is left out of the picture here.

\begin{cryptosequence}
    {dhkex}
    {The Diffie-Hellman Key Exchange protocol}

    \cseqentity{A}{Alice}
    \cseqentity[2.2]{B}{Bob}

    \cseqmessagel{B}{\shortstack[l]{
        $(G, g, q) \pickUAR \groupgen(1^\lambda)$ \\
        $x \pickUAR G$
    }}{$(G, g, q, g^x)$}{A}{}

    \cseqdelay

    \cseqmessager{A}{\shortstack[l]{
        $y \pickUAR G$ \\
        $k = (g^x)^y$
    }}{$g^y$}{B}{$k = (g^y)^x$}

\end{cryptosequence}

Some relationships have been established between these assumptions: it is known that $\ddh \implies \cdh \implies \dl$; and that $\cdh \notimplies \ddh$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 6}

An expression is in disjunctive normal form if it is of the form $e = c_1 \vee c_2 \vee \dots \vee c_m$ where $c_k$ is a clause and of the form $c_k = (z_1^{(k)} \wedge z_2^{(k)} \wedge \dots \wedge z_{n_k}^{(k)})$ in which each $z_j^{(k)}$ is a literal.
Prove that the problem of satisfiability for expressions in disjunctive normal form is in $P$; that is, prove that there is an algorithm which takes as input an expression $e$ with $|e|$ number of symbols in disjunctive normal form and determines whether or not that expression is satisfiable in polynomial time.

\subsection*{Solution}

An expression in Disjunctive Normal Form (DNF) will be evaluated to true if and only if at least one of its clauses is evaluated to true.
Therefore, a DNF expression is satisfiable if boolean assignments can be made to one of its clauses such that the clause is evaluated to true.
It is always possible to evaluate a conjunctive clause as long as a literal and its negation are not present in the clause at the same time.
Consequently, satisfying a DNF expression can be translated to verifying whether there is a clause in that expression that does not contain a variable and its negation at the same time, a procedure which clearly has a runtime linear in the number of symbols $|e|$, thus polynomial time.
Consequently, DNF-SAT is in P.


% tex file for regression
\par \indent A simple and straightforward way to model the voxel time courses
for each subject is to perform linear regression. Initially, we just used the
convolved predicted hemodynamic response (HR) and either all of the
conditions together or each of conditions individually. After realizing that
the HRs themselves did not explain enough of the BOLD ratio, we attempted to
add features in order to reduce or explain the noise we observed. Additional
features that we examined included a linear drift and some of the time
courses' Fourier series and principal components.

\par Linear regression assumes a linear relationship between a response
vector $y$ and a design matrix of predictors $X$. Each element of $y$
represents a single observed response, and each row of $X$ represents a
corresponding vector of predictor values. If one including the intercept as a
term (as we have elected to do), the first column of the $X$ design matrix
should be a vector of $1$s. The linear model can then be expressed as:

\begin{equation}
y = X\beta + \epsilon
\end{equation}

\par It is further assumed that the errors $\epsilon_i$ for each observation
$i$ are independent and identically distributed with $N(0, \sigma^2)$, and
that the errors are independent of $X$. The vector of coefficients $\beta$
with length equal to the number of predictors in $X$ can be estimated with
the closed-form solution:

\begin{equation}
\hat{\beta} =(X^T X)^{-1} X^T y
\end{equation}

\par Even when $(X^T X)$ is not invertible, $\hat{\beta}$ can be estimated
using the pseudo-inverse of $(X^T X)$, represented as $(X^T X)^{-}$ to get a
non-unique value for $\hat{\beta}$.

\par To consider the strength of the effects of these predictors, we will use
t-tests of the corresponding estimated coefficients for each voxel and
subject, as discussed under Section \ref{hypothesis_testing}. The validity of
the model and of the ``p-values'' produced by performing these t-tests is
dependent on whether or not the many assumptions of the linear model are
actually met. In particular, we will discuss the assumption of normal errors
by analyzing the residuals in Section \ref{normality}.

\subsubsection{More about potential features:}
\par Other than the basic HR feature(s) and a column of $1$s (to account
for an ``intercept'' term or non-zero average value), we experimented with
additional predictors for our design matrix $X$. Among these were the
first few principal components of the voxel $\times$ time matrix of voxel time
courses and the first few functions of the Fourier series for the time courses.
As noted above these, additional features helped account for the noise in the
observed BOLD ratio fluctuation.

\vspace{2mm}
\noindent \textbf{Principal Components}
\vspace{2mm}
\par One approach for reducing the noise in the linear model is to include
principal components of the voxel $\times$ time voxel time course matrix.
Instead of using the entire matrix, it may be possible to just include the
first few principal components as features that explain a great deal of the
variance in the entire matrix. To get the principal components, we obtained
the singular value decomposition (SVD) of the time $\times$ time covariance
matrix. We tried this with and without first masking the voxels. To
standardize the voxels, we subtracted the column means (mean across voxels)
from the voxel by time matrix. There is also a very strong effect of mean
over time in the data that dominates other effects, so we subtracted the
row means (mean over time) as well.

As you can see in Figure \ref{fig:pca10}, which compares the variance
explained by including up to ten components, with and without masking the
voxels, masking explains more variance at each component. This trend was
observed across all subjects. So, between the better performance and the
logical rationality of using the masked data (we are not actually interested
in the behvavior of voxels outside the brain), we decided to only work with
the masked data's principal components.

\begin{figure}[ht]
	\centering
	\includegraphics[width=.5\linewidth]{../images/pcacumsumssub010.png}
 	\caption{Comparing proportion of variance explained by Subject 10's
principal components, with and without masking the data.}
 	\label{fig:pca10}
\end{figure}

An important issue to consider is how many principal components to include in
the design matrix. Figure \ref{fig:pcabox} compares the the amount of variance
explained by including successively more principal components across subjects.
A few observations should be noted. First, there is considerable variation
between subjects in how much variance the early principal components capture.
Second, by including only the first six components, it is possible for most
subjects' voxel time course matrix to capture at least 40\% of the variance.
Moving forward, we chose to include six principal components as additional
features when considering models that reduce noise. This cutoff of six
components or 40\% of the variance explained was somewhat arbitrary, with the
idea being we wanted to only include a few components without sacrificing too
much of the variance explained and that we wanted to use the same number of
components for each subject. It will be seen later that the variation between
subjects in how much variance is captured by the first six components has strong
ramifications on the results.

\begin{figure}[ht]
	\centering
	\includegraphics[width=.5\linewidth]{../images/pcaBOX.png}
 	\caption{Boxplots comparing the amount of variance captured by principal
components for each subject. The data was masked beforehand. All subjects are
able to capture at least 40\% of the variance when using just six principal
components.}
 	\label{fig:pcabox}
\end{figure}

\vspace{2mm}
\noindent \textbf{Fourier Series}
\vspace{2mm}
\par We also considered including 6 features related to the first few
functions of the Fourier series.
A full Fourier series is represented as the following:

\begin{equation}
f(x) = \frac{1}{2} \cdot a_0 + \sum_{n=1}^{\infty} a_n \cdot cos(n x) + \sum_{n=1}^{\infty} b_n \cdot  sin(n x)
\end{equation}

\noindent We wanted to represent low level sinesoidal fluctuations, which
requires a few periods over the full range of the time course
$(0, \text{num of TR})$. As such, we changed the Fourier series to:

\begin{equation}
f(x) = \frac{1}{2} \cdot a_0 + \sum_{n=1}^{\infty} a_n \cdot cos(\frac{n}{\text{num of TR}} x) + \sum_{n=1}^{\infty} b_n \cdot sin(\frac{n}{\text{num of TR}} x)
\end{equation}

\noindent We used $ \sum_{n=1}^{3} a_n \cdot cos(\frac{n}{\text{num of TR}} x) +
\sum_{n=1}^{3} b_n \cdot sin(\frac{n}{\text{num of TR}} x)$ to be 6 features to try
to get a low order sinusoidal fluctuations.

\subsection{Model Selection} \label{model_selection}

\par In order to select the best set of features for our $X$ matrix, and also
to compare the use of a single condition feature vs. each of the three
different types of conditions as three separate features, we decided to
utilize model comparison metrics; specifically, the AIC, BIC, and adjusted
$R^2$. Using a small but expressive subset of the subjects ($002$, $003$, and
$014$), we averaged the metrics across all voxels and people, an admittedly not
theoretically sound approach. We visualized values in Figures \ref{fig:AIC},
\ref{fig:BIC}, and \ref{fig:adjr2}.

\par From these plots we can observe that \textbf{(1)} separating the
conditions into individual features to convolve the hemodynamic response did
not provide much gain in these metrics and \textbf{(2)} the inclusion of the 6
principal components comparably tends to create better models than the
inclusion of the 6 Fourier series features. We initially interpreted this as a
vote to include the first 6 principal components and not the Fourier features.
Unfortunately, including the 6 principal components lead to overfitting and
collinearity with the HRF features for some subjects. These problems tended to
arise when the proportion of variance explained by the first 6 principal
components was much greater that 40 \%. Overall, we went with the 6 Fourier
features instead and observed similar t-statistics for the HRF feature in the
models with those when using the first 6 principal components when the variance
explained was around 40 \%.

\begin{figure}
\centering
	\begin{minipage}[b]{0.33\linewidth}
		\centering
		\includegraphics[width=.8\linewidth]{../images/aic_better}

		\caption{AIC}
		\label{fig:AIC}

	\end{minipage}
	\quad
	\begin{minipage}[b]{0.33\linewidth}
		\centering
		\includegraphics[width=.8\linewidth]{../images/bic_better}
		\caption{BIC}
		\label{fig:BIC}

	\end{minipage}

	\begin{minipage}[b]{0.33\linewidth}
		\centering
		\includegraphics[width=.8\linewidth]{../images/adjr2_better}
		\caption{Adjusted $R^2$}
		\label{fig:adjr2}

	\end{minipage}

\end{figure}


\section{Convolutional autoencoder}
A convolutional autoencoder(CAE) consists of two parts, namely the encoding part and decoding part. While the decoded output may not be the same as the input, the goal is to restore the input as much as possible. A sample CAE is shown in the graph. There is only a convolution layer in the encoder part and a de-convolution layer in the decoder part for this example.

\begin{figure}[htbp]
\centering{
\includegraphics[width=.5\textwidth]{CAE.png}}
\caption{Convolutional autoencoder}
\end{figure}

\subsection{Encoder}
We can put convolution layers and pooling layers in the encoding part as the normal CNN to get the encoded output. For this specific CAE here, we use a convolution layer only.
Suppose the input data is denoted as $X$, the kernel for the convolution layer is $K^1$. We can obtain the encoded output $Y$ by convolution as
\begin{align}
    K^1 * X  = Y.
\end{align}
%Then with the backpropagation, the input and the parameters can be updated as
%\begin{align}
%   dY * (K^1)^T = dX,   \\
%   dY * X      = dK^1 .
%\end{align}
If we rewrite the convolution in the form of matrix multiplication, it becomes
\begin{align}
   Y = K^1 * X = T \cdot X.
\end{align}
For the purpose of illustration, we take $K^1 \in \mathbb{R}^4$, $X \in \mathbb{R}^8$ with the stride to be 2. Then we have $Y \in \mathbb{R}^5$ and $T \in \mathbb{R}^{5*8}$.

Suppose $K^1=(k_1, k_2, k_3, k_4)$ and $X=(x_1, x_2, ..., x_8)$. The corresponding coefficient matrix $T$ can be represented as
\begin{align}
   T = P \cdot \tilde{T} &=
%  \begin{bmatrix}
%    (1-2r)  & -r      & 0       & \cdots  & 0  \\
%    -r      & (1-2r)  & -r      & \ddots  & \vdots   \\
%     0      &  \ddots & \ddots  & \ddots  & 0  \\
%    \vdots  &         & -r      & (1-2r)  & -r  \\
%     0      & \cdots  & 0       & -r      & (1-2r)
%  \end{bmatrix}
\left(
  \begin{array}{ccccccccc}
     1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\
     0 & 0 & 1 & 0 & 0 & 0 & 0 & 0 & 0 \\
     0 & 0 & 0 & 0 & 1 & 0 & 0 & 0 & 0 \\
     0 & 0 & 0 & 0 & 0 & 0 & 1 & 0 & 0 \\
     0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 1 \\
  \end{array}
\right)_{5*9}
\left(
  \begin{array}{cccccccc}
     k_3 & k_4 & 0 & 0 & 0 & 0 & 0 & 0  \\
     k_2 & k_3 & k_4 & 0 & 0 & 0 & 0 &  0 \\
     k_1 & k_2 & k_3 & k_4 & 0 & 0 & 0 & 0  \\
     0 & k_1 & k_2 & k_3 & k_4 & 0 & 0 & 0  \\
     0 & 0 & k_1 & k_2 & k_3 & k_4 & 0 & 0  \\
     0 & 0 & 0 & k_1 & k_2 & k_3 & k_4 & 0  \\
     0 & 0 & 0 & 0 & k_1 & k_2 & k_3 & k_4  \\
     0 & 0 & 0 & 0 & 0 & k_1 & k_2 & k_3   \\
     0 & 0 & 0 & 0 & 0 & 0 & k_1 & k_2  \\
  \end{array}
\right)_{9*8}\\ \notag
&=
\left(
  \begin{array}{cccccccc}
     k_3 & k_4 & 0 & 0 & 0 & 0 & 0 & 0  \\
     k_1 & k_2 & k_3 & k_4 & 0 & 0 & 0 &  0 \\
     0 & 0 & k_1 & k_2 & k_3 & k_4 & 0 &  0  \\
     0 & 0 & 0 &0 & k_1 & k_2 & k_3 & k_4   \\
     0 & 0 & 0 & 0 & 0 & 0 & k_1 & k_2    \\
  \end{array}
\right)_{5*8}.
\end{align}
Here $P$ denote the operation of stride being 2 and $\tilde{T}$ is the standard convolution operation.

The procedure can be illustrated as below.
\begin{figure}[htbp]
\centering{
\includegraphics[width=.6\textwidth]{conv.jpeg}}
\caption{Depiction of usual convolution process with 1-d input}
\end{figure}

\subsection{Decoder}
We can put unpooling layers and deconvolution layers in the decoding part to get the decoded output. For this specific CAE here, we use a deconvolution layer only.
Suppose the encoded data from the encoder is denoted as $Y$ and the restored output as $\tilde{X}$. In TensorFlow, $\tilde{X}$ is obtained from $Y$ with the function tf.nn.conv3d$\_$transpose.

Here is what tf.nn.conv3d$\_$transpose does.
It construct another CNN with a convolution layer only, which takes $Y$ as the output and $\tilde{X}$ as the input. Then  the input is updated with the backpropagation by the function gen$\_$nn$\_$ops.conv3d$\_$backprop$\_$input$\_$v2. %This function does the following operation.

If we rewrite the deconvolution in the form of matrix multiplication, it becomes
\begin{align}
   \tilde{X} = K^2 * Y = T^T \cdot Y.
\end{align}

%Suppose $Y$ can be obtained by convolution as
%\begin{align}
%    K^2 * \tilde{X}  = Y.
%\end{align}
%Then with the backpropagation, the input and the parameters can be updated as
%\begin{align}
%   dY *(K^2)^T  = d\tilde{X},   \\
%   dY * \tilde{X}      = dK^2.
%\end{align}
%After $\tilde{X}$ is updated, the $L^2$ loss function is used to compute the difference between $X$ and $\tilde{X}$.

With the same notation as above, the corresponding coefficient matrix $T^T$ can be represented as
\begin{align}
   T^T = \tilde{T}^T\cdot  P^T &=
%  \begin{bmatrix}
%    (1-2r)  & -r      & 0       & \cdots  & 0  \\
%    -r      & (1-2r)  & -r      & \ddots  & \vdots   \\
%     0      &  \ddots & \ddots  & \ddots  & 0  \\
%    \vdots  &         & -r      & (1-2r)  & -r  \\
%     0      & \cdots  & 0       & -r      & (1-2r)
%  \end{bmatrix}
\left(
  \begin{array}{ccccccccc}
     k_3 & k_2 & k_1 & 0 & 0 & 0 & 0 & 0  & 0\\
     k_4 & k_3 & k_2 & k_1 & 0 & 0 & 0 &  0 & 0\\
     0 & k_4 & k_3 & k_2 & k_1 & 0 & 0 & 0 & 0\\
     0 & 0 & k_4 & k_3 & k_2 & k_1 & 0 & 0  & 0\\
     0 & 0 & 0 & k_4 & k_3 & k_2 & k_1 & 0  & 0\\
     0 & 0 & 0 & 0 & k_4 & k_3 & k_2 & k_1  & 0\\
     0 & 0 & 0 & 0 & 0 & k_4 & k_3 & k_2 & k_1  \\
     0 & 0 & 0 & 0 & 0 & 0  & k_4 & k_3 & k_2   \\
  \end{array}
\right)_{8*9}
\left(
  \begin{array}{ccccc}
     1 & 0 & 0 & 0 & 0  \\
     0 & 0 & 0 & 0 & 0  \\
     0 & 1 & 0 & 0 & 0  \\
     0 & 0 & 0 & 0 & 0  \\
     0 & 0 & 1 & 0 & 0  \\
     0 & 0 & 0 & 0 & 0  \\
     0 & 0 & 0 & 1 & 0  \\
     0 & 0 & 0 & 0 & 0  \\
     0 & 0 & 0 & 0 & 1  \\
  \end{array}
\right)_{9*5}
\\ \notag
&=
\left(
  \begin{array}{ccccc}
     k_3 & k_1 & 0 & 0 & 0   \\
     k_4 & k_2 & 0 & 0 & 0   \\
     0 & k_3 & k_1 & 0 & 0  \\
     0 & k_4 & k_2 & 0 & 0  \\
     0 & 0 & k_3 & k_1 & 0  \\
     0 & 0 & k_4 & k_2 & 0  \\
     0 & 0 & 0 & k_3 & k_1  \\
     0 & 0 & 0 & k_4 & k_2  \\
  \end{array}
\right)_{8*5}.
\end{align}
Here $P^T$ denote the operation of padding(adding zeros) and $\tilde{T}$ is the standard deconvolution operation.

The procedure can be illustrated as below.
\begin{figure}[htbp]
\centering{
\includegraphics[width=.6\textwidth]{deconv.jpeg}}
\caption{Deconvolution}
\end{figure}

\section{Denoising by Convolutional Autoencoder}

\paragraph{Autoencoder}
    \begin{itemize}
        \item An Autoencoder is a neural network that is trained to attempt to copy its input to its output.
        \item The network consists of two parts
            \begin{itemize}
                \item An encoder function
                    \begin{equation*}
                        h=f(x).
                    \end{equation*}
                \item A decoder function that produce a reconstruct
                    \begin{equation*}
                        y=g(h).
                    \end{equation*}
            \end{itemize}
        \item The goal for training the network is to minimize the difference between the output $y$ and  the input $x$
            $$
                \min \|g(f(x))-x\|_{\ell^2}.
            $$
    \end{itemize}

A convolutional autoencoder is an autoencoder consists of convolutional layers.

A simple CAE with one hidden layer
\begin{itemize}
\item Convolution
    $$h=f(x)=K\circledast x+\operatorname{diag}(b)I.$$

\item De-convolution

    Inverse operation of the convolution to restore the input
        $$y=g(h)=\hat{K}\circledast h+\operatorname{diag}(\hat b)I.$$

\item We train the model
    $$y=g(f(x))$$
   to solve the following minimization problem
        $$\min\|y-x\|_{\ell^2}$$
\item Then we use $f$ as the encoder, and $g$ as the decoder.

\end{itemize}

    \begin{center}
        \includegraphics[width=0.8\textwidth]{CAE}
    \end{center}

\paragraph{MNIST with noise}
    \begin{itemize}
        \item We add Gaussian noise to MNIST dataset
            \begin{equation*}
                x_{ij} \leftarrow x_{ij}+\eta z,
            \end{equation*}
            where $z$ is the Gaussian random variable given by
            \begin{equation*}
                P(z)= \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{(z-\mu)^2}{2\sigma^2}}
            \end{equation*}
         \item Then construct a CAE model such that given an image with noise, it will output the denoised image.

             \begin{center}
                 \includegraphics[width=0.8\textwidth]{noise}
             \end{center}
    \end{itemize}
    \begin{center}
        \includegraphics[height=0.8\textheight]{denoise}
    \end{center}

    \paragraph{Encoder:}
    \begin{itemize}
    \footnotesize
    \item  First convolutional layer:
    \begin{equation*}
        f^0 = \theta^0(x) \in \mathbb{R}^{16\times 28\times 28},
    \end{equation*}
    with
    \begin{equation*}
        \theta^0(x) = K^0 \circledast x + {\rm{diag}(b^0) }\cdot \bm{1} \quad, \quad K^0 \in R^{16 \times 1 \times 3 \times 3}, \quad b^0 \in \mathbb R^{16}
    \end{equation*}
    \item  Second convolutional layer:
    \begin{equation*}
        f^1 = \theta^1(r^1(\sigma^1 (f^0)))\in \mathbb{R}^{8\times 14\times 14},
    \end{equation*}
            where $\sigma^1: \mathbb{R}^{16\times 28\times 28}\mapsto \mathbb{R}^{16\times 28\times 28}$ is the ReLU activation function; $r^1:\mathbb{R}^{16\times 28\times 28}\mapsto \mathbb{R}^{8\times 14\times 14}$ is the max-pooling function; and
            \begin{equation*}
                \theta^1(x) = K^1 \circledast x + {\rm{diag}(b^1) }\cdot \bm{1} \quad, \quad K^1 \in \mathbb R^{8 \times 16 \times 3 \times 3}, \quad b^1 \in \mathbb R^{8}.
            \end{equation*}
    \item  Third convolutional layer:
    \begin{equation*}
        f^2 = \theta^2(r^2(\sigma^2 (f^1)))\in \mathbb{R}^{8\times 7\times 7},
    \end{equation*}
            where $\sigma^2: \mathbb{R}^{8\times 14\times 14}\mapsto \mathbb{R}^{8\times 14\times 14}$ is the ReLU activation function; $r^2:\mathbb{R}^{8\times 14\times 14}\mapsto \mathbb{R}^{8\times 7\times 7}$ is the max-pooling function; and
            \begin{equation*}
                \theta^2(x) = K^2 \circledast x + {\rm{diag}(b^2) }\cdot \bm{1} \quad, \quad K^1 \in \mathbb R^{8 \times 8 \times 3 \times 3}, \quad b^2 \in \mathbb R^{8}.
            \end{equation*}
    \end{itemize}

    \paragraph{Output of encoder:}
        \begin{equation*}
            f = r^3(\sigma^3(f^2))\in \mathbb{R}^{8\times 4\times 4},
        \end{equation*}
            where $\sigma^3: \mathbb{R}^{8\times 7\times 7}\mapsto \mathbb{R}^{8\times 7\times 7}$ is the ReLU activation function; $r^3:\mathbb{R}^{8\times 7\times 7}\mapsto \mathbb{R}^{8\times 4\times 4}$ is the max-pooling function.

            \begin{center}
                \includegraphics[width=0.5\textwidth]{encoder}
            \end{center}

    \paragraph{Decoder:}
    \begin{itemize}
        \footnotesize
        \item First deconvolutional layer:
            \begin{equation*}
                g^1=\hat\theta^1 (p^1(f))\in \mathbb{R}^{8\times 7\times 7},
            \end{equation*}
            where $p^1: \mathbb{R}^{8\times 4\times 4}\mapsto \mathbb{R}^{8\times 7\times 7}$ is the up-pooling function; and
            \begin{equation*}
                \hat\theta^1(x) = \hat K^1 \circledast x + {\rm{diag}(\hat b^1) }\cdot \bm{1} \quad, \quad \hat K^1 \in \mathbb R^{8 \times 8 \times 3 \times 3}, \quad \hat b^1 \in \mathbb R^{8}.
            \end{equation*}
        \item Second deconvolutional layer:
            \begin{equation*}
                g^2=\hat\theta^2 (p^2(\hat\sigma^2(g)))\in \mathbb{R}^{8\times 14\times 14},
            \end{equation*}
            where $\hat\sigma^2: \mathbb{R}^{8\times 7\times 7}\mapsto \mathbb{R}^{8\times 7\times 7}$ is the ReLU activation function;
            $p^2: \mathbb{R}^{8\times 7\times 7}\mapsto \mathbb{R}^{8\times 14\times 14}$ is the up-pooling function; and
            \begin{equation*}
                \hat\theta^2(x) = \hat K^2 \circledast x + {\rm{diag}(\hat b^2) }\cdot \bm{1} \quad, \quad \hat K^2 \in \mathbb R^{8 \times 8 \times 3 \times 3}, \quad \hat b^2 \in \mathbb R^{8}.
            \end{equation*}
        \item Third deconvolutional layer:
            \begin{equation*}
                g^3=\hat\theta^3(p^3(\hat\sigma^3(g)))\in \mathbb{R}^{16\times 28\times 28},
            \end{equation*}
            where $\hat\sigma^3: \mathbb{R}^{8\times 14\times 14}\mapsto \mathbb{R}^{8\times 14\times 14}$ is the ReLU activation function;
            $p^3: \mathbb{R}^{8\times 14\times 14}\mapsto \mathbb{R}^{16\times 28\times 28}$ is the up-pooling function; and
            \begin{equation*}
                \hat\theta^3(x) = \hat K^3 \circledast x + {\rm{diag}(\hat b^3) }\cdot \bm{1} \quad, \quad \hat K^3 \in \mathbb R^{16 \times 8 \times 3 \times 3}, \quad \hat b^3 \in \mathbb R^{16}.
            \end{equation*}
    \end{itemize}

\paragraph{Output of decoder:}
        \begin{equation*}
            f = \hat\theta^4 (\hat\sigma^4(f^2))\in \mathbb{R}^{1\times 28\times 28},
        \end{equation*}
            where $\hat\sigma^4: \mathbb{R}^{16\times 28\times 28}\mapsto \mathbb{R}^{16\times 28\times 28}$ is the ReLU activation function;  and
            \begin{equation*}
                \hat\theta^4(x) = \hat K^4 \circledast x + {\rm{diag}(\hat b^4) }\cdot \bm{1} \quad, \quad \hat K^4 \in \mathbb R^{1 \times 16 \times 3 \times 3}, \quad \hat b^4 \in \mathbb R^{1}.
            \end{equation*}

            \begin{center}
                \includegraphics[width=0.5\textwidth]{decoder}
            \end{center}
   After training 100 epochs:

    \begin{center}
        \includegraphics[width=0.8\textwidth]{result}
    \end{center}

\section{Linear encoder, nonlinear encoder and ``enhanced'' encoder}
Deep Neural Network can naturally be viewed as nonlinear encoders.

Given the data $x\in \mathbb R^d$, we consider the nonlinear encoder
\begin{equation}\label{nonlinear-encoder0}
f(x)=\sigma (W_1x+b_1),  \quad W_1\in \mathbb R^{c\times d}.
\end{equation}
\begin{equation}\label{nonlinear-encoder}
f(x)=W_2\sigma (W_1x+b_1)+b_2,  \quad W_1\in \mathbb R^{c\times d},
W_2\in\mathbb R^{c\times c}
\end{equation}

We now combine the nonlinear \eqref{nonlinear-encoder} with a linear
encoder to get the following ``enhanced'' encoder:
\begin{equation}\label{enhanced-encoder}
\tilde f(x)=f(x)+\tilde W_1x +\tilde b_1\quad \tilde W_1\in \mathbb R^{c\times d}.
\end{equation}
where
\begin{equation}
\tilde b_1=0 \mbox{ if } b_2=0 \mbox{ or if $f(x)$ is given by \eqref{nonlinear-encoder0}}
\end{equation}

\begin{lemma}
There exists .... such that
$$
W_2\sigma (W_1x+b_1)+b_2\equiv x
$$
\end{lemma}
\begin{theorem}
Nonlinear encoder recovers linear encoder if $d\le c/2$(?)
\end{theorem}
Can we make the following statement:
\begin{remark}
  Nonlinear encoder can recover linear encoder in a smaller dimension?
  or coarser grid?
\end{remark}