\chapter{Linear Algebra}
\textbf{Definition:} Linear algebra is the branch of mathematics concerning linear equations and their representations through matrices and vector spaces. Linear algebra is central to almost all areas of mathematics.

\section{Vectors}

\subsection{Magnitude and direction}
The \textbf{magnitude} of a vector $v=\vec{PQ}$ is the distance between the initial point P and the end point Q. In symbols the magnitude of $\vec{PQ}$ is written as $\Vert\vec{PQ}\Vert$:

\[
\Vert \vec{PQ} \Vert = \sqrt{ \sum_{i=1}^{d} (q_i - p_i)^2}
\]

If the point P is the origin, then the magnitude of $v$ is:
\[
\Vert v \Vert = \sqrt{ \sum_{i=1}^{d} v_i^2 }
\]

The \textbf{direction} of a vector is the measure of the angle it makes with a horizontal line.
In $\mathbb{R}^2$ the direction of $\vec{PQ}$ with $P=(x_1,y_1)$ and $Q=(x_2,y_2)$ can be calculated with:
\[
\theta = \arctan \left( \frac{y_2-y_1}{x_2-x_1} \right)
\]

More generically, the angle between two vectors $\mathbf{a}$ and $\mathbf{b}$ can be calculated from the dot product formula:
\[
\cos(\theta) = \frac{ \mathbf{a} \cdot \mathbf{b} }{ \Vert \mathbf{a} \Vert  \Vert \mathbf{b} \Vert }
\]

\subsection{Dot product or Scalar product}
The dot product of two vectors $\mathbf{a} = [a_1, a_2, ..., a_n]$ and $\mathbf{b} = [b_1, b_2, ..., b_n]$ is defined as:

\[
\mathbf{a} \cdot \mathbf{b} = \sum_{i=1}^{n}a_i b_i = a_1 b_1 + a_2 b_2 + ... + a_n b_n
\]

The dot product can also be written as:

\[ \mathbf{a} \cdot \mathbf{b} =\mathbf{a} ^{T}\mathbf {b} \]

Here, $\displaystyle \mathbf {a} ^{T}$ means the transpose of a $\displaystyle \mathbf {a} $.

Finally, the dot product of two Euclidean vectors  $\mathbf{a}$ and $\mathbf{b}$ is defined by:
\[
\mathbf{a} \cdot \mathbf{b} = \Vert \mathbf{a} \Vert  \Vert \mathbf{b} \Vert \cos(\theta)
\]

where $\theta$ is the angle between $\mathbf{a}$ and $\mathbf{b}$.

\section{Vector spaces}
$\cal G$ is a group for the operation $\otimes$ if:
\begin{enumerate}
	\item $\forall a,b\in{\cal G}\Rightarrow a\otimes b\in\cal G$: a group is
	{\it closed}.
	\item $(a\otimes b)\otimes c = a\otimes (b\otimes c)$: a group is
	{\it associative}.
	\item $\exists e\in{\cal G}$ so that $a\otimes e=e\otimes a=a$: there exists a
	{\it unit element}.
	\item $\forall a\in{\cal G}\exists \overline{a}\in{\cal G}$ so that $a\otimes\overline{a}=e$:
	each element has an {\it inverse}.
\end{enumerate}
If\\
\hspace*{4.5mm}5. $a\otimes b=b\otimes a$

the group is called {\it Abelian} or {\it commutative}.
Vector spaces form an Abelian group for addition and multiplication:
$1\cdot\vec{a}=\vec{a}$, $\lambda(\mu\vec{a})=(\lambda\mu)\vec{a}$,
$(\lambda+\mu)(\vec{a}+\vec{b})=\lambda\vec{a}+\lambda\vec{b}+\mu\vec{a}+\mu\vec{b}$.

$W$ is a {\it linear subspace} if $\forall \vec{w}_1,\vec{w}_2\in W$
holds: $\lambda\vec{w}_1+\mu\vec{w}_2\in W$.

$W$ is an {\it invariant subspace} of $V$ for the operator $A$ if
$\forall\vec{w}\in W$ holds: $A\vec{w}\in W$.

\section{Basis}
For an orthogonal basis holds: $(\vec{e}_i,\vec{e}_j)=c\delta_{ij}$. For an
orthonormal basis holds:  $(\vec{e}_i,\vec{e}_j)=\delta_{ij}$.

The set vectors $\{\vec{a}_n\}$ is linear independent if:
\[
\sum\limits_i\lambda_i\vec{a}_i=0~~\Leftrightarrow~~\forall_i\lambda_i=0
\]
The set $\{\vec{a}_n\}$ is a basis if it is 1. independent and 2.
$V=<\vec{a}_1,\vec{a_2},...>=\sum\lambda_i\vec{a}_i$.

\section{Matrix calculus}
\subsection{Basic operations}
For the matrix multiplication of matrices $A=a_{ij}$ and $B=b_{kl}$ holds
with $^r$ the row index and $^k$ the column index:
\[
A^{r_1k_1}\cdot B^{r_2k_2}=C^{r_1k_2}~~,~~(AB)_{ij}=\sum_ka_{ik}b_{kj}
\]
where $^r$ is the number of rows and $^k$ the number of columns.

The {\it transpose} of $A$ is defined by: $a_{ij}^T=a_{ji}$.
For this holds $(AB)^T=B^TA^T$ and $(A^T)^{-1}=(A^{-1})^T$. For the
{\it inverse matrix} holds: $(A\cdot B)^{-1}=B^{-1}\cdot A^{-1}$. The inverse
matrix $A^{-1}$ has the property that $A\cdot A^{-1}= \mathbb{I}$ and can be found by
diagonalization: $(A_{ij}| \mathbb{I})\sim( \mathbb{I}|A_{ij}^{-1})$.

The inverse of a $2\times2$ matrix is:
\[
\left(\begin{array}{cc}a&b\\ c&d\end{array}\right)^{-1}=\frac{1}{ad-bc}
\left(\begin{array}{cc}d&-b\\ -c&a\end{array}\right)
\]

The {\it determinant function} $D=\det(A)$ is defined by:
\[
\det(A)=D(\vec{a}_{*1},\vec{a}_{*2},...,\vec{a}_{*n})
\]
For the determinant $\det(A)$ of a matrix $A$ holds:
$\det(AB)=\det(A)\cdot\det(B)$. Een $2\times2$ matrix has determinant:
\[
\det\left(\begin{array}{cc}a&b\\ c&d \end{array}\right)=ad-cb
\]
The derivative of a matrix is a matrix with the derivatives of the coefficients:
\[
\frac{dA}{dt}=\frac{da_{ij}}{dt}~~~\mbox{and}~~~\frac{dAB}{dt}=B\frac{dA}{dt}+A\frac{dB}{dt}
\]
The derivative of the determinant is given by:
\[
\frac{d\det(A)}{dt}=D(\frac{d\vec{a}_1}{dt},...,\vec{a}_n)+
D(\vec{a}_1,\frac{d\vec{a}_2}{dt},...,\vec{a}_n)+...+D(\vec{a}_1,...,\frac{d\vec{a}_n}{dt})
\]
When the rows of a matrix are considered as vectors the {\it row rank} of a
matrix is the number of independent vectors in this set. Similar for the
{\it column rank}. The row rank equals the column rank for each matrix.

Let $\tilde{A}:\tilde{V}\rightarrow\tilde{V}$ be the complex extension of the
real linear operator $A:V\rightarrow V$ in a finite dimensional $V$.
Then $A$ and $\tilde{A}$ have the same caracteristic equation.

When $A_{ij}\in \mathbb{R}$ and $\vec{v}_1+i\vec{v_2}$ is an eigenvector of $A$
at eigenvalue $\lambda=\lambda_1+i\lambda_2$, than holds:
\begin{enumerate}
	\item $A\vec{v}_1=\lambda_1\vec{v}_1-\lambda_2\vec{v}_2$ and $A\vec{v}_2=\lambda_2\vec{v}_1+\lambda_1\vec{v}_2$.
	\item $\vec{v}^{~*}=\vec{v}_1-i\vec{v}_2$ is an eigenvalue at $\lambda^*=\lambda_1-i\lambda_2$.
	\item The linear span $<\vec{v}_1,\vec{v}_2>$ is an invariant subspace of $A$.
\end{enumerate}
If $\vec{k}_n$ are the columns of $A$, than the transformed space of $A$ is
given by:
\[
R(A)=<A\vec{e}_1,...,A\vec{e}_n>=<\vec{k}_1,...,\vec{k}_n>
\]
If the columns $\vec{k}_n$ of a $n\times m$ matrix $A$ are independent, than
the nullspace ${\cal N}(A)=\{\vec{0}\}$.

\subsection{Matrix equations}
We start with the equation
\[
A\cdot\vec{x}=\vec{b}
\]
and $\vec{b}\neq\vec{0}$. If $\det(A)=0$ the only solution is $\vec{0}$. If
$\det(A)\neq0$ there exists exactly one solution $\neq\vec{0}$.

The equation
\[
A\cdot\vec{x}=\vec{0}
\]
has exactly one solution $\neq\vec{0}$ if $\det(A)=0$, and if
$\det(A)\neq0$ the solution is $\vec{0}$.

Cramer's rule for the solution of systems of linear equations is: let the
system be written as
\[
A\cdot\vec{x}=\vec{b}\equiv\vec{a}_1x_1+...+\vec{a}_nx_n=\vec{b}
\]
then $x_j$ is given by:
\[
x_j=\frac{D(\vec{a}_1,...,\vec{a}_{j-1},\vec{b},\vec{a}_{j+1},...,\vec{a}_n)}{\det(A)}
\]

\section{Linear transformations}
A transformation $A$ is linear if:
$A(\lambda\vec{x}+\beta\vec{y})=\lambda A\vec{x}+\beta A\vec{y}$.

Some common linear transformations are:
\begin{center}
	\begin{tabular}{||p{7cm}|p{6cm}||}
		\hline
		\bf Transformation type & \bf Equation\\
		\hline
		\hline
		Projection on the line $<\vec{a}>$             &$P(\vec{x})=(\vec{a},\vec{x})\vec{a}/(\vec{a},\vec{a})$\\
		Projection on the plane $(\vec{a},\vec{x})=0$ &$Q(\vec{x})=\vec{x}-P(\vec{x})$\\
		Mirror image in the line $<\vec{a}>$             &$S(\vec{x})=2P(\vec{x})-\vec{x}$\\
		Mirror image in the plane $(\vec{a},\vec{x})=0$&$T(\vec{x})=2Q(\vec{x})-\vec{x}=\vec{x}-2P(\vec{x})$\\
		\hline
	\end{tabular}
\end{center}
For a projection holds: $\vec{x}-P_W(\vec{x})\perp P_W(\vec{x})$ and
$P_W(\vec{x})\in W$.

If for a transformation $A$ holds: $(A\vec{x},\vec{y})=(\vec{x},A\vec{y})=(A\vec{x},A\vec{y})$,
than $A$ is a projection.

Let $A:W\rightarrow W$ define a linear transformation; we define:
\begin{itemize}
	\item If $S$ is a subset of $V$: $A(S):=\{A\vec{x}\in W|\vec{x}\in S\}$
	\item If $T$ is a subset of $W$: $A^\leftarrow(T):=\{\vec{x}\in V|A(\vec{x})\in T\}$
\end{itemize}
Than $A(S)$ is a linear subspace of $W$ and the {\it inverse transformation}
$A^\leftarrow(T)$ is a linear subspace of $V$. From this follows that $A(V)$ is
the {\it image space} of $A$, notation: ${\cal R}(A)$. $A^\leftarrow(\vec{0})=E_0$
is a linear subspace of $V$, the {\it null space} of $A$, notation:
${\cal N}(A)$. Then the following holds:
\[
{\rm dim}({\cal N}(A))+{\rm dim}({\cal R}(A))={\rm dim}(V)
\]

\section{Plane and line}
The equation of a line that contains the points $\vec{a}$ and $\vec{b}$ is:
\[
\vec{x}=\vec{a}+\lambda(\vec{b}-\vec{a})=\vec{a}+\lambda\vec{r}
\]
The equation of a plane is:
\[
\vec{x}=\vec{a}+\lambda(\vec{b}-\vec{a})+\mu(\vec{c}-\vec{a})=\vec{a}+\lambda\vec{r}_1+\mu\vec{r}_2
\]
When this is a plane in $ \mathbb{R}^3$, the {\it normal vector} to this plane is given
by:
\[
\vec{n}_V=\frac{\vec{r}_1\times\vec{r}_2}{|\vec{r}_1\times\vec{r}_2|}
\]
A line can also be described by the points for which the line equation
$\ell$: $(\vec{a},\vec{x})+b=0$ holds, and for a plane V: $(\vec{a},\vec{x})+k=0$.
The normal vector to V is than: $\vec{a}/|\vec{a}|$.

The distance $d$ between 2 points $\vec{p}$ and $\vec{q}$ is given by
$d(\vec{p},\vec{q})=\|\vec{p}-\vec{q}\|$.

In $ \mathbb{R}^2$ holds:
The distance of a point $\vec{p}$ to the line $(\vec{a},\vec{x})+b=0$ is
\[
d(\vec{p},\ell)=\frac{|(\vec{a},\vec{p})+b|}{|\vec{a}|}
\]
Similarly in $ \mathbb{R}^3$:
The distance of a point $\vec{p}$ to the plane $(\vec{a},\vec{x})+k=0$ is
\[
d(\vec{p},V)=\frac{|(\vec{a},\vec{p})+k|}{|\vec{a}|}
\]
This can be generalized for $ \mathbb{R}^n$ and $\mathbb{C}^n$ (theorem from Hesse).

\section{Coordinate transformations}
The linear transformation $A$ from $\mathbb{K}^n\rightarrow\mathbb{K}^m$ is given by
($\mathbb{K}= \mathbb{R}$ of $\mathbb{C}$):
\[
\vec{y}=A^{m\times n}\vec{x}
\]
where a column of $A$ is the image of a base vector in the original.

The matrix $A_\alpha^\beta$ transforms a vector given w.r.t. a basis
$\alpha$ into a vector w.r.t. a basis $\beta$. It is given by:
\[
A_\alpha^\beta=\left(\beta(A\vec{a}_1),...,\beta(A\vec{a}_n)\right)
\]
where $\beta(\vec{x})$ is the representation of the vector $\vec{x}$
w.r.t.\ basis $\beta$.

The {\it transformation matrix} $S_\alpha^\beta$ transforms vectors from
coordinate system $\alpha$ into coordinate system $\beta$:
\[
S_\alpha^\beta:= \mathbb{I}_\alpha^\beta=\left(\beta(\vec{a}_1),...,\beta(\vec{a}_n)\right)
\]
and $S_\alpha^\beta\cdot S_\beta^\alpha= \mathbb{I}$

The matrix of a transformation $A$ is than given by:
\[
A_\alpha^\beta=\left(A_\alpha^\beta\vec{e}_1,...,A_\alpha^\beta\vec{e}_n\right)
\]
For the transformation of matrix operators to another coordinate system holds:
$A_\alpha^\delta=S_\lambda^\delta A_\beta^\lambda S_\alpha^\beta$,
$A_\alpha^\alpha=S_\beta^\alpha A_\beta^\beta S_\alpha^\beta$ and
$(AB)_\alpha^\lambda=A_\beta^\lambda B_\alpha^\beta$.

Further is $A_\alpha^\beta=S_\alpha^\beta A_\alpha^\alpha$,
$A_\beta^\alpha=A_\alpha^\alpha S_\beta^\alpha$. A vector is transformed via
$X_\alpha=S_\alpha^\beta X_\beta$.

\section{Eigen values}
The {\it eigenvalue equation}
\[
A\vec{x}=\lambda\vec{x}
\]
with {\it eigenvalues} $\lambda$ can be solved with
$(A-\lambda \mathbb{I})=\vec{0}\Rightarrow\det(A-\lambda \mathbb{I})=0$. The eigenvalues
follow from this characteristic equation. The following is true:
$\det(A)=\prod\limits_i\lambda_i$ and
${\rm Tr}(A)=\sum\limits_ia_{ii}=\sum\limits_i\lambda_i$.

The eigen values $\lambda_i$ are independent of the chosen basis.
The matrix of $A$ in a basis of eigenvectors, with $S$ the transformation matrix
to this basis, $S=(E_{\lambda_1},...,E_{\lambda_n})$, is given by:
\[
\Lambda=S^{-1}AS={\rm diag}(\lambda_1,...,\lambda_n)
\]
When 0 is an eigen value of $A$ than $E_0(A)={\cal N}(A)$.

When $\lambda$ is an eigen value of $A$ holds: $A^n\vec{x}=\lambda^n\vec{x}$.

\section{Transformation types}
\subsubsection{Isometric transformations}
A transformation is {\it isometric} when: $\|A\vec{x}\|=\|\vec{x}\|$. This
implies that the eigen values of an isometric transformation are given by
$\lambda=\exp(i\varphi)\Rightarrow|\lambda|=1$. Than also holds:
$(A\vec{x},A\vec{y})=(\vec{x},\vec{y})$.

When $W$ is an invariant subspace if the isometric transformation $A$ with
dim$(A)<\infty$, than also $W^\perp$ is an invariante subspace.

\subsubsection{Orthogonal transformations}
A transformation $A$ is {\it orthogonal} if $A$ is isometric {\it and}
the inverse $A^\leftarrow$ exists. For an orthogonal transformation $O$ holds
$O^TO= \mathbb{I}$, so: $O^T=O^{-1}$. If $A$ and $B$ are orthogonal, than $AB$ and
$A^{-1}$ are also orthogonal.

Let $A:V\rightarrow V$ be orthogonal with dim$(V)<\infty$. Than $A$ is:

{\bf Direct orthogonal} if $\det(A)=+1$. $A$ describes a rotation.
A rotation in $ \mathbb{R}^2$ through angle $\varphi$ is given by:
\[
R=
\left(\begin{array}{cc}
\cos(\varphi)&-\sin(\varphi)\\
\sin(\varphi)&\cos(\varphi)
\end{array}\right)
\]
So the rotation angle $\varphi$ is determined by Tr$(A)=2\cos(\varphi)$
with $0\leq\varphi\leq\pi$. Let $\lambda_1$ and $\lambda_2$ be the roots of
the characteristic equation, than also holds:
$\Re(\lambda_1)=\Re(\lambda_2)=\cos(\varphi)$, and $\lambda_1=\exp(i\varphi)$,
$\lambda_2=\exp(-i\varphi)$.

In $ \mathbb{R}^3$ holds: $\lambda_1=1$, $\lambda_2=\lambda_3^*=\exp(i\varphi)$. A
rotation over $E_{\lambda_1}$ is given by the matrix
\[
\left(\begin{array}{ccc}
1&0&0\\
0&\cos(\varphi)&-\sin(\varphi)\\
0&\sin(\varphi)&\cos(\varphi)
\end{array}\right)
\]
{\bf Mirrored orthogonal} if $\det(A)=-1$. Vectors from $E_{-1}$ are mirrored
by $A$ w.r.t.\ the invariant subspace $E^\perp_{-1}$. A mirroring in $ \mathbb{R}^2$
in $<(\cos(\frac{\varphi}{2}),\sin(\frac{\varphi}{2}))>$ is given by:
\[
S=
\left(\begin{array}{cc}
\cos(\varphi)&\sin(\varphi)\\
\sin(\varphi)&-\cos(\varphi)
\end{array}\right)
\]
Mirrored orthogonal transformations in $ \mathbb{R}^3$ are rotational mirrorings:
rotations of axis $<\vec{a}_1>$ through angle $\varphi$ and mirror plane
$<\vec{a}_1>^\perp$. The matrix of such a transformation is given by:
\[
\left(\begin{array}{ccc}
-1&0&0\\
0&\cos(\varphi)&-\sin(\varphi)\\
0&\sin(\varphi)&\cos(\varphi)
\end{array}\right)
\]
For all orthogonal transformations $O$ in $ \mathbb{R}^3$ holds that
$O(\vec{x})\times O(\vec{y})=O(\vec{x}\times\vec{y})$.

$ \mathbb{R}^n$ $(n<\infty)$ can be decomposed in invariant subspaces with dimension
1 or 2 for each orthogonal transformation.

\subsubsection{Unitary transformations}
Let $V$ be a complex space on which an inner product is defined. Than a linear
transformation $U$ is {\it unitary} if $U$ is isometric {\it and} its inverse
transformation $A^\leftarrow$ exists. A $n\times n$ matrix is unitary if
$U^HU= \mathbb{I}$. It has determinant $|\det(U)|=1$. Each isometric transformation
in a finite-dimensional complex vector space is unitary.

{\bf Theorem}: for a $n\times n$ matrix $A$ the following statements are
equivalent:
\begin{enumerate}
	\item $A$ is unitary,
	\item The columns of $A$ are an orthonormal set,
	\item The rows of $A$ are an orthonormal set.
\end{enumerate}

\subsubsection{Symmetric transformations}
A transformation $A$ on $ \mathbb{R}^n$ is {\it symmetric} if
$(A\vec{x},\vec{y})=(\vec{x},A\vec{y})$. A matrix $A\in\mathbb{M}^{n\times n}$
is symmetric if $A=A^T$. A linear operator is only symmetric if its matrix
w.r.t.\ an arbitrary basis is symmetric. All eigenvalues of a symmetric
transformation belong to $ \mathbb{R}$. The different eigenvectors are mutually
perpendicular. If $A$ is symmetric, than $A^T=A=A^H$ on an orthogonal basis.

For each matrix $B\in\mathbb{M}^{m\times n}$ holds: $B^TB$ is symmetric.

\subsubsection{Hermitian transformations}
A transformation $H:V\rightarrow V$ with $V=\mathbb{C}^n$ is {\it Hermitian} if
$(H\vec{x},\vec{y})=(\vec{x},H\vec{y})$. The {\it Hermitian conjugated}
transformation $A^H$ of $A$ is: $[a_{ij}]^H=[a_{ji}^*]$. An alternative
notation is: $A^H=A^\dagger$. The inner product of two vectors $\vec{x}$ and
$\vec{y}$ can now be written in the form: $(\vec{x},\vec{y})=\vec{x}^H\vec{y}$.

If the transformations $A$ and $B$ are Hermitian, than their product $AB$ is
Hermitian if:\\ $[A,B]=AB-BA=0$. $[A,B]$ is called the {\it commutator} of $A$
and $B$.

The eigenvalues of a Hermitian transformation belong to $ \mathbb{R}$.

A matrix representation can be coupled with a Hermitian operator $L$.
W.r.t.\ a basis $\vec{e}_i$ it is given by $L_{mn}=(\vec{e}_m,L\vec{e}_n)$.

\subsubsection{Normal transformations}
For each linear transformation $A$ in a complex vector space $V$ there exists
exactly one linear transformation $B$ so that $(A\vec{x},\vec{y})=(\vec{x},B\vec{y})$.
This $B$ is called the {\it adjungated transformation} of $A$. Notation:
$B=A^*$.  The following holds: $(CD)^*=D^*C^*$. $A^*=A^{-1}$ if $A$ is unitary
and $A^*=A$ if $A$ is Hermitian.

{\bf Definition}: the linear transformation $A$ is {\it normal} in a complex
vector space $V$ if $A^*A=AA^*$. This is only the case if for its matrix $S$
w.r.t.\ an orthonormal basis holds: $A^\dagger A=AA^\dagger$.

If $A$ is normal holds:
\begin{enumerate}
	\item For all vectors $\vec{x}\in V$ and a normal transformation $A$ holds:
	\[
	(A\vec{x},A\vec{y})=(A^*A\vec{x},\vec{y})=(AA^*\vec{x},\vec{y})=(A^*\vec{x},A^*\vec{y})
	\]
	\item $\vec{x}$ is an eigenvector of $A$ if and only if $\vec{x}$ is an
	eigenvector of $A^*$.
	\item Eigenvectors of $A$ for different eigenvalues are mutually perpendicular.
	\item If $E_\lambda$ if an eigenspace from $A$ than the orthogonal complement
	$E_\lambda^\perp$ is an invariant subspace of $A$.
\end{enumerate}
Let the different roots of the characteristic equation of $A$ be $\beta_i$ with
multiplicities $n_i$. Than the dimension of each eigenspace $V_i$ equals
$n_i$. These eigenspaces are mutually perpendicular and each vector
$\vec{x}\in V$ can be written in exactly one way as
\[
\vec{x}=\sum_i\vec{x}_i~~~\mbox{with}~~~\vec{x}_i\in V_i
\]
This can also be written as: $\vec{x}_i=P_i\vec{x}$ where $P_i$ is a projection
on $V_i$. This leads to the {\it spectral mapping theorem}: let $A$ be a normal
transformation in a complex vector space $V$ with dim$(V)=n$. Than:
\begin{enumerate}
	\item There exist projection transformations $P_i$, $1\leq i\leq p$, with the
	properties
	\begin{itemize}
		\item $P_i\cdot P_j=0$ for $i\neq j$,
		\item $P_1+...+P_p= \mathbb{I}$,
		\item ${\rm dim}P_1(V)+...+{\rm dim}P_p(V)=n$
	\end{itemize}
	and complex numbers $\alpha_1,...,\alpha_p$ so that
	$A=\alpha_1P_1+...+\alpha_pP_p$.
	\item If $A$ is unitary than holds $|\alpha_i|=1~\forall i$.
	\item If $A$ is Hermitian than $\alpha_i\in \mathbb{R}~\forall i$.
\end{enumerate}

\subsubsection{Complete systems of commuting Hermitian transformations}
Consider $m$ Hermitian linear transformations $A_i$ in a $n$ dimensional
complex inner product space $V$. Assume they mutually commute.

{\bf Lemma}: if $E_\lambda$ is the eigenspace for eigenvalue $\lambda$ from
$A_1$, than $E_\lambda$ is an invariant subspace of all transformations
$A_i$. This means that if $\vec{x}\in E_\lambda$, than $A_i\vec{x}\in E_\lambda$.

{\bf Theorem}. Consider $m$ commuting Hermitian matrices $A_i$. Than there
exists a unitary matrix $U$ so that all matrices $U^\dagger A_iU$ are diagonal.
The columns of $U$ are the common eigenvectors of all matrices $A_j$.

If all eigenvalues of a Hermitian linear transformation in a $n$-dimensional
complex vector space differ, than the normalized eigenvector is known except
for a phase factor $\exp(i\alpha)$.

{\bf Definition}: a commuting set Hermitian transformations is called
{\it complete} if for each set of two common eigenvectors $\vec{v}_i,\vec{v}_j$
there exists a transformation $A_k$ so that $\vec{v}_i$ and $\vec{v}_j$ are
eigenvectors with different eigenvalues of $A_k$.

Usually a commuting set is taken as small as possible. In quantum physics one
speaks of commuting observables. The required number of commuting obsieervables
equals the number of quantum numbers required to characterize a state.

\section{Homogeneous coordinates}
Homogeneous coordinates are used if one wants to combine both rotations and
translations in {\it one} matrix transformation. An extra coordinate is
introduced to describe the non-linearities. Homogeneous coordinates are derived
from cartesian coordinates as follows:
\[
\left(\begin{array}{c}x\\ y\\ z\end{array}\right)_{\rm cart}=
\left(\begin{array}{c}wx\\ wy\\ wz\\ w\end{array}\right)_{\rm hom}=
\left(\begin{array}{c}X\\ Y\\ Z\\ w\end{array}\right)_{\rm hom}
\]
so $x=X/w$, $y=Y/w$ and $z=Z/w$. Transformations in homogeneous coordinates
are described by the following matrices:
\begin{enumerate}
	\item Translation along vector $(X_0, Y_0, Z_0, w_0)$:
	\[
	T=\left(\begin{array}{cccc}
	w_0&0&0&X_0\\
	0&w_0&0&Y_0\\
	0&0&w_0&Z_0\\
	0&0&0&w_0
	\end{array}\right)
	\]
	\item Rotations of the $x,y,z$ axis, resp. through angles $\alpha,\beta,\gamma$:
	\[
	R_x(\alpha)=\left(\begin{array}{cccc}
	1&0&0&0\\
	0&\cos\alpha&-\sin\alpha&0\\
	0&\sin\alpha&\cos\alpha&0\\
	0&0&0&1
	\end{array}\right)~~~~
	R_y(\beta)=\left(\begin{array}{cccc}
	\cos\beta&0&\sin\beta&0\\
	0&1&0&0\\
	-\sin\beta&0&\cos\beta&0\\
	0&0&0&1
	\end{array}\right)~~~~
	\]
	\[
	R_z(\gamma)=\left(\begin{array}{cccc}
	\cos\gamma&-\sin\gamma&0&0\\
	\sin\gamma&\cos\gamma&0&0\\
	0&0&1&0\\
	0&0&0&1
	\end{array}\right)
	\]
	\item A perspective projection on image plane $z=c$ with the center of
	projection in the origin. This transformation has no inverse.
	\[
	P(z=c)=\left(\begin{array}{cccc}
	1&0&0&0\\
	0&1&0&0\\
	0&0&1&0\\
	0&0&1/c&0
	\end{array}\right)
	\]
\end{enumerate}

\section{Inner product spaces}
A complex inner product on a complex vector space is defined as follows:
\begin{enumerate}
	\item $(\vec{a},\vec{b})=\overline{(\vec{b},\vec{a})}$,
	\item $(\vec{a},\beta_1\vec{b}_1+\beta_2\vec{b}_2)=\beta_1(\vec{a},\vec{b}_1)+\beta_2(\vec{a},\vec{b}_2)$
	for all $\vec{a},\vec{b}_1,\vec{b}_2\in V$ and $\beta_1,\beta_2\in\mathbb{C}$.
	\item $(\vec{a},\vec{a})\geq0$ for all $\vec{a}\in V$,
	$(\vec{a},\vec{a})=0$ if and only if $\vec{a}=\vec{0}$.
\end{enumerate}
Due to (1) holds: $(\vec{a},\vec{a})\in \mathbb{R}$. The {\it inner product space} $\mathbb{C}^n$ is
the complex vector space on which a complex inner product is defined by:
\[
(\vec{a},\vec{b})=\sum_{i=1}^na_i^*b_i
\]
For function spaces holds:
\[
(f,g)=\int\limits_a^bf^*(t)g(t)dt
\]
For each $\vec{a}$ the length $\|\vec{a}\|$ is defined by:
$\|\vec{a}\|=\sqrt{(\vec{a},\vec{a})}$. The following holds:
$\|\vec{a}\|-\|\vec{b}\|\leq\|\vec{a}+\vec{b}\|\leq\|\vec{a}\|+\|\vec{b}\|$,
and with $\varphi$ the angle between $\vec{a}$ and $\vec{b}$ holds:
$(\vec{a},\vec{b})=\|\vec{a}\|\cdot\|\vec{b}\|\cos(\varphi)$.

Let $\{\vec{a}_1,...,\vec{a}_n\}$ be a set of vectors in an inner product space
$V$. Than the {\it Gramian G} of this set is given by: $G_{ij}=(\vec{a}_i,\vec{a}_j)$.
The set of vectors is independent if and only if $\det(G)=0$.

A set is {\it orthonormal} if $(\vec{a}_i,\vec{a}_j)=\delta_{ij}$.
If $\vec{e}_1,\vec{e}_2,...$ form an orthonormal row in an infinite dimensional
vector space Bessel's inequality holds:
\[
\|\vec{x}\|^2\geq\sum_{i=1}^\infty|(\vec{e}_i,\vec{x})|^2
\]
The equal sign holds if and only if
$\lim\limits_{n\rightarrow\infty}\|\vec{x}_n-\vec{x}\|=0$.

The inner product space $\ell^2$ is defined in $\mathbb{C}^\infty$ by:
\[
\ell^2=\left\{\vec{a}=(a_1,a_2,...)~|~\sum_{n=1}^\infty|a_n|^2<\infty\right\}
\]
A space is called a {\it Hilbert space} if it is $\ell^2$ and if also holds:
$\lim\limits_{n\rightarrow\infty}|a_{n+1}-a_n|=0$.

\section{The Laplace transformation}
The class LT exists of functions for which holds:
\begin{enumerate}
	\item On each interval $[0,A]$, $A>0$ there are no more than a finite number of
	discontinuities and each discontinuity has an upper - and lower limit,
	\item $\exists t_0\in[0,\infty>$ and $a,M\in \mathbb{R}$ so that for $t\geq t_0$
	holds: $|f(t)|\exp(-at)<M$.
\end{enumerate}
Than there exists a Laplace transform for $f$.

The Laplace transformation is a generalisation of the Fourier transformation.
The Laplace transform of a function $f(t)$ is, with $s\in\mathbb{C}$ and $t\geq0$:
\[
F(s)=\int\limits_0^\infty f(t){\rm e}^{-st}dt
\]
The Laplace transform of the derivative of a function is given by:
\[
{\cal L}\left(f^{(n)}(t)\right)=-f^{(n-1)}(0)-sf^{(n-2)}(0)-...-s^{n-1}f(0)+s^nF(s)
\]
The operator $\cal L$ has the following properties:
\begin{enumerate}
	\item Equal shapes: if $a>0$ than
	\[
	{\cal L}\left(f(at)\right)=\frac{1}{a}F\left(\frac{s}{a}\right)
	\]
	\item Damping: ${\cal L}\left({\rm e}^{-at}f(t)\right)=F(s+a)$
	\item Translation: If $a>0$ and $g$ is defined by $g(t)=f(t-a)$ if
	$t>a$ and $g(t)=0$ for $t\leq a$, than holds:
	${\cal L}\left(g(t)\right)={\rm e}^{-sa}{\cal L}(f(t))$.
\end{enumerate}
If $s\in \mathbb{R}$ than holds $\Re(\lambda f)={\cal L}(\Re(f))$ and
$\Im(\lambda f)={\cal L}(\Im(f))$.

For some often occurring functions holds:
\begin{center}
	\begin{tabular}{||c||c||}
		\hline
		$f(t)=$&$F(s)={\cal L}(f(t))=$\\
		\hline
		\hline
		$\displaystyle\frac{t^n}{n!}{\rm e}^{at}$&$(s-a)^{-n-1}$\rule{0pt}{15pt}\\
		${\rm e}^{at}\cos(\omega t)$&$\displaystyle\frac{s-a}{(s-a)^2+\omega^2}$\rule{0pt}{15pt}\\
		${\rm e}^{at}\sin(\omega t)$&$\displaystyle\frac{\omega}{(s-a)^2+\omega^2}$\rule{0pt}{15pt}\\
		$\delta(t-a)$&$\exp(-as)$\rule{0pt}{13pt}\\
		\hline
	\end{tabular}
\end{center}

\section{The convolution}
The convolution integral is defined by:
\[
(f*g)(t)=\int\limits_0^tf(u)g(t-u)du
\]
The convolution has the following properties:
\begin{enumerate}
	\item $f*g\in$LT
	\item ${\cal L}(f*g)={\cal L}(f)\cdot{\cal L}(g)$
	\item Distribution: $f*(g+h)=f*g+f*h$
	\item Commutative: $f*g=g*f$
	\item Homogenity: $f*(\lambda g)=\lambda f*g$
\end{enumerate}
If ${\cal L}(f)=F_1\cdot F_2$, than is $f(t)=f_1*f_2$.

\section{Systems of linear differential equations}
We start with the equation $\dot{\vec{x}}=A\vec{x}$. Assume that
$\vec{x}=\vec{v}\exp(\lambda t)$, than follows: $A\vec{v}=\lambda\vec{v}$.
In the $2\times2$ case holds:
\begin{enumerate}
	\item $\lambda_1=\lambda_2$: than $\vec{x}(t)=\sum\vec{v}_i\exp(\lambda_it)$.
	\item $\lambda_1\neq\lambda_2$: than $\vec{x}(t)=(\vec{u}t+\vec{v})\exp(\lambda t)$.
\end{enumerate}
Assume that $\lambda=\alpha+i\beta$ is an eigenvalue with eigenvector $\vec{v}$,
than $\lambda^*$ is also an eigenvalue for eigenvector $\vec{v}^*$. Decompose
$\vec{v}=\vec{u}+i\vec{w}$, than the real solutions are
\[
c_1[\vec{u}\cos(\beta t)-\vec{w}\sin(\beta t)]{\rm e}^{\alpha t}+c_2[\vec{v}\cos(\beta t)+\vec{u}\sin(\beta t)]{\rm e}^{\alpha t}
\]

There are two solution strategies for the equation $\ddot{\vec{x}}=A\vec{x}$:
\begin{enumerate}
	\item Let $\vec{x}=\vec{v}\exp(\lambda t)\Rightarrow\det(A-\lambda^2 \mathbb{I})=0$.
	\item Introduce: $\dot{x}=u$ and $\dot{y}=v$, this leads to $\ddot{x}=\dot{u}$ and
	$\ddot{y}=\dot{v}$. This transforms a $n$-dimensional set of second order
	equations into a $2n$-dimensional set of first order equations.
\end{enumerate}


\section{Kinematics}
Descriptors of linear motion begin with position $\vec{r}$ and of
angular motion with $\theta_q$
\begin{align*}
  \vec{v} &= d\vec{r}/dt  & \vec{\omega}_q &= d\theta_q/dt\\
  \vec{a} &= d\vec{v}/dt  & \vec{\alpha}_q &= d\vec{\omega}_q/dt.
\end{align*}
%-%For general acceleration
%-%\begin{align*}
%-%  v_q &= v_{q_0} + \textstyle\int_0^t a_q dt  & \omega_q &= \omega_{q_0} + \textstyle\int_0^t \alpha_q dt\\
%-%  q &= q_0 + \textstyle\int_0^t v_q dt & \theta_q &= \theta_{q_0} + \textstyle\int_0^t \omega_q dt.
%-%\end{align*}
For constant acceleration
\begin{align*}
q &= q_0 + v_{q_0} t + \tfrac{1}{2}a_q t^2 &\theta_q &= \theta_{q_0} + \omega_{q_0} t + \tfrac{1}{2}\alpha_q t^2\\
v_q &= v_{q_0} + a_q t  & \omega_q &= \omega_{q_0} + \alpha_q t\\
v_q^2 &= v_{q_0}^2 + 2a_q(q-q_0)  &  \omega_q^2 &= \omega_{q_0}^2 + 2\alpha_q(\theta_q-\theta_{q_0})\\
\Delta q &= \tfrac{1}{2}(v_{q_0} + v_q)t  & \Delta \theta_q &= \tfrac{1}{2}(\omega_{q_0} + \omega_q)t.
\end{align*}
Momentum, impulse, Newton's first law, and Newton's second law are
\begin{align*}
  \vec{p} &= m\vec{v}  & \vec{L} &= I\vec{\omega}\\
  \vec{J} &= \Delta \vec{p}  & \vec{J} &= \Delta \vec{L}\\
  \vec{F} &= d\vec{p}/dt  & \vec{\tau} &= d\vec{L}/dt\\
  \textstyle\sum \vec{F} &= m\vec{a}  & \textstyle\sum \vec{\tau} &= I\vec{\alpha}.
\end{align*}
Symmetry of physical laws with respect to translation and rotation begets
conservation of momentum
\begin{align*}
  \vec{p}_1 &= \vec{p}_2 & \vec{L}_1 &= \vec{L}_2.
\end{align*}

Connections between linear and angular kinematics start with the definition
of an angle as the ratio of arc length $s$ to radius $r$
\begin{align*}
  \theta &= s/r\\
  \vec{v}_{\text{tan}} &= \vec{\omega} \times \vec{r} & \vec{L} &= \vec{r} \times \vec{p}\\
  \vec{a}_{\text{tan}} &= \vec{\alpha} \times \vec{r} & \vec{\tau} &= \vec{r} \times \vec{F}
\end{align*}

Moment of inertia $I$ is a measure of the resistance of a body to change its
rotational motion and depends on the distribution of mass \emph{about a
  particular axis}
\begin{align*}
  I_{\text{discrete}} &= \textstyle\sum_j r_j^2 m_j\\
  I_{\text{continuous}} &= \textstyle\int r^2 dm
\end{align*}
About a \emph{parallel} axis the moment of inertia is related to the
distance $d$ from the center of mass
\begin{equation*}
  I = I_{\text{cm}} + M d^2
\end{equation*}


\chapter{Background}\label{ch2:background}
This chapter gives background concepts and notation used in the rest of the thesis.

\section{Partially ordered sets}
This section recalls basic notions of order theory upon which (much of) the abstract interpretation framework is based. For a more comprehensive introduction to order theory, we refer the reader to a text book, such as for instance \cite{order-theory-book} or appendix A of \cite{principles-of-program-analysis-book}.

\begin{definition}[Partial order]
	Given a set $S$, a partial order $\preceq$ on $S$ is a binary relation that, for all $a$, $b$ and $c$ in $S$, satisfies
	\begin{itemize}
		\labelitem{reflexivity} $a \preceq a$
		\labelitem{antisymmetry} if both $a \preceq b$ and $b \preceq a$ then $a = b$
		\labelitem{transitivity} if both $a \preceq b$ and $b \preceq c$ then also $a \preceq c$
	\end{itemize}
\end{definition}
We say that the pair $(S, \preceq)$ is a partially ordered set, or a \textit{poset} for short, and we usually write only the carrier set $S$ when the ordering is unambiguous.

\begin{definition}[Opposite ordering]
	Given a poset $(S, \preceq)$, the \textit{opposite} poset $(S, \preceq^{-1})$ is defined with $a \preceq^{-1} b$ if and only if $b \preceq a$.
\end{definition}
We often use $\succeq$ to indicate $\preceq^{-1}$ and $S^{\op}$ for the carrier set of the opposite poset $(S^{\op}, \succeq)$. Hence $S^{\op} = S$ as sets, but they have different orders defined on them.

\begin{definition}[Upper bounds and least upper bound]
	Given a poset $S$ and one of its subsets $T \subseteq S$, an \textit{upper bound} of $T$ is an element $c \in S$ that is greater or equal than any element of $T$:
	\[
	\forall a \in T.\ a \preceq c
	\]
	The \textit{least upper bound} (or lub for short) of $T$, if it exists, is an upper bound of $T$ that is smaller or equal than all other upper bounds of $T$.
\end{definition}
In general a set $T$ needs not have a least upper bound, but when it does it's unique and we denote it with $\bigsqcup T$. Moreover, when $T = \{ a, b \}$ is made of just two elements, we shall write $a \sqcup b$ for their least upper bound.

The dual notion of lub is that of glb:
\begin{definition}[Lower bounds and greatest lower bound]
	Given a poset $S$ and one of its subsets $T \subseteq S$, a \textit{lower bound} of $T$ is an element $c \in S$ that is smaller or equal than any element of $T$:
	\[
	\forall a \in T.\ c \preceq a
	\]
	The \textit{greatest lower bound} (or glb for short) of $T$, if it exists, is a lower bound of $T$ that is greater or equal than all other upper bounds of $T$.
\end{definition}
Again, if this exists we denote it with $\bigsqcap T$, and if $T = \{ a, b \}$ we use the notation $a \sqcap b$.

\begin{definition}[Lattice]
	A poset $S$ is called a \textit{lattice} if every pair of elements has a lub and glb. It is called a \textit{complete lattice} if every subset has a lub and a glb.
\end{definition}

When the whole set $S$ has an upper bound, that is an element greater than all other elements, that is unique and we denote it with $\top_S$, possibly dropping the subscript if the set is clear from the context. Note that $\top_S$ is the glb of the empty set. Dually, we denote with $\bot_S$ the lower bound of $S$ (if any), that is an element smaller than all other elements, and this is the lub of $\emptyset$.

\begin{definition}[Monotone function]
	Given two poset $S$, $T$, a function $f : S \rightarrow T$ is called \textit{monotone} (or \textit{order-preserving}) if, for any pair $a, b \in S$ of elements of the domain such that $a \preceq_S b$, also their images satisfies $f(a) \preceq_T f(b)$.
\end{definition}

Given a set $S$ and a poset $T$, we can consider the set of functions from $S$ to $T$. This has a natural structure of poset too.
\begin{definition}
	Given two functions $f, g: S \rightarrow T$, we say that $f \preceq g$ if for all elements $a \in S$ we have
	\[
	f(a) \preceq g(a)
	\]
\end{definition}
It's easy to show that this relation among functions is a partial order too. Moreover, if $T$ is a (complete) lattice, the set of functions from $S$ to $T$ is a (complete) lattice too, and this still holds if $S$ is a poset and we restrict ourselves to monotone functions between the two.

An important example of complete lattice are power sets. Given a set $S$, its power set $\pow(S)$ with the ordering induced by set inclusion is a complete lattice. A function $f : S \rightarrow S$ can be lifted to a function on $\pow(S)$ by taking its \textit{additive extension}, that correspond to apply $f$ to all elements of the input set: for a set $T \subseteq S$ we have
\[
f(T) = \{ f(s) \svert s \in T \}
\]
We overloaded the notation using the same symbol for both $f$ and its additive extension, but which of the two we're referring to will always be clear from whether the argument is a set or not.

\section{Galois connections}
The main mathematical tool we use to study abstract interpretations are Galois connections, and the special case of Galois insertions. An introduction to Galois connections from an order theoretic point of view can be found in chapter 7 of \cite{order-theory-book}.

\begin{definition}[Galois connection]\label{ch2:def:gc}
	Let $C$ and $A$ be two partially ordered sets, and $\alpha : C \rightarrow A$, $\gamma : A \rightarrow C$ be a pair of monotone functions between the two.

	We say $\gc{C}{\alpha}{\gamma}{A}$ is a Galois connection if, for any choice of $c \in C$ and $a \in A$ we have
	\[
	\alpha(c) \preceq a \iff c \preceq \gamma(a)
	\]
\end{definition}
For our goals, we will call $C$ the \textit{concrete domain}, $A$ the \textit{abstract domain}, $\alpha$ the \textit{abstraction function} and $\gamma$ the \textit{concretization function}. The two functions $\alpha$ and $\gamma$ are also called \textit{adjoints}\footnote{The term ``adjoint" comes from Category Theory, since Galois connections are adjunctions between the two posets seen as categories. However such a discussion is outside the scope of this thesis.}.

In program analysis we give the following intuitive meaning to those. $C$ is the powerset of the domain of concrete states, $A$ the set of abstract properties we're interested in. Partial orders on $C$ and $A$ represent a relation of ``more precise than": if $a \preceq a'$ for two abstract elements $a, a' \in A$, then the meaning is that property $a$ describes less states than $a'$ (it is stronger or \textit{more precise}). In logical terms, $a \Rightarrow a'$. Analogously, since $C$ is a powerset and the ordering is set inclusion, a more precise concrete element is just a subset, that describes less individual values. With this interpretation, we can say that if an abstract point $a$ correctly describes the concrete $c$ and $a'$ is above $a$ (ie. $a \preceq a'$), then also $a'$ correctly describes $c$, even though it's less precise than $a$.
With this meaning for partial orders, $\alpha$ is the function that maps a concrete state in the most precise (ie. strongest or minimal) abstract property that describes it and $\gamma$ a function that maps an abstract property in the largest (weakest or maximal) concrete state it describes. Since $\alpha(c)$ is the best (smallest) abstract property describing $c$ we have that a generic $a$ correctly describes $c$ if and only if $\alpha(c) \preceq a$. Dually, if $\gamma(a)$ is the largest concrete element described by $a$ we have that $a$ correctly describes a generic $c$ if and only if $c \preceq \gamma(a)$. The Galois connection relation requires exactly these two conditions to be equivalent, as naturally arise when we derive them this way.

\begin{figure}[ht]
	\centering
	\begin{subfigure}{.5\textwidth}
		\centering
		{
			\selectfont
			\def\svgwidth{.8\textwidth}
			\input{images/gc1.pdf_tex}
		}
		\caption{The Galois connection relation}
		\label{ch2:fig:gc-sketch:relation}
	\end{subfigure}%
	\begin{subfigure}{.5\textwidth}
		\centering
		{
			\selectfont
			\def\svgwidth{.8\textwidth}
			\input{images/gc2.pdf_tex}
		}
		\caption{Composing $\alpha$ and $\gamma$}
		\label{ch2:fig:gc-sketch:gamma-alpha-c}
	\end{subfigure}
	\caption{Graphical sketch of a Galois connection}
	\label{ch2:fig:gc-sketch}
\end{figure}
Figure \ref{ch2:fig:gc-sketch} sketches a Galois connection: the lower adjoint $\alpha$ is bent down, and the upper adjoint $\gamma$ is bent up.
Figure \ref{ch2:fig:gc-sketch:relation} represents the relation required by the Galois connection. Figure \ref{ch2:fig:gc-sketch:gamma-alpha-c} shows what happens when abstracting and then concretizing back and element (formalized in Proposition \ref{ch2:th:gc-extensive-charact}): the result is above the initial concrete element. This represent over-approximation: the biggest set described by the property $\alpha(c)$ must contains at least the whole $c$, because it's over-approximating it.

To give an intuition of the role of Galois connections in program analysis, we present the following example.
\begin{example}[Intervals]\label{ch2:ex:intervals}
	We can now formalize the intuitive Example \ref{intr:ex:intervals} of intervals from the Introduction.
	$C$ is the set of possible values of a variable, for instance \code{i}. Since this is an integer value, elements of $C$ are subsets of $\setZ$, so $C = \pow(\setZ)$, with the ordering given by set inclusion. $A$ is the set of abstract properties we track in our analysis, so in our example the set of intervals to which \code{i} may belong. This means
	\[
	A = \Int = \{ [n, m] \svert n \in \setZ \cup \{ -\infty \}, m \in \setZ \cup \{ +\infty \}, n \le m \} \cup \{ [+\infty, -\infty] \}
	\]
	$\alpha$ is the function that allows us to abstract a set $S$ of possible values of \code{i} to the best (ie. most precise) possible abstract property:
	\begin{align*}
		\alpha(S) &= [\min(S); \max(S)]
	\end{align*}
	with the convention that $\min(\emptyset) = +\infty$, $\min(\emptyset) = -\infty$, the minimum of a lower-unbound set is $-\infty$ and the maximum of an upper-unbound set is $+\infty$. This abstraction function is exactly what we expect: no smaller interval can describe the set $S$, since $\min(S)$ and $\max(S)$ are elements of $S$ and hence must also be in the interval. Conversely, this is a correct abstraction of $S$ since all its elements are between $\min(S)$ and $\max(S)$, so are in the interval too.

	$\gamma$ is the function that does the inverse operation: given an interval $[n, m]$, thought as a formal writing that describes the property that the value of \code{i} is between $n$ and $m$, gives back its ``meaning", that is the largest subset of $\setZ$ that matches that property:
	\[
	\gamma([n, m]) = \{ x \in \setZ \svert n \le x \le m \}
	\]
	The set $\{ x \in \setZ \svert n \le x \le m \}$ is exactly what is commonly represented with $[n; m]$: $\gamma$ is simply translating the formal writing (or, in our context, an abstract property) to a semantic set of values.

	The above definition of $\gamma$ is incomplete, missing cases for infinite ends:
	\begin{align*}
		\gamma([-\infty, m]) &= \{ x \in \setZ \svert x \le m \} \\
		\gamma([n, +\infty]) &= \{ x \in \setZ \svert n \le x \} \\
		\gamma([-\infty, +\infty]) &= \setZ \\
		\gamma([+\infty, -\infty]) &= \emptyset
	\end{align*}

	Showing that $\gc{\pow(\setZ)}{\alpha}{\gamma}{\Int}$ is a Galois connection is just a straightforward check. Fixed $S \in \pow(\setZ)$ and the interval $[n, m] \in \Int$ (for simplicity, we assume both $n$ and $m$ finite) we have
	\begin{align*}
		& \alpha(S) \preceq [n, m] \\
		\iff &[\min(S); \max(S)] \preceq [n, m] \\
		\iff &n \le \min(S),\, \max(S) \le m \\
		\iff &\forall x \in S\ .\ n \le x,\, \forall x \in S\ .\ x \le m \\
		\iff &S \subseteq \{ x \in \setZ \svert n \le x \le m \} \\
		\iff &S \subseteq \gamma([n, m])
	\end{align*}
\end{example}
We recall here two properties of Galois connections.
\begin{prop}\label{ch2:th:gc-extensive-charact}
	Let $C$ and $A$ be two partially ordered sets, and $\alpha : C \rightarrow A$, $\gamma : A \rightarrow C$ be a pair of monotone functions between the two.
	Then $\gc{C}{\alpha}{\gamma}{A}$ is a Galois connection if and only if both $\id_C \preceq \gamma \circ \alpha$ and $\alpha \circ \gamma \preceq \id_A$.
\end{prop}
\begin{prop}\label{ch2:th:gc-adjoints-preserve-glb-lub}
	Let $\gc{C}{\alpha}{\gamma}{A}$ be a Galois connection. Then $\gamma$ preserves greatest lower bounds and $\alpha$ preserves least upper bounds.
\end{prop}
In particular, this means that $\gamma$ maps $\top_A$ in $\top_C$ (because they are glb of the empty set) and dually $\alpha$ maps $\bot_C$ in $\bot_A$.

In a Galois connection it may very well happen that two abstract elements have the same concretization, as shown in the following example:
\begin{example}[Product abstraction]
	Consider two concrete domains, for instance two copies of $\pow(\setZ)$ that describes the possible states of two different variables \code{x} and \code{y} that appear in the program.
	Then consider the Galois connection $\gc{\pow(\setZ)}{\alpha}{\gamma}{\Int}$, and suppose we want to use it to abstract both variables. If we abstract separately both the element of $\pow(\setZ_x)$ and that of $\pow(\setZ_y)$ (where subscripts indicates the variable the domain refers to), we get a pair of intervals, one for \code{x} and one for \code{y}, that is an element of $\Int_x \times \Int_y$. It's easy to check that this abstraction function defines a Galois connection between $\pow(\setZ_x \times \setZ_y)$ and $\Int_x \times \Int_y$.

	However, this abstraction has redundant elements: consider
	\begin{align*}
		(\bot, [n, m]) && ([n, m], \bot) && (\bot, \bot)
	\end{align*}
	where $\bot = [+\infty, -\infty]$ describes the empty interval. All these elements are concretized in the concrete $\emptyset \in \pow(\setZ_x \times \setZ_y)$.
\end{example}

We would like not to have those since they are different elements of the abstract domain that describes the same property. In analogy with logic, this is the same kind of redundancy as the possibility to describe the empty set in the following three different ways:
\begin{align*}
\{ (x, y) \svert x \in \emptyset, n \le y \le m \}
&&
\{ (x, y) \svert n \le x \le m, y \in \emptyset \}
&&
\{ (x, y) \svert x \in \emptyset, y \in \emptyset \}
\end{align*}

In order to avoid this issue, we require $\gamma$ to be injective, so that no two different abstract elements can be concretized into the same concrete element, ie. they describe the same property. This turns out to give rise to an interesting definition, that of Galois insertion.
\begin{definition}[Galois insertion]\label{ch2:def:gi}
	Let $\gc{C}{\alpha}{\gamma}{A}$ be a Galois connection. We say this is a \textit{Galois insertion} if $\gamma$ is injective.
\end{definition}

The one proposed above isn't the standard definition of Galois insertion: more commonly, it requires $\alpha \circ \gamma$ to be identity on the abstract domain $\id_A$. However, the two are equivalent, as shown by the following characterization of Galois insertions.
\begin{prop}\label{ch2:th:gi-charact}
	Let $\gc{C}{\alpha}{\gamma}{A}$ be a Galois connection. Then the following are equivalent:
	\begin{enumerate}[label={(\arabic*)}]
		\item $\alpha \circ \gamma = \id_A$
		\item $\alpha$ is surjective
		\item $\gamma$ is injective
	\end{enumerate}
\end{prop}

The interval domain presented above is an example of Galois insertion: since we required $n \le m$ in $[n, m]$, no two intervals describe the same concrete set, and hence $\gamma$ is injective. However not all Galois connections are insertions: as we've seen, composing two independent interval domains gives rise to a Galois connection that isn't an insertion.

In a Galois insertion, since $\gamma$ is injective, we have a bijection between $A$ and $\gamma(A)$. By the definition of Galois insertion we have $\alpha \circ \gamma = \id_A$, hence $\alpha$ is the inverse of $\gamma$ when restricted to $\gamma(A)$. Since both functions are monotone this defines an isomorphism of posets between $A$ and $\gamma(A) \subseteq C$.

Using this isomorphism, whenever we consider a Galois insertion we identify $A$ and its image, so that $A$ becomes a subset of $C$ and $\gamma = \id_A$. In this case, by Proposition \ref{ch2:th:gc-extensive-charact} we have $\id_C \preceq \gamma \circ \alpha = \alpha$, corresponding to the intuitive idea that $\alpha$ must abstract a set of states in something bigger in order to over-approximate it.

A Galois insertion is said to be trivial if $A$ is the concrete domain or it only contains $\bot$: in the former the analysis isn't abstracted at all, and in the latter it can't track any property of concrete states.

Note that this identification simplifies notation, but introduces a pitfall: reasoning as above we may be tempted to say that also $\alpha = \alpha \circ \gamma \preceq \id_A$, always by Proposition \ref{ch2:th:gc-extensive-charact}, so concluding that $\id \preceq \alpha \preceq \id$ and hence $\alpha = \id$. However here we're neglecting the fact that $\gamma = \id_A$, not the identity on the whole set $C$, and actually $\gamma$ is defined only on elements of $A \subseteq C$. This means that the above relation is indeed correct, but only for elements of $A$, as pointed out by the fact that $\alpha \preceq \id_A$ and not $\id_C$. This is a problem that arise in general with this notation: two functions that looks the same are actually different because of their domain, that isn't specified. We'll make sure to always clarify the domain whenever it's not uniquely determined by the context.

Lastly, we recall a constructive way to define a Galois insertion.
\begin{prop}\label{ch2:th:gi-moore-family}
	Let $C$ be a complete lattice, and $\bar{A} \subseteq C$ a \textit{Moore family}, that is a subset such that
	\begin{itemize}
		\item $\bigsqcup C = \top \in \bar{A}$
		\item for any subset $T \subseteq \bar{A}$ of the Moore family, $\bigsqcap T \in \bar{A}$
	\end{itemize}
	Let also
	\[
	\alpha(x) = \bigsqcap \{ a \in \bar{A} \svert x \preceq a \}
	\]
	Then $\gi{C}{\alpha}{\bar{A}}$ is a Galois insertion.
\end{prop}

\section{Under-approximation Galois connections}
The definition of Galois connection is not symmetric, in the sense that it puts $\gamma$ above and $\alpha$ below: in fact the two are also called upper and lower adjoints, respectively. This asymmetry favours one specific direction, that is over-approximations, and is not suited to describe under-approximations. It can be more easily seen from $\id_C \preceq \gamma \circ \alpha$, that means the abstraction of a concrete element $c$ is greater than (ie. an over-approximation of) $c$ itself.
For this reason we introduce the notion of under-approximation Galois connection. Even though it's mathematically equivalent to that of Galois connection, we believe it allows a better notation, helping the reader's intuition.
\begin{definition}[Under-approximation Galois connection]\label{ch2:def:under-gc}
	Let $C$ and $A$ be two partially ordered sets, and $\alpha : C \rightarrow A$, $\gamma : A \rightarrow C$ be a pair of monotone functions between the two.

	We say $\ugc{C}{\alpha}{\gamma}{A}$ is an under-approximation Galois connection if, for any choice of $c \in C$ and $a \in A$ we have
	\[
	a \preceq \alpha(c) \iff \gamma(a) \preceq c
	\]
\end{definition}
This definition is the same as that of Galois connection (\ref{ch2:def:gc}) except that here $\alpha$ is above and $\gamma$ below. Since we are in an under-approximation setting, an abstract property correctly describes a concrete element if it is below it. Again, if $\alpha(c)$ is the best abstract property describing $c$, this is formalized as $a \preceq \alpha(c)$, or dually as $\gamma(c) \preceq a$, yielding the definition.
\begin{figure}[ht]
	\centering
	\begin{subfigure}{.5\textwidth}
		\centering
		{
			\selectfont
			\def\svgwidth{.8\textwidth}
			\input{images/ugc1.pdf_tex}
		}
		\caption{The under-approximation Galois connection relation}
		\label{ch2:fig:ugc-sketch:relation}
	\end{subfigure}%
	\begin{subfigure}{.5\textwidth}
		\centering
		{
			\selectfont
			\def\svgwidth{.8\textwidth}
			\input{images/ugc2.pdf_tex}
		}
		\caption{Composing $\alpha$ and $\gamma$}
		\label{ch2:fig:ugc-sketch:gamma-alpha-c}
	\end{subfigure}
	\caption{Graphical sketch of an under-approximation Galois connection}
	\label{ch2:fig:ugc-sketch}
\end{figure}

Figure \ref{ch2:fig:ugc-sketch} shows an under-approximation Galois connection: $\alpha$, that in this case in the upper adjoint, is bent up and $\gamma$ is bent down. Figure \ref{ch2:fig:ugc-sketch:relation} sketches the relation required, dual to that of normal Galois connections, and Figure \ref{ch2:fig:ugc-sketch:gamma-alpha-c} shows what happens when abstracting and then concretizing back an element: the result her is below the initial concrete element, meaning under-approximation.
\begin{example}\label{ch2:ex:intervals-0}
	Consider the following example of under-approximation Galois insertion (we haven't given its definition yet, but we're confident the reader can anticipate it): the concrete domain is $\pow(\setZ)$, while the abstract domain is the set of all intervals (see Example \ref{ch2:ex:intervals}) containing $0$, plus the empty interval:
	\[
	\Int_0 = \{ I \in \Int \svert 0 \in I \} \cup \{ \bot \}
	\]
	where we used $\bot$ to represent the empty interval $[+\infty, -\infty]$.
	$\gamma$ is the identity since we want an (under-approximation) Galois insertion, and $\alpha(S)$ is the greatest interval fully contained in $S$ that includes $0$. Formally,
	\begin{align*}
		\alpha(S) = \bigcup \{ I \in \Int_0 \svert I \subseteq S \}
	\end{align*}
	The result is in $\Int_0$: if it isn't empty, it does indeed contain $0$, since is the union of intervals in $\Int_0$ that contains $0$ themselves. Moreover it is an interval because union of overlapping intervals is an interval too, and all those intervals intersect at $0$.

	To show this is an under-approximation Galois connection, fix a set $S \subseteq \setZ$ and an interval $[-m, n]$ with $n, m \ge 0$.
	Now the following chain of equivalences hold:
	\begin{align*}
		&[-m, n] \subseteq \alpha(S) \\
		\iff &[-m, n] \subseteq \bigcup \{ I \in \Int_0 \svert I \subseteq S \} \\
		\iff &[-m, 0] \subseteq S, [0, n] \subseteq S \\
		\iff &\gamma([-m, n]) = [-m, n] \subseteq S
	\end{align*}
	hence the one proposed is an under-approximation Galois insertion.

	For simplicity we neglected the case where the interval is $\bot$ and assumed both $n, m$ are non negative integers, but those cases are analogous to the one presented.
\end{example}
With this definition, we could easily prove results analogous to Propositions \ref{ch2:th:gc-extensive-charact} and \ref{ch2:th:gc-adjoints-preserve-glb-lub} and give an analogous definition of under-approximation Galois insertion.
Instead of doing this explicitly, we just observe that an under-approximation Galois connection $\ugc{C}{\alpha}{\gamma}{A}$ is just a Galois connection between opposite domain $\gc{C^{\op}}{\alpha}{\gamma}{A^\op}$ to get those propositions just reversing inequalities:
\begin{prop}\label{ch2:th:under-gc-extensive-charact}
	Let $C$ and $A$ be two partially ordered sets, and $\alpha : C \rightarrow A$, $\gamma : A \rightarrow C$ be a pair of monotone functions between the two.

	Then $\ugc{C}{\alpha}{\gamma}{A}$ is an under-approximation Galois connection if and only if $\gamma \circ \alpha \preceq \id_C$ and $\id_A \preceq \alpha \circ \gamma$.
\end{prop}
From this proposition we have that $\gamma \circ \alpha \preceq \id_C$, that is the abstraction of an element $c$ is lower than (ie. an under-approximation of) $c$ itself.

\begin{prop}\label{ch2:th:under-gc-adjoints-preserve-lub-glb}
	Let $\ugc{C}{\alpha}{\gamma}{A}$ be an under-approximation Galois connection. Then $\gamma$ preserves least upper bounds and $\alpha$ preserves greatest lower bounds.
\end{prop}

This second proposition has an interesting corollary for under-approximation Galois connection. Of course its dual holds for standard Galois connection, but we use it mainly in the under-approximation case.
\begin{corollary}\label{ch2:th:under-gc-union-closure}
	Let $\ugc{\pow(C)}{\alpha}{\gamma}{A}$ be an under-approximation Galois connection. If $a, a' \in A$ then
	\[
	\gamma(a \sqcup a') = \gamma(a) \cup \gamma(a')
	\]
\end{corollary}

This corollary states that $A$ is closed under union: in an under-approximation Galois insertion, where $\gamma = \id_A$, for any pair of sets $S, S' \subseteq C$ that are in $A$ (ie. they are abstract properties) we have that also $S \cup S' \in A$.

We can also get an under-approximation version of Proposition \ref{ch2:th:gi-moore-family} to define a Galois insertion from an opposite Moore family:
\begin{prop}\label{ch2:th:under-gi-moore-family}
	Let $C$ be a complete lattice, and $\bar{A} \subseteq C$ a subset such that
	\begin{itemize}
		\item $\bigsqcap C = \bot \in \bar{A}$
		\item for any subset $T \subseteq \bar{A}$ also $\bigsqcup T \in \bar{A}$
	\end{itemize}
	Let also
	\[
	\alpha(x) = \bigsqcup \{ a \in \bar{A} \svert a \preceq x \}
	\]
	Then $\ugi{C}{\alpha}{\bar{A}}$ is an under-approximation Galois insertion.
\end{prop}

\begin{example}
	Consider a slight variation of the previous example, intervals that contains at least one of $0$ and $1$.
	\[
	\Int_{0,1} = \{ I \in \Int \svert 0 \in \Int \lor 1 \in \Int \} \cup \{ \bot \}
	\]
	To show this is an under-approximation Galois insertion, we use Proposition \ref{ch2:th:under-gi-moore-family} above.
	$\emptyset$, the bottom element of $\pow(\setZ)$ is in $\Int_{0,1}$ by definition. In $\pow(\setZ)$, lubs are unions, so it's enough to show that union of intervals in $\Int_{0,1}$ is still in $\Int_{0,1}$.
	Consider a set $T \subseteq \Int_{0,1}$ of intervals containing either $0$ or $1$. If $T = \{ \bot \}$ then $\bigcup T = \emptyset \in \Int_{0,1}$, and whenever $T$ contains at least one non empty element the union of its elements is the same whether the empty set is in $T$ or not, so we can assume that $\bot \notin T$.

	Let $T_0$ be the set of intervals in $T$ containing $0$ and $T_1$ the set of those containing $1$. If $T_0$ (respectively $T_1$) is not empty, the union of its elements is an interval containing $0$ (resp. $1$) too because all its elements are interval that intersect at $0$ (resp. $1$). If both are empty also $T$ is empty, hence $\bigcup T = \emptyset$ that is in $\Int_{0,1}$. If one of the two is empty, say $T_1$, then $\bigcup T = \bigcup T_0$ that is an interval containing $0$, hence in $\Int_{0,1}$ too. Lastly, if neither is empty, we have
	\[
	\bigcup T = \left( \bigcup T_0 \right) \cup \left( \bigcup T_1 \right)
	\]
	but the former is an interval containing $0$, and the latter is an interval containing $1$, hence their union is still an interval that contains both $0$ and $1$ and so is in $\Int_{0,1}$.
\end{example}

\section{Abstracting functions}
Only defining abstract domains is useless because we want to analyse \textit{programs}, dynamic entities that change the state. Hence the need for a way to abstract functions, used to represent the semantics of programs.

Given a monotone function $f : C \rightarrow C$ on the concrete domain, we want a way to approximate its behaviour in the abstract. In over approximation, we want to make sure the abstract version outputs at least an over-approximation of the concrete one to ensure soundness.
\begin{definition}[Correct abstraction]
	A monotone function $f^{\sharp} : A \rightarrow A$ on the abstract domain is called a \textit{correct abstraction} of $f$ if
	\[
	\alpha \circ f \preceq f^{\#} \circ \alpha
	\]
	The function
	\[
	f^{\alpha} = \alpha \circ f \circ \gamma
	\]
	is called the best abstraction of $f$.
\end{definition}
The best abstraction takes it names from the fact that it's lower than all other correct abstractions of $f$. Intuitively, the best we can do to soundly abstract a function $f$ is first to concretize the input in the optimal way with $\gamma$, then compute the concrete $f$ on it and lastly abstract again the result with $\alpha$.
\begin{example}
	Going back to the Example \ref{ch2:ex:intervals} of intervals, consider the program fragment
\begin{minted}{C}
if (x < 0) {
	x = -x;
}
\end{minted}
	It's semantics is the additive extension of the absolute value, that is
	\[
	f(S) = \{ \abs{x} \svert x \in S \}
	\]
	on the concrete domain $\pow(\setZ)$. A correct abstraction of this function is
	\[
	f^{\sharp}([n, m]) = [0, \max(\abs{n}, \abs{m})]
	\]
	because the interval $[0, \max(\abs{n}, \abs{m})]$ always contains the entire set $f(S)$ when $n = \min(S)$ and $m = \max(S)$. However this is not the best possible abstraction: for instance on $S = \{ 1 \}$ this yields $[0, 1]$ while $f(S) = \{ 1 \}$. Actually the best correct abstraction $f^{\alpha}$ is computed as
	\[
	f^{\alpha}([n, m]) = \alpha \circ f \circ \gamma([n, m]) = \begin{cases*}
		[0, \max(\abs{n}, \abs{m})] &if $n \le 0 \le m$ \\
		[n, m] &if $0 < n$ \\
		[-m, -n] &if $m < 0$ \\
	\end{cases*}
	\]
\end{example}
In this example, we computed the semantics of the whole program fragment and then we proceeded to abstract it. However this process is not what happens in practice because, if we knew the concrete semantics of the program we wouldn't really need the abstraction. An analyser would instead know abstractions of basic transfer functions, the semantics of basic construct of the language, and compose them in order to get a \textit{correct} approximation, but with no guarantee to get the \textit{best} abstraction. In this thesis we won't concern ourselves with this issue because we focus on basic transfer functions; however the reason we're allowed to do so is exactly that those are the basic building blocks used to define the abstract semantics of complex programs.

\begin{example}
	Consider for instance
	\[
	f(S) = \{ x \% 2 \svert x \in S \}
	\]
	where $\%$ is the modulo operator.
	It's best abstraction in the interval domain is
	\[
	f^{\alpha}([n, m]) = \begin{cases*}
		[0, 0] &if $n = m$ is even \\
		[1, 1] &if $n = m$ is odd \\
		[0, 1] &otherwise
	\end{cases*}
	\]
	and this introduces a precision loss: for instance on $S = \{ 1, 3 \}$ we have
	\[
	[1, 1] = \alpha(f(S)) \prec f^{\alpha}(\alpha(S)) = f^{\alpha}([1, 3]) = [0, 1]
	\]
\end{example}

In under-approximation, the definition of correct abstraction simply reverse inequality, as expected, but as notation we use $f^{\flat}$ for a correct under-approximation abstraction instead of $f^{\sharp}$.
\begin{example}
	Consider the under-approximation domain $\Int_0$ of intervals containing 0 (see Example \ref{ch2:ex:intervals-0}) and again the absolute value operation
	\[
	f(S) = \{ \abs{x} \svert x \in S \}
	\]
	In this case it's best abstraction is
	\[
	f^{\alpha}([n, m]) = [0, \max(\abs{n}, \abs{m})]
	\]
	since it's always the case that $n \le 0 \le m$.
\end{example}

It is important to remark that, both in over and under-approximation, with Galois insertions we have
\[
f^{\alpha} = \alpha \circ f \circ \gamma = \alpha \circ f \circ \id_A
\]
that may lead to say $f^{\alpha} = \alpha \circ f$. However, as we observed above, this is not the case since they have different domains: $f^{\alpha}$ takes as input abstract elements only, while $\alpha \circ f$ can be applied to any element of $C$. We'll ensure this condition is satisfied whenever we use $f^{\alpha}$, often because we'll consider $f^{\alpha} \circ \alpha$.

An important notion in abstract interpretation is that of complete abstraction \cite{giacobazzi-making-abstr-complete}. A correct abstraction of a function is said to be complete if it doesn't lose precision with respect to the concrete semantics.
\begin{definition}[Complete abstraction]\label{ch2:def:complete-abstr}
	Given a function $f$ and one of its correct abstractions $f^{\#}$, we say that it is a \textit{complete abstraction} when
	\[
	\alpha \circ f = f^{\#} \circ \alpha
	\]
\end{definition}
There is some interest in complete abstractions \cite{giacobazzi-making-abstr-complete,giacobazzi-analyzing-analyses,bruni-abst-intensionality} because they avoid false alarms, but in this thesis we're just interested in it as a way of constraining an abstract domain to interact with a function. We refer the reader to the discussion in the next chapter for an explanation of why we need this.


%---------------------------Aspect Ratio-----------------------------
\section{Aspect Ratio\label{s:quad-aspect-ratio}}

The aspect ratio of a quadrilateral is:
\[
q = \frac{L_{\max}(L_0+L_1+L_2+L_3)}{4A},
\]
where $A$ is the area of the quadrilateral.

Note that, strictly speaking, the aspect ratio is usually defined for
simplicial elements as the ratio of the maximum edge length to the
inradius (\emph{cf.}~\S\ref{s:tri-aspect-ratio} and
~\S\ref{s:tet-aspect-ratio}). However, a planar quadrilateral does not
have, in general, an inscribed circle: such an incircle exists if and
only if $L_0+L_2=L_1+L_3$. Nonetheless, using the expression
of the triangle aspect ratio as given
in~\eqref{eq:triangle_aspect_ratio}, that is, with no explicit
reference to the inradius but only to the perimeter and the area, one
can then directly extrapolate to obtain a meaningful definition of the
quadrangle aspect ratio.

\trimetrictable{aspect ratio}%
{$1$}%                                                Dimension
{$[1,1.3]$}%                                          Acceptable range
{$[1,DBL\_MAX]$}%                                     Normal range
{$[1,DBL\_MAX]$}%                                     Full range
{$1$}%                                                Unit square
{\cite{pebay:04}}%                                    Reference(s)
{v\_quad\_aspect\_ratio}%                             Verdict function name


\subsection{Bounded sequences}

A function \(f(x)\) on set \(X\) is bounded if:

\(\exists M\in \mathbb{R} [\forall x\in X f(x)\le M]\)

A bounded sequence is a special case of a bounded function where:

\(X=\mathbb{N}\)

That is, a sequence is bounded by \(M\) iff:

\(\forall n\in \mathbb{R} |f(a_n)|\le M\)


% Here we show the equation alignment examples from:
% http://www.orcomplete.com/quick-tip/sertalpbilal/latex-templates-for-optimization-models

\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\usepackage{hyperref}

\title{LaTeX Templates for Optimization Models}
%\author{Sertalp Bilal Çay}
% Modified by Zhiyang Ong to reflect the contributions of John Hammersley, who provided the LaTeX document, and Vince Knight, who made a suggestion for typesetting mathematical optimization models in LaTeX.
\author{Sertalp Bilal {\c{C}}ay and John Hammersley}

\begin{document}
\maketitle

\begin{abstract}
Here we show the equation alignment examples from:\\ \url{http://www.orcomplete.com/quick-tip/sertalpbilal/latex-templates-for-optimization-models}
\end{abstract}

\section{Array} % Example 1

\begin{equation}
\begin{array}{rrclcl}
\displaystyle \min_{x} & \multicolumn{3}{l}{c^T x} \\
\textrm{s.t.} & A x & \leq & b \\
&\displaystyle \sum_{i=0}^{n} x_i & = & 1 \\
& x_j & \geq & 0 & & \forall j \in N \\
\end{array}
\end{equation}

\section{Aligned} % Example 2

\begin{equation}
\begin{aligned}
& \underset{x}{\text{min}}
& & c^T x \\
& \text{s.t.} & &  Ax \leq b_i \\
& & &  \sum_{i=1}^{n} x_i =1 \\
& & &  x_j, \; \forall j \in N. \\
\end{aligned}
\end{equation}

\section{Matrix} % Example 3

\begin{equation}
\begin{matrix}
\displaystyle \min_x & c^T x  \\
\textrm{s.t.} & A x & \leq & b  \\
& \displaystyle \sum_{i=1}^{n} x_i & = & 1  \\
& x_j & \geq & 0 & & \forall j \in N
\end{matrix}
\end{equation}

\section{Align} % Example 4

\begin{align*}
\min_x \quad c^T x \\
Ax &\leq b \\
\sum_{i=1}^n x_i &= 1\\
x_j &\geq 0 \quad \forall j \in N
\end{align*}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Acknowledgements}
\label{sec:Acknowledgements}

John Hammersley provided the LaTeX document; hence, Zhiyang Ong modified the author list to reflect this. In addition, Vince Knight made a suggestion for typesetting mathematical optimization models in \LaTeX, which is incorporated by John Hammersley.

\end{document}


\section{Random Triangles in High Dimensions}

For a triangle $\Delta$, we will let $\mathcal{Q}$ be a measure of the \textit{quality} of the triangle as defined by
\begin{equation}
\label{eq:triangle_score}
\mathcal{Q}(\Delta) = \frac{4\sqrt{3} A}{ \ell_1^2 + \ell_2^2 + \ell_3^2}
\end{equation}
where $A$ is the area of the triangle, and $\ell_1$, $\ell_2$ and $\ell_3$ are the lengths of the three sides.
Without performing any computations, can you determine the maximum and minimum values that $\mathcal{Q}$ can take? Can you describe what kinds of triangles are associated with high, and with low, values of $\mathcal{Q}$?

In the first part of this project, you will write a function that computes the quality of a triangle in $\mathbb{R}^2$, given the coordinates of its vertices.

\begin{itemize}
  \item Write a function, \texttt{tri\_lengths()}, that takes as its arguments the three vertices of a triangle and returns the lengths of the three sides of the triangle.
  \item Write a function, \texttt{tri\_angles()}, that takes as its arguments the three vertices of a triangle and returns the angles of the triangle.
  \item Write a function, \texttt{tri\_area()}, that takes as its arguments the three vertices of a triangle and returns the area of the triangle.
  \item Write a function, \texttt{tri\_score()}, that takes as its arguments the three vertices of a triangle and returns the quality score of the triangle as defined in equation (\ref{eq:triangle_score}).
  \item Test your function \texttt{tri\_score()} on the four triangles below. As a check on your function, the correct values are:
\begin{align*}
	\mathcal{Q}({\color{green}{\Delta}}) &= 0.597\cdots,
&	\mathcal{Q}({ \color{blue}{\Delta}}) &= 0.413\cdots,\\
	\mathcal{Q}({\color{red}{\Delta}}) &=0.172\cdots ,
&	\mathcal{Q}({ \color{purple}{\Delta}}) &= 1.
\end{align*}
\end{itemize}

\begin{center}
\begin{tikzpicture}[scale = 1.8]
  \draw[ultra thick,red!60!white] (0,0)node[below] {$(0,0)$} -- (-1.5,.5)node[above]{$(-3,1)$} -- (1,0)node[right]{$(2,0)$} --cycle;
  \draw[ultra thick, green!60!black] (-0.5,1.5)node[left] {$(-5,0)$} -- (2,1.5)node[right]{$(0,0)$} -- (2,.5)node[right]{$(0,-2)$} -- cycle;
  \draw[ultra thick, red!50!blue!80] (3,-.5)node[above left] {$-1,0$} -- (4,-.5)node[above right] {$(1,0)$} -- (3.5,.2)node[above]{$(0,\sqrt{3})$} -- cycle;
  \draw[ultra thick, blue!80!black] (5,0)node[below]{$(0,0)$} -- (4.8,2)node[left]{$(-0.5,4)$} -- (5.2,2)node[right]{$(0.5,4)$} -- cycle;
\end{tikzpicture}
\end{center}

Many applied mathematicians regularly work with data sets where each data point is associated with many different features (e.g. each medical patient is associated with their own temperature, heart rate, blood pressure, etc).
Sometimes the physical, three-dimensional world can provide useful intuition for the geometry of higher dimensional space. Most of the time, however, we are unable to grasp all the weird ways that the higher dimensional space is `bigger'.

In the second part of this project, you will compare the expected quality of random triangles in $\mathbb{R}^2$ with that of random triangles in $\mathbb{R}^{10}$.
\begin{enumerate}[(a)]
  \item
  \begin{enumerate}[i.]
    \item Use a random number generator to create a random point, \(\bm{x} = (x_1,x_2)\), that is normally distributed in $\mathbb{R}^2$. The functions \texttt{randn()} (Matlab), \texttt{np.random.randn()} (Python), or \texttt{randn()} (Julia) may be useful.
    \item Repeat part i. until you have three points, $\bm{x} = (x_1,x_2)$, \(\bm{y} = (y_1,y_2)\), and \(\bm{z} = (z_1,z_2)\). Let $\Delta$ be the triangle with $\bm{x}$, $\bm{y}$, and $\bm{z}$ as its vertices. Use your function \texttt{tri\_score()} to compute $\mathcal{Q}(\Delta)$.
    \item Repeat parts i. and ii. until you have the quality scores for 100,000 random triangles. Plot a histogram of your data.
  \end{enumerate}
  \item Repeat this process for triangles in $\mathbb{R}^{10}$. That is, use a multivariate normal random number generator to generate 3 points in $\mathbb{R}^{10}$, $\bm{x} = (x_1,x_2,\dots x_{10})$, $\bm{y} = (y_1,y_2,\dots,y_{10})$, $\bm{z} = (z_1,z_2, \dots, z_{10})$. Let $\Delta$ be the triangle with $\bm{x}$, $\bm{y}$, and $\bm{z}$ as its vertices, and compute $\mathcal{Q}(\Delta)$. Repeat for 100,000 random triangles.
 \item Compare the histogram for triangles in $\mathbb{R}^2$ with a histogram for triangles in $\mathbb{R}^{10}$.  Provide a geometric explanation for why the random triangles in higher dimensions are much closer to equilateral than the random triangles in lower dimensions
\end{enumerate}


\chapter{DT Convolution}

\section{Review DT LTI systems and superposition property}

Recall the superposition property of LTI systems. If a DT system is LTI then the superposition property holds. Given a system where
\[
x_i[n] \mapsto y_i[n] \; \forall\; i
\]
then
\[
\sum\limits_{i} a_i x_i[n] \mapsto \sum\limits_{i} a_i y_i[n]
\]

As in CT we can use superposition to enable a problem reduction strategy in DT systems, where we write the input as a weighted sum of simple signals.  In this lecture, the simple signals are weighted, time shifts of one signal, the DT delta function, $\delta[n]$.

\section{Convolution Sum}

To derive this we start with the sifting property of the DT impulse function (from lecture 3)
\[
\sum\limits_{a}^{b} x[n]\delta[n-n_0] = x[n_0]
\]
for any $a < n_0 < b$. A slight change of variables ($n_0 \rightarrow m$) and limits ($a \rightarrow -\infty$ and $b \rightarrow \infty$) gives:
\[
x[n] = \sum\limits_{m = -\infty}^{\infty} x[m]\delta[n-m]
\]
showing that we can write any DT signal as an infinite sum of weighted and time-shifted impluse functions.

Let $h[n]$ be the DT {\it impulse response}, the output due to the input $\delta[n]$, i.e. $\delta[n] \mapsto h[n]$. Then if the system is time-invariant: $\delta[n-m] \mapsto h[n-m]$ and by superposition, if the input is writen as
\[
x[n] = \sum\limits_{m = -\infty}^{\infty} x[m]\delta[n-m]
\]
then the output is given by
\[
y[n] = \sum\limits_{m = -\infty}^{\infty} x[m]h[n-m] = x[n] * h[n]
\]
This is called the \emph{convolution sum} \index{DT Convolution}.

The significance is similar to that in CT convolution. For a LTI DT system, if I know it's impulse response $h[n]$, I can find the response due to \textbf{any} input using convolution. For this reason the impulse response is another way to represent an LTI system.

\section{Graphical View of the Convolution Sum.}

Lets break the convolution expression down into pieces. In it's general form the convolution of two signals $x_1[n]$ and $x_2[n]$ is
\[
x_1[n] * x_2[n] = \sum\limits_{m = -\infty}^{\infty} x_1[m]x_2[n-m]
\]

Suppose $x_1[n]$ and $x_2[n]$ are signals that look like
\begin{center}
  \includegraphics[scale=1]{graphics/dtconvolution-explain1.pdf}
\end{center}

Then $x_1[m]$ and $x_2[-m]$ look like
\begin{center}
  \includegraphics[scale=1]{graphics/dtconvolution-explain2.pdf}
\end{center}

The signal $x_2[n-m]$ is $x_2[-m]$ shifted by $n$ (since $x_2[-m+n]= x_s[n-m]$) and looks like
\begin{center}
  \includegraphics[scale=1]{graphics/dtconvolution-explain3.pdf}
\end{center}

Then the terms of the convolution sum is the product $x_1[m]x_2[n-m]$ whose plot depends of the value of $n$. Some examples, where the individual signals are in grey and their product is in bold:
\begin{center}
  \includegraphics[scale=1]{graphics/dtconvolution-explain4.pdf}
\end{center}

Then convolution is the total sum of the product (bold plots above) for that value of $n$. For the example above we see the sum will be zero for $n$ less than $n_0$ since the two signals do not overlap and their product is zero. For $n_0 \leq n \leq n_1$ the signals overap and the product is non-zero, and the effective bounds of summation are $[n_0,n]$. For $n > n_1$ the signals again overap and the product is non-zero, but the effective bounds of summation are $[n_0,n_1]$.

\section{DT Convolution of Finite-Length Signals}

For finite-length signals, DT convolution gives us an algorithm to determine their convolution. Suppose the signal $x_1$ is non-zero only over the interval $[N_1,M_1]$, and the signal $x_2$ is non-zero only over the interval $[N_2,M_2]$. The \emph{length} of the signals are $L_1 = M_1-N_1+1$ and $L_2 = M_2-N_2+1$ respectively. The non-zero terms of the convolution sum (when the signals overlap) is then the range $[N_1+N_2,M_1+M_2]$ and the sum can be truncated as:

\[
x_1[n] * x_2[n] = \sum\limits_{m = N_1+N_2}^{M_1+M_2} x_1[m]x_2[n-m]
\]

It is common to shift both signals so that they both start at index $0$ (in order to be represented as arrays in a zero-based index programming language like C or C++), zero-padding them both to have length $L=L_1+L_2-1$ (zero-pad means to just add zero values to the end of the sequence). Then the convolution becomes
\[
y = x_1 * x_2 = \sum\limits_{m = 0}^{L} x_1[m]x_2[n-m]
\]
where the indexing of $x_2$ is modulo the signal length, i.e. $x_2[(n-m) \mbox{ mod } L]$. The resulting signal after convolution, $y$, is also of length $L$, and can then be shifted back to start at $N_1+N_2$.

\begin{example} The following C++ code computes the convolution of the DT signals $\{1,-1,1\}$ and $\{1,1,1,1\}$.
\begin{verbatim}
  double x1[6] = {1., -1., 1., 0, 0, 0};
  double x2[6] = {1., 1., 1., 1., 0, 0};
  double y[6];

  for(int n = 0; n < 6; n++){
    double sum = 0.;
    for(int m = 0; m < 6; m++){
      int idx = (n-m) % 6;
      sum += x1[m]*x2[idx];
    }
    y[n] = sum;
  }
\end{verbatim}
Note that $L_1 = 3$, $L_2 = 4$, so that $L=6$.
$\blacksquare$
\end{example}

An interesting aside, convolution of finite length signals is equivalent to multiplication of two polynomials, where the signal values are the coefficients.

\section{Examples of DT Convolution}

\begin{example} Consider the convolution of two unit step functions:
  \[
  u[n] * u[n] = \sum\limits_{m = -\infty}^{\infty} u[m]u[n-m]
  \]
  Note for $n < 0$ the product of the signals $u[m]$ and $u[n-m]$ is zero as shown in the following figure
  \begin{center}
  \includegraphics[scale=1]{graphics/dt-step-step-conv.pdf}
  \end{center}
  so that the resulting sum is zero for any $n < 0$. For $n \geq 0$ the signals $u[m]$ and $u[n-m]$ overlap from $0$ to $n$ as shown below
  \begin{center}
    \includegraphics[scale=1]{graphics/dt-step-step-conv2.pdf}
  \end{center}
  and the convolution sum is
  \[
  \sum\limits_{m = 0}^{n} 1 = (n+1)
  \]
  so that
  \[
  u[n] * u[n] = \left\{ \begin{array}{lc}
    0 & n < 0\\
    n+1 & n \geq 0
  \end{array}
  \right.
  \]
  Putting the piecewise result into a single expression gives
  \[
  u[n] * u[n] = (n+1)u[n]
  \]
  $\blacksquare$
\end{example}

\begin{example}
Consider the convolution of a unit step and the function $\gamma^n\,u[n]$ for some constant $\gamma \neq 1$:
  \[
  \gamma^n\, u[n] * u[n] = \sum\limits_{m = -\infty}^{\infty} \gamma^{m}u[m]u[n-m]
  \]
  Since both signals are multiplied by a step, the product of $\gamma^{m}u[m]u[n-m]$ is non-zero only for $0 \leq m \leq n$ (for the same reason as in the previous example). Thus for $n \geq 0$ the convolution sum is:
  \[
  \sum\limits_{m = 0}^{n} \gamma^{m} = \frac{\gamma^{n+1}-1}{\gamma-1} = \frac{1-\gamma^{n+1}}{1-\gamma}
  \]
  Putting the two peicewise results together gives
  \[
  \gamma^n\, u[n] * u[n] = \frac{1-\gamma^{n+1}}{1-\gamma}\,u[n]
  \]
  $\blacksquare$
\end{example}
\begin{example} Consider the convolution of an arbitrary signal $x[n]$ with the impulse function
  \[
  x[n] * \delta[n] = \sum\limits_{m = -\infty}^{\infty} x[m]\delta[n-m]
  \]
  By the sifting property we get
  \[
  \sum\limits_{m = -\infty}^{\infty} x[m]\delta[n-m] = x[n]
  \]
  Thus the convolution with the impulse gives back the same signal (the $\delta$ is the \emph{identity} signal). $\blacksquare$
\end{example}

The following table lists several DT convolution results.

\begin{center}
  Table of Representative Convolution Sums
  \vspace{1em}

\bgroup
\def\arraystretch{2}
\setlength\tabcolsep{2em}
\begin{tabular}{|c|c|c|}
  \hline
  $x_1[n]$ & $x_2[n]$ & $x_1[n] * x_2[n]$\\
  \hline
  \hline
  $u[n]$ & $u[n]$ & $(n+1)u[n]$\\
  $\gamma^{n}u[n]$ & $u[n]$ & $\frac{1-\gamma^{n+1}}{1-\gamma}u[n]$ for $\gamma \neq 1$\\
  $\gamma_1^{n}u[n]$ & $\gamma_2^{n}u[n]$ & $\frac{\gamma_1^{n+1}-\gamma_2^{n+1}}{\gamma_1-\gamma_2}u[n]$ for $\gamma_1 \neq \gamma_2$\\
  $\gamma^{n}u[n]$ & $\gamma^{n}u[n]$ & $(n+1)\gamma^{n}u[n]$\\
  $|\gamma_1|^{n}\cos\left(\beta n + \theta \right)u[n]$ & $|\gamma_2|^{n}u[n]$ & $\frac{1}{R}\left[ |\gamma_1|^{n+1}\cos\left( \beta (n+1) + \theta - \phi\right) - |\gamma_2|^{n+1}\cos\left( \theta - \phi\right)\right]u[n]$\\
           & & $R = \left[ |\gamma_1|^2 + |\gamma_2|^2 -2|\gamma_1||\gamma_2|\cos(\beta)\right]^{\frac{1}{2}}$\\
  & & $\phi = \arctan\left( \frac{|\gamma_1|\sin(\beta)}{|\gamma_1|\cos(\beta) - |\gamma_2|} \right)$\\

\hline
\end{tabular}
\egroup

\end{center}

\section{Properties of DT Convolution}
There are several useful properties of convolution. We do not prove these here, but it is not terribly difficult to do so. Given signals $x_1[n]$, $x_2[n]$, and $x_3[n]$:

\begin{description}
\item [Communative Property] The ordering of the signals does not matter.
  \[
x_1[n] * x_2[n] = x_2[n] * x_1[n]
  \]
\item [Distributive Propery] Convolution is distributed over addition.
  \[
  x_1[n] * \left(x_2[n] + x_3[n]\right) = \left(x_1[n] * x_2[n] \right) + \left(x_1[n] * x_3[n] \right)
  \]
\item [Associative Property] The order of convolution does not matter.
    \[
  x_1[n] * \left(x_2[n] * x_3[n]\right) = \left(x_1[n] * x_2[n] \right) * x_3[n]
  \]
\item [Index Shift] Given $x_3[n] = x_1[n] * x_2[n]$ then for index shifts $m_1, m_2 \in \mathbb{R}$
  \[
  x_1[n-m_1] * x_2[n-m_2] = x_3[n-m_1 - m_2]
  \]
\item [Multiplicative Scaling] Given $x_3[n] = x_1[n] * x_2[n]$ then for constants $a,b \in \mathbb{C}$
  \[
  \left(a\, x_1[n]\right) * \left(b\, x_2[n]\right) = a\, b\, x_3[n]
  \]
\end{description}

These properties can be used in combination with a table like that above to compute the convolution of a wide variety of signals without evaluating the summations.

\begin{example} Consider the convolution of the causal DT pulse of length $N$, $x_1[n] = u[n] - u[n-N]$, and the signal $x_2[n] = \left( \frac{1}{2}\right)^nu[n]$.

  \begin{align*}
    x_1[n] * x_2[n] &= \left( u[n] - u[n-N]\right) * \left( \left( \frac{1}{2}\right)^nu[n] \right)\\
    &= \left( u[n] \right) * \left( \left( \frac{1}{2}\right)^nu[n] \right) - \left( u[n-N]\right) * \left( \left( \frac{1}{2}\right)^nu[n] \right) \mbox{ using distributive property}\\
    &= \frac{1-\left(\frac{1}{2}\right)^{n+1}}{1-\left(\frac{1}{2}\right)}u[n] - \frac{1-\left(\frac{1}{2}\right)^{n+1}}{1-\left(\frac{1}{2}\right)}u[n] \Big|_{n\rightarrow n-N} \mbox{ from Table row 2 and index shift property}\\
    &= \frac{1-\left(\frac{1}{2}\right)^{n+1}}{\left(\frac{1}{2}\right)}u[n] - \frac{1-\left(\frac{1}{2}\right)^{n-N+1}}{\left(\frac{1}{2}\right)}u[n-N]\\
    &= \frac{1-\left(\frac{1}{2}\right)^{n+1}}{\left(\frac{1}{2}\right)}u[n] - \frac{1-\left(\frac{1}{2}\right)^{-N}\left(\frac{1}{2}\right)^{n+1}}{\left(\frac{1}{2}\right)}u[n-N]\\
    &= 2\left(1-\left(\frac{1}{2}\right)^{n+1}\right)u[n] - 2\left(1-\left(\frac{1}{2}\right)^{-N}\left(\frac{1}{2}\right)^{n+1} \right)u[n-N]\\
    &= \left(2-\left(\frac{1}{2}\right)^{n}\right)u[n] - \left(2-\left(\frac{1}{2}\right)^{-N}\left(\frac{1}{2}\right)^{n} \right)u[n-N]
  \end{align*}
$\blacksquare$
\end{example}


\section{Calculus of Vector and Matrix Valued Functions}

\subsection{Exercise 1}
That $\dot{x}(t) = 0 \implies x(t) = c$ follows immediately from the mean value inequality for vector-valued
functions, $\norm{x(b) - x(a)} \leq (b - a) \norm{x'(t)}$ for some $t \in (a, b)$. My guess is that
Lax was hinting at applying the mean value theorem for real-valued functions to $(x(b) - x(a), y)$.

\subsection{Exercise 2}
We have that
\begin{align*}
        \dv{x} A^{-1} A = 0 = \bigg(\dv{x} A^{-1}\bigg) A + A^{-1} \bigg(\dv{x} A\bigg) \implies \dv{x} A^{-1} = -A^{-1} \bigg( \dv{x} A \bigg) A^{-1}
\end{align*}

\subsection{Exercise 3}
The matrix $A + B$ satisfies $(A + B)^2 = I$, so we have that
\begin{align*}
        e^{A+B} &= \sum_{k = 0}^\infty \frac{(A + B)^k}{k!} \\
                &= \bigg(\sum_{k = 0}^\infty \frac{1}{(2k)!}\bigg) I +  \bigg(\sum_{k = 0}^\infty \frac{1}{(2k + 1)!}\bigg) (A + B) \\
                &= \cosh(1) I + \sinh(1) (A + B)
\end{align*}

\subsection{Exercise 4}
We can use uniform continuity to swap the limits to get
\begin{align*}
        \lim_{m \to \infty} \norm{\dot{E}_m(t) - F(t)} &= \lim_{m \to \infty} \lim_{h \to 0} \norm{\frac{E_m(t + h) - E_m(t)}{h} - F(t)} \\
                                                       &= \lim_{h \to 0} \lim_{m \to \infty} \norm{\frac{E_m(t + h) - E_m(t)}{h} - F(t)} \\
                                                       &= \lim_{h \to 0} \norm{\frac{E(t + h) - E(t)}{h} - F(t)} \\
                                                       &\implies \dot{E}(t) = F(t)
\end{align*}

\subsection{Exercise 5}
Let $M \geq \norm{A(t)}, \norm{\dot{A}(t)}$. Then we can use the expression for the derivative of $A^k(t)$
to get
\begin{align*}
        \norm{\dv{t} A^k} &= \norm{\dot{A} A^{k - 1} + A \dot{A} A^{k - 2} + ... + A^{k - 1}\dot{A}} \\
                   &\leq kM^k \\
        \implies \norm{\dot{E}_m(t) - \dot{E}_n(t)} &\leq \sum_{k = n + 1}^m \frac{kM^k}{k!}
\end{align*}
so $\dot{E}_m(t)$ converges.

\subsection{Exercise 6}
\begin{align*}
        \dv{t} \log\det e^{At} = \Tr(e^{-At} A e^{At}) = \Tr(A)
\end{align*}

\subsection{Exercise 7}
Let $v$ be an eigenvector of $A$ with eigenvalue $a$. Then
\begin{align*}
        e^A v &= \sum_{k = 0}^\infty \frac{A^k v}{k!} \\
              &= \sum_{k = 0}^\infty \frac{a^k v}{k!} \\
              &= e^a v
\end{align*}


\subsection{Subset relation}

\subsubsection{Subset}

If all terms which are members of term \(x\) are also members of term \(y\), then \(x\) is a subset of \(y\).

\(\forall x\forall y[(\forall z(z\in x\rightarrow z\in y))\leftrightarrow (x\subseteq y)]\)

\subsubsection{Proper subset}

If two sets are equal, then each is a subset of the other. A proper subset is one which is a subset, and not equal to the other set.

\(\forall x\forall y[((\forall z(z\in x\rightarrow z\in y)))\land(x\ne y)\leftrightarrow (x\subset y)]\)


\paragraph{Rotation}
$T_\phi(x, y) = (x \cos(\phi) -  y \sin(\phi), x \sin(\phi) + y \cos(\phi))$

\paragraph{Translation} $T_\delta(x, y) = (x + \delta_x, y + \delta_y)$

\paragraph{Scaling}
$T_\lambda(x, y) = (\lambda x, \lambda y)$

\paragraph{Interpolation} To compute pixel value $I_\kappa(x, y)$ after $T_\kappa$: compute pre-image $T_\kappa(x, y)$, then interpolate.

\paragraph{Domain}
$w_l^T \kappa + b_l \le I_\kappa(x, y) \le w_u^T + b_u$, for all $\kappa$

\paragraph{Tightness}
$\begin{array}{r@{~:=~}l}
L(w_l, b_l) & \int_{\kappa\in D} \left(I_\kappa(x, y) - (w_l^T\kappa + b_l)\right) d\kappa \\
U(w_u, b_u) & \int_{\kappa\in D} \left((w_u^T\kappa + b_u) - I_\kappa(x, y)\right) d\kappa
\end{array}$

\paragraph{Optimization}
\begin{itemize}
\item $L(w_l, b_l) \approx \frac{1}{N} \sum_{i=1}^N \left(I_\kappa(x, y) - (w_l^T\kappa^i + b_l)\right)$
\item $w_l^T\kappa^i + b_l \le I_{\kappa^i}(x, y)$, $1 \le i \le N$
\end{itemize}
$\rightarrow$ Solve in poly-time with linear programming.
\paragraph{Soundness}
Find upper bound $\delta$ on violation:
$\begin{array}{r@{~\le~}ll}
(w_l^T\kappa^i + b_l) - I_\kappa(x, y) & \delta_l &\forall \kappa \in D \\
 I_\kappa(x, y) - (w_u^T\kappa + b_u) & \delta_u &\forall \kappa \in D
\end{array}$

Using $b_l-\delta_l$, $b_u+\delta_u$ is sound.

\paragraph{$\rightarrow$ Box}
Use Box to compute upper bound on $f(\kappa)$

\paragraph{$\rightarrow$ MVTh} if $\lvert \partial_i f(\kappa') \rvert \le \lvert L_i \rvert \forall \kappa' \in D = [h_l, h_u]$,
$f(\kappa) = f(\kappa_C) + \nabla f(\kappa')^T(\kappa-\kappa_C) \le f(\kappa_C) + \lvert L \rvert^T (\kappa - \kappa_C) \le f\left(\frac{h_u+h_l}{2}\right) + \lvert L \rvert^T \frac{(h_u-h_l)}{2}$


\section{Mimimization and Least Squares}

\begin{theorem}
  If $K$ is a positive definite (and hence symmetric) matrix, then the quadratic function has a unique minimizer, whcih is the solution to the linear system
  \[
  Kx=f,\text{ namely } x^{*}=K^{-1}f
.\] The minimum value of $p(x)$ is equal to any of the following expressions: \[
p(X^{*})=p(K^{-1}f)=c-f^{T}K^{-1}f=c-f^{T}x^{*}=c-(x^{*})^{T}Kx^{*}
.\]
\end{theorem}


\subsection{Cross Products}
\noindent
A cross product is a way of multiplying two vectors so that the result is a vector.
Although the cross product technically only works for 3D vectors, we will first look a a ``fake'' 2D version to build an intuition.
\begin{equation*}
	\vec{a}\times\vec{b} = a_1b_1-a_2b_2.
\end{equation*}
This ``fake'' 2D cross product gives the area of the parallelogram spanned by $\vec{a}$ and $\vec{b}$.
\begin{equation*}
	\vec{a}\times\vec{b} = \norm{\vec{a}}\norm{\vec{b}}\sin{\theta}
\end{equation*}
where $\theta$ is the angle between $\vec{a}$ and $\vec{b}$.
Another way to think of the magnitude of the cross product, both in 2D and 3D, is as a measure of how perpendicular two vectors are.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.5\textwidth]{../common/vectorsMatrices/CrossProduct.png}
	\caption{Visualization of the cross product}
\end{figure}

\noindent
In 3D, $\vec{a}\times\vec{b}$ is a vector, and similar to the 2D case, the magnitude of $\vec{a}\times\vec{b}$ is equal to the area of the parallelogram spanned by $\vec{a}$ and $\vec{b}$.
\begin{equation*}
	\vec{a}\times\vec{b} = \langle a_2b_3-b_2a_3,a_3b_1-b_3a_1,a_1b_2-b_1a_2 \rangle
\end{equation*}
and
\begin{equation*}
	\norm{\vec{a}\times\vec{b}}=\norm{\vec{a}}\norm{\vec{b}}\sin{\theta}
\end{equation*}
where $\theta$ is the angle between $\vec{a}$ and $\vec{b}$.
Each component of $\vec{a}\times\vec{b}$ gives the area of the parallelogram spanned by $\vec{a}$ and $\vec{b}$ in some plane:
The $x$-component of $\vec{a}\times\vec{b}$ gives the area in the yz-plane ($x$ = 0 plane).
$\vec{a}\times\vec{b}$ is perpendicular, also called ``normal,'' to the plane containing $\vec{a}$ and $\vec{b}$. It's direction, is determined by the right hand rule.\\

\noindent
This cross product table of the standard basis vectors is useful for providing some insight into the properties of the cross product.
\begin{table}[H]
	\centering
	\renewcommand{\arraystretch}{1.5}
	\begin{tabular}{|c||c|c|c|}
		\hline
		$\overrightarrow{\text{row}}\times\overrightarrow{\text{col}}$ & $\hat{i}$  & $\hat{j}$  & $\hat{k}$  \\ \hline\hline
		$\hat{i}$                                & $0$        & $\hat{k}$  & $-\hat{j}$ \\ \hline
		$\hat{j}$                                & $-\hat{k}$ & $0$        & $\hat{i}$  \\ \hline
		$\hat{k}$                                & $\hat{j}$  & $-\hat{i}$ & $0$        \\ \hline
	\end{tabular}
\end{table}

\begin{enumerate}[label=]
	\item \textbf{\underline{NOT} Commutative}, but is antisymmetric
		\begin{equation*}
			\vec{a}\times\vec{b} = -\left(\vec{b}\times\vec{a}\right)
		\end{equation*}
	\item \textbf{Scalar Associative}
		\begin{equation*}
			\left(c\cdot\vec{a}\right)\times\vec{b}=\vec{a}\times\left(c\cdot\vec{b}\right)
		\end{equation*}
	\item \textbf{Distributive}
		\begin{equation*}
			\vec{a}\times\left(\vec{b}\times\vec{c}\right) = \vec{a}\times\vec{b} + \vec{a}\times\vec{c}
		\end{equation*}
\end{enumerate}

\noindent
One can also think of the cross product as the determinant of a matrix.
\begin{equation*}
	\vec{a}\times\vec{b} = \det\begin{bmatrix}
		\hat{i}& \hat{j} & \hat{k} \\
		a_1 & a_2 & a_2\\
		b_1 & b_2 & b_3
	\end{bmatrix}
\end{equation*}


\section{Divided Differences}
\subsection{Divided Differences Notation}
The $k$th divided difference relative to $x_i,\cdots,x_{i+k}$ is
\begin{align*}
  & f[x_i]=f(x_i) \\
  & f[x_i,\cdots,x_{i+k}]=\frac{f[x_{i+1},\cdots,x_{i+k}]-f[x_i,\cdots,x_{i+k-1}]}{x_{i+k}-x_i}
\end{align*}
for each $k=0,1,\cdots,n$, $P_n(x)$ can be rewritten in a form called \emph{Newton's Divided-Difference}:
\[
P_n(x)=f[x_0]+\sum_{k=1}^{n}f[x_0,\cdots,x_k](x-x_0)\cdots (x-x_{k-1})
\]
\begin{theo}
$f\in C^n[a,b]$, $x_i\in [a,b]$ for $i=0:n$, $\exists\xi\in (a,b)$, s.t.
\[
f[x_0,x_1,\cdots,x_n]=\frac{f^{(n)}(\xi)}{n!}
\]
\end{theo}
\begin{proof}
Let $g(x)=f(x)-P_n(x)$, which has $n+1$ distinct zeros in $[a,b]$. According to \emph{Generalized Rolle's Theorem}, $\exists\xi\in (a,b)$, s.t. $g^{(n)}(\xi)=0$.
\begin{align*}
  & 0=g^{(n)}(\xi)=f^{(n)}(\xi)-P_n^{(n)}(\xi)=f^{(n)}(\xi)-n!f[x_0,\cdots,x_k] \\
  \Rightarrow & f[x_0,x_1,\cdots,x_n]=\frac{f^{(n)}(\xi)}{n!}
\end{align*}
\end{proof}

When the nodes are arranged consecutively with equal spacing, then we use $h=x_{i+1}-x_i$  and $x=s\cdot h+x_0$, the equation will become
\begin{align*}
  P_n(x) &= P_n(x_0+sh)=f[x_0]+\sum_{k=1}^{n}s(s-1)\cdots (s-k+1)h^kf[x_0,\cdots,x_k] \\
  &= f[x_0]+\sum_{k=1}^{n}\binom{s}{k}k!h^kf[x_0,\cdots,x_k].
\end{align*}

\subsection{Forward Differences}
\begin{align*}
  f[x_0,x_1]&=\frac{1}{h}\left(f(x_1)-f(x_0)\right)=\frac{1}{h}\triangle f(x_0) \\
  f[x_0,x_1,x_2]&=\frac{1}{2h}\left(\frac{\triangle f(x_1)-\triangle f(x_0)}{h}\right)=\frac{1}{2h^2}\triangle^2f(x_0)
\end{align*}
In general,
\begin{align*}
  & f[x_0,x_1,\cdots,x_k]=\frac{1}{k!h^k}\triangle^kf(x_0) \\
  \Rightarrow & P_n(x)=f[x_0]+\sum_{k=1}^{n}\binom{s}{k}\triangle^kf(x_0)
\end{align*}
\subsection{Backward Differences}
\begin{align*}
  f[x_n,x_{n-1}]&=\frac{1}{h}\nabla f(x_n) \\
  f[x_n,x_{n-1},x_{n-2}]&=\frac{1}{2h^2}\nabla^2f(x_n)
\end{align*}
In general,
\begin{align*}
  & f[x_n,x_{n-1},\cdots,x_{n-k}]=\frac{1}{k!h^k}\nabla^kf(x_n) \\
  \Rightarrow & P_n(x)=f[x_n]+\sum_{k=1}^{n}\frac{s(s+1)\cdots (s+k-1)}{k!}\nabla^kf(x_n)
\end{align*}
Also, we have
\begin{align*}
  & \binom{-s}{k}=\frac{-s(-s-1)\cdots (-s-k+1)}{k!}=(-1)^k\frac{s(s+1)\cdots (s+k-1)}{k!} \\
  \Rightarrow & P_n(x)=f[x_n]+\sum_{k=1}^{n}(-1)^k\binom{-s}{k}\nabla^kf(x_n)
\end{align*}

\subsection{Centered Differences}
\begin{align*}
    P_n(x)=P_{2m+1}(x) &= f[x_0]+\frac{sh}{2}\left(f[x_{-1},x_0]+f[x_0,x_1]\right)+(sh)^2f[x_{-1},x_0,x_1] \\
    &+ \cdots \\
    &+ s^2(s^2-1)\cdots (s^2-(m-1)^2)h^{2m}f[x_{-m},\cdots,x_{m+1}] \\
    &+ \frac{s^2(s^2-1)\cdots(s^2-m^2)h^{2m+1}}{2}\left(f[x_{-m-1},\cdots,x_m]+f[x_{-m},\cdots,x_{m+1}]\right)
\end{align*}
If $n=2m+1$ is odd, we use the above formula, if $n=2m$ is even, we delete the last line and then use the above formula.
\vspace{1cm}\\
\begin{tabular}{@{}llllll@{}}
\toprule
x & f(x) & 1st & 2nd & 3rd & 4th divided differences \\ \midrule
$x_{-2}$ & $f[x_{-2}]$ & $f[x_{-2},x_{-1}]$ & $f[x_{-2},x_{-1},x_{0}]$ & $\underline{f[x_{-2},x_{-1},x_{0},x_{1}]}$ & $\underline{f[x_{-2},x_{-1},x_{0},x_{1},x_{2}]}$ \\
$x_{-1}$ & $f[x_{-1}]$ & $\underline{f[x_{-1},x_{0}]}$ & $\underline{f[x_{-1},x_{0},x_{1}]}$ & $\underline{f[x_{-1},x_{0},x_{1},x_{2}]}$ &  \\
$x_{0}$ & $\underline{f[x_{0}]}$ & $\underline{f[x_{0},x_{1}]}$ & $f[x_{0},x_{1},x_{2}]$ &  &  \\
$x_{1}$ & $f[x_{1}]$ & $f[x_{1},x_{2}]$ &  &  &  \\
$x_{2}$ & $f[x_{2}]$ &  &  &  &  \\ \bottomrule
\end{tabular}