\chapter{projection equations}

The equation for a point object located at ($x_0$, $y_0$, $z_0$) in a circular scan trajectory is given by:

\begin{equation}\label{eq:circularorbit}
\begin{split}
x_{\alpha}& = x_0 \cos \alpha \pm y_0 \sin \alpha, \\
y_{\alpha}& = \mp x_0 \sin \alpha + y_0 \cos \alpha, \\
z& \equiv z_0.
\end{split}
\end{equation}

On a perfectly well aligned detector the projection orbit of the point, $(u_{\alpha}^{(id)}, v_{\alpha}^{(id)})$, in the (x,y,z) coordinate is then given by its projections onto the detector plane,
\begin{equation}\label{eq:idealUVpoints}
u_{\alpha}^{(id)} = \frac{R x_{\alpha}}{y_{\alpha} + R_F}, \hspace{0.4cm} v_{\alpha}^{(id)} = \frac{Rz_0}{y_{\alpha}+R_F},
\end{equation}
where $R$ and $R_F$ are the distances from the focus to the detector and the axis of rotation, respectively.  For a misaligned detector the point (u,v) with (x', y', z') coordinates, the point locations on the detector is given by:

\begin{equation}\label{eq:circularorbit}
\begin{split}
x_{\alpha}& = x_0 \cos \alpha \pm y_0 \sin \alpha, \\
y_{\alpha}& = \mp x_0 \sin \alpha + y_0 \cos \alpha, \\
z& \equiv z_0.
\end{split}
\end{equation}

On a perfectly well aligned detector the projection orbit of the point, $(u_{\alpha}^{(id)}, v_{\alpha}^{(id)})$, in the (x,y,z) coordinate is then given by its projections onto the detector plane,
\begin{equation}\label{eq:idealUVpoints}
u_{\alpha}^{(id)} = \frac{R x_{\alpha}}{y_{\alpha} + R_F}, \hspace{0.4cm} v_{\alpha}^{(id)} = \frac{Rz_0}{y_{\alpha}+R_F},
\end{equation}
where $R$ and $R_F$ are the distances from the focus to the detector and the axis of rotation, respectively.  For a misaligned detector the point (u,v) with (x', y', z') coordinates, the point locations on the detector is given by:

\begin{equation}\label{eq:nonidealUVpoints}
\begin{split}
\begin{pmatrix}
u \\ v
\end{pmatrix} = & \frac{1}{\det \mathbf{Q}}
\begin{pmatrix}
o_{33}-o_{23}v^{(id)}/R & -(o_{13} - o_{23} u^{(id)}/R) \\
-(o_{31} - o_{21} v^{(id)}/R) & o_{11}-o_{21} u^{(id)}/R \\
\end{pmatrix} \\
\times &
\begin{pmatrix}
u^{(id)'} - d_x \\
v^{(id)'} - d_z
\end{pmatrix},
\end{split}
\end{equation}
where the primes denote a simple rescaling of the ideal orbit by

\begin{equation}
\begin{pmatrix}
u^{(id)'} \\ v^{(id)'}
\end{pmatrix}
= \frac{R_y'}{R}
\begin{pmatrix}
u^{(id)} \\ v^{(id)}
\end{pmatrix}.
\end{equation}
where
\begin{equation}
\begin{split}
\det \mathbf{Q} =& (o_{11} - o_{21} u^{(id)}/R)(o_{33}-o_{23}v^{(id)}/R) \\
& -(o_{13}-o_{23}u^{(id)}/R)(o_{31}-o_{21} v^{(id)}/R).
\end{split}
\end{equation}

and $\mathbf{O}$ is the rotation matrix of the misaligned detector shown by
\begin{equation}
\mathbf{O} =
\begin{pmatrix}
\cos \eta \cos \varphi - \sin \eta \sin \theta \sin \varphi & -\cos \theta \sin \varphi & -\cos \varphi \sin \eta - \cos \eta \sin \theta \sin \varphi \\
\cos \varphi \sin \eta \sin \theta + \cos \eta \sin \varphi & \cos \theta \cos \varphi & \cos \eta \cos \varphi \sin \theta - \sin \eta \sin \varphi \\
\cos \theta \sin \eta & -\sin \theta & \cos \eta \cos \theta
\end{pmatrix}
\end{equation}


%---------------------------Aspect Ratio-----------------------------
\section{Aspect Ratio\label{s:tri-aspect-ratio}}

The aspect ratio of a triangle is:
\[
q = \frac{L_{\max}}{2\sqrt{3}r}.
\]
Using~\eqref{eq:Arp}, one can thus write it alternatively as
\begin{equation*}
\label{eq:triangle_aspect_ratio}
q = \frac{L_{\max}(L_0 + L_1 + L_2)}{4\sqrt{3}A}.
\end{equation*}

Note that in earlier versions of \verd{}, triangle aspect ratio
was used to call out what is now called the triangle aspect.

\trimetrictable{aspect ratio}%
{$1$}%                                                Dimension
{$[1,1.3]$}%                                          Acceptable range
{$[1,DBL\_MAX]$}%                                     Normal range
{$[1,DBL\_MAX]$}%                                     Full range
{$1$}%                                                Unit equilateral triangle value
{\cite{pebay:03}}%                                    Reference(s)
{v\_tri\_aspect\_ratio}%                            Verdict function name


\section{Solutions to Differential Equations}
\begin{definition}
	A solution to an ODE on an interval $I$ is a function $f$ that makes the differential equation true on $I$ when $f$ is substituted into the equation.
\end{definition}
\noindent
Although this definition seems straightforward it gives us a formal way to check if a function is a solution to a differential equation.

\begin{example}
	Check that $y = t^2\ln{t}$ ($t > 0$) and $y = t^2$  are solutions to the differential equation
	\begin{equation*}
		t^2y'' - 3ty' + 4y = 0.
	\end{equation*}
\end{example}
\noindent
We'll check $y= t^2\ln{t}$ ($t > 0$) first.
\begin{align*}
	t^2\left(t^2\ln{t}\right)'' - 3t\left(t^2\ln{t}\right)' + 4\left(t^2\ln{t}\right) &= 0 \text{, } t > 0 \\
	t^2\left(3 + 2\ln{t}\right) - 3t\left(t + 2t\ln{t}\right) + 4\left(t^2\ln{t}\right) &= 0 \text{, } t > 0 \\
	3t^2 + 2t^2\ln{t} - 3t^2 - 6t^2\ln{t} + 4t^2\ln{t} &= 0 \text{, } t > 0 \\
	0 &= 0.
\end{align*}
So $y = t^2\ln{t}$ ($t > 0$) is a solution.
Now we'll check $y= t^2$.
\begin{align*}
	t^2\left(t^2\right)'' - 3t\left(t^2\right)' + 4\left(t^2\right) &= 0 \\
	t^2\left(2\right) - 3t\left(2t\right) + 4t^2 &= 0 \\
	2t^2 - 6t^2 + 4t^2 &= 0 \\
	0 &= 0.
\end{align*}
So $y = t^2$ is also a solution.


\section{Link analysis}

\subsection{Power method}
Computing the pagerank index given a connection matrix
consists of finding
a probability distribution over websites in the web graph.

We can initialize a vector with $n$ components, each
with value $\frac{1}{n}$, where $n$ is the number of websites
and multiply it to the connection matrix many times.
We will converge to an eigenvector of the matrix, with eigenvalue 1,
that represent the final probability distribution over websites.

\paragraph{Result is an eigenvector}
We start with a matrix $A = [a_{i,j}]_{n \times n}$, given an eigenpair $(\lambda, \vec{x})$, it holds that $A\vec{x} = \lambda \vec{x}$.

Also, for $A$ dimensionality, we have $n$ eigenpairs, and we can
sort them by non-increasing values of eigenvalue
$$(\lambda_1, \vec{x}_1), \dots, (\lambda_n, \vec{x}_n)$$

Eigenvalues are orthogonal and they constitute a basis for the space
in which vectors such as $\vec{x}$ lives.
If we now consider a starting vector $v_0$ as the starting probability distribution, we can rewrite it as follows.
\begin{equation}
    \begin{aligned}
        \vec{v}_0 &= \alpha_1\vec{x}_1 + \dots + \alpha_n\vec{x}_n
    \end{aligned}
\end{equation}
Thus the matrix vector multiplication to find the final distribution becomes:
\begin{equation}
    \begin{aligned}
         \vec{v}_1 = A\vec{v}_0 &= A(\alpha_1\vec{x}_1 + \dots + \alpha_n\vec{x}_n) && \text{For the above equation}\\
         &= \alpha_1A\vec{x}_1 + \dots + \alpha_nA\vec{x}_n && \text{Distribute multiplication}\\
         &= \alpha_1\lambda_1\vec{x}_1 + \dots + \alpha_n\lambda_n\vec{x}_n && \text{They are eigenvectors}
    \end{aligned}
\end{equation}
Computing $\vec{v}_2$ now becomes:
\begin{equation}
    \begin{aligned}
         \vec{v}_2 = A\vec{v}_1 &= A(\alpha_1\lambda_1\vec{x}_1 + \dots + \alpha_n\lambda_n\vec{x}_n) && \text{Using the previous result}\\
         &= \alpha_1\lambda_1^2\vec{x}_1 + \dots + \alpha_n\lambda_n^2\vec{x}_n && \text{We distribute A, as before}\\
    \end{aligned}
\end{equation}
At the generic $\vec{v}_k$ we have:
\begin{equation}
    \begin{aligned}
         \vec{v}_k &= \alpha_1\lambda_1^k\vec{x}_1 + \dots + \alpha_n\lambda_n^k\vec{x}_n\\
         &= \lambda_1^k\bigg(\alpha_1\vec{x}_1 +
         \alpha_2\bigg(\frac{\lambda_2}{\lambda_1}\bigg)^k\vec{x}_2 \dots\bigg)\\
         &\approx \lambda_1^k\alpha_1\vec{x}_1 &&\text{As k grows, lambdas are sorted}
    \end{aligned}
\end{equation}
We have a practical problem, as if $\lambda_1$ is smaller than 1 this method
diverges.
It holds that the maximum eigenvalue of a row-stochastic matrix
is equals to 1.
\paragraph{Determinant of the transpose}
To prove it, we start to prove that $\mathit{det}A^T = \mathit{det}A$:
\begin{equation}
    \begin{aligned}
        \mathit{det}A &= \sum_i a_{ij} c_{ij} = \sum_j a_{ij} c_{ij}\\
        \mathit{det}A^T &= \sum_i a_{ij}^T c_{ij}^T = \sum_i a_{ji} c_{ji} = \sum_j a_{ij} c_{ij} = \mathit{det}A\\
    \end{aligned}
\end{equation}
It also holds that $A$ and $A^T$ have the same eigenvalues:
\begin{equation}
    \begin{aligned}
        \mathit{det}(A-\lambda I) = 0 &\longleftrightarrow \mathit{det}(A-\lambda I)^T = 0\\
        &\longleftrightarrow \mathit{det}(A^T-\lambda I) = 0
    \end{aligned}
\end{equation}

\paragraph{One as eigenvalue}
We can now show that 1 is always an eigenvalue of a row-stochastic matrix:
\begin{equation}
    \begin{aligned}
        A\cdot \vec{1} = \biggr[\sum_j a_{ij}\cdot1\biggr]_n = \vec{1} && \text{As the components in $A$ row-wise are probabilities}
    \end{aligned}
\end{equation}
We can multiply the unit vector by one showing that it is a valid eigenvalue.
Similarly, this results holds even if $A$ is column-stochastic.

\paragraph{Power of stochastic matrix}
We now prove that if $A$ is row-stochastic, $A^k$ is as well.
\begin{equation}
    \begin{aligned}
        k=1:&& \text{trivial}\\\\
        A^k\;r.s. \rightarrow A^{k+1}\;r.s. : && a_{ij}^{k+1} &=
        \sum_s a_{is}^k \cdot a_{sj}\\
        && \sum_j a_{ij}^{k+1} &= \sum_j
        \sum_s a_{is}^k \cdot a_{sj}\\
        && &= \sum_s a_{is}^k \sum_j a_{sj}\\
        && &= \sum_s a_{is}^k \cdot 1 = 1
    \end{aligned}
\end{equation}

\paragraph{One is highest eigenvalue}
We are left to show 1 is the highest eigenvalue of a column-wise stochastic matrix. Suppose it exists an eigenvalue greater than one.
\begin{equation}
    \begin{aligned}
        A^T = \lambda\vec{v}\\
        (A^T)^k = \lambda^k\vec{v} \\
        \sum_j((a^T)^k)_{ij}v_j = \lambda^kv_i
    \end{aligned}
\end{equation}
We can overestimate the left end with $v_\mathit{max}$, the maximum vector component, we underestimate tghe right end with $G$, a value of choice.
It exists a $k$ that makes the equation true as lambda is greater than one. We then divide by $v_\mathit{max}$.
\begin{equation}
    \begin{aligned}
        \sum_j((a^T)^k)_{ij}v_\mathit{max} &> G\\
        1 = \sum_j((a^T)^k)_{ij} &> \frac{G}{v_\mathit{max}}\\
    \end{aligned}
\end{equation}
$A$ is column wise stochastic, the transpose is row stochastic, thus if we raise to the $k$ we get a sum of one.
This means it is absurd to assume we have an eigenvalue greater than one, as $G$ is arbitrary big.


\subsection{Discrete Regularized Transport}
\label{sec:regsymme}

So far, we have introduced a transport problem where the mass conservation constraint is relaxed. The second step is to define its regularization. A classic way of imposing regularity on a mapping $V: \RR^d \rightarrow \RR^d$ is by measuring the amplitude of its derivatives. Two examples for continuous functions are the quadratic Tikhonov regularizations such as the Sobolev semi-norm $\|\nabla V\|^2$, and the anisotropic total variation semi-norm $\|\nabla V\|_1$ regularization. But, the differential operator $\nabla$  cannot be applied directly to our point clouds due to the lack of neighborhood definition. To extend the definition of the gradient operator, we need to impose a graph structure on each point cloud.

In our setting, we want to regularize the discrete map $T$ defined in~\eqref{eqT}, which is only defined at the location of the points as $X_i \mapsto \tilde{V}_i=X_i - \diag(\Sig \U)^{-1} (\Sig Y)_i $.  To avoid the normalization $\diag(\Sig \U)$ (which typically leads to non-convex optimization problems) we impose a regularity on the map $X_i \mapsto V_i = \diag(\Sig \U)X_i-(\Sigma Y)_i$. This switch also has the advantage of imposing a stronger regularization in regions with large weights and to further regularize the variations of the weights $\Sig \U \in \RR^N$. We believe these are actually interesting features to reduce artifacts for imaging applications.

\paragraph{Gradient on Graphs}

A natural way to define a gradient on a point cloud $X$ is by using  the gradient on a weighted graph $\Gg_X = (X,E_X,W_X)$ where $E_X \subset \{1,\ldots,N\}^2$ is the set of edges and $W_X$ is the set of weights, $W_X = (w_{i,j})_{i,j=1}^N:  \{1,\ldots,N\}^2 \mapsto \RR^+$, satisfying $w_{i,j}=0$ if $(i,j) \notin E_X$. The edges of this graph are defined depending on the application. A typical example is the $n$-nearest neighbor graph, where every vertex $X_i$ is connected to $X_j$ if $X_j$ is one of the $n$-closest points to $X_i$ in $X$, creating the edge $(i,j) \in E_X$, with a weight $w_{i,j}$. Because the edges are directed, the adjacency matrix is not symmetric.

The gradient operator on $\Gg_X$ is defined as $G_X : \RR^{N \times d} \rightarrow \RR^{P \times d}$, where $P=\|E_X\|$ is the number of edges and where, for each $V=(V_i)_{i=1}^N \in \RR^d$,
\eq{
	G_X V = \pa{ w_{i,j}(V_i-V_j) }_{(i,j) \in E_X} \in \RR^{P \times d}.
}
A classic choice for the weights to ensure consistency with the directional derivative is $w_{i,j} = \norm{X_i-X_j}^{-1}$, see for instance~\cite{guilboa07}.

%%%%%%%%%%%%%%%
\paragraph{Regularity Term}

The regularity of a transport map $V \in \RR^{N \times d}$ is then measured according to some norm of $G_X V$, that we choose here for simplicity to be the following
\eq{
	\regul{G_X  V} = \sum_{(i,j) \in E_x} \left(\norm{ w_{i,j} (V_i-V_j) }_q \right)^p,
}
where $\|.\|_q$ is the $\ell^q$ norm in $R^d$.

The case $(p,q)=(1,1)$ is the graph anisotropic total variation, $(p,q)=(2,2)$ is the graph Sobolev semi-norm, and $(p,q)=(1,2)$ is the graph isotropic total variation, see for instance~\cite{elmoataz-graph} for applications of these functionals to imaging problems such as image segmentation and regularization.

\subsection{Symmetric Regular OT Formulation}

Given two point clouds X and Y, our goal is to compute a relaxed OT mapping between them which is regular with respect to both point clouds. To simplify notation, we conveniently re-write the displacement fields we aim to regularize as:
\eq{
	\De_{X,Y}(\Sig) = \diag( \Sigma \U ) X - \Sigma Y
	\qandq
	\De_{Y,X}(\Sig^*) = \diag( \Sigma^* \U ) Y -  \Sigma^* X.
}

Our goal is to obtain a partial matching that is regular according to $X$ and $Y$, so we create two graphs $\Gg_X$ and $\Gg_Y$ as described in Section~\ref{sec:regsymme} and we denote the corresponding gradient operators $G_X \in \RR^{P_X \times N}$ and $G_Y \in \RR^{P_Y \times N}$ where $P_X$ and $P_Y$ are the number of edges in the respective graphs. The symmetric regularized discrete OT energy is defined as:
\eql{\label{eq-symm-reg-energy}
\umin{\Sig \in \Matr_\kappa} \dotp{\Sig}{\Cost{X}{Y}} + \la_X \regul{G_X \De_{X,Y}(\Sig)} + \la_Y \regul{G_Y \De_{Y,X}(\Sig^*)},
}
where $(\la_X,\la_Y) \in (\RR^+)^2$ controls the desired amount of regularity.
The case $\kappa=(1,1,1,1)$ and $(\la_X,\la_Y)=(0,0)$ corresponds to the usual OT defined in~\eqref{eqMK}, and $(\la_X,\la_Y)=(0,0)$ corresponds to the un-regularized formulation~\eqref{eq-relax-map}.

%%%%%%%%%%%%%%%
\subsection{Algorithms}
\label{secalgosymm}

Specific values of the parameters $p$ and $q$ lead to different regularization terms, which in turn necessitate different optimization methods. In the following, for the sake of concreteness, we concentrate on the specific cases $(p,q)=(2,2)$ and $(p,q)=(1,1)$.

%%%
\paragraph{Sobolev regularization}

Defining $q=p=2$ fixes the regularization term as a graph-based Sobolev regularization. In this specific case, the minimization~\eqref{eq-symm-reg-energy} becomes a quadratic programming problem
\begin{equation} \label{eq-symm-S}
	\umin{\Sig \in \Matr_\kappa}
	f(\Sig) =  \dotp{ \Cost{X}{Y}}{\Sig } + \frac{\la_X}{2} \norm{ \Gamma_{X,Y}(\Sig)\|^2 + \frac{\la_Y}{2} \|\Gamma_{Y,X}(\Sig) }^2,
\end{equation}
where $\Gamma_{X,Y}(\Sig) = G_X \De_{X,Y}(\Sig)$ and $\Gamma_{Y,X}(\Sig) = G_Y \De_{Y,X}(\Sig^*)$. The Frank-Wolfe algorithm is well tailored to solve such problems, as noticed for instance in~\cite{Zaslavskiy09}, given that $f$ is convex and differentiable, and $\Matr_\kappa$ is a convex set.
The Frank-Wolfe method (also known as conditional gradient) iterates the following steps until convergence
\begin{equation}
\begin{aligned}\label{eq-frankwolfe-update}
	\tilde\Sig^{(\ell+1)} &\in \uargmin{\tilde\Sig \in \Matr_\kappa} \dotp{\nabla f(\Sig^{(\ell)})}{ \tilde\Sig } \\
	\Sig^{(\ell+1)} &= \Sig^{(\ell+1)} + \tau_\ell ( \tilde\Sig^{(\ell+1)}-\Sig^{(\ell+1)} ),\\
\end{aligned}
\end{equation}
where $\tau_\ell$ is obtained by line-search. The first equation of~\eqref{eq-frankwolfe-update} is a linear program which is efficiently solved using interior point methods~\cite{Nesterov-Nemirovsky-Book}.
The function $f$ is quadratic, and to compute $\nabla f(\Sig)$, we note that $\nabla \frac{1}{2}\norm{\Ga_{X,Y}(\Sig)}^2 = \Ga_{X,Y}^*( \Ga_{X,Y}(\Sig) )$, where the adjoint $\Ga_{X,Y}^*$ of $\Ga_{X,Y}$ is
\eq{
	\Ga_{X,Y}^* = \De^*_{X,Y} \circ G_X^*
	\qwhereq
	\choice{
		\De^*_{X,Y}(U)= \diag^*(U X^*)\U^*-U Y^*, \\
		\De^*_{Y,X}(U)= (\diag^*(U Y^*)\U^*)^*-X U^*,
	}
}
where $\diag^*: \RR^{N \times N} \mapsto \RR^{N}$ is the adjoint of the $\diag$ operator, and given $A \in \RR^{N \times N}$, $\diag^*(A)$ is a vector composed by the elements on the diagonal of $A$.

This leads to the following formula
\eq{
	\nabla f(\Sig) = \Cost{X}{Y} + \la_X  \De^*_{X,Y}(G_X^*  \Gamma_{X,Y}(\Sig)) + \la_Y \De^*_{Y,X}(G_Y^* \Gamma_{Y,X}(\Sig)),
}

The line search optimal step can be explicitly computed as
\eq{
	\tau_{\ell} =
	\frac{ - \dotp{ E^{(\ell)} }{
		\Cost{X}{Y} } -
		\dotp{  \Gamma_{X,Y}(E^{(\ell)}) }{  \Gamma_{X,Y}(\Sig^{(\ell)}) } -
		\dotp{  \Gamma_{Y,X}( E^{(\ell)} )  }{  \Gamma_{Y,X} (\Sig^{(\ell)}) }
	}{
		\la_X \norm{\Gamma_{X,Y}(E^{(\ell)})}^2 +   \la_Y \norm{\Gamma_{Y,X}(E^{(\ell)})}^2
	}
}
where $E^{(\ell)} = \Sig^{(\ell+1)} - \tilde\Sig^{(\ell+1)}$.

%%%%
\paragraph{Anisotropic TV regularization}

We define an anisotropic total variation (TV) norm by setting the parameters $q=p=1$.
Problem~\eqref{eq-symm-reg-energy} can be re-written as a linear program by introducing the auxiliary variables $U_X \in \RR^{P_X \times d}$ and $U_Y \in \RR^{P_Y \times d}$
\eql{
\begin{aligned}
& \underset{\Sig, U_X,U_Y}{\text{min}}  & & \dotp{ \Cost{X}{Y}}{\Sig}  + \lambda_X \dotp{U_X}{\U}+ \lambda_Y \dotp{U_Y}{ \U} \\
& \mbox{subject to} & &
\left\{ \begin{aligned}
-U_X  & \leq G_X ( \Sig Y  - \diag(\Sig \U) X)\leq U_X, \\
-U_Y & \leq G_Y( \Sig^* X - \diag(\Sig^* \U) Y) \leq U_Y, \\
\Sig & \in \Matr_\kappa. \\
\end{aligned}
\right.
\end{aligned}\label{eq-symm-TV}
}

%%%%
\paragraph{Numerical illustrations}

In Fig.~\ref{exlk}, we can observe, on a synthetic example, the influence of the parameters $\kappa$ and $(\lambda_X, \lambda_Y)$, from equation~\eqref{eq-symm-reg-energy}.

\begin{figure*}\label{figlambda}
\centering
\begin{tabular}{@{}|@{}c@{}|@{}c@{}|@{}}
\hline
  \includegraphics[width=.4\linewidth]{../images/syntheticexamples/symmetricsyntheticmapping_l0_KX8_KY8_nn4} &
  \includegraphics[width=.4\linewidth]{../images/syntheticexamples/symmetricsyntheticmapping_l0001_KX8_KY8_nn4} \\
	 {$\la_X=\la_Y=0$} & {$\la_X =\la_Y=0.001$}\\\hline
  \includegraphics[width=.4\linewidth]{../images/syntheticexamples/symmetricsyntheticmapping_l10_KX8_KY8_nn4} &
  \includegraphics[width=.4\linewidth]{../images/graph} \\
  {$\la_X =\la_Y=10$} & Graphs \\\hline
\end{tabular}
\caption{Given two sets of points $X$ (in blue) and $Y$ (in red), we show the points $Z=\diag(\Sigma \U)^{-1} \Sigma Y$ (in green), and the mappings $\Sigma_{i,j}$ as line segments connecting $X_i$ and $Y_j$, which are dashed if $\Sigma_{i,j} \in ]0.1,1[$  and solid if $\Sigma_{i,j}=1$. The results were obtained with the relaxed and regularized OT formulation, setting the parameters to $\kappa=(0.1,8,0.1,8)$. Note the influence of a change in $\la_X$ and $\la_Y$ on the final result: with no regularization ($\la_X=\la_Y=0$) only few points in the data set are matched. The introduction of regularization ($\la_X=\la_Y=0.001$) spreads the connections among the clusters, while maintaining the cluster-to-cluster matching. For  a high value of $\la_X=\la_Y=10$, the regularization tends to match the clusters with similar shape with each other, where the shape is defined by the graph structure. The graphs $\Gg_X$ and $\Gg_Y$ are represented with the nodes on blue and red respectively, and the edges as solid lines.}\label{exlk}
\end{figure*}

For $\la_X=\la_Y=0$ one obtains the relaxed symmetric OT solution, where the transport maps the points in $X$ to the closest point on $Y$, and vice versa. As we increase the values of $\la_X$ and $\la_Y$ to $0.001$, we can see how the regularization affects the mapping. Let us analyze $\regul{G_X \De_{X,Y}(\Sig)} = \|G_X \diag(\Sig \U) X - G_X \Sig Y\|^2 $, for instance. The  term $G_X \diag(\Sig \U) X$ is measuring the regularity of the weights $\diag(\Sig \U)$ on $X$ and the consequence is that for $\la_X=\la_Y=0.001$ there are
plenty of connections with low weight (there are few solid lines), while for $\la_X=\la_Y=0$ there are several mappings with $\Sig_{i,j} = 1$ (solid lines). So, the regularization promotes a spreading of the matchings.

The minimum of $\regul{G_X \De_{X,Y}(\Sig)}$ is reached when $G_X \diag(\Sig \U) X = G_X \Sig Y$, that is, when the graph structure of $X$ has the same shape as the graph structure of $\Sig Y$, which both can be observed in the last column and row. For high values of $\la_X=\la_Y$ the matchings tend to link the clusters by their shape, that is, the big cluster on $X$ with the big cluster of $Y$, and similarly for the small clusters (note that the links with higher value are between the small clusters).


%\documentclass[prc,preprint,superscriptaddress,showpacs,floatfix,widetables]{revtex4}
\documentclass[]{article}
\usepackage[dvips]{graphicx}
\usepackage{epsfig, color}
\usepackage{pst-plot}
\usepackage{bm}
\usepackage{hyperref,amssymb,amsmath,a4wide}
\usepackage{palatino}
\usepackage{dcolumn}% Align table columns on decimal point
\usepackage{graphicx}% Include figure files
\usepackage{listings}
\lstset{language=c++}
\lstset{alsolanguage=[90]Fortran}
\lstset{basicstyle=\small}
\lstset{backgroundcolor=\color{white}}
\lstset{frame=single}
\lstset{stringstyle=\ttfamily}
\lstset{keywordstyle=\color{red}\bfseries}
\lstset{commentstyle=\itshape\color{blue}}
\lstset{showspaces=false}
\lstset{showstringspaces=false}
\lstset{showtabs=false}
\lstset{breaklines}

\fontfamily{phv}
\selectfont

\hypersetup{colorlinks=true, urlcolor=blue, linkcolor=black}

\begin{document}

  \title{FYS 3150/4150: Possible solution of Project 1.}
  \author{Maxim Kartamyshev}
  \date{}

  \maketitle

  Comments and critics are welcome and can be sent to
  \href{mailto::maximk@fys.uio.no}{maximk@fys.uio.no}.

  \section{Part a: Linear equation systems}
  LU decomposition is one of the commonly used methods to solve systems of linear equation:
  \begin{equation}
    \left(\begin{array}{ccccc}
      a_{11}& a_{12}& a_{13} &\dots &a_{1n}  \\
      a_{21}& a_{22}& a_{23} &\dots &a_{2n}  \\
      \dots &\dots  &\dots   &\dots &\dots   \\
      a_{n1}& a_{n2}& a_{n3} &\dots &a_{nn}  \\
    \end{array}
    \right)\left(\begin{array}{c}
      x_1     \\
      x_2     \\
      \dots   \\
      x_n     \\
    \end{array} \right)
    =
    \left(\begin{array}{c}
      y_1     \\
      y_2     \\
      \dots   \\
      y_{n-1} \\
    \end{array} \right)\nonumber
  \end{equation}
  which in compact form reads:
  \begin{equation}
    {\bf A}{\vec x} = {\vec y}  \label{eq:lu_system}
  \end{equation}
  The method consists of decomposing the original matrix {\bf \it A} into a product of lower-diagonal {\bf \it L}
  and upper-diagonal {\bf \it U} matrices:
  \begin{equation}
  A = LU \nonumber
  \end{equation}
  and then solving the linear system (\ref{eq:lu_system}) in two steps:
  \begin{equation}
    L{\vec w} = {\vec y} \;\;\;\;\;\;\;\;\;\;\; U{\vec x} = {\vec w} \label{eq:sys_diag}
  \end{equation}
  Due to the form of {\bf \it L} and {\bf \it U} matrices, the two above equations can be easily solved via back substitution.\\
  \\
  In this project we use supplied library routines for LU-decomposition and back substitution to solve the following
  linear system:
  \begin{equation}
    \left(\begin{array}{ccc}
     \!\!\! -1 &  1& \!\!\! -4 \\
      2  &2 &0  \\
      3  &3 &3  \\
    \end{array}
    \right)\left(\begin{array}{c}
      x_1     \\
      x_2     \\
      x_3     \\
    \end{array} \right)
    =
    \left(\begin{array}{c}
      0     \\
      1     \\
      \frac{1}{2} \\
    \end{array} \right)\nonumber
  \end{equation}
  \\
  which is satisfied with ${\vec x} = [1.25, -0.75, -0.5]^T$. Numerical solution starts with first reading matrix {\bf \it A}
  and vector ${\vec y}$ from a file, LU-decomposition of the matrix with library routine {\bf ludcmp}, and finally
  obtaining the solution with help of routine {\bf lubksb}. Routine {\bf solve\_part\_a} in the attached program listing
  contains implementation of this algorithm and is tested to produce the correct answer.

  \section{Part b: Tri-diagonal linear systems}
  LU-decomposition is a general method and can be used for any non-singular matrix {\bf \it A}.
  There also exist less general methods, applicable only to the matrices of a particular form, but
  allowing for simple and fast computational algorithms.\\
  \\
  In this part of the project we have to solve a linear system governed by the so-called tri-diagonal
  matrix:
  \\
  \begin{equation}
    \left(\begin{array}{ccccccc}
      b_1   & c_1  & 0    &\dots   &0       &0       &0       \\
      a_2   & b_2  & c_2  &\dots   &0       &0       &0       \\
      \dots &\dots &\dots &\dots   &\dots   &\dots   &\dots   \\
      0     &0     &0     &\dots   &a_{n-1} &b_{n-1} &c_{n-1} \\
      0     &0     &0     &\dots   &0       &a_n     &b_n     \\
    \end{array}
    \right)\left(\begin{array}{c}
      x_1     \\
      x_2     \\
      \dots   \\
      x_{n-1} \\
      x_n     \\
    \end{array} \right)
    =
    \left(\begin{array}{c}
      y_1     \\
      y_2     \\
      \dots   \\
      y_{n-1} \\
      y_{n-1} \\
    \end{array} \right) \nonumber
  \end{equation}\\
  \\
  Instead of using LU-decomposition to solve this problem, a much easier method, accounting
  for the specific form of matrix {\bf \it A}, can be employed:
  \\
  \begin{lstlisting}
    // solve tri-diagonal system
    // forward substitution
    btemp = b[0]; y[0] = f[0] / btemp;
    for(i = 1; i < n; i++) {
      temp[i] = c[i-1] / btemp;
      btemp = b[i] - a[i] * temp[i];
      y[i] = (f[i] - a[i] * y[i-1]) / btemp;
    }
    // back substitution
    for(i = n-2; i >= 0; i--) {
      y[i] = y[i] - temp[i+1] * y[i+1];
    }
  \end{lstlisting}
  \ \\
  The routine {\bf solve\_part\_b} in the attached program listing contains implementation of the above algorithm.
  Solution of the $100\times 100$ tri-diagonal system with $a_i = c_i = - 0.01$, $b_i = 0.02$ and $f_i = 0.1*(i+1)$,
  $i=\overline{0,n-1}$, is displayed on Figure \ref{fig:tridiag}. LU-decomposition method has been used to verify
  the tri-diagonal solver.\\
  \begin{figure}[htbp]
    \centering
    \epsfig{file=tri.eps, width=\textwidth}
    \caption{Solution of the 100x100 tri-diagonal system from the project text.}
    \label{fig:tridiag}
  \end{figure}
  \\
  Amount of operations required to solve the $n \times n$ tri-diagonal systems goes as $O(n), n \gg 1$. This stems
  from the fact that implementation of both forward and back substitution in the algorithm require each only
  one (non-nested) loop with index $i$ changing from 0 to $n-1$. Compared with $O(n^3)$ operations in LU-decomposition
  and Gauss elimination, the algorithm for solving tri-diagonal system provides obvious computational advantage,
  but its applicability is limited to the tri-diagonal matrices only.\\
  \\
  It is worth noticing that once one has performed the LU-decomposition of a general non-singular matrix, obtaining
  solution of corresponding linear system requires only $O(n)$ operations (back-substitution). It means that we
  can use the LU-decomposed form of the original matrix to solve the whole class of linear system having the same
  matrix {\bf \it A} and different vectors ${\vec y_i}$.\\
  \\
  Finally, let's compare times used by the tri-diagonal solver and LU-decomposition method. UNIX function 'time' is used
  for precise measurements of time intervals. The Table {\ref{tab:timing}} below summarizes the timing results obtained
  while solving tri-diagonal systems of different dimensionalities $n$.
  \begin{table}[hbp]
    \center
    \caption{Time usage, s}
    \begin{tabular}{ccc}
      \hline
      n & Tri-diagonal solver & LU-decomposition \\
      \hline
      \\
      1000 & $4^{-3}$  &  $32*10^{-3}$  \\
      2000 & $4^{-3}$  &  $114*10^{-3}$ \\
      4000 & $5^{-3}$  &  $442*10^{-3}$  \\
      \\
      \hline
    \end{tabular}
    \label{tab:timing}
  \end{table}
  \\
  The timing results for the tri-diagonal solver show almost no dependence on the dimensionality of the problem.
  This simply means that the time used by tri-diagonal solver is rather small compared to the time spent in other
  parts of the program. One can easily see that the LU-decomposition method uses noticeably longer time to
  solve exactly the same linear system.

\end{document}


\documentclass{article}

\usepackage{amsmath}
\usepackage{numprint}

\author{Daniel Fernandes Martins (danielfmt)}
\title{Question \#1 Solution}

\begin{document}

\maketitle

\section{Solving for $N$}

Substituting the values for $\sigma$, $d$ and $E_{in}$, and putting the
following equation in terms of $N$:

\begin{align*}
E_\mathcal{D}[E_{in}(w_{lin})] &= 0.1^2 \left(1 - \frac{8+1}{N} \right) \\
0.008 &\leq 0.01 \left(1 - \frac{9}{N} \right) \\
0.008 &\leq 0.01 - \frac{0.09}{N} \\
-0.002 &\leq - \frac{0.09}{N} \\
N &\geq 45
\end{align*}

Since we need to pick the least number of examples $N$ among the alternatives
that also satisfy this inequality, we'd have to choose $N=100$.

\end{document}


\chapter{Two Fibonaccis}

$fib(n):$
\begin{algorithmic}
\IF { $ n=0 $ or $ n=1 $ }
\RETURN $n$
\ELSE
\RETURN $fib(n-1) + fib(n-2)$
\ENDIF
\end{algorithmic}

We can state a recurrence for this algorithm:

\begin{align*}
T(n)
&= T(n-1) + T(n-2) + O(1) \\
&\geq fib(n-1) + fib(n-2) \\
&= fib(n)
\end{align*}

$fib2(n):$
\begin{algorithmic}
\IF { $ n = 0 $ }
\RETURN 0
\ENDIF
\STATE create array $f[0 .. n]$
\STATE $f[0] \leftarrow 0, f[1] \leftarrow 1$
\FOR { $ i \leftarrow 2$ to $n$ }
\STATE $f[i] \leftarrow f[i-1] + f[i-2]$
\ENDFOR
\RETURN $f[n]$
\end{algorithmic}

Addition of two numbers in the preceding algorithm takes constant time
until the values exceed the maximum value that can be stored in a
word.  After which, we need to consider how values of arbitrary length
are added.


\subsection{SVD as a function composition}

The first thing to remember, is that matrices are the operational
representation of an special type of function between vector
spaces\footnote{The reader is invited to review any Linear Algebra
  textbook, to recall the definition of a vector space}
called linear transformations (also called linear mappings or linear
  functions). We say is an operational representation, in the
sense that they provide an explicit recipe to apply the
function. Furthermore, the functions they represent are
special, as they have the nice property of preserve algebraic
structure across domain and codomains. Such
property can be summarized as: \\

\[
f(\alpha x + \beta y) = \alpha f(x) + \beta f(y)
\]
\hfill

Where the addition and products mentioned above, are the vector
addition and multiplication by an scalar; defined for vector
spaces. For the specific case of this work, where we restrict
our attention to real matrices, we can tell that they do represent
functions $f: \R{n} \fromto \R{m}$. \\

In this context of linear functions, the matrix multiplication is the
operational representation of the composition of the associated
functions. A matrix factorization is in essence, a way to understand
what the underlying function does; the whole product can be seen as
serial algorithm, where each matrix represents on particular step or
transformation. Each of the four matrices that appear on the SVD
factorization, has its own function as follows:

\begin{itemize}
\item $A$ is a function $\func{F_A}: \R{n} \fromto \R{m}$
\item $V$ is a function $\func{F_V}: \R{n} \fromto \R{n}$
\item Same goes for  \trans{V}, which is $\func{F_{\trans{V}}}: \R{n} \fromto \R{n}$
\item $\Sigma$ is a function $\func{F_\Sigma}: \R{n} \fromto \R{m}$
\item $U$ is a function $\func{F_U}: \R{m} \fromto \R{m}$
\end{itemize}
\hfill

Thus, in the context of function compositions, the SVD factorization
can be restated as: \\

\[
\func{F_A}(\vec{x}) = \func{F_U}(\func{F_\Sigma}(\func{F_{\trans{V}}}(\vec{x})))
\]
\hfill


\input{../common/common.tex}

\title{Math notes - Devil's chessboard}
\author{Uwe Hoffmann}
\hypersetup{colorlinks, pdftitle={Math notes - Devil's chessboard}}

\begin{document}

\setcounter{chapter}{1}
\section*{Devil's chessboard}

\newthought{Hamming codes} are used to solve the problem\footnote{\bibentry{devils_chessboard}} in this note.

You, your friend, and the Devil play a game. You and the Devil are in the room with a chess board with $64$ tokens on it, one on each square. Meanwhile, your friend is outside of the room. The token can either be on an up position or a down position, and the difference in position is distinguishable to the eye. The Devil mixes up the positions (up or down) of the tokens on the board and chooses one of the  squares and calls it the magic square. Next, you may choose one token on a square and flip its position. Then, your friend comes in and must guess what the magic square was by looking on the squares on the board.\footnote{
Details:
\\
\begin{enumerate}
\item You \textbf{may} flip a token. As in, you are not forced to flip a token; you may choose to not flip a token.

\item You can't just tell your friend what square it is. Or point to it. Or text him it. Or... you get the point.

\item Your friend knows the strategy as well (you tell him beforehand).

\item If you don't get it right, the Devil takes your soul. High stakes.
\end{enumerate}}

\vspace{10 mm}
\begin{problem}
Show that there is a winning strategy such that your friend can always know what square the magic square is.
\end{problem}

\begin{marginfigure}[0.0in]
\includegraphics[scale=0.5]{Richard_Hamming.jpg}
\end{marginfigure}

There might be solutions that exploit the chessboard geometry with its black and white fields. We will ignore the chessboard angle though and use this problem as an excuse to dive into the topic of linear codes. We will solve the problem by treating the token information as a 64-bit word and we will devise a winning strategy that involves a Hamming\footnote{Richard Hamming was one of the founders of modern coding theory. \url{http://en.wikipedia.org/wiki/Richard_Hamming}} code (a type of perfect linear code).

But first lets introduce linear codes\index{linear code}. We operate in the field $\mathbb{F}_q$ of integers modulo a prime $q$.

\begin{defn}\label{linearCodeDef}
A \textbf{linear code} $C$ of words of length $n$ is a subspace of the vector space $\mathbb{F}^n_q$. Let $\dim C = k$, then we say that $C$ is a $[n, k]_q$ linear code.
\end{defn}

Given a basis $\{\bm{c_1}, \bm{c_2}, \ldots, \bm{c_k}\}$ of $C$, we can build a matrix $G \in \mathbb{F}_q^{k \times n}$ using the $\bm{c_i}$ basis vectors as rows. Then $C$ is the row space of $G$ and $G$ is called a \textbf{generator matrix}\index{generator matrix} of $C$. We have\footnote{We treat vectors as row vectors in this section. That means that $\bm{x} \in \mathbb{F}_q^k$ is a matrix $\mathbb{F}_q^{1 \times k}$.}

$$
C = \{\bm{x} G: \bm{x} \in \mathbb{F}_q^k\},
$$

\noindent so a code $C$ is made from all linear combinations of the row vectors of its generator matrix.

Let $G'$ be the row reduced echelon form of $G$. By definition $G$ has full row rank, so $G'$ has only nonzero rows. If
$
G' = \left[\begin{smallmatrix}
I_k & | & A_{k \times (n - k)}
\end{smallmatrix}\right]
$
for identity matrix $I_k$ and some matrix $A$ then the generator matrix $G'$ is in \textbf{standard form}\footnote{Not every generator matrix can be row reduced to the standard form. For example
$$
\begin{bmatrix}
1 & 1 & 0 & 0 \\
0 & 0 & 1 & 1
\end{bmatrix}
$$
cannot.}. Row operations preserve the row space, so $G'$ also generates $C$.

\begin{defn}\label{parityCheckDef}
Given a $[n, k]_q$ linear code $C$, matrix $H \in \mathbb{F}_q^{(n-k) \times n}$ is a \textbf{parity check matrix}\index{parity check matrix} for $C$, if $C = nullspace(H)=\{\bm{c} \in \mathbb{F}_q^n: H \bm{c}^T = 0\}$.
\end{defn}

\begin{thm}\label{generatorParityCheckRel}
Given a $[n, k]_q$ linear code $C$ and a generator matrix $
G = \left[\begin{smallmatrix}
I_k & | & A_{k \times (n - k)}
\end{smallmatrix}\right]
$ for $C$ in standard form, then $
H = \left[\begin{smallmatrix}
-A^T_{(n - k) \times k} & | & I_{n - k}
\end{smallmatrix}\right]
$ is a parity check matrix\footnote{With $G$ in standard form this theorem let's us construct a parity check matrix very easily. Also worth noting that in standard form we generate a code word from a message $\bm{x} \in \mathbb{F}_q^k$ by appending $n - k$ parity check bits to the message with $\bm{x} G$. We check if the transmitted and received word $\bm{y} \in \mathbb{F}_q^n$ is a valid code word by verifying $H \bm{y}^T = 0$. If true then the first k positions of $\bm{y}$ are the original message $\bm{x}$.} for $C$.
\end{thm}

\begin{proof}

Let $\bm{c} \in C$ be a code word from $C$. Then there exists an $\bm{x} \in \mathbb{F}_q^k$ such that $\bm{c} = \bm{x} G$.
We have

\begin{align*}
 H \bm{c}^T &= H (\bm{x} G)^T \\
        &= \left[ -A^T_{(n - k) \times k} | I_{n - k} \right] \left(\bm{x} \left[ I_k | A_{k \times (n - k)} \right]\right)^T \\
        &=  \left[ -A^T_{(n - k) \times k} | I_{n - k} \right] \left[ \begin{array}{c}
                                                                           I_k \\ \hline
                                                                           A^T_{(n - k) \times k}
                                                                       \end{array}  \right] \bm{x}^T \\
        &= (-A^T + A^T) \bm{x}^T \\
        &= 0
\end{align*}

This means that $C \subseteq nullspace(H)$. We have $\dim C = k$ and
$$
\dim nullspace(H) = n - rank(H) = n - n + k = k,
$$
so $C = nullspace(H)$ and $H$ is a parity check matrix for $C$.

\end{proof}

What can we do if the generator matrix is not in standard form? Swapping columns in the generator matrix does not preserve the row space, so the linear code generated with the modified matrix is clearly not the same as the original code, but it is an equivalent code\footnote{A $[n, k_1]_q$ linear code $C_1$ is equivalent to a $[n, k_2]_q$ linear code $C_2$ if there is a permutation $\pi \in S_n$ such that when $\pi$ is applied to the coordinate indices of all the code words from $C_1$, it produces all the code words from $C_2$. Equivalent linear codes have the same dimension $k_1=k_2$.}.

\begin{defn}\label{hammingDistDef}
The \textbf{Hamming distance}\index{Hamming distance} $d(\bm{x}, \bm{y})$ between two vectors $\bm{x}, \bm{y} \in \mathbb{F}_q^n$ is the number of positions in which the vectors differ. With $\bm{x} = x_1 x_2 \ldots x_n$ and $\bm{y} = y_1 y_2 \ldots y_n$ we have
$$
d(\bm{x}, \bm{y}) = |\{i : 1 \leq i \leq n : x_i \neq y_i\}|
$$
The \textbf{Hamming weight}\index{Hamming weight} $w(\bm{x})$ is the number of positions that differ from zero:
$$
w(\bm{x}) = |\{i : 1 \leq i \leq n : x_i \neq 0\}| = d(\bm{x}, \bm{0})
$$
\end{defn}

We will use the following properties of Hamming distances:

\marginnote{\textit{Proof} of Lemma \ref{hammingtriangle} \\
The first three properties are obvious from the definition of Hamming distance. For the last property let $i$ be an index where $\bm{x}$ and $\bm{y}$ differ, so $x_i \neq y_i$. For vector $\bm{z}$ we can have the following cases for position $i$:
\begin{align*}
& z_i = x_i \Rightarrow z_i \neq y_i \\
& z_i = y_i \Rightarrow z_i \neq x_i \\
& z_i \neq x_i \land z_i \neq y_i
\end{align*}
In each of these cases the contribution of $z_i$ to $d(\bm{x}, \bm{z}) + d(\bm{z}, \bm{y})$ is at least one, whereas on the left side position $i$ contributes one to $d(\bm{x}, \bm{y})$. A similar analysis holds for indices $i$ where $x_i=y_i$.
\qed}

\begin{lem}\label{hammingtriangle}
\begin{align*}
&\forall \bm{x}, \bm{y} \in \mathbb{F}_q^n : d(\bm{x}, \bm{x}) \geq 0 \\
&\forall \bm{x}, \bm{y} \in \mathbb{F}_q^n : d(\bm{x}, \bm{y}) = 0 \Leftrightarrow x = y \\
&\forall \bm{x}, \bm{y} \in \mathbb{F}_q^n : d(\bm{x}, \bm{y}) = d(\bm{y}, \bm{x}) \\
&\forall \bm{x}, \bm{y}, \bm{z} \in \mathbb{F}_q^n : d(\bm{x}, \bm{y}) \leq d(\bm{x}, \bm{z}) + d(\bm{z}, \bm{y})
\end{align*}
\end{lem}

\begin{defn}\label{hammingDistCodeDef}
The \textbf{minimum distance} of $C$ is:
$$
d(C) = \min \{d(\bm{x}, \bm{x'}) : \bm{x}, \bm{x'} \in C \land \bm{x} \neq \bm{x'} \} = \min \{w(\bm{x}) : \bm{x} \in C \}
$$
\end{defn}

The minimum distance\index{minimum distance} is important enough that we add it to the characteristic notation of a linear code: $[n, k, d]_q$ is a linear  code over field $\mathbb{F}_q$ with bit strings of length $n$, code dimension $k$ and minimum distance between code words $d$.

The next lemma establishes a connection between the minimum distance of a linear code and one of its parity check matrix.

\begin{lem}\label{minCodeParityCheckRel}
The minimum distance of a code $C$ equals the minimum number of linearly dependent columns in one of its parity check matrices.
\end{lem}

\marginnote{\textit{Proof} of Lemma \ref{minCodeParityCheckRel} \\
Let $H$ be a parity check matrix of $[n, k, d]_q$ linear code $C$. There must be a code word $\bm{c}$ with $w(\bm{c}) = d$.
$\bm{c}$ belongs to the nullspace of $H$, so

$$
H \bm{c}^T = \bm{0}
$$

But $H \bm{c}^T$ is a linear combination of column vectors of $H$, with $d$  nonzero coefficients, so the column vectors in this linear combination are linearly dependent.
\qed}

So far we have worked with fields $\mathbb{F}_q$ of any prime $q$. Now we switch to the binary world $q=2$ and $\mathbb{F}_2$. Our vectors are bit strings. We transmit these bit strings over a binary symmetric channel.

\begin{defn}\label{symChanDef}
In a \textbf{binary symmetric channel}\index{binary symmetric channel} each bit sent has the same probability $p < \frac{1}{2}$ of being received incorrectly.
\end{defn}

We send a code word $\bm{x} \in C$ from a $[n, k]_2$ linear code $C$ over a binary symmetric channel and receive a bit string $\bm{y}$. If there were no transmission errors, then $y=x$. If there were errors, we want to find the most likely code word $\bm{x}$ that was transmitted given the errors in $\bm{y}$.

One decoding strategy\footnote{Finding an appropriate code word for the transmitted bit string is called \textit{decoding}. Finding the most likely code word is called \textit{maximum likelihood decoding}.} would be to choose a code word $\bm{x}$ with minimum Hamming distance over all code words from $C$ to received bit string $\bm{y}$. This type of decoding is called \textit{nearest neighbor decoding}. The chosen $\bm{x}$ is not always unique.

\begin{thm}\label{nnDmaximumLike}
In a binary symmetric channel with error probability $p < \frac{1}{2}$ the nearest neighbor decoding is a maximum likelihood decoding.
\end{thm}

\begin{proof}
Given a bit string $\bm{y} \in \mathbb{F}_2^n$ received through the channel, let $P_{\bm{y}}(\bm{x})$ be the probability that the code word $\bm{x}$ was sent when $\bm{y}$ was received. Because the channel is a binary symmetric channel, we have

$$
P_{\bm{y}}(\bm{x}) = p^{d(\bm{x}, \bm{y})} (1 - p)^{n - d(\bm{x}, \bm{y})}
$$

Consider two code words $\bm{x}$ and $\bm{x'}$ such that $d(\bm{x}, \bm{y}) \leq d(\bm{x'}, \bm{y})$. Because $p < \frac{1}{2}$, we then have $P_{\bm{y}}(\bm{x}) \geq P_{\bm{y}}(\bm{x'})$. It follows that
$$
\max_{\bm{x} \in C} P_{\bm{y}}(\bm{x}) = \min_{\bm{x} \in C} d(\bm{x}, \bm{y})
$$
so the likeliest code word is the nearest neighbor to $\bm{y}$.
\end{proof}

For the rest of this section we use nearest neighbor decoding. We want to know if we can detect and possibly correct a transmission with errors. Let's define clearly what we mean by that. A transmission is a pair $(\bm{x}, \bm{y}) \in C \times \mathbb{F}_2^n$, where a code word $\bm{x}$ was sent and a bit strings $\bm{y}$ was received. It has $d(\bm{x}, \bm{y})$ transmission errors. The nearest neighbor decoding $nnd(\bm{y})$ finds a code word (not necessarily unique) closest to $\bm{y}$. The following holds by definition:

$$
d(\bm{y}, nnd(\bm{y})) = \min_{\bm{c} \in C} d(\bm{y}, \bm{c})
$$

If no errors occurred in the transmission, then $\bm{x} = \bm{y}$ and also $d(\bm{x}, \bm{y}) = 0$ and $nnd(\bm{y}) = \bm{x}$. If errors in the transmission occurred we want to:

\begin{enumerate}[label=\textbf{E.\arabic*}]
\item \label{transmissionErrorRecogOne} detect that errors happened, i.e. establish that $\bm{y} \notin C$.
\item \label{transmissionErrorRecogTwo} correct the errors, i.e. establish $nnd(\bm{y}) = \bm{x}$.
\end{enumerate}

The next theorem describes the conditions for \ref{transmissionErrorRecogOne}.

\begin{thm}\label{linearCodeDetectErrors}
Given a $[n, k, d]_2$ linear binary code $C$, we can detect that any transmission with up to $e$ errors was erroneous if and only if $d > e$.
\end{thm}

\begin{proof}

\noindent$(\Rightarrow)$ Let $(\bm{x}, \bm{y})$ be a transmission with $d(\bm{x}, \bm{y}) \leq e < d$ errors. Assume $\bm{y} \in C$. Then $d(\bm{x}, \bm{y}) \leq e < d$ is a contradiction to $d$ being the minimal distance of $C$. It follows that $\bm{y} \notin C$.

\noindent$(\Leftarrow)$ We can detect that any transmission with up to $e$ errors was erroneous. Assume $d \leq e$. Then there exist two code words $\bm{x} \neq \bm{x'}$ such that $d(\bm{x}, \bm{x'}) \leq e$. Now consider transmission $(\bm{x}, \bm{x'})$. It's impossible to detect that it had errors because $\bm{x'}$ is a code word. This is a contradiction with the fact that we can detect that any transmission with up to $e$ errors was erroneous. So $d > e$.
\end{proof}

\begin{marginfigure}[0.5in]
\includegraphics[scale=0.3]{hammingSpheres.pdf}
\caption{A Hamming sphere for code word $\bm{c}$ with radius  $e$ is the set \\
$\{\bm{x}: d(\bm{x}, \bm{c}) \leq e\}$. In this figure the spheres don't overlap, so vectors (blue dots) that fall within a sphere can be error-corrected to code words (red dots).}
\label{fig:hammingSpheres}
\end{marginfigure}

For \ref{transmissionErrorRecogTwo} we have this theorem:

\begin{thm}\label{linearCodeCorrectErrors}
Given a $[n, k, d]_2$ linear binary code $C$, we can correct any transmission with up to $e$ errors if $d > 2 e$.
\end{thm}

\begin{proof}
Let $(\bm{x}, \bm{y})$ be a transmission with $d(\bm{x}, \bm{y}) \leq e$ errors and $d > 2 e$. Assume $nnd(\bm{y}) \neq \bm{x}$. Then $d(\bm{y}, nnd(\bm{y})) \leq e$ (otherwise $\bm{x}$ would be closer than $nnd(\bm{y})$ to $\bm{y}$). We have
$$
d(\bm{x}, nnd(\bm{y})) \leq d(\bm{x}, \bm{y}) + d(\bm{y}, nnd(\bm{y})) \leq e + e = 2 e
$$
which contradicts $d > 2 e$. So $nnd(\bm{y}) = \bm{x}$.

\end{proof}

Theorems \ref{linearCodeDetectErrors} and \ref{linearCodeCorrectErrors} tell us that a large minimum distance $d(C)$ allows us to detect and correct more errors. But a large minimum distance between code words also limits the number of code words. The following theorem puts an upper bound on the number of code words given a minimum distance.

\begin{thm}\label{hammingBound}
Given a $[n, k, 2 t + 1]_2$ linear binary code $C$, we have
$$
|C| \leq \frac{2^n}{\sum_{i = 0}^t \binom{n}{i}}
$$
This upper bound is called \textbf{Hamming bound}\index{Hamming bound}.
\end{thm}

\begin{proof}

Given a bit string $\bm{x}$ and an integer $i \leq n$, there are $\binom{n}{i}$ ways to choose the $i$ positions at which $\bm{x}$ and and another bit string $\bm{y}$ differ. So there are $\binom{n}{i}$ bit strings $\bm{y}$ with $d(\bm{x}, \bm{y}) = i$. This means there are
$$
\sum_{i = 0}^t \binom{n}{i}
$$
bit strings $\bm{y}$ with $d(\bm{x}, \bm{y}) \leq t$.

On the other hand, a bit string $\bm{y}$ with $d(\bm{y}, \bm{x}) \leq t$ to a code word $\bm{x}$ cannot have the same
$d(\bm{y}, \bm{x'}) \leq t$ to a different code word $\bm{x'}$ because then

$$
d(\bm{x}, \bm{x'}) \leq d(\bm{x}, \bm{y}) + d(\bm{y}, \bm{x'}) \leq t + t \leq 2 t
$$

which is a contradiction to $d(C) = 2 t + 1$.

So for each code word, we have at most $\sum_{i = 0}^t \binom{n}{i}$ bit strings with Hamming distance $\leq t$ and we cannot have the same bit strings near two different code words. We have $2^n$ bit strings, so

$$
|C| \sum_{i = 0}^t \binom{n}{i} \leq 2^n
$$

\end{proof}

A binary linear code that achieves equality in the Hamming bound \ref{hammingBound} is called a \textbf{perfect code}\index{perfect code}.

We are now ready to define Hamming codes\index{Hamming code}.

\begin{defn}\label{hammingCode}
A Hamming code $\mathfrak{H}_r$ of order $r$ (where $r$ is a positive integer) is a binary linear code with the parity check matrix
with columns that are all the $2^r - 1$ nonzero bit strings of length $r$.
\end{defn}

Changing the order of the columns in the parity check matrix produces equivalent codes with the same minimum distance. So for easier analysis we now consider Hamming codes with parity check matrix in standard form, ie the last $r$ columns form the identity matrix $I_r$, so $
H = \left[\begin{smallmatrix}
A_{r \times (n - r)} & | & I_{r}
\end{smallmatrix}\right]
$, with $n = 2^r - 1$. From theorem \ref{generatorParityCheckRel} we then know the generator matrix is $
G = \left[\begin{smallmatrix}
I_{n - r} & | & -A_{(n - r) \times r}^T
\end{smallmatrix}\right] = \left[\begin{smallmatrix}
I_{n - r} & | & A_{(n - r) \times r}^T
\end{smallmatrix}\right]
$, since we operate in $\mathbb{F}_2$. We can see that $\dim \mathfrak{H}_r = n - r$. What is the minimum distance of $\mathfrak{H}_r$? All columns are nonzero and distinct, so no two columns are linearly dependent\footnote{Again, this is in $\mathbb{F}_2$. The sum of two distinct columns is always nonzero, so a linear combination that is zero has to have coefficients zero, hence linearly independent.}. But consider the linear combination of the three columns
$$
[1, 1, 0, \ldots, 0]^T + [1, 0, 0, \ldots, 0]^T + [0, 1, 0, \ldots, 0]^T = \bm{0}^T
$$
They are linearly dependent. From lemma \ref{minCodeParityCheckRel} it follows that $d(\mathfrak{H}_r) = 3$, so $\mathfrak{H}_r$ is a $[2^r - 1, 2^r - 1 - r, 3]_2$ binary linear code. According to theorem \ref{linearCodeCorrectErrors} it can correct transmissions with one error.

\begin{thm}\label{hammingPerfect}
$\mathfrak{H}_r$ is a perfect code.
\end{thm}

\begin{proof}

The generator matrix has full row rank, so we need all linear combinations of the rows to get all the code words. This are binary words, so there are $2^{n - r}$ distinct linear combinations. It means $|\mathfrak{H}_r| = 2^{n - r}$.

Inserting into formula of theorem \ref{hammingBound}, we get\footnote{With $t = 1$, because $d(\mathfrak{H}_r) = 3$.}

$$
2^{n - r} \sum_{i = 0}^1 \binom{n}{i} = 2^{n - r} (1 + n) = 2^{n - r} (1 + 2^r - 1) = 2^{n - r} 2^r = 2^n
$$

\end{proof}

This concludes our dive into linear codes and Hamming codes. Let's return to our problem and solve it using Hamming codes. The state of the chessboard is a binary word of length $64$. We use $r = 6$, so Hamming code $\mathfrak{H}_6$. The word length is $2^6 - 1 = 63$. We agree that the devil choosing bit $64$ is a special case which we handle later. For now imagine the chessboard as a $63$-bit binary word and the devil only choosing a magic field between $1$ and $63$.

The winning strategy can be summarized as follows: the first player needs to modify the $63$-bit word (by flipping at most one bit) in such a way that the magic field is the one bit error of a code word in $\mathfrak{H}_6$. Then the second player only has to come in, decode the modified chessboard and point to the corrected error which is the same magic field\footnote{Decoding\label{decodingSideNote} is done as follows: $\bm{x}$ needs to be decoded. It is one bit away from a code word $\bm{c}$ with error at bit $k$. Let $\bm{e_k}$ be the unit vector with bit $k$ set. So $\bm{x} = \bm{c} + \bm{e_k}$ and
$$
H \bm{x} = H (\bm{c} + \bm{e_k}) = H \bm{e_k}
$$
Since $\bm{e_k}$ is be the unit vector with bit $k$ set, $H \bm{e_k}$ is column $k$ from the parity check matrix $H$. To decode we calculate $H \bm{x}$ and look to see which column in $H$ the result is. To save the lookup step we can be even more elegant. Instead of the parity check matrix in standard form, we choose a parity check matrix where column $k$ is the bit representation of $k$. Instead of lookup we just reverse the bit representation back to the integer $k$.
}.

Is this always possible? We know that Hamming codes are perfect codes, so any $63$-bit word is at most one bit away from a code word. We have the following cases for the initial state of the chessboard:

\begin{itemize}
\item  It happens to be a code word in $\mathfrak{H}_6$. Then the first player flips the magic field bit, producing an error there.

\item It happens to be a $63$-bit word that is a one bit error at the magic field. The first player doesn't flip any bit in this case.

\item It happens to be a $63$-bit word with a one bit error different from the magic field.
\end{itemize}

The last case needs a little thinking. Assume $H$ is the parity check matrix for our $\mathfrak{H}_6$ Hamming code and assume the state of the chessboard is $\bm{x}$, which is one bit error from a code word $\bm{c_1}$. Also let $1 \leq m \leq 63$ be the magic field bit and $\bm{e_m}$ the unit vector with bit $m$ set. The one bit error is different from the magic field, so $\bm{x} - \bm{c_1} \neq \bm{e_m}$. Let $\bm{y} = \bm{x} - \bm{e_m}$, which is also one bit away from a code word $\bm{c_2}$, with error bit $k$. So
$\bm{y} = \bm{c_2} + \bm{e_k}$.

Now consider $\bm{x} - \bm{e_k}$:
$$
H (\bm{x} - \bm{e_k}) = H (\bm{y} + \bm{e_m} - \bm{e_k}) = H (\bm{y} - \bm{e_k}) + H \bm{e_m} = H \bm{c_2} + H \bm{e_m} = H \bm{e_m}
$$

So $\bm{x} - \bm{e_k}$ has one bit error at the magic field, which is what we want. Flipping bit $k$ on the initial chessboard $\bm{x}$ achieves that.

In all three cases the modified chessboard is one bit away from a code word with the error at the magic field and the chessboard was modified by flipping at most one bit. The players agree that if the chessboard is a code word instead, then the devil chose bit $64$ as the magic field, which handles the special case. Modifying the chessboard to get a code word can also be done by flipping at most one bit. This scales to any chessboard with size a power of two.

\begin{fullwidth}

What follows is a Mathematica session illustrating the strategy. We use a parity check matrix with column $k$ the bit representation of integer $k$. This simplifies decoding as remarked in the side note \ref{decodingSideNote} above.

The function \textit{hamming} generates the parity check matrix for a Hamming code with a given $r$.

\begin{mmaCell}[morefunctionlocal={i},morepattern={r, r_Integer}]{Code}
  hamming[r_Integer] := Transpose[Table[IntegerDigits[i, 2, r], {i, 1, 2^r - 1}]]
\end{mmaCell}

For example

\begin{mmaCell}[moredefined={hamming}]{Input}
  hamming[4] // MatrixForm
\end{mmaCell}

\begin{mmaCell}{Output}
\\
\end{mmaCell}
$$
\left(
\begin{array}{ccccccccccccccc}
 0 & 0 & 0 & 0 & 0 & 0 & 0 & 1 & 1 & 1 & 1 & 1 & 1 & 1 & 1 \\
 0 & 0 & 0 & 1 & 1 & 1 & 1 & 0 & 0 & 0 & 0 & 1 & 1 & 1 & 1 \\
 0 & 1 & 1 & 0 & 0 & 1 & 1 & 0 & 0 & 1 & 1 & 0 & 0 & 1 & 1 \\
 1 & 0 & 1 & 0 & 1 & 0 & 1 & 0 & 1 & 0 & 1 & 0 & 1 & 0 & 1 \\
\end{array}
\right)
$$

We define $r$ and the corresponding Hamming code $h$ for our chessboard

\begin{mmaCell}{Input}
  r = 6;
\end{mmaCell}

\begin{mmaCell}[moredefined={hamming}]{Input}
  h = hamming[6];
\end{mmaCell}

The function \textit{pos} returns the unit vector with the error bit set from decoding the specified word.

\begin{mmaCell}[morelocal={s},morepattern={w, w_}]{Input}
  pos[w_] := With[\{s = Mod[h.w, 2]\}, UnitVector[2^Length[s] - 1, FromDigits[s, 2]]]
\end{mmaCell}

Function \textit{friendOne} implements the strategy part for the first friend. Given the initial state of the chessboard \textit{cb} and a magic field \textit{mf}, it returns a modified chessboard.

\begin{mmaCell}[morepattern={cb, cb_List, mf, mf_Integer},morelocal={em, ey, y}]{Input}
  friendOne[cb_List, mf_Integer] := Module[\{em, y, ey\},
  em = UnitVector[2^r - 1, mf]; y = Mod[cb - em, 2]; ey = pos[y];
  z = Mod[cb - ey, 2]
  ]
\end{mmaCell}

Function \textit{friendTwo} implements the strategy part for the second friend: decoding the specified chessboard and returning the index of the error bit which is also the magic field.

\begin{mmaCell}[morepattern={cb, cb_List}]{Input}
  friendTwo[cb_List] := Position[pos[cb], 1][[1,1]]
\end{mmaCell}

This next function is returning random initial states for the chessboard.

\begin{mmaCell}{Input}
  rw := RandomInteger[1, \{2^r - 1\}]
\end{mmaCell}

We can now simulate one game with the devil.

\textit{cb} is the initial (random) state of the chessboard.

\begin{mmaCell}{Input}
  cb = rw;
\end{mmaCell}

The magic field is some integer, the devil chose $23$.

\begin{mmaCell}{Input}
  mf = 23;
\end{mmaCell}

The first friend enters the room, modifies the chessboard according to \textit{friendOne}. The returned value is the modified chessboard.

\begin{mmaCell}{Input}
  cb2 = friendOne[cb, mf];
\end{mmaCell}

Let's check that the Hamming distance between initial and modified chessboard is at most one.
\begin{mmaCell}{Input}
  HammingDistance[cb, cb2]
\end{mmaCell}

\begin{mmaCell}{Output}
  1
\end{mmaCell}

The second friend comes in and decodes with \textit{friendTwo}, getting $23$.

\begin{mmaCell}{Input}
  friendTwo[cb2]
\end{mmaCell}

\begin{mmaCell}{Output}
  23
\end{mmaCell}

\end{fullwidth}

\bibliographystyle{plainnat}
\bibliography{../common/math}

\end{document}


\chapter{Linear Regression}

\section{Introduction}
Linear regression is the “work horse” of statistics and (supervised) machine learning. When augmented with kernels or other forms of basis function expansion, it can model also nonlinear relationships. And when the Gaussian output is replaced with a Bernoulli or multinoulli distribution, it can be used for classification, as we will see below. So it pays to study this model in detail.

\section{Representation}

\begin{equation}
p(y|\vec{x},\vec{\theta})=\mathcal{N}(y|\vec{w}^T\vec{x}, \sigma^2)
\end{equation}
where $\vec{w}$ and $\vec{x}$ are extended vectors, $\vec{x}=(1,x)$, $\vec{w}=(b,w)$.

Linear regression can be made to model non-linear relationships by replacing $\vec{x}$ with some non-linear function of the inputs, $\phi(\vec{x})$ \begin{equation}
p(y|\vec{x},\vec{\theta})=\mathcal{N}(y|\vec{w}^T\phi(\vec{x}), \sigma^2)
\end{equation}

This is known as \textbf{basis function expansion}. (Note that the model is still linear in the parameters $\vec{w}$, so it is still called linear regression; the importance of this will become clear below.) A simple example are polynomial basis functions, where the model has the form
\begin{equation}
\phi(x)=(1, x, \cdots, x^d)
\end{equation}

\section{MLE}
Instead of maximizing the log-likelihood, we can equivalently minimize the \textbf{negative log likelihood} or \textbf{NLL}:
\begin{equation}
\text{NLL}(\vec{\theta}) \triangleq -\ell(\vec{\theta})=-\log(\mathcal{D}|\vec{\theta})
\end{equation}

The NLL formulation is sometimes more convenient, since many optimization software packages are designed to find the minima of functions, rather than maxima.

Now let us apply the method of MLE to the linear regression setting. Inserting the definition of the Gaussian into the above, we find that the log likelihood is given by
\begin{align}
\ell(\vec{\theta})& =\sum\limits_{i=1}^N \log\left[\dfrac{1}{\sqrt{2\pi}\sigma}\exp\left(-\dfrac{1}{2\sigma^2}(y_i-\vec{w}^T\vec{x}_i)^2\right)\right] \\
     & =-\dfrac{1}{2\sigma^2}\text{RSS}(\vec{w})-\dfrac{N}{2}\log(2\pi\sigma^2)
\end{align}

RSS stands for \textbf{residual sum of squares} and is defined by
\begin{equation}
\text{RSS}(\vec{w}) \triangleq \sum\limits_{i=1}^N (y_i-\vec{w}^T\vec{x}_i)^2
\end{equation}

We see that the MLE for $\vec{w}$ is the one that minimizes the RSS, so this method is known as \textbf{least squares}.

Let's drop constants wrt $\vec{w}$ and NLL can be written as
\begin{equation}
\text{NLL}(\vec{w}) = \dfrac{1}{2}\sum\limits_{i=1}^N (y_i-\vec{w}^T\vec{x}_i)^2
\end{equation}

There two ways to minimize NLL$(\vec{w})$.

\subsection{OLS}
Define $\vec{y}=(y_1,y_2,\cdots,y_N)$, $\vec{X}=\left(\begin{array}{c}\vec{x}_1^T \\ \vec{x}_2^T \\ \vdots \\ \vec{x}_N^T\end{array}\right)$, then NLL$(\vec{w})$ can be written as
\begin{equation}
\text{NLL}(\vec{w})=\dfrac{1}{2}(\vec{y}-\vec{X}\vec{w})^T(\vec{y}-\vec{X}\vec{w})
\end{equation}

When $\mathcal{D}$ is small(for example, $N < 1000$), we can use the following equation to compute \vec{w} directly
\begin{equation}
\hat{\vec{w}}_{\mathrm{OLS}}=(\vec{X}^T\vec{X})^{-1}\vec{X}^T\vec{y}
\end{equation}

The corresponding solution $\hat{\vec{w}}_{\mathrm{OLS}}$ to this linear system of equations is called the \textbf{ordinary least squares} or \textbf{OLS} solution.

\begin{proof}
We now state without proof some facts of matrix derivatives (we won’t need all of these at this section).
\begin{eqnarray}
trA &\triangleq& \sum\limits_{i=1}^n A_{ii} \nonumber \\
\frac{\partial}{\partial A}AB &=& B^T \\
\frac{\partial}{\partial A^T}f(A) &=& \left[\frac{\partial}{\partial A}f(A)\right]^T \label{eqn:matrix-1} \\
\frac{\partial}{\partial A}ABA^TC &=& CAB+C^TAB^T \label{eqn:matrix-2} \\
\frac{\partial}{\partial A}|A| &=& |A|(A^{-1})^T
\end{eqnarray}

Then,
\begin{eqnarray*}
\text{NLL}(\vec{w}) &=& \frac{1}{2N}(\vec{X}\vec{w}-\vec{y})^T(\vec{X}\vec{w}-\vec{y}) \\
\frac{\partial \text{NLL}}{\vec{w}} &=& \frac{1}{2} \frac{\partial}{\vec{w}} (\vec{w}^T\vec{X}^T\vec{X}\vec{w}-\vec{w}^T\vec{X}^T\vec{y}-\vec{y}^T\vec{X}\vec{w}+\vec{y}^T\vec{y}) \\
                           &=& \frac{1}{2} \frac{\partial}{\vec{w}} (\vec{w}^T\vec{X}^T\vec{X}\vec{w}-\vec{w}^T\vec{X}^T\vec{y}-\vec{y}^T\vec{X}\vec{w}) \\
						   &=& \frac{1}{2} \frac{\partial}{\vec{w}} tr(\vec{w}^T\vec{X}^T\vec{X}\vec{w}-\vec{w}^T\vec{X}^T\vec{y}-\vec{y}^T\vec{X}\vec{w}) \\
						   &=& \frac{1}{2} \frac{\partial}{\vec{w}} (tr\vec{w}^T\vec{X}^T\vec{X}\vec{w}-2tr\vec{y}^T\vec{X}\vec{w})
\end{eqnarray*}

Combining Equations \ref{eqn:matrix-1} and \ref{eqn:matrix-2}, we find that
\begin{equation*}
\frac{\partial}{\partial A^T}ABA^TC = B^TA^TC^T+BA^TC
\end{equation*}

Let $A^T=\vec{w}, B=B^T=\vec{X}^T\vec{X}$, and $C=I$, Hence,
\begin{eqnarray}
\frac{\partial \text{NLL}}{\vec{w}} &=& \frac{1}{2} (\vec{X}^T\vec{X}\vec{w}+\vec{X}^T\vec{X}\vec{w} -2\vec{X}^T\vec{y}) \nonumber \\
						   &=& \frac{1}{2} (\vec{X}^T\vec{X}\vec{w} - \vec{X}^T\vec{y}) \nonumber \\
\frac{\partial \text{NLL}}{\vec{w}} &=& 0 \Rightarrow \vec{X}^T\vec{X}\vec{w} - \vec{X}^T\vec{y} =0 \nonumber \\
\vec{X}^T\vec{X}\vec{w} &=& \vec{X}^T\vec{y} \label{eqn:normal-equation} \\
\hat{\vec{w}}_{\mathrm{OLS}} &=& (\vec{X}^T\vec{X})^{-1}\vec{X}^T\vec{y} \nonumber
\end{eqnarray}
\end{proof}

Equation \ref{eqn:normal-equation} is known as the \textbf{normal equation}.

\subsubsection{Geometric interpretation}

See Figure \ref{fig:graphical-interpretation-of-OLS}.
\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.50]{graphical-interpretation-of-OLS.png}
\caption{Graphical interpretation of least squares for $N=3$ examples and $D=2$ features. $\tilde{\vec{x}}_1$ and $\tilde{\vec{x}}_2$˜ are vectors in $\mathbb{R}^3$; together they define a 2D plane. $\vec{y}$ is also a vector in $\mathbb{R}^3$ but does not lie on this 2D plane. The orthogonal projection of $\vec{y}$ onto this plane is denoted $\hat{\vec{y}}$. The red line from $\vec{y}$ to $\hat{\vec{y}}$ is the residual, whose norm we want to minimize. For visual clarity, all vectors have been converted to unit norm.}
\label{fig:graphical-interpretation-of-OLS}
\end{figure}

To minimize the norm of the residual, $\vec{y}-\hat{\vec{y}}$, we want the residual vector to be orthogonal to every column of $\vec{X}$,so˜ $\tilde{\vec{x}}_j(\vec{y}-\hat{\vec{y}})=0$ for $j=1:D$. Hence
\begin{equation}\begin{split}
\tilde{\vec{x}}_j(\vec{y}-\hat{\vec{y}})=0 & \Rightarrow \vec{X}^T(\vec{y}-\vec{X}\vec{w})=0 \\
                                           & \Rightarrow \vec{w}=(\vec{X}^T\vec{X})^{-1}\vec{X}^T\vec{y}
\end{split}\end{equation}

\subsection{SGD}
When $\mathcal{D}$ is large, use stochastic gradient descent(SGD).

\begin{align}
\because \dfrac{\partial}{\partial w_i}\text{NLL}(\vec{w})=& \sum\limits_{i=1}^N (\vec{w}^T\vec{x}_i-y_i)x_{ij} \\
\therefore w_j=& w_j - \alpha\dfrac{\partial}{\partial w_j}\text{NLL}(\vec{w}) \nonumber \\
                  =& w_j - \sum\limits_{i=1}^N \alpha(\vec{w}^T\vec{x}_i-y_i)x_{ij} \\
\therefore \vec{w}=& \vec{w}-\alpha(\vec{w}^T\vec{x}_i-y_i)\vec{x}
\end{align}

\section{Ridge regression(MAP)}
One problem with ML estimation is that it can result in overfitting. In this section, we discuss a way to ameliorate this problem by using MAP estimation with a Gaussian prior.

\subsection{Basic idea}
We can encourage the parameters to be small, thus resulting in a smoother curve, by using a zero-mean Gaussian prior:
\begin{equation}
p(\vec{w})=\prod\limits_j \mathcal{N}(w_j|0,\tau^2)
\end{equation}
where $1/\tau^2$ controls the strength of the prior. The corresponding MAP estimation problem becomes
\begin{equation}
\arg\max_{\vec{w}} \sum\limits_{i=1}^N \log{\mathcal{N}(y_i|w_0+\vec{w}^T\vec{x}_i,\sigma^2)}+\sum\limits_{j=1}^D \log{\mathcal{N}(w_j|0,\tau^2)}
\end{equation}

It is a simple exercise to show that this is equivalent to minimizing the following
\begin{equation}\label{eqn:Ridge-regression-J}
J(\vec{w})=\dfrac{1}{N}\sum\limits_{i=1}^N (y_i-(w_0+\vec{w}^T\vec{x}_i))^2+\lambda\lVert\vec{w}\rVert^2 , \lambda \triangleq \dfrac{\sigma^2}{\tau^2}
\end{equation}

Here the first term is the MSE/ NLL as usual, and the second term, $\lambda \geq 0$, is a complexity penalty. The corresponding solution is given by
\begin{equation}\label{eqn:Ridge-regression-solution}
\hat{\vec{w}}_{\mathrm{ridge}}=(\lambda\vec{I}_D+\vec{X}^T\vec{X})^{-1}\vec{X}^T\vec{y}
\end{equation}

This technique is known as \textbf{ridge regression},or \textbf{penalized least squares}. In general, adding a Gaussian prior to the parameters of a model to encourage them to be small is called $\ell_2$ \textbf{regularization} or \textbf{weight decay}. Note that the offset term $w_0$ is not regularized, since this just affects the height of the function, not its complexity.

We will consider a variety of different priors in this book. Each of these corresponds to a different form of \textbf{regularization}. This technique is very widely used to prevent overfitting.

\subsection{Numerically stable computation *}

\begin{equation}\label{eqn:Ridge-regression-SVD}
\hat{\vec{w}}_{\mathrm{ridge}}=\vec{V}(\vec{Z}^T\vec{Z}+\lambda\vec{I}_N)^{-1}\vec{Z}^T\vec{y}
\end{equation}

\subsection{Connection with PCA *}

\subsection{Regularization effects of big data}
Regularization is the most common way to avoid overfitting. However, another effective approach — which is not always available — is to use lots of data. It should be intuitively obvious that the more training data we have, the better we will be able to learn.

In domains with lots of data, simple methods can work surprisingly well (Halevy et al. 2009). However, there are still reasons to study more sophisticated learning methods, because there will always be problems for which we have little data. For example, even in such a data-rich domain as web search, as soon as we want to start personalizing the results, the amount of data available for any given user starts to look small again (relative to the complexity of the problem).

\section{Bayesian linear regression}
TODO


\section{Question 1}
System is:
$$
G(s) = \frac{-s + 3}{(s+1)(s+2)(s^2+2s+4)}
$$


\section*{Exercises}

\begin{ex}
  Which of the following are quadratic forms?
  \begin{enumerate}
  \item $f_1(x,y,z) = (x+y+z)^2$.
  \item $f_2(x,y,z) = x^2 + y^2 + z^2$.
  \item $f_3(x,y,z) = (x+y)^2 + (x+z)^2 + (y+z)^2$.
  \item $f_4(x,y,z) = x^2 - y^2$.
  \item $f_5(x,y,z) = x^2 + 2xyz + (y+z)^2$.
  \item $f_6(x,y,z) = x(y+z)$.
  \item $f_7(x,y,z) = x^2y^2z^2$.
  \end{enumerate}
  \begin{sol}
    All of them except (e) and (g).
  \end{sol}
\end{ex}

\begin{ex}
  Find the coefficients of the quadratic form
  $f(x,y,z) = \vect{v}^T A\vect{v}$, where
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      0 &  1 &  2 \\
      1 & -3 & -4 \\
      2 & -4 &  5 \\
    \end{mymatrix}.
  \end{equation*}
\end{ex}

\begin{ex}
  Write the quadratic form
  \begin{equation*}
    f(x,y,z) = (x+y)^2 + 2(x-z)^2 + 3yz
  \end{equation*}
  in matrix form.
  \begin{sol}
    $f(x,y,z) = \vect{v}^T A\vect{v}$, where
    \begin{equation*}
      A = \begin{mymatrix}{ccc}
        3  &  1  & -2  \\
        1  &  1  & 1.5 \\
        -2 & 1.5 &  2  \\
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Apply the change of variables $x = u+2w$, $y = v$, $z = w-v$ to the
  quadratic form
  \begin{equation*}
    x^2 + 7y^2 + 5z^2 - 4xy - 4xz + 11yz.
  \end{equation*}
  \begin{sol}
    $3u^2 + v^2 + vw + w^2$.
  \end{sol}
\end{ex}

\begin{ex}
  Perform a change of variables so that the quadratic form
  $f(x,y) = 3x^2 - 2xy + 3y^2$ becomes diagonal.
  \begin{sol}
    $x=u+v$ and $y=u-v$ gives $f = 4u^2 + 8v^2$.
  \end{sol}
\end{ex}

\begin{ex}
  Diagonalize the quadratic form
  \begin{equation*}
    f(x,y,z) = 3x^2 + 3y^2 + 4z^2 + 2xz - 2yz.
  \end{equation*}
\end{ex}

\begin{ex}
  Find the principal axes of the following curves, and sketch them:
  \begin{enumerate}
  \item $x^2+\frac{1}{2}y^2=1$,
  \item $2x^2 + 4xy + 5y^2 = 1$.
  \item $3x^2 + 2xy + 3y^2 = 1$.
  \end{enumerate}
  \begin{sol}
    \begin{enumerate}
    \item Eigenvalues: $\eigenvar_1=1$, $\eigenvar_2=\frac{1}{2}$.
      Principal axes:
      $\vect{u}_1 = \begin{mymatrix}{r} 1 \\ 0 \end{mymatrix}$ and
      $\vect{u}_2 = \begin{mymatrix}{r} 0 \\ 1 \end{mymatrix}$.
      $\vect{u}_1$-intercept: $1$, $\vect{u}_2$-intercept: $\sqrt{2}$.
    \item Eigenvalues: $\eigenvar_1=1$, $\eigenvar_2=6$.
      Principal axes:
      $\vect{u}_1 = \frac{1}{\sqrt{5}}\begin{mymatrix}{r} 2 \\ -1 \end{mymatrix}$ and
      $\vect{u}_2 = \frac{1}{\sqrt{5}}\begin{mymatrix}{r} 1 \\ 2 \end{mymatrix}$.
      $\vect{u}_1$-intercept: $1$, $\vect{u}_2$-intercept: $\frac{1}{\sqrt{6}}$.
    \item Eigenvalues: $\eigenvar_1=2$, $\eigenvar_2=4$.
      Principal axes:
      $\vect{u}_1 = \frac{1}{\sqrt{2}}\begin{mymatrix}{r} 1 \\ -1 \end{mymatrix}$ and
      $\vect{u}_2 = \frac{1}{\sqrt{2}}\begin{mymatrix}{r} 1 \\ 1 \end{mymatrix}$.
      $\vect{u}_1$-intercept: $\frac{1}{\sqrt{2}}$, $\vect{u}_2$-intercept: $\frac{1}{2}$.
    \end{enumerate}
    \begin{equation*}
      \begin{tikzpicture}[scale=1.5]
        \begin{scope}[color=black]
          \draw[->] (-1.5,0) -- (1.5,0) node[right] {$x$};
          \draw[->] (0,-1.5) -- (0,1.5) node[below right] {$y$};
          \draw (-1,0) -- +(0,-.1) node[below] {$-1$};
          \draw (1,0) -- +(0,-.1) node[below] {$1$};
          \draw (0,-1) -- +(-.1,0) node[left] {$-1$};
          \draw (0,1) -- +(-.1,0) node[left] {$1$};
        \end{scope}
        \begin{scope}[color=blue,cm={(1,0,0,1,(0,0))}]
          \draw[thick,red] (0,0) circle [x radius={1}, y radius={sqrt(2)}];
          \draw[red] (1,0) +(0.15,0.2) node {$1$};
          \draw[red] (0,{sqrt(2)}) +(-0.4,0.15) node {$\sqrt{2}$};
          \draw[thick,blue,->] (0,0) -- node[below] {$\vect{u}_1$} (1,0);
          \draw[thick,blue,->] (0,0) -- node[left] {$\vect{u}_2$} (0,1);
        \end{scope}
      \end{tikzpicture}
      \begin{tikzpicture}[scale=1.5]
        \begin{scope}[color=black]
          \draw[->] (-1.5,0) -- (1.5,0) node[right] {$x$};
          \draw[->] (0,-1.5) -- (0,1.5) node[below right] {$y$};
          \draw (-1,0) -- +(0,-.1) node[below] {$-1$};
          \draw (1,0) -- +(0,-.1) node[below] {$1$};
          \draw (0,-1) -- +(-.1,0) node[left] {$-1$};
          \draw (0,1) -- +(-.1,0) node[left] {$1$};
        \end{scope}
        \begin{scope}[color=blue,cm={(2/sqrt(5),-1/sqrt(5),1/sqrt(5),2/sqrt(5),(0,0))}]
          \draw[thick,red] (0,0) circle [x radius={1}, y radius={1/sqrt(6)}];
          \draw[red] (1,0) +(0.1,-0.2) node {$1$};
          \draw[red] (0,{1/sqrt(6)}) +(0.2,0.2) node {$\frac{1}{\sqrt{6}}$};
          \draw[thick,blue,->] (0,0) -- (1,0) node[right] {$\vect{u}_1$};
          \draw[thick,blue,->] (0,0) -- (0,1) node[above] {$\vect{u}_2$};
        \end{scope}
      \end{tikzpicture}
      \begin{tikzpicture}[scale=1.5]
        \begin{scope}[color=black]
          \draw[->] (-1.5,0) -- (1.5,0) node[right] {$x$};
          \draw[->] (0,-1.5) -- (0,1.5) node[below right] {$y$};
          \draw (-1,0) -- +(0,-.1) node[below] {$-1$};
          \draw (1,0) -- +(0,-.1) node[below] {$1$};
          \draw (0,-1) -- +(-.1,0) node[left] {$-1$};
          \draw (0,1) -- +(-.1,0) node[left] {$1$};
        \end{scope}
        \begin{scope}[color=blue,cm={(1/sqrt(2),-1/sqrt(2),1/sqrt(2),1/sqrt(2),(0,0))}]
          \draw[thick,red] (0,0) circle [x radius={1/sqrt(2)}, y radius={1/2}];
          \draw[red] ({1/sqrt(2)},0) +(0.2,-0.2) node {$\frac{1}{\sqrt{2}}$};
          \draw[red] (0,{1/2}) +(-0.2,0.2) node {$\frac{1}{2}$};
          \draw[thick,blue,->] (0,0) -- (1,0) node[right] {$\vect{u}_1$};
          \draw[thick,blue,->] (0,0) -- (0,1) node[above] {$\vect{u}_2$};
        \end{scope}
      \end{tikzpicture}
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Find the principal axes of the ellipsoid
  $2x^2 + 2y^2 + 3z^2 + 2xz - 2yz$.
  \begin{sol}
    The principal axes are
    $\frac{1}{\sqrt{3}}\begin{mymatrix}{r} 1 \\ -1 \\ -1 \end{mymatrix}$,
    $\frac{1}{\sqrt{2}}\begin{mymatrix}{r} 1 \\ 1 \\ 0 \end{mymatrix}$,
    $\frac{1}{\sqrt{6}}\begin{mymatrix}{r} 1 \\ -1 \\ 2 \end{mymatrix}$.
  \end{sol}
\end{ex}


\section{Linearization \& Newton's Method}

\subsection{Linearization}
As we've seen, tangent lines intersect their function at most once: at the point of tangency.
However, we know that differentiable functions are locally linear, so we'd expect the tangent line to be a decent approximation of the function near the point of tangency.

\begin{definition}
	If $f$ is differentiable at $a$, then the approximating function
	\begin{equation*}
		L(x) = f^\prime(a)(x-a) + f(a)
	\end{equation*}
	is the linearization of $f$ at $a$.
\end{definition}

\begin{example}
	Find the linearization of $f(x) = \ln{(x+1)}$ at $x=0$.
	How accurate is this approximation at $x=0.1$?
\end{example}
\begin{answer}
	Following the definition,
	\begin{align*}
		f(0) = \ln{(0+1)} = 0 \\
		f^\prime(x) &= \frac{1}{x+1} \\
		f^\prime(0) &= \frac{1}{0+1} = 1 \\
		L(x) &= 1(x-0) + 0 = x.
	\end{align*}

	Calculating the error at $x=0.1$,
	\begin{align*}
		L(0.1) &= 0.1 \\
		f(0.1) &\approx .0953 \\
		\text{\% error} &= \frac{\abs{L(0.1)-f(0.1)}}{L(0.1)}100\text{\%} \approx 4.7\text{\%}.
	\end{align*}

	So, we can see the linear approximation is pretty good.
\end{answer}

We call the difference in $x$ between the point of tangency and the point we're trying to approximate the differential.
\begin{definition}
	Let $y=f(x)$ be a differentiable function.
	The differential $\d{x}$ is an independent variable.
	The differential $\d{y}$ is $\d{y} = f^\prime(x)\d{x}$.
\end{definition}

$\mathrm{d}y$ is the approximated change in $y$ expected by the linearization for some given change in $x$, $\d{x}$.

\begin{example}
	Find $\d{y}$ for $y=\frac{2x}{1+x^2}$, $x=-2$, and $\d{x} = 0.1$.
\end{example}
\begin{answer}
	\begin{align*}
		y^\prime &= \frac{-4x^2}{\left(1+x^2\right)^2} + \frac{2}{1+x^2} \\
		y^\prime(-2) &= -6/25 \\
		\d{y} &= (-6/25)(0.1) = -0.024.
	\end{align*}
\end{answer}

\subsection{Newton's Method}
We can use the fact that the tangent line approximates the function to find the zeroes of functions.
Starting with an initial guess $x_0$ for the $x$ value of the zero, we look at the the tangent line at $x_0$ and find where it intersects the $x$-axis.
\begin{align*}
	L_0(x) &= f^\prime(x_0)(x - x_0) + f(x_0) \\
	0 &= f^\prime(x_0)(x - x_0) + f(x_0) \\
	-f^\prime(x_0)(x - x_0) &= f(x_0) \\
	x - x_0 &= -\frac{f(x_0)}{f^\prime(x_0)} \\
	x &= x_0 - \frac{f(x_0)}{f^\prime(x_0)}.
\end{align*}
This $x$ value serves as our next guess for the zero.
We repeat this process until we find the zero or are satisfied with our error, that is, successive approximations match up to a predetermined number of decimal places.\footnote{For most well-behaved functions, Newton's Method can get within a small margin of error or a zero relatively quickly. There is also a generalized, sometimes faster version of Newton's Method that approximates the function with higher-order polynomials than just lines.}
This yields a recursive formula
\begin{equation*}
	x_{n+1} = x_n - \frac{f(x_n)}{f^\prime(x_n)}.
\end{equation*}

\begin{example}
	Find an approximation for the zero of $x^3-2x-5$ accurate to four decimal places.
\end{example}
\begin{answer}
	In this particular case, we have
	\begin{equation*}
		x_{n+1} = x_{n} - \frac{x^3_n - 2x_n - 5}{3x^2_n - 2}.
	\end{equation*}

	A quick sketch of the graph of this function suggests that $x_0=2$ is a reasonable initial guess.
	\begin{align*}
		x_1 &= x_0 - \frac{f(x_0)}{f^\prime(x_0)} = 2 - \frac{2^3 - 2(2) - 5}{3(2)^2 - 2} = 2.1 \\
		x_2 &= x_1 - \frac{f(x_1)}{f^\prime(x_1)} = 2.1 - \frac{2.1^3 - 2(2.1) - 5}{3(2.1)^2 - 2} \approx 2.0946 \\
		x_3 &= x_2 - \frac{f(x_2)}{f^\prime(x_2)} = 2.0946 - \frac{2.0946^3 - 2(2.0946) - 5}{3(2.0946)^2 - 2} \approx 2.0946.
	\end{align*}

	Since we have obtained 2.0946 twice consecutively, it must be accurate to four decimal places.
\end{answer}


\subsection{Poisson distribution}

\subsection{Definition}

We can use the Poisson distribution to model the number of indepedent events that occur in an a time period.

For a very short time period the chance of us observing an event is a Bernoulli trial.

\(P(1)=p\)

\(P(0)=1-p\)

\subsection{Chance of no observations}

Let's consider the chance of repeatedly getting \(0\): \(P(0;t)\).

We can see that: \(P(0;t+\delta t)=P(0;t)(1-p)\).

And therefore:

\(P(0;t+\delta t)-P(0;t)=-pP(0;t))\)

By setting \(p=\lambda \delta t\):

\(\dfrac{P(0;t+\delta t)-P(0;t)}{\delta t}=-\lambda P(0;t))\)

\(\dfrac{\delta P(0;t)}{\delta t}=-\lambda P(0;t)\)

\(P(0;t)=Ce^{-\lambda t}\)

If \(t=0\) then \(P(0;t)=0\) and so \(C=1\).

\(P(0;t)=e^{-\lambda t}\)

\subsection{Deriving the Poisson distribution}


\subsection{Endomorphisms form a vector space}

An endomorphism maps a vector space onto itself.

\( end (V)=\hom (V, V)\)

Need to show that endomorphism is a vector space

Essentially

\(v\in V\)

\(f\in F\)

\(av = f\)

\(bv = g\)

\((a\oplus b)v=f+g\)

\((a\oplus b)v=av + bv\)

so there is some operation we can do on two members of endo

linear in addition. That is, if we have two dual "things", we can define the addition of functions as the operation which results int he outputs being added.

what about linear in scalar? same approach.

Well we define

\(c\odot a)=cav\)

There is a unique endomorphism which results in two other endomorphisms being added together. define this as addition
\subsection{Dimension of endomorphisms}

\(\dim (end(V))=(\dim V )^2\)

\subsection{Basis of endomorphisms}


%%lecture 7

%% 20.9
\newpage
\section{Hilbert spaces}
\label{sec:hilbert_spaces}
\begin{beispiel}
	Consider $\mathbb{C}^n = \set[(x_1,x_2,\dots,x_n)]{x_i \in \mathbb{C}}$ and $x,y \in \mathbb{C}^n$ with
	$x= (x_1,\dots,x_n)$, $y = (y_1,\dots,y_n)$. Define the inner product of $x,y$ (scalar product)
	\[
		\skal{x}{y} = \sum^{n}_{i=1}x_i \bar{y}_i \in \mathbb{C}.
	\]
	We have a map
	\begin{align*}
		\mathbb{C}^n \times \mathbb{C}^n &\to \mathbb{C} \\
		(x,y) &\mapsto \skal{x}{y}.
	\end{align*}
	This mapping has properties:
	\begin{itemize}
		\item $x \neq 0$ folgt $\skal{x}{x} = \sum^{n}_{i=1}x_i \bar{x}_i = \sum^{n}_{i=1} \abs{x_i}^2 >0$
		\item $\skal{\lambda x}{y} = \lambda \skal{x}{y}$ for $x,y \in \mathbb{C}^n$, $\lambda \in \mathbb{C}$.
		\item $\skal{x}{y} = \sum^{n}_{i=1} x_i \bar{y}_i = \overline{\sum^{n}_{i=1}y_i \bar{x}_i}$ for $x,y \in \mathbb{C}^n$. \\
		In particular $\skal{x}{\lambda y} = \bar{\lambda} \skal{x}{y}$ for $\lambda \in \mathbb{C}$.
		\item $\skal{x+y}{z} = \skal{x}{z}+ \skal{y}{z}$ for $x,y,z \in \mathbb{C}^n$.
	\end{itemize}
\end{beispiel}
\begin{definition*}
	An inner product space $V$ is a complex vector space with an inner product which is a map
	\[
		\skal{.}{.}: V \times V \to \mathbb{C}.
	\]
	Satisfying
	\begin{itemize}
		\item $\skal{\lambda x}{y} = \lambda \skal{x}{y}$ for any $x,y \in V$, $\lambda \in \mathbb{C}$.
		\item $\skal{x+y}{z} = \skal{x}{z}+ \skal{y}{z}$ for any $x,y,z \in V$.
		\item $\skal{x}{y} = \overline{\skal{x}{y}}$ for any $x,y \in V$.
		\item $\skal{x}{x}>0$ for any $x \in V, x \neq 0$.
	\end{itemize}
\end{definition*}
Can we generalize $\mathbb{C}^n$? \\
\[
	\mathbb{C}^{\mathbb{N}} \set[(x_1,x_2, \dots)]{x_i \in \mathbb{C}}
\]
with
\[
	\skal{x}{y} = \sum^{\infty}_{i=1} x_i \bar{y}_i.
\]
This is not necessarily convergent.
\begin{beispiele}
	\begin{enumerate}[(1)]
		\item 	\[
		l^2 = \set[(x_1,x_2, \dots)]{ \sum^{\infty}_{i=1} \abs{x_i}^2 < \infty}.
	\]
	We have with Cauchy Schwarz
	\[
		\sum^{n}_{i=1} \abs{x_i \bar{y}_i} \leq \left( \sum^{n}_{i=1} \abs{x_i}^2 \right)^{\frac{1}{2}} \left( \sum^{n}_{i=1} \abs{y_i}^2 \right)^{\frac{1}{2}}
	\]
	if $x \in l^2$ and $y \in l^2$ we get
	\begin{align*}
		\sum^{n}_{i=1}\abs{x_i \bar{y}_i} \leq \left( \sum^{\infty}_{i=1} \abs{x_i}^2 \right)^{\frac{1}{2}} \left( \sum_{i=1}^{\infty} \abs{y_i}^2. \right)^{\frac{1}{2}} < \infty.
	\end{align*}
	It follows that $\sum_{i=1}^{\infty} x_i \bar{y}_i$ converges absolutely and hence it is convergent. The following
	\[
		\skal{x}{y} = \sum^{\infty}_{i=1} x_i \bar{y}_i
	\]
	is well-defined for vectors $x,y \in l^2$. Like for $\mathbb{C}^n$ one can easily check that $\skal{.}{.}$ satisfies the axioms for inner products. \\
	$(l^2, \skal{.}{.})$ is an inner product space.
	\item Consider $C([0,1])$ with the inner product
	\[
		\skal{f}{g} = \int_{0}^{1}f(t) \overline{g(t)} \,\mathrm{d}t \qquad \forall\, f,g \in C([0,1]).
	\]
	\begin{itemize}
		\item 	\[
			\skal{\lambda f}{g} = \int_{0}^{1}\lambda f(t) \overline{g(t)} \,\mathrm{d}t = \lambda \int_{0}^{1}f(t) \overline{g(t)} \,\mathrm{d}t = \lambda \skal{f}{g}.
		\]
		\item \[
			\skal{f}{f} = \int_{0}^{1}f(t) \overline{f(t)} \,\mathrm{d}t = \int_{0}^{1} \abs{f(t)}^2 \,\mathrm{d}t >0.
		\]
		\item $\dots$.
	\end{itemize}
	\end{enumerate}
\end{beispiele}
If we take $\mathbb{R}^3$ with the Eukledian norm on $\mathbb{R}^3$
\[
	\norm{(x_1,x_2,x_3)} = \sqrt{x_1^2 + x_2^2 + x_3^2} = \left( \sum_{i=1}^{3} \abs{x_i}^2 \right)^{\frac{1}{2}} = \skal{x}{x}^{\frac{1}{2}}.
\]
Let $V$ be an inner product space with $\skal{.}{.}$ as the inner product. Let for $x \in V$
\[
	\norm{x} := \skal{x}{x}^{\frac{1}{2}}.
\]
\begin{satz}
	The $x \mapsto  \norm{x}$ with $\norm{.}$ defined above is a norm.
\end{satz}
We are going to prove the norm axioms but first we need another theorem.
	\begin{theorem}[Cauchy-Schwarz inequalitiy]
		For any $x,y \in V$ (inner product space)
		\[
			\abs{\skal{x}{y}} \leq \skal{x}{x}^{\frac{1}{2}} \skal{y}{y}^{\frac{1}{2}}.
		\]
		The equality holds iff $x,y$ are linearly dependent.
	\end{theorem}
	\begin{beweis}
		Assume $x,y$ linearly dependent. We can assume that $x= \lambda y$ for some $\lambda \in \mathbb{C} $.
		\[
			\abs{\skal{x}{y}} = \abs{ \skal{\lambda y}{y}} = \abs{\lambda} \skal{y}{y}
		\]
		and
		\begin{align*}
					\skal{x}{x}^{\frac{1}{2}} \skal{y}{y}^{\frac{1}{2}} &= \skal{\lambda y}{\lambda y}^{\frac{1}{2}} \skal{y}{y}^{\frac{1}{2}} \\
					&= \abs{\lambda} \skal{y}{y}^{\frac{1}{2}} \skal{y}{y}^{\frac{1}{2}} \\
					&= \abs{\lambda} \skal{y}{y}.
		\end{align*}
		Hence \[
			\abs{\skal{x}{y}} = \skal{x}{x}^{\frac{1}{2}} \skal{y}{y}^{\frac{1}{2}}.
		\]
		Assume $x,y$ are linearly independent. Hence $x + \lambda y \neq 0$ for any $\lambda \in \mathbb{C}$. By an axiom for inner product we get
		\[
			0< \skal{x+ \lambda y}{x + \lambda y} = \skal{x}{x} + \lambda \skal{y}{x} + \bar{\lambda} \skal{x}{y} + \abs{\lambda}^2 \skal{y}{y}.
		\]
		Pick now
		\[
			\lambda = - \frac{\skal{x}{y}}{\skal{y}{y}}.
		\]
		(Note that $y \neq 0$ as $x,y$ linearly independent.)
		We have \begin{align*}
						0 &< \skal{x}{x} - \frac{\overset{= \abs{\skal{x}{y}}^2}{\overbrace{\skal{x}{y}\skal{y}{x}}}}{\skal{y}{y}} - \frac{\overset{= \abs{\skal{x}{y}}^2}{\overbrace{\overline{\skal{x}{y}}\skal{x}{y}}}}{\skal{y}{y}}+ \frac{\abs{\skal{x}{y}}^2}{\skal{y}{y}^2} \skal{y}{y} \\
						&= \skal{x}{x} - \frac{\abs{\skal{x}{y}}^2}{\skal{y}{y}}.
		\end{align*}
		This gives
		\[
			\frac{\abs{\skal{x}{y}}^2}{\skal{y}{y}} < \skal{x}{x}
		\]
		and it follows
		\[
			\abs{\skal{x}{y}}^2 < \skal{x}{x} \skal{y}{y}.
		\]
	\end{beweis}
Now we can use this inequality to proof the statement above:
\begin{beweis}
	\begin{enumerate}[(i)]
		\item $\norm{x} >0$ for all $x \neq 0$ in $V$ (Exercise).
		\item $\norm{\lambda x} = \abs{\lambda} \norm{x}$ for all $x \in V$, $\lambda \in \mathbb{C}$ (Exercise).
		\item Let $x,y \in V$. Then
		\begin{align*}
			\norm{x+y}^2 &= \skal{x+y}{x+y} \\ &= \skal{x}{x}+ \skal{x}{y}+ \skal{y}{x} + \skal{y}{y} \\
			&= \skal{x}{x} + 2 \re( \skal{x}{y}) + \skal{y}{y} \\
			&\leq  \skal{x}{x}+ 2 \abs{\skal{x}{y}} + \skal{y}{y} \\
			&\leq  \skal{x}{x} + 2 \skal{x}{x}^{\frac{1}{2}}\skal{y}{y}^{\frac{1}{2}} + \skal{y}{y} \\
			&= \left( \skal{x}{x}^{\frac{1}{2}} + \skal{y}{y}^{\frac{1}{2}} \right)^2.
		\end{align*}
		So
		\[
			\norm{x+y}^2 \leq \left( \norm{x} + \norm{y} \right)^2.
		\]
	\end{enumerate}
\end{beweis}
\begin{theorem}[The Parallelogram Law]
	Let $(V, \skal{.}{.})$ be an inner product space. Let $\norm{x} = \skal{x}{x}^{\frac{1}{2}}$. Then
	\[
		\norm{x+y}^2 + \norm{x-y}^2 = 2 (\norm{x}^2 + \norm{y}^2) \qquad \forall\, x,y \in V.
	\]
\end{theorem}
\begin{satz}
	$l^p$ has inner product $\skal{.}{.}_{l^p}$ such that
	\[
		\norm{x}_p = \sqrt{\skal{x}{x}_{l^p}}
	\]
	iff $p =2$.
\end{satz}
\begin{beweis}
	Enough to show that $\norm{.}_p$-norm does not satisfy the parallelogram law for some $x,y \in l^p$ if $p \neq 2$. Take for example $x = (1,0,0, \dots)$
	and $y= (0,1,0, \dots)$. Note that $\norm{x}_{l^p} = \norm{y}_{l^p} = 1$
	\begin{align*}
		\norm{x+y}^2_{l^p} &= \norm{(1,1,0, \dots)}_{l^p} = 2^{\frac{2}{p}} \\
		\norm{x-y}^2_{l^p} &= \norm{(1,-1,0,\dots)}_{l^p} = 2^{\frac{2}{p}} \\
		\norm{x+y}^2_{l^p} + \norm{x-y}_{l^p}^2 &= 2 \cdot 2^{\frac{2}{p}} = 2( \norm{x}^2_{l^p}+ \norm{y}^2_{l^p}) = 2 \cdot 2.
	\end{align*}
\end{beweis}
All $l^p$ with $p \neq 2$ are not inner product spaces.
\minisec{Exercise:}Show that $(C([0,1]),\norm{.}_{\infty})$ is not an inner product space.
\begin{bemerkung}
	Whenever a norm satisfies the parallelogram law then there exists an inner product on $V$ such that
	\[
		\norm{x} = \skal{x}{x}^{\frac{1}{2}}.
	\]
\end{bemerkung}
\begin{theorem}[The Polarization Identity]
	Let $(V,\skal{.}{.})$ be an inner product space. Then
	\[
		4 \skal{x}{y}  = \norm{x+y}^2- \norm{x-y}^2 + i \norm{x+ iy}^2 - i \norm{x - iy}^2.
	\]
\end{theorem}
\begin{definition}
	Let $(V, \skal{.}{.})$ be an inner product space. We say that $x,y$ in $V$ are orthogonal if $\skal{x}{y} = 0$ (We write $x \perp y$). Let $M \subseteq V$
	Define the orthogonal complement
	\[
		M^{\perp} = \set[x \in V]{x \perp y \text{ for any }y \in M}.
	\]
\end{definition}
\begin{proposition}
	If $M \subseteq V$ then $M^{\perp}$ is a subspace of $V$.
\end{proposition}
\begin{theorem}[Pythagorean formula]
	$x,y \in V$ (inner product space). Then
	\[
		x \perp y \qquad \text{iff} \qquad \norm{x+y}^2 = \norm{x}^2 + \norm{y}^2.
	\]
\end{theorem}

\subsection{Orthogonal Systems}
\label{sub:orthogonal_systems}

Let $(V, \skal{.}{.})$ be an inner product space $\set{u_n} \subseteq V$ is called orthogonal system (with $n$ finite or infinite) if $u_n \perp u_m$ for all $n \neq m$. It is an orthonormal system if in addition $\norm{u_n}=1$.

\begin{beispiele}
	\begin{enumerate}[1)]
		\item $\set{e_k}_{k=1}^{\infty} \subseteq l^2$ with
		\[
			\skal{x}{y} = \sum^{\infty}_{i=1} x_i \bar{y}_i
		\]
		with
		\[
			e_k = (0,\dots,1,0,\dots).
		\]
		$\Rightarrow$ $\set{e_k}$ is an ON-system.
		\item $C([-\pi,\pi])$ with
		\[
			\skal{f}{g} = \int_{- \pi}^{\pi} f(t) \overline{g(t)} \,\mathrm{d}t.
		\]
		\[
			\set[\frac{1}{\sqrt{2 \pi}} e^{-int}]{n \in \mathbb{Z}}
		\]
		is an orthonormal system.
	\end{enumerate}
\end{beispiele}

\begin{definition}
	Let $\set[a_n]{n \in \mathbb{N}}$ be an orthonormal system in $V$. The formal series
	\[
		\sum_{n=1}^{\infty} \skal{x}{a_n}a_n
	\]
	is called a fourier series of $x$ corresponding $\set[a_n]{n \in \mathbb{N}}$ and $\skal{x}{a_n}$ are called fourier coefficients of $x$ corresponding to $\set[a_n]{n \in \mathbb{N}}$.
\end{definition}

\begin{theorem}[Bessel's Equality and Inequality]
	If $\set{a_n}$ orthonormal system in an inner product space $V$, then for all $x \in V$
	\[
		\norm{x- \sum_{k=1}^{n} \skal{x}{a_k}a_k}^2 = \norm{x}^2 - \sum_{k=1}^{n} \abs{\skal{x}{a_k}}^2
	\]
	and
	\[
		\sum_{k=1}^{\infty} \abs{\skal{x}{a_k}}^2 \leq \norm{x}^2.
	\]
\end{theorem}
\begin{beweis}
	\begin{align*}
		\norm{x- \sum_{k=1}^{n} \skal{x}{a_k}a_k}^2 &= \skal{x- \sum_{k=1}^{n} \skal{x}{a_k}a_k }{x - \sum_{k=1}^{n} \skal{x}{a_k}a_k} \\
		&= \skal{x}{x} - \sum_{k=1}^{n} \overline{\skal{x}{a_k}} \skal{x}{a_k} - \sum_{k=1}^{n} \skal{x}{a_k}\skal{a_k}{x}
		\\ & \qquad \qquad + \skal{\sum_{k=1}^{n}\skal{x}{a_k}a_k}{\sum_{k=1}^{n}\skal{x}{a_k}a_k} \\
		& = \norm{x}^2 - \sum_{k=1}^{n} \abs{\skal{x}{a_k}}^2 - \sum_{k=1}^{n} \abs{\skal{x}{a_k}}^2 + \sum_{k=1}^{n}\abs{\skal{x}{a_k}}^2  \\
		& = \norm{x}^2 - \sum_{k=1}^{n} \abs{\skal{x}{a_k}}^2.
	\end{align*}
	This gives also:
	\[
		\sum_{k=1}^{n}\abs{\skal{x}{a_k}}^2 = \norm{x}^2 - \norm{x - \sum_{k=1}^{n} \skal{x}{a_k}a_k} \leq \norm{x}^2
	\]
	for all $n \in \mathbb{N}$. Hence
	\[
		\sum_{k=1}^{\infty} \abs{\skal{x}{a_k}}^2 \leq \norm{x}^2.
	\]
\end{beweis}

\begin{definition}[Hilbert space]
	A Hilbert space is an inner product space which is complete w.r.t. the norm is defined through the inner product.
\end{definition}

\begin{beispiele}
	\begin{itemize}
		\item $\mathbb{C}^n$ is an inner product space and complete w.r.t the Eukledean norm. Hence $\mathbb{C}^n$ is a Hilbert space.
		\item $l^2$ is a Banach space w.r.t.
		\[
			\norm{x}_{l^2} = \left( \sum_{i=1}^{\infty} \abs{x_i}^2 \right)^{\frac{1}{2}}
		\]
		and
		\[
			\norm{x}_{l^2} = \skal{x}{x}^{\frac{1}{2}},
		\]
		where
		\[
			\skal{x}{y} = \sum_{i=1}^{\infty} x_i \bar{y}_i.
		\]
		\item $(C([0,1]), \norm{.}_{\infty})$ is a Banach space but not an inner product space. Hence it is no Hilbert space.
		\item $(C([0,1]),\skal{.}{.})$ is an inner product space $f,g \in C([0,1])$ with
		\[
			\skal{f}{g} = \int_{0}^{1} f(t) \overline{g(t)} \,\mathrm{d}t
		\]
		and the corresponding
		\[
			\norm{f}_2 = \skal{f}{f} = \int_{0}^{1} \abs{f(t)}^2 \,\mathrm{d}t.
		\]
	\end{itemize}
\end{beispiele}
\begin{bemerkung}
	Other $l^p$ spaces are not Hilbert spaces!!!! They are not inner product spaces.
\end{bemerkung}

\begin{satz}
	$(C([0,1]), \skal{.}{.})$ is not a Hilbert space since $(C([0,1]), \norm{.}_2)$ is not complete.
\end{satz}

\begin{beweis}
	Sketch: Show that $f_n(t)$, which is defined as a piecewise continuous function for example
	\[
		f_n(x)= \begin{cases}
			1, &\text{ if }x \in [0,\frac{1}{2}]\\
			0, &\text{ if }x \in [\frac{1}{2} + \frac{1}{n}] \\
			\text{continuous}, & \text{elsewhere}
		\end{cases}
	\] is a Cauchy sequence w.r.t $\norm{.}_2$ but has no limit in $C([0,1])$.
\end{beweis}

Consider
\[
	C_F = \set[(x_1,x_2,\dots)]{\text{only finite }x_i \neq 0}
\]
with \[
	\skal{x}{y} = \sum_{i=1}^{\infty} x_i \bar{y}_i.
\]
Show that $(C_F, \skal{.}{.})$ is not a Hilbert space.

\begin{definition}[strongly and weakly convergent]
	A sequence $\set{x_n} \subseteq H$, where $H$ is a Hilbert space, is called strongly convergent $(x_n \to x \in H)$ if
	\[
		\norm{x_n -x} \to 0, \qquad n \to  \infty.
	\]
	(Norm induced by an inner product) \\ We say that $x_n$ is weakly convergent ($x_n \rightharpoonup x$) if
	\[
		\skal{x_n}{y} \to \skal{x}{y}, \qquad \forall\, y \in H.
	\]
\end{definition}

\begin{satz}
	$x_n \to x$$ \qquad \Rightarrow \qquad $ $x_n \rightharpoonup x$.
\end{satz}

\begin{beweis}
	Assume strong convergence for $(x_n)_{n \in \mathbb{N}}$. Then
	\begin{align*}
		\abs{\skal{x_n}{y}- \skal{x}{y}} &= \abs{\skal{x_n-x}{y}} \\
		&\leq \underset{=\norm{x_n-x}}{\underbrace{\skal{x_n-x}{x_n-x}^{\frac{1}{2}}}} \underset{= \norm{y}}{\underbrace{\skal{y}{y}^{\frac{1}{2}}}} \\
		&= \underset{ \to 0}{\underbrace{x_n-x}} \norm{y} \to 0, \qquad n \to \infty.
	\end{align*}
	Hence $\skal{x_n}{y} \to \skal{x}{y}$.
\end{beweis}

\begin{bemerkung}
	The converse is not true in general: \\
	Take $H=l^2$ and
	\begin{align*}
		x_n &= e_n = (0, \dots,1,0,\dots) \\
		y &= (y_1,y_2,\dots) \in l^2.
	\end{align*}
	We have for all $y \in H$
	\[
		\skal{e_n}{y} = y_n \to 0, \qquad n \to \infty
	\]
	as
	\[
		\norm{e_n -0}_{l^2} = \norm{e_n}_{l^2} =1.
	\]
\end{bemerkung}

\begin{satz}
	$x_n \to x$ and $y_n \to y$ yields
	\[
		\skal{x_n}{y_n} \to \skal{x}{y}.
	\]
	In particular
	\[
		x_n \to x \qquad \Rightarrow \qquad \norm{x_n} \to \norm{x}.
	\]
\end{satz}
\begin{beweis}
	\begin{align*}
		\abs{\skal{x_n}{y_n}-\skal{x}{y}} &= \abs{\skal{x_n}{y_n}-\skal{x}{y_n} + \skal{x}{y_n} - \skal{x}{y}} \\
		&= \abs{\skal{x_n -x}{y_n} + \skal{x}{y_n-y}} \\
		&\leq \abs{\skal{x_n-x}{y_n}} + \abs{\skal{x}{y_n-y}} \\
		&\leq \underset{\to 0}{\underbrace{\norm{x_n-x}}} \underset{< \infty}{\underbrace{\norm{y_n}}} + \underset{< \infty}{\underbrace{\norm{x}}} \underset{\to 0}{\underbrace{\norm{y_n-y}}} \to 0, \qquad  n \to \infty.
	\end{align*}
	Check $\set{\norm{y_n}}$ is bounded
	\[
		\norm{y_n} = \norm{y_n -y +y} \leq \underset{\to 0}{\underbrace{\norm{y_n -y}}}+ \underset{< \infty}{\underbrace{\norm{y}}} \to 0, \qquad n \to \infty.
	\]
\end{beweis}

\begin{satz}
	$x_n \rightharpoonup x$ and $\norm{x_n} \to \norm{x}$ yields
	\[
		x_n \to x.
	\]
\end{satz}
\begin{beweis}
	\begin{align*}
		\norm{x_n-x}^2 &= \skal{x_n -x}{x_n-x} \\
		&= \underset{= \norm{x_n}^2}{\underbrace{\skal{x_n}{x_n}}} - \skal{x}{x_n}- \skal{x_n}{x} + \skal{x}{x} \\
		&= \norm{x_n}^2 - \overline{\skal{x_n}{x}} - \skal{x_n}{x} + \norm{x}^2 \\
		&\to \norm{x}^2 - \norm{x}^2 - \norm{x}^2 + \norm{x}^2 = 0.
	\end{align*}
\end{beweis}

We have proved
\[
	x_n \to x \qquad \Rightarrow \qquad \set{\norm{x_n}} \text{ is bounded}.
\]

\begin{theorem}
	\[
		x_n \rightharpoonup x \qquad \Rightarrow \qquad \sup_{n \in \mathbb{N}}\norm{x_n} < \infty.
	\]
\end{theorem}
\begin{beweis}
	Let $x_n \rightharpoonup x$. Consider $f_n: H \to \mathbb{C}$ where
	\[
		f_n(y) = \skal{y}{x_n}, \qquad y \in H.
	\]
	\begin{itemize}
		\item $f_n$ is a linear functional for every $n \in \mathbb{N}$.
		\item $\forall\, n \in \mathbb{N}$ $f_n$ is a bounded ($\Leftrightarrow$ continuous) linear functional as if
		\[
			y_k \stackrel{k \to \infty}{\to }y \qquad \Rightarrow \qquad f_n(y_k) = \skal{y_k}{x_n} \to \skal{y}{x_n} = f_n(y), \qquad k \to \infty.
		\]
		\item $f_n(y) \to \skal{y}{x}$. \\
		$\set{f_n(y)}_n$ is a convergent sequence in $\mathbb{C}$ and hence bounded for all $y \in H$. \\
		Hence it exists $M_y$ such that
		\[
			\abs{f_n(y)} \leq M_y.
		\]
		By Banach-Steinhaus-Theorem it holds
		\[
			\norm{f_n} \leq M \text{ for some }M >0.
		\]
		We are done if we proof that $\norm{f_n} = \norm{x_n}$.
		\[
			\abs{f_n(y)}= \abs{\skal{y}{x_n}} \leq \norm{y} \norm{x_n}, \qquad \forall\, y \in H.
		\]
		Hence
		\[
			\norm{f_n} \leq \norm{x_n} \qquad \qquad (1).
		\]
		On the other Hand we have
		\[
			f_n(x_n) = \skal{x_n}{x_n}= \norm{x_n}^2
		\]
		and thus
		\[
			\norm{f_n} = \sup_{x \in H} \frac{\abs{f_n(x)}}{\norm{x}} \geq \frac{\abs{f_n(x_n)}}{\norm{x_n}} = \norm{x_n} \qquad \qquad (2)
		\]
		With (1) and (2) we are finished.
	\end{itemize}
\end{beweis}

\subsection{Orthogonal decomposition in Hilbert spaces}
\label{sub:orthogonal_decomposition_in_hilber_spaces}

Remember Linear Algebra. Take $\mathbb{R}^n$ and a subspace $M \subseteq \mathbb{R}^n$
\[
	\Rightarrow \qquad \forall\, x \in \mathbb{R}^n \qquad x = z + y, \qquad \text{where }z \in M, y \in M^{\perp}.
\]
This can be done in a unique way
\begin{align*}
	M &= \spn \set{e_z} \\
	M^{\perp} &= \spn\set{e_y}
\end{align*}
and
\[
	z = \proj_{M^{\perp}}x, \qquad \qquad \norm{x - \proj_Mx} = \min_{y \in M}\norm{x-y}.
\]

\minisec{General Hilbert space case}

\begin{proposition}
	$M \subseteq H$, then $M^{\perp}$ is a closed subspace and
	\[
		\left( M^{\perp} \right)^{\perp} = \overline{\spn M}.
	\]
\end{proposition}

\begin{satz}
	$H$ Hilbert space and $M$-closed subspace of $H$ and $x \in H$. Then there exists a unique $z \in M$ such that
	\[
		\norm{x-z} = \dist(x,M) := \inf_{y \in M} \norm{x-y}.
	\]
	($z$ analog of the $\text{proj}_Mx$ in the other case).
\end{satz}

\begin{proposition}
	Taking $z \in M$ from the previous proposition. We have $x - z \in M^{\perp}$, i.e.
	\[
		x = \underset{\in M}{\underbrace{z}} + \underset{\in M^{\perp}}{\underbrace{(x - z)}}.
	\]
\end{proposition}


\subsection{Dot product}

Given two vectors we can calculate:

\(\dfrac{a.b}{||a|| ||b||}\)

If the two vectors are identical, this is \(1\). If they are orthogonal this is \(0\). If they are opposite, this is \(-1\).