%% -*- coding:utf-8 -*-
\chapter{Base definitions}

\section{Definitions}

\subsection{Object}

\begin{definition}[Class]
  A class is a collection of sets (or sometimes other mathematical
  objects) that can be unambiguously defined by a property that all
  its members share.
  \label{def:class}
\end{definition}

\begin{definition}[Object]
  \label{def:object}
  In category theory object is considered as something that does not
  have internal structure (aka point) but has a property that makes
  different objects belong to the same \mynameref{def:class}
\end{definition}

\begin{remark}[Class of Objects]
  \label{rem:objclass}
  The \mynameref{def:class} of \mynameref{def:object}s will be marked as
  $\catob{C}$ (see \cref{fig:class_of_objects}).
\end{remark}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,4) {$\cat{C}$};

    \node[ele,label=left:$a$] (a) at (2,2) {};
    \node[ele,label=left:$b$] (b) at (2,1) {};
    \node[ele,label=left:$c$] (c) at (0,2) {};
    \node[ele,label=left:$d$] (d) at (0,1) {};

    \node[draw,fit= (a) (b) (c) (d),minimum width=4cm, minimum
      height=4cm] {}  ;

  \end{tikzpicture}
  \caption{Class of objects $\catob{C}=\{a,b,c,d\}$}
  \label{fig:class_of_objects}
\end{figure}

\subsection{Morphism}
Morphism is a kind of relation between 2 \mynameref{def:object}s.
\begin{definition}[Morphism]
  \label{def:morphism}
  A relation between two \mynameref{def:object}s $a$ and $b$
  \[
  f_{ab}: a \rightarrow b
  \]
  is called
  \textit{morphism}. Morphism assumes a direction i.e. one \mynameref{def:object}
  ($a$) is called \textit{source} and another one ($b$)
  \textit{target}.

  The \mynameref{def:set} of all morphisms between objects $a$ and $b$
  is denoted as $\hom\left(a, b\right)$.
\end{definition}

The important remark about morphisms is below
\begin{remark}[Morphism]
\label{rem:morphism}
The morphism has to be considered as a relation between objects. We
will avoid standard (from set theory) notation for morphisms: $f(a) =
b$. The reason for this is the following. Let $f_1: a \to b$ and $f_2:
a \to b$ are 2 different morphisms. The notation $f_1(a) = b, f_2(a) =
b$ leads to incorrect conclusion that $f_1 = f_2$.

For instance if $a
= b = \mathbb{R}$ then 2 functions $f_1(x) = x, f_2(x) = -x$ set 2
different ordering on $\mathbb{R}$ and as result have not to be
considered as the same functions.
\end{remark}

\begin{definition}[Domain]
  \label{def:domain}
  Given a \mynameref{def:morphism} $f: a \to b$, the
  \mynameref{def:object} $a$ is called domain and denoted as $\dom f$.
\end{definition}

\begin{definition}[Codomain]
  \label{def:codomain}
  Given a \mynameref{def:morphism} $f: a \to b$, the
  \mynameref{def:object} $b$ is called codomain and denoted as $\cod f$.
\end{definition}

\mynameref{def:morphism}s have several properties. \footnote{The
  properties don't have any proof and postulated as axioms}
\begin{axiom}[Composition]
  \label{axm:composition}
  If we have 3 \mynameref{def:object}s $a, b$ and $c$ and 2
  \mynameref{def:morphism}s
  \[
  f_{ab} : a \rightarrow b
  \]
  and
  \[
  f_{bc} : b \rightarrow c
  \]
  then there exists \mynameref{def:morphism}
  \[
  f_{ac} : a \rightarrow c
  \]
  such that
  \[
  f_{ac} = f_{bc} \circ f_{ab}
  \]
\end{axiom}

\begin{remark}[Composition]
  \label{rem:composition}
  The equation
  \[
  f_{ac} = f_{bc} \circ f_{ab}
  \]
  means that we apply $f_{ab}$ first and then we apply $f_{bc}$ to the
  result of the application i.e. if our objects are sets and $x \in a$
  then
  \[
  f_{ac} ( x ) = f_{bc} ( f_{ab} ( x ) ),
  \]
  where $f_{ab} ( x ) \in b$.
\end{remark}

\begin{axiom}[Associativity]
  \label{axm:associativity}
  The \mynameref{def:morphism}s \mynameref{axm:composition}s should
  follow associativity property:
  \[
  f_{ce} \circ (f_{bc} \circ f_{ab}) = (f_{ce} \circ f_{bc}) \circ
  f_{ab} = f_{ce} \circ f_{bc} \circ f_{ab}.
  \]
\end{axiom}

\begin{definition}[Identity morphism]
  \label{def:id}
  For every \mynameref{def:object} $a$ we define a special
  \mynameref{def:morphism} $\idm{a} : a \rightarrow a$ with the
  following properties: $\forall f_{ab} : a \rightarrow b$
  \begin{equation}
    \idm{a} \circ f_{ab} = f_{ab}
    \label{eq:leftid}
  \end{equation}
  and
  $\forall f_{ba} : b \rightarrow a$
  \begin{equation}
    f_{ba} \circ \idm{a}  = f_{ba}.
    \label{eq:rightid}
  \end{equation}
  This morphism is called as \textit{identity morphism}.
\end{definition}

Note that \mynameref{def:id} is unique, see
\mynameref{thm:identity_unique} below.

\begin{definition}[Commutative diagram]
  A commutative diagram is a diagram of \mynameref{def:object}s (also known as
  vertices) and \mynameref{def:morphism}s (also known as arrows or
  edges) such that all directed paths in the diagram with the same
  start and endpoint lead to the same result by composition
  \label{def:commutative_diagram}

  The following diagram commutes if $f_{ab} = f_{cb} \circ f_{ac}$.

  \begin{center}
    \begin{tikzpicture}[description/.style={fill=white,inner sep=2pt}]
      \matrix (m) [matrix of math nodes, row sep=3em,
        column sep=2.5em, text height=1.5ex, text depth=0.25ex]
              { a& & b \\
                & c & \\ };
              %\draw[double,double distance=5pt] (m-1-1) – (m-1-3);
              \path[->]
              (m-1-1) edge node[description] {$ f_{ab} $} (m-1-3)
              edge node[description] {$  f_{ac} $} (m-2-2)
              (m-2-2) edge node[description] {$  f_{cb} $} (m-1-3);
    \end{tikzpicture}
  \end{center}
\end{definition}

\begin{remark}[Class of Morphisms]
  \label{rem:morphclass}
  The \mynameref{def:class} of \mynameref{def:morphism}s will be marked as
  $\cathom{C}$ (see \cref{fig:class_of_morphisms})
\end{remark}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,4) {$\cat{C}$};

    \node[ele,label=left:$a$] (a) at (0,2) {};
    \node[ele,label=left:$b$] (b) at (0,1) {};
    \node[ele,label=right:$c$] (c) at (2,2) {};
    \node[ele,label=right:$d$] (d) at (2,1) {};

    \node[draw,fit= (a) (b) (c) (d),minimum width=4cm, minimum
      height=4cm] {}  ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (a) to
    node[pos=0.5,above]{$f$} (c);
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (b) to
    node[pos=0.5,left]{$g$} (a);
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (b) to
    node[pos=0.5,right]{$h$} (c);

  \end{tikzpicture}
  \caption{Class of morphisms $\cathom{C}=\{f,g,h\}$, where $h = f
    \circ g$}
  \label{fig:class_of_morphisms}
\end{figure}

\begin{definition}[Monomorphism]
  \label{def:monomorphism}
  If $\forall g_1, g_2$ the equation
  \[
  f \circ g_1 = f \circ g_2
  \]
  leads to
  \[
  g_1 = g_2
  \]
  then $f$ is called \textit{monomorphism}.
\end{definition}

\begin{definition}[Epimorphism]
  \label{def:epimorphism}
  If $\forall g_1, g_2$ the equation
  \[
  g_1 \circ f = g_2 \circ f
  \]
  leads to
  \[
  g_1 = g_2
  \]
  then $f$ is called \textit{epimorphism}.
\end{definition}

\begin{definition}[Isomorphism]
\label{def:isomorphism}
A \mynameref{def:morphism} $f: a \to b$ is called \textit{isomorphism} if
$\exists g: b \to a$ such that $f \circ g = \idm{a}$
and $g \circ f = \idm{b}$.
If there is an isomorphism $f$ between objects $a$ and $b$
then it is denoted as $a \cong_f b$.
\end{definition}

\begin{remark}[Isomorphism]
\label{rem:isomorphism}
There are can be many different \mynameref{def:isomorphism}s between 2
\mynameref{def:object}s.

If there is an unique isomorphism between 2 objects then the objects
can be treated as the same object.
\end{remark}

\subsection{Category}

\begin{definition}[Category]
  \label{def:category}
  A category $\cat{C}$ consists of
  \begin{itemize}
  \item \mynameref{def:class} of
    \mynameref{def:object}s $\catob{C}$
  \item \mynameref{def:class} of \mynameref{def:morphism}s $\cathom{C}$
    defined for $\catob{C}$, i.e. each morphism $f_{ab}$ from
    $\cathom{C}$ has both source
    $a$ and target $b$ from $\catob{C}$
  \end{itemize}
  For any \mynameref{def:object} $a$ there should be unique
  \mynameref{def:id} $\idm{a}$. Any morphism should satisfy
  \mynameref{axm:composition} and \mynameref{axm:associativity}. See
  \cref{fig:category}
\end{definition}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,4.5) {$\cat{C}$};

    \node[ele,label=left:$a$] (a) at (0,2) {};
    \node[ele,label=left:$b$] (b) at (0,1) {};
    \node[ele,label=right:$c$] (c) at (2,2) {};
    \node[ele,label=right:$d$] (d) at (2,1) {};

    \node[draw,fit= (a) (b) (c) (d),minimum width=5cm, minimum
      height=5cm] {}  ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (a) to
    node[pos=0.5,above]{$f$} (c);
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (b) to
    node[pos=0.5,left]{$g$} (a);
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (b) to
    node[pos=0.5,right]{$h$} (c);

    \draw (a) to [out=45,in=135,looseness=20] node[above] {$\idm{a}$} (a);
    \draw (b) to [out=-45,in=-135,looseness=20] node[below] {$\idm{b}$} (b);
    \draw (c) to [out=45,in=135,looseness=20] node[above] {$\idm{c}$} (c);
    \draw (d) to [out=-45,in=-135,looseness=20] node[below] {$\idm{d}$} (d);

  \end{tikzpicture}
  \caption{Category $\cat{C}$. It consists of 4 objects
    $\catob{C} = \{a,b,c,d\}$ and 7 morphisms
    $\catob{C} = \{f,g,h = f \circ g, \idm{a}, \idm{b},
    \idm{c}, \idm{d}\}$}
  \label{fig:category}
\end{figure}

\begin{definition}[Set of morphisms]
\label{def:morphism_set}
  The set of morphisms between objects $a$ and $b$ in the $\cat{C}$
  will be denoted as $\hom_{\cat{C}}(a, b)$
\end{definition}

The \mynameref{def:category} can be considered as a way to represent a
structured data. \mynameref{def:morphism}s are the ones which form the
structure.

\begin{definition}[Opposite category]
\label{def:op_category}
\index{Category!opposite}
\index{Category!dual}
If $\cat{C}$ is a \mynameref{def:category} then opposite (or dual) category
$\cat{C}^{op}$ is constructed in the following way: \mynameref{def:object}s
are the same, but the \mynameref{def:morphism}s are inverted i.e.
if $f \in \cathom{C}$ and $\dom f = a, \cod f = b$, then the
corresponding morphism $f^{op} \in \cathom{C^{op}}$ has $\dom f^{op} =
b, \cod f^{op} = a$ (see \cref{fig:op_category})
\end{definition}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,4.5) {$\cat{C}$};

    \node[ele,label=left:$a$] (a) at (0,2) {};
    \node[ele,label=left:$b$] (b) at (0,1) {};
    \node[ele,label=right:$c$] (c) at (2,2) {};
    \node[ele,label=right:$d$] (d) at (2,1) {};

    \node[draw,fit= (a) (b) (c) (d),minimum width=5cm, minimum
      height=5cm] {}  ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (c) to
    node[pos=0.5,above]{$f^{op}$} (a);
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (a) to
    node[pos=0.5,left]{$g^{op}$} (b);
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (c) to
    node[pos=0.5,right]{$h^{op}$} (b);

    \draw (a) to [out=45,in=135,looseness=20] node[above] {$\idm{a}$} (a);
    \draw (b) to [out=-45,in=-135,looseness=20] node[below] {$\idm{b}$} (b);
    \draw (c) to [out=45,in=135,looseness=20] node[above] {$\idm{c}$} (c);
    \draw (d) to [out=-45,in=-135,looseness=20] node[below] {$\idm{d}$} (d);

  \end{tikzpicture}
  \caption{Opposite category $C^{op}$ to the category from
    \cref{fig:category} . It consists of 4 objects
    $\catob{C^{op}} = \catob{C} = \{a,b,c,d\}$ and 7 morphisms
    $\cathom{C^{op}} = \{f^{op},g^{op},h^{op} = g^{op} \circ
    f^{op}, \idm{a}, \idm{b},
    \idm{c}, \idm{d}\}$}
  \label{fig:op_category}
\end{figure}

\begin{remark}{Composition on $C^{op}$}
\label{rem:op_composition}
\index{Composition!opposite category}
As you can see from \cref{fig:op_category} the
\mynameref{axm:composition} is reverted for
\mynameref{def:op_category}. If $f,g,h = f \circ g \in
\cathom{C}$ then $f \circ g$ translated into $g^{op} \circ
f^{op}$ in opposite category.
\end{remark}

\begin{definition}[Small category]
\label{def:small_category}
\index{Category!small}
A category $\cat{C}$ is called \textit{small} if both $\catob{C}$ and
$\cathom{C}$ are \nameref{def:set}s
\end{definition}

\begin{definition}[Large category]
\label{def:large_category}
\index{Category!large}
A category $\cat{C}$ is not \mynameref{def:small_category} then it is
called \textit{large}. The example of large category is
\mynameref{def:setcategory}
\end{definition}

\section{\textbf{Set} category example}

There are several examples of categories that will also be used later

\begin{definition}[Set]
  \label{def:set}
  Set is a collection of distinct object. The objects are called the
  elements of the set.
\end{definition}

\begin{definition}[Binary relation]
  \label{def:binary_relation}
  If $A$ and $B$ are 2 \mynameref{def:set}s then a subset of $A \times B$ is
  called binary relation $R$ between the 2 sets, i.e. $R \subset A \times B$.
\end{definition}

\begin{definition}[Function]
  \label{def:function}
  Function $f$ is a special type of \mynameref{def:binary_relation}. I.e.
  if $A$ and $B$ are 2 \mynameref{def:set}s then a subset of $A \times B$ is
  called function $f$ between the 2 sets if $\forall a \in A \, \exists!
  b \in B$ such that $(a,b) \in f$. In other words function definition
  does not allow ``multi value''.
\end{definition}

\begin{definition}[Cartesian product]
  \label{def:cartesian_product}
  If $A$ and $B$ are two sets then we can define a new set $A \times B
  = \left\{(a,b)|a \in A, b \in B\right\}$ that is called as the
  \textit{cartesian product}.
\end{definition}

\begin{definition}[\textbf{Set} category]
  \label{def:setcategory}
  \index{Object!\textbf{Set} category}
  \index{Morphism!\textbf{Set} category}
  \index{Category!\textbf{Set}}
  In the set category we consider a \mynameref{def:set} of
  \mynameref{def:set}s where
  \mynameref{def:object}s are the \mynameref{def:set}s and
  \mynameref{def:morphism}s are \mynameref{def:function}s between the
  sets.

  The \mynameref{def:id} is trivial function such that $\forall x \in
  X: \idm{X}(x) = x$.

  In general case when we say \textbf{Set} category we assume the set
  of all sets. But the result is inconsistent because famous Russell's
  paradox \cite{wiki:russell_paradox} can be applied. To avoid such
  situations we consider a limitation that is applied on our
  construction, for instance
  ZFC \cite{wiki:zfc}. If we apply the limitation we have that set of
  all sets is not a set itself and as result the  \textbf{Set}
  category is a \mynameref{def:large_category}
  \index{Category!large}
\end{definition}

\begin{remark}[Set vs Category]
  \label{rem:set_vs_category}
  There is an interesting relation between sets and categories. In both
  we consider objects(sets) and relations between
  them(morphisms/functions).

  In the set theory we can get info about functions by looking inside
  the objects(sets) aka use ``microscope'' \cite{bib:milewski2018category}

  Contrary in the category theory we initially don't have any info about object
  internal structure but can get it using the relation between the
  objects i.e. using \mynameref{def:morphism}s. In other words we can use
  ``telescope'' \cite{bib:milewski2018category}  there.
\end{remark}

\begin{definition}[Categorical approach]
\label{def:categorical_approach}
The description of a system via its communications we will call as
\textit{categorical approach}.

This description is contrary to an ordinary system description via its
internal structure.
\end{definition}

\begin{definition}[Singleton]
\label{def:singleton_set}
The \textit{singleton} is a \mynameref{def:set} with only one element.
\end{definition}

\begin{example}[Domain]
  \label{ex:domain_set}
  Given a function $f: X \to Y$, the set $X$ is the domain. I.e. $\dom
  f = X$
\end{example}

\begin{example}[Codomain]
  \label{ex:codomain_set}
  Given a function $f: X \to Y$, the set $Y$ is the codomain. I.e.
  $\cod f = Y$
\end{example}

\begin{definition}[Surjection]
  \label{def:surjection}
  The function $f: X \rightarrow Y$ is surjective (or onto) if
  $\forall y \in Y$, $\exists x \in X$ such that
  $f\left(x\right) = y$ (see \cref{fig:surjection,fig:bijection}).
\end{definition}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,5) {$X$};
    \node at (4,5) {$Y$};

    \node[ele,label=left:$x_1$] (x1) at (0,4) {};
    \node[ele,label=left:$x_2$] (x2) at (0,3) {};
    \node[ele,label=left:$x_3$] (x3) at (0,2) {};
    \node[ele,label=left:$x_4$] (x4) at (0,1) {};

    \node[ele,,label=right:$y_1$] (y1) at (4,4) {};
    \node[ele,,label=right:$y_2$] (y2) at (4,3) {};
    \node[ele,,label=right:$y_3$] (y3) at (4,2) {};

    \node[draw,fit= (x1) (x2) (x3) (x4),minimum width=2cm] {} ;
    \node[draw,fit= (y1) (y2) (y3),minimum width=2cm] {} ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (x1) -- (y2);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x2) -- (y1);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x3) -- (y3);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x4) -- (y3);
  \end{tikzpicture}
  \caption{A surjective (non-injective) function from domain $X$ to
    codomain $Y$ }
  \label{fig:surjection}
\end{figure}

\begin{remark}[Surjection vs Epimorphism]
  \label{rem:surjection_epimorphism}
  \mynameref{def:surjection} and \mynameref{def:epimorphism} are
  related each other. Consider a non-surjective function $f: X
  \rightarrow Y' \subset Y$ (see \cref{fig:surjection_epimorphism}). One can
  conclude that there is not an \mynameref{def:epimorphism} because
  $\exists g_1: Y' \to Y'$ and  $g_2 : Y \to Y$ such
  that $g_1 \ne g_2$ because they operates on different
  \mynameref{def:domain}s but from other hand $g_1(Y') = g_2(Y')$. For
  instance we can choose $g_1 = \idm{Y'}, g_2=\idm{Y}$. As
  soon as $Y'$ is \mynameref{def:codomain} of $f$ we always have
  $g_1(f(X)) = g_2(F(X))$.

  As result we can say that an \mynameref{def:surjection} is a
  \mynameref{def:epimorphism} in the \textbf{Set} category. Moreover
  there is a proof
  \cite{bib:proofwiki:Surjection_iff_Epimorphism_in_Category_of_Sets}
  of that fact.

\end{remark}

% https://tex.stackexchange.com/questions/19987/
% drawing-a-bijective-map-with-tikz
\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,5) {$X$};
    \node at (4,5) {$Y$};
    \node at (5.5,3.5) {$Y'$};

    \node[ele,label=left:$x_1$] (x1) at (0,4) {};
    \node[ele,label=left:$x_2$] (x2) at (0,3) {};
    \node[ele,label=left:$x_3$] (x3) at (0,2) {};
    \node[ele,label=left:$x_4$] (x4) at (0,1) {};

    \node[ele,,label=right:$y_1$] (y1) at (4,4) {};
    \node[ele,,label=right:$y_2$] (y2) at (4,3) {};
    \node[ele,,label=right:$y_3$] (y3) at (4,2) {};
    \node[ele,,label=right:$y_4$] (y4) at (4,1) {};

    \node[draw,fit= (x1) (x2) (x3) (x4),minimum width=2cm] {} ;
    \node[draw,fit= (y1) (y2) (y3) (y4),minimum width=2cm] {} ;
    \node[draw,fit= (y1) (y2) (y3),minimum width=2cm] {} ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (x1) -- (y2);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x2) -- (y1);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x3) -- (y3);

  \end{tikzpicture}
  \caption{A non-surjective function $f$ from domain $X$ to
    codomain $Y' \subset Y$.
    $\exists g_1: Y' \rightarrow Y', g_2: Y \rightarrow Y$ such that
    $g_1(Y') = g_2(Y')$, but as soon as $Y'
    \ne Y$ we have $g_1 \ne g_2$. Using the fact that $Y'$ is codomain
    of $f$ we got $g_1 \circ f = g_2 \circ f$.
    I.e. the function $f$ is not epimorphism. }
  \label{fig:surjection_epimorphism}
\end{figure}

\begin{definition}[Injection]
  \label{def:injection}
  The function $f: X \rightarrow Y$ is injective (or one-to-one function) if
  $\forall x_1, x_2 \in X$, such that $x_1 \ne x_2$ then
  $f\left(x_1\right) \ne f\left(x_2\right)$ (see
  \cref{fig:injection,fig:bijection}).
\end{definition}

% https://tex.stackexchange.com/questions/19987/
% drawing-a-bijective-map-with-tikz
\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,5) {$X$};
    \node at (4,5) {$Y$};

    \node[ele,label=left:$x_1$] (x1) at (0,4) {};
    \node[ele,label=left:$x_2$] (x2) at (0,3) {};
    \node[ele,label=left:$x_3$] (x3) at (0,2) {};
    \node[ele,label=left:$x_4$] (x4) at (0,1) {};

    \node[ele,,label=right:$y_1$] (y1) at (4,4) {};
    \node[ele,,label=right:$y_2$] (y2) at (4,3) {};
    \node[ele,,label=right:$y_3$] (y3) at (4,2) {};
    \node[ele,,label=right:$y_4$] (y4) at (4,1) {};

    \node[draw,fit= (x1) (x2) (x3) (x4),minimum width=2cm] {} ;
    \node[draw,fit= (y1) (y2) (y3) (y4),minimum width=2cm] {} ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (x1) -- (y2);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x2) -- (y1);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x3) -- (y3);
  \end{tikzpicture}
  \caption{A injective (non-surjective) function from domain $X$ to
    codomain $Y$ }
  \label{fig:injection}
\end{figure}

\begin{remark}[Injection vs Monomorphism]
  \label{rem:injection_monomorphism}
  \mynameref{def:injection} and \mynameref{def:monomorphism} are
  related each other. Consider a non-injective function $f: X
  \rightarrow Y$ (see \cref{fig:injection_monomorphism}). One can
  conclude that it is not monomorphism because $\exists g_1, g_2$ such
  that $g_1 \ne g_2$ and $f(g_1(a_1)) = y_3 = f(g_2(b_1))$.

  As result we can say that an \mynameref{def:injection} is a
  \mynameref{def:monomorphism} in \textbf{Set} category. Moreover
  there is a proof
  \cite{bib:proofwiki:Injection_iff_Monomorphism_in_Category_of_Sets}
  of that fact.
\end{remark}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node at (0,5) {$A$};
    \node at (0,0) {$B$};
    \node at (4,5) {$X$};
    \node at (8,5) {$Y$};

    \node[ele,label=left:$a_1$] (a1) at (0,4) {};
    \node[ele,label=left:$a_2$] (a2) at (0,3) {};

    \node[ele,label=left:$b_1$] (b1) at (0,2) {};
    \node[ele,label=left:$b_2$] (b2) at (0,1) {};

    \node[ele,label=left:$x_1$] (x1) at (4,4) {};
    \node[ele,label=left:$x_2$] (x2) at (4,3) {};
    \node[ele,label=left:$x_3$] (x3) at (4,2) {};
    \node[ele,label=left:$x_4$] (x4) at (4,1) {};

    \node[ele,,label=right:$y_1$] (y1) at (8,4) {};
    \node[ele,,label=right:$y_2$] (y2) at (8,3) {};
    \node[ele,,label=right:$y_3$] (y3) at (8,2) {};
    \node[draw,fit= (a1) (a2),minimum width=2cm] {} ;
    \node[draw,fit= (b1) (b2),minimum width=2cm] {} ;

    \node[draw,fit= (x1) (x2) (x3) (x4),minimum width=2cm] {} ;
    \node[draw,fit= (y1) (y2) (y3),minimum width=2cm] {} ;

    \draw[->,thick,shorten <=2pt,shorten >=2pt] (x1) -- (y2);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x2) -- (y1);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x3) -- (y3);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x4) -- (y3);

    \draw[->,thick,shorten <=2pt,shorten >=2] (a1) -- (x3);
    \draw[->,thick,shorten <=2pt,shorten >=2] (a2) -- (x3);

    \draw[->,thick,shorten <=2pt,shorten >=2] (b1) -- (x4);
    \draw[->,thick,shorten <=2pt,shorten >=2] (b2) -- (x4);

  \end{tikzpicture}
  \caption{A non-injective function $f$ from domain $X$ to
    codomain $Y$. $\exists g_1: A \rightarrow X, g_2: B
    \rightarrow X$ such that $g_1 \ne g_2$ but $f
    \circ g_1 = f \circ g_2$. I.e. the function $f$ is not monomorphism. }
  \label{fig:injection_monomorphism}
\end{figure}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]
    % the texts
    \node at (0,5) {$X$};
    \node at (4,5) {$Y$};

    \node[ele,label=left:$x_1$] (x1) at (0,4) {};
    \node[ele,label=left:$x_2$] (x2) at (0,3) {};
    \node[ele,label=left:$x_3$] (x3) at (0,2) {};
    \node[ele,label=left:$x_4$] (x4) at (0,1) {};

    \node[ele,,label=right:$y_1$] (y1) at (4,4) {};
    \node[ele,,label=right:$y_2$] (y2) at (4,3) {};
    \node[ele,,label=right:$y_3$] (y3) at (4,2) {};
    \node[ele,,label=right:$y_4$] (y4) at (4,1) {};

    \node[draw,fit= (x1) (x2) (x3) (x4),minimum width=2cm] {} ;
    \node[draw,fit= (y1) (y2) (y3) (y4),minimum width=2cm] {} ;
    \draw[->,thick,shorten <=2pt,shorten >=2pt] (x1) -- (y4);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x2) -- (y2);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x3) -- (y1);
    \draw[->,thick,shorten <=2pt,shorten >=2] (x4) -- (y3);
  \end{tikzpicture}
  \caption{An injective and surjective function (bijection)}
  \label{fig:bijection}
\end{figure}

\begin{definition}[Bijection]
  \label{def:bijection}
  The function $f: X \rightarrow Y$ is bijective (or one-to-one
  correspondence) if it is an \mynameref{def:injection} and a
  \mynameref{def:surjection} (see \cref{fig:bijection}).
\end{definition}

There is a question what is the categorical analog of a single
\mynameref{def:set}. Main characteristic of a category is a structure
but the set by definition does not have a structure. Which category
does not have any structure? The answer is
\mynameref{def:discrete_category}.

\begin{definition}[Discrete category]
  \label{def:discrete_category}
  Discrete category is a \mynameref{def:category} where
  \mynameref{def:morphism}s are only \mynameref{def:id}s.
\end{definition}

\section{Programming languages examples}

In the programming languages we consider types as
\mynameref{def:object}s and functions as
\mynameref{def:morphism}s. The critical requirements for such
consideration is that the functions have to be pure functions (without
side effects). This requirement mainly is satisfied by functional
languages such as Haskell and Scala. From other side the functional
languages use lazy evaluation to improve their performance. The laziness
can also make category theory axiom invalid (see
\mynameref{rem:hask_lazy_eval}).

Strictly speaking neither Haskell (pure functional language) nor C++
can be considered as a category in general. For the first approximation
a functional language (Haskell, Scala) can be considered as a
category if we avoid to use functions with side effects (mainly for
Scala) and use strict (for both Haskell and Scala) evaluations. Take
the fact into consideration and define categories for 3 languages

\begin{definition}[\textbf{Hask} category]
\label{def:haskcategory}
The objects in the \textbf{Hask} category are Haskell types and
morphisms are functions
\end{definition}

\begin{definition}[Pure function]
\label{def:pure_function}
The function is pure if it's execution give the same results
independently from the environment.
\end{definition}

\begin{definition}[\textbf{Scala} category]
\label{def:scalacategory}
The objects in the \textbf{Scala} category are Scala types and
morphisms are functions. We don't define functions that have a state
in the category. I.e. the functions are \mynameref{def:pure_function}s.
\end{definition}

\begin{definition}[\textbf{C++} category]
\label{def:cppcategory}
The objects in the \textbf{C++} category are Scala types and
morphisms are functions. We don't define functions that have a state
in the category. I.e. the functions are \mynameref{def:pure_function}s.
\end{definition}

In any case we can construct a simple toy category that can be easy
implemented in any language. Particularly we will look into category
with 3 objects that are types: Int, Bool, String. There are also
several functions between them (see \cref{fig:pl_example}).

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]
    % the texts
    \node[ele,label=right:Int] (int) at (6,4) {};
    \node[ele,label=right:Bool] (bool) at (6,0) {};
    \node[ele,label=left:String] (string) at (0,0) {};

    \draw[->,thick,shorten <=2pt,shorten >=2pt] (int) to
    node[right]{isEven} (bool);
    \draw[->,thick,shorten <=2pt,shorten >=2] (string) to
    node[pos=0.5,sloped,above]{stringLength} (int);
    \draw[->,thick,shorten <=2pt,shorten >=2] (string) to
    node[pos=0.5,sloped,above]{isStringLengthEven} (bool);
    \draw[->,thick,shorten <=2pt,shorten >=2] (string) -- (bool);
    \draw (int) to [out=45,in=135,looseness=50] node[above]
          {$id_{Int}$} (int);
    \draw (string) to [out=-45,in=-135,looseness=50] node[below]
          {$id_{String}$} (string);
    \draw (bool) to [out=-45,in=-135,looseness=50] node[below]
          {$id_{Bool}$} (bool);

  \end{tikzpicture}
  \caption{Programming language category example. Objects are types: Int,
    Bool, String. Morphisms are several functions}
  \label{fig:pl_example}
\end{figure}

\subsection{\textbf{Hask} toy category}
\begin{example}[\textbf{Hask} toy category]
  \label{ex:haskcategory}
  \index{Object!\textbf{Hask} example}
  \index{Morphism!\textbf{Hask} example}
  Types in Haskell are considered as \mynameref{def:object}s.
  Functions are considered as \mynameref{def:morphism}s.
  We are going to implement \mynameref{def:category} from
  \cref{fig:pl_example}.

  The function \mintinline{haskell}{isEven}
  converts\mintinline{haskell}{Int} type
  into Bool.
  \begin{minted}{haskell}
    isEven :: Int -> Bool
    isEven x = x `mod` 2 == 0
  \end{minted}

  There is also \mynameref{def:id} that is defined as follows
  \begin{minted}{haskell}
    id :: a -> a
    id x = x
  \end{minted}

  If we have an additional function
  \begin{minted}{haskell}
    stringLength :: String -> Int
    stringLength x = length x
  \end{minted}
  then we can create a \mynameref{axm:composition}
  \begin{minted}{haskell}
    isStringLengthEven :: String -> Bool
    isStringLengthEven = isEven . stringLength
  \end{minted}

  %% If we consider pure (without effects) functions then
  %% \mynameref{axm:associativity} is also satisfied.
\end{example}

%http://math.andrej.com/2016/08/06/hask-is-not-a-category/
\begin{remark}[Haskell lazy evaluation]
  \label{rem:hask_lazy_eval}
  Each Haskell type has a special value $\bot$. The fact that the
  value and lazy evaluations are part of the language, make several
  category law invalid, for instance
  \mynameref{def:id} behaviour become invalid in specific cases:

  The following code
  \begin{minted}{haskell}
    seq undefined True
  \end{minted}
  produces \textit{undefined}
  But the following
  \begin{minted}{haskell}
    seq (id.undefined) True
    seq (undefined.id) True
  \end{minted}
  produces \textit{True} in both cases.
  As result we have
  (we cannot compare compare functions in Haskell, but if we
  could we can get the following)
  \begin{minted}{haskell}
    id . undefined /= undefined
    undefined . id /= undefined,
  \end{minted}
  i.e. \eqref{eq:leftid} and
  \eqref{eq:rightid} are not satisfied.
\end{remark}

\subsection{\textbf{C++} toy category}
\begin{example}[\textbf{C++} toy category]
  \label{ex:cppcategory}
  \index{Object!\textbf{C++} example}
  \index{Morphism!\textbf{C++} example}
  We will use the same trick as in \mynameref{ex:haskcategory} and
  will assume
  types in C++ as \mynameref{def:object}s,
  functions as \mynameref{def:morphism}s.
  We also are going to implement
  \mynameref{def:category} from \cref{fig:pl_example}.

  We  also define 2 functions:
  \begin{minted}{c++}
    auto isEven = [](int x) {
      return x % 2 == 0;
    };

    auto stringLength = [](std::string s) {
      return static_cast<int>(s.size());
    };
  \end{minted}

  Composition can be defined as follows:
  \begin{minted}{c++}
    // h = g . f
    template <typename A, typename B>
    auto compose(A g, B f) {
      auto h = [f, g](auto a) {
        auto b = f(a);
        auto c = g(b);
        return c;
      };
      return h;
    };
  \end{minted}

  The \mynameref{def:id}:
  \begin{minted}{c++}
    auto id = [](auto x) { return x; };
  \end{minted}

  The usage examples are the following:
  \begin{minted}{c++}
    auto isStringLengthEven = compose<>(isEven, stringLength);

    auto isStringLengthEvenL = compose<>(id, isStringLengthEven);

    auto isStringLengthEvenR = compose<>(isStringLengthEven, id);
  \end{minted}

  Such construction will always provides us the category as soon as we
  use pure function (functions without effects).
\end{example}

\subsection{\textbf{Scala} toy category}
\begin{example}[\textbf{Scala} toy category]
  \label{ex:scalacategory}
  \index{Object!\textbf{Scala} example}
  \index{Morphism!\textbf{Scala} example}

  We will use the same trick as in \mynameref{ex:haskcategory} and
  will assume
  types in Scala as \mynameref{def:object}s,
  functions as \mynameref{def:morphism}s.
  We also are going to implement
  \mynameref{def:category} from \cref{fig:pl_example}.

  \begin{minted}{scala}
    object Category {
      def id[A]: A => A = a => a
      def compose[A, B, C](g: B => C, f: A => B):
          A => C = g compose f

      val isEven = (i: Int) => i % 2  == 0
      val stringLength = (s: String) => s.length
      val isStringLengthEven = (s: String) =>
          compose(isEven, stringLength)(s)
    }
  \end{minted}

  The usage example is below
  \begin{minted}{scala}

    class CategorySpec extends Properties("Category") {
      import Category._
      import Prop.forAll

      property("composition") = forAll { (s: String) =>
        isStringLengthEven(s)  == isEven(stringLength(s))
      }

      property("right id") = forAll { (i: Int) =>
        isEven(i)  == compose(isEven, id[Int])(i)
      }

      property("left id") = forAll { (i: Int) =>
        isEven(i)  == compose(id[Boolean], isEven)(i)
      }
    }
  \end{minted}
\end{example}

\section{Quantum mechanics examples}
The most critical property of quantum system is the superposition
principle. The \mynameref{def:setcategory} cannot be used for it
because it does not satisfy the principle. but a simple modification
of the \textbf{Set} category does.
\begin{definition}[\textbf{Rel} category]
  \label{def:relcategory}
  \index{Object!\textbf{Rel} category}
  \index{Morphism!\textbf{Rel} category}
  We will consider a set of sets (same as \mynameref{def:setcategory})
  i.e. \mynameref{def:set}s as \mynameref{def:object}s. Instead of
  \mynameref{def:function}s we will use \mynameref{def:binary_relation}s as
  \mynameref{def:morphism}s.

  The \textbf{Rel} category is similar to the finite dimensional
  Hilber space especially because it assumes some kind of superposition.
  Really consider $\cat{Rel}$ - the \textbf{Rel} category. $X, Y \in
  \catob{Rel}$ - 2 sets which consists of different elements. Let $f: X
  \to X$ - \mynameref{def:morphism}. Each element $x \in X$ is
  mapped to a subset $Y' \subset Y$. The $Y'$ can be
  \mynameref{def:singleton_set}  (in this case no differences with
  \mynameref{def:setcategory}) but there can be a situation when $Y'$
  consists of several elements. In the case we will get some kind of
  superposition that is analogiest to quantum systems.
\end{definition}

In the quantum mechanics we say about Hilber spaces.
\begin{definition}[Hilbert space]
  \label{def:hilbert_space} The Hilbert space is a complex vector space
  with an inner product as a complex number ($\mathbb{C}$).

  Later we will consider only finite dimensional Hilber spaces.
  We will denote a Hilbert space of dimensional $n$ as
  $\mathcal{H}_n$. Obviously $\mathcal{H}_1 = \mathbb{C}$.
\end{definition}

\begin{definition}[Dual space]
\label{def:dual_space}
Each Hilber space $\mathcal{H}$ has an associated with it dual space
$\mathcal{H}^\ast$ that consists of linear functionals
\end{definition}

\begin{example}[Dirac notation]
\label{ex:dirac_notation}
Consider a ket-vector $\ket{\psi} \in \mathcal{H}$. Then the
corresponding vector from \mynameref{def:dual_space} is called
bra-vector $\bra{\psi} \in \mathcal{H}^\ast$. From the definition of
dual space the bra-vector is a linear functional i.e.
\[
\bra{\psi} : \mathcal{H} \to \mathbb{C},
\]
$\forall \ket{\phi} \in \mathcal{H}$ we have
\(
\bra{\psi}\left(\ket{\phi}\right) = \left(\ket{\psi}, \ket{\phi}\right)
\) - inner product that is often written as $\bra{\psi}\ket{\phi}$.
\end{example}

The
transformation between 2 \mynameref{def:hilbert_space}s that preserves
the structure is called
linear map or linear transformations.
\begin{definition}[Linear map]
\label{def:linear_map}
The linear map between2 \mynameref{def:hilbert_space}s $\mathcal{A}$
and $\mathcal{B}$ is a mapping $f: \mathcal{A} \to \mathcal{B}$ that
preserves additions
\[
f(a_1 + a_2) = f(a_1) + f(a_2),
\]
and scalar multiplications:
\[
f(c \cdot a) = c \cdot f(a)
\]
where $a,a_{1,2} \in \mathcal{A}$ and $f(a), f(a_{1,2}) \in \mathcal{B}$.
\end{definition}

\begin{remark}[Linear map]
\label{rem:linear_map}
Note that \mynameref{def:linear_map} does not preserve inner product.
TBD (verify the statement ???)
\end{remark}

If we want to combine 2 Hilbert spaces into one we use a notion of
direct sum.
\begin{definition}[Direct sum of Hilber spaces]
  \label{def:fdhilb_direct_sum}
  Let $\mathcal{A}, \mathcal{B}$ are 2 Hilber spaces. The
  direct sum $\mathcal{A} \oplus \mathcal{B}$ is defined as follows
  \[
  \mathcal{A} \oplus \mathcal{B} = \{a \oplus b | a \in \mathcal{A}, b
  \in \mathcal{B}\}.
  \]
  The inner product is defined as follows
  \[
  \bra{a_1 \oplus b_1}\ket{a_2 \oplus b_2} =
  \bra{a_1}\ket{a_2} + \bra{b_1}\ket{b_2}.
  \]
\end{definition}

\begin{definition}[\textbf{FdHilb} category]
  \label{def:fdhilbcategory}
  \index{Object!\textbf{FdHilb} category}
  \index{Morphism!\textbf{FdHilb} category}

  Most common case in quantum mechanics is the case of quantum states
  in the finite dimensional Hilbert space. We can consider the set of
  all finite dimensional Hilbert spaces as a category. The
  \mynameref{def:object}s in the category are finite dimensional
  \mynameref{def:hilbert_space}s and \mynameref{def:morphism}s are
  \mynameref{def:linear_map}s. The category is denoted as
  \textbf{FdHilb}. It is very similar to   \mynameref{def:relcategory}.
  The brief relation is   described in the
  \cref{tab:set_vs_rel_vs_fdhilb}.
  \begin{table}
    \centering
    \caption{Relations between \textbf{Set}, \textbf{Rel} and \textbf{FdHilb} categories}
    \label{tab:set_vs_rel_vs_fdhilb}
    \begin{adjustbox}{width=1\textwidth}
      \small
      \begin{tabular}{l|l|l|l}
        \toprule
        & \textbf{Set} & \textbf{Rel} & \textbf{FdHilb}\\
        \midrule
        \mynameref{def:object} & \mynameref{def:set} &
        \mynameref{def:set} &
        finite dimensional \mynameref{def:hilbert_space}\\
        \mynameref{def:morphism} & \mynameref{def:function} &
          \mynameref{def:binary_relation} &
          \mynameref{def:linear_map}\\
          \mynameref{def:initial_object} & empty set & empty set & trivial
          \mynameref{def:hilbert_space} of dimensional 0 \\
          \mynameref{def:terminal_object} & \mynameref{def:singleton_set} &
          \mynameref{def:singleton_set} & $\mathbb{C}$ \\
          \mynameref{def:product} & \mynameref{def:cartesian_product} &
          \mynameref{def:cartesian_product}& \mynameref{def:fdhilb_direct_sum} \\
          \mynameref{def:sum} & \mynameref{ex:set_sum} &
          \mynameref{ex:set_sum} & \mynameref{def:fdhilb_direct_sum} \\
          \bottomrule
      \end{tabular}
    \end{adjustbox}
  \end{table}
\end{definition}

\begin{example}[Rabi oscillations]
  \label{ex:rabioscillations}
  For our example we consider a 2 level atom with states $\ket{a}$ -
  excited and $\ket{b}$ - ground.
  As soon as we consider a 2-level system we are in the 2 dimensional
  \mynameref{def:hilbert_space} i.e. have only one
  \mynameref{def:object}. Lets call
  it as $\ket{\psi}$. The category in the example will be called as
  $\cat{Rabi}$. I.e. $\catob{Rabi} = \mathcal{H}_2
  \{\ket{\psi}\}$.

  The atom interacts with light beam of
  frequency $\omega = \omega_{ab}$. The state of the system is
  described by the following equation \cite{bib:quantum_optics_mine}:
  \begin{equation}
    \ket{\psi} = \cos{\frac{\omega_R t}{2}} \ket{a} -
      i \sin{\frac{\omega_R t}{2}} \ket{b},
    \nonumber
  \end{equation}
  where $\omega_R$ - Rabi frequency \cite{bib:quantum_optics_mine}.

  The interaction time $t$ is fixed and corresponds to $\omega_R t =
  \pi$ i.e. the interaction can be described a linear operator $\hat{L}$.

  There are 4 different states and as result 4
  \mynameref{def:morphism}s:
  \begin{eqnarray}
    \ket{\psi}_0 = \ket{a},
    \nonumber \\
    \ket{\psi}_1 = \hat{L} \ket{\psi}_0 = -i \ket{b},
    \nonumber \\
    \ket{\psi}_2 = \hat{L}^2 \ket{\psi}_0 = - \ket{a},
    \nonumber \\
    \ket{\psi}_3 = \hat{L}^3 \ket{\psi}_0 = i \ket{b},
    \nonumber
  \end{eqnarray}

\begin{figure}
  \centering
  \begin{tikzpicture}[ele/.style={fill=black,circle,minimum
        width=.8pt,inner sep=1pt},every fit/.style={ellipse,draw,inner
        sep=-2pt}]

    % the texts
    \node[ele,label=right:$\ket{\psi}$] (0) at (2,2) {};

    \draw (0) to [out=45,in=135,looseness=50] node[pos=0.5,above]
          {$\hat{L}$} (0);
    \draw (0) to [out=45,in=135,looseness=100] node[pos=0.5,above]
          {$\hat{L}^2$} (0);
    \draw (0) to [out=45,in=135,looseness=150] node[pos=0.5,above]
          {$\hat{L}^3$} (0);
    \draw (0) to [out=45,in=135,looseness=200] node[pos=0.5,above]
          {$\idm{\ket{\psi}} = \hat{L}^4$} (0);

  \end{tikzpicture}
  \caption{Rabi oscillations as a category $\cat{Rabi}$}
  \label{fig:example_quantum}
\end{figure}
\end{example}


\chapter{Measurement}

\paragraph{Quantity, magnitude, and unit}

The quantity 1 kg has magnitude 1 and unit kg (kilogram).
The magnitude is a number.
The quantity 2 kg is twice 1 kg.
The quantity 10 kg is ten times 1 kg.

\section{SI units and prefixes}

SI stands for \emph{syst\`eme international d'unit\'es}
(international system of units).

The units are grouped into two kinds: base units and derived units.
Base units don't depend on other units.
Derived units depend on other units.
Example base units are kilogram (kg), meter (m), and second (s).
Example derived units are newton (N, which is \si{kg.m.s^{-2}})
and joule (J, which is \si{N.m}).

A prefix can be put before a unit to enlarge or shrink it.
Example prefixes are
\si{\micro} (micro, \(10^{-6}\)),
m (milli, \(10^{-3}\)),
k (kilo, \(10^3\)),
M (mega, \(10^6\)).
The quantities \SI{1}{kg}, \SI{1000}{g}, and \SI{1000000}{mg} are the same quantity.

Some units are rarely used.
An example rare unit is megagram (1 million grams).

For the complete list of SI units and prefixes, search the Internet%
\footnote{\url{https://en.wikipedia.org/wiki/International_System_of_Units\#Units_and_prefixes}}%
\footnote{\url{https://physics.nist.gov/cuu/Units/units.html}}%
.

\paragraph{A psychological effect}
Someone sounds heavier in grams than in kilograms.
For example, one friend of mine weighs one hundred \emph{thousand} grams.
We compare numbers more easily than we compare units.

\section{Writing numbers in scientific notation}

% https://en.wikipedia.org/wiki/Significant_figures
An example number in scientific notation is \( 1.23 \times 10^{50} \).
Without scientific notation, we would need to write out that number with its 47 trailing zeros.

The number \( 1.23 \times 10^{50} \) has three significant figures.

\section{Reporting a measurement and its uncertainty}

How do we measure quantities?
How do report a measurement?

Every measurement has an uncertainty.
Tools have limited precision.
We report a measurement by writing \SI{1.00(5)}{cm} or \SI[separate-uncertainty]{1.00+-0.05}{cm}
to mean that the actual quantity is somewhere between 0.95 cm and 1.05 cm.
We don't know the actual quantity.
We only know that it's between those.

\section{Sanity check with dimensional analysis}

Dimensional analysis can sanity-check a calculation.
If the unit is wrong, the calculation is wrong.
If the unit is right, the calculation may be right.

Dimensional analysis is a test.
It can detect falsehood.
It can't prove correctness.


\section{Distance Metric}
\label{sec:dist}

We represent each context with a high dimensional probability vector
called the substitute vector as described in the previous section.  In
this section we compare various distance metrics in this high
dimensional space with the goal of discovering one that will judge
vectors that belong to the same syntactic category similar and vectors
that belong to different syntactic categories distant.  The distance
metrics we have considered are listed in Table~\ref{tab:metrics}.

\begin{table}[ht] \centering
\small
\begin{tabular}{|lll|}
\hline
Cosine($\mathbf{p}, \mathbf{q}$) & = & $<\mathbf{p},\mathbf{q}> / (\|\mathbf{p}\|_{2} \|\mathbf{q}\|_{2})$ \\
Euclid($\mathbf{p}, \mathbf{q}$) & = & $\|\mathbf{p} - \mathbf{q}\|_{2}$ \\
Manhattan($\mathbf{p}, \mathbf{q}$) & = & $\|\mathbf{p} - \mathbf{q}\|_{1}$ \\
Maximum($\mathbf{p}, \mathbf{q}$) & = & $\|\mathbf{p} - \mathbf{q}\|_{\infty}$ \\
KL2($\mathbf{p}, \mathbf{q}$) & = & $\sum_i p_iln(p_i/q_i) + q_iln(q_i/p_i) $\\
JS($\mathbf{p}, \mathbf{q}$) & = & $\sum_i p_iln(p_i/m_i) + q_iln(q_i/m_i) $\\
& & where $m_i = (p_i + q_i) / 2$\\
\hline
\end{tabular}
\caption{Similarity metrics.  JS is the Jensen-Shannon divergence and
  KL2 is a symmetric implementation of Kullback-Leibler divergence.}
\label{tab:metrics}
\end{table}

To judge the merit of each distance metric we obtained supervised
baseline scores using leave-one-out cross validation and the weighted
k-nearest-neighbor algorithm\footnote{Neighbors were weighted using
  1/distance, $k=30$ was chosen empirically.} on the gold tags of the
test corpus.  The results are listed in Table~\ref{tab:distscores}
sorted by score.

\begin{table}[ht] \centering
\begin{tabular}{|l|c|}
\hline
Metric & Accuracy(\%) \\
\hline
KL2 & 0.6889 \\
Manhattan & 0.6865 \\
Jensen & 0.6801 \\
Cosine & 0.6706 \\
Maximum & 0.6663 \\
Euclid & 0.6255 \\
lg2-Maximum & 0.5361 \\
lg2-Cosine & 0.4847 \\
lg2-Euclid & 0.4038 \\
lg2-Manhattan & 0.3729 \\
\hline
\end{tabular}
\caption{Supervised baseline scores with different distance metrics.
  Log-metric indicates that metric applied to the log of the
  probability vectors.}
\label{tab:distscores}
\end{table}

% K=30
% KL2 0.688884263114072
% Manhattan 0.686511240632806
% Jensen 0.680099916736053
% Cosine 0.670566194837635
% Maximum 0.666278101582015
% Euclid 0.625478767693589
% lg2-Maximum 0.536136552872606
% lg2-Cosine 0.484721065778518
% lg2-Euclid 0.403788509575354
% lg2-Manhattan 0.37285595337219

% K=20
% KL2 & 68.95\\
% Manhattan & 68.75\\
% Jensen & 68.43\\
% Cosine & 67.45\\
% Maximum	& 66.55\\
% Euclid & 63.33\\
% log-Maximum & 54.20\\
% log-Cosine & 49.33\\
% log-Euclid & 41.21\\
% log-Manhattan & 37.59\\

The entries with the log- prefix indicate a metric applied to the log
of the probability vectors.  Distance metrics on log probability
vectors performed poorly compared to their regular counterparts
indicating differences in low probability words are relatively
unimportant and high probability substitutes determine syntactic
category.  The surprisingly good result achieved by the simple Maximum
metric (which identifies the dimension with the largest difference
between two vectors) also support this conclusion.  The maximum score
of 69\% can be taken as a rough upper bound for an unsupervised
learner using this space on the 45-tag 24K test corpus because 31\% of
the instances are assigned to the wrong part of speech by the majority
of their closest neighbors.  We will discuss ways to push this upper
bound higher by including other features in
Section~\ref{sec:sparsity}.

% moreover we are using probability vectors so its more natural to use
% kl2


\section{Root: Roots of Functions}
\subsection{Quadratics}
\subsection*{quadreal, quadcmpx}
These address $ax^2+bx+c=0$ for real a,b,c and for complex a,b,c.
Internally, quadreal calls quadcmpx if the discriminant is less than zero.

We follow the analysis in NR92 pg 184.  However, we add generation of
alpha and beta terms via:
\begin{eqnarray}
  x_1    & = & \alpha + \beta\\
  x_2    & = & \alpha - \beta \mbox{ therefore, }\\
  \alpha & = & \frac{x_1+x_2}{2}\\
  \beta  & = & \frac{x_1-x_2}{2}
\end{eqnarray}

[HGG: Initially I couldn't find a way to extract $\alpha$ and
$\beta$ from the NR92 approach.  I even implemented quadreal in
EXTENDED reals (80-bit), to cover potential truncations.  Finally I went
back and looked for an extraction.  The problem of course was that
I was initially looking for $\alpha$ and $\beta$ as sources for
$x_1$ and $x_2$, not as results.]

\subsection{Nonlinear Functions}
In one dimension, root finding means finding a value x such
that $f(x) = 0$.  Generally you need a clue to get started,
e.g., from examining a graph.  Examination allows selection
of a bracketing pair, x1 and x2, which straddles the x-axis
(and thus straddles at least one root).  NR92 argues
persuasively that you should {\em always} bracket the root
before applying a numerical technique.

That brings up the next question.  How can you recognize a
bracket pair?  f(x1) will have the opposite sign from f(x2).
Here are some approaches:
\begin{verbatim}
     Given: y1:=f(x1); y2:=f(x2);

     a) IF (y1>0.0 AND y2<0.0) OR (y1<0.0 AND y2>0.0) THEN (*bracketed*)

     b) IF y1*y2<0.0 THEN (*bracketed*)

     c) IF sgn(y1) = -sgn(y2) THEN (*bracketed*)
\end{verbatim}

I haven't timed these out, but on the basis of short-circuit
evaulation of relationals, and on the basis of no procedure
calls and no multiplies, "a" should be the best.  I'll use
it in the following routines.

Given an arbitrary pair, we can reach out further and
further trying to get a bracket, or we can close the gap
narrower and narrower.  Both can be useful.

\subsection*{bracket\_out}
Inspired by NR92's zbrac.  The idea is to start with two
points, expand by the golden ratio iteratively, and see if
there comes a time when the y's are of opposite signs.

NR92 uses 1.6 as the growth factor.  Just to be different,
I'll use the Golden constant.  Also,
I'll require $x1<x2$.  NR92's algorithm actually works for
$x2<x1$ also, but it isn't as obvious.  I think the slight
reduction in generality is more than paid back in
readability.

\subsection*{bracket\_in}
Inspired by NR92's zbrak.  In testing, I noticed that when
the segments just happen to line up exactly on a root, they
miss it.  So doing different ranges is a good idea,
or doing n's which are not multiples of one another.  [HGG: All
things considered, I like this routine better than
bracket\_out, but they can work together.]

\subsection*{root\_bisect}
Inspired by NR92's rtbis.  Also covered by Hopk88, pg 67.

The basic trick is to note that the segment sizes get
smaller by 1/2 every time, so we can just do:
\begin{verbatim}
     h:=x2-x1;
     ...
     h:=0.5*h;
\end{verbatim}

The exit criterion is:
\begin{verbatim}
     IF h<tol THEN RETURN x; END;
\end{verbatim}

Each iteration requires a decision on where to go next,
based on y values.  But to do this, we need to know which
direction in x corresponds to which direction in y.  So we
need to orient the function.  NR92 does $f>0$ at $x+dx$.  Just
to be different, I'll do the opposite:
\begin{verbatim}
     y:=func(x1);
     IF y>0.0 THEN
       x:=x2; h:=x1-x2;
     ELSE
       x:=x1; h:=x2-x1;
     END;
     (*initialize*)
     h:=h*0.5; x:=x+h;
     FOR i:=1 TO maxiter DO
       y:=func(x);
       IF y<0 THEN
         x:=x+h;
       ELSE
         x:=x-h;
       END;
       IF h<xacc THEN RETURN x; END;
       h:=h*0.5;
     END;
\end{verbatim}

I decided to precalc y1 and y2 in order to do the bracketing
check.  That led to slight modifications in the
initialization.  Next, after running into cases of hitting
the root dead on, I decided to add a check for y=0.  I also
had a nice bug in the $h < {\mbox xacc}$ line.  I was vaguely thinking h
would always be positive, so I skipped doing an ABS(h) for
the comparison with xacc.  That of course failed, because if
func has a negative slope, h is negative.  Putting in ABS
did the job.

\subsection*{root\_brent}
Brent's algorithm is used in the netlib matrix libraries.
The quadratic formula is given in NR92, eqn 9.3.1. \dots 9.3.5.
But NR92 only gives code, not the algorithm.  To demonstrate
derivation from the ideas rather than the raw code, I have
used more descriptive naming than NR92, and changed the use
of temporaries.  I also added quick victory checks at the
start.

The bracket pair $a \dots b$ is always the biggest interval of
interest.  Sometimes we also have a point c which is known
be on a's side but a little closer to b (thus giving a
smaller bracket).  At worst, c is identical to a.  So the
best shot for the next interval is $b-c$, called diffnext.

The hard part is deciding what to do about tolerances.  The
problem is that Brent's algorithm is carefully crafted with
truncation errors in mind.  You can't just go around pulling
temp variables out of loops etc. [HGG: Which I did at one point,
then thought better of it.]

After building the code, I tested with $x1>x2$, $x2>x1$, $x's <0$,
$x's > 0$, and x's straddling 0.  Does just fine as long as
there is only one root in the x1 \dots x2 range.  But I can get
$root=0.0$ or a genuine error if there are $>1$ roots.  I
suspect that is due to confusing the c as it moves back and
forth between a and b.  Maybe I'll look at this more later.

\subsection*{root\_newtraph}
The basic newton-raphson root finder is analyzed by Hopk88,
pg 83 and an algorithm is given in Krey88, pg 953.  NR92
also covers it, pg .  The formula is:
\begin{equation}
  x_{i+1}:=x_i-f(x_i)/f'(x_i)
\end{equation}
Thus, you must be able to provide $f'(x)$.

The various authors agree that this algorithm can go wrong
sometimes.  NR92 addresses that by using Brent's idea of
dropping back to bisection in a pinch.  We will follow their
lead.  The question is, when should you go to bisection?
Given brackets a and b, and r for rootnext:
\begin{verbatim}
     a........r.........b
\end{verbatim}

Then $(a-r)*(r-b)$ should be positive no matrixter which
direction the axis points.  That is, it could be 2 positives
or 2 negatives.  If we get a negative from the multiply, r
is out of bounds.

When we need bisection, we can do: $\mbox{root}:=0.5*(a+b)$.
Again, we don't care whether $a<b$ or $a>b$.  However, to have a
delta for checking exit criteria, we need:
\begin{verbatim}
     tmp:=root;
     root:=0.5*(a+b);
     delta:=root-tmp;
\end{verbatim}

We start with an arbitrary r set at a, and work from there.
To assure that a and b remain bracketing, we need to stick
to a convention.  Let a be such that $f(a)<0$ and b such that
$f(b)>0$.  Then when a new root is formed, find its f, and set
the proper a or b to root.

NR92 also provides tests for converging too slowly.  My
algorithm just relies on bisection to safely drag the
solution to the root.  I have not found conditions which
make this an issue --- if some arise, we'll deal with it
then.


\subsubsection{Dirichlet inversion}
Define the Dirichlet convolution $f*g(n)$ as:

$$f*g(n)=\sum^n_{d=1}[d|n]f(n)g(\frac{n}{d})$$

Assume we are going to calculate some function $S(n)=\sum^n_{i=1}f(i)$,
where $f(n)$ is a multiplicative function.
Say we find some $g(n)$ that is simple to calculate,
and $\sum^n_{i=1}f*g(i)$ can be figured out in $O(1)$ complexity.
Then we have

\begin{equation*}
\begin{split}
\sum^n_{i=1}f*g(i)	&=\sum^n_{i=1}\sum_d[d|i]g(\frac{i}{d})f(d)\\
					&=\sum^n_{\frac{i}{d}=1}\sum^{\floor*{\frac{n}{\frac{i}{d}}}}_{d=1}g(\frac{i}{d})f(d)\\
					&=\sum^n_{i=1}\sum^{\floor*{\frac{n}{i}}}_{d=1}g(i)f(d)\\
					&=g(1)S(n)+\sum^n_{i=2}g(i)S(\floor*{\frac{n}{i}})\\
S(n)				&=\frac{\sum^n_{i=1}f*g(i)-\sum^n_{i=2}g(i)S(\floor*{\frac{n}{i}})}{g(1)}\\
\end{split}
\end{equation*}

It can be proven that $\floor*{\frac{n}{i}}$ has at most $O(\sqrt{n})$ possible values.
Therefore, the calculation of $S(n)$ can be reduced to $O(\sqrt{n})$ calculations of $S(\floor*{\frac{n}{i}})$.
By applying the master theorem, it can be shown that the complexity of such method is $O(n^{\frac{3}{4}})$.

Moreover, since $f(n)$ is multiplicative, we can process the first $n^{\frac{2}{3}}$ elements via linear sieve,
and for the rest of the elements, we apply the method shown above. The complexity can thus be enhanced to $O(n^{\frac{2}{3}})$.

For the prefix sum of Euler's function $S(n)=\sum^n_{i=1}\varphi(i)$, notice that $\sum_{d|n}\varphi(d)=n$.
Hence $\varphi*I=id$. ($I(n)=1,id(n)=n$)
Now let $g(n)=I(n)$, and we have $S(n)=\sum^n_{i=1}i-\sum^n_{i=2}S(\floor*{\frac{n}{i}})$.

For the prefix sum of Mobius function $S(n)=\sum^n_{i=1}\mu(i)$, notice that $\mu*I=(n)\{[n=1]\}$.
Hence $S(n)=1-\sum^n_{i=2}S(\floor*{\frac{n}{i}})$.

Some other convolutions include $(p^k)\{1-p\}*id=I$, $(p^k)\{p^k-p^{k+1}\}*id^2=id$ and $(p^k)\{p^{2k}-p^{2k-2}\}*I=id^2$.

Usage:
\begin{enumerate}
	\item \texttt{CUBEN} should be $N^{\frac{1}{3}}$.
	\item Pass \texttt{p\_f} that returns the prefix sum of $f(x)(1\le x<th)$.
	\item Pass \texttt{p\_g} that returns the prefix sum of $g(x)(0\le x\le N)$.
	\item Pass \texttt{p\_c} that returns the prefix sum of $f*g(x)(0\le x\le N)$.
	\item Pass \texttt{th} as the thereshold, which generally should be $N^{\frac{2}{3}}$.
	\item Pass \texttt{mod} as the module number, \texttt{inv} as the inverse of $g(1)$ regarding \texttt{mod}.
	\item \textbf{Remember that $x$ in \texttt{p\_g(x)} and \texttt{p\_c(x)} may be larger than \texttt{mod}!}
	\item Run \texttt{init(n)} first.
	\item Use \texttt{ans(x)} to fetch answer for $\frac{n}{x}$.
\end{enumerate}

\lstinputlisting{src/mathematics/computation/dirichlet-convolution.cpp}


% !TeX spellcheck = en_US
\documentclass[]{report}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{tikz}
\usepackage{amsmath}
\usepackage{amssymb}

\begin{document}

\section{Full FDTD for the 1D Case}
% TODO: Reference to that
We're starting with the previously [REF] derived six equations that stem from Maxwell. They're listed here again for convenience:
\begin{align}
	\partial_y E_z - \partial_z E_y &= -\mu \partial_t H_x \\
	\partial_z E_x - \partial_x E_z &= -\mu \partial_t H_y \\
	\partial_x E_y - \partial_y E_x &= -\mu \partial_t H_z \\
	\partial_y H_z - \partial_z H_y &= \varepsilon \partial_t E_x + \sigma E_x \\
	\partial_z H_x - \partial_x H_z &= \varepsilon \partial_t E_y + \sigma E_y \\
	\partial_x H_y - \partial_y H_x &= \varepsilon \partial_t E_z + \sigma E_z \text{ .}
\end{align}
The operator \( \partial_x \) is a shorthand for the partial derivative \( \frac{\partial}{\partial x} \).

Again, we're only considering a discretization along one axis (\textit{z-axis}), but now we won't force the conditions a plane wave. Thus spatial derivatives along other axes (\textit{x-axis} and \textit{y-axis}) will be disregarded, but the electric and magnetic field will not be constricted to a single axis. This reasoning leads to the following modified equations:
\begin{align}
	\partial_z E_y &= \mu \partial_t H_x \\
	\partial_z E_x &= -\mu \partial_t H_y \\
	0 &= -\mu \partial_t H_z \\
	- \partial_z H_y &= \varepsilon \partial_t E_x + \sigma E_x \\
	\partial_z H_x  &= \varepsilon \partial_t E_y + \sigma E_y \\
	0 &= \varepsilon \partial_t E_z + \sigma E_z \text{ .}
\end{align}
Note that equations 3 and 6 [REF] can be dropped, since they only state that there is time-variance in the electric and magnetic field along the z-axis. The discretization is done in a staggered manner (\textit{Yee-grid}) as in the case of the plane wave and leads to the equations
% TODO: Add mention of Yee grid to plane wave note
\begin{align}
	\frac{E_y^{n+1/2}(k+1)-E_y^{n+1/2}(k)}{\Delta z} &= \mu \frac{H_x^{n+1}(k+1/2)-H_x^{n}(k+1/2)}{\Delta t} \text{ ,} \\
	\frac{E_x^{n+1/2}(k+1)-E_x^{n+1/2}(k)}{\Delta z} &= -\mu \frac{H_y^{n+1}(k+1/2)-H_y^{n}(k+1/2)}{\Delta t} \text{ ,} \\
	-\frac{H_y^{n}(k+1/2)-H_y^{n}(k-1/2)}{\Delta z} &= \varepsilon \frac{E_x^{n+1/2}(k)-E_x^{n-1/2}(k)}{\Delta t} + \sigma E_x^n(k) \text{ ,} \\
	\frac{H_x^{n}(k+1/2)-H_x^{n}(k-1/2)}{\Delta z} &= \varepsilon \frac{E_y^{n+1/2}(k)-E_y^{n-1/2}(k)}{\Delta t} + \sigma E_y^n(k) \text{ ,}
\end{align}
where a new unknown variable \( E_x^n(k) \) appears -- originally -- due to the contribution of the current density term in Ampere's circuital law [REF]. This new unknown variable can be eliminated by averaging between two time steps as in
% TODO: Ref
\begin{equation}
	E_x^n(k) = \frac{E_x^{n+1/2}(k)-E_x^{n-1/2}(k)}{2} \text{ .}
\end{equation}

Rearranging gives the iterative algorithm:
\begin{align}
	H_x^{n+1}(k+1/2) &= H_x^{n}(k+1/2) + \frac{\Delta t}{\mu \Delta z}\left( E_y^{n+1/2}(k+1)-E_y^{n+1/2}(k) \right) \\
	H_y^{n+1}(k+1/2) &= H_y^{n}(k+1/2) - \frac{\Delta t}{\mu \Delta z}\left( E_x^{n+1/2}(k+1)-E_x^{n+1/2}(k) \right) \\
	E_x^{n+1/2}(k) &= E_x^{n-1/2}(k) - \frac{\Delta t}{\varepsilon \Delta z + \frac{1}{2}\sigma \Delta z \Delta t} \left( H_y^{n}(k+1/2)-H_y^{n}(k-1/2) \right) \\
	E_y^{n+1/2}(k) &= E_y^{n-1/2}(k) + \frac{\Delta t}{\varepsilon \Delta z + \frac{1}{2}\sigma \Delta z \Delta t} \left( H_x^{n}(k+1/2)-H_x^{n}(k-1/2) \right) \text{ .}
\end{align}
% TODO: Refs
In each time step, first [20] and [21] are determined for the electric field, then the magnetic field is determined with [18] and [19]. For numerical stability, the CFL condition
\begin{equation}
	\Delta t \leq \frac{\Delta z}{c}
\end{equation}
has to also be fulfilled. For this case we arrive at the following factors
\begin{align}
	\frac{\Delta t}{\mu \Delta z} &= \frac{1}{Z_0 \mu_r} \\
	\frac{\Delta t}{\varepsilon \Delta z + \frac{1}{2}\sigma \Delta z \Delta t} &= \frac{\frac{\Delta z}{c}}{\varepsilon \Delta z + \frac{1}{2}\sigma \Delta z \frac{\Delta z}{c}} = \frac{Z_0}{\varepsilon_r + \frac{1}{2} Z_0 \sigma \Delta z}
\end{align}
for the spatial derivative terms in 18-21.
% TODO: Refs

\section{Simulations}

\end{document}


\subsection{Profit}

The profit of a firm is the difference between revenue and costs.

\(\pi = pq-c\)

Where \(q\) is the amount producted, and \(p\) is the price, and \(c\) is a function of production.

\subsection{Maximising profit}

\(\pi = pq-c\)

The firm's production \(q\) affects the market price \(p\).

\(\dfrac{\delta \pi }{\delta q}= \dfrac{\delta }{\delta q} [pq-c]\)

\(\dfrac{\delta \pi }{\delta q}= p+q\dfrac{\delta p}{\delta q}-\dfrac{\delta c}{\delta q}\)

The firm chooses \(Q\) to maximise profits.

\(p+q\dfrac{\delta p}{\delta q}=\dfrac{\delta c}{\delta q}\)

The right side is marginal costs (MC), the left is marginal revenue.

\(p[1+\dfrac{q}{p}\dfrac{\delta p}{\delta q}]=MC\)

We know that the price elasticity of demand is: \(\epsilon = \dfrac{p}{q}\dfrac{\delta q}{\delta p}\)

So we have:

\(p[1+\dfrac{1 }{\epsilon }]=MC\)

\(p=\dfrac{\epsilon }{1+\epsilon }MC\)

\subsection{Intensive and extensive margins}

\(revenue = pq\)

\(MR=p +q\dfrac{\delta p}{\delta q}\)

\(p\) is the extensive margin.

\(q\dfrac{\delta p}{\delta q}\) is the (negative) intensive margin.

monopoly pricing. when lower prices, gain money on extensive margin. lose money on intensive margin.


\chapter{Appendix Statistics}

\section*{Concepts from Mathematical Statistics}The sampling distribution of an estimator is the probability distribution of the estimator under repeated sampling.  The standard error of a measurement is essentially the standard deviation of the process by which the measurement was generated.  When the underlying probability distribution of the generating process is known the standard error can be used to calculate confidence intervals.  Otherwise Chebyshev's inequality can be used. The standard error of a sample from a population is the standard deviation of the sampling distribution and may be estimated as $\frac{\sigma}{\sqrt{n}}$

Completeness is the ability of a statistical estimator to ensures that the parameters of the underlying probability distribution representing the model can be estimated from the statistic.

Consider a family of probability distributions for a random variable $X$ parameterized by a parameter $\theta$. A quantity $T(X)$ that depends on the random variable $X$ but not on the parameter $\theta$ is called a statistic.  A statistic that captures all of the information in X that is relevant to the estimation of $\theta$ is called a sufficient statistic. Since the conditional distribution of $X$ given $T(X)$ does not depend on $\theta$, neither does the conditional expected value of $g(X)$ given $T(X)$. The conditional expected value is itself a statistic and so is available for use in estimation. If $g(X)$ is an estimator of $\theta$, then typically the conditional expectation of $g(X)$ given $T(X)$ is a better estimator of $\theta$. The Rao-Blackwell theorem makes this precise.

\section*{L-Estimators \& M-Estimators}Order statistics of a sample ${X_i}$ can be used to estimate quantiles. An L statistic is a linear combination of an order statistic.  L statistics are used in Box plots.  An M-Estimator is the minima of sums of functions of the sample data.  Maximum Likelihood and Least Squares are examples of M-Estimators.

\section*{Testing for normality and other distributions } Powerful inference methods can be employed when data is generated by a Gaussian process. This section describes techniques for testing the normality of a sample and comparing two samples.  Kolmogorov-Smirnov test uses the fact that the empirical cumulative distribution function is normal in the limit. It is a non-parametric and distribution free test. Given the empirical distribution
 \[F_n(x) = \frac{1}{n} \sum\limits_{n}^{i=1} \biggl\{\begin{array}{c}1 :x_i\leq x \\0 : x_i>x\\ \end{array}\]
 , and a test CDF\[ F(x)\] the K-S test statistics are $D_n^+ = max(F_n(x)-F(x) ) $ and $D_n^- = min(F_n(x)-F(x) )$ The generality of this test comes at a loss in precision near the tails of a distribution.  The K-S statistics are more sensitive near points close to the median, and are only valid for continuous distributions.  The Kuipers test uses the statistic $D_n^+ + D_n^- $ and is useful for detecting changes in time series since the statistic is invariant in ???? transformation of the dependent variable $ F_n$.  The Anderson-Darling test is based on the K-S test and uses the specific distribution to specify the ????critical values??? of the test.  The chi-squared is based on the sample histogram and allows comparison against a discrete distribution, but has the potential drawback of being sensitive to how the histogram is binned and requires more samples to be valid.  The Shapiro-Wilk test uses the expected values of the order statistics of $F(x)$ to calculate the test statistic.  It is sensitive to data that are very close together, and numerical implementations may suffer from a loss of accuracy for large sample sizes.

 K-S [Chakravarti, Laha, and Roy, (1967). Handbook of Methods of Applied Statistics, Volume I, John Wiley and Sons, pp. 392-394].  Shapiro-Wilk [Shapiro, S. S. and Wilk, M. B. (1965). "An analysis of variance test for normality (complete samples)", Biometrika, 52, 3 and 4, pages 591-611.]

\section*{Regression Methods}Standard least squares regression consists in fitting a line through the data points (training points in learning theory) that minimizes the sum of square residuals.  The underlying assumption is that the data and the response can be modeled by a linear relationship.  In the event that the model accurately captures the functional dependence of the response generated by the data, and under the assumptions that the data is corrupted by Gaussian noise, precise statistical inferences can be made on the model parameters.  Modifications to this standard model include nonlinear mapping of the input data, local fitting, biased estimators, subset selection, coefficient shrinking, weighted least squares, and basis expansion transformations.

The multiple regression model in matrix notation can be expressed
as
(BBCREVISIT)

These equations hold for the univariate and the multivariate case.

$\textbf{Y} = \textbf{X}{\beta} + \textbf{e}$.

where $\textbf{e} =_d N(\mathbf{\mu},\mathbf{\Sigma})$

$E \textbf{Y} = \mathbf{\mu} X \mathbf{\beta}$

$var \textbf{Y} = \sigma^2 \textbf{I}$

Maximum Likelihood and Least Squares give same estimator;

$\widehat{\mathbf{\beta}}=(X^TX)^{-1} X^T Y$

$\hat{\sigma}^2 = frac{1}{n} || Y- \hat{\mu} ||^2$

Multivariate regression is the extension to $YM = Xb + e$

Here $Y, X, b$, and $e$ are as described for the multivariate regression model and $M$ is an $m x s$ matrix of coefficients defining linear transformation of the dependent variables. The normal equations are $X'Xb = X'YM$ and a solution for the normal equations is given by $b = (X'X)-X'YM$. Here the inverse of $X'X$ is a generalized inverse if $X'X$ contains
redundant columns.

\section*{Generalized Linear Models}Suppose we have $n$ observations of $k$ dimensional data denoted $\{x_i\}_{i=1}^{k}$ and for each observation we have a response $y_i$. We wish to fit the observations to the responses. Generalized Linear Regression is a modeling technique that allows for non normal distributions and models non-linear relationships in the training data. M-estimators are used to fit a generalized linear model Ref Huber (1964).
A linear model $ Y =\Lambda(X)=X\beta + \epsilon$ fits a linear relationship between the dependent variables $Y_i$ and the predictor variables $X_i$ \begin{equation}Y_i=\Lambda(X_i)=b_o + b \circ X_i.\end{equation}
A generalized linear model $Y= g(\Lambda(X) ) + \epsilon $ fits the data to $ Y = g (X \circ W)$. Fitting the model consists of minimizing the objective function $\sum\limits_{i=1}^{n} g(e_i)=\sum\limits_{i=1}^{n} g(y_i- x_i \beta)$
, where $e_i$ are the residuals $y_i-x_i \beta$. We see that for ordinary least squares $g(e_i)=e_i^2$, and the usual matrix equations fall out by differentiating with respect to $\beta$. Carrying this out for general $g$ \begin{equation} \sum\limits_{i=1}^{n} \frac{\partial  g(y_i-x_i \beta)}{\partial \beta}=0 \end{equation} gives the system of $k+1$ equations to solve for estimating the coefficients $b_i$.  If we set$\alpha(x)=\frac{g'(x)}{x}$ and calculate the derivative above, we have to solve \begin{equation} \sum\limits_{i=1}^{n} \omega(e_i) (y_i-x_i \beta) x_i = 0 \end{equation}. Which gives rise to a weighted least squares where the weights depend on the residuals - which depend on the coefficients - which depend on the weights.  This suggests an iterative algorithm; \begin{equation} \beta^\tau = ( X^{t} W^{(\tau-1)} X )^{-1} X^{t} W^{\tau-1} y \end{equation} where $W_{ij}^{(\tau-1)}=\alpha(e_{i}^{(\tau-1})$.  Several parameterizations are popular for the exponential family. The most general form of the distribution \[ p(x,\theta) = f(x,\theta)e^{g(x,\theta)} \in C^2(\dblr \otimes \dblr ) \otimes C^2(\dblr \otimes \dblr )\].  The estimators derived below assume that $f$ and $g$ are separable, \[ p(x,\theta) = f(x) h(\theta) e^{\alpha(x) \beta(\theta)} \in C^2(\dblr) \otimes C^2(\dblr ) \otimes C^2(\dblr ) \otimes C^2(\dblr ) \].  From \[ \int\limits_{x=-\infty}^{x=+\infty} p(x,\theta) dx \: =1\] we get \[ \fderiv{\theta}{p(x,\theta)} = 0 = \sderiv{\theta}{p(x,\theta)}d \] Since the parametrization we have chosen for the exponential family allows, in the sequel we drop the notation for dependent variable and denote the derivative with a prime. \[ \fderiv{\theta}{p(x,\theta)} = \fderiv{\theta}{f h e^{\alpha \beta}} = h' f e^{\alpha \beta} + f h \alpha \beta' e^{\alpha \beta} = \bigl( \frac{h'}{h} + \alpha \beta' \bigr) p(x,\theta) \] which gives \[\int \fderiv{\theta}{p(x,\theta)} dx = \int \bigl( \frac{h'}{h} + \alpha \beta' \bigr) p(x,\theta) dx= \frac{h'}{h} \int p(x,\theta) dx \:+\: \beta' \int \alpha(x) p(x,\theta) dx = \frac{h'}{h}+\beta' E[\alpha(x)] \] so that \[E[\alpha(x)]=-\frac{h'}{h \beta'}\]. Continuing along this vein,\begin{gather*} 0 =\int \sderiv{\theta}{p(x,\theta)} dx = \int \fderiv{\theta}{\bigl( \frac{h'}{h} + \alpha \beta' \bigr) p(x,\theta) } dx =\\  \int \bigr(\frac{h''}{h}-\frac{(h')^2}{h^2}+\alpha \beta'' \bigl ) p(x,\theta) + (\frac{h'}{h}+\alpha \beta') \fderiv{\theta}{p(x,\theta)} \: dx = \\ \int \bigr(\frac{h''}{h}-\frac{(h')^2}{h^2}+\alpha \beta'' \bigl ) p(x,\theta) + (\frac{h'}{h}+\alpha \beta')^2 p(x,\theta) \: dx = \\ \int \bigr(\frac{h''}{h}-\frac{(h')^2}{h^2}+\alpha \beta'' \bigl ) p(x,\theta) + (\frac{h'}{h}+\alpha \beta')^2 p(x,\theta) \: dx = \\ \int \bigr(\frac{h''}{h}-\frac{(h')^2}{h^2}+\alpha \beta'' \bigl ) p(x,\theta) + (\alpha \beta'- E[\alpha(x)]\beta')^2 p(x,\theta) \: dx \end{gather*}  Keeping in mind that  \[ Var[a x ]= E[ (ax-E(ax)^2 ] = a^2 E [ (x-E[x])^2] =a^2 Var[x]  \]  we get the variance via\[ \bigr(\frac{h''}{h}-\frac{(h')^2}{h^2}+E[\alpha(x)] \beta'' \bigl )+ Var[\alpha(x) \beta'(\theta)] = \bigr(\frac{h''}{h}-\frac{(h')^2}{h^2}+E[\alpha(x)] \beta'' \bigl )+ (\beta')^2 Var[\alpha(x)] = 0 \]  The score $U(x)$ is given by \[ U(x)=\pfderiv{\theta}{L(\theta,x)} = \pfderiv{\theta}{\log \: p(x,\theta)}=\pfderiv{\theta}{\bigr( \log h(\theta) + \log f(x) + \alpha(x) \beta(\theta)\bigl) } = \frac{h'}{h} + \alpha \beta'\] so \[E[U(x)]=\beta'E[\alpha(x)] + \frac{h'}{h} =0 \].  The Fisher Information $\mathcal{F}$ is defined \[\mathcal{F}=Var[U(x)] =Var[ \alpha \beta' + \frac{h'}{h}]= Var[ \alpha \beta'] \] So from above we have \[Var[U(x)]= Var[ \alpha \beta'] =\bigr(-\frac{h''}{h}+\frac{(h')^2}{h^2}-E[\alpha(x)] \beta'' \bigl )\].  Now differentiating,  \[ \fderiv{\theta}{U(\theta,x)} = \frac{h''}{h} - \frac{(h')^2}{h^2} + \alpha \beta''\] \[E[U'(\theta,x)]=\frac{h''}{h} - \frac{(h')^2}{h^2} + E[\alpha] \beta'' =  \frac{h''}{h} - \frac{(h')^2}{h^2} -\frac{ \beta'' h'}{\beta'}= - Var[U(x)] \].  Note that if we write the parametrization of the separable exponential family as \[ p(x,\theta) =  e^{\alpha(x) \beta(\theta)+\log(f(x))+\log(h(\theta))}\] then, \[\sderiv{\theta}{\log(h(\theta))}=\fderiv{\theta}{\frac{h'}{h}}=\frac{h''}{h}-\frac{(h')^2}{h^2}\].     (BBCREVISIT - this duplicates and has notation clash with material above).

A general form of the exponential distribution \begin{equation} \rho(x;\theta) = exp( \frac{x \theta - \xi(\theta) }{\sigma} ) \nu ( x) \end{equation} has a log likelihood for a random sample $\{ X_i  \}_{i=1 \hdots N}$ given functionally by \begin{equation} \mathcal{L} (\theta) =  \sum\limits_{i=1}^{N} [ X_i  \theta  - \xi (\theta) + log   ( \nu ( X_i ) ) ] \end{equation} The scale parameter $\sigma$ and $\theta$ are orthogonal parameters in that E [ ] The Generalized Linear model can  $\rho'$ is referred to as a link function in the statistical literature.  If $\rho'(x)= x\field(1)$ and $\epsilon=(\epsilon_1, \hdots ,\epsilon_n)$ are iid $N(\mu,\sigma)$ we have multiple linear regression.  In classification problems or binomial models the logit $\rho'(x)=log(x/(1-x))$ link function is used. The logit is extended to the $k$ category case by \begin{equation}\rho'( x_i | x_j j \neq i)= log ( \frac{x_i}{1- \sum\limits_{j \neq i} x_j})\end{equation}. The posterior probability densities ${p_i(?)}$ bbcrevisit (or $p_i$ the probability of observing class $i$)  of k classes are modeled by linear functions of the input variables $x_i$.

\section*{Fitting the GLM}Iteratively re-weighted least squares (IRLS) is used to for fitting generalized linear models and in finding M-estimators.  The objective function

\begin{equation}
J(\beta^{i+1}) = arg min \sum w_i ( \beta) | y_i - f_i (\beta) |
\end{equation}

is solved iteratively using a Gauss-Newton or Levenberg-Marquardt (LM) algorithm. LM is an iterative technique that finds a local minimum of a function that is expressed as the sum of squares of nonlinear functions. It is a combination of steepest descent and the Gauss-Newton method. When the current solution is far from the minimum the next iterate is in the direction of steepest descent. When the current solution is close to the minimum the next iterate is a Gauss-Newton step.

Linear least-squares estimates can behave badly when the error is not normal.  Outliers can be removed, or accounted for by employing a robust regression that is less sensitive to outliers than least squares.  M-Estimators introduced by Huber generalize maximum likelihood estimation and are less biased and more efficient.  Instead of trying to minimize the log likelihood

\begin{equation}
L(\theta) = \sum - log ( p(x_i, \theta)
\end{equation}

Huber proposed minimizing

\begin{equation}
M(\theta) = \sum  \rho(x_i, \theta)
\end{equation}

where $\rho$ reduces the effect of outliers. Common loss function are the Huber, and Tukey Bisquare.  For $\rho(x) = x^2$ we have the familiar least squares loss.

M estimators arise from the desire to apply Maximum Likelihood Estimators to noisy normal data, and to model more general distributions. They provide a regression that is robust against outliers in the training set, and allow for modeling of non-Gaussian processes. When $\rho$ above is a probability distribution, we are preforming a maximum likelihood estimation.

The Huber function which is a hybrid $L^2$ $L^1$ norm
\begin{equation}
\rho_\eta(e_i)=\biggl\{\begin{array}{cc}
\frac{e_i^2}{2} & |e_i| \leq \eta \\
  \eta |e_i| - \frac{\eta^2}{2} & |e_i| > \eta \\
\end{array}
\end{equation}
The  Tukey Bisquare estimator is given by
\begin{equation}
g_\eta(e_i)=\biggl\{ \begin{array}{cc} \frac{\eta^2}{6} (
1-[1-\frac{e_i}{\eta}_2]^3) & |e_i| \leq \eta \\
\frac{\eta^2}{6} & |e_i| > \eta \\
\end{array}
\end{equation}

Numerical procedures for doing this calculation are the Newton-Raphson method [see the section on root finding below ], and Fisher-Scoring method [ replace $ \frac{\partial^2 \mathcal{L}(\mathbf{\theta})}{\partial \mathbf{\theta} \partial \mathbf{\theta}^{t} }$ with $E[ \frac{\partial^2 \mathcal{L}(\mathbf{\theta})}{\partial \mathbf{\theta} \partial \mathbf{\theta}^{t} }  ]$. For high dimensional data, many models may be fit in an attempt to find the simplest one that can explain the data.  In the language of statistical learning theory, the choice of a norm $\rho$ is tantamount to choosing a loss function. Restricting the admissible functions to the one parameter family of exponential probability distributions defines the capacity via a functional form of the law of large numbers. \cite{Scholkopf B. (2002)}

\section*{Feature Subset Selection (FSS)}The goal of feature selection techniques to to improve the model building process by eliminating features that do not have discriminative power. Algorithms for feature selection either rank features or create subsets of increasing optimality.  FSS should be contrasted with feature extraction techniques such as PCA, LLE, or Laplacian eigenmaps.  The goal of feature extraction is to transform data from a high dimensional space to a low dimensional one while preserving the relevant information.

The statistical approach to feature selection most commonly used is stepwise regression.  Common optimality criteria are FSS schemes the Kolmogorov-Smirnov Test ,the t-test, the f-test, the Wilks Lambda Test and Wilcoxon Rank Sum Test.

It's important to distinguish the FSS process from a data dimension reduction process such as PCA which requires all the original measurements to compute the projection. The better FSS algorithms are recursive

Construct a $p x M$ basis matrix $H^{T}$ and transform feature vector $x' = H^{T} x$.

Generalize to $L^{2}$ with smoothing splines

Smoothing spline $RSS(f,\lambda)= \sum\limits_{i=1}^{N} (y_{i} -f(x_{i}) )^{2} + \lambda \int f''(t)^{2} dt$. where $f \in C^{2}(\field{R} )$ This is minimized in $L^{2}$ the first term measuring closeness of fit, and the second term penalizes curvature. $\lambda \rightarrow 0$ gives any function interpolating the data points ${x_i}_{i  \in {1, ... N} } $ an $\lambda \rightarrow \infty$ constrains $f$ to be linear.

\section*{Longitudinal Data Analysis} Longitudinal data analysis is the observation of multiple subjects over repeated intervals. Binary repeated responses are typically modelled with a marginal or random effects model, which will be made precise below. Marginal Models are a generalization of the GLM presented above for correlated data.  Here, the correlation is inter subject across time.  Statistical analysis of longitudinal data must take into account that serial observations of a subject are likely to be correlated, time may be an explanatory variable, and that missing response data my induce a bias in the results.  Let ${X_{ij}}$ be time varying or fixed covariates for the binary response ${Y_{ij}}$ of subject $i \in {1,...n}$ at time intervals $j \in {t_1,...t_m}$. By convention $X_{ij} \in \field{R} x \field{R^p}$ where the first dimension is the intercept. The marginal model is; $logit (E(Y_{ij} | X_{ij}) ) = X_{ij}^{\dagger} \beta$ and enforces the assumption that the relationship between the covariates and the response is the same for all subjects. Recall that for a binary response, $E(Y_{ij} | X_{ij}) = P(Y_{ij}=1 | X_{ij})$.  The random effects model takes into account that the relationship between the covariates and response varies between subjects; $logit (P(Y_{ij}=1 | X_{ij}) ) = X_{ij}^{\dagger} \beta_i$ If it is know that only a subset of the covariates are involved in the inter-subject variability, we can set $\beta_i= \beta + \beta_i$ and write $logit (P(Y_{ij}=1 | X_{ij}, \beta_i) ) = X_{ij}^{\dagger} \beta + O X_{ij} \beta_i$ Where the kernel of $O : \dblr^n \rightarrow \dblr^{n'}$ is the span of the covariates that do not change between subjects.  If $\lambda_i =_d N(0,\sigma)$ then the difference in the parameter vectors $\beta$ in the two models differ according to $\sigma$.

The GEE method of fitting the marginal model is described in: \cite{Liang, K-Y and Zeger, S. L.(1986)}  The Survival Analysis is a form of longitudinal analysis that takes into consideration the amount of time an observation is made on a subject.  GLM's can be used to fit discrete longitudinal hazard models derived from survival analysis, see  \cite{Prentice and Gloeckler (1978)}.   \cite{Meyer, B.D. (1990)} generalized that approach to account for an unobserved subject heterogeneity.  \cite{Holmen, M (2005)} applied the hazard model of \cite{Prentice, R. and L. Gloeckler (1978)} to the takeover hazard of large firms.  A negative relationship between dual class ownership and value is empirically known, and that relationship can be explained by the lower takeover probability of the dual class firms.  Dual class entities had a higher risk for takeover, but the hazard is lower since these firms use the dual class structure to change the capital structure in a way that allows the controlling shareholders to remain in control by reducing firm value.  The proportional hazards model can be discretized, but it is important to identify whether the process is truly a discrete process.  In that case the link function should be the logit as the Marginal Model above specifies, rather than the log-log function of the discretized proportional model.  The difference is the modelling of a probability transition in the former case versus a rate for the latter case.  Variable selection techniques for longitudinal data are relatively limited and most seem to rely on Wald type tests. Wald tests to include a variable are based on already computed maximum likelihood values. The Rao score test is used to include a covariate in the model building process.  The Wald test calculates \[z^2=\frac{\widehat{\beta}}{stderr}=_d  \chi^2\]  The likelihood ratio statistic for comparing two models $L_0 \in L_1$ \[-2 \frac{L_0}{L_1} =_d \chi^2\] is useful for backward stepwise variable subset selection. The degrees of freedom of the of the statistic is equal to the difference in dimension of the two models.

\section*{Discretization \& Sheppard's Correction}W. Sheppard (1898) Derived an approximate relationship between the moments of a continuous distribution and it's discrete approximation. This provides a transformation to statistical estimators that correct for the binning of continuous data.  As the scale at which datum are collected is increased, the variance of an estimate can become biased.  It is important to assess  bias caused by grouping and to correct it if necessary. The  bias of the approximate maximum likelihood estimator where observations are approximated by interval midpoints $O(w^2)$, where $w$ is the bin width. A Sheppards correction can be used to reduce the bias to order $O(w^3)$,  Signal processing engineers often have to deal with such a quantization effect when designing finite precision systems, image processing being a particularly relevant example. The engineering community typically models the quantization noise $Q=[X]-X$, where $[X]$ is the quantized realization of $X$. One might be tempted to apply a Sheppard's correction to the moments of the quantized data, thinking that $Var(X)<Var([X])$ but it is possible to construct examples where $Q$ and $[X]$ are independent, or where $Cov(X,Q)$ is such that $Var(X)>Var([X])$.  Shepard's correction is limited in that is doesn't apply to the first moment, and the frequencies of the first and last bins need to be low.  Expand $p(x;\theta)$ in a Taylor series and substitute in the Maximum Likelihood equations. \cite{Lindley, D. V. (1950)}  Suppose we have n realizations of iid RV's ${X_1, \hdots , X_n}$ and the data is collected on a discrete grid on the range of $X$ $Ran(X)=\{[y_i-d_i/2,y_i+d_i/2]\}_{i=1}^{i=m}$ where the intervals are centered on the location where a measurement. The realized values ${y_1, \hdots , y_m}$ have probabilities $p_i=\int\limits_{y_i - d_i /2}^{y_i+d_i /2} p(x;\theta) \;\; dx$ Expanding $p(x;\theta)$ in a Taylor series about $y$, $p(x;\theta)= \sum\limits_{i=0}^{\infty} \frac{p^{(i)}(y) }{i!} (x-y)^i$.

\section*{Multidimensional Scaling}Multidimensional scaling (MDS) is an alternative to factor analysis. The aim of MDS and factor analysis is to detect meaningful underlying dimensions that explain similarities or dissimilarities data points. In factor analysis, the similarities between points are expressed via the correlation matrix. With MDS any kind of similarity or dissimilarity matrix may be used.  Given $n$ observations ${x_i}_{i=1}^{n} \in \dblr^k$ and $n^2$ distances $d_{ij}$ between them, MDS looks for $n$ points ${\xi_i}_{i=1}^{n}$ in $dblr^l : l<k$ that preserve the distance relations. When a metric $\rho()$ exists for the similarity measure, gradient descent is used to minimize the MDS functional $S(\xi_1, \ldots , \xi_l)=\biggl( \sum_{i \neq j} d_{ij}-||\xi_i-\xi_j||_{\rho}\biggr)^\frac{1}{2}$.

\section*{Principal Components} For a data set $\textbf{X} \in
M_{(N,m)}(\mathbb{R}) = { x_1, x_2, \ldots x_N | x_i \in
\mathbb{R}^m } $, the first k principal components provided the
best k dimensional linear approximation to that data set.
Formally, we model the data via $f(\theta) = \mu + \textbf{V}_k
\theta | \mu \in \mathbb{R}^m, V_k \in O_{m,k}(\mathbb{R}),
\theta \in \mathbb{R}^k$ so $f(\theta)$ is an affine hyperplane
in $\mathbb{R}^m$

\section*{Evaluating classifier performance} Multi-class problems can be treated simultaneously or broken in to a sequence of two class problems.  Cross validation is used both for classifier parameter tuning and for feature subset selection.  Student-t and ANOVA can be used to evaluate the performance of classifiers against one another.  The Student-t test compares two classifiers, while the ANOVA test can compare multiple classifiers against one another.  Confusion matrices and ROC graphs are commonly employed visualization tools for assessing classifier performance. The rows of a confusion matrix add to the total population for each class, and the columns represent the predicted class.  An ROC curve plots the TP rate against the FP rate. Often a curve in ROC space is drawn using classifier parameters for tuning purposes.

\begin{table}[h]
\begin{tabular}{|c|c|}
  \hline
  % after \\: \hline or \cline{col1-col2} \cline{col3-col4} ...
  TN &  FP \\
  \hline
  FN & TP \\
  \hline
\end{tabular}
\caption{Two class confusion matrix where the proportions are
specified}
\end{table}

Common performance metrics for the two class problem are sensitivity (TP), specificity (TN), precision (the proportion of predicted cases within a class that were correct), and accuracy (the overall proportion of correct predictions). These metric can be extended to more than two classes by defining $A=tr ( C ) / || C ||_{L^\infty}$ where $C$ is the confusion matrix. TP, FN, FP, TN are proportions defined for the two class problem.

\section*{Covariance Matrix Estimation } For numerical stability in regression algorithms, the covariance matrix needs to be positive definite.  An well conditioned estimator for the covariance matrix of a process can be obtained by mixing the sample covariance with the identity matrix.  This is a linear shrinkage estimator based on a modified Frobenius norm for $A \in M_{mn}$  \begin{equation} ||\mathbf{A}||_{\cal{F}}= \sqrt{ \frac{tr (A A^t)}{n}} \end{equation}  Without loss of generality, set $\mu =0$ and let $\widehat{\Sigma} = \alpha \mathbb{I} + \beta \mathbf{S}$ where $\mbf{S}=\frac{\mbf{X}^T \mbf{X}}{n}$ is the sample covariance.  We seek to minimize $E( ||\widehat{\Sigma} - \Sigma||^2)$, but since we don't know the true population covariance matrix, we have to form an approximation.

Many applications in statistics and machine learning require an estimate of the covariance matrix or it's inverse. Generally we avoid taking inverses of matrices in practice for stability reasons. The sample covariance matrix usually performs poorly as a proxy for the underlying covariance matrix.  There is a large literature on this subject; particularly fomr the finance industry where this is an important part of portfolio theory.two major challenges in covariance estimation are the positive-defniteness constraint and the high-dimensionality where the number of parameters grows quadratically in the dimension.

\section*{Testing For Normality}In this section we will use the term $EPDF_X$ to mean the empirical probability density function. There are a variety of univariate tests to help determine which parametric distribution your data belongs to. These fall under the category of Goodness of Fit testing. For a parametric family the null hypothesis $H_o : X=_d p(x| \theta)$ is tested against the alternative that $X$ does not belong to the family $p(x|\theta)$ There are also family of test to determine whether two $EPDF$'s come from the same distribution.  Keep in mind that there are many transformations ( polynomial, logarithmic, \& rational )  to transform data to look more Normal.  The presence of tails and skew will be the most problematic to deal with.

The Anderson-Darling test determines whether a sample comes from a specified distribution. The sample data can is transformed to a uniform distribution and then a uniformity test is then done on the transformed data. The test statistic is compared against pre-computed values for the assumed probability distribution.

The Kolmogorov-Smirnov is non-parametric a form of minimum distance estimation.  It can be used to test a sample against a reference or to compare two samples against each other.  In the one sided case the KS statistic calculates the distance between the $EPDF$ of a sample and a reference.  In the two sided case the distance between the $EPDS$'f of the two samples are calculated.  The KS test is robust to location and shape, making it  Omnibus tests evaluate whether the explained variance in a set of data is significantly greater than the unexplained variance. For example is the F-test in ANOVA. Omnibus tests of normality based on the likelihood ratio outperform the Anderson-Darling test statistic.