%%%%%%%%%%%%%%%%%%%%%%
% Useful Observation %
%%%%%%%%%%%%%%%%%%%%%%

\section{Useful Observations}

\begin{observation}{}{Bounding Measures}

    The following inequalities to bound measures are \emph{always} applicable, for \emph{any} sets $A, B, C \in \CalF$:

    \begin{enumerate}
        \setlength{\parskip}{0em}
        \item ``Dropping a set in an intersection gives an upper bound'' $\Leftrightarrow$ ``Relaxing constraints'':

            \begin{align*}
                \mu(A \cap B) \leq \mu(A).
            \end{align*}
        \item ``Dropping a set in a union gives an lower bound'':

            \begin{align*}
                \mu(A \cup B) \geq \mu(A).
            \end{align*}
        \item ``Adding a set in a union gives an upper bound'' $\Leftrightarrow$ ``Adding constraints'':

            \begin{align*}
                \mu(A \cup B) \leq \mu(A \cup B \cup C).
            \end{align*}
        \item ``Intersections are less than a set and a set is less than a union'':

            \begin{align*}
                \mu(A \cap B) \leq \mu(A) \leq \mu(A \cup B).
            \end{align*}
    \end{enumerate}

\end{observation}

\begin{observation}{}{Adding $\Omega$ by Intersection}

    If you would like to introduce a property to an existing set $A$ to make it easier to work with, for instance easier to bound, you can add an intersection with $\Omega$:

        \begin{align*}
            \mu(A) = \mu(\Omega \cap A).
        \end{align*}

    Then $\Omega$ can be split into the set $B$ that represents the property and $B^C$ that does not have the property, where $\Omega = B \cup B^C$. Then:

        \begin{align*}
            \mu(A) = \mu(\Omega \cap A) = \mu((B \cup B^C) \cap A) = \\
            \mu((B \cup B^C) \cap A) = \mu((B \cap A) \cup (B^C \cap A)).
        \end{align*}

    Using $\sigma$-additivity, we get:

        \begin{align*}
            \mu(A) = \mu(B \cap A) + \mu(B^C \cap A).
        \end{align*}

    Then by the observation on bounding measures, this can be made into an inequality:

        \begin{align*}
            \mu(A) &= \mu(B \cap A) + \mu(B^C \cap A) \\
            &\leq \mu(B \cap A) + \mu(B^C).
        \end{align*}

\end{observation}

\begin{observation}{}{Increasing Sequence of Sets}

    For an \emph{increasing} sequence of sets $(A_n)_{n=1}^{\infty}$ we can define:

        \begin{align*}
            \lim_{n \to \infty} A_n \coloneqq \bigcup_{n=1}^{\infty} A_n
        \end{align*}

\end{observation}

\begin{observation}{}{Decreasing Sequence of Sets}

    For an \emph{decreasing} sequence of sets $(A_n)_{n=1}^{\infty}$ we can define:

        \begin{align*}
            \lim_{n \to \infty} A_n \coloneqq \bigcap_{n=1}^{\infty} A_n
        \end{align*}

\end{observation}

\begin{observation}{}{$\mu$-Almost Everywhere Finite, I}

    If $f: \Omega \to \mathbb{R}$ is $\mu$-a. e. finite, then note that if $A_n \coloneqq \{ |f| \geq n \}$, then $(A_n)_{n=1}^{\infty}$ is a decreasing sequence and so:

        \begin{align*}
            \mu\left(\bigcap_{n=1}^{\infty} A_n\right) = \mu\left(\lim_{n \to \infty} A_n\right) = \mu(|f| = \infty) \\ = 0.
        \end{align*}

\end{observation}

\begin{observation}{}{$\mu$-Almost Everywhere Finite, II}

    If $f: \Omega \to \mathbb{R}$ is $\mu$-a. e. finite, then observe

        \begin{align*}
            \mu(|f| = \infty) = \lim_{R \to \infty} \mu(|f| \geq R) = 0.
        \end{align*}

\end{observation}

\begin{observation}{}{Almost Surely Finite, II}

    If $f: \Omega \to \mathbb{R}$ is a.s. finite, then observe

        \begin{align*}
            P(|f| = \infty) = \lim_{R \to \infty} P(|f| \geq R) = 0. \\
            \iff P(|f| < \infty) = \lim_{R \to \infty} P(|f| < R) = 1.
        \end{align*}

\end{observation}

\begin{observation}{}{Almost Surely Finite}

    If $f: \Omega \to \mathbb{R}$ is a. s. finite, then note that if $A_n \coloneqq \{ |f| \geq n \}$, then $(A_n)_{n=1}^{\infty}$ is a decreasing sequence and so:

        \begin{align*}
            P\left(\bigcap_{n=1}^{\infty} A_n\right) = P\left(\lim_{n \to \infty} A_n\right) = P(|f| = \infty) \\ = 0.
        \end{align*}

\end{observation}

\begin{observation}{}{$\mu$-Almost Everywhere Convergence I}

    If $f_n \to f$ $\mu$-a.e., then $\mu(f_n \not\to f) = 0$.

\end{observation}

\begin{observation}{}{$\mu$-Almost Everywhere Convergence II}

    If $A \in \CalF$ is a set such that $\mu(A) = 0$ and

        \begin{align*}
            \lim_{n \to \infty} |f_n(\omega) - f(\omega)| = 0 \quad \forall \omega \in A^C,
        \end{align*}

    then $f_n \to f$ $\mu$-almost everywhere.

\end{observation}

\begin{observation}{}{Almost Sure Convergence}

    If $f_n \to f$ a.s., then $P(f_n \not\to f) = 0$ or equivalently $P(f_n \to f) = 1$.

\end{observation}

\begin{observation}{}{Splitting Measures of Inequalities}

    Let $f,g$ be measurable functions and $a \in \mathbb{R}$, then observe that:

        \begin{align*}
            \mu(|f| \geq a) \leq \mu\left( |f - g| \geq \frac{a}{2} \right) + \mu\left( |g| \geq \frac{a}{2} \right)
        \end{align*}

\end{observation}

\begin{observation}{}{Using Borel-Cantelli}

    If you can define sets $(A_k)_{k=1}^{\infty}$ such that $\mu(A_k) \leq 1/k^2$, then you can use Borel-Cantelli as:

        \begin{align*}
            \sum_{k=1}^{\infty} \mu(A_k) \leq \sum_{k=1}^{\infty} \frac{1}{k^2} < \infty.
        \end{align*}

    In fact, the choice of $1/k^2$ is more or less arbitrary. This technique would work with any $r_k$ s.t. $\sum_{k=1}^{\infty} r_k < \infty$ and $\mu(A_k) \leq r_k$. Caution: $r_k = 1/k$ does \emph{not} work.

\end{observation}

\begin{observation}{}{Function As Integral}

    Let $f: \Omega \to \overline{\mathbb{R}}$ be a \emph{non-negative} measurable function, the obvserve that

        \begin{align*}
            f(\omega) = \int\limits_0^{f(\omega)} \, dx = \int\limits_0^{\infty} \Indicator{x \leq f(\omega)} \, dx
        \end{align*}

\end{observation}

\begin{observation}{}{Bounding Complement Probabilities}

    Note that $1 - x \leq e^{-x}$. Therefore, we can bound probabilities of a product of complement events, for instance:

        \begin{align*}
            \prod_{n = 1}^{\infty} P(A_n^C) = \prod_{n = 1}^{\infty} [1 - P(A_n)] \leq \\ \prod_{n = 1}^{\infty} e^{-P(A_n)} = e^{\sum_{n = 1}^{\infty} -P(A_n)}
        \end{align*}

\end{observation}

\begin{observation}{}{Interchanging Expectation \& Infinite Sum}

    Observe that if $f$ is \emph{non-negative}, then:

        \begin{align*}
            E\left(\sum_{n=1}^{\infty} f(X_n)\right) =
            E\left(\lim_{N \to \infty} \sum_{n=1}^{N} f(X_n)\right) = \\
            = \lim_{N \to \infty} \sum_{n=1}^{N} E f(X_n) =
            \sum_{n=1}^{\infty} E f(X_n),
        \end{align*}

    where pulling the expectation through the sum can be done due to the Monotone Convergence Theorem, as $\sum_{n=1}^{N} f(X_n)$ is an increasing sequence of \emph{non-negative} random variables.

\end{observation}

\begin{observation}{}{Markov-Chebyshev's Inequality \& Norm}

    The following is the general Markov-Chebyshev Inequality rewritten using the norm instead of an integral. Let $f: \Omega \to \overline{\mathbb{R}}$ be a \emph{non-negative}, measurable function in $L_{\alpha}(\Omega, \CalF, \mu)$, then

        \begin{align*}
            \mu(f \geq \lambda) \leq \lambda^{-\alpha} \Norm{f}_{\alpha}^{\alpha} \, d\mu \quad \forall \lambda > 0, \alpha > 0.
        \end{align*}

\end{observation}

\begin{observation}{}{Distribution Function as Expectation}

    Let $X$ be a random variable and $F_X$ its distribution function. Then:

        \begin{align*}
            F_X(a) = P(X \leq a) = \int_{\Omega} \Indicator{X \leq a} \, dP = E\Indicator{X \leq a}.
        \end{align*}

\end{observation}

\begin{observation}{}{Distribution Function as Expectation, II}

    Let $X$ be a random variable and $F_X$ its distribution function. Then:

        \begin{align*}
            F_X(x + a) - F_X(x) = E\Indicator{x < X \leq x + a}.
        \end{align*}

\end{observation}

\begin{observation}{}{Tightening/Relaxing Expectations}

    Let $X$ be a random variable and $\lambda \in \mathbb{R}$. Then the following  holds:

        \begin{align*}
            EX \geq E(\Indicator{X \geq \lambda}X) \geq E(\Indicator{X \geq \lambda}\lambda).
        \end{align*}

    Left-to-right can be thought of as ``tightening'' the constraints and thus (potentially) decreasing the area that is integrated over, right-to-left as ``loosening'' and thus (potentially) increasing the area that is integrated over.

\end{observation}

\begin{observation}{}{Identical Distribution Giving Equal Probability}

    Let $(X_n)_{n=1}^{\infty}$ be a sequence of \emph{independent, identically distributed} random variables. Let $A_n$ be an event depending on $X_n$, for instance $A_n \coloneqq \{ X_n \geq K \}$ for some $K \in \mathbb{R}$, then all $P(A_n)$ are equal due to $X_n$ being identically distributed, i.e.

        \begin{align*}
            P(A_n) = p \quad \textrm{for } n \geq 1, p \in [0,1].
        \end{align*}

\end{observation}

\begin{observation}{}{Identical Distribution \& Infinite Sum}

    Let $(X_n)_{n=1}^{\infty}$ be a sequence of \emph{independent, identically distributed} random variables. Let $A_n$ be an event depending on $X_n$, for instance $A_n \coloneqq \{ X_n \geq K \}$ for some $K \in \mathbb{R}$, then

        \begin{align*}
            \sum_{n=1}^{\infty} P(A_n) < \infty \Rightarrow P(A_n) = 0 \textrm{ for } n \geq 1.
        \end{align*}

\end{observation}


\section{Definitions}
Note, in our previous two examples of rotational symmetrical groups, the notion of composition is quite important:
In both of our proofs that the tetrahedron group and the isocagonal cone group are different, we have harness this notion.
Naturally, we should have it in our definition of group.
\begin{definition}
    Let $X$ be a set. A binary operation $\cdot$ is a function $:X\times X\to X$.
\end{definition}
\begin{definition}
    A group $G$ is a triple $(G,\cdot,e)$, where $G$ is a nonempty set, $\cdot$ is a binary operation on $G$ and $e\in G$, that satisfies\\
    G1. (Law of Associativity) $\forall a,b,c\in G, (a\cdot b)\cdot c=a\cdot (b\cdot c)$.\\
    G2. (Identity) $\forall a\in G, a\cdot e=a$.\\
    G3. (Inverse) $\forall a\in G, \exists b\in G, a\cdot b=e$
\end{definition}
Most of the time we write $a\cdot b$ as $ab$.
\begin{theorem}
    Let $(G, \cdot, e)$ be a group, then\\
    1. $ab=e\implies ba=e$.\\
    2. $ea=a$.\\
    3. $ab=e\land ab^\prime=e\implies b=b^\prime$.\\
    4. $\exists a, ae^\prime=a\implies e^\prime=e$.
\end{theorem}
\begin{proof}
    1. Choose $c$ such that $(ba)c=e$, so $e=bac=beac=b(ab)ac=ba((ba)c)=bae=ba$.\\
    2. By part 1, we can choose $b$ such that $ab=ba=e$, so $ea=aba=ae=a$.\\
    3. $b=bab^\prime=eb^\prime$ (due to part 1) $=b^\prime$ (due to part 2).\\
    4. By part 1, we choose $b$ such that $ba=e$, so $e^\prime=ee^\prime$ (by part 2) $=bae^\prime=ba=e$.
\end{proof}
\begin{remark}
    There are a LOT of proofs to the preceding theorem.
    For example, the lecturer used a somewhat different proof for part 1 of the theorem.
    Therefore, you should try and come up with your own -- it's great fun.
\end{remark}
By part 3 and axiom G3, any $a$ in the group has a unique $b$ in the group such that $ab=ba=e$.
Then we write $a^{-1}=b$ for this element.
This is called the \textit{inverse} of $a$.
Note as well that $(a^{-1})^{-1}=a$.
In addition, $(ab)^{-1}=b^{-1}a^{-1}$.
\begin{definition}
    For any element $a\in G$, declare $a^0=e$.
    Say inductively that $a^n=a(a^{n-1})$ for a positive integer $n$.
    Similarly for a negative integer $n$, $a^n=(a^{-1})^{-n}$
\end{definition}
One can check that the usual laws of indices apply.
\begin{proposition}
    1. $a^na^m=a^{m+n}$.\\
    2. $(a^n)^m=a^{nm}$.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
There are some `fake axioms' that we do not actually have to be stated.
For example, the definition of binary operation includes the axiom of closure.
However, we do have to verify the closure property to show that something is a group.


\subsection{Compact spaces}

A space \(X\) is compact if each open cover has a finite subcover.

If we can define a cover which does not have a finite subcover, then the space is not compact.

For example an infinite cover could be tend towards \((0,1)\), eg as \(\dfrac{1}{n},1-\dfrac{1}{n}\)

This covers \((0,1)\), but there is no finite subcover. As a result \((0,1)\) is not compact.


\section{Sparse Matrices and Solid Boolean Algebras}\label{boolean-algebras}
%=================================================

Manuscript~\cite{paoluzzi2019finite} shows that the set of join-irreducible atoms of the Boolean Algebra generated by a partition of $\E^3$ are one-to-one with the basis of 3-chain space also generated by the partition, and hence with the columns of the $[\partial_3]$ matrix, which provides a boundary representation (as 2-cycles) of the independent elements of 3-space.
An example of such space decomposition is shown in Figures~\ref{fig:image1} and~\ref{fig:image2}. The 3-cells are not in scale, and are suitably rotated to better exhibit their
complex structure, even containing internal holes. Their assembly gives the union of the five cubes. Each
\emph{3-cell is generated} as \emph{a column} of the \emph{sparse matrix} of
\emph{chain map} $\partial_3: C_3\to C_2$,  with values in $\{-1,0,1\}$.
They are the join-irreducible \emph{atoms} of the CSG algebra with closed regular cells.

\begin{figure}[htbp]
    \centering
    \begin{minipage}{0.475\textwidth}
        \centering
       \includegraphics[width=0.6\linewidth]{../figs/image1.png}
       \caption{A collection $\mathcal{S}$ of five random cubes in 3D Euclidean space.}
       \label{fig:image1}
    \end{minipage}\hfill
    \begin{minipage}{0.475\textwidth}
        \centering
       \includegraphics[width=0.8\linewidth]{../figs/image2.png}
        \caption{3-cells of the generated arrangement $\mathcal{A}(\mathcal{S})$ of $\E^3$.
        They are given by columns of $[\partial_3]$ as \emph{atoms} of a solid Boolean algebra.}
       \label{fig:image2}
    \end{minipage}
\end{figure}

To generate the $d$-space arrangement induced by a collection of cellular ($d$--1)-complexes, some numerical algebra and basic tools of linear algebra and algebraic topology are used. In particular, (sparse) matrices of operators and matrix multiplication and transposition. Interval-trees and $kd$-trees are also introduced for acceleration of clustering of face cells into subsets of congruent shape.

Using our approach with sparse matrices, the validity of topological computations is guaranteed, since these operator matrices must satisfy by construction the (graded) constraints: $[\partial_2][\partial_3]=[0]$ and $[\partial_1][\partial_2]=[0]$.  Similarly, we have $[\delta_1][\delta_0]=[0]$ and $[\delta_2][\delta_1]=[0]$.

With our approach based on boundary operators, the distinctions are removed between manifold and non-manifold representations (fairly standard in Solid Modeling), so allowing for mixing B-reps, cellular decompositions of elementary solids, and/or regular grids. This allows, e.g., for the generation of internal structures and mixed-dimensional objects needed in many applications, for example to make stronger and more resilient the ones to be produced via 3D printing.

Even more, the evaluation of CSG expressions of arbitrary complexity is done in a novel way, by combining the binary columns of 3D basis elements (i.e., the elements of the $U_3$ basis) with native Julia's operators for bitwise operations. In other words, once the $3$-space partition is generated, and $3$-cells are classified w.r.t.~all solids terms, via a single point-set containment test,  \emph{all CSG algebraic  expressions}---of any complexity---can be  evaluated simply by bitwise vectorized logical operations.
Finally, the sparse matrix approach can be extended to general dimensions and/or implemented on highly parallel computational engines, even using standard GPU computing kernels.


\subsection{Height of Semilattice:}
\begin{itemize}
    \item \textbf{Definition}: It is defined as the largest number of greater than($>$) relations that would fit in a descending order.
    \par It can also be defined as the longest path length in the semilattice diagram.
    \item \textbf{Set of values $V$ and a partial order} For eg. in a semilattice with power set of $n$ definitions as the set of values $V$ and set intersection($\cap$) as the meet operator there would be 2\textsuperscript{n} number of elements but the height of the semilattice would be $n$ as that is the maximum path length in the graph(as can be seen in the figure \ref{fig:height_semilattice}). As we go down on each edge the cardinality reduces by 1 and the longest path is from complete set to null set of length $n$.
    \begin{figure}
    \centering
    \includegraphics[width=1\linewidth]{images/semilattice_height.png}
    \caption{Semilattice height example}
    \label{fig:height_semilattice}
    \end{figure}
    \par Similarly, for DFA of reaching definitions the height of the semilattice is the number of definitions whereas the number of values is exponential in number of definitions.
    \item Now, There could be infinite height semilattices also having infinite no. of values(eg. set of integers with $max$ as the meet operator) but even with infinite no. of elements in the semilattice it’s height can still be finite, for eg. in case of constant propagation.
    \item Height of the semilattice is an important property because, one of the requirements for the fixed point iteration to converge is that the semilattice height is finite, if it’s infinite then the iteration is not guaranteed to converge.
\end{itemize}


\chapter{Directed graphical models (Bayes nets)}
\label{chap:DGM}

\section{Introduction}

\subsection{Chain rule}
\begin{equation}
p(x_{1:V}) = p(x_1)p(x_2|x_1)p(x_3|x_{1:2})\cdots p(x_N|x_{1:V-1})
\end{equation}

\subsection{Conditional independence}
$X$ and $Y$ are \textbf{conditionally independent} given $Z$, denoted $X \perp Y|Z$, iff the conditional joint can be written as a product of conditional marginals, i.e.
\begin{equation}
X \perp Y|Z \Longleftrightarrow p(X,Y|Z)=p(X|Z)p(Y|Z)
\end{equation}

first order \textbf{Markov assumption}: the future is independent of the past given the present,
\begin{equation}
x_{t+1} \perp x_{1:t-1}|x_t
\end{equation}

first-order \textbf{Markov chain}
\begin{equation}
p(x_{1:V}) = p(x_1)\prod\limits_{t=2}^V p(x_t|x_{t-1})
\end{equation}

\subsection{Graphical models}
A \textbf{graphical model}(GM) is a way to represent a joint distribution by making CI assumptions. In particular, the nodes in the graph represent random variables, and the (lack of) edges represent CI assumptions.

There are several kinds of graphical model, depending on whether the graph is directed, undirected, or some combination of directed and undirected. In this chapter, we just study directed graphs. We consider undirected graphs in Chapter 19.

\subsection{Directed graphical model}
A \textbf{directed graphical model}or \textbf{DGM} is a GM whose graph is a DAG. These are more commonly known as \textbf{Bayesian networks}. However, there is nothing inherently “Bayesian” about Bayesian networks: they are just a way of defining probability distributions. These models are also called \textbf{belief networks}. The term “belief” here refers to subjective probability. Once again, there is nothing inherently subjective about the kinds of probability distributions represented by DGMs.

\textbf{Ordered Markov property}
\begin{equation}
x_s \perp x_{\mathrm{pred}(s)\ \mathrm{pa}(s)} \perp x_{\mathrm{pa}(s)}
\end{equation}
where pa$(s)$ are the parents of nodes, and pred$(s)$ are the predecessors of nodes in the DAG.

\textbf{\textbf{Markov chain}} on a DGM
\begin{equation}
p(x_{1:V}|G)=\prod\limits_{t=1}^V p(x_t|x_{\mathrm{pa}(t)})
\end{equation}

\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.50]{graphical-models.png}
\caption{(a) A simple DAG on 5 nodes, numbered in topological order. Node 1 is the root, nodes 4 and 5 are the leaves. (b) A simple undirected graph, with the following maximal cliques: {1,2,3}, {2,3,4}, {3,5}.}
\label{fig:graphical-models}
\end{figure}

\section{Examples}

\subsection{Naive Bayes classifiers}
\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.60]{naive-bayes-as-DGM.png}
\caption{(a) A naive Bayes classifier represented as a DGM. We assume there are $D=4$ features, for simplicity. Shaded nodes are observed, unshaded nodes are hidden. (b) Tree-augmented naive Bayes classifier for $D=4$ features. In general, the tree topology can change depending on the value of $y$.}
\label{fig:naive-bayes-as-DGM}
\end{figure}

\subsection{Markov and hidden Markov models}
\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.50]{second-order-Markov-chain.png}
\caption{A first and second order Markov chain.}
\label{fig:second-order-Markov-chain}
\end{figure}

\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.60]{first-order-HMM.png}
\caption{A first-order HMM.}
\label{fig:first-order-HMM}
\end{figure}

\section{Inference}
Suppose we have a set of correlated random variables with joint distribution $p(\vec{x}_{1:V}|\vec{\theta})$. Let us partition this vector into the \textbf{visible variables} $\vec{x}_v$, which are observed, and the \textbf{hidden variables}, $\vec{x}_h$, which are unobserved. Inference refers to computing the posterior distribution of the unknowns given the knowns:
\begin{equation}
p(\vec{x}_h|\vec{x}_v,\theta) = \frac{p(\vec{x}_h, \vec{x}_v|\vec{\theta})}{p(\vec{x}_v|\vec{\theta})}=\frac{p(\vec{x}_h, \vec{x}_v|\vec{\theta})}{\sum_{\vec{x}_h'}p(\vec{x}_h', \vec{x}_v|\vec{\theta})}
\end{equation}

Sometimes only some of the hidden variables are of interest to us. So let us partition the hidden variables into \textbf{query variables}, $\vec{x}_q$, whose value we wish to know, and the remaining \textbf{nuisance variables}, $\vec{x}_n$, which we are not interested in. We can compute what we are interested in by \textbf{marginalizing out} the nuisance variables:
\begin{equation}
p(\vec{x}_q|\vec{x}_v,\vec{\theta}) = \sum_{\vec{x}_n}p(\vec{x}_q, \vec{x}_n|\vec{x}_v, \vec{\theta})
\end{equation}

\section{Learning}
MAP estimate:
\begin{equation}
\hat{\vec{\theta}} =\arg\max_{\vec{\theta}}\sum\limits_{i=1}^N \log p(\vec{x}_{i,v}|\vec{\theta})+\log p(\vec{\theta})
\end{equation}

\subsection{Learning from complete data}
If all the variables are fully observed in each case, so there is no missing data and there are no hidden variables, we say the data is \textbf{complete}. For a DGM with complete data, the likelihood is given by
\begin{equation}\begin{split}
p(\mathcal{D}|\vec{\theta}) & = \prod_{i=1}^N p(\vec{x}_i|\vec{\theta}) \\
& = \prod_{i=1}^N\prod_{t=1}^V p(\vec{x}_{it}|\vec{x}_{i, \mathrm{pa}(t)}, \vec{\theta}_t) \\
& = \prod_{t=1}^V p(\mathcal{D}_t|\vec{\theta}_t)
\end{split}\end{equation}
where $\mathcal{D}_t$ is the data associated with node $t$ and its parents, i.e., the $t$'th family.

Now suppose that the prior factorizes as well:
\begin{equation}
p(\vec{\theta})=\prod\limits_{t=1}^V p(\vec{\theta}_t)
\end{equation}

Then clearly the posterior also factorizes:
\begin{equation}
p(\vec{\theta}|\mathcal{D}) \propto p(\mathcal{D}|\vec{\theta})p(\vec{\theta}) = \prod\limits_{t=1}^V p(\mathcal{D}_t|\vec{\theta}_t)p(\vec{\theta}_t)
\end{equation}

\subsection{Learning with missing and/or latent variables}
If we have missing data and/or hidden variables, the likelihood no longer factorizes, and indeed it is no longer convex, as we explain in detail in Section TODO. This means we will usually can only compute a locally optimal ML or MAP estimate. Bayesian inference of the parameters is even harder. We discuss suitable approximate inference techniques in later chapters.

\section{Conditional independence properties of DGMs}

\subsection{d-separation and the Bayes Ball algorithm (global Markov properties)}
\begin{enumerate}
\item P contains a chain
\begin{equation}\begin{split}
p(x,z|y) & = \frac{p(x,y,z)}{p(y)}
= \frac{p(x)p(y|x)p(z|y)}{p(y)} \\
 & = \frac{p(x,y)p(z|y)}{p(y)} = p(x|y)p(z|y)
\end{split}\end{equation}

\item P contains a fork
\begin{equation}
p(x,z|y) = \frac{p(x,y,z)}{p(y)}
= \frac{p(y)p(x|y)p(z|y)}{p(y)}
= p(x|y)p(z|y)
\end{equation}
\item P contains v-structure
\begin{equation}
p(x,z|y) = \frac{p(x,y,z)}{p(y)}
= \frac{p(x)p(z)p(y|x,z)}{p(y)}
\neq p(x|y)p(z|y)
\end{equation}
\end{enumerate}

\subsection{Other Markov properties of DGMs}

\subsection{Markov blanket and full conditionals}

\begin{equation}
mb(t) = ch(t)\cup pa(t)\cup copa(t)
\end{equation}

\subsection{Multinoulli Learning}
Multinoulli Distribution：
\begin{equation}\label{eqn:discrite}
Cat(x|\mu) = \prod_{k=1}^K\mu_k^{x_k}
\end{equation}
then from \ref{bayes_net} and \ref{eqn:discrite}:
\begin{equation}
p(x|G,\theta) = \prod_{v=1}^V\prod_{c=1}^{C_v}\prod_{k=1}^K
\theta_{vck}^{y_{vck}}
\end{equation}
Likelihood
\begin{equation}
p(D|G,\theta) = \prod_{n=1}^N p(x_n|G,\theta)
=\prod_{n=1}^N\prod_{v=1}^V\prod_{c=1}^{C_{nv}}\prod_{k=1}^K
\theta_{vck}^{y_{nvck}}
\end{equation}
where $y_{nv} = f(pa(x_{nv}))$, f(x) is a map from x to a vector, there is only one element in the vector is 1.

\section{Influence (decision) diagrams *}


\chapter{Associative Memory Network}
Associative memory networks are single input, single output layer networks in which weights are determined in order to store a set of pattern associations. They are able to recall the desired response patterns when a given pattern is similar to the stored pattern.
$$\mathbf{{(x_1 y_1), (x_2 y_2), \ldots, (x_p y_p)}}$$
Association vectors are defined as:
$$\mathbf{x_k \rightarrow y_k}$$
\begin{description}
\item[Auto-associative] $\mathbf{x_k = y_k}$
\item[Hetero-associative] $\mathbf{x_k \ne y_k}$
\end{description}

\section{Linear Feed-forward Associative Memory network}
Linear Feed-forward Associative Memory (LAM) network is a single input, single output layer with linear activation function. \\
The synaptic input:
$$\mathbf{u=Wx}$$
The output activation:
$$\mathbf{y=\beta u}$$
\begin{center} where $\beta$ is the slope of the linear function\end{center}
\clearpage
\noindent LAM networks learn using the Hebbian rule. The weight matrix is \emph{given} by:
$$\mathbf{W_k = y_k x_k^{T}}$$
\begin{center} where $\mathbf{W_k}$ weight matrix learns the $k$ th pattern association\end{center}
The weight matrix to learn all the associations is \emph{given} by:
\begin{equation*}
\begin{split}
\mathbf{W} &= \sum_{k=1}^{P} \mathbf{W_k} \\
&= \sum_{k=1}^{P} \mathbf{y_k x_k^{T}}
\end{split}
\label{weight_lam}
\end{equation*}

\section{Cross-Talk versus Perfect Recall}
If inputs are \emph{uncorrelated or orthogonal} the Hebbian rule make \emph{perfect recall}. Otherwise, the output is corrupted or has \emph{cross-talk}. \\
Substituting \ref{weight_lam} to output, with $\mathbf{x_l}$ as input:
\begin{equation}
\begin{split}
\mathbf{y} &= \mathbf{Wx_l} \\
&= \Big(\sum_{k=1}^{P} \mathbf{y_k x_k^{T}} \Big) \mathbf{x_l} \\
&= \mathbf{y_l x_l^{T} x_l} + \sum_{k=1,k \ne l} \mathbf{y_k x_k^{T} x_l}
\end{split}
\end{equation}
If $\mathbf{x_l}$ and $\mathbf{x_k}$ are orthogonal:
$$\mathbf{x_l^{T} x_k} = 0$$
Then, perfect recall of the learned pattern:
$$\mathbf{y = y_l \| x_l \|^{2}}$$
$$\mathbf{y \propto y_l}$$
Otherwise, cross-talk:
$$\sum_{k=1,k \ne l} \mathbf{y_k x_k^{T} x_l}$$

\section{Problem with LAM Network}
Both auto and heterogeneous associative memory network do not function well in the presence of noise.
Let distorted input as:
$$\mathbf{x}_l^{i} = \mathbf{x}_l + \Delta_l$$
Then the output:
\begin{equation*}
\begin{split}
\mathbf{y} &= \mathbf{W} \mathbf{x}_l' \\
&= \mathbf{W} (\mathbf{x}_l + \Delta_l) \\
&= \Big(\sum_{k=1}^{P} \mathbf{y_k x_k^{T}}\Big) (\mathbf{x_l} + \Delta_l) \\
&= \mathbf{y_l x_l^{T} x_l + y_l x_l^{T} \Delta_l + \Big(\sum_{k \ne l}^{P} \mathbf{y_k x_k^{T} x_l}\Big) + \Big(\sum_{k \ne l}^{P} \mathbf{y_k x_k^{T} \Delta_l}\Big)}
\end{split}
\end{equation*}
Since orthogonal:
$$\mathbf{y} = \mathbf{y_l x_l^{T} x_l + y_l x_l^{T} \Delta_l +  \Big(\sum_{k \ne l}^{P} \mathbf{y_k x_k^{T} \Delta_l}\Big)}$$

\section{Recurrent/Hopfield AMN}
Suppressing noise component by thresholding the outputs and recycling of the output to the input. Weight is still based on Hebbian's learning rule, but $w_{ii} = 0$

\section{Bidirectional Associative Memory Network}
Bidirectional Associative Memory (BAM) network is a \emph{recurrent hetero-associative memory} of two layers of neurons. \\
Synaptic input to the output layer:
$$\mathbf{u = Wx}$$
The output:
$$
y_j =
\begin{cases}
1 & u_j > 0 \\
-1 & u_j \le 0
\end{cases}
$$
Synaptic input back to the input layer:
$$\mathbf{v = W^{T} y}$$
The input:
$$
x_j =
\begin{cases}
1 & v_i > 0 \\
-1 & v_i \le 0
\end{cases}
$$
Repeat until convergence


\subsection{Incidence matrix}

An incidence matrix has \(m_{ij}\) representing the number of connections from vertex \(i\) to edge \(j\).


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\begin{document}

%--------------------------------------------------------------------------------
\section{First example}

The well known Pythagorean theorem \(x^2 + y^2 = z^2\) was
proved to be invalid for other exponents.
Meaning the next equation has no integer solutions:

\[ x^n + y^n = z^n \]

%--------------------------------------------------------------------------------

\section{Second example}

In physics, the mass-energy equivalence is stated by the equation $E=mc^2$, discovered in 1905 by Albert Einstein.

The mass-energy equivalence is described by the famous equation
\[ E=mc^2 \]
discovered in 1905 by Albert Einstein.
In natural units ($c$ = 1), the formula expresses the identity
\begin{equation}
E=m
\end{equation}

\section{Third example}

This is a simple math expression \(\sqrt{x^2+1}\) inside text.
And this is also the same:
\begin{math}
\sqrt{x^2+1}
\end{math}
but by using another command.

This is a simple math expression without numbering
\[\sqrt{x^2+1}\]
separated from text.

This is also the same:
\begin{displaymath}
\sqrt{x^2+1}
\end{displaymath}

\ldots and this:
\begin{equation*}
\sqrt{x^2+1}
\end{equation*}

    In physics, the mass-energy equivalence is stated
    by the equation $E=mc^2$, discovered in 1905 by Albert Einstein.

    The mass-energy equivalence is described by the famous equation
    \[ E=mc^2 \]
    discovered in 1905 by Albert Einstein.
    In natural units ($c = 1$), the formula expresses the identity
    \begin{equation}
        E=m
    \end{equation}

    Subscripts in math mode are written as $a_b$ and superscripts are written as $a^b$. These can be combined an nested to write expressions such as

    \[ T^{i_1 i_2 \dots i_p}_{j_1 j_2 \dots j_q} = T(x^{i_1},\dots,x^{i_p},e_{j_1},\dots,e_{j_q}) \]

    We write integrals using $\int$ and fractions using $\frac{a}{b}$. Limits are placed on integrals using superscripts and subscripts:

    \[ \int_0^1 \frac{dx}{e^x} =  \frac{e-1}{e} \]

    Lower case Greek letters are written as $\omega$ $\delta$ etc. while upper case Greek letters are written as $\Omega$ $\Delta$.

    Mathematical operators are prefixed with a backslash as $\sin(\beta)$, $\cos(\alpha)$, $\log(x)$ etc.

\end{document}


\documentclass[12pt]{scrartcl}

\input{preamble}

\makeatletter
\title{Hack 6.0}\let\Title\@title
\subtitle{Computer Science I\\
{\small
\vskip1cm
Department of Computer Science \& Engineering \\
University of Nebraska--Lincoln}
\vskip-1cm}
%\author{Dr.\ Chris Bourke}
\date{~}
\makeatother

\begin{document}

\maketitle

\hrule

\input{instructions.tex}

\section*{Problem Statement}

In this hack you'll get some more practice writing functions that utilize
pass-by-reference (pointers), error handling and enumerated types.  There
are several different
ways to model colors including RGB and CMYK.  RGB is generally used in displays
and models a color with three values in the range $[0, 255]$ corresponding to
the red, green and blue ``contribution'' to the color.  For example, the
triple $(255, 255, 0)$ corresponds to a full red and green (additive) value
which results in yellow.  CMYK or Cyan-Magenta-Yellow-Black is a model used
in printing where four colors of ink are combined to make various colors.
In this system, the four values are on the scale $[0, 1]$.  Write
functions to convert between these models.

\begin{enumerate}
\item Write a function to convert from an RGB color model to CMYK.  To
convert to CMYK, you first need to scale each integer value to the range
$[0, 1]$ by simply computing
	$$r' = \frac{r}{255}, \quad g' = \frac{g}{255}, \quad b' = \frac{b}{255}$$
	and then using the following formulas:
\begin{align*}
k & = 1-\max\{r', g', b'\} \\
c & = \frac{(1-r'-k)}{(1-k)} \\
m & = \frac{(1-g'-k)}{(1-k)} \\
y & = \frac{(1-b'-k)}{(1-k)} \\
\end{align*}
Your function should have the following signature:

\mintinline{c}{int rgbToCMYK(int r, int g, int b, double *c, double *m, double *y, double *k)}

Identify any and all error conditions and use the return value to indicate
an error code (0 for no error, non-zero value(s) for error conditions).  Note
that one edge case is black, when $(r,g,b) = (0,0,0)$ which would lead to a
division by zero in the formulas.  The equivalent CMYK values are $(0,0,0,1)$.

\item Write a function to convert from CMYK to RGB using the following formulas.
\begin{align*}
r & = 255 \cdot (1 - c) \cdot (1-k) \\
g & = 255 \cdot (1 - m) \cdot (1-k) \\
b & = 255 \cdot (1 - y) \cdot (1-k) \\
\end{align*}
Results should be rounded.  Your function should have the following signature:

\mintinline{c}{int cmykToRGB(double c, double m, double y, double k, int *r, int *g, int *b)}

Identify any and all error conditions and use the return value to indicate
an error code (0 for no error, non-zero value(s) for error conditions).

\end{enumerate}

\section*{Instructions}

\begin{itemize}
  \item You are encouraged to collaborate any number of students
  before, during, and after your scheduled hack session.
  \item Design at least 3 test cases for each function
  \emph{before} you begin
  designing or implementing your program.  Test cases are
  input-output pairs that are known to be correct using means
  other than your program.
  \item You may (in fact are encouraged) to define any additional
  ``helper'' functions that you find useful.
  \item Include the name(s) of everyone who worked together on
  this activity in your source file's header.
  \item Place your prototypes and documentation in a header file
  named \mintinline{text}{colorUtils.h} and your source in a file
  named \mintinline{text}{colorUtils.c}.

  \item A testing file, \mintinline{text}{utilsTester.c} has been
  provided that uses cmocka (\url{https://cmocka.org/}), a unit testing
  framework for C.  We have already written several (17) test cases
  for you.  Using these examples, implement your test cases using cmocka
  for your two functions.  You should have at least 3 test cases for
  each function for a total of 23 test cases.

  The starter file should be sufficient to demonstrate how to use
  cmocka, but the full documentation can be found here: \url{https://api.cmocka.org/}.
  A \mintinline{text}{makefile} has also been provided to help
  you easily compile your files.  Note that cmocka is already
  installed on the CSE server.  If you compile on your own machine, you will
  have to install and troubleshoot cmocka yourself.
  \item Turn in all of your files via webhandin, making sure that
  it runs and executes correctly in the webgrader.  Each individual
  student will need to hand in their own copy and will receive
  their own individual grade.
\end{itemize}

\end{document}


\section{Linear Support Vector Regression} \label{section:svr}

In the case of regression the goal is to predict a real-valued output for $y'$ so that our training data is of the form:

\begin{equation}
	\{(x_i,y_i), x\in\Re^m, y_i\in\Re, i=1, \dots, n\} \label{eq:svr_data}
\end{equation}

The regression SVM use a loss function that not allocating a penalty if the predicted value $y'_i$ is less than a distance $\epsilon$ away from the actual value $y_i$, i.e., if $|y_i-y'_i| \leq \epsilon$, where $y'_i = w^T x_i + b$. The region bound by $y'_i\pm\epsilon \ \forall_i$ is called an $\epsilon$-insensitive tube. The output variables which are outside the tube are given one of two slack variable penalties depending on whether they lie above, $\xi^+$, or below, $\xi^-$, the tube, provided $\xi^+ \geq 0$ and $\xi^- \geq 0 \ \forall_i$:

\begin{equation} \label{eq:svr_consts}
	\begin{aligned}
		& y_i\leq y'_i+\epsilon+\xi^+ \ \forall_i \\
    	& y_i\geq y'_i-\epsilon-\xi^- \ \forall_i \\
    	& \xi_i^+, \xi_i^- \geq 0 \ \forall_i
	\end{aligned}
\end{equation}

The objective function for SVR can then be written as:

\begin{equation} \label{eq:quad_svr_obj}
    \begin{aligned}
        \min_{w,b,\xi^+,\xi^-} \quad & \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n (\xi_i^+ + \xi_i^-) \\
            \text{subject to} \quad & y_i - w^T x_i - b \leq \epsilon + \xi_i^+ \ \forall_i \\ & w^T x_i + b - y_i \leq \epsilon + \xi_i^- \ \forall_i \\ & \xi_i^+, \xi_i^- \geq 0 \ \forall_i
    \end{aligned}
\end{equation}

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.6]{img/linear_dual_l1_svr_hyperplane}
  	\caption{Linear SVR hyperplane}
  	\label{fig:linear_dual_l1_svr_hyperplane}
\end{figure}

\pagebreak

\subsection{Epsilon-insensitive loss}

The \emph{epsilon-insensitive} loss is defined as:

\begin{equation} \label{eq:eps_loss1}
	\mathcal{L}_\epsilon^1 = \max(0, |y - (w^T x + b)| - \epsilon)
\end{equation}

or, equivalently:

\begin{equation} \label{eq:eps_loss2}
	\mathcal{L}_\epsilon^1 =
	\begin{cases}
		0 & \text{if} \ |y - (w^T x + b)| \leq \epsilon \\
		|y - (w^T x + b)| - \epsilon & \text{otherwise} \\
	\end{cases}
\end{equation}

As the \emph{hinge} loss, also the \emph{epsilon-insensitive} loss is a nondifferentiable convex function due to its nonsmoothness in $\pm\epsilon$, but has a subgradient that is given by:

% http://juliaml.github.io/LossFunctions.jl/stable/losses/distance/#L1EpsilonInsLoss-1

\begin{equation} \label{eq:eps_loss_der}
	\partial_w \mathcal{L}_\epsilon^1=
		\begin{cases}
            \displaystyle \frac{y - (w^T x + b)}{|y - (w^T x + b)|}x & \text{if} \ |y - (w^T x + b)| \geq \epsilon \\
            0 & \text{otherwise} \\
        \end{cases}
\end{equation}

\subsubsection{Primal formulation}

The general primal unconstrained formulation takes the same form of~\eqref{eq:primal_svm}.

The quadratic optimization problem~\eqref{eq:quad_svr_obj} can be equivalently formulated as:

\begin{equation} \label{eq:primal_l1_svr}
	\min_{w,b} \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n \max(0, |y_i - (w^T x_i + b)| - \epsilon)
\end{equation}

where we make use of the \emph{epsilon-insensitive} loss~\eqref{eq:eps_loss1} or~\eqref{eq:eps_loss2}.

The above formulation penalizes slacks $\xi$ linearly and is called $\mathcal{L}_1$-SVR.

The $\mathcal{L}_1$-SVR objective~\eqref{eq:primal_l1_svr} can be rewritten in form~\eqref{eq:reg_bias_primal_svm1} or~\eqref{eq:reg_bias_primal_svm2} as:

\begin{equation} \label{eq:reg_bias_primal_l1_svr}
	\min_{w,b} \frac{1}{2} (\| w \|^2 + b^2) + C \sum_{i=1}^n \max(0, |y_i - (w^T x_i + b)| - \epsilon)
\end{equation}

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.4]{img/l1_svr_loss}
  	\caption{Epsilon-insensitive loss with different optimization steps}
  	\label{fig:l1_svr_loss}
\end{figure}

\subsubsection{Wolfe dual formulation}

To reformulate the~\eqref{eq:quad_svr_obj} as a \emph{Wolfe dual}, we introduce the Lagrange multipliers $\alpha_i^+, \alpha_i^-, \mu_i^+, \mu_i^- \geq 0 \ \forall_i$:

\begin{equation} \label{eq:svr_wolfe_dual}
	\begin{aligned}
    	\max_{\alpha^+,\alpha^-,\mu^+,\mu^-} \min_{w,b,\xi^+,\xi^-} \mathcal{W}(w,b,\xi^+,\xi^-,\alpha^+,\alpha^-,\mu^+,\mu^-) = \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n (\xi_i^+ + \xi_i^-)-\sum_{i=1}^n (\mu_i^+ \xi_i^+ + \mu_i^- \xi_i^-) \\ -\sum_{i=1}^n \alpha_i^+(\epsilon+\xi_i^+ + y'_i-y_i)-\sum_{i=1}^n \alpha_i^-(\epsilon+\xi_i^- - y'_i+y_i)
	\end{aligned}
\end{equation}

Substituting for $y_i$, differentiating wrt $w, b, \xi^+$, $\xi^-$ and setting the derivatives to $0$ gives:

\begin{equation} \label{eq:svr_wolfe_der_w}
	\frac{\partial \mathcal{W}}{\partial w}=w-\sum_{i=1}^n (\alpha_i^+ - \alpha_i^-) x_i \Rightarrow w=\sum_{i=1}^n (\alpha_i^+ - \alpha_i^-) x_i
\end{equation}

\begin{equation} \label{eq:svr_wolfe_der_b}
	\frac{\partial \mathcal{W}}{\partial b}=-\sum_{i=1}^n (\alpha_i^+ - \alpha_i^-)\Rightarrow \sum_{i=1}^n (\alpha_i^+ - \alpha_i^-)=0
\end{equation}

\begin{equation}\label{eq:svr_wolfe_der_xip}
	\frac{\partial \mathcal{W}}{\partial\xi_i^+}=0\Rightarrow C=\alpha_i^+ + \mu_i^+
\end{equation}

\begin{equation} \label{eq:svr_wolfe_der_xim}
	\frac{\partial \mathcal{W}}{\partial\xi_i^-}=0\Rightarrow C=\alpha_i^- + \mu_i^-
\end{equation}

Substituting~\eqref{eq:svr_wolfe_der_w} and~\eqref{eq:svr_wolfe_der_b} in, we now need to maximize $\mathcal{W}$ wrt $\alpha_i^+$ and $\alpha_i^-$, where $\alpha_i^+ \geq 0,\ \alpha_i^- \geq 0 \ \forall_i$:

\begin{equation} \label{eq:svr_max_wolfe_dual}
    \max_{\alpha^+,\alpha^-} \mathcal{W}(\alpha^+,\alpha^-) = \sum_{i=1}^n y_i(\alpha_i^+ - \alpha_i^-)-\epsilon\sum_{i=1}^n (\alpha_i^+ + \alpha_i^-)-\frac{1}{2}\sum_{i,j}(\alpha_i^+ - \alpha_i^-)\langle x_i, x_j \rangle(\alpha_j ^+ - \alpha_j ^-)
\end{equation}

Using $\mu_i^+ \geq 0$ and $\mu_i^- \geq 0$ together with~\eqref{eq:svr_wolfe_der_w} and~\eqref{eq:svr_wolfe_der_b} means that $\alpha_i^+ \leq C$ and $\alpha_i^- \leq C$. We therefore need to find:

\begin{equation} \label{eq:svr_min_wolfe_dual}
    \begin{aligned}
        \min_{\alpha^+,\alpha^-} \quad & \frac{1}{2}(\alpha^+ - \alpha^-)^TK(\alpha^+ - \alpha^-)+\epsilon e^T(\alpha^+ + \alpha^-)-y^T(\alpha^+ - \alpha^-) \\
            \text{subject to} \quad & 0\leq\alpha_i^+,\alpha_i^- \leq C \ \forall_i \\ & e^T(\alpha^+ - \alpha^-)=0
    \end{aligned}
\end{equation}

where $e^T = [1, \dots, 1]$.

We can write the~\eqref{eq:svr_min_wolfe_dual} in a standard quadratic form as:

\begin{equation}
    \begin{aligned} \label{eq:svr_min_qp_wolfe_dual}
        \min_{\alpha} \quad & \frac{1}{2}\alpha^T Q\alpha-q^T\alpha \\
            \text{subject to} \quad & 0\leq\alpha_i\leq C \ \forall_i \\ & e^T\alpha=0
    \end{aligned}
\end{equation}

where the Hessian matrix $Q =
\begin{bmatrix}
K & -K\\
-K & K
\end{bmatrix}$
, $\alpha =
\begin{bmatrix}
\alpha^+\\
\alpha^-
\end{bmatrix}$
, $q =
\begin{bmatrix}
-y\\
y
\end{bmatrix} + \epsilon$
, and $e =
\begin{bmatrix}
1\\
-1
\end{bmatrix}$.

Each new predictions $y'$ can be found using:

\begin{equation} \label{eq:svr_pred}
    y'= \sum_{i=1}^n (\alpha_i^+ - \alpha_i^-)\langle x_i, x' \rangle+b
\end{equation}

A set $S$ of support vectors $x_s$ can be created by finding the indices $i$ where $0\leq\alpha\leq C$ and $\xi_i^+=0$ or $\xi_i^-=0$.

This gives us:

\begin{equation} \label{eq:svr_b}
    b=y_s-\epsilon-\sum_{m\in S}(\alpha_m^+ -\alpha_m^-) \langle x_m, x_s \rangle
\end{equation}

As before it is better to average over all the indices $i$ in $S$:

\begin{equation} \label{eq:svr_b_avg}
    b=\frac{1}{N_s}\sum_{s\in S}y_s-\epsilon-\sum_{m \in S}(\alpha_m^+ - \alpha_m^-)\langle x_m, x_s \rangle
\end{equation}

From~\eqref{eq:svr_min_qp_wolfe_dual} we can notice that the equality constraint $e^T \alpha = 0$ arises form the stationarity condition $\partial_{{b}} \mathcal{W}=0$. So, again, for simplicity, we can again consider the bias term $b$ embedded into the weight vector. We report below the box-constrained dual formulation~\cite{hsu2002simple} that arises from the primal~\eqref{eq:reg_bias_primal_svm1} or~\eqref{eq:reg_bias_primal_svm2} where the bias term $b$ is embedded into the weight vector $w$:

\begin{equation} \label{eq:svr_min_bcqp_wolf_dual}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T (Q + ee^T)\alpha+q^T\alpha \\
            \text{subject to} \quad & 0\leq\alpha_i\leq C \ \forall_i
    \end{aligned}
\end{equation}

\subsubsection{Lagrangian dual formulation}

In order to relax the constraints in the \emph{Wolfe dual} formulation~\eqref{eq:svr_min_wolfe_dual} we define the problem as a \emph{Lagrangian dual} relaxation by embedding them into objective function, so we need to allocate the Lagrange multipliers $\mu$ and $\lambda_+, \lambda_- \geq 0$:

\begin{equation} \label{eq:l1_svr_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda_+,\lambda_-} \min_{\alpha} \mathcal{L}(\alpha,\mu,\lambda_+,\lambda_-) &= \frac{1}{2} \alpha^T Q\alpha+q^T\alpha + \mu^T (e^T \alpha) + \lambda_+^T (ub - \alpha) - \lambda_-^T \alpha \\
    &= \frac{1}{2} \alpha^T Q\alpha + (q + \mu e^T + \lambda_+ - \lambda_-)^T \alpha + \lambda_+^T ub \\
    \text{subject to} \quad & \,\, \lambda_+, \lambda_- \geq 0
	\end{aligned}
\end{equation}

where the upper bound $ub^T = [C, \dots, C]$.

Taking the derivative of the Lagrangian $\mathcal{L}$ wrt $\alpha$ and settings it to 0 gives:

\begin{equation} \label{eq:svr_lagrangian_der_a}
	\frac{\partial \mathcal{L}}{\partial \alpha}=0\Rightarrow Q \alpha + (q + \mu e^T + \lambda_+ - \lambda_-) = 0
\end{equation}

With $\alpha$ optimal solution of the linear system:

\begin{equation} \label{eq:l1_svr_lagrangian_sol}
    Q \alpha = - (q + \mu e^T + \lambda_+ - \lambda_-)
\end{equation}

the gradients wrt $\mu$, $\lambda_+$ and $\lambda_-$ are:

\begin{equation} \label{eq:svr_lagrangian_der_mu}
	\frac{\partial \mathcal{L}}{\partial \mu}=-e \alpha
\end{equation}

\begin{equation} \label{eq:svr_lagrangian_der_lp}
	\frac{\partial \mathcal{L}}{\partial \lambda_+}=\alpha - ub
\end{equation}

\begin{equation} \label{eq:svr_lagrangian_der_lm}
    \frac{\partial \mathcal{L}}{\partial \lambda_-}=-\alpha
\end{equation}

From~\eqref{eq:svr_min_qp_wolfe_dual} we can notice that the equality constraint $e^T \alpha = 0$ arises form the stationarity condition $\partial_{{b}} \mathcal{W}=0$. So, again, for simplicity, we can again consider the bias term $b$ embedded into the weight vector. In this way the dimensionality of~\eqref{eq:l1_svr_lagrangian_dual} is reduced by removing the multipliers $\mu$ which was allocated to control the equality constraint $e^T \alpha=0$, so we will end up solving exactly the problem~\eqref{eq:svr_min_bcqp_wolf_dual}.

\begin{equation} \label{eq:l1_svr_bcqp_lagrangian_dual}
	\begin{aligned}
    	\max_{\lambda_+,\lambda_-} \min_{\alpha} \mathcal{L}(\alpha,\lambda_+,\lambda_-) &= \frac{1}{2} \alpha^T (Q + ee^T)\alpha+q^T\alpha + \lambda_+^T (ub - \alpha) - \lambda_-^T \alpha \\
    &= \frac{1}{2} \alpha^T (Q + ee^T)\alpha + (q + \lambda_+ - \lambda_-)^T \alpha + \lambda_+^T ub \\
    \text{subject to} \quad & \,\, \lambda_+, \lambda_- \geq 0
	\end{aligned}
\end{equation}

where, again, the upper bound $ub^T = [C, \dots, C]$.

Now, taking the derivative of the Lagrangian $\mathcal{L}$ wrt $\alpha$ and settings it to 0 gives:

\begin{equation} \label{eq:l1_svr_bcqp_lagrangian_der_a}
	\frac{\partial \mathcal{L}}{\partial \alpha}=0\Rightarrow (Q + ee^T) \alpha + (q + \lambda_+ - \lambda_-) = 0
\end{equation}

With $\alpha$ optimal solution of the linear system:

\begin{equation} \label{eq:l1_svr_bcqp_lagrangian_sol}
    (Q + ee^T) \alpha = - (q + \lambda_+ - \lambda_-)
\end{equation}

the gradients wrt $\lambda_+$ and $\lambda_-$ are:

\begin{equation} \label{eq:l1_svr_bcqp_lagrangian_der_lp}
	\frac{\partial \mathcal{L}}{\partial \lambda_+}=\alpha - ub
\end{equation}

\begin{equation} \label{eq:l1_svr_bcqp_lagrangian_der_lm}
    \frac{\partial \mathcal{L}}{\partial \lambda_-}=-\alpha
\end{equation}

\bigskip

Note that since the Hessian matrix $Q$ of the $\mathcal{L}_1$-SVR is not strictly positive definite, i.e., the Lagrangian function is not strictly convex since it will be linear along the eigenvectors correspondent to the null eigenvalues and so it will be unbounded below, the Lagrangian dual relaxation, i.e.,~\ref{eq:l1_svr_lagrangian_sol} and~\ref{eq:l1_svr_bcqp_lagrangian_sol}, will be nondifferentiable, so it will have infinite solutions and for each of them it will have a different subgradient. In order to compute an approximation of the gradient, we will choose $\alpha$ in such a way as the one that minimizes the 2-norm since it is good almost like the gradient:

\begin{equation} \label{eq:svr_lagrangian_krylov_sol}
	\min_{\alpha_n \in K_n(Q, b)} \| Q \alpha_n - b \|
\end{equation}

Since we are dealing with a symmetric system we will choose a well-known Krylov method that performs the Lanczos iterate, i.e., symmetric Arnoldi iterate, called \emph{minres}, i.e., symmetric \emph{gmres}, to compute the vector $\alpha_n$ that minimizes the norm of the residual $r_n = Q \alpha_n - b$ among all vectors in $K_n(Q, b) = span(b, Qb, Q^2b, \dots, Q^{n-1}b)$.

\bigskip

Since the linear algebra methods in the ML context are crucial and also in order to deal with a per-iteration cost equals to the other algorithms described later to provide a coherent comparison of all at the end, we will solve it with a primal-dual optimization method and we modify its definition by adding a strictly convex augmentation term, i.e., a penalty term, in order to improve the actual convergence of the algorithms. So, if we consider a general quadratic optimization problem subject to linear constraints, i.e., equality and inequality constraints, defined as:

\begin{equation}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T Q \alpha + q^T \alpha \\
            \textrm{subject to} \quad & A \alpha = b \\ & G \alpha \leq h \\ & lb \leq \alpha \leq ub
    \end{aligned}
\end{equation}

or, equivalently:

\begin{equation}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T Q \alpha + q^T \alpha \\
            \textrm{subject to} \quad & A \alpha = b \\ & \hat{G} \alpha \leq \hat{h}
    \end{aligned}
\end{equation}

where $\hat{G} =
\begin{bmatrix}
 G \\
-I \\
 I
\end{bmatrix}$ and $\hat{h} =
\begin{bmatrix}
h & -lb & ub
\end{bmatrix}$; we give the following \emph{augmented Lagrangian dual}:

\begin{equation} \label{eq:l1_svr_gen_aug_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda} \min_{\alpha} \quad & \frac{1}{2} \alpha^T Q \alpha + q^T \alpha + \mu^T (A \alpha - b) + \lambda^T (\hat{G} \alpha - \hat{h}) + \frac{\rho}{2} \| A \alpha - b \|^2 + \frac{\rho}{2} \| \hat{G} \alpha - \hat{h} \|^2 \\
    \text{subject to} \quad & \lambda \geq 0
	\end{aligned}
\end{equation}

with $\rho > 0$.

\bigskip

According to this definition, we change the formulation~\ref{eq:l1_svr_lagrangian_dual} as:

\begin{equation} \label{eq:l1_svr_aug_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda} \min_{\alpha} \mathcal{L}(\alpha,\mu,\lambda) &= \frac{1}{2} \alpha^T Q\alpha+q^T\alpha + \mu^T (e^T \alpha) + \lambda^T (\hat{G} \alpha - \hat{h}) + \frac{\rho}{2} \| e^T \alpha \|^2 + \frac{\rho}{2} \| \hat{G} \alpha - \hat{h} \|^2 \\
    \text{subject to} \quad & \,\, \lambda \geq 0
	\end{aligned}
\end{equation}

and the formulation~\ref{eq:l1_svr_bcqp_lagrangian_dual} as:

\begin{equation} \label{eq:l1_svr_bcqp_aug_lagrangian_dual}
	\begin{aligned}
    	\max_{\lambda} \min_{\alpha} \mathcal{L}(\alpha,\lambda) &= \frac{1}{2} \alpha^T (Q + ee^T) \alpha + q^T \alpha + \lambda^T (\hat{G} \alpha - \hat{h}) + \frac{\rho}{2} \| \hat{G} \alpha - \hat{h} \|^2 \\
    \text{subject to} \quad & \,\, \lambda \geq 0
	\end{aligned}
\end{equation}

where $\hat{G} =
\begin{bmatrix}
-I \\
 I
\end{bmatrix}$ and $\hat{h} =
\begin{bmatrix}
-lb & ub
\end{bmatrix}$ with $lb^T = [0, \dots, 0]$, $ub^T = [C, \dots, C]$ and $\rho > 0$.

\pagebreak

\subsection{Squared epsilon-insensitive loss}

The \emph{squared epsilon-insensitive} loss is defined as:

\begin{equation} \label{eq:squared_eps_loss1}
	\mathcal{L}_\epsilon^2 = \max(0, |y - (w^T x + b)| - \epsilon)^2
\end{equation}

or, equivalently:

\begin{equation} \label{eq:squared_eps_loss2}
	\mathcal{L}_\epsilon^2 =
	\begin{cases}
		0 & \text{if} \ |y - (w^T x + b)| \leq \epsilon \\
		(|y - (w^T x + b)| - \epsilon)^2 & \text{otherwise} \\
	\end{cases}
\end{equation}

As the \emph{squared hinge} loss, also the \emph{squared epsilon-insensitive} loss is a strictly convex function and its gradient is given by:

% http://juliaml.github.io/LossFunctions.jl/stable/losses/distance/#L2EpsilonInsLoss-1

\begin{equation} \label{eq:squared_eps_loss_der}
	\nabla_w \mathcal{L}_\epsilon^2=
		\begin{cases}
            \displaystyle 2 \operatorname{sign}(y - (w^T x + b)) (|y - (w^T x + b)| - \epsilon) x & \text{if} \ |y - (w^T x + b)| \geq \epsilon \\
            0 & \text{otherwise} \\
        \end{cases}
\end{equation}

\subsubsection{Primal formulation}

To provide a continuously differentiable function the optimization problem~\eqref{eq:primal_l1_svr} can be formulated as:

\begin{equation} \label{eq:primal_l2_svr}
    \min_{w,b} \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n \max(0, |y_i - (w^T x_i + b)| - \epsilon)^2
\end{equation}

where we make use of the \emph{squared epsilon-insensitive} loss that quadratically penalized slacks $\xi$ and is called $\mathcal{L}_2$-SVR.

The $\mathcal{L}_2$-SVR objective~\eqref{eq:primal_l2_svr} can be rewritten in form~\eqref{eq:reg_bias_primal_svm1} or~\eqref{eq:reg_bias_primal_svm2} as:

\begin{equation} \label{eq:reg_bias_primal_l2_svr}
	\min_{w,b} \frac{1}{2} (\| w \|^2 + b^2) + C \sum_{i=1}^n \max(0, |y_i - (w^T x_i + b)| - \epsilon)^2
\end{equation}

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.4]{img/l2_svr_loss}
  	\caption{Squared epsilon-insensitive loss with different optimization steps}
  	\label{fig:l2_svr_loss}
\end{figure}

\subsubsection{Wolfe dual formulation}

As done for the $\mathcal{L}_1$-SVR we can derive the \emph{Wolfe dual} formulation of the $\mathcal{L}_2$-SVR by obtaining:

\begin{equation} \label{eq:wolfe_dual_l2_svr}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T (Q + D) \alpha + q^T \alpha \\
            \text{subject to} \quad & \alpha_i \geq 0 \ \forall_i \\ & e^T \alpha=0
    \end{aligned}
\end{equation}

or, alternatively, with the regularized bias term by obtaining:

\begin{equation} \label{eq:reg_bias_wolfe_dual_l2_svr}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2}\alpha^T (Q + ee^T + D) \alpha + q^T \alpha \\
            \text{subject to} \quad & \alpha_i \geq 0 \ \forall_i
    \end{aligned}
\end{equation}

where the diagonal matrix $\displaystyle D_{ii} = \frac{1}{2C} \ \forall_i$.

\subsubsection{Lagrangian dual formulation}

In order to relax the constraints in the $\mathcal{L}_2$-SVR \emph{Wolfe dual} formulation~\eqref{eq:wolfe_dual_l2_svr} we define the problem as a \emph{Lagrangian dual} relaxation by embedding them into objective function, so we need to allocate the Lagrange multipliers $\mu$ and $\lambda \geq 0$:

\begin{equation} \label{eq:l2_svr_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda} \min_{\alpha} \mathcal{L}(\alpha,\mu,\lambda) &= \frac{1}{2} \alpha^T (Q+D)\alpha+q^T\alpha + \mu^T (e^T \alpha) - \lambda^T \alpha \\
    &= \frac{1}{2} \alpha^T (Q+D)\alpha + (q + \mu e^T - \lambda)^T \alpha \\
    \text{subject to} \quad & \,\, \lambda \geq 0
	\end{aligned}
\end{equation}

Taking the derivative of the Lagrangian $\mathcal{L}$ wrt $\alpha$ and settings it to 0 gives:

\begin{equation} \label{eq:l2_svr_lagrangian_der_a}
	\frac{\partial \mathcal{L}}{\partial \alpha}=0\Rightarrow (Q+D) \alpha + (q + \mu e^T - \lambda) = 0
\end{equation}

With $\alpha$ optimal solution of the linear system:

\begin{equation} \label{eq:l2_svr_lagrangian_sol}
    (Q+D) \alpha = - (q + \mu e^T - \lambda)
\end{equation}

the gradients wrt $\mu$ and $\lambda$ are:

\begin{equation} \label{eq:l2_svr_lagrangian_der_mu}
	\frac{\partial \mathcal{L}}{\partial \mu}=-e \alpha
\end{equation}

\begin{equation} \label{eq:l2_svr_lagrangian_der_lalbda}
    \frac{\partial \mathcal{L}}{\partial \lambda}=-\alpha
\end{equation}

From~\eqref{eq:svr_min_qp_wolfe_dual} we can notice that the equality constraint $e^T \alpha = 0$ arises form the stationarity condition $\partial_{{b}} \mathcal{W}=0$. So, again, for simplicity, we can again consider the bias term $b$ embedded into the weight vector. In this way the dimensionality of~\eqref{eq:l2_svr_lagrangian_dual} is reduced by removing the multipliers $\mu$ which was allocated to control the equality constraint $e^T \alpha=0$, so we will end up solving exactly the problem~\eqref{eq:reg_bias_wolfe_dual_l2_svr}.

\begin{equation} \label{eq:l2_svr_lb_lagrangian_dual}
	\begin{aligned}
    	\max_{\lambda} \min_{\alpha} \mathcal{L}(\alpha,\lambda) &= \frac{1}{2} \alpha^T (Q + ee^T + D) \alpha+q^T\alpha - \lambda^T \alpha \\
    &= \frac{1}{2} \alpha^T (Q + ee^T + D)\alpha + (q - \lambda)^T \alpha \\
    \text{subject to} \quad & \,\, \lambda \geq 0
	\end{aligned}
\end{equation}

Now, taking the derivative of the Lagrangian $\mathcal{L}$ wrt $\alpha$ and settings it to 0 gives:

\begin{equation} \label{eq:l2_svr_lb_lagrangian_der_a}
	\frac{\partial \mathcal{L}}{\partial \alpha}=0\Rightarrow (Q + ee^T + D) \alpha + (q - \lambda) = 0
\end{equation}

With $\alpha$ optimal solution of the linear system:

\begin{equation} \label{eq:l2_svr_lb_lagrangian_sol}
    (Q + ee^T + D) \alpha = - (q - \lambda)
\end{equation}

the gradient wrt $\lambda$ is:

\begin{equation} \label{eq:l2_svr_lb_lagrangian_der_l}
    \frac{\partial \mathcal{L}}{\partial \lambda}=-\alpha
\end{equation}

\bigskip

Note that since the Hessian matrix $Q$ of the $\mathcal{L}_2$-SVR is symmetric and strictly positive definite, we can find the unique solution of the Lagrangian dual relaxation, i.e.,~\ref{eq:l2_svr_lagrangian_sol} and~\ref{eq:l2_svr_lb_lagrangian_sol}, by solving the system with the Cholesky factorization.

\bigskip

Since the linear algebra methods in the ML context are crucial and also in order to deal with a per-iteration cost equals to the other algorithms described later to provide a coherent comparison of all at the end, we will solve it with a primal-dual optimization method and we modify its definition by adding a strictly convex augmentation term, i.e., a penalty term, in order to improve the actual convergence of the algorithms. So, if we consider a general quadratic optimization problem subject to linear constraints, i.e., equality and inequality constraints, defined as:

\begin{equation}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T Q \alpha + q^T \alpha \\
            \textrm{subject to} \quad & A \alpha = b \\ & G \alpha \leq h \\ & lb \leq \alpha \leq ub
    \end{aligned}
\end{equation}

or, equivalently:

\begin{equation}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T Q \alpha + q^T \alpha \\
            \textrm{subject to} \quad & A \alpha = b \\ & \hat{G} \alpha \leq \hat{h}
    \end{aligned}
\end{equation}

where $\hat{G} =
\begin{bmatrix}
 G \\
-I \\
 I
\end{bmatrix}$ and $\hat{h} =
\begin{bmatrix}
h & -lb & ub
\end{bmatrix}$; we give the following \emph{augmented Lagrangian dual}:

\begin{equation} \label{eq:l2_svr_gen_aug_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda} \min_{\alpha} \quad & \frac{1}{2} \alpha^T Q \alpha + q^T \alpha + \mu^T (A \alpha - b) + \lambda^T (\hat{G} \alpha - \hat{h}) + \frac{\rho}{2} \| A \alpha - b \|^2 + \frac{\rho}{2} \| \hat{G} \alpha - \hat{h} \|^2 \\
    \text{subject to} \quad & \lambda \geq 0
	\end{aligned}
\end{equation}

with $\rho > 0$.

\bigskip

According to this definition, we change the formulation~\ref{eq:l2_svr_lagrangian_dual} as:

\begin{equation} \label{eq:l2_svr_aug_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda} \min_{\alpha} \mathcal{L}(\alpha,\mu,\lambda) &= \frac{1}{2} \alpha^T (Q + D) \alpha+q^T\alpha + \mu^T (e^T \alpha) + \lambda^T (\hat{G} \alpha - \hat{h}) + \frac{\rho}{2} \| e^T \alpha \|^2 + \frac{\rho}{2} \| \hat{G} \alpha - \hat{h} \|^2 \\
    \text{subject to} \quad & \,\, \lambda \geq 0
	\end{aligned}
\end{equation}

and the formulation~\ref{eq:l2_svr_lb_lagrangian_dual} as:

\begin{equation} \label{eq:l2_svr_lb_aug_lagrangian_dual}
	\begin{aligned}
    	\max_{\lambda} \min_{\alpha} \mathcal{L}(\alpha,\lambda) &= \frac{1}{2} \alpha^T (Q + ee^T + D) \alpha + q^T \alpha + \lambda^T (\hat{G} \alpha - \hat{h}) + \frac{\rho}{2} \| \hat{G} \alpha - \hat{h} \|^2 \\
    \text{subject to} \quad & \,\, \lambda \geq 0
	\end{aligned}
\end{equation}

where $\hat{G} =
\begin{bmatrix}
-I
\end{bmatrix}$ and $\hat{h} =
\begin{bmatrix}
-lb
\end{bmatrix}$ with $lb^T = [0, \dots, 0]$ and $\rho > 0$.


\subsection{Membership relation}

Say we have a preterite \(P(x)\) which is true for some values of \(x\). Sets allow us to explore the properties of these values.

We may want to talk about a collection of terms for which \(P(x)\) is true, which we call a set.

To do this we need to introduce new axioms, however first we can add (conservative) definitions to help us do this.

We introduce a new relation: membership. If element \(x\) is in set \(s\) then the following relation is true, otherwise it is false:

\(x\in s\)

Sets are also terms. In first-order logic they will be included in quantifiers. Indeed, in set theory, we aim to treat everything as sets.

If a term is not a member of another term, we can write this using the non-member relation as follows:

\(\forall x \forall s [\neg (x\in s)\leftrightarrow x\not\in s]\)


\documentclass[a4paper]{scrartcl}
\usepackage{amsmath}

\title{Important ML formulas}
\date{}

\begin{document}
\maketitle

\section{Gaussians}

Definition:

\begin{align}
    \mathcal{N}(x|\mu, C) &= \frac{1}{|C| (2 \pi)^{d/2}} \exp\left[ -\frac{1}{2} (x - \mu)^T C^{-1} (c - \mu) \right]
\end{align}

Marginals and conditionals (see Bishop, (2.113) - (2.117)):

\begin{align}
    p(x) &= \mathcal{N}(x|\mu, \Lambda^{-1}) \\
    p(y|x) &= \mathcal{N}(y|A x + b, L^{-1}) \\
    p(y) &= \int \mathrm{d}x\; p(y|x)p(x)
        = \mathcal{N}(y|A\mu + b, L^{-1} + A \Lambda^{-1} A^T) \\
    p(x|y) &= \frac{p(y|x)p(x)}{p(y)}
        = \mathcal{N}(y|\Sigma \left\{ A^T L (y - b) + \Lambda \mu \right\}, \Sigma) \\
    \Sigma &= \left( \Lambda  + A^T L A \right)^{-1}
\end{align}

Conditioning on blocks

\begin{align}
    p(x) &= \mathcal{N}(x|\mu, C) \\
    C &= \begin{pmatrix}
        C_N & k \\
        k^T & c
    \end{pmatrix} \\
    x &= (x_N, x_{N+1})^T \\
    \mu &= (\mu_N, \mu_{N+1})^T
\end{align}

Then:

\begin{align}
    p(x_{N+1}|x_{N}) &=
        \mathcal{N}\left( x_{N+1} | \tilde{\mu}_{N+1}, \tilde{C}_{N+1} \right) \\
    \tilde{\mu}_{N+1} &= \mu_{N+1} + k^T C_N^{-1} (x_N - \mu_N) \\
    \tilde{C}_{N+1} &= c - k^T C_N^{-1} k
\end{align}

\section{Laplace approximation}
\begin{align}
    \log p(x)
        &\approx \log p(x^\star) + \frac{1}{2} \left(x - x^\star \right)^T H \left(x - x^\star\right) \\
        &= \mathcal{N}(x|x^\star, H^{-1}) \\
    &\left. \frac{\partial}{\partial x} \log p(x) \right|_{x^\star} = 0 \\
    &H_{ij} = \frac{\partial^2}{\partial x_i \partial x_j} \log p(x)
\end{align}

\section{Sigmoids}
\begin{align}
    \sigma(x) &= \frac{1}{1 - \exp(-x)} \\
    \sigma(-x) &= 1 - \sigma(x)  \\
    \log \frac{\sigma(x)}{1 - \sigma(x)} &= x \\
    \frac{\partial}{\partial x}\sigma(x) &= \sigma(x) (1 - \sigma(x))
\end{align}

\section{Block Matrix Inverse}
\begin{align}
    \begin{bmatrix}
            A_{00} & A_{01} \\
            A_{10} & A_{11}
    \end{bmatrix}^{-1}
    &= \begin{bmatrix}
        A_{00}^{-1} + F E^{-1} G & -F E^{-1} \\
        -E^{-1} G & E^{-1}
    \end{bmatrix} \\
    E &= A_{11} - A_{10} A_{00}^{-1} A_{01} \\
    F &= A_{00}^{-1}A_{01} \\
    G &= A_{10} A_{00}^{-1}
\end{align}

\end{document}