\documentclass[../main.tex]{subfiles}

\begin{document}

\setcounter{chapter}{0}
\chapter{Reflections on the Notion of Space I}

The purpose of these first lectures is to understand the notion of a manifold in different contexts (topological/differential/analytic manifolds). We begin in this first lecture by looking at the case of topological manifolds.

\section{Reminders on topological manifolds}

\begin{defn}

\begin{enumerate}
    \item A \textbf{topological manifold} is a topological space $X$ that has an open cover $\{U_i\}_{i \in I}$, such that for each $i \in I$, there exists a homeomorphism from $U_i$ to an open set of $\mathbb R^{n_i}$ (for some $n_i\ge 0$ depedent on $i$).
    \item The category of topological manifolds is the full subcategory of topological spaces whose objects are topological manifolds. It is denoted by $\mathbf{TopMfd}$.
\end{enumerate}

\end{defn}

Let $X$ be a topological manifold and $\{U_i\}_{i \in I}$ an open cover as in definition $1.1 (1)$. We write, for $i$ and $j$ in $I$, $U_{i,j} = U_n \cap U_j$. There's a diagram of topological spaces

\[
    \bigsqcup_{(i, j) \in I^2} U_{i, j} \rightrightarrows \bigsqcup_{i \in I} U_i
\]

The first morphism sends $U_{i, j}$ to $U_i$ via the natural inclusion $U_{i,j}
subset U_i$, and the second morphism sends $U_{i, j}$ to $U_j$ via the natural inclusion $U_{i, j}
\subset U_j$. There's also another morphism

\[
    \bigsqcup_{i \in I} U_i \to X
\]

which are inclusions $U_i \subset X$. This coequalizes the two morphisms above. So we get a morphism of topological spaces

\[
Colim\left(\bigsqcup_{(i, j) \in I^2} U_{i, j} \rightrightarrows \bigsqcup_{i \in I} U_i\right) \to X
\]

The important fact is the following

\begin{lem}

The morphism

\[
Colim\left(\bigsqcup_{(i, j) \in I^2} U_{i, j} \rightrightarrows \bigsqcup_{i \in I} U_i\right) \to X
\]

is an isomorphism.
\end{lem}

\begin{proof}
The lemma says for a topological space $Y$, to give a morphism $f: X \to Y$ is the same as choosing, for each given $i \in I$, a morphism $f_i: U_i \to Y$ such that $(f_i)|_{U_{i, j}} = (f_j)|_{U_{i, j}}$ for all $(i, j) \in I^2$.

(Exercise: provide the details.)
\end{proof}

One can interpret the above lemma as follows: all topological manifolds are obtains as the colimit of a diagram of opens in $\mathbb R^n$ (for some $n$). We draw from this the following principle:

\begin{prcp*}
The category $TopMfd$ of topological manifolds is deduced from the category of opens in $\mathbb R^n$ (and continiuous maps.).
\end{prcp*}

this is the principle that we're going to clarify in the following.

\section{Manifolds and sheaves}

Let $C$ be the full subcategory of $TopMfd$ whose objects are opens of $\mathbb R^n$. We denote by $Pr(C)$ the category of presheaves of sets over $C$ (also denoted $\hat C$). We consider the Yoneda embedding restricted to $C$:

\begin{align}
h_{(-)}: VarTop &\to Pr(C)\\
X &\mapsto h_X
\end{align}

where the presheaf $h_X$ is defined by

\[
h_X(Y) := Hom_{TopMfd}(Y, X)
\]

\begin{lem}
    The functor $h_{(-)}$ above is fully faithful.
\end{lem}

\begin{proof}
The functor is faithful: ...

The functor is full: ...

TODO: finish
\end{proof}

The lemma 2.1 is a good point of depature, for $TopMfd$ is identified as (is equivalent to) a full subcategory of $Pr(C)$. We seek a characterization of this subcategory.

We start with $C$ a Grothendieck site by declaring a family of morphisms $\{U_i \to U\}_{i \in I}$ in C to be a covering family if each morphism $U_i \to U$ is an open immersion, and the total morphism $\sqcup_{i \in I} U_i \to U$ is surjective. This defines a pre-topology on $C$ (exercise: verify). The associated topology is denoted $\tau$.

\begin{lem}
    For all $X \in TopMfd$, the presheaf $h_X \in Pr(C)$ is a sheaf for the topology $\tau$.
\end{lem}

\begin{proof}
    ...
\end{proof}

Lemma 2.2 implies we have a full and faithful functor

\[
h_{(-)}: TopMfld \to Sh(C, \tau)
\]

A sheaf isomorphic to $h_X$ is said to be representable by $X$. More generally we identify the category $TopMfd$ with its image in $Sh(C, \tau)$

To chacaterize this image we make a definition

\begin{defn}
    \begin{enumerate}
            \item A morphism $f: F \to G$ in $Sh(C, \tau)$ is a \textbf{local homeomphism} if for all $X \in C$, and all morphisms $h_X \to G$, the sheaf $F \times_G h_X$ is representable by some $Y \in TopMfd$, and the induced morphism $Y \to X$ by the projection $h_y \simeq F \times_G h_X \to h_X$ is a local homeomorphism of topological spaces \footnote{Recall: a continuous map of topological spaces is a local homeomorphism if for each $x \in X$, there exists $U$ an open neighborhood of $x$ in $X$ and $V$ and open neighborhood of $f(x)$ in Y, such that $f$ induces a homeomorphism from $U \to V$}.

            \item A morphism in $Sh(C, \tau)$ is an \textbf{open immersion} if it's a monomorphism and a local homeomorphism.
    \end{enumerate}
\end{defn}

It's easy to verify that open immersions in $Sh(C, \tau)$ are stable under compositions (Exercise: verify). One can also verify that local heomorphisms are stable under compositions, but this needs corrolary 2.5 below (Exercise: verify). We see also that a morphism of topological manifolds is a local homeomorphism if and only if $h_X \to h_Y$ is a local homeomorphism in the sense of the above definition (Exercise: verify).

We then have the following proposition

\begin{prop}
    A sheaf $F \in Sh(C, \tau)$ is representable by a topological manifold (i.e. $F \simeq h_X$ for some $X \in TopMfd$), if there exists a family $\{U_i\}_{i \in I}$ of objects in $C$, and a morphism of sheaves

    \[
        p: \bigsqcup_{i \in I} h_{U_i} \to F
    \]
    satisfying the following two conditions

    \begin{enumerate}
        \item The morphism $p$ is an epimorphism of sheaves.
        \item For all $i \in I$, the morphism $U_i \to F$ is an open immersion (in the sense of definition 2.3)
    \end{enumerate}
\end{prop}

\begin{proof}
\end{proof}

\begin{cor}
    Let $X \in TopMfd$, and $F \to h_X$ a morphism of sheaves. If there exists an open cover of $X$ such that for all $i \in I$, the sheaf $f \times_{h_X} h_{U_i}$ is representable by a topological manifold, then $F$ is representable by a topological manifold.
\end{cor}
\begin{proof}
    For all $i \in I$, choose ${V_{i, j}}_{j \in J}$...
\end{proof}

\section{Quotient manifolds}

Let G be a (discrete) group acting on a topological manifold $X \in TopMfd$. By functoriality, the group $G$ acts on the sheaf $h_X$. Recall a group action on X is ... . Recall also a group action $G$ on $X$ is properly discontinuous if all points $x \in X$ has an open neighborhood $U \subset X$ such that for all $g \in G$, we have

\[
    g(U) \cap U \ne \emptyset \implies (g = e)
\]

In the following, we will take care not to confuse the sheaf quotient $h_X/G$ and the sheaf $h_{X/G}$ represented by the quotient topological space.

\begin{prop}
    \begin{enumerate}
            \item If the action $G$ on $X$ is free, the the quotient morphism
            \[h_X \to h_X/G\] is a local homeomorphism
            \item If the action $G$ on $X$ is properly discontinuous, then the quotient $h_X/G \in Sh(C, \tau)$ is a topological manifold.
    \end{enumerate}
\end{prop}
\begin{proof}
...
\end{proof}

\section{Shortcomings of manifolds}

The proposition 3.1 is a good reason for cosntructing examples of topological manifolds by properly discontinuous actions. However, when $G$ acts on a manifold $X$ but the action is not propertly discontinuous, the topological space quotient $X/G$ is in general very pathological. The sheaf quotient $h_X/G$. The sheaf quotient $h_X/G$ has good properties (e.g. point (1) of proposition 3.1) similar to representability of a topological manifold.

An example is the following: we take the action of the discrete group $\mathbb Q$ (under addition on the topological space $\mathbb R$ via the morphism

\[\mathbb R \times \mathbb Q \to \mathbb R\]

given by $(x, t) \mapsto x + t$. We note this action is free, but not properly discontinuous. IN addition, the morphism $\mathbb R \to \mathbb R / \mathbb Q$ is not a local homeomorphism nor is it locally injective. Finally, the topological space quotient $\mathbb R / \mathbb Q$ has a gross topology. We see that the quotient $\mathbb R/ \mathbb Q$ is not a reasonable object from the point of view of geometry. On the otherhand, the sheaf quotient $h_\mathbb R/\mathbb Q$ is more interesting, for the morphisms $h_\mathbb R \to h_\mathbb R / \mathbb Q$ is a local homeomorphism. The sheaf $h_\mathbb R / \mathbb Q$ is a primary example of a geometric space

\begin{defn}
    A sheaf $F \in Sh(C, \tau)$ is a geometric space if there exists a family of objects $\{U_i\}_{i \in I}$ of $C$, and a morphism of sheaves

    \[
        p: \bigsqcup_{i \in I} h_{U_i} \to F
    \]
    satisfies the following two conditions

    \begin{enumerate}
        \item The morphism $p$ is an epimorphism of sheaves
        \item For all $i\in I$, the morphism $U_i \to F$ is a local homeomorphism.
    \end{enumerate}
\end{defn}

\end{document}


The implementation of Binomial Heaps we used,
along with Trees,
can be found in pages 68-72 of the reference book (\cite{Okasaki}).

\subsection{Data Structure Overview}
A Binomial Heap is an implementation of mergeable (or meldable) priority queue.
It uses Trees that we had to implement as well.
Let us first see their implementation.

\subsubsection{Trees Overview}
A Tree is simply represented as a Node with an element
(the priority, of totally ordered type), a rank and a list of Trees as children.
The rank, which is $\geq 0$, is to be understood as follows:
a Node of rank $k$ has $k$ children of ranks $k$, $k-1$, ..., $0$,
a rank $0$ meaning a Node without children.
From that it comes that a Tree with a root Node of rank $k$
contains in total $2^k$ elements,
so a Tree can represent a power of two,
like the \emph{k-th} bit in a binary number.

The Trees must satisfy the \emph{Minimum Heap Property},
which means that the element of a parent Node
is less or equal than any of its children Node's elements.
Also, in the implementation, the children list is maintained in decreasing order of rank.
These properties are thus two invariants to check on Trees,
along with the domain of the rank.

The \verb|link| operation on Trees takes two Trees of rank $k$,
makes the one with bigger root element the first child of the other
so that both invariants are satisfied.
As a result, the returned tree is of rank $k+1$.
It is thus like the addition of two bits of a binary number.
The operation is done in constant time.

Let us now see how the Trees are used in the Binomial Heap implementation.

\subsubsection{Binomial Heaps Overview}
A Binomial Heap is implemented as a list of Trees
which is kept in increasing order of rank.
There must also not be more than one Tree of a particular rank.
These properties are thus invariants to check.

We can then visualize a well-formed Binomial Heap as the binary number of its size,
with the Trees in the list as the $1$-valued bits of the number.

With the help of the \verb|link| operation on Trees,
we can see a \verb|merge| of two Binomial Heaps
as an addition of the two binary numbers that represent them.
In this view, \verb|link| is used to add two bits of same position,
and the result of the operation is the carry.

The other operations on Binomial Heaps are \verb|insert|, \verb|findMin| and \verb|deleteMin|.
The \verb|insert| operation creates a $0$-ranked Tree with the element to insert
and puts the Tree in the Heap, using an \verb|insertTree| function,
also used by \verb|merge| to insert the carry in the recursively merged Heaps,
at each carry step.
This function inserts the Tree only if its rank is smaller or equal to the smallest rank in the Tree list of the Heap
(either it adds it to the list, either it recursively \verb|link|s and \verb|insertTree|s),
so it had to be checked that the function is not called with a Tree of bigger rank in the implementation.

The \verb|findMin| operation needs only to look for the minimum element
in the root Nodes of the Trees in the Tree list.
This is because the Trees satisfy the \emph{Minimum Heap Property}.

Finally, the \verb|deleteMin| operation finds the Tree with the minimum root in the list,
removes its root
and reverses its children so that the children list is of the correct form for a Tree list of a Binomial Heap.
After that, it \verb|merge|s this new Heap with the Heap composed of
the remaining Trees of the original Binomial Heap.

All operation are done in amortized logarithmic time.
Let us see now the verification of the implementations
for both Trees and Binomial Heaps with Leon.

\subsection{Verification With Leon}
As for the first data structure verification presented in this report,
 \verb|size|, \verb|content| and \verb|toList| functions were used
 for pre- and post-conditions of operations,
as well as the invariants for Trees and Binomial Heaps we saw above.

Unfortunately we managed to prove only 46 properties over 53.
We have tried to tweak our model in order to help Leon proving some of them,
but unfortunately the recursive structure of Trees prevented us to reduce the number of \verb|Unknown|s.

Some tests were written for the Binomial Heaps and
it helped to find some mistakes we made in writing the code and
which were not found by Leon.
For example, an invariant was too strong regarding the ordering of the Trees in the Heap and
the error could be seen only with tests,
when a precondition for an operation was not satisfied.

We had some difficulties to state the total order of the elements' type
(\texttt{T <: Ordered[T]} was not understood by Leon),
so we decided to use \verb|BigInt|s to be able to verify the data structure,
as it can be done without loss of generality regarding the logic of the implementation.
We also had to adapt a bit our implementation in order to circumvent some of Leon's limitations.
For instance, a Heap is represented as a \texttt{List[Tree]}.
It could therefore have been interested to use an implicit class to operate on such structure,
a construct that is not yet available in Leon.

Moreover, our data structures used standard constructs such as lists,
but with some specific properties.
In order to enforce those properties,
we had to verify the invariants as \texttt{require}s at the beginning of each \texttt{def}.
It would have been nice instead to be able to write requirements directly inside \texttt{case class}es,
as it would have remove some boilerplate code.

Leon could unfortunately not state as valid most of complicated, crucial operations,
which are still \verb|Unknown|s.


\section*{Bayesian asymptotics}
\begin{frame}{Asymptotics}
A major part of a statistical approach is understanding what happens in the limit of many many observations.
Consider the joint conditional density of the data, $f_n(\boldsymbol{x} \mid \theta)$ and a prior $\pi(\theta)$.
What happens to $p_n(\theta \mid \boldsymbol{x}) = f_n(\boldsymbol{x} \mid \theta)\pi(\theta)/m_n(\boldsymbol{x})$ as $n \to \infty$ ?
\begin{idea}[Asymptotics is about understanding]
 Infinity is a big ``number''.
 Considering what happens  as $n \to \infty$ is less a statement about a real world situation than about the structure and regularity of a model.
 Doing asymptotics is about understanding what makes a model tick rather than getting useful results for a regime seldom achieved in practice.
\end{idea}
Another important aspect to consider is the \textbf{rate} at which things converge asymptotically.
Studying rates provides complementary information about the structure of the model and gives hints as to the accuracy of asymptotic approximations.
 \end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Bayesian asymptotics I: consistency}
\begin{theo}[The posterior concentrates around the ``true'' value]
Let $(S, \mathcal{A}, \mu)$ be a probability space and let $(\Omega, \tau)$ be a finite-dimensional parameter space equipped with a Borel $\sigma$-field.
Suppose there exist measurable $h_n: \mathcal{X}^n \to \Omega$ such that $h_n(\boldsymbol{X})$ converges in probability to $\Theta$.
Writing $\mu_{\boldsymbol{\Theta} \mid \boldsymbol{X}}(\cdot \mid \boldsymbol{x})$ for the posterior measure, we have
\begin{equation*}
 \lim_{n \to \infty} \mu_{\boldsymbol{\Theta} \mid \boldsymbol{X}}(A \mid \boldsymbol{X}) = I_A(\Theta), \: \mu-\textrm{a.s.}
\end{equation*}
\end{theo}
\textbf{Please} see Theorem 7.78 in \cite{Schervish1995} (pg 429) for all of the \textit{many} details.

\textbf{Discussion:} what we are essentially saying here is that if there exists a consistent (sequence of) estimator(s) for $\theta$, then the posterior will concentrate around the true generating distribution of the parameter asymptotically.
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Remember Cromwell's law?}
Here is another neat little theorem with a cumbersome proof.
\begin{theo}[A ``nice'' prior ensures posterior consistency]
Define $\operatorname{KL}(\theta, \theta^\prime)$ as the Kullback-Leibler divergence between $P_{\theta}$ and $P_{\theta^\prime}$.
Let $\theta_0$ be the true data-generating parameter and define $C_\epsilon = \{\theta : \operatorname{KL}(\theta_0, \theta) < \epsilon\}$, $\epsilon > 0$.
 Let $\Pi$ be a prior measure such that $\Pi(C_\epsilon) > 0$ for every $\epsilon > 0$.
 Take $N_0$ open such that $C_\epsilon \subset N_0$.
 Then
 \begin{equation*}
  \lim_{n \to \infty} \mu_{\boldsymbol{\Theta} \mid \boldsymbol{X}}(N_0 \mid \boldsymbol{X}) = 1, \: P_{\theta_0}-\textrm{a.s.}
 \end{equation*}
\end{theo}
Again, \textbf{please} see Theorem 7.80 in \cite{Schervish1995} (pg 430) for the details.
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[allowframebreaks]{Interlude: regularity conditions}
Before we proceed, we will need to make things nice.
Consider the following regularity conditions
\begin{itemize}
 \item[1] The parameter space is $\boldsymbol{\Theta} \subset \mathbb{R}^d$ for some finite $d$;
 \item[2] We have $\theta_0$ an an interior point of $\boldsymbol{\Theta}$;
 \item[3] The prior distribution has a density w.r.t. Lebesgue which is positive and continuous at $\theta_0$;
 \item[4] There exists $N_0 \subseteq \boldsymbol{\Theta}$ with $\theta_0 \in N_0$ such that the log-likelihood, $l_n(\theta)$, is twice-differentiable with respect to all coordinates of $\theta$, $P_{\theta}$-a.s.
 \item[5] The largest eigenvalue of the inverse observed Fisher information, $\Sigma_n$, vanishes in probability.
 \item[6] The MLE is consistent;
 \item[7] The Fisher information is a smooth function of $\theta$.
\end{itemize}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Bayesian asymptotics II: asymptotic normality}
We can now state a nice result which characterises the asymptotic form of the posterior.
\begin{theo}[Bernstein von-Mises\footnote{Named after Austrian mathematician Richard Edler von Mises (1883--1953) and Russian mathematician Sergei Natanovich Bernstein (1880--1968).}]
Under the regularity conditions we have discussed, take $\hat{\theta}$ to be the MLE.
Put $\boldsymbol{\Psi}_n = \left(\Sigma_n\right)^{-1/2}(\theta- \hat{\theta})$.
Then the posterior distribution of $\boldsymbol{\Psi}_n$ conditional on $\boldsymbol{X}$ converges in probability \textbf{uniformly} on compact sets to the multivariate normal distribution $\operatorname{Normal}_d\left(\boldsymbol{0}, \boldsymbol{I}_d\right)$ with density $\phi_d$.
More precisely,
\begin{equation*}
  \lim_{n \to \infty} P_{\theta_0}\left(\sup_{\psi \in B} \bigg\rvert f_{\boldsymbol{\Psi}_n \mid \boldsymbol{X}}(\psi) - \phi_d(\psi)  \bigg\lvert > \epsilon \right) = 0,
\end{equation*}
for all $B \subset \mathbb{R}^d$ compact and $\epsilon > 0$.
\end{theo}
See Theorem 7.89 in \cite{Schervish1995} (page 437).
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Dabbling with normal approximations}
\begin{exercise}[Cauchy location posterior]
 Take $X_i \sim \operatorname{Cauchy}(\theta, 1)$, $i = 1, 2,\ldots, 10$.
 In particular, suppose $\boldsymbol{x} = \{-5, -3, 0, 2, 4, 5, 7, 9, 11, 14\}$.
 \begin{itemize}
  \item[i)] Compute the MLE and $l^{\prime\prime}$;
  \item[ii)] Deduce the parameters of the normal approximation to $p(\theta \mid \boldsymbol{x})$;
  \item[iii)] Use an MCMC\footnote{The instructor can assist with this step.} routine to sample from $p(\theta \mid \boldsymbol{x})$, obtain a posterior approximation to its density and compare it to the normal approximation;
  \item[iv)] Simulate data sets of sizes $n=20, 50, 100, 500, 1000$ and $10, 000$ and repeat iii.
  \item[v)] See if you can reduce/increase the discrepancy between the posterior and its approximation by fiddling with the prior (without breaking the regularity assumptions!).
 \end{itemize}
\end{exercise}
See example 7.104 in \cite{Schervish1995} (page 444).
 \end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Recommended reading}
\begin{itemize}
  \item[\faBook] \cite{Schervish1995} Ch. 7.4.
%  \item
 \item[\faForward] Next lecture: \cite{Raftery1988} and~\cite{Gelman2002}.
 \end{itemize}
\end{frame}


\documentclass[]{article}

\usepackage{amsmath,amssymb,amsbsy}
\usepackage[letterpaper,total={6in,8in}]{geometry}
\usepackage[colorlinks]{hyperref}

\usepackage{color}

\ifpdf
  \RequirePackage{graphicx}
  \RequirePackage{epstopdf}
  \DeclareGraphicsExtensions{.pdf,.jpeg,.png,.jpg}
\else
  \RequirePackage[dvipdfmx]{graphicx}
  \RequirePackage{bmpsize}
  \DeclareGraphicsExtensions{.eps,.pdf,.jpeg,.png,.jpg}
\fi
\graphicspath{{pics/},}

\RequirePackage{algorithm}
\RequirePackage{algorithmic}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
\providecommand{\algorithmautorefname}{Algorithm}

\RequirePackage{subfigure}
\RequirePackage{empheq}
\providecommand{\subfigureautorefname}{\figureautorefname}

%opening
\title{Theory Report for ``Multichannel Deconvolution of Skin Conductance Data: Concurrent Separation of Tonic and Phasic Component''}
\author{Yuchen Jin, Jin Lu}

\providecommand{\od}{\mathrm{d}}

\begin{document}

\maketitle

\section{Theory}

For any channel $n$, the skin conductance (SC) could be decomposed by a phasic component and a tonic component,
\begin{align}
  y_{\rm{SC}_n}(t) = p_n(t) + s_n(t) + \nu_n(t),
\end{align}
where the phasic component $p_n(t)$ is stimulated by the autonomic nervous system (ANS), and the tonic component $s_n(t)$ is influenced by the thermoregulation. In this work, we will model both components by different methods. The phasic component is decomposed by a series of state-space models sharing the same input, while the tonic component is described as a combination of different spline functions. Based on the theory-based multichannel forward modeling, the stimuli from the ANS could be solved by a joint optimization. In the meanwhile, the tonic component would be extracted from the data.

\subsection{Model the phasic component with a multichannel state-space representation}
According to the presumption in \cite{alexander2005separating,society2012publication,amin2019robust}, the phasic components detected in different regions of the body are regulated by the same ANS signal, the detected phasic component for channel $n$ could be formulated by
\begin{align}
  p_n(t) = \alpha_n \zeta_n (t),
\end{align}
{\color{red} where $\alpha_n$ denotes the attenuation caused by the conduction of the EDA,} $\zeta_n(t)$ is a internal variable representing the conducted signal from the same ANS. Mathematically, $\zeta_n(t)$ is modeled by a smoothing filter applied to the neural stimuli $u(t-\beta_n)$. The conduction delay $\beta_n$ shows that the same stimuli would be detected at different moments. Generally, the neural stimuli could be defined as a time-series spike signal, i.e. $u(t) = \sum_{i=1}^N q_i \delta(t - \Delta_i)$. Given $u(t)$, \textit{Alexander et al.}~\cite{alexander2005separating} provides a second-order ordinary differential equation to describe $\zeta_n(t)$,
\begin{align} \label{fml:the:ode}
  \tau_r \tau_d \frac{\od^2 \zeta_n(t)}{\od t^2} + (\tau_r + \tau_d) \frac{\od \zeta_n(t)}{\od t} + \zeta_n(t) = u(t - \beta_n).
\end{align}

The difference between different channels in \eqref{fml:the:ode} is only the time delay. In other words, the solutions of $\zeta_n(t)$ for different $n$ are the same except the time delay. To solve \eqref{fml:the:ode}, \textit{Faghih et al.}~\cite{faghih2015characterization} propose a state space model. Incorporating the time delay into the internal signal $\zeta_n(t)$, we denote another internal state $x^{(n)}_2 (t)$ as $\zeta_n(t + \beta_n)$. The state-space model could be formulated as the following form,
\begin{subequations} \label{fml:the:state-raw}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    \dot{x}^{(n)}_1 (t) &= a x^{(n)}_1(t) + b u(t),\\
    \tau_r \tau_d \dot{x}^{(n)}_2 (t) &= c x^{(n)}_1(t) + d x^{(n)}_2(t), \label{fml:the:state-raw-2}\\
    y_n (t) &= \alpha_n x^{(n)}_2(t).
  \end{empheq}
\end{subequations}
where $x^{(n)}_1(t)$ is another internal state. To solve the coefficients $a, b, c, d$, we eliminate $x^{(n)}_1(t)$ in \eqref{fml:the:state-raw-2},
\begin{equation} \label{fml:the:ode-from-state}
  \begin{aligned}
    \tau_r \tau_d \ddot{x}_2 (t) &= c \dot{x}^{(n)}_1(t) + d \dot{x}^{(n)}_2(t) = c \left(a x^{(n)}_1(t) + b u(t) \right) + d \dot{x}^{(n)}_2(t) \\
    &= a ( \tau_r \tau_d \dot{x}^{(n)}_2 (t) - d x^{(n)}_2(t) ) + bc u(t) + d \dot{x}^{(n)}_2(t), \\
    &= (a \tau_r \tau_d + d) \dot{x}^{(n)}_2 (t) - a d x^{(n)}_2(t) + bc u(t).
  \end{aligned}
\end{equation}

To ensure the coherence between \eqref{fml:the:ode} and \eqref{fml:the:ode-from-state}, we have,
\begin{subequations}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    a \tau_r \tau_d + d &= - \tau_r - \tau_d,\\
    ad &= 1, \\
    bc &= 1.
  \end{empheq}
\end{subequations}

Given $a = -\frac{1}{\tau_r}$ and $b = \frac{1}{\tau_r}$, we could solve the above equations, and substitute the solutions into \eqref{fml:the:state-raw},
\begin{subequations} \label{fml:the:state-sing}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    \begin{bmatrix}
      \dot{x}^{(n)}_1 (t) \\ \dot{x}^{(n)}_2 (t)
    \end{bmatrix} &= \begin{bmatrix}
      -1/{\tau_r} & 0 \\ 1/{\tau_d} & -1/{\tau_d}
    \end{bmatrix} \begin{bmatrix}
      x^{(n)}_1(t) \\ x^{(n)}_2 (t)
    \end{bmatrix} + \begin{bmatrix}
      1/{\tau_r} \\ 0
    \end{bmatrix} u(t), \\
    y_n (t) &= \alpha_n x^{(n)}_2(t).
  \end{empheq}
\end{subequations}

Given the single-channel transmission matrix $\phi=\begin{bmatrix}
-1/{\tau_r} & 0 \\ 1/{\tau_d} & -1/{\tau_d}
\end{bmatrix}$, we could derive \eqref{fml:the:state-sing} into the multichannel form. When we have $\chi$ channels of data, the multichannel state-space model is
\begin{subequations} \label{fml:the:state-mul}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    \dot{\mathbf{x}} (t) &= \mathbf{A} \mathbf{x} (t) + \mathbf{B} u (t), \\
    \mathbf{y} (t) &= \mathbf{C} \mathbf{x} (t),
  \end{empheq}
\end{subequations}
where $\mathbf{x} (t) = \begin{bmatrix}
  x^{(1)}_1 (t) \\ x^{(1)}_2 (t) \\ \vdots \\ x^{(\chi)}_1 (t) \\ x^{(\chi)}_2 (t)
\end{bmatrix}$, $\mathbf{y} (t) = \begin{bmatrix}
  y_1 (t) \\ y_2 (t) \\ \vdots \\ y_{\chi} (t)
\end{bmatrix}$, $\mathbf{A} = \begin{bmatrix}
  \phi & \mathbf{0} & \cdots & \mathbf{0} \\
  \mathbf{0} & \phi & \cdots & \mathbf{0} \\
  \vdots & \vdots & \ddots & \vdots \\
  \mathbf{0} & \mathbf{0} & \cdots & \phi
\end{bmatrix}$, $\mathbf{B} = \begin{bmatrix}
  1/{\tau_r} \\ 0 \\ \vdots \\ 1/{\tau_r} \\ 0
\end{bmatrix}$, and\\$\mathbf{C} = \begin{bmatrix}
  0 & \alpha_1 & 0 & 0 & \cdots & 0 & 0 \\ 0 & 0 & 0 & \alpha_2 & \cdots & 0 & 0 \\ \vdots & \vdots & \vdots & \vdots & \ddots & \vdots & \vdots \\ 0 & 0 & 0 & 0 & \cdots & 0 & \alpha_{\chi}
\end{bmatrix}$.

The above model could be rewritten as a discrete model. Denote the sampling rates of the observation and the neural stimuli as $T_y$ and $T_u$ respectively, we could formulate the discretion of the time as $t_k = k T_y$ and $\Delta_i = i T_u$ for both signals. When $T_u$ is small enough, we could approximate the discrete neural stimuli by $\mathbf{u} = \begin{bmatrix}
  q_1 & q_2 & \cdots & q_N
\end{bmatrix}^T$, where we use $q_i=0$ to represent no signal at the time step $\Delta_i$. Based on these configurations, we could derive the discrete model matrices by
\begin{subequations} \label{fml:the:state-dis-mat}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    \mathcal{A} &= \mathcal{L}^{-1}\{ s \mathbf{I} - \mathbf{A} \}(T_u), \\
    \mathcal{B} &= \mathbf{A}^{-1}(\mathcal{A} - \mathbf{I})\mathbf{B}, \\
    \mathcal{C} &= \mathbf{C},
  \end{empheq}
\end{subequations}
where $\mathcal{L}^{-1}$ is the inverse Laplace transform.

The discrete form of \eqref{fml:the:state-mul} is
\begin{subequations}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    \mathbf{x} [k+1] &= \mathcal{A} \mathbf{x} [k] + \mathcal{B} \mathbf{u} [k], \\
    \mathbf{y} [k] &= \mathcal{C} \mathbf{x} [k],
  \end{empheq}
\end{subequations}

In practice, we assume that $T_y = L T_u$, where $L$ is a integer. In this case, we could denote the state vector based on the sampling rate $T_y$ by $\mathbf{z}[k] = \mathbf{x}[Lk]$. Then we have $\mathcal{A}_d = \mathcal{A}^L$, $\mathcal{B}_d = \begin{bmatrix}
  \mathcal{A}^{L-1} \mathcal{B} & \mathcal{A}^{L-2} \mathcal{B} & \cdots \mathcal{B}
\end{bmatrix}$, $\mathbf{u}_d[k] = \begin{bmatrix}
  \mathbf{u}[Lk] & \mathbf{u}[Lk+1] & \cdots \mathbf{u}[Lk+L-1]
\end{bmatrix}^T$. By this way, the discrete model is finally formulated as
\begin{subequations} \label{fml:the:state-dis}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{empheq}[left=\empheqlbrace]{align}
    \mathbf{z} [k+1] &= \mathcal{A}_d \mathbf{z} [k] + \mathcal{B}_d \mathbf{u}_d [k], \\
    \mathbf{y} [k] &= \mathcal{C} \mathbf{z} [k],
  \end{empheq}
\end{subequations}

Since the system is casual, we have
\begin{align} \label{fml:the:out-dis}
  \mathbf{y}[k] = \mathcal{F}[k] \mathbf{z}_0 + \mathcal{D}[k] \mathbf{u},
\end{align}
where $\mathcal{F}[k] = \mathcal{C}\mathcal{A}_d^k$, $\mathcal{D}[k] = \mathcal{C} \begin{bmatrix}
  \mathcal{A}_d^{k-1} \mathcal{B}_d & \mathcal{A}_d^{k-2} \mathcal{B}_d & \cdots \mathcal{B}_d & \mathbf{0}_{N-kL}
\end{bmatrix}$ and \\$\mathbf{u} = \begin{bmatrix}
  \mathbf{u}_d[0] & \mathbf{u}_d[1] & \cdots & \mathbf{u}_d[M-1]
\end{bmatrix}^T$. The initial condition is configured as \\$\mathbf{z}_{0} = \mathbf{z}[0] = \begin{bmatrix}
  0 & y_1(0) & 0 & \frac{y_2(0)}{\alpha_2} & \cdots & 0 & \frac{y_{\chi}(0)}{\alpha_{\chi}}
\end{bmatrix}^T$.

Let $\mathbf{y} = \begin{bmatrix}
\mathbf{y}[1]^T & \mathbf{y}[2]^T & \cdots & \mathbf{y}[M]^T
\end{bmatrix}^T$, $\mathcal{F}_{\boldsymbol{\theta}} = \begin{bmatrix}
\mathcal{F}[0] & \mathcal{F}[1] & \cdots & \mathcal{F}[M-1]
\end{bmatrix}^T$ and \\$\mathcal{D}_{\boldsymbol{\theta}} = \begin{bmatrix}
\mathcal{D}[0] & \mathcal{D}[1] & \cdots & \mathcal{D}[M-1]
\end{bmatrix}^T$, we can represent the whole multichannel phasic component by
\begin{align} \label{fml:the:out-dis-all}
  \mathbf{y} = \mathcal{F}_{\boldsymbol{\theta}} \mathbf{z}_{0} + \mathcal{D}_{\boldsymbol{\theta}} \mathbf{u},
\end{align}
where we use $\boldsymbol{\theta}$ to denote the learnable vector $\begin{bmatrix}
  \tau_r & \tau_d & \alpha_1 & \alpha_2 & \cdots & \alpha_{\chi}
\end{bmatrix}^T$.

\subsection{Model the tonic component with spline functions}

The tonic signal of the $n^{\mathrm{th}}$ channel could be viewed as the coefficients $q_n (t)$ convolved with the cubic B-spline function $\psi(t)$,
\begin{align}
  s_n(t) =  \psi(t) \otimes q_n (t).
\end{align}

The coefficients could be viewed as a time-series signal composed of several spikes, i.e. $q_n (t) = \sum_{j=1}^N q_j \delta(t - jT_s)$, where $T_s$ is the sampling rate of the coefficients. $T_s$ could be used to control the smoothness of the B-spline function. To ensure the smoothness, we usually use a longer period for $T_s$ like 6s. With these configurations, we could discretize the coefficients as $\mathbf{q} = \begin{bmatrix}
  q_1 & q_2 & \cdots q_n
\end{bmatrix}$. The convolution on $\psi(t)$ could be formulated by a Toeplitz matrix $\mathbf{C}$. The $k^{\mathrm{th}}$ row of the matrix could be formulated as
\begin{align}
  \mathbf{c}_k = \begin{bmatrix}
    \psi(kT_y + T_s) & \psi(kT_y) & \psi(kT_y - T_s) & \cdots & \psi(T_u - T_s)
  \end{bmatrix}.
\end{align}
With the discretion, we could formulate the tonic component as
\begin{align}
  \mathbf{s}_n = \mathbf{C} \mathbf{q}_n.
\end{align}

\subsection{Preprocess the data}

Preprocessing aims at removing the outliers in the raw data. The whole process is shown in \autoref{alg:preprocessing}. We find the peaks of the differentiated raw data. The searching method is based on comparing the 2 values near the center point. The data patches with outliers are all replaced by their spline approximations. Most of the outliers could be removed by this way.
\begin{algorithm}[tb]
  \caption{The preprocessing applied to the raw data.}
  \label{alg:preprocessing}
  \begin{algorithmic}[1]
    \REQUIRE The raw data $\{\mathbf{y}_{\mathrm{SC}_n}\}_{n=1}^\chi$ for $\chi$ channels.
    \ENSURE The preprocessed data $\{\hat{\mathbf{y}}_{\mathrm{SC}_n}\}_{n=1}^\chi$ for $\chi$ channels.
    % if-then-else
    \FOR{$n$ from $1$ to $\chi$}
      \STATE Find the local positive and negative peaks of the differential data $\dot{\mathbf{y}}_{\mathrm{SC}_n}$, where peaks means the data with a value larger than the around values by 0.1;
      \FORALL{founded peaks}
        \STATE Select 4 points near the peak, the range is limited in 100 points around the peak;
        \STATE Use the spline interpolation of the selected 4 points to replace the 100 points;
      \ENDFOR
      \STATE Perform the 64 order low-pass FIR filter, the cut-off frequency is 3 Hz;
    \ENDFOR
  \end{algorithmic}
\end{algorithm}

\subsection{Solve the deconvolution}

After modeling the data by the aforementioned two methods, we could formulate the $n^{\mathrm{th}}$ channel of the SC data as follows,
\begin{align}
  \mathbf{y}_n = \mathcal{F}_{\boldsymbol{\theta}_n} \mathbf{z}_{0} + \mathcal{D}_{\boldsymbol{\theta}_n} \mathbf{u} + \mathbf{C} \mathbf{q}_n + \boldsymbol{\nu},
\end{align}
where $\boldsymbol{\theta}_n = \begin{bmatrix}
\tau_r & \tau_d & \alpha_n
\end{bmatrix}^T$ is the subset of the learnable vector $\boldsymbol{\theta}$, and $\mathbf{q}_n$ is the decomposition coefficients of the $n^{\mathrm{th}}$ tonic component. For the phasic decomposition, the coefficients, i.e. the neural stimuli $\mathbf{u}$ is shared, and the model parameters are not totally shared crossing the channels. For the tonic decomposition, the modeling function $\mathbf{C}$ is shared crossing the channels, while the coefficients $\mathbf{q}_n$ are different among different channels.

The multichannel deconvolution with the tonic separation could be formulated by the following constrained joint optimization problem,
\begin{subequations} \label{fml:the:optimization}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{align}
    \arg\min\limits_{\substack{\boldsymbol{\theta},~\mathbf{u}, \\ \{\mathbf{q}_n\}_{n=1}^{\chi}, \\ \lambda,~\mu}} & \frac{1}{\chi} \left( \sum_{n=1}^{\chi} \mathcal{J}(\boldsymbol{\theta}_n,~\mathbf{u},~\mathbf{q}_n) + \mu \lVert \mathbf{q}_n \rVert_2^2 \right) + \lambda \lVert \mathbf{u} \rVert^p_p, \\
    \mathrm{s.t.}~&\mathcal{J}(\boldsymbol{\theta}_n,~\mathbf{u},~\mathbf{q}_n) = \lVert \mathbf{y}_n - \mathcal{F}_{\boldsymbol{\theta}_n} \mathbf{z}_{0} - \mathcal{D}_{\boldsymbol{\theta}_n} \mathbf{u} - \mathbf{C} \mathbf{q}_n \rVert^2_2, \label{fml:loss-channel}\\
    & \boldsymbol{\Gamma} \boldsymbol{\theta} \preccurlyeq \mathbf{b},~ \mathbf{u} \succcurlyeq \mathbf{0} \label{fml:cons-phasic}\\
    & \forall~n,~\mathbf{C}\mathbf{q}_n \preccurlyeq \mathbf{y}_n, \label{fml:cons-tonic}
  \end{align}
\end{subequations}
where \eqref{fml:loss-channel} represents the loss function for the $n^{\mathrm{th}}$ channel, \eqref{fml:cons-phasic} is the constraint of the phasic decomposition, and \eqref{fml:cons-tonic} is the constraint of the tonic extraction. In \eqref{fml:cons-phasic}, the Tikhonov matrix $\boldsymbol{\Gamma}$ and the boundary vector $\mathbf{b}$ are defined as
\begin{subequations} \label{fml:the:constraint}
  \renewcommand{\theequation}
  {\theparentequation-\arabic{equation}}
  \begin{align}
    \boldsymbol{\Gamma} &= \begin{bmatrix}
      1 & -1 & 0 & 0 & \cdots & 0 & 0 \\
      0 & 0 & 1 & -1 & \cdots & 0 & 0 \\
      \vdots & \vdots & \vdots & \vdots & \ddots & \vdots & \vdots \\
      0 & 0 & 0 & 0 & \cdots & 1 & -1
    \end{bmatrix}, \\
    \mathbf{b} &= \begin{bmatrix}
      1.4 & -0.1 & 6.0 & -1.5 & 100.0 & -0.01 & \cdots & 100.0 & -0.01
    \end{bmatrix}^T
  \end{align}
\end{subequations}

Conventionally, it is difficult to decide the proper the regularization coefficients $\lambda,~\mu$. In our work, we take the optimization of the coefficients into the consideration. The optimization is based on Focal Underdetermined System Solver (FOCUSS+) algorithm~\cite{murray2005visual}, while the estimation of the tunable regularization coefficients is based on  generalized cross-validation (GCV) method~\cite{zdunek2008improved}.

The whole algorithm for the optimization is discussed in \autoref{alg:deconvolution}. The optimization could be divided into an outer loop and an inner loop. In the outer loop, we solve the modeling parameters $\boldsymbol{\theta}$. The deconvolution and with the tonic extraction is performed in the inner loop.
\begin{algorithm}[!tb]
  \caption{The preprocessing applied to the raw data.}
  \label{alg:deconvolution}
  \begin{algorithmic}[1]
    \REQUIRE The preprocessed multichannel data $\{\hat{\mathbf{y}}_{\mathrm{SC}_n}\}_{n=1}^\chi$, the decomposition coefficients $\mathbf{u}, \{\mathbf{q}_{n}\}_{n=1}^\chi$, and the initialized parameters $\boldsymbol{\theta}$, where $\tau_d \sim U(0.10,~1.4)$, $\tau_r \sim U(1.5,~6.0)$, $\alpha_n \sim U(0.01,~1.0)$, $\mathbf{u} \sim 0$, and $\mathbf{q}_n \sim \mathcal{N}(0.1, 0.02)$.
    % if-then-else
    \FOR {$j$ from $1$ to $30$}
      \STATE Let $\boldsymbol{\theta},~\{\mathbf{q}_{n}\}$ fixed, use FOCUSS+ to solve $\tilde{\mathbf{u}}^{(j)}$. Set $\mathbf{u} = \tilde{\mathbf{u}}^{(j)}$;
      \STATE Let $\mathbf{u},~\{\mathbf{q}_{n}\}$ fixed, solve $\tilde{\boldsymbol{\theta}}^{(j)}$ by the interior method. Set $\boldsymbol{\theta} = \tilde{\boldsymbol{\theta}}^{(j)}$;
      \STATE Let $\boldsymbol{\theta},~\mathbf{u}$ fixed, use FOCUSS+ to solve $\{\tilde{\mathbf{q}}_{n}\}^{(j)}$. Set $\{\mathbf{q}_{n}\} = \{\tilde{\mathbf{q}}_{n}\}^{(j)}$;
    \ENDFOR
    \STATE Initialize $\hat{\boldsymbol{\theta}}^{(0)}=\boldsymbol{\theta}$, $\hat{\mathbf{u}}^{(0)}=\mathbf{u}$, $\{\hat{\mathbf{q}}_{n}\}^{(0)}=\{\mathbf{q}_{n}\}$;
    \FOR{$i$ from $0$ until converge}
      \STATE Let $\boldsymbol{\theta},~\{\mathbf{q}_{n}\}$ fixed, solve $\hat{\mathbf{u}}^{(j)}$ by the following steps;
      \STATE Let $\hat{\lambda}^{(i)(0)} = 2 \times 10^{-3}$,
      \FOR{$m$ from $1$ until converge}
        \STATE Let $\lambda = \hat{\lambda}^{(i)(m-1)}$, $\boldsymbol{\theta},~\{\mathbf{q}_{n}\}$ fixed, use FOCUSS+ to solve $\hat{\mathbf{u}}^{(i)(m)}$. Set $\mathbf{u} = \hat{\mathbf{u}}^{(i)(m)}$;
        \STATE Let $\mathbf{u},~\boldsymbol{\theta}~\{\mathbf{q}_{n}\}$ fixed, use FOCUSS+ to solve $\hat{\lambda}^{(i)(m)}$. Set $\lambda = \hat{\lambda}^{(i)(m)}$;
      \ENDFOR
      \STATE Set $\hat{\mathbf{u}}^{(i)} = \hat{\mathbf{u}}^{(i)(m)}$.
      \STATE Let $\hat{\mu}^{(i)(0)} = 2 \times 10^{-3}$,
      \FOR{$m$ from $1$ until converge}
        \STATE Let $\mu = \hat{\mu}^{(i)(m-1)}$, $\boldsymbol{\theta},\mathbf{u}$ fixed, use FOCUSS+ to solve $\{\hat{\mathbf{q}}_{n}\}^{(i)(m)}$. Set $\{\mathbf{q}_{n}\} = \{\hat{\mathbf{q}}_{n}\}^{(i)(m)}$;
        \STATE Let $\mathbf{u},~\boldsymbol{\theta}~\{\mathbf{q}_{n}\}$ fixed, use FOCUSS+ to solve $\hat{\mu}^{(i)(m)}$. Set $\mu = \hat{\mu}^{(i)(m)}$;
      \ENDFOR
      \STATE Set $\{\hat{\mathbf{q}}_{n}\}^{(i)} =  \{\hat{\mathbf{q}}_{n}\}^{(i)(m)}$.
    \ENDFOR
  \end{algorithmic}
\end{algorithm}

\bibliographystyle{ieeetr}
\bibliography{ref}

\end{document}


\documentclass[../Notes/main.tex]{subfiles}

\begin{document}
\section{Geometry}
\subsection{Vectors/Points}
\lstinputlisting[firstline=2]{vector2D/vector2D.cpp}

\subsection{Calculate Areas}
\subsubsection{Integration via Simpson's Method}
\lstinputlisting[firstline=2]{simpsonsMethod/simpsonsMethod.cpp}

\subsubsection{Green's Theorem}
\lstinputlisting[firstline=2]{greenTheorem/greenTheorem.cpp}

\subsection{Pick's Theorem}
Given a simple polygon (no self intersections) in a lattice such that all vertices are grid points. Pick's theorem relates the Area \(A\), points inside of the polygon \(i\) and the points of the border of the polygon \(b\), in the following way:
\begin{equation*}
    A=i+\frac{b}2-1
\end{equation*}

\end{document}


\chapter{Summary}
\label{ch:summary}
\begin{figure}
  \centering
  \inputTikZ{figures/overview}
  \caption{Overview over the various important concepts, equations and adaptive filters covered in these lecture notes.}
  \label{fig:overview}
\end{figure}
\noindent Fig.~\ref{fig:overview} shows an overview over the various important concepts, equations and adaptive filters covered in these lecture notes. All relationships indicated by the arrows have been covered with the exception of the arrow from the steepest descent block to the NLMS block. The interested reader may establish this relationship by solving Problem 1 of Chapter 6 in \cite{Haykin2001}.

Table~\ref{tab:summary} shows a summary of the adaptive filters covered in these lecture notes. Most the expressions for the mean-square stability, the excess mean-square error (EMSE), the misadjustment, and the mean-square deviation (MSD) are approximations, and they can therefore only be used as a rule of thumb. For more accurate expressions see the description of the adaptive filters and the references therein.

\begin{sidewaystable}
  \centering
  \small
  \begin{tabular}{@{}Sl Sr<{ = }@{ }Sl Sc Sc Sc Sc Sc Sc@{}}
    \toprule
    Name & \multicolumn{2}{c}{Algorithm} & Cost & Mean-Square Stability & EMSE & Misadjustment & MSD\\
    \midrule
    SD  & $\vect{g}(\vect{w}(n))$ & $2\vect{R}_u\vect{w}(n)-2\vect{r}_{ud}(n)$       & $\mathcal{O}(M)$   & $0 < \mu < \displaystyle\frac{2}{\lambda_\textup{max}}$ & 0 & 0 & 0\\
        & $\vect{w}(n+1)$         & $\vect{w}(n)-\displaystyle\frac{\mu}{2}\vect{g}(\vect{w}(n))$ &      & & & &\\[4mm]
    LMS & $e(n)$                  & $d(n)-\vect{u}^T(n)\vect{w}(n)$                 & $\mathcal{O}(M)$   & $0 < \mu < \displaystyle\frac{2}{\tr{\vect{R}_u}}$ & $\displaystyle\frac{\mu}{2}J_\textup{min}\tr{\vect{R}_u}$ & $\displaystyle\frac{\mu}{2}\tr{\vect{R}_u}$ & $\displaystyle\frac{\mu}{2}J_\textup{min}M$\\
        & $\vect{w}(n+1)$         & $\vect{w}(n)+\mu\vect{u}(n)e(n)$                &                    & & & &\\[4mm]
    NLMS& $e(n)$                  & $d(n)-\vect{u}^T(n)\vect{w}(n)$                 & $\mathcal{O}(M)$   & $0 < \beta < 2$& $\displaystyle\frac{\beta}{2}J_\textup{min}$ & $\displaystyle\frac{\beta}{2}$ & $\displaystyle\frac{\beta J_\textup{min}}{2\tr{\vect{R}_u}}$\\
        & $\vect{w}(n+1)$         & $\vect{w}(n)+\displaystyle\frac{\beta}{\epsilon+\|\vect{u}(n)\|^2}\vect{u}(n)e(n)$ & & & & &\\[4mm]
    APA & $\vect{e}(n)$           & $\vect{d}(n)-\vect{U}^T(n)\vect{w}(n)$          & $\mathcal{O}(MK^2)$& $0 < \beta < 2$&$\displaystyle\frac{\beta}{2}J_\textup{min}K$ & $\displaystyle\frac{\beta}{2}K$ & no simple expression\\
        & $\vect{w}(n+1)$         & \multicolumn{2}{@{}Sl}{$\vect{w}(n)+\beta\vect{U}(n)[\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n)]^{-1}\vect{e}(n)$} & & & &\\[4mm]
    RLS & $\vect{\pi}(n)$         & $\vect{P}(n-1)\vect{u}(n)$                      & $\mathcal{O}(M^2)$ & $0 < \lambda \leq 1$&$\displaystyle\frac{J_\textup{min}\frac{1-\lambda}{1+\lambda}M}{1-\frac{1-\lambda}{1+\lambda}M}$ & $\displaystyle\frac{\frac{1-\lambda}{1+\lambda}M}{1-\frac{1-\lambda}{1+\lambda}M}$ & $\displaystyle\frac{J_\textup{min}\frac{1-\lambda}{1+\lambda}M}{1-\frac{1-\lambda}{1+\lambda}M}\sum_{m=1}^M\frac{1}{\lambda_m}$\\
        & $\vect{k}(n)$           & $\displaystyle\frac{\vect{\pi}(n)}{\lambda+\vect{u}^T(n)\vect{\pi}(n)}$& & & & &\\
        & $\xi(n)$                & $d(n)-\vect{u}^T(n)\vect{w}(n-1)$               &                    & & & &\\
        & $\vect{w}(n)$         & $\vect{w}(n-1)+\vect{k}(n)\xi(n)$               &                    & & & &\\
        & $\vect{P}(n)$           & \multicolumn{2}{@{}Sl}{$\lambda^{-1}\left[\vect{P}(n-1)-\vect{k}(n)\vect{\pi}^T(n)\right]$}& & & &\\
    \bottomrule
  \end{tabular}
  \caption{Overview over the basic adaptive filters and their properties. Most of the expressions for the mean-square stability, the excess mean-square error (EMSE), the misadjustment, and the mean-square deviation (MSD) are approximations.}
  \label{tab:summary}
\end{sidewaystable}


\documentclass{article}
\usepackage[top=1in, bottom=1in, left=1in, right=1in]{geometry}
\usepackage{amsmath}
\begin{document}
    \subsection*{Nonhomogeneous linear equation solution set}
    The solution of the nonhomogeneous linear equation is consist of two parts; one is a any special solution, and the other is the general solution of the homogeneous linear equation. That means:
    \begin{equation}
        Soltion\ Set = \{Special\ Solution + General\ Solution\}
    \end{equation}
    \subsection*{Range}
    The range of a matrix (A) is the column space of the matrix.
    \subsection*{Diagonalizable matrix}
    If a n-dimension matrix can be diagonalized, it must have n eigenvectors, which are independent with each other.

    Precedure:
    \begin{enumerate}
        \item Find its eigenvalues $\lambda_1$, $\lambda_2$, $\cdots$, $\lambda_m$;
        \item Find the correspoding eigenvectors $\vec{a_1}$, $\vec{a_2}$, $\cdots$, $\vec{a_n}$;
        \item $V=\{\vec{a_1}, \cdots, \vec{a_n}\}$;
        \item $A=V\ diag(\lambda_1, \cdots, \lambda_m)\ V^{-1}$.
    \end{enumerate}
    \subsection*{Inverse matrix}
    \begin{enumerate}
        \item $\{A|I\}\Rightarrow\{I|A^{-1}\}$;
        \item $A^{-1}=\frac{A^{*}}{|A|}$, $A^{*}$ is adjugate matrix.
    \end{enumerate}
    \subsection*{Orthogonal complement}
    \begin{itemize}
        \item Row $(A)^{\perp}=$ Null $(A)$;
        \item Range $(A)^{\perp}=$ Null $(A^{T})$.
    \end{itemize}
    \subsection*{Geometric multiplicity and algebraic multiplicity}
    \begin{itemize}
        \item Geometric multiplicity of $\lambda$ is the dimension of the Null($A-\lambda I$);
        \item Algebraic multiplicity of $\lambda$ is the number of times $\lambda$ appears in the equation $|\lambda I-A|=0$.
        \item Geometric multiplicity is always not exceed Algebraic multiplicity; And all algebraic multiplicities of different eigenvalues sum up should be n (for a n$\times$n matrix);
        \item if every Geometric multiplicity equals to Algebraic multiplicity, we say that the matrix is diagonalizable.
    \end{itemize}
    \subsection*{Vandermonde matrix}
    \begin{equation*}
        V=\left(\begin{matrix}
            1 & x_0 & x_0^2 & \cdots & x_0^n\\
            1 & x_1 & x_1^2 & \cdots & x_1^n\\
            \vdots & \vdots & \vdots &  & \vdots\\
            1 & x_n & x_n^2 & \cdots &x_n^n
        \end{matrix}\right)
    \end{equation*}
    \begin{equation*}
        V\Rightarrow\left(\begin{matrix}
            1 & 0 & 0 & \cdots & 0\\
            1 & x_1-x_0 & x_1(x_1-x_0) & \cdots & x_1^{n-1}(x_1-x_0)\\
            \vdots & \vdots & \vdots &  & \vdots\\
            1 & x_n-x_0 & x_n(x_n-x_0) & \cdots &x_n^{n-1}(x_n-x_0)
        \end{matrix}\right)
    \end{equation*}
    \begin{equation*}
        \det(V)=\left|\begin{matrix}
             x_1-x_0 & x_1(x_1-x_0) & \cdots & x_1^{n-1}(x_1-x_0)\\
             \vdots & \vdots &  & \vdots\\
             x_n-x_0 & x_n(x_n-x_0) & \cdots &x_n^{n-1}(x_n-x_0)
        \end{matrix}\right|=(x_n-x_0)\cdots(x_1-x_0)\left|\begin{matrix}
            1 & x_1 & x_1^2 & \cdots & x_1^n\\
            1 & x_2 & x_2^2 & \cdots & x_2^n\\
            \vdots & \vdots & \vdots &  & \vdots\\
            1 & x_n & x_n^2 & \cdots &x_n^n
        \end{matrix}\right|
    \end{equation*}
    \begin{equation*}
        \det(V)=\prod_{0\leq i<j\leq n}(x_j-x_i)
    \end{equation*}
    \subsection*{Characteristic polynomial}
    The characteristic polynomial of a matrix A is $|A-\lambda I|$.
    \subsection*{Representing linear transformations by matrices}
    For a transformation T and a basis a, it can be represented by a matrix T as follows:
    \begin{equation*}
        T(x)=Tx=x^{T}T^T,\ T=\{T(a_1),\cdots,T(a_n)\}^T
    \end{equation*}
    \subsection*{Vector projection}
    For a vector u, the projection of u on v is as follows:
    \begin{equation*}
        a_1=\frac{<u,v>}{<v,v>}v
    \end{equation*}
    So, the projection of u on a plane with normal vector v is as follows:
    \begin{equation*}
        a_2=u-a_1
    \end{equation*}
    So, the reflection of u on a plane with normal vector v is as follows:
    \begin{equation*}
        a_2=u-2a_1
    \end{equation*}
    \subsection*{Dimension formula}
    \begin{equation*}
        \dim(V_1+V_2)=\dim(V_1)+\dim(V_2)-\dim(V_1\cap V_2)
    \end{equation*}
    \subsection*{Rotation and reflection}
    For a transformation, if its matrix is orthogonal and the determinant is 1, then, it is a Rotation. Otherwise, if the determinant is -1, then, it is a reflection.
    \begin{equation*}
        \left(\begin{matrix}
            \cos(\theta)&-\sin(\theta)\\
            \sin(\theta)&\cos(\theta)
        \end{matrix}\right)
    \end{equation*}
    \subsection*{Gram–Schmidt process}
    For a basis $(\alpha_1,\cdots,\alpha_n)$, the orthogonal basis can be gotten with the following steps:
    \begin{enumerate}
        \item $\beta_1=\alpha_1$;
        \item $\beta_2=\alpha_2-k\beta_1$, $k=\frac{<\alpha_2,\beta_1>}{<\beta_1, \beta_1>}$;
        \item $\cdots \cdots$
        \item $\beta_n=\alpha_n-k_1\beta_1-k_2\beta_2-\cdots-k_{n-1}\beta_{n-1}$, $k_{i}=\frac{<\alpha_n,\beta_i>}{<\beta_i,\beta_i>}$.
    \end{enumerate}
\end{document}