% $Id$
%
% Author: David Fournier
% Copyright (c) 2008 Regents of the University of California
%

The Vector Autoregressive Moving Average Garch process combines
the Vector Autoregressive Moving Average (\textsc{varma})
processes with the Vector Garch (generalized
autoregressive conditional heteroscedastic) processes.

\section{Formulation of the \textsc{varma garch} process}

The \textsc{varma garch} process of type $(p,q,r,s)$ is given by a
series $Y_t$ for $t=-p+1,\ldots,n$, where for each value of $t$,
$Y_t$ is an $m$-dimensional vector.
For $t>0$, the $Y_t$ are assumed to satisfy a relationship of the form
\begin{equation}
  Y_t=\mu +\sum_{l=1}^p A_l (Y_{t-l}-\mu)
    +\sum_{l=0}^q B_l \epsilon_{t-l}
  \label{eq1}
\end{equation}
where the $\mu$ is an $m$-dimensional vector,
$A_l$ and $B_l$ are $m\times m$ matrices, $B_0$ is the identity
matrix,  and the
$\epsilon_t$ are multivariate (normal) random vectors,
with means~$0$.  Covariance matrices $\Sigma_t$ and
 $\hbox{\rm E}(\epsilon_t\epsilon_{t^\prime})=0$ if $t\ne t^\prime$.
Let
\begin{equation}
  r_t=Y_t-\mu -\sum_{l=1}^p A_l (Y_{t-l}-\mu)
  \label{eqc1}
\end{equation}
be the vector of model residuals or ``shocks.''\footnote{For the moving
average model ($q>0$), one might argue that since the previous values
of $r_t$ have been observed, the shock part of $r_t$, say,
$d_t$, is given by $d_t=r_t-\sum_{l=1}^q B_l r_{t-l}$,
but this has not been done at present.}
These residuals are
assumed to contribute to the covariance matrix in the next time period.
The $\Sigma_t$ evolve according to one of several relationships.

\subsubsection{The \textsc{dvec} relationship}
\begin{equation}
  \Sigma_t=\Omega+
        \sum_{l=1}^r F_l\bigotimes r_{t-l}r_{t-l}^\prime
         +\sum_{l=1}^sG_l\bigotimes \Sigma_{t-l}
  \label{eqc}
\end{equation}
In equation~(\ref{eqc}), the matrices
$F_l$ and $G_l$ are symmetric and the operator
`$\bigotimes$' denotes the element-wise product of matrices.
This parameterization does not restrict the resulting matrix to
be positive definite, so some care is necessary to ensure the
stability of the resulting~model.

\subsubsection{The \textsc{bekk} relationship}
\begin{equation}
\Sigma_t=\Omega+
       \sum_{l=1}^r F_l\, r_{t-l}\, r_{t-l}^\prime\, F_l^\prime
        + \sum_{l=1}^s\, G_l\, \Sigma_{t-l}\, G_l^\prime
  \label{eq2}
\end{equation}

\subsubsection{\textsc{dveci} and \textsc{bekkai} parameterizations}
The basic \textsc{dvec} and \textsc{bekk} parameterizations can be extended by  modifying
components of the $r_t$ to reflect the asymmetric response
to positive and negative values.
\begin{align}
    \nonumber \eta_{ij} =&\epsilon_{ij}/\alpha_j &\textrm{if } \epsilon_{ij} \ge 0\\
    \eta_{ij} =&\epsilon_{ij}\alpha_j                    &\textrm{if }  \epsilon_{ij} < 0
\end{align}
This modified form will be referred to as the \textsc{dveci} and \textsc{bekkai} parameterizations.
\begin{equation}
\Sigma_t=\Omega+
        \sum_{l=1}^r F_t\bigotimes \eta_{t-l}\, \eta_{t-l}^\prime
         +\sum_{l=1}^sG_l\bigotimes \Sigma_{t-l}
  \label{eqd}
\end{equation}
\begin{equation}
  \Sigma_t=\Omega+
       \sum_{l=1}^r F_l\, \eta_{t-l}\, \eta_{t-l}^\prime\, F_l^\prime
        + \sum_{l=1}^sG_l\, \Sigma_{t-l}\, G_l^\prime
   \label{eqe}
\end{equation}

\section{Setting a value for $\Sigma_1$}

The value for the parameters in $\Sigma_1$ are often poorly determined
and simply letting them be free parameters can lead to
instability and initial
transient effects in the model. To stabilize the parameterization, we have
calculated $\Sigma_1$ through $\Sigma_{\max\{r,s\}}$ from the condition
\begin{equation}
  \widehat\Omega=\Sigma+\sum_{l=1}^q B_l\Sigma B_l^\prime
\end{equation}
where
\begin{equation}
  \widehat\Omega=\frac{1}{n}\sum_{t=1}^n\hat\epsilon_t\, \hat\epsilon_t^\prime
\end{equation}
denotes the empirical covariance matrix formed from the
models residuals
\begin{equation}
  \hat\epsilon_t=Y_t-\mu-\sum_{l=1}^p A_l (Y_{t-l}-\mu).
\end{equation}

\section{Ensuring that the $\Sigma_t$ are positive definite}

The \textsc{dvec} parameterization can produce matrices that are not
positive definite, and the \textsc{bekk} parameterization can produce matrices
that are almost not positive definite (much as a positive number can
get arbitrarily close to zero). At worst, this will lead to a failure
in the model to converge and at best, it makes the estimation somewhat
unstable. To improve model performance, the \textsc{bekk} and \textsc{dvec}
operations are followed by a modification of the resulting $\Sigma_t$ that
 makes them more positive definite. The first problem is to get a notion of what is meant by ``small'' for a particular problem.
This is accomplished by first scaling the
$\Sigma_t$ to produce a matrix $\Lambda_t$ where
\begin{equation}
  \Lambda_{ij}=\frac{\Sigma_{t_{ij}}}{\sqrt{\Sigma_{t_{ij}}\Sigma_{t_{ij}}}}
\end{equation}
The terms $\Lambda_{ii}$ are then bounded above {$1.0\e{-3}$,  i.e., they are replaced in a differentiable fashion with numbers that are $\ge 1.0\e{-3}$ using
the \texttt{posfun} function.
In addition, the correlation matrix
$\Lambda_{ij}/\sqrt{\Lambda_{t_{ij}}\Lambda_{t_{ij}}}$ is
decomposed via a Choleski decomposition, the divisors of
 which is forced to be $> 0.3$ in a differentiable fashion using the
\texttt{posfun} function.
\X{\fontindexentry{tt}{posfun} function}
The
above operations leave a matrix that is sufficiently positive definite
and close enough to $\Sigma_1$ unchanged.

\section{Missing data}

Missing data points are included into the model as parameters to
be estimated. If there are a substantial number of missing data
points, this will induce bias into the estimates.

\section{The likelihood function}

The model was fit by maximum-likelihood or, more correctly, by
finding the mode of the Bayesian posterior distribution.
A robust likelihood function that is a mixture of a normal
distribution and a Cauchy distribution is employed.  The amount of robustness
can be changed by the user.

\section{Model selection}

Model selection consists of fitting the model to the data for various values
of the parameters $(p,q,r,s)$ and trying to determine the simplest model
that adequately fits the data, if any.

The two criteria which are used for this are the likelihood ratio test
and investigation of the residuals in the form of the Box-Ljung statistic.
The likelihood-ratio test is used for general model selection, while
the Box-Ljung statistic is used to investigate whether or not the model
can adequately fit the changes in the covariance matrices $\Sigma_t$
that occur over time.

\section{The Box-Ljung statistic}

The following Box-Ljung
statistic was employed
to test the ability of the model to model the time varying covariance
structure of the time series. This statistic is calculated from the
estimated standardized residuals $z_t$, for $t=1,\ldots n$,
where for each $t$, $z_t$ is an $m$-dimensional vector.
The $z_i$ are obtained in the calculations necessary to calculate the
log-likelihood~\mbox{function}.
\begin{align}
  \hat\mu_j&=\frac{1}{ n}\sum_{i=1}^n z_{ij}\\
  z_{ij}^{\prime}&=z_{ij}-\hat\mu_j\\
  \hat\sigma_{jk}&=\frac{1}{ n}\sum_{i=1}^n z_{ij}^\prime z_{ik}^\prime\\
  \gamma_{ijk}&=
       \frac{
            \frac{1}{ n-l}\sum_{i=1}^{n-l}
         (z_{ij}^\prime z_{ik}^\prime-\hat\sigma_{jk})
         (z_{i,j+l}^\prime z_{i,k+l}^\prime-\hat\sigma_{jk})
            }{
            \frac{1}{ n}\sum_{i=1}^n
         (z_{ij}^\prime z_{ik}^\prime-\hat\sigma_{jk})^2
       }
\end{align}

Under the null hypothesis that the model is adequate, and if the
$z_i$ are normally distributed, then the sum
$$LB(K)_{ij}=n\sum_{k=1}^k w_k \gamma_{ijk}^2$$
is asymptotically distributed as a $\chi^2$ random variables
with~$K$ degrees of freedom. Here, $w_k=(n+2)/(n-k)$.

\section{Analysis of simulated data}

One method to get an idea how well a statistical model works is to
use it with simulated data where the true values of the parameters being estimated
are known.
A simple simulator that can generate data sets is included with the
\texttt{mgarch} package. The simulator generated a 4-dimensional set of
1,000 observations. A type~$1,1,1,1$ process was simulated and
analyzed.

The following plots show the actual and predicted values for the
diagonal variance and correlation terms for the analysis with
a type $1,1,1,1$~model.

\includegraphics[height=2.5in, width=\textwidth]{covarplot11.png}

\includegraphics[height=2.5in, width=\textwidth]{covarplot11.png}

\includegraphics[height=2.5in, width=\textwidth]{covarplot22.png}

\includegraphics[height=2.5in, width=\textwidth]{covarplot33.png}

\includegraphics[height=2.5in, width=\textwidth]{covarplot44.png}

\includegraphics[height=2.5in, width=\textwidth]{corrplot21.png}

\includegraphics[height=2.5in, width=\textwidth]{corrplot31.png}

\includegraphics[height=2.5in, width=\textwidth]{corrplot32.png}

\includegraphics[height=2.5in, width=\textwidth]{corrplot41.png}

\includegraphics[height=2.5in, width=\textwidth]{corrplot42.png}

\includegraphics[height=2.5in, width=\textwidth]{corrplot43.png}

\section{Analysis of real data}

The data consist of daily observations of the German Mark/US Dollar and Japanese Yen/US dollar exchange rates, as well as the SP-500 and Tokyo
 (\textsc{tokyose}) stock exchange indices. For this data set, $m=4$, and there
were $1301$ time periods with $211$ missing values.

\subsection{Model Parameters log-likelihood directory $p,q,r,s$.}
\begin{lstlisting}
VARMA
0,0,0,0   221         -1382.56      000
1,0,0,0   241         -1322.47      100
1,1,0,0   253         -1308.57      110

VARMA with DVEC
0,0,1,1   241         -1050.87
1,1,2,1   283         -928.518
1,2,1,1   289         -924.755

VARMA with DVECI
1,1,2,1a  287         -887.764
1,2,1,1a  293         -888.531
\end{lstlisting}

\bigskip
\subsection{Ljung-Box statistic ($\chi^2$ with 10 degrees of freedom).}
\begin{lstlisting}
VARMA
0,0,0,0         138.953  52.902  26.351 19.171
                 52.902 222.502 154.708 59.681
                 26.351 154.708  60.328 87.031
                 19.171  59.681  87.031 68.889

1,1,0,0         154.977  74.511  19.329 11.472
                 74.511 177.016 130.295 59.084
                 19.329 130.295  47.499 72.724
                 11.472  59.084  72.724 49.329

VARMA with DVEC
0,0,1,1           3.728 13.180 17.682 17.633
                 13.180 11.496 25.846  6.016
                 17.682 25.846  7.610  6.791
                 17.633  6.016  6.791 13.398

1,1,1,1           4.239  9.519 10.866 10.885
                  9.519  6.434 21.734  8.047
                 10.866 21.734  4.089  8.318
                 10.885  8.047  8.318 12.814

1,2,1,1           5.660 10.348 11.029 11.557
                 10.348  7.011 21.727  7.039
                 11.029 21.727  4.819  7.908
                 11.557  7.039  7.908 10.684

1,1,2,1           6.920 11.698 11.464 12.593
                 11.698  4.914 24.677  8.179
                 11.464 24.677  2.399  8.855
                 12.593  8.179 8.855 10.985

VARMA with DVECI
1,2,1,1a          4.352  9.658  9.402  8.542
                  9.658  7.646 17.516  8.416O
                  9.402 17.516  5.795  7.108
                  8.542  8.416  7.108 10.483

1,1,2,1a          4.785 17.077  8.760 10.487
                 17.077  6.718 18.563  9.736
                  8.760 18.563  3.270  8.525
                 10.487  9.736  8.525 10.253
\end{lstlisting}

While the model $1,1,2,1a$ produced almost as high a
log-likelihood value as model $1,2,1,1a$, the superior
performance of the latter model with respect to the
Box-Ljung statistic might prompt us to consider it the model of choice.

The following plots show the actual and predicted values for the diagonal variance and
correlation terms. %fixed

\includegraphics[height=2.5in, width=\textwidth]{example/covarplot11.png}

\includegraphics[height=2.5in, width=\textwidth]{example/covarplot22.png}

\includegraphics[height=2.5in, width=\textwidth]{example/covarplot33.png}

\includegraphics[height=2.5in, width=\textwidth]{example/covarplot44.png}

\includegraphics[height=2.5in, width=\textwidth]{example/corrplot21.png}

\includegraphics[height=2.5in, width=\textwidth]{example/corrplot31.png}

\includegraphics[height=2.5in, width=\textwidth]{example/corrplot32.png}

\includegraphics[height=2.5in, width=\textwidth]{example/corrplot41.png}

\includegraphics[height=2.5in, width=\textwidth]{example/corrplot42.png}

\includegraphics[height=2.5in, width=\textwidth]{example/corrplot43.png}

\bigskip

\section{Input format}

By default, the stand-alone version of the model attempts to read in the
data from a file named \texttt{mgarch.dat}.  This can be changed by a command line option.
A reasonable command line option would be:
\begin{lstlisting}
  mgarch -ind datafile -nox -nohess
\end{lstlisting}
The command
\begin{lstlisting}
  mgarch -?
\end{lstlisting}
will print out a list of command line options.

Part of a data file is shown below. The first line describes the data
and specifies the form of the model. The \texttt{delta} flag determines whether or not
the parameters that measure asymmetric response in the \textsc{arch} component of the
model are estimated. The robustness number controls the amount of
robustness in the likelihood function. A value between~0.0 and~0.5 is
probably appropriate. A value of~10,000 is used to indicate missing values
in the~data.
\begin{lstlisting}
# number of     dimension    p q r s  delta  robustness
# observations                        flag
  1301            4          0 0 1 1   0        0
-0.473592      -0.30815      -0.199577      -0.154677
0.140859      -0.256788      -0.823379      -0.947582
-0.80579      -0.833361      2.439394      1.208645
-1.542773      -0.855767      0.469603      -0.433346
10000          10000          1.445218      0.462298
6.014853      4.964597      -0.189343      -0.231408
-1.064758      -1.700819      -3.143447      0.62605
-0.608465      -2.618972      1.641306      2.127487
-5.242424      2.902805      2.60543        -0.668981
2.205787      2.026769      -1.42811      -0.141505
 // ..........................................

10000           10000        -1.072871      0.365977
-0.208759      1.51892      -0.59516        -0.029556
-2.413508      -2.578193      2.045828      -0.2362
0.890598      0.119672      -0.415587      0.029485
-2.797967      0.943489      -1.377787      0.283592
1.405865      1.915963      -0.696567      0.450668
10000          10000        -0.31709      0.255885
2.971906      -1.704494      0.835319      0.286408
-3.451449      1.233715      -1.144007      -0.322157
1.342827      0.376409      0.279213      0.226602
\end{lstlisting}

\section{Output files}

Since the model produces a lot of output, it is a good idea to run it in its own directory,
so files can be easily deleted. The independent variables of the
optimization are in a file named \texttt{mgarch.par}. (\texttt{mgarch.bar} is an equivalent binary file.)
A more user-friendly report is in the file \texttt{mgarch.rep}.
The estimated covariances and correlations are in files named \texttt{covar.XX} and \texttt{correl.XX}.

The model selection criteria have identified the model~1.1.1a
as the best of the models considered for these data.

Some parameters of interest are:
\begin{lstlisting}
# alpha:
 0.394461 0.538978 1.19337 0.969918
# A1:
 -0.0531276 0.772529 -0.187692 -0.131215
 0.0898897 -0.399746 -0.142294 -0.106079
 0.0740065 0.248345 -0.0629680 -0.212928
 0.0918460 -0.0575289 -0.0700786 0.0537302
# B1:
 -0.00719450 -0.344793 0.115020 0.00516664
 -0.0943230 0.474330 0.147845 0.0936166
 -0.0860047 -0.243294 0.0187986 0.217280
 -0.0815394 0.0843289 0.00782384 -0.0539808
# F:
 -0.109526 -0.0292127 -0.122395 0.0548869
 0.000332929 -0.0964482 0.0153865 -0.0346829
 -0.0148848 -0.0146167 -0.117961 -0.0529385
 -0.00504663 -0.000547068 0.00413882 -0.133956
# G:
 0.930026 -0.269741 0.136401 0.427047
 -0.0345549 -0.981606 0.00281003 -0.00411475
 0.0976783 -0.0132467 -0.976596 0.0306139
 0.117971 -0.0134465 0.00647385 -0.955173
\end{lstlisting}
For the four parameters $\alpha_i$, a value $<1$ indicates
that negative values seem to have a larger effect on changes
in the covariance structure than do positive ones. This effect seems
to be much larger in the dollar cross rates than in the
stock exchanges indices.

\section{The code for the \textsc{bekkgarch} model}

The code for the \textsc{bekkgarch} model follows. Additional
comments have been added to the~code.
\begin{lstlisting}
DATA_SECTION
  // This section describes the data inputs to the model.
  // By default they are read in from the file bekkgarch.dat.
  init_int na  // number of time periods
  init_int m   // dimension of the vector time series
  init_int p   // degree of autoregression must be >=0
  init_int q   // degree of moving average mult be >=0
  init_int ra   // degree of arch must be>=0
  init_int sg   // degree of arch must be>=0
  init_int delta_switch // turns on asymmetric response to shocks
  init_number robustness  // amount of robustness probably something
              // between 0 and .05 is right
  int n
  int msquared
 !!  n=na-p;  // number of obs for conditional likelihood
  init_matrix cY(-p+1,n,1,m)  // the vector time series of observations
  int nmiss
 LOC_CALCS
  msquared=m*m;
  int ii=0;
  int j;
  int i;
  for (i=-p+1;i<=n;i++)
    for (j=1;j<=m;j++)
      if (cY(i,j)==10000) ii++;
  nmiss=ii;
 END_CALCS
  ivector rowmiss(1,nmiss)
  ivector colmiss(1,nmiss)
 LOC_CALCS
  ii=1;
  for (i=1;i<=n;i++)
    for (j=1;j<=m;j++)
      if (cY(i,j)==10000) {
        rowmiss(ii)=i;
        colmiss(ii)=j;
        ii++;
      }
 END_CALCS
  int m1
  int beginSigma
  matrix Idm(1,m,1,m)
  ivector beginF(1,ra)
  ivector beginG(1,sg)
  ivector beginA(1,p)
  ivector beginB(1,q)
  int nstart
 LOC_CALCS
  nstart=0;
  if (nmiss>0) nstart=1;
  m1=m*(m+1)/2;  // "size" of n x n symmetric matrix
  Idm=identity_matrix(1,m);
  if (p) beginA.fill_seqadd(1+nstart,1);
  if (q) beginB.fill_seqadd(p+1+nstart,1);
  if (ra) beginF.fill_seqadd(p+q+1+nstart,1);
  if (sg) beginG.fill_seqadd(p+q+1+nstart,1);
  beginSigma=p+q+1+nstart;
 END_CALCS
PARAMETER_SECTION
 LOC_CALCS
  int mm=m;
  int dstart;
  if (delta_switch>0)
    dstart=p+q+2+nstart;
  else
    dstart=-1;
 END_CALCS
  // the initial values for the time series and disturbance as
  // estimated parameters
  init_bounded_vector mudev(1,m,-100,100,-1)
  init_bounded_vector delta(1,m,.1,1.9,dstart)
  vector mu(1,m)
  vector esd(1,m)
  vector muemp(1,m)  // empirical covariance matrix
  matrix Y(-p+1,n,1,m)  // the vector time series of observations
  init_matrix_vector A1(1,p,1,mm,1,mm,beginA)  // pars for the AR coff matrices
  3darray A(1,p,1,m,1,m)  // pars for the AR coff matrices
  init_matrix_vector B1(1,q,1,mm,1,mm,beginB) // pars for the MA coff matrices
  3darray B(0,q,1,m,1,m)  // has the additional B(0)=Id
  3darray Bt(0,q,1,m,1,m)  // has the additional B(0)=Id
  4darray cov(1,n,0,q,1,m,1,m)  // the m x m blocks for the covariance
  3darray TB_S(0,q,1,m,1,m)
  matrix Omega(1,m,1,m)
  3darray B_S(0,q,1,m,1,m)
  init_bounded_vector v_Sigma(1,m1,-10,10.1,beginSigma);  // pars for Sigma
  matrix Semp(1,m,1,m)
  matrix SSemp(1,m,1,m)
  matrix ch_Sigma(1,m,1,m)
  init_bounded_vector_vector Fcoff(1,ra,1,msquared,-1.000,1.0,beginF)
  3darray F(1,ra,1,m,1,m)
  3darray tF(1,ra,1,m,1,m)
  init_bounded_vector_vector Gcoff(1,sg,1,msquared,-.98,.98,beginG)
 LOC_CALCS
  if (ra)
  {
    int mmin,mmax;
    mmin=Fcoff(1).indexmin();
    mmax=Fcoff(1).indexmax();
    for (int ij=mmin;ij<=mmax;ij++) if (value(Fcoff(1)(ij))==0.0)
      Fcoff(1)(ij)=0.02;
  }
  if (sg)
  {
    int mmin,mmax;
    mmin=Gcoff(1).indexmin();
    mmax=Gcoff(1).indexmax();
    for (int ij=mmin;ij<=mmax;ij++) if (value(Gcoff(1)(ij))==0.0)
      Gcoff(1)(ij)=0.001;
  }
 END_CALCS
  3darray G(1,sg,1,m,1,m)
  3darray tG(1,sg,1,m,1,m)
  3darray Sigma(1,n,1,m,1,m)
  init_bounded_vector missvals(1,nmiss,-5.0,5.0);
  vector arpart(1,m)
  matrix r(1,n,1,m)
  3darray rr(1,n,1,m,1,m)
  vector vecr(1,n*m)   // VEC[r]
  vector y(1,n*m)
  number ldet
  objective_function_value f
  matrix Yv(-p+1,n,1,m)  // after subtracting off the mean
 !! int q1m=(q+1)*m;
 !!CLASS  banded_symmetric_dvar_matrix S(1,n*m,q1m);
PROCEDURE_SECTION
  int t; int i; int j;
  fill_matrices_with_independent_parameters();
  Y=cY;
  add_missing_values();
  calculate_time_series_mean();
  calculate_the_residuals();
  calculate_the_empirical_covariance();
  SSemp=get_initial_sigma();
  dvariable fpen=calculate_the_sub_variances_BEKK();
  calculate_the_sub_covariances();
  calculate_the_covariance_matrix();
  int ierr=0;
  // choleski decomposition of a banded symmetric matrix
  // produces a banded lower triangular matrix
  dvariable fpen1=0.0;
  banded_lower_triangular_dvar_matrix blt=choleski_decomp_positive(S,
    1.e-6,fpen1);
  fpen+=fpen1;
  // solve for y=inv(blt)*vecr
  y=solve(blt,vecr);
  int ss=0;
  dvariable lno=ln_det(Sigma(1),ss);
  f+=norm2(log(delta));
  for (i=1;i<=q;i++) f+=norm2(B1(i));
  f+=norm2(v_Sigma);
  for (i=1;i<=ra;i++) f+=norm2(F(i));
  for (i=1;i<=sg;i++) f+=norm2(G(i));
  dvariable lndet=0.0;
  for (i=1;i<=n*m;i++) lndet+=log(blt(i,i));
  // robust log-likelihood function -- mixture of normal and
  // tiny bit of cauchy
  dvar_vector y2= square(y);
  if (robustness>1.e-20)
    f+= lndet - sum(log(mfexp(-0.5*y2)+robustness/(1.0+y2)));
  else
    f+= lndet - sum(log(mfexp(-0.5*y2)+.0001/(1.0+y2)));

  f+=fpen;
  (*ad_printf)("f = %lf\n",value(f));
FUNCTION fill_matrices_with_independent_parameters
  int ii=1; int i=1; int iii;
  double d=sqrt(0.1);
  if  (!sg) d=1.0;
  ch_Sigma.initialize();
  // this is the choleski decomp parameterization of the
  // covariance matrix
  for (i=1;i<=m;i++)
    for (int j=1;j<=i;j++) {
      if (i==j)ch_Sigma(i,i)+=d;
      ch_Sigma(i,j)+=v_Sigma(ii++);
    }
  for (iii=1;iii<=ra;iii++) {
    if (iii==1 || active(Fcoff(iii))) {
      ii=1;
      F(iii).initialize();
      tF(iii).initialize();
      for (i=1;i<=m;i++) {
        for (int j=1;j<=m;j++)
          F(iii)(i,j)=Fcoff(iii)(ii++);
      }
    }
    tF(iii)=trans(F(iii));
  }

  for (iii=1;iii<=sg;iii++) {
    if (iii==1 || active(Gcoff(iii))) {
      ii=1;
      G(iii).initialize();
      tG(iii).initialize();
      for (i=1;i<=m;i++) {
        for (int j=1;j<=m;j++) {
          G(iii)(i,j)=Gcoff(iii)(ii++);
        }
        if (iii==1) G(iii)(i,i)+=0.90;
      }
    }
    tG(iii)=trans(G(iii));
  }
  B.initialize();
  A.initialize();
  // B(0) is the identity matrix
  B(0)=Idm;
  for (i=1;i<=p;i++) A(i)=A1(i);
  for (i=1;i<=q;i++) B(i)=B1(i);
  for (i=0;i<=q;i++) Bt(i)=trans(B(i));

FUNCTION calculate_the_residuals
  int t; int j;
  mu=muemp+mudev;
  for (t=-p+1;t<=0;t++) Yv(t)=Y(t)-mu;
  for (t=1;t<=n;t++) {
    Yv(t)=Y(t)-mu;
    calculate_autoregressive_part(t);
    r(t)=Yv(t)-arpart;
  }
  int ii=0;
  // this corresponds to the VEC operator
  for (int i=1;i<=n;i++)
    for (j=1;j<=m;j++) vecr(++ii)=r(i,j);

FUNCTION void calculate_the_residuals2(dvar_matrix& e)
  int t; int j;
  mu=muemp+mudev;
  for (t=-p+1;t<=0;t++) Yv(t)=Y(t)-mu;
  for (t=1;t<=n;t++) {
    Yv(t)=Y(t)-mu;
    calculate_autoregressive_part(t);
    e(t)=Yv(t)-arpart;
    for (j=1;j<=q;j++) {
      if (t<=j) break;
      e(t)-=B(j)*e(t-j);
    }
  }

FUNCTION calculate_the_sub_covariances
  int i; int k; int l;
  int qq=0;
  for ( l=1;l<=q;l++) {
    if (!active(B1(l))) break;
    qq=l;
  }
  cov.initialize();
  for (i=1;i<=n;i++) {
    for (int l=0;l<=qq;l++) {
      if (i<=l) break;
      for (int k=0;k<=qq-l;k++) {
        int ilk=i-l-k;
        if (ilk<1) ilk=1;
        cov(i,l)+=B(l+k)*Sigma(ilk)*Bt(k);
      }
    }
  }

FUNCTION calculate_the_empirical_covariance
  int i;
  Semp.initialize();
  ivector sgn(1,m);
  esd.initialize();
  for (i=1;i<=n;i++) {
    esd+=square(r(i));
  }
  esd/=n;
  esd=sqrt(esd);
  if (active(delta)) {
    dvar_vector mult_neg=elem_div(esd,delta);
    dvar_vector mult_pos=elem_prod(esd,delta);
    for (i=1;i<=n;i++) {
      sgn.initialize();
      dvar_vector sr=sfabs(elem_div(r(i),esd));
      for (int j=1;j<=m;j++)
        if (r(i,j)<0)
          sr(j)=-sr(j)*mult_neg(j);
        else
          sr(j)=sr(j)*mult_pos(j);
      rr(i)=outer_prod(sr,sr);
    }
    Semp=empirical_covariance(r);
  }
  else
  {
    for (i=1;i<=n;i++) {
      rr(i)=outer_prod(r(i),r(i));
      Semp+=rr(i);
    }
    Semp/=n;
  }
  for (int j=1;j<=m;j++)
    esd(j)=sqrt(Semp(j,j));

FUNCTION dvariable calculate_the_sub_variances_diagonal_vector_garch(void)
  int i; int k; int ii; int jj;
  dvar_vector norms(1,n);
  dvariable fpen=0.0;
  dvariable fpen1;
  Omega=ch_Sigma*SSemp*trans(ch_Sigma);
  Sigma.initialize();
  // set the first Sigma equal to the empirical covariance
  int rsmax=mymax(ra,sg);
  Sigma(1)=SSemp;
  //cout << Sigma(1) << endl;
  dvar_matrix SS=scale(Sigma(1),esd);
  //cout << SS << endl;
  fpen+=positivize_sigma(SS);
  //cout << SS << endl;
  dvariable ns=norm(SS);
   norms(1)=ns;
  fpen1=0.0;
  dvariable bn=mf_upper_bound(ns,1000.0,fpen1);
  if (fpen1>0.0) {
    SS*=(bn/ns);
    fpen+=fpen1;
  }
  Sigma(1)=unscale(SS,esd);
  for (i=2;i<=rsmax;i++) {
    Sigma(i)=Sigma(1);
    norms(i)=norms(1);
  }

  int mmin=Sigma(1).indexmin();
  int mmax=Sigma(1).indexmax();
  dvar_vector s(mmin,mmax);
  for (i=rsmax+1;i<=n;i++) {
    Sigma(i)=Omega;
    if (ra) Sigma(i)+=elem_prod(F(1),rr(i-1));
    for (ii=2;ii<=ra;ii++) {
      if (active(Fcoff(ii)))
        Sigma(i)+=elem_prod(F(ii),rr(i-ii));
    }

    if (sg) Sigma(i)+=elem_prod(G(1),Sigma(i-1));
    for (ii=2;ii<=sg;ii++) {
      if (active(Gcoff(ii)))
        Sigma(i)+=elem_prod(G(ii),Sigma(i-ii));
    }

    // "positivize" the
    // correlation matrix
    dvar_matrix SS=scale(Sigma(i),esd);
    fpen+=positivize_sigma(SS);
    dvariable ns=norm(SS);
    norms(i)=ns;
    fpen1=0.0;
    dvariable bn=mf_upper_bound(ns,1000.0,fpen1);
    if (fpen1>0.0) {
      SS*=(bn/ns);
      fpen+=fpen1;
    }
    Sigma(i)=unscale(SS,esd);
  }
  dvector trend(1,n);
  trend.fill_seqadd(-1,2.0/(n-1));
  cout << "norms*trend/norm(norms)" << endl;
  dvariable npen=norms*trend/norm(norms);
  cout << norms*trend/norm(norms) << endl;
  fpen+=npen;
  if (fpen>1.0)
    cout << " fpen = " << fpen << endl;
  return fpen;

FUNCTION dvariable calculate_the_sub_variances_BEKK(void)
  int i; int k; int ii; int jj;
  dvar_vector norms(1,n);
  dvariable fpen=0.0;
  dvariable fpen1;
  Omega=ch_Sigma*SSemp*trans(ch_Sigma);
  Sigma.initialize();
  // set the first Sigma equal to the empirical covariance
  int rsmax=mymax(ra,sg);
  Sigma(1)=SSemp;
  dvar_matrix SS=scale(Sigma(1),esd);
  fpen+=positivize_sigma(SS);
  dvariable ns=norm(SS);
   norms(1)=ns;
  fpen1=0.0;
  dvariable bn=mf_upper_bound(ns,1000.0,fpen1);
  if (fpen1>0.0) {
    SS*=(bn/ns);
    fpen+=fpen1;
  }
  Sigma(1)=unscale(SS,esd);
  for (i=2;i<=rsmax;i++) {
    Sigma(i)=Sigma(1);
    norms(i)=norms(1);
  }

  int mmin=Sigma(1).indexmin();
  int mmax=Sigma(1).indexmax();
  dvar_vector s(mmin,mmax);
  for (i=rsmax+1;i<=n;i++) {
    Sigma(i)=Omega;
    if (ra) Sigma(i)+=F(1)*rr(i-1)*tF(1);
    for (ii=2;ii<=ra;ii++) {
      if (active(Fcoff(ii)))
        Sigma(i)+=F(ii)*rr(i-ii)*tF(ii);
    }

    if (sg) Sigma(i)+=G(1)*Sigma(i-1)*tG(1);
    for (ii=2;ii<=sg;ii++) {
      if (active(Gcoff(ii)))
        Sigma(i)+=G(ii)*Sigma(i-ii)*tG(ii);
    }

    // "positivize" the
    // correlation matrix
    dvar_matrix SS=scale(Sigma(i),esd);
    fpen+=positivize_sigma(SS);
    dvariable ns=norm(SS);
    norms(i)=ns;
    fpen1=0.0;
    dvariable bn=mf_upper_bound(ns,1000.0,fpen1);
    if (fpen1>0.0) {
      SS*=(bn/ns);
      fpen+=fpen1;
    }
    Sigma(i)=unscale(SS,esd);
  }
  dvector trend(1,n);
  trend.fill_seqadd(-1,2.0/(n-1));
  cout << "norms*trend/norm(norms)" << endl;
  dvariable npen=norms*trend/norm(norms);
  cout << norms*trend/norm(norms) << endl;
  fpen+=npen;
  if (fpen>1.0)
    cout << " fpen = " << fpen << endl;
  {
    //ofstream ofs("sigma");

    //for (int i=1;i<=n;i++)
      //ofs << Sigma(i) << endl << endl;
  }
  return fpen;

FUNCTION dvar_matrix scale(dvar_matrix& M,dvar_vector& sd)
   int mmin=sd.indexmin();
   int mmax=sd.indexmax();
   dvar_matrix SM(mmin,mmax,mmin,mmax);
   for (int i=mmin;i<=mmax;i++)
     for (int j=mmin;j<=mmax;j++)
       SM(i,j)=M(i,j)/(sd(i)*sd(j));
   return SM;

FUNCTION dvar_matrix unscale(dvar_matrix& M,dvar_vector& sd)
   int mmin=sd.indexmin();
   int mmax=sd.indexmax();
   dvar_matrix SM(mmin,mmax,mmin,mmax);
   for (int i=mmin;i<=mmax;i++)
     for (int j=mmin;j<=mmax;j++)
       SM(i,j)=M(i,j)*(sd(i)*sd(j));
   return SM;

FUNCTION dvar_matrix get_initial_sigma(void)
   int i,j,k,l,ll,m2,r,s;
   m2=m*m;
   dvar_matrix M(1,m2,1,m2);

   dvar_vector v=VEC(Semp);
   M=identity_matrix(1,m2);
   for (ll=1;ll<=q;ll++)
   {
     for (i=1;i<=m;i++)
     {
       for (j=1;j<=m;j++)
       {
         int col=(i-1)*m+j;
         for (r=1;r<=m;r++)
         {
           for (s=1;s<=m;s++)
           {
             int row=(r-1)*m+s;
             M(row,col)+=B(ll)(r,i)*B(ll)(s,j);
           }
         }
       }
     }
   }
   v=solve(M,v);
   dvar_matrix tmp= MAT(v,m,m);
   return tmp;

FUNCTION calculate_the_covariance_matrix
  int ioffset; int joffset; int i1; int i; int j1;
  int k; int l;
  int qq=0;
  for ( l=1;l<=q;l++) {
    if (!active(B1(l))) break;
    qq=l;
  }
  S.initialize();
  for (i=1;i<=n;i++) {
    ioffset=(i-1)*m;
    for (int k=0;k<=qq;k++) {
      //if (k>0 && !active(B1)) break;
      joffset=(i-k-1)*m;
      if (joffset<0) break;
      for (i1=1;i1<=m;i1++) {
        int up;
        if (k==0)
	  up=i1;
	else
	  up=m;
        for (j1=1;j1<=up;j1++) {
          int i2=i1+ioffset;
	  int j2=j1+joffset;
          S(i1+ioffset,j1+joffset)=cov(i,k,i1,j1);
        }
      }
    }
    if (S(1,1) < 0)
      cout << S(1,1) << endl;
  }

FUNCTION void calculate_autoregressive_part(int t)
  // The user can put in any (nonlinear) function desired here
  arpart.initialize();
  for (int j=1;j<=p;j++) arpart+=A(j)*Yv(t-j);

REPORT_SECTION
  int i; int ii; int jj, t;
  for (ii=1;ii<=m;ii++)
  {
    for (jj=1;jj<=ii;jj++)
    {
      ofstream ofs((char*)("covar." + str(ii) +str(jj)));
      ofstream ofs1((char*)("correl." + str(ii) +str(jj)));
      dvar_matrix Covariance(1,m,1,m);
      for (i=1;i<=n;i++) {
        Covariance=Sigma(i);
        for(int j=1;j<=q;j++)
          if ( (i-j)>0 ) Covariance+=B(j)*Sigma(i-j)*trans(B(j));
        ofs << Covariance(ii,jj) << endl;
        ofs1 << Covariance(ii,jj)/
          sqrt(Covariance(ii,ii)*Covariance(jj,jj))<< endl;
      }
    }
  }

  {
    dvar_matrix ymat=MAT(y,n,m);
    for (int i=1;i<=m;i++) {
       ofstream ofs1((char*)("yres." + str(i)));
       dvector tmp(1,n);
       for (t=1;t<=n;t++)
         tmp(t)=value(ymat(t,i));
       ivector hist=histogram(-20,20,81,tmp);
         ofs1 << column_print(hist) << endl;
    }
  }
  report << "The means" << endl;
  report << mu << endl;
  for (i=1;i<=p;i++) {
    report  << "A("<< i << ")" << endl;
    report << setfixed() << setprecision(3) << A(i) << endl;
  }
  report << endl;
  for (i=1;i<=q;i++) {
    report  << "B("<< i << ")" << endl;
    report << setfixed() << setprecision(3) << B(i) << endl;
  }
  report << endl;
  report  << "delta" << endl;
  report << setfixed() << setprecision(3) << delta << endl;
  report << endl;
  report  << "Omega" << endl;
  report << setfixed() << setprecision(3) << Omega << endl;
  report << endl;
  for (i=1;i<=ra;i++) {
    report  << "F("<< i << ")" << endl;
    report << setfixed() << setprecision(3) << F(i) << endl;
  }
  report << endl;
  for (i=1;i<=sg;i++) {
    report  << "G("<< i << ")" << endl;
    report << setfixed() << setprecision(3) << G(i) << endl;
  }
  report << endl;
  //report << setfixed() << setw(8) << setprecision(1) << S << endl;
  {  // calculate predicted observations for next 20 time periods
     // for graphs  results are in t.1 t.2 etc
     int npreds=20;
     dvar_matrix e(1,n,1,m);
     calculate_the_residuals2(e);

     dvar_matrix Z(n+1,n+npreds,1,m);
     Z.initialize();

     for (int t=n+1;t<=n+npreds;t++) {
       Z(t)+=mu;
       for (int i=1;i<=p;i++) {
         if (t-i>n)
           Z(t)+=A(i)*(Z(t-i)-mu);
         else
           Z(t)+=A(i)*(Y(t-i)-mu);
       }
       for (i=1;i<=q;i++) {
         if (t-i<=n && t-i>0)
           Z(t)+=B(i)*e(t-i);
       }
     }
     for (i=1;i<=m;i++) {
       ofstream ofs1((char*)("pred" + str(i)));
       for (t=-p+1;t<=n;t++)
         ofs1 << Y(t,i) << endl;
       for (t=n+1;t<=n+npreds;t++)
         ofs1 << Z(t,i) << endl;
     }
   }
   dmatrix T(1,m,1,m);
   dmatrix chi(1,m,1,m);
   T.initialize();
   dmatrix Aut(1,m,1,m);
   Aut.initialize();
   const int K=10;
   d3_array gamma(1,K,1,m,1,m);
   dmatrix cr(1,n,1,m);
   for (i=1;i<=n;i++)
   {
     cr(i)=value(y((i-1)*m+1,i*m).shift(1));
     T+=outer_prod(cr(i),cr(i));
   }
   T=T/n;
   for (i=2;i<=n;i++)
   {
     Aut+=outer_prod(cr(i-1),cr(i));
   }
   Aut=Aut/(n-1);
   for (i=1;i<=m;i++)
   {
     for (int j=1;j<=m;j++)
     {
       Aut(i,j)/=sqrt(T(i,i)*T(j,j));
     }
   }
   gamma.initialize();
   for (int j=1;j<=m;j++)
   {
     for (int k=1;k<=m;k++)
     {
       for (int l=1;l<=10;l++)
       {
         double tmp=0;
         for (i=1;i<=n-l;i++)
         {
           gamma(l,j,k)+=(cr(i,j)*cr(i,k)-T(j,k))*(cr(i+l,j)*cr(i+l,k)-T(j,k));
           tmp+=square(cr(i,j)*cr(i,k)-T(j,k));
         }
         gamma(l,j,k)/=tmp;
       }
     }
   }

   chi.initialize();
   for (int l=1;l<=K;l++)
   {
     chi+=n*(n+2)/(n-l)*square(gamma(l));
   }

   report << "Covariance of standardized residuals" << endl;
   report << T << endl;
   report << "Lag 1 autocorellation of standardized residuals" << endl;
   report << Aut << endl;
   report << "Ljung Box statistic based on chi squared with " << K
          << " degrees of freedom" << endl;
   report << chi << endl;

FUNCTION add_missing_values
  for (int ii=1;ii<=nmiss;ii++) {
    Y(rowmiss(ii),colmiss(ii))=missvals(ii);
  }
  ofstream ofs("testy");
  ofs << Y << endl;

FUNCTION calculate_time_series_mean
  muemp.initialize();
  for (int i=-p+1;i<=n;i++) muemp+=Y(i);
  muemp/=(n+p);

FUNCTION dvariable positivize_sigma(dvar_matrix& TS)
  int ii,jj;
  dvariable fpen=0.0;
  int mmin=TS.indexmin();
  int mmax=TS.indexmax();
  dvar_vector s(mmin,mmax);
  for (ii=mmin;ii<=mmax;ii++)
    s(ii)=sqrt(posfun(TS(ii,ii),1.e-3,fpen));
  for (ii=mmin;ii<=mmax;ii++)
    for (jj=mmin;jj<=mmax;jj++)
      TS(ii,jj)/=(s(ii)*s(jj));
  TS=positive_definite_matrix(TS,.3,fpen);
  for (ii=mmin;ii<=mmax;ii++)
    for (jj=mmin;jj<=mmax;jj++)
      TS(ii,jj)*=(s(ii)*s(jj));
  return fpen;

GLOBALS_SECTION
  // some C++ compilers don't supply this!
  int mymax(int x,int y)
  {
    if (x>y)
      return x;
    else
      return y;
  }

TOP_OF_MAIN_SECTION

  ofstream ofs("Error.log");
  arrmblsize=5000000;
  gradient_structure::set_GRADSTACK_BUFFER_SIZE(560000);
  gradient_structure::set_CMPDIF_BUFFER_SIZE(15000000);
  gradient_structure::set_MAX_NVAR_OFFSET(1000);
\end{lstlisting}


\chapter{Normalised LMS and Affine Projection Algorithm}
\label{ch:NLMS_APA}
\section{Review of the Basics}
\subsection{Inversion of a 2 by 2 Block Matrix}
Let $\vect{A}$ and $\vect{M}_1=\vect{D}-\vect{C}\vect{A}^{-1}\vect{B}$ be non-singular matrices. Then
\bmath
  \bbmtx
    \vect{A} & \vect{B}\\
    \vect{C} & \vect{D}
  \ebmtx^{-1} =
  \bbmtx
    \vect{A}^{-1}+\vect{A}^{-1}\vect{B}\vect{M}_1^{-1}\vect{C}\vect{A}^{-1} & -\vect{A}^{-1}\vect{B}\vect{M}_1^{-1}\\
    -\vect{M}_1^{-1}\vect{C}\vect{A}^{-1} & \vect{M}_1^{-1}
  \ebmtx\ .
  \label{eq:block_inv1}
\emath
Let instead $\vect{D}$ and $\vect{M}_2=\vect{A}-\vect{B}\vect{D}^{-1}\vect{C}$ be non-singular matrices. Then
\bmath
  \bbmtx
    \vect{A} & \vect{B}\\
    \vect{C} & \vect{D}
  \ebmtx^{-1} =
  \bbmtx
    \vect{M}_2^{-1} & -\vect{M}_2^{-1}\vect{B}\vect{D}^{-1}\\
    -\vect{D}^{-1}\vect{C}\vect{M}_2^{-1} & \vect{D}^{-1}+\vect{D}^{-1}\vect{C}\vect{M}_2^{-1}\vect{B}\vect{D}^{-1}
  \ebmtx\ .
  \label{eq:block_inv2}
\emath
The matrices $\vect{M}_1$ and $\vect{M}_2$ are said to be the \textit{Schur complement} of $\vect{A}$ and $\vect{D}$, respectively.

\subsection{The Method of Lagrange Multipliers}
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/const_opt}
  \caption{Optimisation problem with a single equality constraint. The points $\vect{x}_u$ and $\vect{x}_c$ are the unconstrained and the constrained minimum, respectively.}
  \label{fig:const_opt}
\end{figure}
\noindent The method of Lagrange multipliers can be used to solve optimisation problems with equality constraints\footnote{The method of Lagrange multipliers can also be used to solve the more general problem with inequality constraints, but we do not consider this here.}
\begin{gather}
  \begin{alignedat}{2}
      &\underset{\vect{x}}{\text{optimise}} &\quad f(\vect{x}) &\\
      &\text{subject to}&\quad \vect{h}(\vect{x}) &= \vect{0}
  \end{alignedat}
  \label{eq:constr_opt}
\end{gather}
Consider the optimisation problem in Fig.~\ref{fig:const_opt} with the cost function $f(\vect{x})$ and the constraints $\vect{h}(\vect{x})=\vect{0}$. At the constrained optimum $\vect{x}_c$, the gradients of $f(\vect{x})$ and $\vect{h}(\vect{x})$ are parallel. Thus,
\begin{alignat*}{2}
  \vect{\nabla}f(\vect{x}) &= \lambda \vect{\nabla}h(\vect{x}) &\qquad &\text{(single constraint)}\\
  \vect{\nabla}f(\vect{x}) &= \sum_{m=1}^M\lambda_m \vect{\nabla}h_m(\vect{x})= \vect{\nabla}\vect{h}(\vect{x})\vect{\lambda} &\qquad &\text{(multiple constraints)}
\end{alignat*}
The vector $\vect{\lambda}$ contains the Lagrange multipliers, and they ensure that the gradients have the same direction and length.

If we define the Lagrangian function
\bmath
  L(\vect{x},\vect{\lambda}) = f(\vect{x}) - \vect{\lambda}^T\vect{h}(\vect{x})\ ,
\emath
then its critical points satisfy
\begin{alignat}{2}
  \vect{\nabla}_{\vect{x}} L(\vect{x},\vect{\lambda}) &= \vect{0}\quad\iff&\quad\vect{\nabla}f(\vect{x}) &= \vect{\nabla}\vect{h}(\vect{x})\vect{\lambda}\label{eq:grad_lagr_func}\\
  \vect{\nabla}_{\vect{\lambda}} L(\vect{x},\vect{\lambda}) &= \vect{0}\quad\iff&\quad\vect{h}(\vect{x}) &= \vect{0}\ .\label{eq:grad_lagr_func_const}
\end{alignat}
\eq{eq:grad_lagr_func} is the same as what we previously derived from Fig.~\ref{fig:const_opt}, and \eq{eq:grad_lagr_func_const} is simply the constraints of our optimisation problem. Thus, the solutions to the system of equations given by \eq{eq:grad_lagr_func} and \eq{eq:grad_lagr_func_const} are indeed the critical points of our optimisation problems. In order to find the optimum, we need to classify the critical points as either minimums, maximums, or saddle points. The classification of the critical points for any pair of cost function and constraints is beyond the scope of these lecture notes. We refer the interested reader to \cite[pp.~294--308]{Antoniou2007}. Alternatively, we may formulate the constrained optimisation problem in \eq{eq:constr_opt} as the sequential unconstrained optimisation problem \cite[ch.~5]{Boyd2004}
\bmath
  \max_{\vect{\lambda}}\left\{\underset{\vect{x}}{\text{optimise}}\ L(\vect{x},\vect{\lambda}) \right\}\ .
  \label{eq:constr_opt_nested}
\emath
We could solve this optimisation problem by applying the five step recipe for unconstrained optimisation in Sec.~\ref{ssec:optimisation} to first the inner optimisation problem and then the outer optimisation problem.

We are here not concerned with these general methods. Instead, we consider the simpler problem of minimising a quadratic cost function with linear equality constraints.

\subsubsection{Quadratic Cost Function with Linear Equality Constraints}
We consider the following equality constrained minimisation problem
\begin{gather}
  \begin{alignedat}{2}
      &\min_{\vect{x}} &\quad f(\vect{x}) &= \frac{1}{2}\vect{x}^T\vect{Px}+\vect{q}^T\vect{x}+r\\
      &\text{subject to}&\quad \vect{h}(\vect{x}) &= \vect{Ax}-\vect{b}=\vect{0}
  \end{alignedat}
  \label{eq:constr_opt_quad_lin}
\end{gather}
in which the cost function is quadratic and the equality constraints are linear. We assume that $\vect{P}$ is an $N\times N$ p.d. matrix so that $f(\vect{x})$ is a convex function. Moreover, we assume that $\vect{A}$ is a full rank $M\times N$ matrix with $M\leq N$. The Langrangian function is
\bmath
  L(\vect{x},\vect{\lambda}) = \frac{1}{2}\vect{x}^T\vect{Px}+\vect{q}^T\vect{x}+\vect{r} - \vect{\lambda}^T(\vect{Ax}-\vect{b})
\emath
and its gradient w.r.t. $\vect{x}$ is
\bmath
  \vect{\nabla}_{\vect{x}} L(\vect{x},\vect{\lambda}) = \vect{Px}+\vect{q}-\vect{A}^T\vect{\lambda}\ .
\emath
The system of equations given by \eq{eq:grad_lagr_func} and \eq{eq:grad_lagr_func_const} are therefore
\bmath
  \bbmtx
     \vect{P} & -\vect{A}^T\\
    -\vect{A} &  \vect{0}
  \ebmtx
  \bbmtx
    \vect{x}\\
    \vect{\lambda}
  \ebmtx =
  \bbmtx
    -\vect{q}\\
    -\vect{b}
  \ebmtx\ .
\emath
Since the cost function $f(\vect{x})$ is assumed to be convex, there is only one point satisfying this system of equations. By using the $2\times 2$ block matrix inversion rule in \eq{eq:block_inv1}, we obtain that this point $(\vect{x}_c,\vect{\lambda}_c)$ is
\begin{align}
  \vect{\lambda}_c &= (\vect{AP}^{-1}\vect{A}^T)^{-1}(\vect{AP}^{-1}\vect{q}+\vect{b})\label{eq:constr_opt_quad_lin_minlam}\\
  \vect{x}_c &= \vect{P}^{-1}(\vect{A}^T\vect{\lambda}_c-\vect{q})\ .
  \label{eq:constr_opt_quad_lin_minx}
\end{align}
Thus, the solution to the constrained minimisation problem in \eq{eq:constr_opt_quad_lin} is $\vect{x}_c$ given by \eq{eq:constr_opt_quad_lin_minx}.

\section{Overview over Adaptive Filters based on the Mean-Squared Error Cost Function}
An overview over the adaptive filters is shown in Fig.~\ref{fig:overview}. Recall that we have the following important properties for the steepest descent algorithm and the stochastic gradient methods.
\subsubsection{Steepest Descent (SD)}
\begin{itemize}
  \item Since the statistics is assumed known, the gradient and filter coefficients are deterministic functions.
  \item SD is of limited practical usage, but illustrates nicely how adaptive filters function an how they are analysed.
  \item SD is the "ideal" (first-order) adaptive filter.
\end{itemize}
\subsubsection{Stochastic Gradient Methods (SGMs)}
\begin{itemize}
  \item Since the statistics is unknown, the gradient and filter coefficients are stochastic processes.
  \item They are generally easy to implement and hard to analyse.
  \item They are approximations to the steepest descent algorithm.
\end{itemize}

\subsection{Model for the Analysis of SGMs}
\label{ssec:analysis_model}
In order to make the SGMs easier to analyse, a model is typically assumed for the relationship between the input signal $u(n)$ and the desired signal $d(n)$. Fig.~\ref{fig:sys_id_block_diagram} shows one of the most popular models used for the analysis of adaptive filters.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/sys_id_block_diagram}
  \caption{Block diagram showing the model used to analyse adaptive filters.}
  \label{fig:sys_id_block_diagram}
\end{figure}
Define
\begin{align}
  \vect{w}_o &= \bbmtx w_{o,0} & \cdots & w_{o,M-1} \ebmtx^T\\
  \vect{w}(n) &= \bbmtx w_0(n) & \cdots & w_{M-1}(n) \ebmtx^T\\
  \vect{u}(n) &= \bbmtx u(n) & \cdots & u(n-M+1) \ebmtx^T\\
  \vect{\Delta}\vect{w}(n) &= \vect{w}_o-\vect{w}(n)
\end{align}
where $\vect{\Delta}\vect{w}(n)$ is referred to as the weight error.
\subsubsection{Assumptions}
From Fig.~\ref{fig:sys_id_block_diagram}, we have that
\bmath
  d(n) = z(n)+v(n) = \vect{u}^T(n)\vect{w}_o + v(n)\ .
\emath
Moreover, we assume that
\begin{enumerate}
  \item $v(n)$ is white noise with variance $\sigma_v^2$.
  \item $v(n)$ and $u(n)$ are uncorrelated.
  \item $\vect{u}(n)$ is a white process with correlation matrix $\vect{R}_u$. This assumption implies that $\vect{u}(n)$ and $\vect{w}(n)$ are uncorrelated.
\end{enumerate}
\subsubsection{Equations}
We have that
  \begin{align}
    e(n) &= d(n) - y(n) =  v(n) + \vect{u}^T(n)\vect{w}_o - \vect{u}^T(n)\vect{w}(n)\\
         &= \vect{u}^T(n)\vect{\Delta}\vect{w}(n)+ v(n)
  \end{align}
Thus, at the optimum where $\vect{\Delta}\vect{w}(n)=\vect{0}$, we have that
\bmath
  J_1(\vect{w}_o) = J_\textup{min} = J_1(v^2(n)) = \sigma_v^2\ .
\emath
The cost function can be written as
\begin{align}
  J_1(\vect{w}(n)) &= E[e^2(n)] = E[v^2(n)] + E[\vect{\Delta}\vect{w}^T(n)\vect{u}(n)\vect{u}^T(n)\vect{\Delta}\vect{w}(n)]\notag\\
  &= \sigma_v^2 + E[E[\vect{\Delta}\vect{w}^T(n)\vect{u}(n)\vect{u}^T(n)\vect{\Delta}\vect{w}(n)|\vect{\Delta}\vect{w}(n)]]\notag\\
  &= J_\textup{min} + E[\vect{\Delta}\vect{w}^T(n)\vect{R}_u\vect{\Delta}\vect{w}(n)]\ .
\end{align}
When viewed as a function of $n$, the cost function is called the learning curve.

\subsection{How to Analyse Adaptive Filters}
Adaptive filters can be analysed in several ways.
\begin{description}
  \item[Transient Performance] How does the filter handle an abrupt change in the statistics of $u(n)$, and how fast does it converge to steady-state?
  \item[Steady-state Performance] How does the filter perform in a WSS environment after all transients have died out?
  \item[Tracking Performance] How does the filter handle slow variations in the statistics of $u(n)$ and/or $d(n)$?
  \item[Numerical Precision Effects] What happens when the filter is implemented on a finite precision computer?
  \item[Computational Complexity] How much processing time does the algorithm require?
\end{description}
\subsubsection{Important Values and Functions}
When analysing adaptive filters, we typically quantify their performance in terms of the following values and functions.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/learning_curve_sgm}
  \caption{The cost function (same as the MSE), the minimum MSE, and the excess MSE.}
  \label{fig:learning_curve_sgm}
\end{figure}
\begin{alignat}{2}
  &J_1(\vect{w}(n))  &\quad &\text{(Mean-Square Error (MSE))}\\
  &J_1(\vect{w}_o) = J_\textup{min} &\quad &\text{(Minimum MSE (MMSE))}\\
  &J_\textup{ex} = J_1(\vect{w}(\infty))-J_\textup{min} &\quad &\text{(Excess MSE (EMSE))}\\
  &\mathcal{M} = \frac{ J_\textup{ex}}{J_\textup{min}} &\quad &\text{(Misadjustment)}\\
  &E[\|\vect{\Delta}\vect{w}(\infty)\|^2]  &\quad &\text{(Mean-Square Deviation (MSD))}\\
  &\lambda_\textup{max}  &\quad &\text{(Maximum Eigenvalue of $\vect{R}_u$)}\\
  &\lambda_\textup{min}  &\quad &\text{(Minimum Eigenvalue of $\vect{R}_u$)}\\
  &\kappa(\vect{R}_u) = \frac{\lambda_\textup{max}}{\lambda_\textup{min}} &\quad &\text{(Condition number of $\vect{R}_u$)}
\end{alignat}
In adaptive filtering, some people use the term eigenvalue spread $\chi(\vect{R}_u)$ instead of the condition number. The MSE, the EMSE, the MSD, and the misadjustment depend on the adaptive filtering algorithm. Conversely, the remaining parameters depend on the desired signal and the input signal and are thus independent of the adaptive filtering algorithm. In Fig.~\ref{fig:learning_curve_sgm}, some of the quantities are shown.

\section{LMS Revisited}
Recall the steepest descent algorithm
\bmath
  \vect{w}(n+1) = \vect{w}(n)-\frac{\mu}{2}\vect{g}(\vect{w}(n))
\emath
where
\bmath
  \vect{g}(\vect{w}(n)) = 2\vect{R}_u\vect{w}(n)-2\vect{r}_{ud} = -2E[\vect{u}(n)e(n)]
\emath
is the gradient of $J_1(\vect{w}(n))$. Replacing the gradient with the simple estimate
\bmath
  \hat{\vect{g}}(\vect{w}(n)) = -2\vect{u}(n)e(n)
\emath
leads to the LMS algorithm
\begin{align}
  \vect{w}(n+1) &= \vect{w}(n) + \mu\vect{u}(n)e(n)\\
                &= (\vect{I}-\mu\vect{u}(n)\vect{u}^T(n))\vect{w}(n) + \mu\vect{u}(n)d(n)\ .
\end{align}
In order to analyse the LMS algorithm, we use the analysis model in Sec.~\ref{ssec:analysis_model}.

\subsection{Transient Analysis}
We would like that
\begin{alignat*}{2}
  \lim_{n\to\infty} E[\Delta\vect{w}(n)] &= 0 &\qquad &\text{(Convergence in mean)}\\
  \lim_{n\to\infty} E[\|\Delta\vect{w}(n)\|^2] &= c < \infty &\qquad &\text{(Convergence in mean-square)}
\end{alignat*}
\subsubsection{Convergence in the Mean}
We have that
\bmath
  \vect{\Delta}\vect{w}(n+1) = (\vect{I}-\mu\vect{u}(n)\vect{u}^T(n))\vect{\Delta}\vect{w}(n) + \mu\vect{u}(n)v(n)
\emath
from which we get
\bmath
  E[\vect{\Delta}\vect{w}(n+1)] = (\vect{I}-\mu\vect{R}_u)E[\vect{\Delta}\vect{w}(n)]\ .
\emath
Thus, the LMS algorithm is stable in the mean if (same as for SD)
\bmath
  0 < \mu < \frac{2}{\lambda_\textup{max}}\ .
\emath
Since $\lambda_\textup{max}$ is unknown and hard to estimate, we can bound it by
\bmath
  \lambda_\textup{max} \leq \textup{tr}(\vect{R}_u) = ME[u^2(n)] \leq M S_\textup{max}\ .
\emath
Estimating the power $E[u^2(n)]$ or the maximum is the power spectral density $S_\textup{max}$ is fairly easy.

\subsubsection{Convergence in the Mean-Square}
We assume that $\vect{u}(n)$ has a Gaussian distribution with zero-mean and covariance matrix $\vect{R}_u$. Then, as we have shown in Appendix~\ref{app:lms_analysis}, we get that the LMS algorithm is stable in the mean-square if the step-size satisfies
\bmath
  f(\mu) = \frac{\mu}{2}\sum_{m=1}^M \frac{\lambda_m}{1-\mu\lambda_m} < 1\ .
\emath
For small step-sizes $\mu \ll 1/\lambda_\textup{max}$, we have
\bmath
  f(\mu) \approx \frac{\mu}{2}\sum_{m=1}^M\lambda_m = \frac{\mu}{2}\tr{\vect{R}_u} < 1\ .
\emath
Thus, stability in the mean-square requires that the step-size satisfies
\bmath
  0 < \mu < \frac{2}{\tr{\vect{R}_u}}\ .
\emath

\subsection{Steady-State Analysis}
From Appendix~\ref{app:lms_analysis}, we have that
\begin{alignat}{2}
  &\text{EMSE:}&\quad J_\textup{ex} &= J_1(\vect{w}(\infty))-J_\textup{min} = J_\textup{min}\frac{f(\mu)}{1-f(\mu)}\\
  & &\quad &\approx \frac{\mu}{2}J_\textup{min}\tr{\vect{R}_u}\\
  &\text{Misadjustment:}&\quad \mathcal{M} &= \frac{ J_\textup{ex}}{J_\textup{min}} = \frac{f(\mu)}{1-f(\mu)}\\
  & &\quad &\approx \frac{\mu}{2}\tr{\vect{R}_u}\\
  &\text{MSD:} &\quad E[\|\vect{\Delta}\vect{w}(\infty)\|^2] &\approx \frac{\mu}{2} J_\textup{min} M
\end{alignat}
Here, the approximations are valid for small step-sizes. We see that the EMSE is approximately proportional to the input power. This is a problem and referred to as \textit{gradient noise amplification}.

\section{Normalised LMS Adaptive Filters}
Recall that we can bound the maximum eigenvalue by
\bmath
  \lambda_\textup{max} \leq \textup{tr}(\vect{R}_u) = ME[u^2(n)]\ .
\emath
An estimate of the power could be
\bmath
  \hat{E}[u^2(n)] = \frac{1}{M}\vect{u}^T(n)\vect{u}(n) = \frac{1}{M}\|\vect{u}(n)\|^2\ .
\emath
This leads to the following bound on the step-size
\bmath
  0 < \mu < \frac{2}{\|\vect{u}(n)\|^2}\quad\iff\quad 0 < \mu\|\vect{u}(n)\|^2 < 2\ .
\emath
We may define the time-varying step-size
\bmath
  \mu(n) = \frac{\beta}{\|\vect{u}(n)\|^2}\ ,\qquad 0<\beta<2\ .
\emath
Then, the LMS algorithm is
\bmath
  \vect{w}(n+1) = \vect{w}(n) + \frac{\beta}{\|\vect{u}(n)\|^2}\vect{u}(n)e(n)
\emath
which is called the normalised LMS (NLMS) algorithm. The NLMS bypasses the problem of gradient noise amplification by normalising the gradient estimate by the input power. However, this normalisation causes numerical problems when the input power is close to zero. In order to avoid this, we introduce the regularisation parameter $\epsilon$, which is a small positive constant, into the NLMS algorithm
\bmath
  \boxed{\vect{w}(n+1) = \vect{w}(n) + \frac{\beta}{\epsilon+\|\vect{u}(n)\|^2}\vect{u}(n)e(n)}\ .
\emath
\subsubsection{Fast Computation}
The power estimate may be computed very efficiently by writing it in a recursive manner
\begin{align}
  \|\vect{u}(n)\|^2 &= \sum_{m=0}^{M-1}u^2(n-m) = u^2(n) - u^2(n-M) + \sum_{m=1}^{M}u^2(n-m)\notag\\
  &= u^2(n) - u^2(n-M) + \|\vect{u}(n-1)\|^2\ .
\end{align}
It should be noted that the use of this recursion can be problematic in practice due to accumulated rounding errors \cite[p.~227]{Sayed2003}. These rounding errors may potentially cause the norm to be negative.

\subsection{Transient Analysis}
In order to analyse the NLMS algorithm, we use the analysis model in Sec.~\ref{ssec:analysis_model}. If $\sigma_v^2=0$, the NLMS algorithm converges in the mean and the mean-square if \cite[p.~325]{Haykin2001}
\bmath
  \boxed{0<\beta<2}\ .
\emath
Moreover, the optimal step-size is
\bmath
  \beta_o = 1\ .
\emath
If $\sigma_v^2>0$, the optimal step-size is more complicated and can be found in \cite[p.~325]{Haykin2001}.

\subsection{Steady-State Analysis}
Assuming the analysis model in Sec.~\ref{ssec:analysis_model}, it can be shown that \cite[pp.~300--302,p.~474]{Sayed2003}
\begin{alignat}{2}
  &\text{EMSE:}&\quad J_\textup{ex} &= J_1(\vect{w}(\infty))-J_\textup{min}\notag\\
  & &\quad &\approx \frac{\beta}{2} J_\textup{min}\tr{\vect{R}_u}E\left[\frac{1}{\|\vect{u}(n)\|^2}\right]\geq\frac{\beta}{2} J_\textup{min}\\
  &\text{Misadjustment:}&\quad \mathcal{M} &= \frac{ J_\textup{ex}}{J_\textup{min}} \approx \frac{\beta}{2}\tr{\vect{R}_u}E\left[\frac{1}{\|\vect{u}(n)\|^2}\right]\geq\frac{\beta}{2}\\
  &\text{MSD:} &\quad E[\|\vect{\Delta}\vect{w}(\infty)\|^2] &\approx \frac{\beta}{2} J_\textup{min} E\left[\frac{1}{\|\vect{u}(n)\|^2}\right] \geq \frac{\beta J_\textup{min}}{2\tr{\vect{R}_u}}
\end{alignat}
where the inequality follows from that $E[x^{-1}]\geq E[x]^{-1}$. The approximations are valid for small values of $\beta$ and $\epsilon$. Note that the approximation of the misadjustment no longer depends on the input power. This is a consequence of the normalisation of the LMS gradient with $\|\vect{u}(n)\|^2$. Moreover, it should be noted that there exist several other approximations to the EMSE and the MSD than presented above \cite[p.~474]{Sayed2003}.

\subsection{Computational Cost}
Table~\ref{tab:comp_cost_nlms} shows the computational cost of the NLMS algorithm in terms of the number of multiplications, additions or subtractions, and divisions. From the table, we see that the total number of flops is $6M+2$. If we use the fast computation of the normalisation constant, the computational cost is reduced to $4M+7$ flops as shown in Table~\ref{tab:comp_cost_nlms_fast}. Thus, the NLMS algorithm has a linear complexity in the filter length $\mathcal{O}(M)$ for both versions of the NLMS algorithm.
\begin{table}[htbp]
  \centering
  \begin{tabular}{l c c c}
    \toprule
    Term & $\times$ & $+$ or $-$ & $/$ \\
    \midrule
    $\|\vect{u}(n)\|^2$ & $M$ & $M-1$ & \\
    $\vect{u}^T(n)\vect{w}(n)$ & $M$ & $M-1$ & \\
    $e(n)=d(n)-\vect{u}^T(n)\vect{w}(n)$ & & $1$ & \\
    $\beta e(n)/(\epsilon+\|\vect{u}(n)\|^2)$ & $1$ & $1$ & $1$\\
    $\beta\vect{u}(n)e(n)/(\epsilon+\|\vect{u}(n)\|^2)$ & $M$ & & \\
    $\vect{w}(n)+\beta \vect{u}(n)e(n)/(\epsilon+\|\vect{u}(n)\|^2)$ &  & $M$ &\\
    \midrule
    Total & $3M+1$ & $3M$ & $1$ \\
    \bottomrule
  \end{tabular}
  \caption{Computational cost of the NLMS algorithm.}
  \label{tab:comp_cost_nlms}
\end{table}
\begin{table}[htbp]
  \centering
  \begin{tabular}{l c c c}
    \toprule
    Term & $\times$ & $+$ or $-$ & $/$ \\
    \midrule
    $\|\vect{u}(n)\|^2$ & 2 & 2 & \\
    $\vect{u}^T(n)\vect{w}(n)$ & $M$ & $M-1$ & \\
    $e(n)=d(n)-\vect{u}^T(n)\vect{w}(n)$ & & $1$ & \\
    $\beta e(n)/(\epsilon+\|\vect{u}(n)\|^2)$ & $1$ & $1$ & $1$\\
    $\beta\vect{u}(n)e(n)/(\epsilon+\|\vect{u}(n)\|^2)$ & $M$ & & \\
    $\vect{w}(n)+\beta \vect{u}(n)e(n)/(\epsilon+\|\vect{u}(n)\|^2)$ &  & $M$ & \\
    \midrule
    Total & $2M+3$ & $2M+3$ & $1$ \\
    \bottomrule
  \end{tabular}
  \caption{Computational cost of the NLMS algorithm with fast update of the normalisation factor.}
  \label{tab:comp_cost_nlms_fast}
\end{table}

\subsection{Another Derivation of the NLMS Algorithm}
We have two requirements to the adaptive filter.
\begin{enumerate}
  \item If we filter the input sample at time $n$ through the filter at time $n+1$, the error should be zero. That is,
  \bmath
    d(n) - \vect{u}^T(n)\vect{w}(n+1) = 0\ .
  \emath
  There are an infinite number of solutions that satisfy this requirement if $M>1$.
  \item Among all the filter coefficient vectors $\vect{w}(n+1)$ satisfying the first requirements, we select the vector resulting in the smallest change from iteration $n$ to $n+1$. That is, $\|\vect{w}(n+1)-\vect{w}(n)\|^2$ should be as small as possible.
\end{enumerate}
These two requirements lead to the following constrained minimisation problem
\begin{gather}
  \begin{alignedat}{2}
      &\min_{\mathclap{\vect{w}(n+1)}} &\quad f(\vect{w}(n+1))&=\|\vect{w}(n+1)-\vect{w}(n)\|^2\\
      &\text{s.t.}&\quad h(\vect{w}(n+1)) &= d(n) - \vect{u}^T(n)\vect{w}(n+1) = 0
  \end{alignedat}
  \label{eq:nlms_opt}
\end{gather}
This optimisation problem has a quadratic cost function and a single linear equality constraint, and it is therefore on the same form as \eq{eq:constr_opt_quad_lin} with
\begin{align}
  \vect{x} &= \vect{w}(n+1)\\
  \vect{A} &= -\vect{u}^T(n)\\
  \vect{b} &= -d(n)\\
  \vect{P} &= 2\vect{I}\\
  \vect{q} &= -2\vect{w}(n)\\
  \vect{r} &= \|\vect{w}(n)\|^2\ .
\end{align}
Thus, from \eq{eq:constr_opt_quad_lin_minlam} and \eq{eq:constr_opt_quad_lin_minx}, we readily obtain the solution to the optimisation problem in \eq{eq:nlms_opt}. The solution is given by
\begin{align}
  \lambda &= -\frac{2}{\|\vect{u}(n)\|^2}e(n)\\
  \vect{w}(n+1) &= \vect{w}(n)+\frac{1}{\|\vect{u}(n)\|^2}\vect{u}(n)e(n)\ .
\end{align}
The last equation is identical to the NLMS algorithm with $\beta=1$ and $\epsilon=0$.

\section{Affine Projection Adaptive Filters}
We may extend the requirements to the adaptive filter in the following way.
\begin{enumerate}
  \item If we filter the input sample at time $n-k$ for $k=0,1,\ldots,K-1<M$ through the filter at time $n+1$, the corresponding errors should be zero. That is,
  \bmath
    d(n-k) - \vect{u}^T(n-k)\vect{w}(n+1) = 0\ ,\quad k=0,1,\ldots,K-1<M\ .
  \emath
  If we define
  \begin{align}
    \vect{U}(n) &= \bbmtx\vect{u}(n) & \vect{u}(n-1) & \cdots & \vect{u}(n-K+1)\ebmtx\\
    \vect{d}(n) &= \bbmtx d(n) & d(n-1) & \cdots & d(n-K+1)\ebmtx^T\ ,
  \end{align}
  we may write
  \bmath
    \vect{d}(n)-\vect{U}^T(n)\vect{w}(n+1) = \vect{0}\ .
  \emath
  If $K=M$ and $\vect{U}(n)$ has full rank, there is only one filter coefficient vector $\vect{w}(n+1)$ satisfying this requirement. It is given by
  \bmath
    \vect{w}(n+1) = \vect{U}^{-T}(n)\vect{d}(n)\ .
  \emath
  When this is not the case, there are an infinite number of solutions that satisfy this requirement, and we therefore need a second requirement.
  \item Among all the filter coefficient vectors $\vect{w}(n+1)$ satisfying the first requirements, we select the vector resulting in the smallest change from iteration $n$ to $n+1$. That is, $\|\vect{w}(n+1)-\vect{w}(n)\|^2$ should be as small as possible.
\end{enumerate}
Note that for $K=1$, the requirements are identical to the requirements leading to the NLMS algorithm. The affine projections algorithm (APA) can therefore be viewed as a generalisation of the NLMS algorithm. We may formulate the requirements as a constrained optimisation problem
\begin{gather}
  \begin{alignedat}{2}
      &\min_{\mathclap{\vect{w}(n+1)}} &\quad f(\vect{w}(n+1)) &=\|\vect{w}(n+1)-\vect{w}(n)\|^2\\
      &\text{s.t.}&\quad \vect{h}(\vect{w}(n+1)) &=  \vect{d}(n)-\vect{U}^T(n)\vect{w}(n+1) = \vect{0}
  \end{alignedat}
  \label{eq:apa_opt}
\end{gather}
This optimisation problem has a quadratic cost function and $K\leq M$ linear equality constraints, and it is therefore on the same form as \eq{eq:constr_opt_quad_lin} with
\begin{align}
  \vect{x} &= \vect{w}(n+1)\\
  \vect{A} &= -\vect{U}^T(n)\\
  \vect{b} &= -\vect{d}(n)\\
  \vect{P} &= 2\vect{I}\\
  \vect{q} &= -2\vect{w}(n)\\
  \vect{r} &= \|\vect{w}(n)\|^2\ .
\end{align}
Thus, from \eq{eq:constr_opt_quad_lin_minlam} and \eq{eq:constr_opt_quad_lin_minx}, we readily obtain the solution to the optimisation problem in \eq{eq:apa_opt}. The solution is given by
\begin{align}
  \vect{\lambda} &= -2\left(\vect{U}^T(n)\vect{U}(n)\right)^{-1}\vect{e}(n)\\
  \vect{w}(n+1) &= \vect{w}(n)+\vect{U}(n)\left(\vect{U}^T(n)\vect{U}(n)\right)^{-1}\vect{e}(n)\ .
\end{align}

From this result, we see that the APA reduces to the NLMS algorithm for $K=1$. Usually, we add a regularisation parameter $\epsilon$ and a step-size parameter $\beta$ to the algorithm and obtain
\bmath
  \boxed{\vect{w}(n+1) = \vect{w}(n)+\beta\vect{U}(n)\left(\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n)\right)^{-1}\vect{e}(n)}\ .
\emath
As for the NLMS algorithm, $\epsilon$ is a small positive value which bypasses numerical problems when $\vect{U}^T(n)\vect{U}(n)$ is ill-conditioned.

For $K>1$, The computational complexity of the APA is higher than that of the NLMS algorithm since we have to invert a $K\times K$ matrix. Although there exist fast ways of doing this \cite[pp.~339--340]{Haykin2001}, the APA is more expensive than the NLMS algorithm.

\subsection{Transient Analysis}
In order to analyse the APA, we use the analysis model in Sec.~\ref{ssec:analysis_model}. If $\sigma_v^2=0$, the APA converges in the mean and the mean-square if \cite[p.~337]{Haykin2001}
\bmath
  \boxed{0<\beta<2}\ .
\emath
Moreover, the optimal step-size is
\bmath
  \beta_o = 1\ .
\emath
If $\sigma_v^2>0$, the optimal step-size is more complicated and can be found in \cite[p.~337]{Haykin2001}.

\subsection{Steady-State Analysis}
Assuming the analysis model in Sec.~\ref{ssec:analysis_model}, it can be shown that \cite[p.~327]{Sayed2003}
\begin{alignat}{2}
  &\text{EMSE:}&\quad J_\textup{ex} &= J_1(\vect{w}(\infty))-J_\textup{min}\notag\\
  & &\quad &\approx \frac{\beta}{2} J_\textup{min}\tr{\vect{R}_u}E\left[\frac{K}{\|\vect{u}(n)\|^2}\right]\geq\frac{\beta}{2} J_\textup{min}K\\
  &\text{Misadjustment:}&\quad \mathcal{M} &= \frac{ J_\textup{ex}}{J_\textup{min}} \approx \frac{\beta}{2}\tr{\vect{R}_u}E\left[\frac{K}{\|\vect{u}(n)\|^2}\right]\geq\frac{\beta K}{2}
\end{alignat}
where the inequality follows from that $E[x^{-1}]\geq E[x]^{-1}$. The approximations are valid for small values of $\beta$ and $\epsilon$. It should be noted that there exist several other approximations to the EMSE than presented above \cite[p.~325]{Sayed2003}. For more accurate expressions and for an expression for the mean-square deviation, see \cite[pp.~510--512]{Sayed2003} and \cite{Shin2004}.

\subsection{Computational Cost}
Table~\ref{tab:comp_cost_apa} shows the computational cost of the APA algorithm in terms of the number of multiplications and additions or subtractions. We have assumed that the cost of inverting a $K\times K$ matrix is $K^3$ multiplications and additions \cite[p.~240]{Sayed2003}. From the table, we see that the total number of flops is $2M(K^2+2K)+2K^3+K^2+M$. Thus, the APA has a complexity of $\mathcal{O}(MK^2)$. Note that there exist faster ways of implementing the APA \cite[pp.~339-340]{Haykin2001}.
\begin{table}[htbp]
  \centering
  \begin{tabular}{@{}l c c c@{}}
    \toprule
    Term & $\times$ & $+$ or $-$ \\
    \midrule
    $\vect{U}^T(n)\vect{w}(n)$ & $MK$ & $(M-1)K$ \\
    $\vect{e}(n)=\vect{d}(n)-\vect{U}^T(n)\vect{w}(n)$ & & $K$ \\
    $\vect{U}^T(n)\vect{U}(n)$ & $MK^2$ & $(M-1)K^2$ \\
    $\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n)$ & & $K$ \\
    $(\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n))^{-1}$ & $K^3$ & $K^3$ \\
    $(\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n))^{-1}\vect{e}(n)$ & $K^2$ & $K(K-1)$ \\
    $\vect{U}(n)(\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n))^{-1}\vect{e}(n)$ & $MK$ & $M(K-1)$ \\
    $\beta\vect{U}(n)(\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n))^{-1}\vect{e}(n)$ & $M$ &  \\
    $\vect{w}(n)+\beta\vect{U}(n)(\epsilon\vect{I}+\vect{U}^T(n)\vect{U}(n))^{-1}\vect{e}(n)$ &  & $M$\\
    \midrule
    Total & $M(K^2+2K+1)+K^3+K^2$ & $M(K^2+2K)+K^3$\\
    \bottomrule
  \end{tabular}
  \caption{Computational cost of the APA.}
  \label{tab:comp_cost_apa}
\end{table}


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{locfreq}
\section*{\hspace*{-1.6cm} locfreq}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Frequency localization characteristics.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
[fm,B] = locfreq(x)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty locfreq} computes the frequency localization characteristics of
        signal {\ty x}. The definition used for the averaged frequency
        and the frequency spreading are the following\,:
\begin{eqnarray*}
f_m &=& \frac{1}{E_x}\ \int_{-\infty}^{+\infty} \nu\ |X(\nu)|^2\ d\nu\\
B &=& 2\ \sqrt{\frac{\pi}{E_x}\ \int_{-\infty}^{+\infty} (\nu-f_m)^2\
|X(\nu)|^2\ d\nu }
\end{eqnarray*}
where $E_x$ is the energy of the signal and $X(\nu)$ the Fourier transform
of $x(t)$. With this definition (and the one used in {\ty loctime}), the
Heisenberg-Gabor inequality writes $B\ T\geq 1$.\\

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8.5cm} c}
Name & Description & Default value\\
\hline
        {\ty x}     & signal\\
\hline  {\ty fm}    & averaged normalized frequency center\\
        {\ty B}     & frequency spreading\\
\hline
\end{tabular*}

\end{minipage}

\vspace*{1cm}

{\bf \large \sf Example}
\begin{verbatim}
         z=amgauss(160,80,50).*fmconst(160,0.2);
         [fm,B]=locfreq(z); [fm,B]
         ans =
               0.2000    0.0200
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
loctime.
\end{verbatim}
\end{minipage}


%!TEX root = ../notes.tex
\section{May 3, 2022}
\subsection{Minkowski, Lagrange, and Waring Walk Into a Bar\dots}
\subsubsection{Four Squares Theorem and Waring's Problem}
\begin{theorem}[Lagrange, 1770]
    Every nonnegative integer can be written as a sum of four square integers.
\end{theorem}
In the same year, Waring asserted, in his book, that
\begin{theorem}[Waring, 1770]
    For every integer $k\geq 2$, there is a $g(k)$ such that every nonnegative integer can be written as a sum of at most $g(k)$ $k$th powers.
\end{theorem}
He claimed that $g(3) = 9$ and $g(4) = 19$. We note that $23$ and $239$ require $9$ cubes, and $79$ requires $19$ fourth powers. $g(3) = 9$ is a result of Wieferich-Kemper (1909) and $g(4) = 19$ is a result of Balasubramanian, Dress, Deshouillers (1986). Hilbert proved this theorem in $1909$.

There is also a `capital $G$' version of this question, which is an asymptotic best bound of $g(k)$.

$4 \leq G(3) \leq 7$. This is currently unknown.

$G(4) = 16$, which we do know. So, your mileage may very much vary on these.

\subsubsection{Lattices and Minkowski's Theorem}
\begin{definition}[Lattice]
    Let $e_1, e_2, \dots, e_n$ be a set of basis vectors for $\RR^n$. Then the additive subgroup of $(\RR^n, +)$ generated by $e_1, e_2, \dots, e_n$ is called a \ul{lattice}.
\end{definition}
\begin{example}
    The most obvious lattice in $\RR^n$ is $\ZZ^n$, where we take $e_i$ to be the standard basis vectors.

    We can take $\alpha\ZZ^n$ by some $0\neq \alpha\in\RR$ is also an obvious lattice.

    We can also take things like $\frac{1}{2}\ZZ\times \ZZ\subseteq\RR^2$.
\end{example}

\begin{definition}[Fundamental Domain]
    If $L$ is a lattice generated by $e_1, e_2, \dots, e_n$ in $\RR^n$, then the \ul{fundamental domain} of $L$ is the set
    \[\left\{ \sum a_i e_i \mid a_i\in\RR, 0\leq a_i < 1 \right\}\]
\end{definition}
\begin{example}
    If we take $\ZZ^2\subseteq\RR^2$, then the fundamental domain is the square with corners $(0, 0)$ to $(1, 1)$, with some dotted lines.
\end{example}
\begin{definition}[Convex, Symmetric]
    A set $X\subseteq\RR^n$ is \ul{convex} if forall $x, y\in X$,
    \[\lambda x + (1 - \lambda)y \in X\]
    for all $0\leq \lambda \leq 1$.

    A set $X$ is \ul{symmetric} if $x\in X$ implies $-x\in X$.
\end{definition}
\begin{example}
    A triangle is convex but not symmetric. An ellipse about the origin is convex and symmetric. An annulus is symmetric but not convex.
    \begin{center}
        \includegraphics[width=0.9\textwidth]{images/shapes.png}
    \end{center}
\end{example}
\begin{theorem}[Minkowski, p.140 \cite{stewart2015algebraic}]
    Let $L$ be an $n$-dimensional lattice in $\RR^n$ with fundamental domain $T$, and let $X$ be a bounded symmetric convex subset of $\RR^n$. If
    \[\Vol(X) > 2^n\Vol(T)\]
    then $X$ contains a non-zero point of $L$.
\end{theorem}
\begin{example}
    If we have the $\ZZ^2$ example, if we have a `dotted' open square with area $4$, we just about not contain any nonzero points. If we add anything in, we'll have a nontrivial point.
    \begin{center}
        \includegraphics[width=0.4\textwidth]{images/square.png}
    \end{center}
\end{example}

\begin{proof}[Proof of 4 squares theorem using Minkowski's Theorem]
    We first prove this statement for primes, then extend to all positive integers. We first have
    \[2 = 1^2 + 1^2 + 0^2 + 0^2.\]
    So suppose we're trying to prove $p\in\ZZ_+$ is an odd prime.
    \begin{claim*}
        The equation $r^2 + s^2 + 1\equiv 0\pmod{p}$ has a solution $(r, s)\in\ZZ^2$.
    \end{claim*}
    \emph{Why?} Every element of $\ZZ/p\ZZ$ is a sum of $2$ squares\dots

    So let us select such an $r, s$. Consider the lattice $\Lambda\subseteq\ZZ^4$ that is given by
    \[\Lambda = A\ZZ^4,\]
    where
    \[A = \begin{pmatrix}
            p & 0 & r & s  \\
            0 & p & s & -r \\
            0 & 0 & 1 & 0  \\
            0 & 0 & 0 & 1
        \end{pmatrix}.\]
    We make an observation of the particular nature of the points in this lattice.

    If $\bvec{t}= (t_1, t_2, t_3, t_4)\in\ZZ^4$, and $\bvec{x} = (x_1, x_2, x_3, x_4)$ where $\bvec{x} = A\bvec{t}$, then
    \begin{align*}
        x_1^2 + x_2^2 + x_3^2 + x_4^2
         & = (pt_1 + rt_3 + st_4)^2 + (pt_2 + st_3 - rt_4)^2 + t_3^2 + t_4^2 \\
         & \equiv (rt_3 + st_4)^2 + (st_3 - rt_4)^2 + t_3^2 + t_4^2\pmod{p}  \\
         & = (1 + r^2 + s^2)(t_3^2 + t_4^2)\pmod{p}                          \\
         & = 0\pmod{p}
    \end{align*}
    Geometrically, the idea is if we can get the norm to be small enough, we know the sum is exactly $p$.

    Let's find an appropriate radius for our ball. A $4$-dimensional ball of radius $R$ has volume
    \[\pi^2R^4/2\]
    choose $R$ such that
    \begin{align*}
        16p^2 & < \frac{\pi^2 R^4}{2}\approx 4.93 R^4 \\
        4p    & < \approx 2.22 R^2                    \\
        2p    & < \approx 1.11 R^2
    \end{align*}
    The next condition is $R^2 < 2p$. So we want $R$ such that
    \[R^2 < 2p < \approx 1.11 R^2\]
    If we take $R^2 = 1.9p$ works.

    Let $X$ be the ball centered at the origin with radius $\sqrt{1.9p}$. Let $T$ be the fundamental domain for $\Lambda = A\ZZ^4$. Then we have $\Vol(T) = \det(A) = p^2$.

    Then $\Vol(X) > 2^n\Vol(T) = 16p^2$. Applying Minkowski's theorem, we conclude that $X$ \emph{must} contain some nonzero lattice point of $\Lambda$.

    If we let $\bvec{x} = (x_1, x_2, x_3, x_4)$ be this point. Then since $\bvec{x}\in X$, so
    \[x_1^2 + x_2^2 + x_3^2 + x_4^2 < R^2 < 2p.\]
    so $p\mid x_1^2 + x_2^2 + x_3^2 + x_4^2$ and $x_1^2 + x_2^2 + x_3^2 + x_4^2 < 2p$, and it's a nonzero point, so this forces
    \[x_1^2 + x_2^2 + x_3^2 + x_4^2 = p,\]
    which is as desired for primes.

    In the arbitrary case for arbitrary $n$, note that
    \begin{align*}
         & (a^2 + b^2 + c^2 + d^2)(A^2 + B^2 + c^2 + D^2)                \\
         & = (aA - bB - cC - dD)^2 + (aB + bA + cD - dC)^2 +             \\
         & \phantom{=}\ \ (aC - bD + cA + dB)^2 + (aD + bC - cB + dA)^2.
    \end{align*}
    so composite $n$ is also a sum of 2 squares.
\end{proof}

\begin{center}
    \emph{\dots and that's the course! }
\end{center}


\documentclass[aps,superscriptaddress,preprint,amsmath,floatfix,byrevtex]{article}
\usepackage[pdftex]{graphicx}
\usepackage{textcomp}
\usepackage{amsmath}
\usepackage{dcolumn}

\oddsidemargin 0.1in
\evensidemargin 0.1in
\textwidth 6.7in

\newcommand{\lrbar}{\bar{l}_{\rho}^{\mathrm{2D}}}

\begin{document}
\title{Derivation of the mean radial displacement, $\lrbar$, in 2 dimensions}
\author{Markus Dittrich}
\date{\today}

%\pacs{}
\maketitle

\section{Introduction}
In the following we derive an expression for the mean radial displacement, $\lrbar$,
for diffusing 2D molecules in analogy to the well known expression for $\bar{l}_r$
in three dimensions.

\section{Derivation}
Starting point is the solution of the diffusion equation in 2 dimensions
\begin{equation}
\frac{\partial c(\rho,t)}{\partial t} = D \nabla^2 c(\rho,t)
\end{equation}
which, for a point source of $M$ molecules released at the origin at time $t=0$
can be shown to be
\begin{equation}
c(\rho,t) = \frac{M}{4 \pi Dt} e^{- \frac{\rho^2}{4Dt}}
\end{equation}
(see also Eqs. 3.1, 3.2 in \cite{KERR2008}). Therefore, the probability
density for a molecule being in a radial shell of thickness $d\rho$ around the
origin is given by
\begin{equation}
p^{\mathrm{2D}} (\rho,t) = \frac{(2 \pi \rho) d\rho}{4\pi Dt}  e^{- \frac{\rho^2}{4Dt}}
\end{equation}
or, in normalized coordinates $\tilde{\rho} = \frac{\rho}{\lambda}$ with the
normalization constant $\lambda = \sqrt{4Dt}$
\begin{equation}
p^{\mathrm{2D}} (\tilde{\rho},t) = 2 e^{-\tilde{\rho}^2} (\tilde{\rho} d\tilde{\rho})
  \;\;\;\; .
\end{equation}
Hence, the mean radial displacement of a 2D molecule is given by
\begin{equation}
\lrbar = 2 \lambda \int_0^{\infty}  \tilde{\rho}^2 e^{- \tilde{\rho}^2} d\tilde{\rho}
\end{equation}
Since
\begin{eqnarray}
\int_0^{\infty} t^{2n} e^{-at^2} dt &=& \frac{\Gamma(n+\frac{1}{2})}{2a^{n+\frac{1}{2}}}
  \nonumber \\
\Rightarrow \int_0^{\infty}  \tilde{\rho}^2 e^{- \tilde{\rho}^2} d\tilde{\rho} &=&
  \frac{\Gamma(\frac{3}{2})}{2} = \frac{\sqrt{\pi}}{4}
\end{eqnarray}
and from this

\begin{equation}
\boxed{
\lrbar = 2 \lambda \frac{\sqrt{\pi}}{4} = \sqrt{\pi D t}
}
\end{equation}

\begin{thebibliography}{99}
\bibitem{KERR2008} R. Kerr, T.M. Bartol, B. Kaminsky, M. Dittrich, J.C.J. Chang, S. Baden,
T.J. Sejnowski, and J.R. Stiles, Fast Monte Carlo Simulation Methods for Biological
Reaction-Diffusion Systems in Solution and on Surfaces (2008),
SIAM J. Sci. Comput., 30:3126-3149.
\end{thebibliography}

\end{document}


\section{Lines}
In this section we will discuss how to represent lines in 3D, and some related problems, including finding the line at the intersection of two planes, and finding the intersection point of a plane and a line.

\subsection{Line representation}
Unlike 2D lines and planes, 3D lines don't have a nice representation as a single equation on coordinates like $ax+by=c$ or $ax+by+cz=d$. We could represent them as the intersection of two planes, like
\begin{align*}
a_1x+b_1y+c_1z &= d_1 \\
a_2x+b_2y+c_2z &= d_2
\end{align*}
but this representation is not very convenient to work with, and there are many pairs planes we could choose.

\centerFig{line3d-0}

Instead, we will work with a parametric representation: we take a point $O$ on the line and a vector $\vv{d}$ parallel to the line and say that the points belonging to the line are all points
\[P = O + k \vv{d}\]
where $k$ is a real parameter.

\centerFig{line3d-1}

Note that here, $\vv{d}$ plays the same role as $\vv{v}$ did for 2D lines (see section~\ref{ss:line-rep}).

If we are given two points $P,Q$ on the line, we can set $O=P$ and $\vv{d}=\vv{PQ}$, so we implement the structure like this:
\begin{lstlisting}
struct line3d {
    p3 d, o;
    // From two points P, Q
    line3d(p3 p, p3 q) : d(q-p), o(p) {}
    // From two planes p1, p2 (requires T = double)
    line3d(plane p1, plane p2); // will be implemented later

    // Will be defined later:
    // - these work with T = int
    double sqDist(p3 p);
    double dist(p3 p);
    bool cmpProj(p3 p, p3 q);
    // - these require T = double
    p3 proj(p3 p);
    p3 refl(p3 p);
    p3 inter(plane p) {return o - d*p.side(o)/(d|p.n);}
};
\end{lstlisting}

\subsection{Distance from a line}
A point $P$ is on a line $l$ described by $O,\vv{d}$ if and only if $\vv{OP}$ is parallel to $\vv{d}$. This is the case when $\crossv{d}{OP} = \vv{0}$.

\centerFig{line3d-2}

More generally, this product $\crossv{d}{OP}$ also gives us information about the distance to the line: the distance from $l$ to $P$ is equal to
\[\frac{\big\|\crossv{d}{OP}\big\|}{\big\|\vv{d}\big\|}\]
%Note that this expression also works for 2D lines; indeed, the norm of $\crossv{d}{OP}$ is equivalent to the
\begin{lstlisting}
double sqDist(p3 p) {return sq(d*(p-o))/sq(d);}
double dist(p3 p) {return sqrt(sqDist(p));}
\end{lstlisting}

\subsection{Sorting along a line}\label{ss:sort-line3d}
Just like we did with 2D lines in section~\ref{ss:sort-line}, we can sort points according to their position along a line $l$. To find out if a point $P$ should come before another point $Q$, we simply need to check whether
\[\vv{d}\cdot P < \vv{d}\cdot Q\]
so we can use the following comparator:
\begin{lstlisting}
bool cmpProj(p3 p, p3 q) {return (d|p) < (d|q);}
\end{lstlisting}

\subsection{Orthogonal projection and reflection}
Let's say we want to project $P$ on a line $l$, that is find the closest point to $P$ on $l$.
Our usual approach to projecting things, which is to move $P$ perpendicularly until it touches $l$, doesn't work as well here: indeed, there are many possible directions that are perpendicular to $l$.

\centerFig{line3d-3}

Instead we will start from $O$ and move along the line until we reach the projection of $P$. We have seen in the previous section that taking the dot product with $\vv{d}$ tells us how far some point is along $l$. In fact, if we compute $\dotv{d}{OP}$, this tells us the (signed) distance from $O$ to the projection of $P$, multiplied by $\normv{d}$. So we can find the projection this way:
\begin{lstlisting}
p3 proj(p3 p) {return o + d*(d|(p-o))/sq(d);}
\end{lstlisting}

Once we've found the projection $P'$, we can find the reflection $P''$ easily, since it is twice as far in the same direction: we have $P'' = P' + \vv{PP'}$, which becomes $2P'-P$ if we allow vector operations on points.
\begin{lstlisting}
p3 refl(p3 p) {return proj(p)*2 - p;}
\end{lstlisting}

\subsection{Plane-line intersection}\label{ss:plane-line}
Let's say we have a plane $\Pi$, represented by vector $\vv{n}$ and real $d$, and a line $l$, represented by point $O$ and $\vv{d}$. To find an intersection between them, we need to find a point $O+k\vv{d}$ that lies on $\Pi$, that is, such that
\[\vv{n}\cdot\left(O+k\vv{d}\right) = d\]

\centerFig{line3d-4}

Solving for $k$, we find
\[k = \frac{d - \vv{n}\cdot O}{\dotv{n}{d}} = \frac{-\side_{\Pi}(O)}{\dotv{n}{d}}\]
which can be implemented directly:
\begin{lstlisting}
p3 inter(plane p) {return o - d*p.side(o)/(p.n|d);}
\end{lstlisting}

Note that this is undefined when $\dotv{n}{d}=0$, that is, when $\Pi$ and $l$ are parallel.\footnote{We take a closer look at criteria for parallelism and perpendicularity in section~\ref{s:angles}.}

\subsection{Plane-plane intersection}
If we are given two non-parallel planes $\Pi_1$ and $\Pi_2$ (defined by $\vv{n_1}$, $d_1$ and $\vv{n_2}$, $d_2$), how can we find their common line?

First we need to find its direction $\vv{d}$. Clearly, the direction needs to be parallel to both planes, so it must be perpendicular to both $\vv{n_1}$ and $\vv{n_2}$. Thus we take $\vv{d} = \crossv{n_1}{n_2}$.

Then we need to find an arbitrary point $O$ that is on both planes. Here, we will actually compute the closest such point to the origin. It is given by
\[O = \frac{\big(d_1\vv{n_2} - d_2\vv{n_1}\big)\times\vv{d}}{\big\|\vv{d}\big\|^2}\]

Let's analyze this expression. It is the sum of two vectors,
\begin{align*}
\vv{v_1} &= \frac{d_1}{\big\|\vv{d}\big\|^2}\left(\crossv{n_2}{d}\right) \\
\vv{v_2} &= -\frac{d_2}{\big\|\vv{d}\big\|^2}\left(\crossv{n_1}{d}\right)
\end{align*}
Because $\vv{v_1}$ is perpendicular to $\vv{n_2}$, it is parallel to $\Pi_2$, while $\vv{v_2}$ is perpendicular to $\vv{n_1}$ and thus parallel to $\Pi_1$. So we can see $\vv{v_1}$ as the vector that leads from the origin to $\Pi_1$ while staying parallel to $\Pi_2$, and $\vv{v_2}$ as the vector that leads from the origin to $\Pi_2$ while staying parallel to $\Pi_1$.

\centerFig{line3d-5}

Let's verify that $O$ is on $\Pi_1$, that is, $\vv{n_1}\cdot O = \vv{n_1}\cdot(\vv{v_1} + \vv{v_2}) = d_1$. Since $\vv{v_2}$ is perpendicular to $\vv{n_1}$, clearly $\dotv{n_1}{v_2} = 0$. What remains is
\begin{align*}
\dotv{n_1}{v_1}
&= \frac{d_1}{\big\|\vv{d}\big\|^2}\left(\crossv{n_2}{d}\right)\cdot\vv{n_1} \\
&= \frac{d_1}{\big\|\vv{d}\big\|^2}\Big(\crossv{n_1}{n_2}\Big)\cdot\vv{d} \\
&= \frac{d_1}{\big\|\vv{d}\big\|^2}\left(\dotv{d}{d}\right) \\
&= d_1
\end{align*}
where between the first two lines, we used the fact that the mixed product is conserved if we swap its arguments in a circular way (see exercise~\ref{ex:mixed-props}).

We prove similarly that $O$ is on $\Pi_2$. Finally we note that both $\vv{v_1}$ and $\vv{v_2}$ are perpendicular to $\vv{d}$, so the vector from the origin to $O$ is perpencular to $\vv{d}$, the direction of the line. Therefore it must necessarily arrive on the line at the closest point to the origin.

We can implement this as the following constructor:
\begin{lstlisting}
line3d(plane p1, plane p2) {
    d = p1.n*p2.n;
    o = (p2.n*p1.d - p1.n*p2.d)*d/sq(d);
}
\end{lstlisting}

\subsection{Line-line distance and nearest points}
Consider line $l_1$ defined by $O_1 + k\vv{d_1}$ and line $l_2$ defined by $O_2 + k\vv{d_2}$.
If they are parallel, the distance between them is easy to find: just find the distance from $l_1$ to $O_2$.
Otherwise, they are either intersecting or skew, in which case the question is a bit more complex.

\centerFig{line3d-6}

Let's call $C_1$ the point of $l_1$ that is closest to $l_2$, and $C_2$ the point on $l_2$ that is closest to $l_1$.
Direction $\vv{C_1C_2}$ should be perpendicular to both $l_1$ and $l_2$.
Indeed, if it were not, it would be possible to get a smaller distance by moving either $C_1$ or $C_2$.
So $\vv{C_1C_2}$ is parallel to $\vv{n}=\crossv{d_1}{d_2}$.

\centerFig{line3d-7}

Because of this, we could compute the distance as
\[|C_1C_2| = \frac{\big|\dotv{C_1C_2}{n}\big|}{\normv{n}}\]
We don't know $C_1$ or $C_2$ yet, but since the dot product doesn't change when one of the vectors moves perpendicular to the other, we can move $C_1$ to $O_1$ and $C_2$ to $O_2$, so that we get
\[\frac{\big|\dotv{C_1C_2}{n}\big|}{\normv{n}} = \frac{\big|\dotv{O_1C_2}{n}\big|}{\normv{n}} = \frac{\big|\dotv{O_1O_2}{n}\big|}{\normv{n}}\]
which gives the following implementation:
\begin{lstlisting}
double dist(line3d l1, line3d l2) {
    p3 n = l1.d*l2.d;
    if (n == zero) // parallel
        return l1.dist(l2.o);
    return abs((l2.o-l1.o)|n)/abs(n);
}
\end{lstlisting}

Now, how can we find $C_1$ and $C_2$ themselves? Let's call $\Pi_2$ the plane that contains $l_2$ and is parallel to $n_2$; its normal is $\vv{n_2} = \crossv{d_2}{n}$. Then $C_1$ is the intersection between that plane and line $l_1$, so we can use the formula obtained in section~\ref{ss:plane-line} to find it:
\[C_1 = O_1 + \frac{\dotv{O_1O_2}{n_2}}{\dotv{d_1}{n_2}}\]

\centerFig{line3d-8}

This is implemented in a straightforward way. If we want $C_2$ instead we just have to swap $l_1$ and $l_2$.
\begin{lstlisting}
p3 closestOnL1(line3d l1, line3d l2) {
    p3 n2 = l2.d*(l1.d*l2.d);
    return l1.o + l1.d*((l2.o-l1.o)|n2)/(l1.d|n2);
}
\end{lstlisting}


\documentclass{article}

\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}

\begin{document}

\section*{Limits}

\[\lim_{x \to 0^{+}} \frac{1}{x} = \infty\]

\[\lim_{x \to 0^{-}} \frac{1}{x} = -\infty\]

\[\lim_{x \to \pm\infty} \frac{1}{x} = 0\]

\end{document}


\chapter{Conclusion and Future Works}
\label{chap_conc}
This thesis presents a study of deep multi-layer kernels on supervised and unsupervised learning algorithms. In the supervised learning settings, multi-layer kernels are studied on structured output prediction problems and discriminant analysis methods. In the unsupervised learning settings, multi-layer kernels are used for feature learning task. In particular, we used MKMs and MKL for building a feature learning framework with kernel machines whose architecture and training is equivalent to existing deep learning algorithms.

In the unsupervised feature learning model with MKMs we experimented with single kernel (arc-cosine kernel) and mixed kernels. In some cases, the mixed kernel version was found to be superior than their single kernel counterparts. Features learned by the MKMs were visualized with tSNE algorithm to understand the separability of different classes.

Instead of using single kernel in each layer, a convex combination of multiple kernels were tried out using an unsupervised MKL formulation. The training of this model was done in a greedy layer-by-layer fashion. Experimental results indicates that, this ML-MKL model is superior to single-kernel MKMs. Though we had conducted empirical study on object recognition datasets like \textit{cifar10} and text classification datasets like \textit{20-newsgroups}, the results were not very impressive. In the case of \textit{20-newsgroups} classification, multiple layers might be making the similarity information in the kernel matrix more noisy, since the performance was degrading after each layer is added.  The empirical studies on supervised learning settings also gave fruitful results. The KFDA algorithm with multi-layer kernels showed competitive performance with state of the art deep learning algorithms. Structured output learning algorithms also works well when multi-layer kernels were used.

\section{Future Works}
The empirical study conducted in this thesis is giving many insights, from which we can list out some potential future directions to explore.

The complexity of the model in terms of the number of layers and number of kernels in each layer are set by using  cross-validation techniques in our experiments. The optimal number of layers and optimal number of kernels in each layer characterizes the structure of ML-MKL. A thoretical study on the optimal structure of ML-MKL model is a future extension for this work.

The training of ML-MKL framework in the current implementation is performed in a greedy layer-by-layer fashion, by iteratively adding layers. This can be thought of as unsupervised pre-training done in typical deep learning algorithms. Then we can devise a supervised fine-tuning mechanism for this model, by taking class labels into account, which may change either the kernel parameters or weights.

The deep learning algorithms are found to be particularly effective for learning complex decision functions. Structured output prediction tasks like parse tree prediction, protein segment alignment prediction etc. need very complex decision functions. Thus, the deep multi-layer kernel machines can be studied in such domains.

The results from KFDA with multi-layer kernels is having some interesting aspects. The algorithm is performing well, either when using a highly non-linear arc-cosine kernel or when very large number of layers are used in the kernel function. The obscurity behind the need of such complex kernels for
discriminant analysis can be explored further.


\documentclass[11 pt]{scrartcl}
\usepackage[header, margin, koma]{tyler}

\newcommand{\hwtitle}{Hard Counting Problems}

\newif\ifproblemsol
\problemsolfalse

\pagestyle{fancy}
\fancyhf{}
\fancyhead[l]{\hwtitle{}}
\fancyhead[r]{Tyler Zhu}
\cfoot{\thepage}

\begin{document}
\title{\Large \hwtitle{}}
\author{\large Tyler Zhu}
\date{\large\today}

\maketitle

These are some pretty arbitrary, hard counting problems. They are more so for testing your general ability to see connections in counting well than how well you bookkeep. Safe travels.

\section{Warm up}
\begin{problem}
    How many cubic polynomials $f(x)$ with positive integer coefficients are there such that $f(1) = 9$?
\end{problem}
\ifproblemsol
\begin{proof}[Solution]
    Let $f(x) = ax^3+bx^2+cx+d$. The only condition on $f(x)$ is that $f(1) = 9$, which means $a + b + c + d = 9$. Since $a,b,c,d$ are positive integers, there are $\binom{8}{3} = 56$ such polynomials.
\end{proof}
\fi

\section{Assorted Candies}
\begin{problem}
    Prove $\sum_{k=0}^n \binom{n}{k} 2^k = 3^n$ with a combinatorial argument.
\end{problem}
\ifproblemsol
\begin{proof}[Solution]
    Suppose we're trying to create a $n$-bead string with 3 colors; red, blue and green. We can either have 3 chocies for each spot for a total of $3^n$, or pick the $n-k$ spots which will be red, and then fill in the remaining $k$ spots with either blue or green for a total of $\binom{n}{k}2^k$ for each choice of $k$.
\end{proof}
\fi

\begin{problem}
    Let $a_1, a_2, \dots, a_n$ be a sequence of arbitrary natural numbers. Define $b_k$ to be the number of elements $a_i$ for which $a_i \geq k$. Prove that $a_1 + a_2 + \dots + a_n = b_1 + b_2 + \cdots$.
\end{problem}
\ifproblemsol
\begin{proof}[Solution]
    The idea is to double count. Drawing a picture is the best way to see this: for each $a_i$, draw $a_i$ circles vertically. Then the LHS is counting the number of circles going vertically, while the RHS is counting them horizontally.

    Formally, one way to count is simply to sum the $a_i$'s. Another way to count uses the fact that they are natural numbers. All the $b_i$ start at 0. Then, for any given $a_i$, $b_1$ through $b_{a_i}$ will have all their values increased by $1$, increasing the total on the RHS by $a_i$. Hence the total contributions of all $a_i$ to the RHS is just $a_1 + \dots + a_n$.
\end{proof}
\fi

\begin{problem}
    How many ways are there to insert $+$’s between the digits of 111111111111111 (fifteen 1’s) so that the result will be a multiple of 30?
\end{problem}
\ifproblemsol
\begin{proof}[Solution]
    No matter how many $+$'s we insert, the result will always be a multiple of $3$ since there are fifteen 1's. For it to be a multiple of 10, we need exactly 10 numbers, which means we're adding 9 $+$'s. There are 14 gaps, so our answer is $\binom{14}{9}$.
\end{proof}
\fi

\begin{problem}
    Compute
    \[ \sum_{n_{60}=0}^2\sum_{n_{59}=0}^{n_{60}}\dots \sum_{n_2=0}^{n_3} \sum_{n_1=0}^{n_2}\sum_{n_0=0}^{n_1} 1.\]
\end{problem}
\ifproblemsol
\begin{proof}[Solution]
    Another way of phrasing the problem is to find the number of solutions to $0\leq n_0 \leq n_1 \leq n_2 \leq \dots \leq n_{60} \leq 2$. This corresponds to the number of right-up walks on a $61\times 2$ grid from the bottom left to the top right, which we all know to be $\binom{63}{2}$.

    Alternatively, notice that every solution is of the form $(0,\dots, 0, 1, \dots, 1, 2, \dots, 2)$, so we only need to specify the number of 0s, 1s, and 2s. This is equivalent to solving the equation $x+y+z = 61$ for nonnegative $x,y,z$, which (by stars and bars) has $\binom{63}{2}$ solutions.
\end{proof}
\fi

\begin{problem}
    There’s a new (virtual) game show featuring $N$ people where a few lucky contestants get to compete for the ultimate prize: a roll of toilet paper. The game works as follows: everyone lines up in front of a jar of ping pong balls numbered 1 through 100 and one-by-one randomly select a ball until everyone has one. Then the winning number is announced, and anyone with the winning number wins the prize.
    \alphanum
        \ii Suppose the winning number is 42. What’s the probability that if $N=3$, then the third person wins the game?
        \ii If $N = 100$, what’s the probability that the third person wins the game now?
        \ii You and a friend are watching the game (for $N = 100$) and after all of the balls have been drawn, you both decide to bet on the results. Your friend picks contestant 42, thinking they must have the winning number, but before you can pick a contestant, 98 of them groan in realization that they have losing numbers, which leaves you with no choice but to bet on contestant 20. What’s the probability that you win the bet?
    \enumend
\end{problem}
\ifproblemsol
\begin{proof}[Solution]
\alphanum
    \ii By symmetry, $\frac{1}{100}$. One can also compute it out to be $\frac{99}{100}\cdot \frac{98}{99}\frac{1}{98} = \frac{1}{100}$.
    \ii As above, by symmetry, $\frac{1}{100}$. Computation also works.
    \ii This is just the Monty Hall problem in disguise, where you choose to switch! Once the other 98 doors (contestant's numbers) were revealed, all of the $\frac{99}{100}$ probability went into contest 20's chances, so you have a $\frac{99}{100}$ probability of winning the bet.
\enumend
\end{proof}
\fi

\section{Dessert}
These aren't actually relevant to course material; please don't do them unless you really want to!

\begin{exercise}
    Let $(a_1,a_2, \dots, a_{12})$ be a permutation of $(1,2,\dots,12)$ for which
    \[ a_1>a_2>a_3>a_4>a_5>a_6 \text{ and } a_6<a_7<a_8<a_9<a_{10}<a_{11}<a_{12}. \]
    An example of such a permutation is $(6,5,4,3,2,1,7,8,9,10,11,12)$. Find the number of such permutations.
\end{exercise}
\ifproblemsol
\begin{proof}[Solution]
    First, $a_6 = 1$ in every permutation, as it must be the smallest number. We can build any such permutation by picking the five numbers that go on the left and filling them in decreasing order. Then the remaining numbers must fill out the other six on the right, so there are $\binom{11}{5}$ such ways to pick them.
\end{proof}
\fi

\begin{exercise}
    Let $N$ denote the number of $7$-tuples of sets $S_1, S_2, \dots,S_7$, not necessarily distinct, for which
    \[ S_1 \subseteq S_2 \subseteq \cdots \subseteq S_7 \subseteq \{1,2,3,4,5,6,7\}.\]
    Find $N$.
\end{exercise}
\ifproblemsol
\begin{proof}[Solution]
    For each number, if it appears in a set $S_i$, then it appears in all sets $S_j$ where $j \geq i$. So for each number, we have 7 ways to pick the first set it starts appearing in, or none at all, for a total of 8 choices. Thus $N = 8^7$.
\end{proof}
\fi

\begin{exercise}
    In a shooting match a marksman must break eight targets arranged in three hanging columns of 3, 3 and 2 targets respectively. Whenever a target is broken, it must be the lowest unbroken target in its column. In how many different orders can the eight targets be broken?
\end{exercise}
\ifproblemsol
\begin{proof}[Solution]
    Consider a string such as $AAABBBCC$ as a set of instructions, where the appearance of an $A$ means shoot at the lowest hanging target in the first column, $B$ means shoot in the second column, and $C$ means shoot in the third column. Then every permutation of this string gives a new way to break the eight targets, for a total of $\frac{8!}{3!3!2!}$.
\end{proof}
\fi

\end{document}


This document contains a list of steps for approximating uncertainties of measured magnitudes in ASP 3231 project.

\section{Find uncertainty of star's flux}

\begin{itemize}
  \item Measure twenty background fluxes by choosing apertures in the regions without stars.
  \item Note that the radius of the apertures for measuring fluxes should be equal to the radius used to measure the fluxes of the stars in that image.
  \item Calculate standard deviation of these fluxes. We will use it to approximate the uncertainty of the flux u(f) from the stars for that image (I don't know why we can do this, but ok).
\end{itemize}

\section{Calculate uncertainty of reference star’s magnitude in photometric image}

\begin{itemize}
  \item The magnitude $m_2$ of a reference star is calculated from photometric calibration equation:
  \begin{equation}
  m_2 = z - 2.5 \log_{10}(f/t) + k \ A,
  \end{equation}
  where z, and k are constants, $f$ is flux of a reference star measured in photometric image, $A$ is the average air mass index of the stacked images.
  \item Next we calculate $u(m_2)$ (uncertainty of $m_2$) by propagating uncertainties  $u(f)$ and $u(A)$. The uncertainty of the air mass $u(A)$ is standard deviation of air mass indices of the stacked images.

  \item Assuming $f$ and $A$ are independent, we calculate uncertainty of m2 using first order approximation:

  \begin{equation}
    u(m_2) = \sqrt{ \left[ \frac{2.5}{f} \frac{1}{\ln(10)} u(f) \right]^2 + \left[ a \ u(A)\right]^2}.
  \end{equation}
\end{itemize}

\section{Calculate uncertainty of star’s magnitude in non-photometric image for one reference star}

\begin{itemize}
  \item The magnitude $m_1$ of a star in non-photometric image is calculated from equation
  \begin{equation}
  m_1 = m_2 - 2.5 \log_{10} \left( \frac{f_1}{f_2} \right),
  \label{eq_star_magnitude}
  \end{equation}
  where $m_2$ is magnitude of a reference star, $f_1$ is the flux of the star we want, and $f_2$ is flux of a reference star. Both fluxes $f_1$ and $f_2$ are measured in non-photometric image.

  \item We calculate the uncertainty of $m_1$, assuming all variables in \autoref{eq_star_magnitude} are independent, and thus using first order approximation
  \begin{equation}
  u(m_1) = \sqrt{[u(m_2)]^2 + \left[ 2.5 \frac{u(f)}{\ln(10)} \right]^2 \left( \frac{1}{f_1^2} + \frac{1}{f_2^2} \right)},
  \label{eq_uncertainty_of_magnitude}
  \end{equation}
  where u(f) is the uncertainty of flux measurement in non-photometric image for the star we want.

\end{itemize}

\section{Calculate final uncertainty of star's magnitude}

\begin{itemize}
   \item We use multiple reference stars to calculate a star's magnitudes using \autoref{eq_star_magnitude}. So if we have three reference stars $a$, $b$ and $c$, we will calculate three magnitudes for each star: $m_a, m_b, m_c$.

  \item We calculate the final magnitude of the star $m$ by taking the middle magnitude of $m_a, m_b, m_c$.

  \item For the uncertainty of the magnitude we take  the uncertainty of the middle magnitude. Not that we do not calculate the average of $m_a, m_b, m_c$, because if we do so and use the simple error propagation formula we will get:
  \[
  u(m) = \frac{\sqrt{u(m_a)^2 + u(m_b)^2 + u(m_c)^2}}{3}.
  \]

  This approach assumes $m_a, m_b, m_c$ are independent measurements. But this $m_a, m_b, m_c$  are not independent, since in each measurement the star's flux $f_1$ is the same (\autoref{eq_star_magnitude}), and $m_2$ and $f_2$ are also the same for all stars. Therefore, if we did that, our uncertainty would be significantly underestimated.

  \item We report star's magnitude $m$, its uncertainty $u(m)$ and publish in The International Journal Of Tourism Sciences.

\end{itemize}