\documentclass[letterpaper,11pt]{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[hmargin=1.25in,vmargin=1in]{geometry}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{lmodern}
\usepackage{microtype}
\usepackage{pdflscape}
\usepackage{subcaption}

\title{Coursework 3: STAT 570}
\author{Philip Pham}
\date{\today}

\begin{document}
\maketitle

\begin{enumerate}
\item Consider the Poisson-gamma random effects model given by
  \begin{align}
    Y_i \mid \mu_i, \theta_i
    &\sim \operatorname{Poisson}\left(\mu_i\theta_i\right) \\
    \theta_i
    &\sim \operatorname{Gamma}\left(b, b\right),
  \end{align}
  which leads to a negative binomial marginal model with the variance a
  quadratic function of the mean. Design a simulation study, along the lines of
  that which produced Table 2.3 in the book (overdispersed Poisson example) to
  investigate the efficiency and robustness under
  \begin{itemize}
  \item a Poisson model;
  \item quasi-likelihood with $\mathbb{E}\left[Y\right] = \mu$ and
    $\operatorname{Var}\left(Y\right) = \alpha\mu$; and
  \item sandwich estimation.
  \end{itemize}

  Use a log-linear model
  \begin{equation}
    \log \mu_i = \beta_0 + \beta_1x_i,
  \end{equation}
  with $x_i \sim_\mathrm{iid} \mathcal{N}\left(0, 1\right)$ for
  $i = 1,2,\ldots,n$, and $\beta_0 = -2$ and $\beta_1 = \log 2$.

  Simulate for:
  \begin{itemize}
  \item $b \in \left\{0.2,1,10,1000\right\}$.
  \item $n \in \left\{10, 20, 50, 100, 250\right\}$.
  \end{itemize}

  Summarize what your take away message is after carrying out these simulations.

  \begin{description}
  \item[Solution:] Note that
    \begin{align}
      \mathbb{P}\left(
      Y_i = y \mid \mu_i
      \right)
      &= \int_0^\infty\mathbb{P}\left(
      Y_i = y \mid \mu_i, \theta_i = \theta
      \right)\mathbb{P}\left(
      \theta_i = \theta
      \mid b
        \right)
        \,\mathrm{d}\theta \nonumber\\
      &= \int_0^\infty
        \left(
        \frac{\left(\mu_i\theta\right)^y}{y!}\exp\left(-\mu_i\theta\right)
        \right)
        \left(
        \frac{b^b}{\Gamma(b)}\theta^{b-1}
        \exp\left(-b\theta\right)
        \right)
        \,\mathrm{d}\theta \nonumber\\
      &= \frac{\mu_i^y b^b}{y!\Gamma(b)}
        \int_0^\infty
        \theta^{b + y - 1}\exp\left(-\theta(b + \mu_i)\right)
        \,\mathrm{d}\theta \nonumber\\
      &= \frac{\Gamma(y + b)}{y!\Gamma(b)}
        \frac{\mu_i^y b^b}{\left(\mu_i + b\right)^{b + y}}
        = \frac{\Gamma(y + b)}{y!\Gamma(b)}
        \left(\frac{b}{\mu_i + b}\right)^b
        \left(\frac{\mu_i}{\mu_i + b}\right)^y
        \nonumber\\
      &\sim \operatorname{NegativeBinomial}\left(
        b,
        \frac{\mu_i}{\mu_i + b}
        \right).
        \label{eqn:p1_negative_binomial}
    \end{align}

    By properties of the negative binomial distribution, we have that
    \begin{align}
      \mathbb{E}\left[
      Y_i
      \mid x_i
      \right]
      &= \mu_i = \exp\left(\beta_0 + \beta_1x_i\right) \nonumber\\
      \operatorname{Var}\left(Y_i \mid x_i\right)
      &= \mu_i\left(
        1 + \frac{\mu_i}{b}
        \right).
      \label{eqn:p1_y_mean_variance}
    \end{align}
    Thus, variance has a quadratic dependence on the mean, and smaller values of
    $b$ correspond to more dispersion.

    \subsection*{Poisson Model}

    In the Poisson model, we assume that
    $\operatorname{Var}\left(Y_i \mid x_i\right) = \mu_i$, e.g.
    $b \rightarrow \infty$, so we neglect the overdispersion parameter.

    In this case, the log-likelihood function is
    \begin{equation}
      l\left(\beta\right) =
      \sum_{i=1}^n\left[
        y_i\left(\beta_0 + \beta_1x_i\right) -
        \exp\left(\beta_0 + \beta_1x_i\right) -
        \sum_{k=1}^{y_i}\log k
      \right],
    \end{equation}
    which gives us the score function
    \begin{equation}
      S\left(\beta\right) = \sum_{i=1}^n\begin{pmatrix}
        y_i - \exp\left(\beta_0 + \beta_1x_i\right) \\
        x_iy_i - x_i\exp\left(\beta_0 + \beta_1x_i\right)
      \end{pmatrix}.
      \label{eqn:p1_score_function}
    \end{equation}
    We can estimate $\beta$ by solving for
    $S\left(\hat{\beta}\right) = \mathbf{0}$, numerically.

    We can estimate the variance of the estimates from the Fisher information,
    \begin{align}
      \operatorname{Var}\left(
      \hat{\beta}
      \right)
      &\approx
        I_n\left(\hat{\beta}\right)^{-1} \label{eqn:p1_beta_hat_variance}\\
      &= \left(
        \sum_{i=1}^n\begin{pmatrix}
          \exp\left(\hat{\beta}_0 + \hat{\beta}_1x_i\right) &
          x_i\exp\left(\hat{\beta}_0 + \hat{\beta}_1x_i\right) \\
          x_i\exp\left(\hat{\beta}_0 + \hat{\beta}_1x_i\right) &
          x_i^2\exp\left(\hat{\beta}_0 + \hat{\beta}_1x_i\right)
        \end{pmatrix} \right)^{-1}  \nonumber\\
      &= \frac{1}{
        \left(\sum_{i=1}^n \hat{\mu}_i\right)\left(\sum_{i=1}^n x_i^2\hat{\mu}_i\right)
        - \left(\sum_{i=1}^n x_i\hat{\mu}_i\right)^2}
        \begin{pmatrix}
          \sum_{i=1}^n x_i^2\hat{\mu}_i &
          -\sum_{i=1}^n x_i\hat{\mu}_i \\
          -\sum_{i=1}^n x_i\hat{\mu}_i &
          \sum_{i=1}^n \hat{\mu}_i
        \end{pmatrix}, \nonumber
    \end{align}
    where $\hat{\mu}_i = \exp\left(\hat{\beta}_0 + \hat{\beta}_1x_i\right)$.

    \subsection*{Quasi-likelihood}

    In a quasi-likelihood model, we specify the mean and variance as
    \begin{align}
      \mathbb{E}\left[
      Y_i
      \mid x_i
      \right]
      &= \mu_i = \exp\left(\beta_0 + \beta_1x_i\right) \nonumber\\
      \operatorname{Var}\left(Y_i \mid x_i\right)
      &= \alpha \mu_i
      \label{eqn:p1_y_quasi_mean_variance}
    \end{align}
    From Equation \ref{eqn:p1_y_mean_variance}, we see that this is not quite
    correct, still, but it is closer to the real model than the Poisson model.

    Then, by Equation 2.30 of Wakefield's \emph{Bayesian and Frequentist
      Regression Methods} our estimating function is
    \begin{align}
      U\left(\beta\right)
      &= D^\intercal V^{-1}\left(y - \mu\right)/\alpha \nonumber\\
      &= \sum_{i=1}^n\begin{pmatrix}
        \exp\left(\beta_0 + \beta_1x_i\right) \\
        x_i\exp\left(\beta_0 + \beta_1x_i\right)
      \end{pmatrix}
      \frac{y_i - \exp\left(\beta_0 + \beta_1x_i\right)}{\alpha\exp\left(\beta_0 + \beta_1x_i\right)}
      \nonumber\\
      &= \frac{1}{\alpha}\sum_{i=1}^n\begin{pmatrix}
        y_i - \exp\left(\beta_0 + \beta_1x_i\right) \\
        x_iy_i - x_i\exp\left(\beta_0 + \beta_1x_i\right)
      \end{pmatrix} = \frac{1}{\alpha}S\left(\beta\right)
      \label{eqn:p1_quasi_likelihood_score_function}
    \end{align}
    from Equation \ref{eqn:p1_score_function}. Thus, the maximum
    quasi-likelihood estimate will be the same as the maximum likelihood
    estimate from the Poisson model.

    Having solved for $\hat{\beta}$, we have
    \begin{equation}
      \hat{\mu} = \exp\left(\hat{\beta}_0 + \hat{\beta}_1x_i\right).
      \label{eqn:p1_mu_hat}
    \end{equation}

    by Equation 2.31 of Wakefield's \emph{Bayesian and Frequentist Regression
      Methods}, we can then compute
    \begin{equation}
      \hat{\alpha}_n
      = \frac{1}{n-2}\sum_{i=1}^n \frac{\left(y_i - \hat{\mu}_i\right)^2}{\hat{\mu}_i}
      \label{eqn:p1_alpha_hat}
    \end{equation}

    Then, the variance of our estimates is
    \begin{align}
      \operatorname{Var}\left(
      \hat{\beta}
      \right)
      &\approx
      \hat{\alpha}_n\left(\hat{D}^\intercal \hat{V}^{-1} \hat{D}\right)^{-1} \nonumber\\
      &= \hat{\alpha}_n\left(\sum_{i=1}^n
        \begin{pmatrix}
          \hat{\mu_i} & x_i\hat{\mu_i} \\
          x_i\hat{\mu_i} & x_i^2\hat{\mu_i}.
        \end{pmatrix}\right)^{-1}
        \nonumber\\
      &= \hat{\alpha}_nI_n\left(\hat{\beta}\right)^{-1}
        \label{eqn:p1_beta_hat_quasi_variance}
    \end{align}
    from Equation \ref{eqn:p1_beta_hat_variance}.

    \subsection*{Sandwich Estimation}

    In sandwich estimation, we only need to specify an estimating function
    $G\left(\beta\right)$. Then, we can apply Equation 2.43 of Wakefield's
    \emph{Bayesian and Frequentist Regression Methods} to compute the variance
    of our estimates:
    \begin{align}
      \operatorname{Var}\left(\hat{\beta}\right)
      &= \frac{1}{n}\hat{A}^{-1}\hat{B}\left(\hat{A}^{-1}\right)^\intercal \nonumber\\
      \hat{A}
      &= -\frac{1}{n}\sum_{i=1}^n
        \frac{\partial}{\partial\beta}G\left(\hat{\beta}, X_i, Y_i\right) \nonumber\\
      \hat{B}
      &= \frac{1}{n}\sum_{i=1}^n
        G\left(\hat{\beta}, X_i, Y_i\right)G\left(\hat{\beta}, X_i, Y_i\right)^\intercal.
        \nonumber
    \end{align}

    We can reuse the score function from the quasi-likelihood estimate in
    Equation \ref{eqn:p1_quasi_likelihood_score_function} without $\alpha$, so
    \begin{equation}
      G\left(\hat{\beta}, X_i, Y_i\right)
      = \begin{pmatrix}
        Y_i - \hat{\mu}_i \\
        X_i\left(Y_i - \hat{\mu}_i\right)
      \end{pmatrix}
    \end{equation}
    Thus, our estimate for $\hat{\beta}$ will remain the same.

    From Equations \ref{eqn:p1_beta_hat_variance} and
    \ref{eqn:p1_beta_hat_quasi_variance}, we have that
    \begin{equation}
      \hat{A} = \frac{1}{n}\hat{D}\hat{V}^{-1}\hat{D} = \frac{1}{n}I_n\left(\hat{\beta}\right)
    \end{equation}

    From Equation \ref{eqn:p1_quasi_likelihood_score_function}, we have that
    \begin{align}
      \hat{B}
      &= \frac{1}{n}
        \hat{D}^\intercal\hat{V}^{-1}\operatorname{diag}\left(RR^\intercal\right)\hat{V}^{-1}\hat{D}
      \nonumber\\
      &= \frac{1}{n}\hat{D}^\intercal\begin{pmatrix}
        \frac{\left(y_1 - \hat{\mu}_1\right)^2}{\hat{\mu}_1^2} & & \\
        & \ddots &\\
        & &\frac{\left(y_n - \hat{\mu}_n\right)^2}{\hat{\mu}_n^2}
      \end{pmatrix}\hat{D} =
            \frac{1}{n}\sum_{i=1}^n \frac{\left(y_i - \hat{\mu}_i\right)^2}{\hat{\mu}_i^2}
      \begin{pmatrix}
        \hat{\mu}_i^2 & x_i\hat{\mu}_i^2 \\
        x_i\hat{\mu}_i^2 & x_i^2\hat{\mu}_i^2
      \end{pmatrix} \nonumber\\
      &= \frac{1}{n}\sum_{i=1}^n\left(y_i-\hat{\mu}_i\right)^2
        \begin{pmatrix}
          1 & x_i\\
          x_i & x_i^2
        \end{pmatrix} = \frac{1}{n}\sum_{i=1}^n
                G\left(\hat{\beta},x_i,y_i\right)G\left(\hat{\beta},x_i,y_i\right)^\intercal.
    \end{align}

    \subsection*{Results}

    The results of estimating $\beta_0$ and $\beta_1$ under various models for
    different $b$ and $n$ can be seen in Tables \ref{tab:p1_results_beta_0} and
    \ref{tab:p1_results_beta_1} for $\beta_0$ and $\beta_1$, respectively. How
    often the 95\% confidence interval contains the true value of $\beta_j$ was
    calculated for each simulation.

    When $b \in \{10,1000\}$, there is not much dispersion, and the Poisson
    model does well for all $n$. At smaller values of $b$, the coverage is not
    very good.

    The quasi-likelihood model does better than the Poisson model for
    $n \geq 50$. For smaller values of $n$, its confidence intervals often fail
    to cover the true value. The quasi-likelihood model does a poorer job as
    covering $\beta_1$, however.

    For $\beta_1$, only sandwich estimation produces good confidence intervals
    for small $b$. However, it needs a lot of samples to do so, typically
    $n = 250$ or $n = 1000$. When $n$ is smaller it often performs worse than
    both the Poisson and quasi-likelihood models.

    Code for the simulations can be found at
    \href{https://nbviewer.jupyter.org/github/ppham27/stat570/blob/master/hw3/poisson\_gamma\_random\_effects.ipynb}{\texttt{poisson\_gamma\_random\_effects.ipynb}}.
  \end{description}

  \begin{landscape}
    \begin{table}
      \tiny
      \begin{subtable}{9in}
        \input{p1_beta_0.tex}
        \caption{95\% confidence interval converage for $\beta_0$.}
        \label{tab:p1_results_beta_0}
      \end{subtable}

      \begin{subtable}{9in}
        \input{p1_beta_1.tex}
        \caption{95\% confidence interval converage for $\beta_1$.}
        \label{tab:p1_results_beta_1}
      \end{subtable}

      \caption{The results of fitting various models to the data from a
        Poisson-gamma random effects model. 100,000 simulations were done for
        each $(\text{model}, b, n)$ tuple.}
      \label{tab:p1_results}
    \end{table}
  \end{landscape}

\item The data in Table \ref{tab:p2_data} contain data on a typical reliability
  experiment and give the failure stresses (in GPa) of four samples of carbon
  fibers of lengths 1, 10, 20 and 50mm.

  \begin{table}
    \tiny
    \centering
    \input{p2_data.tex}
    \caption{Failure stress data for four groups of fibers.}
    \label{tab:p2_data}
  \end{table}

  \begin{enumerate}
  \item The exponential distribution
    $Y \mid \lambda \sim_\mathrm{iid} \operatorname{Exponential}(\lambda)$, is a
    simple model for reliability data:
    \begin{equation}
      p\left(y \mid \lambda \right) =
      \lambda \exp\left(-\lambda y\right),
      \label{eqn:p2_exponential_pdf}
    \end{equation}
    with $\lambda, y > 0$. The hazard function is the probability of imminent
    failure and is given by
    \begin{equation}
      h\left(y\mid\lambda\right) =
      \frac{p\left(y \mid \lambda\right)}{S\left(y\mid\lambda\right)},
      \label{eqn:p2_hazard_definition}
    \end{equation}
    where
    $S\left(y\mid\lambda\right) = \mathbb{P}\left(Y > y \mid \lambda\right)$ is
    the probability of failure beyond $y$. Derive the hazard function for the
    exponential distribution. Suppose we have a sample $y_1,\ldots,y_n$, of size
    $n$ from an exponential distribution. Find the form of the MLE of $\lambda$
    and the asymptotic variance.
      \begin{description}
      \item[Solution:] The survival function can be derived with Equation
        \ref{eqn:p2_exponential_pdf} as
        \begin{align}
          S\left(y\mid\lambda\right)
          &= \mathbb{P}\left(Y > y \mid \lambda\right) \nonumber\\
          &= \int_y^\infty \lambda \exp\left(-\lambda t\right)\,\mathrm{d}t \nonumber\\
          &= \left. -\exp\left(-\lambda t\right) \right|^\infty_y \nonumber \\
          &= \exp\left(-\lambda y\right).
            \label{eqn:p2_survival}
        \end{align}

        With Equations \ref{eqn:p2_hazard_definition} and \ref{eqn:p2_survival},
        the hazard function is
        \begin{equation}
          h\left(y\mid\lambda\right) =
          \frac{p\left(y \mid \lambda\right)}{S\left(y\mid\lambda\right)}
          = \frac{\lambda \exp\left(-\lambda y\right)}{\exp\left(-\lambda y\right)} = \lambda.
          \label{eqn:p2_hazard}
        \end{equation}
      \end{description}

      Given $y_1,\ldots,y_n$, the log-likelihood function is
      \begin{equation}
        l\left(\lambda\right) = n\log\lambda - \lambda\sum_{i=1}^n y_i.
        \label{eqn:p2_exponential_log_likelihood}
      \end{equation}

      From Equation \ref{eqn:p2_exponential_log_likelihood}, the score function is
      \begin{equation}
        S\left(\lambda\right) = \frac{n}{\lambda} - \sum_{i=1}^n y_i.
        \label{eqn:p2_exponential_score}
      \end{equation}
      Solving $S\left(\hat{\lambda}\right) = 0$, gives use the MLE,
      $\boxed{\hat{\lambda} = \frac{n}{\sum_{i=1}^n y_i} = \frac{1}{\bar{y}}.}$

      The asymptotic variance can be derived from the Fisher information
      \begin{equation}
        I_n\left(\hat{\lambda}\right)
        = \operatorname{Var}\left(S\left(\hat{\lambda}\right)\right)
        = \frac{n}{\hat{\lambda}^2}.
      \end{equation}

      Thus, we have that
      $\boxed{\operatorname{Var}\left(\hat{\lambda}\right) =
        \frac{\hat{\lambda}^2}{n}.}$

    \item For each of the four groups in Table \ref{tab:p2_data}, estimate a
      separate $\lambda$, with an associated standard error. Examine the
      appropriateness of the exponential model via Q-Q plots.

      \begin{description}
      \item[Solution:] The estimates and standard errors for $\lambda$ are in
        Table \ref{tab:p2_exponential_estimates}.

        \begin{table}
          \centering
          \input{p2_exponential_estimates.tex}
          \caption{Results of fitting an exponential model for each length.}
          \label{tab:p2_exponential_estimates}
        \end{table}

        From the Q-Q plots in Figure \ref{fig:p2_qq_exponential}, we see that
        the exponential model is a very poor fit for the data. The sampled
        quantiles are very different than the theoretical quantiles. Thus, the
        model appears to be misspecified.

        \begin{figure}
          \centering
          \includegraphics{p2_qq_exponential.pdf}
          \caption{Q-Q plots at each length when fitting an exponential model
            using the MLE.}
          \label{fig:p2_qq_exponential}
        \end{figure}
      \end{description}

      Code for calculations and plots can be found in
      \href{https://nbviewer.jupyter.org/github/ppham27/stat570/blob/master/hw3/failure\_stresses.ipynb}{\texttt{failure\_stresses.ipynb}}.

    \item Consider a quasi-likelihood approach to inference for $\lambda$ under
      the model with
      \begin{align}
        \mathbb{E}\left[Y \mid \lambda \right]
        &= \lambda^{-1} \nonumber\\
        \operatorname{Var}\left(Y \mid \lambda\right)
        &= \alpha\lambda^{-2}
        \label{eqn:p2_quasi_likelihood_model}
      \end{align}
      with $\alpha > 0$. Suggest an estimator for $\alpha$.  Estimate $\lambda$,
      $\alpha$, and the standard errors, separately for each of the four groups
      in Table \ref{tab:p2_data}. What do the results suggest to you about the
      fit of the exponential model?

      \begin{description}
      \item[Solution:] The quasi-likelihood score is
        \begin{align}
          U\left(\lambda\right)
          &= D^\intercal V^{-1}
            \left(Y - \mathbb{E}\left[Y \mid \lambda \right]\right)/\alpha \nonumber\\
          &= \begin{pmatrix}
            -\lambda^{-2} & \cdots & -\lambda^{-2}
          \end{pmatrix} \begin{pmatrix}
            \lambda^{2} & & \\
            & \ddots & \\
            & & \lambda^{2}
          \end{pmatrix} \begin{pmatrix}
            \frac{Y_1 - \lambda^{-1}}{\alpha} \\
            \vdots \\
            \frac{Y_n - \lambda^{-1}}{\alpha}
          \end{pmatrix} \nonumber\\
          &= -\frac{1}{\alpha}\sum_{i=1}^n\left(Y_i - \lambda^{-1}\right)
            = -\frac{1}{\alpha}\left(\sum_{i=1}^nY_i - n\lambda^{-1}\right).
            \label{eqn:p2_quasi_score}
        \end{align}

        Solving Equation \ref{eqn:p2_quasi_score},
        $U\left(\hat{\lambda}\right) = 0$, we get
        $\hat{\lambda} = \bar{Y}^{-1}$, which is the same as the MLE estimate.

        $\hat{\alpha}$ is given by Equation 2.31 of Wakefield's \emph{Bayesian
          and Frequentist Regression Methods}:
        \begin{equation}
          \hat{\alpha}
          = \frac{1}{n - 1}\sum_{i=1}^n\frac{\left(Y_i - \hat{\mu}\right)^2}{V\left(\hat{\mu}\right)}
          = \frac{\hat{\lambda}^2}{n - 1}\sum_{i=1}^n\left(Y_i - \hat{\lambda}^{-1}\right)^2.
        \end{equation}

        We have that
        \begin{equation}
          \operatorname{Var}\left(U\left(\lambda\right)\right)
          = \mathbb{E}\left[- \frac{\partial U}{\partial \lambda}(\lambda)\right]
          = \frac{n\lambda^{-2}}{\alpha},
        \end{equation}
        we can estimate
        \begin{equation}
          \operatorname{Var}\left(\hat{\lambda}\right) =
          \operatorname{Var}\left(U\left(\hat{\lambda}\right)\right)^{-1}
          \approx \frac{\hat{\alpha}\hat{\lambda}^2}{n},
        \end{equation}
        which is the same as the variance for the MLE estimate multiplied by
        $\hat{\alpha}$. The results of fitting the quasi-likelihood model can be
        seen in Table \ref{tab:p2_quasi_likelihood_estimates}.

        \begin{table}
          \centering
          \input{p2_quasi_likelihood_estimates.tex}
          \caption{Results of fitting a quasi-likelihood model for each length.}
          \label{tab:p2_quasi_likelihood_estimates}
        \end{table}

        From Equation \ref{eqn:p2_quasi_score}, we see that a quadratic variance
        function leads to the same score function as Gamma distribution
        with fixed shape parameter $\alpha^{-1}$ and rate parameter
        $\lambda\alpha^{-1}$. $\alpha = 1$ would correspond to the
        exponential distribution, so it is unsurprising to see that our estimate
        for $\lambda$ is the same as the MLE estimate.

        \begin{figure}
          \centering
          \includegraphics{p2_qq_quasi_likelihood.pdf}
          \caption{Q-Q plots at each length with a Gamma distribution. The shape
            parameter $\hat{\alpha}^{-1}$ and rate parameter
            $\hat{\lambda}\hat{\alpha}^{-1}$ were estimated using
            quasi-likelihood.}
          \label{fig:p2_qq_quasi_likelihood}
        \end{figure}

        Standard errors are much smaller than those estimated in Table
        \ref{tab:p2_exponential_estimates}. From Figure
        \ref{fig:p2_qq_exponential}, we see that the residuals are
        underdispersed relative to an exponential model, so $\hat{\alpha} < 1$,
        which leads to the smaller standard error estimates.

        Q-Q plots with the theoretical quantiles derived from
        $\operatorname{Gamma}\left(\hat{\alpha}^{-1},
          \lambda\hat{\alpha}^{-1}\right)$ in Figure
        \ref{fig:p2_qq_quasi_likelihood}. The points lie close the $y = x$
        line. This suggests that the Gamma and quasi-likelihood model are more
        appropriate. They better capture the variance model compared to the
        exponential model.

        Code for calculations and plots can be found in
        \href{https://nbviewer.jupyter.org/github/ppham27/stat570/blob/master/hw3/failure\_stresses.ipynb}{\texttt{failure\_stresses.ipynb}}.
      \end{description}
    \item Obtain the form of the sandwich estimate for the variance of
      $\hat{\lambda}$. Numerically evaluate sandwich standard errors for the
      estimate of $\lambda$ in each of the four groups.

      \begin{description}
      \item[Solution:] We apply Equation 2.43 of Wakefield's \emph{Bayesian and
          Frequentist Regression Methods} to compute the variance of our
        parameter estimate:
    \begin{align}
      \operatorname{Var}\left(\hat{\lambda}\right)
      &= \frac{1}{n}\hat{A}^{-1}\hat{B}\left(\hat{A}^{-1}\right)^\intercal \nonumber\\
      \hat{A}
      &= -\frac{1}{n}\sum_{i=1}^n
        \frac{\partial}{\partial\lambda}G\left(\hat{\lambda}, Y_i\right) \nonumber\\
      \hat{B}
      &= \frac{1}{n}\sum_{i=1}^n
        G\left(\hat{\lambda}, Y_i\right)G\left(\hat{\lambda}, Y_i\right)^\intercal, \nonumber
    \end{align}
    where we resue the quasi-score from Equation \ref{eqn:p2_quasi_score} to
    specify
    \begin{equation}
      G\left(\lambda, Y_i\right) = \frac{1}{\lambda} - Y_i
      \label{eqn:p2_sandwich_score}
    \end{equation}
    as our estimating function.

    Thus, we'll have that
    \begin{align}
      \hat{A}
      &= \frac{1}{n}I\left(\hat{\lambda}\right) = \frac{1}{\hat{\lambda}^2} \nonumber\\
      \hat{B}
      &= \frac{1}{n}\sum_{i=1}^n\left(Y_i - \frac{1}{\hat{\lambda}}\right)^2.
    \end{align}

    Thus, our sandwich estimate will be
    \begin{equation}
      \boxed{
        \operatorname{Var}\left(\hat{\lambda}\right)
        = \frac{\hat{\lambda}^4}{n^2}\sum_{i=1}^n\left(Y_i - \frac{1}{\hat{\lambda}}\right)^2.
      }
      \label{eqn:p2_sandwich_variance}
    \end{equation}
    The results of applying Equations \ref{eqn:p2_sandwich_score} and
    \ref{eqn:p2_sandwich_variance} can be seen in Table \ref{tab:p2_sandwich
      estimates}.

    \begin{table}
      \centering \input{p2_sandwich_estimates.tex}
      \caption{Results of fitting a model with sandwich estimation for each
        length.}
      \label{tab:p2_sandwich estimates}
    \end{table}

    The estimates for $\hat{\lambda}$ are of course the same as in Tables
    \ref{tab:p2_exponential_estimates} and
    \ref{tab:p2_quasi_likelihood_estimates}, since we reused the same score
    function. The standard errors are smaller than the exponential model since
    the data is underdispersed in that model. However, they are quite similar to
    those in the quasi-likelihood model despite not specifying a variance
    model. From Figure \ref{fig:p2_qq_quasi_likelihood}, we have evidence that
    the quasi-likelihood model fits the data well, so it is unsurprising that an
    empirical estimate would yield similar results.
  \end{description}

\item The Weibull distribution is a common model for survival or reliability
  data:
  $Y \mid \eta,\alpha \sim_\mathrm{iid}
  \operatorname{Weibull}\left(\eta,\alpha\right)$, with $\eta > 0$, and
  $\alpha > 0$. The random variable $Y$ has a Weibull distribution if its
  density can be written in the form
  \begin{equation}
    p\left(y \mid \eta,\alpha\right) = \eta\alpha^{-\eta}y^{\eta-1}\exp\left[
      -\left(\frac{y}{\alpha}\right)^\eta
    \right].
    \label{eqn:p2_weibull_pdf}
  \end{equation}

  Find the mean, variance and hazard function of a Weibull distribution. For
  what value of the parameters does the exponential distribution result?

  \begin{description}
  \item[Solution:] The mean can be calculated in terms of the Gamma function
    \begin{align}
      \mathbb{E}\left[
      Y \mid \eta,\alpha
      \right] = \int_0^\infty y p\left(y \mid \eta,\alpha\right)\,\mathrm{d}y
      &= \eta\int_0^\infty \left(\frac{y}{\alpha}\right)^{\eta}
        \exp\left[
        -\left(\frac{y}{\alpha}\right)^\eta
        \right]\,\mathrm{d}y\nonumber\\
      &= \alpha\int_0^\infty u^{1 + 1/\eta - 1}\exp\left(-u\right)\,\mathrm{d}u \nonumber\\
      &= \alpha\Gamma\left(1 + 1/\eta\right).
        \label{eqn:p2_weibull_mean}
    \end{align}

    We can calculate the second moment similarly,
    \begin{align}
      \mathbb{E}\left[
      Y^2 \mid \eta,\alpha
      \right] = \int_0^\infty y^2 p\left(y \mid \eta,\alpha\right)\,\mathrm{d}y
      &= \eta\int_0^\infty y\left(\frac{y}{\alpha}\right)^{\eta}
        \exp\left[
        -\left(\frac{y}{\alpha}\right)^\eta
        \right]\,\mathrm{d}y\nonumber\\
      &= \alpha^2\int_0^\infty u^{2 + 1/\eta - 1}\exp\left(-u\right)\,\mathrm{d}u \nonumber\\
      &= \alpha^2\Gamma\left(2 + 1/\eta\right).
    \end{align}

    Thus, we have that
    \begin{equation}
      \operatorname{Var}\left(
        Y \mid \eta,\alpha
      \right) = \alpha^2\left(
        \Gamma\left(2 + 1/\eta\right) - \left[\Gamma\left(1 + 1/\eta\right)\right]^2
      \right).
    \end{equation}

    The survival function is
    \begin{align}
      S\left(y \mid \eta,\alpha\right)
      &= \int_y^\infty p\left(t \mid \eta,\alpha\right)\,\mathrm{d}t =
        \int_{\left(y/\alpha\right)^\eta}^\infty\exp\left(-u\right)\,\mathrm{d}u \nonumber\\
      &= \exp\left[-\left(\frac{y}{\alpha}\right)^\eta\right].
        \label{eqn:p2_weibull_survival}
    \end{align}

    Thus, the hazard function simplifies to
    \begin{equation}
      h\left(y \mid \eta,\alpha\right) =
      \frac{p\left(y \mid \eta,\alpha\right)}{S\left(y \mid \eta,\alpha\right)} =
      \eta\alpha^{-\eta}y^{\eta-1}.
      \label{eqn:p2_weibull_hazard}
    \end{equation}

    When $\eta = 1$, this is just the exponential distribution with rate
    parameter $\lambda = \alpha^{-1}$.
  \end{description}
\item Is the Weibull distribution with unknown parameters $\eta$, $\alpha$ a
  member of the exponential family? What are the implications for inference?
  \begin{description}
  \item[Solution:] No, the Weibull distribution with unknown parameters $\eta$,
    $\alpha$ is not a member of the exponential family. Mainly, if take the
    $\log$ of the probability density function, we have the term
    $\left(\frac{y}{\alpha}\right)^\eta$. This can not be written in the form
    $\theta^\intercal T(y)$, where $\theta$ are parameters and $T(y)$ is a
    transformation of $y$ into a finite-dimensional vector.

    There are implications in inference. The
    \href{https://en.wikipedia.org/wiki/Sufficient_statistic#Exponential\_family}{Pitman-Koopman-Darmois
      theorem} states that only in exponential families is there a sufficient
    statistic whose dimension remains bounded as sample size increases. Thus,
    when inferring the parameters from a sample with maximum likelihood, all the
    data must be used, which may make the computation intractable for large
    datasets. Morever, when doing a Bayesian inference, the posterior must be
    conditioned on all the data rather than a finite set of sufficient
    statistics, so no conjugate prior will exist.

    In particular, generalized linear models require that the response be
    generated from a distribution in the exponential family. The special
    structure of the exponential family make it so the dispersion estimate is an
    ancillary statistic: its estimation is independent of the estimation of the
    mean.
  \end{description}

\item For the Weibull model and a random sample of size $n$ obtain: the
  log-likelihood, the score, and the observed information matrix.

  \begin{description}
  \item[Solution:] The log-likelihood function is
    \begin{equation}
      l\left(\eta,\alpha\right)
      = \sum_{i = 1}^n\left(
        \log\eta - \eta\log\alpha + \left(\eta - 1\right)\log y_i
        - \exp\left[\eta\left(\log y_i - \log\alpha\right)\right]\right)
    \end{equation}

    From which, we have the score function
    \begin{equation*}
      S\left(\eta,\alpha\right) = \sum_{i=1}^n
      \begin{pmatrix}
        \frac{1}{\eta} - \log\alpha + \log y_i -
        \left(\log y_i - \log\alpha\right)
        \exp\left[\eta\left(\log y_i - \log\alpha\right)\right]
        \\
        -\frac{\eta}{\alpha}
        +\frac{\eta}{\alpha}\exp\left[\eta\left(\log y_i - \log\alpha\right)\right]
      \end{pmatrix}.
    \end{equation*}

    The observed information for a single observation is
    \begin{align*}
      I_{y_i}\left(\eta, \alpha\right)
      &= -\nabla \nabla^\intercal l\left(\eta,\alpha\right) = \nabla S\left(\eta,\alpha\right) \\
      &= \begin{pmatrix}
        \frac{1}{\eta^2} + \left(\log \frac{y_i}{\alpha}\right)^2
        \left(\frac{y_i}{\alpha}\right)^\eta
        &
        \frac{1}{\alpha} - \frac{1}{\alpha}\left(\frac{y_i}{\alpha}\right)^\eta -
        \frac{\eta}{\alpha}\left(\log \frac{y_i}{\alpha}\right)\left(\frac{y_i}{\alpha}\right)^\eta
        \\
        \frac{1}{\alpha} - \frac{1}{\alpha}\left(\frac{y_i}{\alpha}\right)^\eta -
        \frac{\eta}{\alpha}\left(\log \frac{y_i}{\alpha}\right)\left(\frac{y_i}{\alpha}\right)^\eta
        &
        -\frac{\eta}{\alpha^2} + \frac{\eta}{\alpha^2}\left(\frac{y_i}{\alpha}\right)^\eta
        + \left(\frac{\eta}{\alpha}\right)^2\left(\frac{y_i}{\alpha}\right)^\eta
      \end{pmatrix}.
    \end{align*}

    The total observed information is then $\displaystyle
    I_n\left(\eta,\alpha\right) = \sum_{i=1}^nI_{y_i}\left(\eta,\alpha\right)$.
  \end{description}

\item Solve the score equations in order to obtain the maximum likelihood
  estimators (MLEs). You should obtain a single equation that needs to be
  numerically solved.

  \begin{description}
  \item[Solution:] We want to solve the two equations given by
    $S\left(\hat{\eta}, \hat{\alpha}\right) = \mathbf{0}$.

    We can solve for $\hat{\alpha}$ in terms of $\hat{\eta}$ from the second
    entry:
    \begin{equation}
      \hat{\alpha} = \left(\frac{1}{n}\sum_{i=1}^n y_i^{\hat{\eta}}\right)^{1/\hat{\eta}}.
      \label{eqn:p2_alpha_hat}
    \end{equation}

    Substituting $\hat{\alpha}$ with Equation \ref{eqn:p2_alpha_hat}, we can
    numerically solve for $\hat{\eta}$ in the first entry of the score function.
  \end{description}
\item Obtain the MLEs and standard errors for the parameters of the Weibull
  model, for each of the groups in Table \ref{tab:p2_data}.
  \begin{description}
  \item[Solution:] The estimates and standard errors can be found in Table
    \ref{tab:p2_weibull estimates}. One can see that mean of each estimated
    distribution $\hat{\alpha}\Gamma\left(1 + 1/\hat{\eta}\right)$ is quite
    close to corresponding mean for the exponential distribution
    $\hat{\lambda}^{-1}$.

    \begin{table}
      \centering \input{p2_weibull_estimates.tex}
      \caption{Results of fitting a Weibull model to each length by maximizing
        likelihood.}
      \label{tab:p2_weibull estimates}
    \end{table}

    \begin{figure}
      \centering
      \includegraphics{p2_qq_weibull.pdf}
      \caption{Q-Q plots at each length with a Weibull distribution with
        parameters in Table \ref{tab:p2_weibull estimates}.}
      \label{fig:p2_qq_weibull}
    \end{figure}

    The Q-Q plots in Figure \ref{fig:p2_qq_weibull} indicate the Weibull
    distribution fits the data quite well. However, it appears slightly worse
    than the Gamma distribution: compared to Figure
    \ref{fig:p2_qq_quasi_likelihood}, the lower tail is further away from the
    $y = x$ line. That is, the data are underdispered when fitted to the Weibull
    distribution.

    Code for calculations and plots can be found in
    \href{https://nbviewer.jupyter.org/github/ppham27/stat570/blob/master/hw3/failure\_stresses.ipynb}{\texttt{failure\_stresses.ipynb}}.
  \end{description}
\end{enumerate}
\end{enumerate}

\end{document}


%mainfile: ../lecture_notes.tex
\chapter{Recursive Least-Squares Adaptive Filters}
\label{ch:RLS}
\section{Review of the Basics}
\subsection{The Matrix Inversion Lemma}
\label{sec:woodbury}
Let $\vect{X}$, $\vect{Y}$, $\vect{X}+\vect{UYV}$, and $\vect{Y}^{-1}+\vect{VX}^{-1}\vect{U}$ all be non-singular matrices. By equating element $(1,1)$ of the two block matrices in \eq{eq:block_inv1} and \eq{eq:block_inv2} and setting
\begin{align}
  \vect{X} &= \vect{A}\\
  \vect{U} &= \vect{B}\\
  \vect{V} &= \vect{C}\\
  \vect{Y} &= -\vect{D}^{-1}\ ,
\end{align}
we obtain the matrix inversion lemma
\bmath
  (\vect{X}+\vect{UYV})^{-1} = \vect{X}^{-1}-\vect{X}^{-1}\vect{U}(\vect{Y}^{-1}+\vect{VX}^{-1}\vect{U})^{-1}\vect{V}\vect{X}^{-1}\ .
  \label{eq:woodbury}
\emath
The matrix inversion lemma is also sometimes called the \textit{Woodbury matrix identity} or the \textit{Woodbury's identity}.

\section{Method of Least-Squares}
The adaptive filtering problem is shown in Fig.~\ref{fig:nonwss_block_diagram2}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/nonwss_block_diagram}
  \caption{Block diagram of adaptive filtering in a non-WSS environment.}
  \label{fig:nonwss_block_diagram2}
\end{figure}
From the figure, we have that
\begin{itemize}
  \item[$u(n)$:] zero-mean input signal
  \item[$w_m(n)$:] $M$-tap FIR-filter with impulse response $w_0(n),w_1(n),\ldots,w_{M-1}(n)$
  \item[$y(n)$:] output signal given by $y(n) = \sum_{m=0}^{M-1}w_m(n) u(n-m)$
  \item[$d(n)$:] zero-mean desired signal
  \item[$e(n)$:] error signal
\end{itemize}
Define
\begin{align}
  \vect{w}(n) &= \bbmtx w_0(n) & w_{1}(n) & \cdots & w_{M-1}(n)\ebmtx^T\\
  \vect{u}(n)&= \bbmtx u(n) & u(n-1) & \cdots & u(n-M+1)\ebmtx^T\ .
\end{align}
For $i=1,2,\ldots,n$ with $n\geq M$, define
\begin{align}
  \vect{A}(n) &= \bbmtx \vect{u}(1) & \vect{u}(2) & \cdots & \vect{u}(n)\ebmtx^T\\
  \vect{d}(n) &= \bbmtx d(1) & d(2) & \cdots & d(n)\ebmtx^T\\
  \vect{e}(n) &= \bbmtx e(1) & e(2) & \cdots & e(n)\ebmtx^T\ .
\end{align}
Then
\bmath
  \vect{e}(n) = \vect{d}(n)-\vect{A}(n)\vect{w}(n)\quad\iff\quad\vect{d}(n)=\vect{A}(n)\vect{w}(n)+\vect{e}(n)\ .
\emath
Note that we have made $\vect{A}(n)$, $\vect{d}(n)$, and $\vect{e}(n)$ time-dependent in order to emphasise that we are here concerned with an online algorithm. Therefore, we also formulate the squared error cost function $J_2(\vect{w}(n))$ in a slightly different way compared to \eq{eq:cf_squared_error}. Here, we define it as
\bmath
  J_2(\vect{w}(n)) = \sum_{i=1}^{n}e^2(i)=\vect{e}^T(n)\vect{e}(n)\ .
\emath

In the method of least-squares, we wish to minimise $J_2(\vect{w}(n))$ which we can write as
\begin{align}
  J_2(\vect{w}(n)) &= \vect{e}^T(n)\vect{e}(n) = (\vect{d}(n)-\vect{A}(n)\vect{w}(n))^T(\vect{d}(n)-\vect{A}(n)\vect{w}(n))\notag\\
  &= \vect{d}^T(n)\vect{d}(n)+\vect{w}^T(n)\vect{A}^T(n)\vect{A}(n)\vect{w}(n)-2\vect{w}^T(n)\vect{A}^T(n)\vect{d}(n)\ .
\end{align}
The minimiser
\bmath
  \vect{w}_o(n) = (\vect{A}^T(n)\vect{A}(n))^{-1}\vect{A}^T(n)\vect{d}(n)
\emath
is referred to as the least-squares solution, and it is the unique solution to the normal equations
\bmath
  \vect{A}^T(n)\vect{A}(n)\vect{w}(n) = \vect{A}^T(n)\vect{d}(n)\ ,
\emath
provided that $\vect{A}(n)$ has full rank.

\subsection{Weighted Least-Squares}
When the statistics of $u(n)$ and/or $d(n)$ is time dependent, the minimisation of the squared error $J_2(\vect{w}(n))$ may fail to give a good estimate at time $n$ since all data affect the value of $J_2(\vect{w}(n))$ with the same weight. Ideally, we would like that the new data are assigned a larger weight than the old data. In order to do this, we reformulate $J_2(\vect{w}(n))$ as a weighted cost function
\bmath
  J_\beta(\vect{w}(n)) = \sum_{i=1}^{n}\beta(n,i) e^2(i) = \vect{e}^T(n)\vect{B}(n) \vect{e}(n)
\emath
where $\beta(n,i)$ contains the weight pertaining to the $i$'th error at time $n$, and $\vect{B}(n)$ is a diagonal matrix given by
\bmath
  \vect{B}(n) = \diag{\beta(n,1), \beta(n,2), \ldots , \beta(n,n)}\ .
\emath
We use the five step recipe in Sec.~\ref{ssec:optimisation} to minimise $J_\beta(\vect{w}(n))$ w.r.t $\vect{w}(n)$.
\begin{enumerate}
  \item Construct the cost function
  \begin{align}
    J_\beta(\vect{w}(n)) &= \vect{e}^T(n)\vect{B}(n) \vect{e}(n) = (\vect{d}(n)-\vect{A}(n)\vect{w}(n))^T\vect{B}(n)(\vect{d}(n)-\vect{A}(n)\vect{w}(n))\notag\\
    &= \vect{d}(n)^T\vect{B}(n)\vect{d}(n)+\vect{w}^T(n)\vect{\Phi}(n)\vect{w}(n)-2\vect{w}^T(n)\vect{\varphi}(n)
  \end{align}
  where we have defined $\vect{\Phi}(n)$ and $\vect{\varphi}(n)$ as
\begin{align}
  \vect{\Phi}(n) &= \vect{A}^T(n)\vect{B}(n)\vect{A}(n)\\
  \vect{\varphi}(n) &= \vect{A}^T(n)\vect{B}(n)\vect{d}(n)\ .
\end{align}
We refer to $\vect{\Phi}(n)$ and $\vect{\varphi}(n)$ as the correlation matrix and the cross-correlation vector, respectively, since they are scaled and weighted estimates of $\vect{R}_u$ and $\vect{r}_{ud}$.
  \item Find the gradient
  \bmath
    \vect{g}(\vect{w}(n)) = (\vect{\Phi}(n)+\vect{\Phi}^T(n))\vect{w}(n)-2\vect{\varphi}(n)=2\vect{\Phi}(n)\vect{w}(n)-2\vect{\varphi}(n)
  \emath
  \item Solve $\vect{g}(\vect{w}(n))=\vect{0}$ for $\vect{w}(n)$
  \begin{alignat}{3}
    &&\qquad \vect{g}(\vect{w}(n)) &= \mathrlap{2\vect{\Phi}(n)\vect{w}(n)-2\vect{\varphi}(n)}\notag\\
    \ArrowBetweenLines
    &&\qquad \alignedbox{\vect{\Phi}(n)\vect{w}(n)}{=\vect{\varphi}(n)}  &\qquad &\text{(Weighted Normal Equations)}\\
    \ArrowBetweenLines
    &&\qquad           \vect{w}_o(n) &= \vect{\Phi}^{-1}(n)\vect{\varphi}(n) &\qquad &\text{(If }\vect{\Phi}(n)\text{ is invertible)}
  \end{alignat}
  \item Find the Hessian
  \bmath
    \vect{H}(\vect{w}) = 2\vect{\Phi}(n)
  \emath
  which is p.d. for all $\vect{w}(n)$ if $\vect{A}(n)$ has full rank and $\beta(n,i)>0$ for all $n\geq i > 0$.
  \item This implies that
  \begin{itemize}
    \item $J_\beta(\vect{w}(n))$ is a convex function, and
    \item $\vect{w}_o(n) = \vect{\Phi}^{-1}(n)\vect{\varphi}(n)$ is the global minimiser.
  \end{itemize}
\end{enumerate}
The solution $\vect{w}_o(n)$ is often referred to as the weighted least-squares solution.

\subsubsection{Estimation of the (Cross-)Correlation}
Comparing the weighted normal equations with the Wiener-Hopf equations, we see that
\begin{align}
  \hat{\vect{R}}_u(n) &= c(n,\beta)\vect{\Phi}(n) = c(n,\beta)\vect{A}^T(n)\vect{B}(n)\vect{A}(n) = c(n,\beta)\sum_{i=1}^{n}\beta(n,i)\vect{u}(i)\vect{u}^T(i)\label{eq:wls_Ru_est}\\
  \hat{\vect{r}}_{ud}(n) &= c(n,\beta)\vect{\varphi}(n) = c(n,\beta)\vect{A}^T(n)\vect{B}(n)\vect{d}(n) = c(n,\beta)\sum_{i=1}^{n}\beta(n,i)\vect{u}(i)d(i)\label{eq:wls_rud_est}
\end{align}
are the estimates of the correlation matrix and the cross-correlation vector, respectively. The constant $c(n,\beta)$ depends on $n$ and the weighting function $\beta(n,i)$, and it can be selected such that $\hat{\vect{R}}_u(n)$ and $\hat{\vect{r}}_{ud}(n)$ are unbiased estimates of $\vect{R}_u(n)$ and $\vect{r}_{ud}(n)$.

\subsection{Weight Functions}
We consider three simple weight functions.

\subsubsection{Growing Window Weight Function}
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/growing_window}
  \caption{The growing window weight function.}
  \label{fig:growing_window}
\end{figure}
\noindent If we select the weight function as
\bmath
  \beta(n,i) =
  \begin{cases}
    1 & 0 < i \leq n\\
    0 & \text{otherwise}
  \end{cases}\ ,
\emath
we obtain the growing window weight function, and it is sketched in Fig.~\ref{fig:growing_window}. Selecting the growing window weight function reduces the weighted least-squares problem to the standard least-squares problem. In order to obtain unbiased estimates of $\vect{R}_u(n)$ and $\vect{r}_{ud}(n)$ in \eq{eq:wls_Ru_est} and \eq{eq:wls_rud_est}, respectively, we have to use
\bmath
  c(n,\beta) = \frac{1}{n}\ .
\emath

\subsubsection{Sliding Window Weight Function}
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/sliding_window}
  \caption{The sliding window weight function.}
  \label{fig:sliding_window}
\end{figure}
\noindent  If we select the weight function as
\bmath
  \beta(n,i) =
  \begin{cases}
    1 & n-L < i \leq n\\
    0 & \text{otherwise}
  \end{cases}
\emath
for $0 < L \leq n$, we obtain the sliding window weight function, and it is sketched in Fig.~\ref{fig:sliding_window}. If we select $L=n$, the sliding window weight function reduces to the growing window weight function. In order to obtain unbiased estimates of $\vect{R}_u(n)$ and $\vect{r}_{ud}(n)$ in \eq{eq:wls_Ru_est} and \eq{eq:wls_rud_est}, respectively, we have to use
\bmath
  c(n,\beta) = \frac{1}{L}\ .
\emath

\subsubsection{Exponential Weight Function}
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/exponential_window}
  \caption{The exponential weight function.}
  \label{fig:exponential_window}
\end{figure}
\noindent If we select the weight function as
\bmath
  \beta(n,i) =
  \begin{cases}
    \lambda^{n-i} & 0 < i \leq n\\
    0 & \text{otherwise}
  \end{cases}
\emath
for $0<\lambda\leq 1$, we obtain the exponential weight function, and it is sketched in Fig.~\ref{fig:exponential_window}. The parameter $\lambda$ is called \textit{the forgetting factor}. If we select $\lambda=1$, the exponential weight function reduces to the growing window weight function. In order to obtain unbiased estimates of $\vect{R}_u(n)$ and $\vect{r}_{ud}(n)$ in \eq{eq:wls_Ru_est} and \eq{eq:wls_rud_est}, respectively, we have to use
\bmath
  c(n,\beta) =
  \begin{cases}
    \displaystyle\frac{1-\lambda}{1-\lambda^n} & 0<\lambda< 1\\
    \displaystyle\frac{1}{n} & \lambda=1
  \end{cases}\ .
\emath

\section{The Recursive Least-Squares Algorithm with an Exponential Weight Function}
In an online algorithm, we have to solve the weighted normal equations
\bmath
  \vect{\Phi}(n)\vect{w}(n) = \vect{\varphi}(n)
\emath
for $\vect{w}(n)$ at every time index $n$. However, solving this equation directly as
\bmath
  \vect{w}(n) = \vect{\Phi}^{-1}(n)\vect{\varphi}(n)
\emath
yields a high computational complexity of the algorithm for the following two reasons.
\begin{enumerate}
  \item The matrices $\vect{A}(n)$ and $\vect{B}(n)$, and the vector $\vect{d}(n)$ grows with $n$. Computing $\vect{\Phi}(n)$ and $\vect{\varphi}(n)$ directly would therefore be infeasible for an online algorithm, unless we use a weight function with a finite duration window.
  \item For an $M$-tap FIR filter, it requires in the order of $\mathcal{O}(M^3)$ operations to solve the normal equations for the filter coefficient vector $\vect{w}(n)$.
\end{enumerate}
The recursive least-squares (RLS) algorithm bypasses these two problems. Below, we consider how this is obtained for an exponential weight function, which is the most common weight function.

\subsubsection{Recursive Computation of $\vect{\Phi}(n)$ and $\vect{\varphi}(n)$}
We may compute the correlation matrix recursively by rewriting it as
\begin{align}
  \vect{\Phi}(n) &= \vect{A}^T(n)\vect{B}(n)\vect{A}(n) = \sum_{i=1}^{n}\lambda^{n-i}\vect{u}(i)\vect{u}^T(i)\\
  &= \lambda^0 \vect{u}(n)\vect{u}^T(n) + \sum_{i=1}^{n-1}\lambda^{n-i}\vect{u}(i)\vect{u}^T(i) = \vect{u}(n)\vect{u}^T(n) + \lambda\sum_{i=1}^{n-1}\lambda^{n-1-i}\vect{u}(i)\vect{u}^T(i)\notag\\
  &= \vect{u}(n)\vect{u}^T(n) + \lambda\vect{\Phi}(n-1)\ .
  \label{eq:rls_corr_recurs}
\end{align}
Similarly, we may compute the cross-correlation vector recursively by rewriting it as
\begin{align}
  \vect{\varphi}(n) &= \vect{A}^T(n)\vect{B}(n)\vect{d}(n) = \sum_{i=1}^{n}\lambda^{n-i}\vect{u}(i)d(i)\\
  &= \lambda^0 \vect{u}(n)d(n) + \sum_{i=1}^{n-1}\lambda^{n-i}\vect{u}(i)d(i) = \vect{u}(n)d(n) + \lambda\sum_{i=1}^{n-1}\lambda^{n-1-i}\vect{u}(i)d(i)\notag\\
  &= \vect{u}(n)d(n) + \lambda\vect{\varphi}(n-1)\ .
\end{align}
These recursive formulations of the correlation matrix and the cross-correlation vector clearly reduce the computational complexity and are suitable to use in an online algorithm.

\subsubsection{Inversion of $\vect{\Phi}(n)$}
For the inversion of the correlation matrix, we use the matrix inversion lemma stated in Sec.~\ref{sec:woodbury}. Comparing the recursive formulation of the correlation matrix in \eq{eq:rls_corr_recurs} with the left side of \eq{eq:woodbury}, we obtain
\begin{align}
  \vect{X} &= \lambda\vect{\Phi}(n-1)\\
  \vect{U} &= \vect{u}(n)\\
  \vect{V} &= \vect{u}^T(n)\\
  \vect{Y} &= 1\ .
\end{align}
Thus, invoking the matrix inversion lemma, we have that
\bmath
  \vect{\Phi}^{-1}(n) = \lambda^{-1}\vect{\Phi}^{-1}(n-1)-\lambda^{-2}\frac{\vect{\Phi}^{-1}(n-1)\vect{u}(n)\vect{u}^T(n)\vect{\Phi}^{-1}(n-1)}{1+\lambda^{-1}\vect{u}^T(n)\vect{\Phi}^{-1}(n-1)\vect{u}(n)}\ .
  \label{eq:rls_inv_phi}
\emath
Note that the computational complexity of the right side of the equation is much lower than that of the left side of the equation when $\vect{\Phi}^{-1}(n-1)$ is known. In order to simplify the notation, we define
\begin{alignat}{2}
  \vect{P}(n) &= \vect{\Phi}^{-1}(n)&\quad&\text{(inverse correlation matrix)}\\
  \vect{k}(n) &= \frac{\vect{P}(n-1)\vect{u}(n)}{\lambda+\vect{u}^T(n)\vect{P}(n-1)\vect{u}(n)}&\quad&\text{(gain vector)}
\end{alignat}
which leads to that we may write \eq{eq:rls_inv_phi} as
\bmath
  \vect{P}(n) = \lambda^{-1}\left[\vect{P}(n-1)-\vect{k}(n)\vect{u}^T(n)\vect{P}(n-1)\right]\ .
\emath
By rearranging the expression for the gain vector, we obtain
\begin{align}
  \vect{k}(n) &= \lambda^{-1}\left[\vect{P}(n-1)-\vect{k}(n)\vect{u}^T(n)\vect{P}(n-1)\right] \vect{u}(n)\notag\\
  &= \vect{P}(n)\vect{u}(n)\ .
\end{align}

\subsubsection{Recursive computation of $\vect{w}(n)$}
We can now develop the recursive update equation of the filter coefficient vector. We have that
\begin{align}
  \vect{w}(n) &= \vect{P}(n)\vect{\varphi}(n)\\
  &= \vect{P}(n)\left[\vect{u}(n)d(n) + \lambda\vect{\varphi}(n-1)\right]=\lambda\vect{P}(n)\vect{P}^{-1}(n-1)\vect{w}(n-1)+\vect{k}(n)d(n)\notag\\
  &= \left[\vect{P}(n-1)-\vect{k}(n)\vect{u}^T(n)\vect{P}(n-1)\right]\vect{P}^{-1}(n-1)\vect{w}(n-1)+\vect{k}(n)d(n)\notag\\
  &= \vect{w}(n-1)-\vect{k}(n)\vect{u}^T(n)\vect{w}(n-1)+\vect{k}(n)d(n)\notag\\
  &= \vect{w}(n-1)+\vect{k}(n)\xi(n)
\end{align}
where we have defined the a priori error as
\bmath
  \xi(n) = d(n)-\vect{u}^T(n)\vect{w}(n-1)\ .
\emath

\subsubsection{The RLS Algorithm}
The RLS algorithm may now be formulated as the following set of equations
\begin{align}
  \vect{\pi}(n) &= \vect{P}(n-1)\vect{u}(n)\\
  \vect{k}(n) &= \frac{\vect{\pi}(n)}{\lambda+\vect{u}^T(n)\vect{\pi}(n)}\\
  \xi(n) &= d(n)-\vect{u}^T(n)\vect{w}(n-1)\\
  \vect{w}(n) &= \vect{w}(n-1)+\vect{k}(n)\xi(n)\\
  \vect{P}(n) &= \lambda^{-1}\left[\vect{P}(n-1)-\vect{k}(n)\vect{\pi}^T(n)\right]\ .
\end{align}
For an $M$-tap FIR filter, it requires in the order of $\mathcal{O}(M^2)$ operations to run one iteration of the RLS algorithm.

\subsubsection{Initialisation}
In order to start the RLS algorithm, we need to select values for the initial inverse correlation matrix $\vect{P}(0)$, the initial filter coefficient vector $\vect{w}(0)$, and the input samples $u(n)$ for $n=-M+1,-M+2,\ldots,1$. Typically, we assume that
\begin{align}
  \vect{P}(0) &= \delta^{-1}\vect{I}\\
  \vect{w}(0) &= \vect{0}\\
  u(n) & = 0\ ,\quad\text{for } -M+1<n<1\ .
\end{align}
The first assumption implies that we assume that $\vect{u}(n)$ for $n<1$ is a white random process with covariance matrix $\delta\vect{I}$. The value of $\delta$ should reflect the SNR of the input data with $\delta$ being small for a high SNR and $\delta$ being large for a low SNR \cite[pp.~444--446]{Haykin2001}. This assumption introduces bias into the correlation matrix $\vect{\Phi}(n)$. However, this bias decreases to zero for an increasing $n$. An alternative initialisation, which does not introduce bias, is to estimate the correlation matrix and the cross-correlation vector as \cite[pp.~545--546]{Hayes1996}
\begin{align}
  \vect{P}(0) &= \left[\sum_{i=-M+1}^{0}\lambda^{-i}\vect{u}(i)\vect{u}^T(i)\right]^{-1}\\
  \vect{\varphi}(0) &= \sum_{i=-M+1}^{0}\lambda^{-i}\vect{u}(i)\vect{d}(i)
\end{align}
prior to starting the RLS algorithm at time $n=1$. The initial value of the filter coefficient vector can be set to $\vect{w}(0)=\vect{P}(0)\vect{\varphi}(0)$. Note, that this approach requires that we know the input signal from time $n=-2M+2$ and the desired signal from time $n=-M+2$.

\subsection{Selection of the Forgetting Factor}
At time $n$, the memory of the sliding window RLS algorithm is the $L$ newest samples indexed by $n-L+1,\ldots, n$. For the exponentially weighted RLS algorithm, the memory is controlled by the forgetting factor $\lambda$. Whereas the interpretation of $L$ is simple, the corresponding interpretation of $\lambda$ is not that intuitive when we have to investigate the memory of the exponentially weighted RLS algorithm. That is, we would like to interpret the forgetting factor as a sliding window length. We call this window length for the effective window length and denote it by $L_\textup{eff}$. A simple way of connecting $L_\textup{eff}$ and $\lambda$ is by requiring that
\bmath
  \lim_{n\to\infty} \sum_{i=1}^{n}\lambda^{n-i} = \lim_{n\to\infty} \sum_{i=n-L_\textup{eff}+1}^{n}1\ .
\emath
That is, when the RLS algorithm has reached steady-state, the area under the sliding window curve should equal the area under the exponential window curve. This leads to
  \bmath
     \lim_{n\to\infty} \sum_{i=n-L_\textup{eff}+1}^{n}1 = L_\textup{eff}
  \emath
  and
  \bmath
    \lim_{n\to\infty} \sum_{i=1}^{n}\lambda^{n-i}=\lim_{n\to\infty} \sum_{k=0}^{n-1}\lambda^{k} = \lim_{n\to\infty} \sum_{k=0}^{n-1}\lambda^{k} = \lim_{n\to\infty}\frac{1-\lambda^n}{1-\lambda} = \frac{1}{1-\lambda}
  \emath
  where the second last equality follows for $\lambda\neq 1$ from the geometric series, and the last equality follows if $0<\lambda<1$. Thus, we have that
  \bmath
    L_\textup{eff} = \frac{1}{1-\lambda}\ .
  \emath

\subsection{Transient Analysis}
The RLS algorithm is stable in the mean and the mean-square if $0<\lambda\leq 1$\ . It may also be shown that the rate of the convergence of the RLS algorithm is typically an order of magnitude faster than the rate of the convergence of the LMS algorithm. Moreover, the rate of the convergence of the RLS algorithm is invariant to the condition number of the correlation matrix $\vect{R}_u$ of the input signal \cite[p.~463, ch.~14]{Haykin2001}.

\subsection{Steady-State Analysis}
It can be shown that \cite[p.~510]{Sayed2003}
\begin{alignat}{2}
  &\text{EMSE:}&\quad J_\textup{ex} &= J_2(\vect{w}(\infty))-J_\textup{min}\notag\\
  & &\quad &\approx J_\textup{min}\frac{(1-\lambda)M}{1+\lambda-(1-\lambda)M}\\
  &\text{Misadjustment:}&\quad \mathcal{M} &= \frac{ J_\textup{ex}}{J_\textup{min}} \approx \frac{(1-\lambda)M}{1+\lambda-(1-\lambda)M}\\
  &\text{MSD:} &\quad E[\|\vect{\Delta}\vect{w}(\infty)\|^2] &\approx J_\textup{ex}\sum_{m=1}^M\frac{1}{\lambda_m}
\end{alignat}
where $\lambda_m$ is the $m$'th eigenvalue of the correlation matrix $\vect{R}_u$ not to be confused with the forgetting factor $\lambda$. The approximations hold under certain conditions which may be found in \cite[pp.~508--510]{Sayed2003}.

\subsection{Computational Cost}
Table~\ref{tab:comp_cost_rls} shows the computational cost of the RLS algorithm in terms of the number of multiplications, additions or subtractions, and divisions. From the table, we see that the total number of flops is $5M^2+5M+1$. Thus, the RLS algorithm has a complexity of $\mathcal{O}(M^2)$. Note, that there exist faster ways of implementing the RLS algorithm \cite[pp.~247]{Sayed2003}. Some of them even achieve linear complexity \cite{Slock1991}.
\begin{table}[htbp]
  \centering
  \begin{tabular}{l c c c}
    \toprule
    Term & $\times$ & $+$ or $-$ & $/$\\
    \midrule
    $\vect{\pi}(n)=\vect{P}(n-1)\vect{u}(n)$ & $M^2$ & $M(M-1)$ &\\
    $\vect{k}(n) = \vect{\pi}(n)/(\lambda+\vect{u}^T(n)\vect{\pi}(n))$ & $M$ & $M$ & $1$\\
    $\xi(n)=d(n)-\vect{u}^T(n)\vect{w}(n-1)$ & $M$ & $M$ & \\
    $\vect{w}(n) = \vect{w}(n-1)+\vect{k}(n)\xi(n)$ & $M$ & $M$ & \\
    $\vect{P}(n) = (\vect{P}(n-1)-\vect{k}(n)\vect{\pi}^T(n))/\lambda$ & $M^2$ & $M^2$ & $M^2$ \\
    \midrule
    Total & $2M^2+3M$ & $2M^2+2M$ & $M^2+1$\\
    \bottomrule
  \end{tabular}
  \caption{Computational cost of the RLS algorithm.}
  \label{tab:comp_cost_rls}
\end{table}


\chapter{Poison Processes}
\emph{“好想退休哦。”}
\newpage

\section{Poisson Distribution}
    \subsection{Poisson Distibution}
    \begin{definition}[Poisson Distribution]
        $X$ has a \textbf{Poisson distribution} with rate $\lambda$, denoted by $Poisson(\lambda)$, if
        \[ \mathbb{P}[X=k] = \frac{\lambda^k}{k!}e^{-\lambda} \]
    \end{definition}
    \begin{remark}
        If we view $\lambda$ as a constant, then the Poisson distribution can be seen as the limitation of a Binomial distribution $Bermoulli(n, p)$ with $p = \frac{\lambda}{n}$, as $n\to\infty$.
    \end{remark}
    \begin{theorem}
        If $n$ is large, then $Bernoulli(n, \lambda/n)$ is approximately $Poisson(\lambda)$.
    \end{theorem}

    Suppose we want to compute the number of customers arriving at some restaurant. If we divide the time interval to be sufficiently small s.t. at most one customer arrives at each interval and the probability is uniformly $p$, then the total number of arrivals has a Poisson distribution.

    \subsection{Expectation and Variance of Poisson Distribution}
    Let $X$ be a random variable with a Poisson distribution, then
    \[ \mathbb{E}[X] = \lambda \]
    \[ Var[X] = \lambda \]

    \subsection{Additivity of Poisson Distributions}
    \begin{theorem}[Additivity of Poisson Distributions]
        Let $X_1$, $X_2$ be two independent random variables, $X_1\sim Poisson(\lambda_1)$, $X_2\sim Poisson(\lambda_2)$, then
        \[ X_1 + X_2 \sim Poisson(\lambda_1 + \lambda_2) \]
    \end{theorem}
    \begin{proof}
        \begin{align*}
            \mathbb{P}[X_1=X_2=n] &=\sum_{m=0}^n \mathbb{P}[X_1=m, X_2=n-m]\\
            &= \sum_{m=0}^n\mathbb{P}[X_1=m]\cdot\mathbb{P}[X_2=n-m]\\
            &= \sum_{m=0}^n\frac{\lambda_1^m}{m!}e^{-\lambda_1}\cdot\frac{\lambda_2^{n-m}}{(n-m)!}e^{-\lambda_2}\\
            &= \frac{e^{-(\lambda_1+\lambda_2)}}{n!}\sum_{m=0}^n\frac{n!}{n!(n-m)!}\lambda_1^m\lambda_2^{n-m}\\
            &= \frac{e^{-(\lambda_1+\lambda_2)}}{n!}\sum_{m=0}^n\mathrm{C}_m^n \lambda_1^m \lambda_2^{n-m}\\
            &= \frac{(\lambda_1+\lambda_2)^n}{n!}e^{-(\lambda_1+\lambda_2)}
        \end{align*}
    \end{proof}

\section{Exponential Distributions}

    \subsection{Definition of Exponential Distributions}
    \begin{definition}[Exponential Distribution]
        A random variable $X$ is said to have an \textbf{exponential distribution} with rate $\lambda$, $X \sim Exp(\lambda)$ if
        \[ \forall t \ge 0 \quad \mathbb{P}[X \le t] = 1 - e^{-\lambda t} \]
    \end{definition}

    \subsection{Properties of Exponential Distribution}

        \subsubsection{Probability Density Function}
        The exponential distribution $Exp(\lambda)$ has probability density function
        \[ p(x) = \lambda e^{-\lambda x} \]

        \subsubsection{Expectation}
        \[ \mathbb{E}[X] = \int_0^{+\infty} t\lambda e^{-\lambda t}\mathrm{d}t = \frac{1}{\lambda} \]

        \subsubsection{Variance}
        \[ Var[X] = \mathbb{E}[X^2] - \mathbb{E}[X] = \frac{1}{\lambda^2} \]

        \subsubsection{Lack of Memory}
        \[ \mathbb{P}[X>t+s|X>s] = \mathbb{P}[X \ge t] \]
        \begin{remark}
            If we have waited for $s$ units of time, then the probability that we have to wait for $t$ more units is the same as that of we have not waited at all.\footnote{白~{}等~{}了}
        \end{remark}

        \subsubsection{Exponential Races}\label{subs:ExponentialRace}
        Let $X_1 \sim Exp(\lambda_1)$, $X_2 \sim Exp(\lambda_2)$ be two independent random variables. Let $Y = \min(X_1, X_2)$, then
        \[ Y \sim Exp(\lambda_1 + \lambda_2) \]
        \begin{proof}
            \begin{align*}
                \mathbb{P}[Y \ge k] &= \mathbb{P}[\min\{X_1,X_2\} \ge k]\\
                &= \mathbb{P}[X_1 \ge k, X_2 \ge k]\\
                &= \mathbb{P}[X_1 \ge k]\cdot \mathbb{P}[X_2 \ge k]\\
                &= e^{-(\lambda_1 + \lambda_2)k}
            \end{align*}
        \end{proof}

        If we consider the problem of ``Who finishes first between $X_1$ and $X_2$'', then
        \[ \mathbb{P}[Y=X_1]=\sum_{s=0}^\infty \mathbb{P}[X_1=s,X_2>s] = \int_0^{+\infty}p_1(s)\mathbb{P}[X_2>s]\mathrm{d}s = \frac{\lambda_1}{\lambda_1 + \lambda_2} \]
        And an exponential race among $n$ random variables is
        \[ \mathbb{P}[Y=X_i] = \frac{\lambda_i}{\sum_j \lambda_j} \]

\section{Poisson Processes}

    \subsection{Definition of Poisson Processes}
    \begin{definition}[Poisson Process]\label{def:PoissonProcess}
        $\{N(s)|s \ge 0\}$ is a \textbf{Poisson Process}, if
        \begin{enumerate}
            \item $N(0)=0$
            \item $\forall t, s \ge 0 \quad N(t+s) - N(s) \sim Poisson(\lambda t)$
            \item $n$ has \textbf{independent increments}: $\forall t_0 \le t_1 \le \dots \le t_n$,
            \[ N(t_1)-N(t_0), N(t_2)-N(t_1), \dots, N(t_n)-N(t_{n-1}) \]
            are \textbf{mutually independent}.
        \end{enumerate}
    \end{definition}

    \subsection{Alternative Interpretation of Poisson Process}
    We can construct a Poisson process as the sum of multiple random variables with Exponential distributions.
    \begin{proposition}\label{prop:AltdefOfPoissonProcess}
        Let $\tau_1,\tau_2,\dots,\tau_n$ be independent random variables with exponential distribution $Exp(\lambda)$. Let $T_n=\sum_{i=1}^n\tau_i$, $N(s)=\max\{n|T_n \le s\}$

        Then $N(s)$ is a Poisson process with rate $\lambda$.
    \end{proposition}
    \begin{remark}
        If we think of $\tau_i$ as the time interval between the arrival time of customers in a store, then $N(s)$ is the number of customer arrivals before time $s$.
    \end{remark}

    To prove Proposition \ref{prop:AltdefOfPoissonProcess}, we first introduce a theorem.

    \begin{theorem}\label{thm:SumOfExpHasGammaDistribution}
        Let $\tau_1, \tau_2, \dots, \tau_n$ be independent random variables with Exponential distribution $Exp(\lambda)$. Let $T_n = \sum_{i=1}^n\tau_n$, then $T_n$ has a \emph{Gamma distribution} $\Gamma(n, \lambda)$
        \[ f_{T_n}(t) = \lambda e^{-\lambda t}\frac{(\lambda t)^{n-1}}{(n-1)!} \quad \forall t \ge 0 \]
    \end{theorem}
    \begin{proof}
        Prove by induction on $n$.
        \begin{itemize}
            \item \textbf{Base. } $n=1$. $f_{T_1}(t) = \lambda e^{-\lambda t}$ obviouly holds.
            \item \textbf{Hypothesis. } Suppose it holds for $n$.
            \item \textbf{Step. }
            \begin{align*}
                f_{T_{n+1}}(t) &= \int_0^t f_{T_n}(s)\cdot\lambda e^{-\lambda(t-s)}\mathrm{d}s\\
                &= \int_0^t \lambda e^{-\lambda s}\frac{(\lambda s)^{n-1}}{(n-1)!}\lambda e^{-\lambda(t-s)}\mathrm{d}s \quad \text{(Plug in the hypothesis)}\\
                &= \frac{\lambda^{n+1}}{(n-1)!}e^{-\lambda t}\int_0^t s^{n-1}\mathrm{d}s \quad \text{(Move the constants out)}\\
                &= \frac{\lambda^{n+1}}{n!}e^{-\lambda t}t^n
            \end{align*}
            which is exactly $\Gamma(n+1, \lambda(n+1))$, so we are done.
        \end{itemize}
    \end{proof}
    \begin{remark}
        Theorem \ref{thm:SumOfExpHasGammaDistribution} states that the arrival time of the $n$-th customer has a Gamma distribution.
    \end{remark}

    We can now prove Proposition \ref{prop:AltdefOfPoissonProcess}.
    \subsection{Proof of Equivalence of the two definitions}
        \begin{proof}
            To show that the two definitions are equivalent, we need to show that the three requirements in definition \ref{def:PoissonProcess} can be satisfied by the alternative definition \ref{prop:AltdefOfPoissonProcess}.
            \begin{enumerate}
                \item $N(0)=0$ is trivial.
                \item We begin proving (2) of definition \ref{def:PoissonProcess} from the case $s=0$. Notice that $N(t)=n \Leftrightarrow T_n \le t \wedge T_{n+1} > t$, i.e. the $n$-th customer has arrived, but the $(n+1)$-th has not. So we only need to prove the latter has Poisson distribution.
                \begin{align*}
                    \mathbb{P}[N(t)=n] &= \mathbb{P}[T_n \le t, T_{n+1} > t]\\
                    &= \int_0^t\mathbb{P}[T_n = s]\cdot\mathbb{P}[\tau_{n+1} > t-s]\mathrm{d}s \quad \text{(Enumerate all possible values of $t$)}\\
                    &= \int_0^t\lambda e^{-\lambda s}\frac{(\lambda s)^{n-1}}{(n-1)!}\cdot e^{-\lambda(t-s)}\mathrm{d}s \quad \text{(Theorem \ref{thm:SumOfExpHasGammaDistribution})}\\
                    &= \frac{\lambda^n}{(n-1)!}e^{-\lambda t}\int_0^t s^{n-1}\mathrm{d}s \quad \text{(Move out the constants)}\\
                    &= \frac{(t\lambda)^n}{n!}\cdot e^{-\lambda t}
                \end{align*}
                which is exactly the expression of $Poisson(\lambda t)$, so we are done.

                When $s > 0$, by the lack of memory property, $N(t+s)-N(s)$ must have the same distribution as $N(t) - N(0) = N(t)$. Of course this can also be verified by some moderate calculations.
                \item (3) of definitiion \ref{def:PoissonProcess} can be proved using a similar argument. Notice that the lack-of-memory property and (2) implies that the ``number of arrivals'' after $s$ is independent of the arrivals before $s$. Therefore $N(t_n) - N(t_{n-1})$ is independent of $N(r)$ for all $r < t_{n-1}$. So the result can be proved by induction.
            \end{enumerate}
        \end{proof}

\section{Thinning}
    Thinning is associating another random variable with a Poisson Process.

    \subsection{Thinning}
        We can associate some i.i.d. random variables $Y_i$ with each arrival. For example, $Y_i$ can be the gender of the $i$-th customer arriving at some restaurant; or if customers arrive in cars, $Y_i$ can be the number of passengers in a car. Let $P_j = \mathbb{P}[Y_i = j]$. Let $N_j(t)$ be the total number of $i \le N(t)$ with $Y_i = j$.

        \begin{theorem}\label{thm:ThinningOfPoissonProcess}
            $N_j(t)$ are \emph{independent} Poisson Processes with rate $\lambda P_j$.
        \end{theorem}
        \begin{proof}
            We will prove the result using the first definition of Poisson Distribution \ref{def:PoissonProcess}. We only need to consider the simplest cases where $Y_i \in \{0,1\}$.
            \begin{align*}
                \mathbb{P}[N_0(t) = j, N_1(t) = k] &= e^{-\lambda t}\frac{(\lambda t)^{j+k}}{(j+k)!}\cdot C^{j+k}_j P_0^j P_1^k \\
                &= e^{-\lambda t}\frac{(\lambda t)^{j+k}}{(j+k)!}\frac{(j+k)!}{j! \cdot k!}P_0^j P_1^k\\
                &= e^{-P_0\lambda t}\frac{(P_0\lambda t)^j}{j!} \cdot e^{-P_1\lambda t} \frac{(P_1 \lambda t)^k}{k!}
            \end{align*}
            This is exactly the product of two Poisson Processes, with rate $P_0\lambda$ and $P_1\lambda$ respectively. Therefore we have completed our proof.
        \end{proof}
        \begin{remark}
            One tricky fact behind this theorem is that, suppose we are interested in the gender of customers arriving at a restaurant. Assume that the probability that the customer is male or female are equal. If one day 40 males came to the restaurant, this does not give any implication to the number of females coming to the restaurant because they are independent.
        \end{remark}

    \subsection{Examples}
        \subsubsection{Finding typos in a book.} Two editors read a 300-page manuscript. Editor $A$ finds $100$ typos, Editor $B$ finds $120$ typos, and $80$ of these typos are the same. Suppose $A$ and $B$ have probability $P_A$ and $P_B$ of discovering a typo, and suppose the typos in the book is a rate $\lambda$ Poisson Process. How can we estimate $\lambda$, $P_A$ and $P_B$?

        We can associate a random variable with each typos appearing in the manuscript.
        \begin{enumerate}
            \item Neither $A$ or $B$ found it. $(1-P_A)(1-P_B)$
            \item Only $A$ found it. $P_A(1-P_B)$
            \item Only $B$ found it. $P_B(1-P_A)$
            \item Both $A$ and $B$ found it. $P_AP_B$
        \end{enumerate}
        From Theorem \ref{thm:ThinningOfPoissonProcess}, we know that the four cases follow four independent Poisson Processes. So we can estimate the parameters by solving
        \[
        \begin{cases}
            300P_A(1-P_B)\lambda &= 20\\
            300P_B(1-P_B)\lambda &= 40\\
            \lambda P_AP_B &= 80
        \end{cases}
        \quad \Longrightarrow \quad
        \begin{cases}
            P_A &= 2/3\\
            P_B &= 4/5\\
            \lambda &= 1/2
        \end{cases}
        \]

        \subsubsection{Coupon Collector Once Again.}
        We revisit the Coupon Collector's problem, but this time we assume the probability of the $j$-th coupon is $P_j$.
        \begin{itemize}
            \item Let $N_j$ be the number of gachas before we first get a coupon of the $j$-th category.
            \item Let $N = \max_{1 \le j \le n}\{N_j\}$ is the total number of gachas we need to collect all categories of coupons.
        \end{itemize}
        We want to find $\mathbb{E}[N]$.

        We re-formulate the problem into the following: suppose we owns a restaurant, and the customers coming to the restaurant is a Poisson Process of rate $\lambda = 1$. Each customer brings a coupon, let $Y_j$ be the category of the coupon brought by the $j$-th customer. Therefore $N_j(t)$ is a Poisson Process with rate $P_j$. Let $X_j$ be the random variable \emph{in this Poisson Process setting} denoting the first time to meet a customer with a category $j$ coupon, and we now alternatively consider $X = \max \{X_j\}$.

        We are interested in $\mathbb{P}[X \le t]$, i.e. the probability that we have got all categories of coupons before time $t$.
        \begin{align*}
            \mathbb{P}[X \le t] &= \mathbb{P}[X_1 \le t, X_2 \le t, \dots X_n \le t]\\
            &= \mathbb{P}[X_1 \le t] \cdot \mathbb{P}[X_2 \le t] \cdot \cdots \cdot \mathbb{P}[X_n \le t]
        \end{align*}
        By the alternative definition of Poisson Process \ref{prop:AltdefOfPoissonProcess}, $X_j$ has an Exponential Distribution with rate $P_j$. Therefore
        \[ \mathbb{P}[X \le t] = \prod_{i=1}^n (1-e^{-P_jt}) \]
        \[ \mathbb{P}[X > t] = 1 - \prod_{i=1}^n (1-e^{-P_jt}) \]
        \begin{proposition}\label{prop:AltComputationOfExpectation}
            If $X \ge 0$, then
            \[ \mathbb{E}[X] = \sum_{t=0}^{\infty} \mathbb{P}[X \ge t] = \sum_{i=0}^{\infty}i\cdot\mathbb{P}[X=i] \]
            \[ \mathbb{E}[X] = \int_0^{\infty} \mathbb{P}[X > t]\mathrm{d}t\]
        \end{proposition}
        By Proposition \ref{prop:AltComputationOfExpectation},
        \[ \mathbb{E}[X] = \int_0^{\infty} \mathbb{P}[X > t]\mathrm{d}t = \int_0^{\infty}1 - \prod_i (1-e^{-P_it})\mathrm{d}t \]
        We have figured out $\mathbb{E}[X]$, but how is $\mathbb{E}[X]$ related to $\mathbb{E}[N]$?

        Notice that $X$ denotes the exact time at which the customer carrying the $N$-th coupon arrives. If $\tau_i$ denotes the time interval between the arrival of the $(i-1)$-th and the $i$-th customers, then
        \[ X = \sum_{i=1}^N \tau_i \]
        Take expectation on both sides
        \[ \mathbb{E}[X] = \mathbb{E}\left[ \sum_{i=1}^N \tau_i \right] = \mathbb{E}[N]\cdot\mathbb{E}[\tau_1] = \mathbb{E}[N] \]
        Notice that $N$ is a random variable, but we somehow still exchanged the summation and expectation in the second equation, this holds because of Wald's Equation, which will be covered in the future. For now, notice that once $N$ is determined, $\mathbb{E}[X|N]$ is determined.
        \[ \mathbb{E}[X|N] = N\mathbb{E}[\tau_1] = N \]
        Take expectations on both sides again
        \[ \mathbb{E}[\mathbb{E}[X|N]] = \mathbb{E}[X] = \mathbb{E}[N] \]
        \begin{remark}
            The original Coupon Collector's problem is a special case where $P_j$ are uniform. It can be proved\footnote{“我也不会积，但是我用Mathematica跑了一下他们确实相等” -- Chihao} that the integral equals to the sum of harmonic series.\footnote{“这个积分我会积” -- Kuan}\footnote{“不愧是常州数学帝” -- Chihao}
        \end{remark}

    \subsection{Nonhomogeneous Thinning}
        Now consider another case. Suppose $P_i$ now depends on the arrival time $s$.
        \begin{theorem}[Nonhomogeneous Thinning]
            \label{thm:NonhomogeneousThinning}
            Let $Y_1,\dots,Y_k$ be random variables, and let $p_1,\dots,p_k$ be nonnegative functions of time $s$ such that for all $s$, $\sum_{j=1}^k p_j(s)=1$. Assume $\mathbb{P}[Y_i=j]=p_j(s)$ at $s$. Then $N_i(t)$'s have \emph{independent} Poisson distribution with mean $\lambda\int_0^t P_i(s)\mathrm{d}s$.
        \end{theorem}
        \begin{proof}
            Suppose there are $k$ categories. Suppose until time $t$, each type $i$ have total number $n_i$.
            \begin{align*}
                &\mathbb{P}[N_1(t)=n_1,\dots,N_k(t)=n_k]\\
                &=\mathbb{P}\left[ N_1(t)=n_1,\dots,N_k(t)=n_k|N(t)=\sum_i n_i \right]\mathbb{P}\left[ N(t) = \sum_i n_i \right] \quad \text{Always holds}\\
                &= \frac{\left(\sum_{i=1}^kn_i\right)!}{n_1!\cdots n_k!}p_1^{n_1}\cdots p_k^{n_k} \cdot e^{-\lambda t}\frac{(\lambda t)^{\sum_in_i}}{(\sum_i n_i)!}\\
                &= \prod_{i=1}^k e^{-\lambda t P_i} \frac{(\lambda t P_i)^{n_i}}{n_i!}
            \end{align*}
            where
            \[ P_i = \frac{1}{t}\int_0^t P_i(s)\mathrm{d}s \]
        \end{proof}
        \begin{remark}
            Actually it is a non-homogeneous Poisson Process with rate
            \[ \lambda(t) = \lambda\cdot P_i(t) \]
        \end{remark}

        \begin{definition}[Nonhomogeneous Poisson Process]\label{def:NonhomogeneousPoissonProcess}
            $\{ N(t) | t \ge 0 \}$ is a \textbf{nonhomogeneous Poisson Process} with rate $\lambda(s)$ if
            \begin{enumerate}
                \item $N(0)=0$
                \item $N(t)$ has independent increments.
                \item $N(t)-N(s)$ has distribution $Poisson\left(\int_s^t \lambda(\tau)\mathrm{d}\tau\right)$
            \end{enumerate}
        \end{definition}
        \begin{remark}
            In this case, the time intervals are not exponential distributions, and they are not independent.
        \end{remark}

    \subsection{Applications of Nonhomogeneous Thinning}

        \subsubsection{M/$G$/$\infty$ Queue}
        Consider a queue at some counters in a bank. M stands for a Poisson Process representing the incoming flow of customers. $G$ is a distribution of the time to serve the customers, $G(t) = \mathbb{P}[ServiceTime \le t]$. $\infty$ is the number of counters. Let
        \begin{itemize}
            \item $X(t)$: the number of customers completed services before time $t$.
            \item $Y(t)$: the number of customers being served at time $t$.
        \end{itemize}

        Consider the customers arriving before $t$, there are two types
        \begin{itemize}
            \item Those who have completed their services before $t$. The number is $X(t)$.
            \item Those who are being served at $t$. The number is $Y(t)$.
        \end{itemize}

        Notice that
        \[ p_1(s) = G(t-s) \quad p_2(s) = 1 - G(t-s) = \bar{G}(t-s) \]
        Therefore
        \[ \mathbb{E}[X(t)] = \mathbb{E}[N_1(t)] = \lambda\int_0^t G(t-s)\mathrm{d}s = \lambda\int_0^t G(s)\mathrm{d}s \]
        \[ \mathbb{E}[Y(t)] = \mathbb{E}[N_2(t)] = \lambda\int_0^t \bar{G}(s)\mathrm{d}s \]

        \subsubsection{Cars in a Tunnel}
        Consider a tunnel\footnote{“刚才Kuan提醒我隧道内不能超车” -- Chihao} of length $l$. Suppose we are driving a car, once the car enters the tunnel, its speed cannot change. Assume other NPC cars enter the tunnel with a Poisson distribution of rate $\lambda$. Suppse the speeds of NPC cars have a distribution $G$, and are also fixed once an NPC car enters the tunnel.

        We want to choose an initial speed $x$ to minimize the time we 超车 or 被超车。

        Let $t_0 = l/x$. If we enter the tunnel at time $s$, then we must exit the tunnel at time $s + t_0$. For an NPC car with speed $X$, the time interval that the NPC car stays in the tunnel is $T=l/X$. Let $F$ be the distribution of $T$.

        \[ F(t) = \mathbb{P}[T < t] = \mathbb{P}[l/X < t] = \mathbb{P}[X > l/t] = \bar{G}[l/t] \]

        Let $t$ denote the time when an NPC car enters the tunnel. We divide NPC cars into $3$ categories:
        \begin{enumerate}
            \item 超车。$t > s$ and $t + T < s + t_0$
            \item 被超车。$t < s$ and $t + T > s + t_0$
            \item No thing happens. Omitted.
        \end{enumerate}
        At any given time $t$, let $p$ be the probability of overtaking or being overtaken, we have
        \[
            p(t) =
            \begin{cases}
                \mathbb{P}[t + T > s + t_0] = \bar{F}(s+t_0-t) \quad &(t < s)\\
                \mathbb{P}[t + T < s + t_0] = \bar{F}(s+t_0-t) \quad &(s < t < s+t_0)\\
                0 \quad &(t > s + t_0)
            \end{cases}
        \]
        Notice that only $T$ in the above equation is a random variable.
        \begin{align*}
            \mathbb{E}[N_{encountered}(s+t_0)] &= \lambda \left( \int_0^s \bar{F}(s-t_0-t)\mathrm{d}t + \int_s^{s+t_0} F(s+t_0-t)\mathrm{d}t \right)\\
            &= \lambda \left( \int_{t_0}^{s+t_0} \bar{F}(t)\mathrm{d}t + \int_0^{t_0} F(t)\mathrm{d}t \right)
        \end{align*}
        To maximize the expectation, we take the derivative w.r.t. $t_0$.
        \[ \frac{\mathrm{d}}{\mathrm{d}t_0} = \lambda \left( \bar{F}(s+t_0) - \bar{F}(t_0) + F(t_0) \right) = 0 \]
        If the system has been running for a long time, i.e. $s$ is sufficiently large, $\bar{F}(s+t_0) \approx 0$, so
        \[ F(t_0) = \bar{F}(t_0) = \frac{1}{2} \]
        and
        \[ G\left(\frac{l}{t_0}\right) = \frac{1}{2} \]
        So we choose an initial speed $x$ such that
        \[ G(x) = \frac{1}{2} \]

        \subsubsection{HIV Patients}
        We want to estimate the number of HIV patients, suppose a patient may or may not show symptoms. Assume the number of patients with HIV virus has a Poisson distribution with an unknown rate $\lambda$. Suppose we know a distribution $G$ representing ``how long a patient with HIV virus shows symptoms''.

        We divide people into 2 types by whether they show symptoms or not. Suppose the patient is infected at time $s$ and we observe if they have shown symptoms at time $t$.
        \begin{equation}\label{eq:NumOfPatientsWithSymptoms} \mathbb{E}[N_1(t)] = \lambda \int_0^t G(t-s)\mathrm{d}s = \lambda\int_0^t G(s)\mathrm{d}s \end{equation}
        \[ \mathbb{E}[N_2(t)] = \lambda \int_0^t \hat{G}(t-s)\mathrm{d}s = \lambda\int_0^t \hat{G}(s)\mathrm{d}s \]
        where $\lambda$ is unknown.

        However, we do know the number of patients who have already shown symptoms. Therefore we can estimate $\lambda$ with Eq (\ref{eq:NumOfPatientsWithSymptoms}), and estimate $\mathbb{E}[N_2(t)]$ with $\hat{\lambda}$.

\section{Conditioning}

    \subsection{Conditioning}
        Let $T_1, T_2, \dots, T_n$ be the arrival times of a Poisson Process with rate $\lambda$. Let $U_1, U_2, \dots, U_n$ be independent uniform random variables on $[0,t]$. Let $V_1, V_2, \dots, V_n$ be sequence $U_i$ re-arranged in increasing order, then
        \begin{theorem}[Conditioning]\label{thm:ConditioningOfPoissonProcess}
            If we condition on $N(t)=n$, then the distribution of $T_1, \dots, T_n$ is the same as the distribution of $V_1, \dots, V_n$.
        \end{theorem}
        \begin{sketchproof}~{}
            \begin{itemize}
                \item The joint distribution of $T_1,\dots,T_n$ given $N(t)=n$ is $n!/t^n$ (by brutal force calculation).
                \item The resulting distribution is uniform over $[0,t]$ because the space has volume $t^n$ and $n!$ possible orderings.
            \end{itemize}
        \end{sketchproof}
        \begin{remark}
            This property is useful when computing $\mathbb{P}[N(s)=m|N(t)=n]$
            \[ \mathbb{P}[N(s)=m|N(t)=n] = C^n_m \left(\frac{s}{t}\right)^m\left(1-\frac{s}{t}\right)^{n-m} \]
        \end{remark}

\section{Poisson Approximation}
    \subsection{Motivating Example: Max Load}\label{subs:m-balls-in-n-bins}
        Consider a m-ball-into-n-bin problem. Suppose we throw balls randomly into several bins, and we want to know how many balls there are in the bin with the most balls.

        If we consider the number of balls in each bin, the number follows a Bernoulli distribution. However, since the number of balls in different bins are not independent, analyzing the problem in this way can be complicated.

        Alternatively, we can analyze the problem by constructing a Poisson Process.

        \begin{itemize}
            \item Let $X_i$ be the number of balls in the $i$-th bin.
            \item $X_i \sim Bernoulli(m, \frac{1}{n})$.
            \item $\sum_i X_i = m$.
            \item We are interested in $X = \max X_i$.
        \end{itemize}

    \subsection{Poisson Approximation}
        \begin{theorem}[Poisson Approximation]\label{thm:PoissonApproximation}
            Let $(X_1,\dots,X_n)$ be a sequence of random variables where each $X_i$ has a Bernoulli distribution, then its distribution is the same as $(Y_1, \dots, Y_n)$ conditioned on $\sum_i Y_i = m$, where $Y_i \sim Poisson(\lambda)$ are \emph{independent} Poisson variables with rate $\lambda$.
        \end{theorem}
        \begin{proof}
            Let $a_1, \dots, a_n \ge 0$ be $n$ positive integers s.t. $\sum_i a_i = m$. We calculate $\mathbb{P}[X_1 = a_1, \dots, X_n = a_n]$ and $\mathbb{P}[Y_1 = a_1, \dots, Y_n = a_n|\sum_i Y_i=m]$ to show that they are equal to each other.

            Consider throwing $m$ balls into these $n$ bins. There are $n^m$ different kind of throws and $m!$ different permutations of different balls. However, the order of balls in the same bin does not matter, so we shoud divide $m!$ by $a_1!\cdot \cdots \cdot a_n!$.
            \begin{align*}
                \mathbb{P}[X_1 = a_1, \dots, X_n = a_n]
                &= \frac{1}{n^m} \cdot \frac{m!}{a_1! \dots a_n!}
            \end{align*}

            \begin{align*}
                \mathbb{P}[Y_1 = a_1, \dots, Y_n = a_n | \sum_iY_i = m]
                &= \frac{\mathbb{P}[Y_1=a_1,\dots,Y_n=a_n]}{\mathbb{P}[\sum_i Y_i = m]}\\
                &= \frac{\prod_{i=1}^n \mathbb{P}[Y_i = a_i]}{\mathbb{P}[\sum_i Y_i = m]} \quad \text{(Independence of $Y_i$)}\\
                &= \frac{\prod_{i=1}^n e^{-\lambda}\frac{\lambda^{a_i}}{a_i!}}{e^{-\lambda n} \cdot \frac{(\lambda n)^m}{m!}} \quad \text{(Denominator is a sum of Poisson)}\\
                &= \frac{1}{n^m} \cdot \frac{m!}{a_1! \dots a_n!} \quad \text{(Mafs)}
            \end{align*}

        \end{proof}

    \subsection{m-balls-in-n-bins: MaxLoad Revisited}
        With Theorem \ref{thm:PoissonApproximation}, we can solve the motivating example in Section \ref{subs:m-balls-in-n-bins}.
        \begin{theorem}
            Let $m=n$. Let $X = \max_i X_i$. Then there exist constants $c_1$, $c_2$ such that
            \[ \mathbb{P}\left[\frac{c_1\log n}{\log\log n} < X < \frac{c_2\log n}{\log\log n}\right] = 1 - O\left(\frac{1}{n}\right) \]
        \end{theorem}
        \begin{proof}
            We first proof the \emph{upper bound}. Proving the upper bound is equivalent to proving
            \[\mathbb{P}\left[ X \ge \frac{c_2\log n}{\log\log n} \right] = O\left(\frac{1}{n}\right) \]
            \begin{align*}
                \mathbb{P}\left[ X \ge \frac{c_2\log n}{\log\log n} \right] &= \mathbb{P}[\exists i: X_i \ge k]\\
                &\le n\mathbb{P}[X_1 \ge k] \quad \text{(Union Bound)}\\
                &\le n C^k_n\left(\frac{1}{n}\right)^k\\
                &\le n\cdot \left(\frac{en}{k}\right)^k n^{-k} \quad \text{(Mafs)}\\
                &= n\cdot\left(\frac{e}{k}\right)^k
            \end{align*}
            Therefore it is sufficient to show that $n\cdot e^k / k^k < 1/n^{1+\epsilon}$ for some $\epsilon > 0$. Take logarithm on both sides,
            \[ (2+\epsilon)\log n + k - k\log k < 0 \]
            Hence $k\log k > 3\log n$ suffices. Plug in $k = \frac{c_2\log n}{\log\log n}$.
            \begin{align*}
                k\log k &= \frac{c_2\log n}{\log\log n} (\log\log n - \log\log\log n + \log c_2)\\
                &> c_2\log n\left(1- \frac{\log\log\log n}{\log\log n}\right)\\
                &> \frac{c_2}{2}\log n
            \end{align*}
            So we can choose $c_2 = 6$.

            The proof of \emph{lower bound} requires Theorem \ref{thm:BadEventBound}. Let $X < \frac{c_1\log n}{\log\log n}$ be a ``bad event''. This is equivalent to
            \[ \forall i, \quad X_i < \frac{c_1\log n}{\log\log n} \]
            Denote this bad event by $\mathcal{B}$, let $f = \mathbb{I}[\mathcal{B}|X_1,\dots,X_n]$ be an indicator of the bad event. By Theorem \ref{thm:BadEventBound},
            \begin{align*}
                \mathbb{E}[f] &= \mathbb{P}[\mathcal{B}]\\
                &\le e\sqrt{n} \mathbb{E}[\mathbb{I}[\mathcal{B}|(Y_1,\dots,Y_n)]]\\
                &= e\sqrt{n} \mathbb{P}[\forall i \quad Y_i < \frac{c_1\log n}{\log\log n}] \quad \text{(Independence of $Y_i$)}\\
                &= e\sqrt{n}\prod_{i=1}^n \mathbb{P}[Y_i < RHS]\\
                &= e\sqrt{n}(1-\mathbb{P}[Y_i \ge RHS])^n\\
                &\le e\sqrt{n}(1-\mathbb{P}[Y_i = RHS])^n\\
                &= e\sqrt{n}\left(1-\frac{1}{ek!}\right)^n\\
                &\le e\sqrt{n} e^{-\frac{n}{ek!}} < \frac{1}{n} \quad \text{(Mafs)}
            \end{align*}
        \end{proof}
        \begin{theorem}\label{thm:BadEventBound}
            Let $X_i$ be Bernoulli random variables, and let $Y_i$ be independent Poisson random variables with rate $\lambda = m/n$. For any function $f$ from $\mathbb{N}^n$ to $\mathbb{N}$,
            \[ \mathbb{E}[f(X_1, \dots, X_n)] \le e\sqrt{m}\cdot\mathbb{E}[f(Y_1,\dots,Y_m)] \]
        \end{theorem}
        \begin{proof}
            \begin{align*}
                \mathbb{E}[f(Y_1,\dots,Y_n)] &= \sum_{k=0}^{\infty}\mathbb{E}[f(Y_1,\dots,Y_n) | \sum_i Y_i = k] \cdot \mathbb{P}[\sum_i Y_i = k] \quad \text{(Total Probability)}\\
                &\ge \mathbb{E}[f(Y_1,\dots,Y_n)|\sum Y_i = m]\cdot \mathbb{P}[\sum Y_i = m]\\
                &= \mathbb{E}[f(X_1,\dots,X_m)]\cdot\mathbb{P}[\sum Y_i = m] \quad \text{(thm \ref{thm:PoissonApproximation})}
            \end{align*}
            If we let each $Y_i$ has a Poisson distribution of rate $m/n$, by additivity of Poisson variables we have $\sum Y_i \sim Poisson(m)$, and then
            \begin{align*}
                \mathbb{E}[f(Y_1,\dots,Y_n)] &= \mathbb{E}[f(X_1,\dots,X_n)]\cdot e^{-m}\frac{m^m}{m!}\\
                &\ge \mathbb{E}[f(X_1,\dots,X_n)]\cdot\frac{1}{e\sqrt{m}}
            \end{align*}
        \end{proof}
        \begin{remark}~{}
            \begin{itemize}
                \item $f$ is usually chosen to be some indicator function $\mathbb{I}$ of some ``bad event''. (In order to show that the probability that this ``bad event'' happens is very small.)
                \item Choosing $\lambda = m/n$ is to maximize $\mathbb{P}[\sum_i Y_i=m]$.
            \end{itemize}
        \end{remark}


\newpage
\subsection{Goal}\label{subsec:goal}

The use of stochastic and metaheuristic algorithms (Simulated Annealing, Differential Evolution, Particle Swarm Optimization) in the tasks of unconstrained nonlinear optimization and the experimental comparison of them with Nelder-Mead and Levenberg-Marquardt algorithms.

\subsection{Formulation of the problem}\label{subsec:formulation-of-the-problem}

Generate the noisy data $(x_k, y_k)$, where $k = 0, \dots, 1000$, according to the rule:

\begin{equation*}
y = \begin{cases}
-100 + \delta_k, & f(x_k) < -100, \\
f(x_k) + \delta_k & -100 \leq f(x_k) \leq 100, \\
100 + \delta_k & f(x_k) > 100,
\end{cases}
\end{equation*}

\begin{equation*}
    x_k = \frac{3k}{1000},
\end{equation*}

\begin{equation*}
    f(x) = \frac{1}{x^2 - 3x + 2},
\end{equation*}

where $\delta_x \approx N(0, 1)$ are values of a random variable with standard normal distribution.
Approximate the data by the rational function

\begin{equation*}
    F(x, a, b, c, d) = \frac{ax + b}{x^2 + cx + d},
\end{equation*}

by means of least squares through the numerical minimization of the following function:

\begin{equation*}
    D(a, b, c, d) = \sum^{1000}_{k=0}(F(x_k, a, b, c, d) - y_k)^2.
\end{equation*}

To solve the minimization problem, use Nelder-Mead algorithm, Levenberg-Marquardt algorithm and at least two of the methods among Simulated Annealing, Differential Evolution and Particle Swarm Optimization.
If necessary, set the initial approximations and other parameters of the methods.
Use $\varepsilon = 0.001$ as the precision;
at most 1000 iterations are allowed.
Visualize the data and the approximants obtained in a single plot.
Analyze and compare the results obtained (in terms of number of iterations, precision, number of function evaluations, etc.).

\subsection{Brief theoretical part}\label{subsec:brief-theoretical-part}

Optimization methods are numerical methods for finding optimal (in some sense) values of objective functions, for example, in the framework of mathematical models of certain processes.

\textit{Stochastic algorithms} are a broad class of algorithms that rely on repeated random sampling to solve an optimization problem.
These methods are useful when it is impossible or difficult to apply others.

\textit{Metaheuristic algorithms} are algorithms inspired by natural phenomena that solve the optimization problem by trial and error.
Metaheuristic methods do not guarantee that a solution to the optimization problem will be found.

\paragraph{Nelder-Mead.}

If the function $f(x)$ is a ravine function, the efficiency of the simplex method in solving problem:
\begin{equation*}
    \min_{x \in R^n} f(x) = f(x*) = f*
\end{equation*}
is significantly reduced due to the fact that the regular simplex cannot be "extended" along the ravine.
The Nelder-Mead method (the deformable polyhedron method) is a development of the simplex method and uses the deformation of the current simplex (not necessarily regular) in the search process.
The method uses the following operations on simplexes:

\begin{itemize}
    \item reflection;
    \item reduction;
    \item compression;
    \item stretching.
\end{itemize}

\paragraph{Levenberg-Marquardt algorithm}

The \textit{Levenberg-Marquardt algorithm} (\textit{LMA}) is a popular trust region algorithm that is used to find a minimum of a function (either linear or nonlinear) over a space of parameters.
Essentially, a trusted region of the objective function is internally modeled with some function such as a quadratic.
When an adequate fit is found, the trust region is expanded.
As with many numerical techniques, the Levenberg-Marquardt method can be sensitive to the initial starting parameters.

\paragraph{Simulated annealing}
\textit{Simulated annealing} is a random-search technique which exploits an analogy between the way in  which a metal cools and freezes into a minimum energy crystalline structure (the annealing process) and  the search for a minimum in a more general system;
it forms the basis of an optimisation technique for combinatorial and other problems.

\paragraph{Differential evolution}
\textit{Differential evolution} is one of the most influential optimization algorithms up-to-date.
Differential evolution works through analogous computational steps as used by a standard evolutionary algorithm.

\subsection{Results}\label{subsec:results}

In the case of a \textit{rational approximation}, significant nonlinearities occur, so the result depends on the initial approximations (Figure~\ref{ris:plot}).

\begin{figure}[H]
    \center
    \includegraphics[width=\textwidth]{img/plot.png}
    \caption{Stochastic and metaheuristic algorithms.}
    \label{ris:plot}
\end{figure}

From the table below (Table,~\ref{tbl:stohmeta}) it is evident that methods \textit{Nelder-Mead} and \textit{Levenberg-Marquardt} require an order of magnitude less number of computations.
However, despite the fact that metaheuristic methods do not guarantee finding the result, they allow you to get a solution of fairly good quality in an acceptable time without knowing the search space.

Note that both metaheuristic methods produced extremely similar results for different numbers of iterations.

\begin{table}[ht]
\caption{Stochastic and metaheuristic algorithms}
\begin{tabular}{l|l|c|c|}
\cline{2-4}
                                                         & \textbf{method}                 & \multicolumn{1}{l|}{\textbf{iterations}} & \multicolumn{1}{l|}{\textbf{function eval}} \\ \hline
\multicolumn{1}{|l|}{\multirow{4}{*}{\textbf{rational}}} & \textit{Nelder-Mead}            & 480                                      & 795                                         \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & \textit{Levenberg-Marquardt}    & nan                                      & 3                                           \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & \textit{Simulated Annealing}    & 1000                                     & 9111                                        \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & \textit{Differential Evolution} & 149                                      & 9105                                        \\ \hline
\end{tabular}
\label{tbl:stohmeta}
\end{table}

\paragraph{Levenberg-Marquardt algorithm gives poor results}

\begin{theorem}
    \label{theorem:conv_newton}
    For doubly continuously differentiable functions (i.e.\ for $f \in C^2(D)$) with a non-degenerate matrix $\nabla^2 f(y^*)$, there exists a $\varepsilon$-neighborhood of the stationary point $y^*$ of the function $f(y)$ such that for any initial point $y^0$ from this neighborhood, the Newton method will converge superlinearly, and if the Lipschitz condition is met in this neighborhood for Hesse matrices, it will converge quadratically.
\end{theorem}

Since the Levenberg-Marquardt algorithm is a hybrid of gradient descent and the Newton method, it therefore has the disadvantages of the Newton method.
Based on the theorem above [1], we can conclude that if the Hesse matrix is not degenerate, but not sign-positive, then the Newton method can converge to a stationary point that is not a minimum point, but a maximum point or a saddle point.
Accordingly, a more careful choice of initial approximations is necessary.

\subsection{Conclusion}\label{subsec:conclusion}

In the course of the laboratory work, stochastic and metaheuristic algorithms were implemented and analyzed within the problem of the unconstrained optimization problem.
This paper also provides a comparative analysis of stochastic and metaheuristic algorithms.

\subsection{References}\label{subsec:references}
\begin{enumerate}[label={[\arabic*]}]
    \item С.Ю. Городецкий, Лабораторный практикум по методам локальной оптимизации в программной системе LocOpt, 2007.
\end{enumerate}

\subsection{Appendix}\label{subsec:appendix}

The source code is located \href{https://github.com/vanSultan/anal_dev_algo/tree/lab_04}{here}: \url{https://github.com/vanSultan/anal_dev_algo/tree/lab_04}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Written By Michael Brodskiy
% Class: Analytic Geometry & Calculus III (Math-292)
% Professor: V. Cherkassky
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[12pt]{article}
\usepackage{alphalph}
\usepackage[utf8]{inputenc}
\usepackage[russian,english]{babel}
\usepackage{titling}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{enumitem}
\usepackage{amssymb}
\usepackage[super]{nth}
\usepackage{everysel}
\usepackage{ragged2e}
\usepackage{geometry}
\usepackage{fancyhdr}
\geometry{top=1.0in,bottom=1.0in,left=1.0in,right=1.0in}
\newcommand{\subtitle}[1]{%
  \posttitle{%
    \par\end{center}
    \begin{center}\large#1\end{center}
    \vskip0.5em}%

}
\usepackage{hyperref}
\hypersetup{
colorlinks=true,
linkcolor=blue,
filecolor=magenta,
urlcolor=blue,
citecolor=blue,
}

\urlstyle{same}

\title{Lecture XII Notes}
\date{July 2, 2020}
\author{Michael Brodskiy\\ \small Professor: V. Cherkassky}

% Mathematical Operations:

% Sum: $$\sum_{n=a}^{b} f(x) $$
% Integral: $$\int_{lower}^{upper} f(x) dx$$
% Limit: $$\lim_{x\to\infty} f(x)$$

\begin{document}

\maketitle

\section{The Multivariable Chain Rule $-$ 14.5}

In three dimensions, the formula for the chain rule looks as follows:

$$\frac{dz}{dt}=\frac{\partial f}{\partial x}\frac{dx}{dt} + \frac{\partial f}{\partial y}\frac{dy}{dt}$$

Conditions $z=f(x,y); x=x(t), y=y(t)$\\

\textit{Example:}

$$z=x^2y+3xy^4; x=\sin2t,y=\cos t$$
$$\frac{dz}{dt}=(2xy+3y^4)(2\cos2t)+(x^2+12xy^3)(-\sin t)$$

Another case looks as follows:

If $z=f(x,y); x=g(s,t), y=h(s,t)$, then:

$$\frac{\partial z}{\partial s}=\frac{\partial z}{\partial x}\frac{\partial x}{\partial s} + \frac{\partial z}{\partial y}\frac{\partial y}{\partial s}$$

$$\frac{\partial z}{\partial t}=\frac{\partial z}{\partial x}\frac{\partial x}{\partial t} + \frac{\partial z}{\partial y}\frac{\partial y}{\partial t}$$

The general case of the chain rule, where $u$ is a multivariable function of $n$ variables $x_1$, $x_2$,\dots $x_n$, and each $x$ is a function of $m$ variables $t_1$,\dots$t_m$, then:

$$\frac{\partial u}{\partial t_i}=\frac{\partial u}{\partial x_1}\frac{\partial x_1}{\partial t_i}+\frac{\partial u}{\partial x_2}\frac{\partial x_2}{\partial t_i}+\dots+\frac{\partial u}{\partial x_n}\frac{\partial x_n}{\partial t_i}$$

\subsection{Implicit Differentiation}

If $\frac{\partial f}{\partial y}\neq 0$:

$$\frac{dy}{dx}=-\frac{\frac{\partial f}{\partial x}}{\frac{\partial f}{\partial y}}$$

In addition to this:

$$\frac{\partial z}{\partial x}=-\frac{\frac{\partial f}{\partial x}}{\frac{\partial f}{\partial z}}$$

and

$$\frac{\partial z}{\partial y}=-\frac{\frac{\partial f}{\partial y}}{\frac{\partial f}{\partial z}}$$

\section{The Directional Derivative $-$ 14.6}

The definition of a directional derivative is:

$$ D_{\hat{u}}=\lim_{h\to0} \frac{f(x_o+ha, y_o + hb)-f(x_o,y_o)}{h} $$

In more simplified terms:

$$D_{\hat{u}}=\frac{\partial f}{\partial x} a + \frac{\partial f}{\partial y} b$$

\subsection{The Gradient Vector}

The gradient vector is written $\overrightarrow{\nabla} f$, and:

$$\overrightarrow{\nabla} f = \frac{\partial f}{\partial x}\hat{\bold{i}} + \frac{\partial f}{\partial y}\hat{\bold{j}}$$

In the case that $u = f(x,y,z)$:

$$\overrightarrow{\nabla} f = \frac{\partial f}{\partial x}\hat{\bold{i}} + \frac{\partial f}{\partial y}\hat{\bold{j}}+\frac{\partial f}{\partial z}\hat{\bold{k}}$$

and

$$D_{\hat{u}}=\overrightarrow{\nabla}f(x,y,z)\cdot\hat{u}$$

The greatest value of the gradient, written as $|\overrightarrow{\nabla}f|$, occurs in the same direction for $\hat{\bold{u}}$ and $\overrightarrow{\nabla}f$

\end{document}