\section{Bootstrap}
The {\bf bootstrap} is a statistical method for estimating standard errors and confidence sets of statistics, such as estimators.

\subsection{Non-parametric Bootstrap for Confidence Sets}\label{S:NPBootstrap}
Let $T_n := T_n((X_1,X_2,\ldots,X_n))$ be a statistic, i.e.~any function of the data $X_1,X_2,\ldots,X_n \overset{\IID}{\sim} F^*$.  Suppose we want to know its variance $\V_{F^*}(T_n)$, which clearly depends on the fixed and possibly unknown DF $F^*$.

If our statistic $T_n$ is one with an analytically unknown variance, then we can use the bootstrap to estimate it.  The bootstrap idea has the following two basic steps:
\begin{enumerate}
\item[$\mathsf{Step~1}$:] Estimate $\V_{F^*}(T_n)$ with $\V_{\widehat{F}_n}(T_n)$.
\item[$\mathsf{Step~2}$:] Approximate $\V_{\widehat{F}_n}(T_n)$ using simulated data from the ``Bootstrap World.''
\end{enumerate}
For example, if $T_n=\overline{X}_n$, in $\mathsf{Step~1}$, $\V_{\widehat{F}_n}(T_n) = s_n^2/n$, where $s_n^2=n^{-1} \sum_{i=1}^n (x_i-\overline{x}_n)$ is the sample variance and $\overline{x}_n$ is the sample mean.  In this case,  $\mathsf{Step~1}$ is enough.  However, when the statistic $T_n$ is more complicated (e.g.~$T_n=\widetilde{X}_n = F^{[-1]}(0.5)$), the sample median, then we may not be able to find a simple expression for $\V_{\widehat{F}_n}(T_n)$ and may need $\mathsf{Step~2}$ of the bootstrap.

\begin{eqnarray}
\text{Real World Data come from} &  F^* \quad \implies X_1,X_2,\ldots,X_n & \implies
T_n((X_1,X_2,\ldots,X_n))=t_n \notag \\
\text{Bootstrap World Data come from} & \widehat{F}_n \quad  \implies X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n & \implies
T_n((X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n))=t^{\bullet}_n \notag
\end{eqnarray}

Observe that drawing an observation from the ECDF $\widehat{F}_n$ is equivalent to drawing one point at random from the original data (think of the indices $[n] := \{ 1,2,\ldots,n \}$ of the original data $X_1,X_2,\ldots,X_n$ being drawn according to the equi-probable $\demoivre(1/n,1/n,\ldots,1/n)$ RV on $[n]$).  Thus, to simulate $X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n$ from $\widehat{F}_n$, it is enough to drawn $n$ observations with replacement from $X_1,X_2,\ldots,X_n$.

In summary, the algorithm for Bootstrap Variance Estimation is:

\begin{enumerate}
\item[$\mathsf{Step~1}$:] Draw $X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n \sim \widehat{F}_n$
\item[$\mathsf{Step~2}$:] Compute $t_n^{\bullet} = T_n((X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n))$
\item[$\mathsf{Step~3}$:] Repeat $\mathsf{Step~1}$ and $\mathsf{Step~2}$ $B$ times, for some large $B$, say $B>1000$, to get $t_{n,1}^{\bullet}, t_{n,2}^{\bullet},\ldots,t_{n,B}^{\bullet}$
\item[$\mathsf{Step~4}$:] Several ways of estimating the bootstrap confidence intervals are possible:
\begin{enumerate}
\item The  $1-\alpha$ Normal-based bootstrap confidence interval is:
\[
C_n = [T_n - z_{\alpha/2} \widehat{se}_{boot}, T_n + z_{\alpha/2} \widehat{se}_{boot}] \ ,
\]
where the bootstrap-based standard error estimate is:
\[
\widehat{se}_{boot} = \sqrt{v_{boot}} = \sqrt{ \frac{1}{B} \sum_{b=1}^B \left(  t_{n,b}^{\bullet} - \frac{1}{B} \sum_{r=1}^B t_{n,r}^{\bullet} \right)^2}
\]
\item The $1-\alpha$ percentile-based bootstrap confidence interval is:\\
\[
C_n=[\widehat{G^{\bullet}}_{n}^{-1}(\alpha/2),\widehat{G^{\bullet}}_{n}^{-1}(1-\alpha/2)] ,
\]
where $\widehat{G^{\bullet}}_{n}$ is the empirical DF of the bootstrapped $t_{n,1}^{\bullet}, t_{n,2}^{\bullet},\ldots,t_{n,B}^{\bullet}$ and $\widehat{G^{\bullet}}_{n}^{-1}(q)$ is the $q^{\text{th}}$ sample quantile \eqref{E:qthSampleQuantile} of $t_{n,1}^{\bullet}, t_{n,2}^{\bullet},\ldots,t_{n,B}^{\bullet}$.
\end{enumerate}
\end{enumerate}

\begin{labwork} [Confidence Interval for Median Estimate of Inter Earth Quake Times]\label{LW:NZSIEQTimesMedianBootstrap}
Let us find the $95\%$ Normal-based bootstrap confidence interval as well as the  $95\%$ percentile-based bootstrap confidence interval for our plug-in estimate of the median of inter earth quake times from \hyperref[LW:PlugInEstimatesEarthQuakes]{Labwork~\ref*{LW:PlugInEstimatesEarthQuakes}} using the following script:
\VrbMf[label=NZSIEQTimesMedianBootstrap.m]{scripts/NZSIEQTimesMedianBootstrap.m}

We get the following output when we call the script file.
\begin{VrbM}
>> NZSIEQTimesMedianBootstrap
n =        6127
Medianhat =   25.5092
B =        1000
ans =        6127        1000
ConfInt95BootNormal =   24.4383   26.5800
ConfInt95BootPercentile =   24.4057   26.4742
\end{VrbM}
\end{labwork}

\begin{labwork}[Confidence Interval for Median Estimate of Web Login Data]
Find the $95\%$ Normal-based bootstrap confidence interval as well as the  $95\%$ percentile-based bootstrap confidence interval for our plug-in estimate of the median for each of the data arrays:
\[
\text{{\tt WebLogSeconds20071001035730} and {\tt WebLogSeconds20071002035730} .}
\]
Once again, the arrays can be loaded into memory by following the commands in the first $13$ lines of the script file {\tt WebLogDataProc.m} of \hyperref[DA:WebLogs]{Section~\ref*{DA:WebLogs}}.  Produce four intervals (two for each data-set).  Do the confidence intervals for the medians for the two days intersect?
\begin{VrbM}
>> WebLogDataProc % load in the data
>> Medianhat = median(WebLogSeconds20071001035730) % plug-in estimate of median
Medianhat =       37416
>> % store the length of data array
>> K=length(WebLogSeconds20071001035730)
K =       56485
>> B= 1000 % Number of Bootstrap replications
B =        1000
>> BootstrappedDataSet = WebLogSeconds20071001035730([ceil(K*rand(K,B))]);
>> size(BootstrappedDataSet) % dimension of the BootstrappedDataSet
ans =       56485        1000
>> BootstrappedMedians=median(BootstrappedDataSet); % get the statistic in Bootstrap world
>> % 95% Normal based Confidence Interval
>> SehatBoot = std(BootstrappedMedians); % std of BootstrappedMedians
>> % 95% C.I. for median from Normal approximation
>> ConfInt95BootNormal = [Medianhat-1.96*SehatBoot, Medianhat+1.96*SehatBoot]
ConfInt95BootNormal =    37242    37590
>> % 95% Percentile based Confidence Interval
ConfInt95BootPercentile = ...
    [qthSampleQuantile(0.025,sort(BootstrappedMedians)),...
    qthSampleQuantile(0.975,sort(BootstrappedMedians))]
ConfInt95BootPercentile =   37239    37554
\end{VrbM}
\end{labwork}

\begin{labwork}[Confidence interval for correlation]\label{LW:LSATGPACorrBoot}
Here is a classical data set used by Bradley Efron (the inventor of bootstrap) to illustrate the method.  The data are LSAT (Law School Admission Test in the U.S.A.) scores and GPA of fifteen individuals.

Thus, we have bivariate data of the form $(Y_i,Z_i)$, where $Y_i={\rm LSAT}_i$ and $Z_i={\rm GPA}_i$.  For example, the first individual had an LSAT score of  $y_1=576$ and a GPA of $z_1=3.39$ while the fifteenth individual had an LSAT score of $y_{15}=594$ and a GPA of $z_{15}=3.96$.  We supose that the bivariate data $(Y_i,Z_i) \overset{IID}{\sim} F^*$, such that $F^* \in \{ \text{all bivariate DFs} \}$.  This is a bivariate non-parametric experiment.  The bivariate data are plotted in Figure .

The law school is interested in the correlation between the GPA and LSAT scores:
$$
\theta^* = \frac{\int \int (y-\E(Y))(z-\E(Z))dF(y,z)}{\sqrt{\int (y-\E(Y))^2 dF(y) \int (z-\E(Z))^2 dF(z)}}
$$
The plug-in estimate of the population correlation $\theta^*$ is the sample correlation:
$$
\widehat{\Theta}_n = \frac{\sum_{i=1}^n(Y_i-\overline{Y}_n)(Z_i-\overline{Z}_n)}{\sqrt{\sum_{i=1}^n(Y_i-\overline{Y}_n)^2 \sum_{i=1}^n(Z_i-\overline{Z}_n)^2}}
$$
\VrbMf[label=LSATGPACorrBootstap.m]{scripts/LSATGPACorrBootstap.m}

We get the following output when we call the script file.
\begin{VrbM}
>> LSATGPACorrBootstap
SampleCorrelation =    0.5459
ConfInt95BootNormal =    0.1770    0.9148
ConfInt95BootPercentile =    0.2346    0.9296
\end{VrbM}

\begin{figure}
\caption{Data from Bradley EfronÕs LSAT and GPA scores for fifteen individuals (left).  The confidence interval of the sample correlation, the plug-in estimate of the population correlation, is obtained from the sample correlation of one thousand bootstrapped data sets (right).\label{F:LSATGPACorrBootstrap}}
\begin{center}
\makebox{\includegraphics[width=4.5in]{figures/LSATGPACorrBootstrap}}
\end{center}
\end{figure}

\end{labwork}

\subsection{Parametric Bootstrap for Confidence Sets}\label{S:PBootstrap}

The {\bf bootstrap} may also be employed for estimating standard errors and confidence sets of statistics, such as estimators, even in a parametric setting.  This is much easier than the the variance calculation based on Fisher Information and/or the Delta method.

The only difference in the {\bf parametric bootstrap} as opposed to the {\bf non-parametric bootstrap} we saw earlier is that our statistic of interest $T_n := T_n((X_1,X_2,\ldots,X_n))$ is a function of the data:
$$X_1,X_2,\ldots,X_n \overset{\IID}{\sim} F(x;\theta^*) \enspace .$$
That is, our data come from a parametric distribution $F(x;\theta^*)$ and we want to know the variance of our statistic $T_n$, i.e.~$\V_{\theta^*}(T_n)$.

The parametric bootstrap concept has the following two basic steps:
\begin{enumerate}
\item[$\mathsf{Step~1}$:] Estimate $\V_{\theta^*}(T_n)$ with $\V_{\widehat{\theta}_n}(T_n)$, where $\widehat{\theta}_n$ is an estimate of $\theta^*$ based on maximum likelihood or the method of moments.
\item[$\mathsf{Step~2}$:] Approximate $\V_{\widehat{\theta}_n}(T_n)$ using simulated data from the ``Bootstrap World.''
\end{enumerate}
For example, if $T_n=\overline{X}_n$, the sample mean, then in $\mathsf{Step~1}$, $\V_{\widehat{\theta}_n}(T_n) = n^{-1} \sum_{i=1}^n (x_i-\overline{x}_n)$ is the sample variance.  Thus, in this case,  $\mathsf{Step~1}$ is enough.  However, when the statistic $T_n$ is more complicated, say $T_n=\widetilde{X}_n = F^{[-1]}(0.5)$, the sample median, then we may not be able to write down a  simple expression for $\V_{\widehat{\theta}_n}(T_n)$ and may need $\mathsf{Step~2}$ of the bootstrap.
\begin{eqnarray}
{\scriptsize \text{Real World Data come from}} &  F(\theta^*) \quad \implies X_1,X_2,\ldots,X_n & \implies
T_n((X_1,X_2,\ldots,X_n))=t_n \notag \\
{\scriptsize \text{Bootstrap World Data come from}} & F(\widehat{\theta}_n) \quad  \implies X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n & \implies
T_n((X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n))=t^{\bullet}_n \notag
\end{eqnarray}
To simulate $X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n$ from $F(\widehat{\theta}_n)$, we must have a simulation algorithm that allows us to draw IID samples from $F(\theta)$, for instance the inversion sampler.  In summary, the algorithm for Bootstrap Variance Estimation is:
\begin{enumerate}
\item[$\mathsf{Step~1}$:] Draw $X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n \sim F(\widehat{\theta}_n)$
\item[$\mathsf{Step~2}$:] Compute $t_n^{\bullet} = T_n((X^{\bullet}_1,X^{\bullet}_2,\ldots,X^{\bullet}_n))$
\item[$\mathsf{Step~3}$:] Repeat $\mathsf{Step~1}$ and $\mathsf{Step~2}$ $B$ times, for some large $B$, say $B\geq1000$, to get $t_{n,1}^{\bullet}, t_{n,2}^{\bullet},\ldots,t_{n,B}^{\bullet}$
\item[$\mathsf{Step~4}$:] We can estimate the bootstrap confidence intervals in several ways:
\begin{enumerate}
\item The  $1-\alpha$ normal-based bootstrap confidence interval is:
\[
C_n = [T_n - z_{\alpha/2} \widehat{se}_{boot}, T_n + z_{\alpha/2} \widehat{se}_{boot}] \ ,
\]
where the bootstrap-based standard error estimate is:
\[
\widehat{se}_{boot} = \sqrt{v_{boot}} = \sqrt{ \frac{1}{B} \sum_{b=1}^B \left(  t_{n,b}^{\bullet} - \frac{1}{B} \sum_{r=1}^B t_{n,r}^{\bullet} \right)^2}
\]
\item The $1-\alpha$ percentile-based bootstrap confidence interval:\\
\[
C_n=[\widehat{G^{\bullet}}_{n}^{-1}(\alpha/2),\widehat{G^{\bullet}}_{n}^{-1}(1-\alpha/2)] ,
\]
where $\widehat{G^{\bullet}}_{n}$ is the empirical DF of the bootstrapped $t_{n,1}^{\bullet}, t_{n,2}^{\bullet},\ldots,t_{n,B}^{\bullet}$ and $\widehat{G^{\bullet}}_{n}^{-1}(q)$ is the $q^{\text{th}}$ sample quantile \eqref{E:qthSampleQuantile} of $t_{n,1}^{\bullet}, t_{n,2}^{\bullet},\ldots,t_{n,B}^{\bullet}$.
\end{enumerate}
\end{enumerate}

Let us apply the bootstrap method to the previous problem of estimating the standard error of the coefficient of variation from $n=100$ samples from $\normal(100,10^2)$ RV.  The confidence intervals from bootstrap-based methods are similar to those from the Delta method.
{\VrbMf[label=CoeffOfVarNormalBoot.m]{scripts/CoeffOfVarNormalBoot.m}}
\begin{VrbM}
>> CoeffOfVarNormal
Muhat =  100.3117
Sigmahat =   10.9800
Psihat =    0.1095
Sehat =    0.0077
ConfInt95DeltaMethod =    0.0943    0.1246
ConfInt95BootNormal =    0.0943    0.1246
ConfInt95BootPercentile =    0.0946    0.1249
\end{VrbM}


\section{The Chain Rule}\label{sec:ChainRule}
Let $h(x)=\sqrt{625-x^2}$.
The rules stated previously do not allow us to find $h'(x)$.
However, $h(x)$ is a composition of two functions.
Let $f(x)=\sqrt x$ and $g(x)=625-x^2$.
Then we see that
$$h(x)=(f\circ g)(x).$$
From our rules we know that $f'(x)=\frac{1}{2}x^{-1/2}$ and $g'(x)=-2x$, thus it would be convenient to have a rule which allows us to differentiate $f\circ g$ in terms of $f'$ and $g'$.
This gives rise to the chain rule.

\begin{formulabox}[The Chain Rule]
If $g$ is differentiable at $x$ and $f$ is differentiable at $g(x)$, then the composite function $h=f\circ g$ [recall $f\circ g$ is defined as $f(g(x))$] is differentiable at $x$ and $h'(x)$ is given by:
$$h'(x)=f'(g(x))\cdot g'(x).$$
\end{formulabox}

The chain rule has a particularly simple expression if we use the
Leibniz notation for the derivative. The quantity $f'(g(x))$ is the
derivative of $f$ with $x$ replaced by $g$; this can be written
$df/dg$. As usual, $g'(x)=dg/dx$. Then the chain rule becomes
$${df\over dx} = {df\over dg}{dg\over dx}.$$
This looks like trivial arithmetic, but it is not: $dg/dx$ is not a
fraction, that is, not literal division, but a single symbol that
means $g'(x)$. Nevertheless, it turns out that what looks like trivial
arithmetic, and is therefore easy to remember, is really true.

It will take a bit of practice to make the use of the chain rule come
naturally---it is more complicated than the earlier differentiation
rules we have seen.

\begin{example}{Chain Rule}{ChainRule}
Compute the derivative of $\ds \sqrt{625-x^2}$.
\end{example}

\begin{solution}
 We already know that the
answer is $\ds -x/\sqrt{625-x^2}$, computed directly from the limit. In
the context of the chain rule, we have $\ds f(x)=\sqrt{x}$,
$\ds g(x)=625-x^2$. We know that $\ds f'(x)=(1/2)x^{-1/2}$, so $\ds f'(g(x))=
(1/2)(625-x^2)^{-1/2}$. Note that this is a two step computation:
first compute $f'(x)$, then replace $x$ by $g(x)$. Since $g'(x)=-2x$
we have
$$f'(g(x))g'(x)={1\over 2\sqrt{625-x^2}}(-2x)={-x\over
    \sqrt{625-x^2}}.
$$
\end{solution}

\begin{example}{Chain Rule}{ChainRule2}
Compute the derivative of $\ds 1/\sqrt{625-x^2}$.
\end{example}

\begin{solution}
This is a quotient with
a constant numerator, so we could use the quotient rule, but it is
simpler to use the chain rule. The function is $\ds (625-x^2)^{-1/2}$, the
composition of $\ds f(x)=x^{-1/2}$ and $\ds g(x)=625-x^2$. We compute
$\ds f'(x)=(-1/2)x^{-3/2}$ using the power rule, and then
$$f'(g(x))g'(x)={-1\over 2(625-x^2)^{3/2}}(-2x)={x\over (625-x^2)^{3/2}}.$$
\end{solution}

In practice, of course, you will need to use more than one of the
rules we have developed to compute the derivative of a complicated
function.

\begin{example}{Derivative of Quotient}{DerivativeQuotient}
Compute the derivative of $$f(x)={x^2-1\over x\sqrt{x^2+1}}.$$
\vspace{-0.5cm}
\end{example}

\begin{solution}
The ``last'' operation here is division, so to get started we need to
use the quotient rule first. This gives
\begin{eqnarray*}
f'(x)&=&{(x^2-1)'x\sqrt{x^2+1}-(x^2-1)(x\sqrt{x^2+1})'\over
x^2(x^2+1)}\cr
&=&{2x^2\sqrt{x^2+1}-(x^2-1)(x\sqrt{x^2+1})'\over
x^2(x^2+1)}.\cr
\end{eqnarray*}
Now we need to compute the derivative of $\ds x\sqrt{x^2+1}$. This is a
product, so we use the product rule:
$${d\over dx}x\sqrt{x^2+1}=x{d\over dx}\sqrt{x^2+1}+\sqrt{x^2+1}.$$
Finally, we use the chain rule:
$${d\over dx}\sqrt{x^2+1}={d\over dx}(x^2+1)^{1/2}=
{1\over 2}(x^2+1)^{-1/2}(2x)={x\over \sqrt{x^2+1}}.$$
And putting it all together:
\begin{eqnarray*}
f'(x)&=&{2x^2\sqrt{x^2+1}-(x^2-1)(x\sqrt{x^2+1})'\over
x^2(x^2+1)}.\cr
&=&{2x^2\sqrt{x^2+1}-(x^2-1)\left(x{\ds{x\over \sqrt{x^2+1}}}
+\sqrt{x^2+1}\right)\over
x^2(x^2+1)}.\cr
\end{eqnarray*}
This can be simplified of course, but we have done all the calculus,
so that only algebra is left.
\end{solution}

\begin{example}{Chain of Composition}{ChainComposition}
Compute the derivative of $\ds \sqrt{1+\sqrt{1+\sqrt{x}}}$.
\end{example}

\begin{solution}
Here we have a
more complicated chain of compositions, so we use the chain rule
twice.
At the outermost ``layer'' we have the function
$\ds g(x)=1+\sqrt{1+\sqrt{x}}$ plugged into $\ds f(x)=\sqrt{x}$, so applying
the chain rule once gives
$${d\over dx}\sqrt{1+\sqrt{1+\sqrt{x}}}=
{1\over 2}\left(1+\sqrt{1+\sqrt{x}}\right)^{-1/2}{d\over dx}
\left(1+\sqrt{1+\sqrt{x}}\right).$$
Now we need the derivative of $\ds \sqrt{1+\sqrt{x}}$. Using the chain
rule again:
$${d\over dx}\sqrt{1+\sqrt{x}}={1\over
  2}\left(1+\sqrt{x}\right)^{-1/2}{1\over 2}x^{-1/2}.$$
So the original derivative is
\begin{eqnarray*}
{d\over dx}\sqrt{1+\sqrt{1+\sqrt{x}}}&=&
{1\over 2}\left(1+\sqrt{1+\sqrt{x}}\right)^{-1/2}
{1\over
  2}\left(1+\sqrt{x}\right)^{-1/2}{1\over 2}x^{-1/2}.\cr
&=&{1\over 8 \sqrt{x}\sqrt{1+\sqrt{x}}\sqrt{1+\sqrt{1+\sqrt{x}}}}
\end{eqnarray*}
\end{solution}

Using the chain rule, the power rule, and the product rule, it is
possible to avoid using the quotient rule entirely.

\begin{example}{Derivative of Quotient without Quotient Rule}{DerivativeQuotientWithoutQuotientRule}
Compute the derivative of $\ds f(x)={x^3\over x^2+1}$.
\end{example}

\begin{solution}
Write
$\ds f(x)=x^3(x^2+1)^{-1}$, then
\begin{eqnarray*}
f'(x)&=&x^3{d\over dx}(x^2+1)^{-1}+3x^2(x^2+1)^{-1}\cr
\\
&=&x^3(-1)(x^2+1)^{-2}(2x)+3x^2(x^2+1)^{-1}\cr
\\
&=&-2x^4(x^2+1)^{-2}+3x^2(x^2+1)^{-1}\cr
\\
&=&{-2x^4\over (x^2+1)^{2}}+{3x^2\over x^2+1}\cr
\\
&=&{-2x^4\over (x^2+1)^{2}}+{3x^2(x^2+1)\over (x^2+1)^{2}}\cr
\\
&=&{-2x^4+3x^4+3x^2\over (x^2+1)^{2}}={x^4+3x^2\over (x^2+1)^{2}}
\end{eqnarray*}
Note that we already had the derivative on the second line; all the
rest is simplification. It is easier to get to this answer by using
the quotient rule, so there's a trade off: more work for fewer
memorized formulas.
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for Section \ref{sec:ChainRule}}

\begin{enumialphparenastyle}

Find the derivatives of the functions. For extra practice, and to
check your answers, do some of these in more than one way if
possible.

\begin{ex}  $\ds x^4-3x^3+(1/2)x^2+7x-\pi$
\begin{sol}
$\ds 4x^3-9x^2+x+7$
\end{sol}
\end{ex}

\begin{ex}  $\ds x^3-2x^2+4\sqrt{x}$
\begin{sol}
$\ds 3x^2-4x+2/\sqrt{x}$
\end{sol}
\end{ex}

\begin{ex}  $\ds (x^2+1)^3$
\begin{sol}
$\ds 6(x^2+1)^2x$
\end{sol}
\end{ex}

\begin{ex}  $\ds x\sqrt{169-x^2}$
\begin{sol}
$\ds \sqrt{169-x^2}-x^2/\sqrt{169-x^2}$
\end{sol}
\end{ex}

\begin{ex}  $\ds (x^2-4x+5)\sqrt{25-x^2}$
\begin{sol}
$\ds  (2x-4)\sqrt{25-x^2}-$\hfill\break$(x^2-4x+5)x/\sqrt{25-x^2}$
\end{sol}
\end{ex}

\begin{ex}  $\ds \sqrt{r^2-x^2}$, $r$ is a constant
\begin{sol}
$\ds -x/\sqrt{r^2-x^2}$
\end{sol}
\end{ex}

\begin{ex}  $\ds \sqrt{1+x^4}$
\begin{sol}
$\ds 2x^3/\sqrt{1+x^4}$
\end{sol}
\end{ex}

\begin{ex}  $\ds {1\over\sqrt{5-\sqrt{x}}}$.
\begin{sol}
$\ds{1\over 4\sqrt{x}(5-\sqrt{x})^{3/2}}$
\end{sol}
\end{ex}

\begin{ex}  $\ds (1+3x)^2$
\begin{sol}
$\ds  6+18x$
\end{sol}
\end{ex}

\begin{ex}  $\ds{(x^2+x+1)\over(1-x)}$
\begin{sol}
$\ds {2 x + 1\over1 - x }+{x^2  + x + 1\over(1 - x)^2}$
\end{sol}
\end{ex}

\begin{ex}  $\ds{\sqrt{25-x^2}\over x}$
\begin{sol}
$\ds  -1/\sqrt{25-x^2}-\sqrt{25-x^2}/x^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds\sqrt{{169\over x}-x}$
\begin{sol}
$\ds{1\over2}\left({-169\over x^2}-1\right)\Big/\sqrt{{169\over x}-x}$
\end{sol}
\end{ex}

\begin{ex}  $\ds \sqrt{x^3-x^2-(1/x)}$
\begin{sol}
$ \ds{3x^2-2x+1/x^2\over 2\sqrt{x^3-x^2-(1/x)}}$
\end{sol}
\end{ex}

\begin{ex}  $\ds 100/(100-x^2)^{3/2}$
\begin{sol}
$ \ds{300 x \over(100-x^2)^{5/2}}$
\end{sol}
\end{ex}

\begin{ex}  $\ds {\root 3 \of{x+x^3}}$
\begin{sol}
$ \ds{ 1 + 3 x^2\over3(x+x^3)^{2/3}}$
\end{sol}
\end{ex}

\begin{ex}  $\ds \sqrt{(x^2+1)^2+\sqrt{1+(x^2+1)^2}}$
\begin{sol}
$\ds \left(4x(x^2+1)+{4x^3+4x\over2\sqrt{1+(x^2+1)^2}}\right)\Big/$\hfill\break$2\sqrt{(x^2+1)^2+\sqrt{1+(x^2+1)^2}}$
\end{sol}
\end{ex}

\begin{ex}  $\ds (x+8)^5$
\begin{sol}
$\ds 5(x+8)^4$
\end{sol}
\end{ex}

\begin{ex}  $\ds (4-x)^3$
\begin{sol}
$\ds -3(4-x)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds (x^2+5)^3$
\begin{sol}
$\ds 6x(x^2+5)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds (6-2x^2)^3$
\begin{sol}
$\ds -12x(6-2x^2)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds (1-4x^3)^{-2}$
\begin{sol}
$\ds 24x^2(1-4x^3)^{-3}$
\end{sol}
\end{ex}

\begin{ex}  $\ds 5(x+1-1/x)$
\begin{sol}
$\ds 5+5/x^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds 4(2x^2-x+3)^{-2}$
\begin{sol}
$\ds -8(4x-1)(2x^2-x+3)^{-3}$
\end{sol}
\end{ex}

\begin{ex}  $\ds {1\over 1+1/x}$
\begin{sol}
$\ds 1/(x+1)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds {-3\over 4x^2-2x+1}$
\begin{sol}
$\ds 3(8x-2)/(4x^2-2x+1)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds (x^2+1)(5-2x)/2$
\begin{sol}
$\ds -3x^2+5x-1$
\end{sol}
\end{ex}

\begin{ex}  $\ds (3x^2+1)(2x-4)^3$
\begin{sol}
$\ds 6x(2x-4)^3+6(3x^2+1)(2x-4)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds{x+1\over x-1}$
\begin{sol}
$\ds -2/(x-1)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds{x^2-1\over x^2+1}$
\begin{sol}
$\ds 4x/(x^2+1)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds{(x-1)(x-2)\over x-3}$
\begin{sol}
$\ds (x^2-6x+7)/(x-3)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds{2x^{-1}-x^{-2}\over 3x^{-1}-4x^{-2}}$
\begin{sol}
$\ds -5/(3x-4)^2$
\end{sol}
\end{ex}

\begin{ex}  $\ds 3(x^2+1)(2x^2-1)(2x+3)$
\begin{sol}
$\ds 60x^4+72x^3+18x^2+18x-6$
\end{sol}
\end{ex}

\begin{ex}  $\ds{1\over (2x+1)(x-3)}$
\begin{sol}
$\ds (5-4x)/((2x+1)^2(x-3)^2)$
\end{sol}
\end{ex}

\begin{ex}  $\ds ((2x+1)^{-1}+3)^{-1}$
\begin{sol}
$\ds 1/(2(2+3x)^2)$
\end{sol}
\end{ex}

\begin{ex}  $\ds (2x+1)^3(x^2+1)^2$
\begin{sol}
$\ds 56x^6+72x^5+110x^4+100x^3+60x^2+28x+6$
\end{sol}
\end{ex}

\begin{ex}   Find an equation for the tangent line to
$\ds f(x) = (x-2)^{1/3}/(x^3 + 4x - 1)^2$ at $x=1$.
\begin{sol}
$y=23x/96-29/96$
\end{sol}
\end{ex}

\begin{ex}  Find an equation for the tangent line to $\ds y=9x^{-2}$ at $(3,1)$.
\begin{sol}
$y=3-2x/3$
\end{sol}
\end{ex}

\begin{ex}  Find an equation for the tangent line to $\ds (x^2-4x+5)\sqrt{25-x^2}$
at $(3,8)$.
\begin{sol}
$y=13x/2-23/2$
\end{sol}
\end{ex}

\begin{ex}  Find an equation for the tangent line to $\ds \ds{(x^2+x+1)\over(1-x)}$
at $(2,-7)$.
\begin{sol}
$y=2x-11$
\end{sol}
\end{ex}

\begin{ex}  Find an equation for the tangent line to
$\ds \sqrt{(x^2+1)^2+\sqrt{1+(x^2+1)^2}}$
at $\ds (1,\sqrt{4+\sqrt{5}})$.
\begin{sol}
$\ds y={20+2\sqrt5\over5\sqrt{4+\sqrt5}}\,x+{3\sqrt5\over5\sqrt{4+\sqrt5}}$
\end{sol}
\end{ex}

\begin{ex}
Let $y=f(x)$ and $x=g(t)$. If $g(1)=2$, $f(2)=3$, $g'(1)=4$ and $f'(2)=5$, find the derivative of $f\circ g$ at 1.
\begin{sol}
	$(f(g(1)))'=20$
\end{sol}
\end{ex}

\begin{ex}
Express the derivative of $g(x)=x^2f(x^2)$ in terms of $f$ and the derivative of $f$.
\begin{sol}
	$g'(x)=2x(f(x^2)+x^2f'(x^2))$
\end{sol}
\end{ex}

\end{enumialphparenastyle}


\subsection{Compact spaces}\label{subsec:compact_spaces}

\begin{definition}\label{def:centered_family}\mcite[123]{Engelking1989}
  The nonempty family \( \mscrF \) of subsets of the topological space \( X \) is said to be a \term{centered family of sets} or to have the \term{finite intersection property} if the intersection \( F_1 \cap \cdots \cap F_n \) of any finite collection of sets is nonempty.
\end{definition}

\begin{definition}\label{def:compact_space}\mcite[123]{Engelking1989}
  The space \( X \) is called \term{compact} if any of the following equivalent finiteness conditions hold:
  \begin{thmenum}
    \thmitem{def:compact_space/finite_subcover} Every open cover of \( X \) has a finite subcover.
    \thmitem{def:compact_space/centered_family} Every centered \hyperref[def:centered_family]{family} \( \mscrF \) of closed subsets of \( X \) has a nonempty intersection.
    \thmitem{def:compact_space/convergent_nets} Every \hyperref[def:topological_net]{net} has a cluster point or, \hyperref[thm:def:net_convergence/properties/cluster_point_iff_subnet_limit_point]{equivalently}, a \hyperref[def:net_convergence]{convergent} subnet. This property is also called \enquote{generalized sequential compactness} or, when restricted to sequences instead of general nets, simply \enquote{sequential compactness}.
  \end{thmenum}
\end{definition}
\begin{proof}
  \ImplicationSubProof{def:compact_space/finite_subcover}{def:compact_space/centered_family} Assume that every open cover of \( X \) has a finite subcover. Let \( \mscrF \) be a centered family of closed subsets of \( X \). Aiming at a contradiction, suppose that \( \bigcap \mscrF = \varnothing \). Then
  \begin{equation*}
    X
    =
    X \setminus \bigcap \mscrF
    =
    \bigcup_{F \in F} (X \setminus F),
  \end{equation*}
  which has a finite subcover indexed by, say, \( \mscrF' \subseteq F \). But \( \mscrF \) is a centered family and \( \bigcap \mscrF' \) is nonempty, hence
  \begin{equation*}
    X
    =
    \bigcup_{F \in F'} (X \setminus F)
    =
    X \setminus \bigcap \mscrF'
    \neq
    X.
  \end{equation*}

  The obtained contradiction shows that \( \bigcap \mscrF \) is nonempty.

  \ImplicationSubProof{def:compact_space/centered_family}{def:compact_space/finite_subcover} Assume that every centered family of closed sets has a nonempty intersection. Let \( \{ U_k \}_{k \in \mscrK} \) be an open cover of \( X \). By putting \( F_k \coloneqq U_k \) for all \( k \in \mscrK \), we obtain a family \( \{ F_k \}_{k \in \mscrK} \) of closed sets with an empty intersection. Therefore, it is not a centered family. Then there exists at least one finite subfamily \( \{ F_k \}_{k \in \mscrK'} \) with an empty intersection. The complement of this subfamily is then a finite cover of \( X \), which proves our statement.

  \ImplicationSubProof{def:compact_space/finite_subcover}{def:compact_space/convergent_nets} Assume that every open cover of \( X \) has a finite subcover. Fix a net \( \{ x_k \}_{k \in \mscrK} \subseteq X \).

  Aiming at a contradiction, suppose that the net has no cluster points. For any point \( x \in X \) and any neighborhood \( U_x \) of \( x \), the net is not frequently in \( U_x \). Obviously \( \{ U_x \}_{x \in X} \) is an infinite open cover of \( X \). Then it has a finite subcover indexed by, say, \( X' \subseteq X \).

  Therefore, every element of the net \( \{ x_k \}_{k \in \mscrK} \) if contained in one of the finitely many neighborhoods \( \{ U_x \}_{x \in X'} \) and the net itself is frequently in at least one of the neighborhoods.

  Thus, one of the finitely many points in \( X' \) is a cluster point of \( \{ x_k \}_{k \in \mscrK} \).

  \ImplicationSubProof{def:compact_space/convergent_nets}{def:compact_space/centered_family}\mcite[thm. 3.1.23]{Engelking1989} Assume that every net has a cluster point. Let \( \{ F_k \}_{k \in \mscrK} \) be a central family of closed sets.

  Denote by \( \mathcal C \) the family of all finite subsets of \( \mscrK \). For each \( C \in \mathcal C \), the set \( \bigcap_{c \in C} F_c \) is a finite intersection of members of a central family and is hence nonempty. Choose an element \( x_C \in \bigcap_{c \in C} F_c \) for each \( C \in \mathcal C \).

  If we order \( \mathcal C \) by reverse inclusion, that is, if \( C \leq C' \iff \bigcap_{c \in C'} F_c \subseteq \bigcap_{c \in C} F_c \), then the family \( \{ x_C \}_{C \in \mathcal C} \) becomes a net. Our assumption is that it has a cluster point, say \( x_0 \).

  It remains to show that \( x_0 \) belongs to the intersection of the centered family \( \{ F_k \}_{k \in \mscrK} \) itself. Fix an element \( F_0 \) of this family and denote by \( C_0 \) the singleton set \( \{ k_0 \} \in \mathcal C \). Because \( x_0 \) is a cluster point, for every neighborhood \( U \) of \( x_0 \), there exists an index \( C \in \mathcal C \) such that \( C \geq C_0 \) and \( x_C \in U \). Then \( x_C \in \bigcap_{c \in C} F_c \subseteq \bigcap_{c \in C_0} F_c = F_0 \).

  Therefore, \( F_0 \cap U \neq \varnothing \) for all neighborhoods \( U \) of \( x_0 \). By \fullref{thm:def:topological_closure_operator/properties/neighborhood_intersection}, \( x_0 \in F_0 \). Since \( F_0 \) was an arbitrary set from the centered family \( \{ F_k \}_{k \in \mscrK} \), we conclude that the intersection of the family is not empty. This proves the theorem.
\end{proof}

\begin{remark}\label{rem:precompact_set}
  If the closure of \( A \) is compact, we call \( A \) \term{relatively compact} or \term{precompact} (although the term \enquote{precompact} is also used for totally bounded sets, see \fullref{def:totally_bounded_set}).
\end{remark}

\begin{proposition}\label{thm:def:compact_space/properties}
  \hyperref[def:compact_space]{Compact spaces} have the following basic properties:
  \begin{thmenum}
    \thmitem{thm:def:compact_space/properties/closed} If \( X \) is any topological space and \( A \subseteq X \) is a compact subspace, then \( A \) is closed in \( X \).
    \thmitem{thm:def:compact_space/properties/closed_subspace} A closed subspace of a compact space is compact.
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:def:compact_space/properties/closed} If \( A \) is compact, by \fullref{def:compact_space/convergent_nets} every net in \( A \) has a cluster point in \( A \). By \fullref{thm:def:derived_set/properties/closed_iff_contains_all_cluster_points}, \( A \) is closed in \( X \).

  \SubProofOf{thm:def:compact_space/properties/closed_subspace} Let \( X \) be \hyperref[def:compact_space]{compact} and let \( X' \subseteq X \) be a closed subspace.

  Fix an open cover \( \{ U_k \}_{k \in \mscrK} \) of \( X' \). By definition of the subspace \hyperref[def:topological_subspace]{topology} for each \( U_k \) there exists a set \( V_k \) that is open in \( X \) and \( U_k = V_k \cap X' \). Since \( X' \) is closed, \( X \setminus X' \) is open and hence the family \( \{ V_k \}_{k \in \mscrK} \) together with \( X \setminus X' \) is an open cover of the space \( X \).

  By \fullref{def:compact_space/finite_subcover}, there exists a finite subcover \( \{ V_k \}_{k \in \mscrK'} \) that, along with \( X \setminus X' \), still covers \( X' \). Therefore, \( X' \) is also compact.
\end{proof}

\begin{theorem}[Tychonoff's product theorem]\label{thm:tychonoffs_product_theorem}\mcite[thm. 3.2.4]{Engelking1989}
  Let \( { (X_k, \mscrT_k) }_{k \in \mscrK} \) be a family of topological spaces. Their \hyperref[def:topological_product]{product} \( (\prod_{k \in \mscrK} X_k, \prod_{k \in \mscrK} \mscrT_k) \) is \hyperref[def:compact_space]{compact} if and only if \( (X_k, \mscrT_k) \) is compact for every \( k \in \mscrK \).

  In \hyperref[def:zfc]{\logic{ZF}} this theorem is equivalent to the \hyperref[def:zfc/choice]{axiom of choice} --- see \fullref{thm:axiom_of_choice_equivalences/tychonoff}.
\end{theorem}

\begin{theorem}[Weierstrass' extreme value theorem]\label{thm:weierstrass_extreme_value_theorem}\mcite[cor. 3.2.9]{Engelking1989}
  Let \( X \) be a compact topological space and let \( f: X \to \BbbR \) be a continuous function into the \hyperref[def:set_of_real_numbers]{real numbers}.

  Then \( f \) is \hyperref[def:metric_space/bounded_function]{bounded} and there exist \( m, M \in X \) such that
  \begin{balign*}
    f(m) = \min_{x \in X} f(x)
     &  &
    f(M) = \max_{x \in X} f(x).
  \end{balign*}
\end{theorem}

\begin{definition}\label{def:locally_compact_space}\mcite[148]{Engelking1989}
  A topological space is called \term{locally compact} if every point has a relatively compact neighborhood.
\end{definition}


\section{Principal Component Analysis(PCA)}
	Sometimes, the dimension of input data is very large, we need to reduce the dimension before analyzing it. The \textbf{Principal Component Analysis} is a popular method of dimensional reduction.

	Suppose $\mathbf x_1,...,\mathbf x_n\in\mathbb R^d$ are the input data ($d\gg 1$), and satisfy $\sum_{i=1}^{n}\mathbf x_i=\mathbf 0$. Write $\mathbf X = (\mathbf x_1,...,\mathbf x_n)$. We want to find a subspace $\mathbb F$ (dim$\mathbb F$ = k) of $\mathbb{R}^d$ such that each projections of $\mathbf x_i$ in $\mathbb F$ is as close as possible to $\mathbf x_i$. The effect of PCA on MNist Dataset is showed in Figure \ref{fig:mnistpca}.

	\begin{figure}[h]
		\centering
		\includegraphics[width=0.7\linewidth]{MNistPCA}
		\caption{The effect of PCA on MNist Dataset, where $\dim \mathbb F=3$}
		\label{fig:mnistpca}
	\end{figure}

\subsection{The Case of k=1: The First Component}

	In the 1-d case, the subspace $\mathbb F$ is a line which goes through the original point in $\mathbb R^d$, let's write the direction vector of $\mathbb F$ as $\mathbf w$, where $\|\mathbf w\|_2=1$. Then $\mathbb F=\text{span}\{\mathbf w\}$ $\sum_{i=1}^N x_i=0$. The projections of $\mathbf x_1,...,\mathbf x_n$ in $\mathbb F$ is $\mathbf y_1,...,\mathbf y_n$, i.e.
	\begin{align}
	y_i&=\alpha_i \mathbf w \\\notag
	(y_i,\mathbf w)&=(x_i ,\mathbf w) \\\notag
	\alpha_i &=(x_i ,\mathbf w)=\mathbf w^T\mathbf x_i \\\notag
	\end{align}

	 $$\mathbf y_i=(\mathbf w^T\mathbf x_i)\mathbf w=\mathbf w \mathbf w^T\mathbf x_i.$$
	  So we can easily get
	$$\sum_{i=1}^{n}\mathbf y_i=\sum_{i=1}^{n}(\mathbf w^T\mathbf x_i)\mathbf w=\sum_{i=1}^{n}\mathbf w^T(\sum_{i=1}^{n}\mathbf x_i)\mathbf w=\bm 0.$$

\begin{itemize}
\item Maximize the Variance (from the point of statistical learning)

	Because the more point $\mathbf y_1,...,\mathbf y_n$ is decentralized, the easily we classify them. And we can use the variance to measure the degree of decentralization of these points. So our goal is to find the line, maximizes the variance of  $\mathbf y_1,...,\mathbf y_n$, i.e.

	\begin{align*}
	\max_{\|\mathbf w\|^2_2=1}f(\textbf w)
	&=Var(\textbf y) = \sum_i \mathbf ||\mathbf y_i-\dfrac{1}{n}\sum_{i=1}^{n}\mathbf y_i||_2^2\\
	&= \sum_i \mathbf ||\mathbf y_i-\mathbf 0||_2^2= \sum_i(\textbf w^T \textbf x_i)^2 \hspace{1cm} (\text{use} \|\mathbf w\|_2=1)
	\end{align*}

\item Minimize distance (form the point of autoencoder)

	In other opinion, we want the distance between $\mathbf x_i$ and $\mathbf y_i$ can be as small as possible. So we can minimize
	\begin{align*}
	\min_{\|\mathbf w\|^2_2=1}g(\textbf w)
	=&\sum_i \mathbf ||\mathbf y_i-\mathbf x_i||_2^2\\
	=&\sum_i \mathbf( ||\mathbf y_i||_2^2-2\mathbf x_i^T\mathbf y_i + ||\mathbf x_i||_2^2)\\
	=&-\sum_i(\textbf w^T \textbf x_i)^2+\sum_i||\mathbf x_i||_2^2
	\end{align*}
	So our goal is to find the optimal $\mathbf w$
	\begin{align}
	\mathbf w&=\arg\max_{||\mathbf w||^2=1} f(\mathbf w)=\arg\max_{||w||^2=1} \sum_i(\textbf w^T \textbf x_i)^2 \\\notag
	&= \arg\max_{||\mathbf w||^2=1} \sum_i\textbf w^T \textbf x_i \textbf x_i^T  \sum_i\textbf w
	= \arg\max_{||\mathbf w||^2=1} \textbf w^T XX^T  \sum_i\textbf w   \hspace{1cm}
	\end{align}

\item Solve this problem:

	Its Lagrange multiplier: $$L(\textbf w,\mu )=\sum_i(\textbf w^T \textbf x_i)^2-\mu(\textbf w^T \textbf w-1),$$
	Then we get gradient of $L$:
	$$\nabla_{\mathbf w} L=2(\sum_i \textbf x_i \textbf x_i^T)\mathbf w-2\mu  \mathbf w=0,$$
	$$\nabla_{\mu} L=\mathbf w^T \mathbf w-1=0,$$
	and we can obtain $$S\mathbf w=(\sum_i \textbf x_i \textbf x_i^T)\textbf w=\mu \mathbf w,$$
	where $\textbf S=\sum_i \textbf x_i \textbf x_i^T=\mathbf X\mathbf X^T \in \mathbb{R}^{d\times d}$.v Then $\mu$ and $\textbf w$ is the eigenvalue and corresponding eigenvector of $\mathbf S$. It is easy to calculator that
	 $$f(\mathbf w)=\mathbf w^T \mathbf S \mathbf w =\mu \mathbf w^T\mathbf w =\mu.$$
\end{itemize}
	 So we can get a theorem:
	\begin{theorem}\label{PCA}  $\mathbf w$ is the corresponding eigenvector of the max eigenvalue of $\mathbf S=\mathbf X\mathbf X^T$.
	\end{theorem}

	Besides, we call $\mathbf x_i\cdot \mathbf w$ the first component of data $\mathbf X$, and $\mathbf w$ is the first component vector.

\subsection{The Case of High Dimensions: Further Components}
	Suppose $\dim \mathbb F = k$, and we have calculated the first component, then we reduce the first component: $\hat{\mathbf X}=\mathbf X - \mathbf w\mathbf w^T\mathbf X$. Now we use  $\mathbf w_{2}^T\mathbf x = \bm 0$ to represent $\mathbb F_2$ and do the same work as the 1-d case.
	By \textbf{theorem} \ref{PCA} we can get $\mathbf w_2$ is the corresponding eigenvector of the max eigenvalue of $\hat{\mathbf S}=\hat{\mathbf X}\hat{\mathbf X}^T$.

	Let $\mathbf X = \mathbf U \bm \Sigma \mathbf V^T$ is the SVD decomposition of $\mathbf X$, where $\mathbf U=(\mathbf u_1,...,\mathbf u_n), \bm \Sigma = \text{diag}(\sigma_1,...,\sigma_n)$, $\mathbf  V=(\mathbf v_1,...,\mathbf v_n)$. We have
	$$\mathbf S = \mathbf U \bm \Sigma^2\mathbf U^T,$$ so $\mathbf w =\mathbf u_1$, and

	 \begin{align*}\hat{\mathbf X} =& \mathbf X - \mathbf w\mathbf w^T\mathbf X\\
	 =&\mathbf U \bm \Sigma \mathbf V^T-\sigma_1 \mathbf u_1\mathbf v_1^T\\
	 =&\sum_{i=1}^n \sigma_i \mathbf u_i\mathbf v_i^T-\sigma_1 \mathbf u_1\mathbf v_1^T\\
	 =&\sum_{i=2}^n \sigma_i \mathbf u_i\mathbf v_i^T\\
	 =&(\bm 0,\mathbf u_2,...,\mathbf u_n)\text{diag} (0,\sigma_2,...,\sigma_n)(\bm 0,\mathbf v_2,...,\mathbf v_n)^T
	 \end{align*}

	 So the corresponding eigenvector of the max eigenvalue of $\hat{\mathbf S}$ is the corresponding eigenvector of the second big eigenvalue of $\mathbf S$. If we generalize this conclusion, we can get the theorem:
	\begin{theorem}
	 	The $i$-th component vector $\mathbf w_i$ is the corresponding eigenvector of the $i$-th big eigenvalue of $\mathbf S=\mathbf X\mathbf X^T$, i.e. the $i$-th big left singular value of $\mathbf S=\mathbf X$.
	\end{theorem}
	\begin{remark}
	Suppose the eigenvalues are listed in a descending order as $$\mu_1\geq \mu_2\geq ...\geq...$$ Take $\sigma_i=\sqrt{\mu_i}$, then $\sigma$s are called the corresponding  singular values.
	\end{remark}

\subsection{A General Proof}
	Suppose $\text{dim}\mathbb F = k$,  $\mathbf{W}=(\mathbf{w}_1,...,\mathbf{w}_k)$, and $\mathbb F=\text{span}\{\mathbf{w}_1,...,\mathbf{w}_k\}$ ($\mathbf{w}_j^T\mathbf{w}_j=1,\mathbf{w}_i^T\mathbf{w}_j=0, i\neq j$), then the projection of $\mathbf x_i$ is
	\begin{equation}
		\mathbf{y}_i=\sum_{j=1}^{k}(\mathbf{w}_j^T \mathbf{x}_i)\mathbf{w}_j=\mathbf W\mathbf W^T\mathbf x_i=\mathbf W \tilde{\mathbf x}_i.
	\end{equation}

	By the previous discussion, our goal is also to maximize the variance of $\mathbf y_i$, $i=1,...,n$.

	Because
	\begin{align*}
	f(\textbf W)
	&=Var(\textbf y) = \sum_i \mathbf ||\mathbf y_i-\dfrac{1}{n}\sum_{i=1}^{n}\mathbf y_i||_2^2\\
	&= \sum_i \mathbf ||\mathbf y_i-\mathbf 0||_2^2\\
	&= \sum_i\sum_j^k(\textbf w_j^T \textbf x_i)^2 \hspace{1cm} (\text{use} \|\mathbf w_j\|_2=1)\\
	&=\sum_j^k\textbf w_j^T \mathbf{S} \textbf w_j, \\
	\end{align*}
	since $w_i^T w_j=\delta_{ij}$, we get a optimal problem:

	\begin{equation}
	\max_{\mathbf W^T\mathbf W=\mathbf I}f(\mathbf W)=\sum_j^k\textbf w_j^T \mathbf{S} \textbf w_j= \text{tr}(\mathbf W^T\mathbf S\mathbf W)
	\end{equation}
	The Lagrange multiplier of $f(\mathbf W)$ is:
	\begin{equation}
	L(\textbf W,\bm \mu)
	=\sum_j\textbf w_j^T \mathbf{S} \textbf w_j-\sum_j\mu_{jj}(\textbf w_j^T \textbf w_j-1)-2\sum_{j< s} \mu_{js}(\textbf w_j^T \textbf w_s-0)
	\end{equation}
	The derivative of $L$ is
	\begin{equation}
	0=\dfrac{\partial L}{\partial \mathbf w_j}=2(\mathbf{S}\mathbf w_j-\sum_s \mu_{js}  \mathbf w_s), \hspace{1cm} \mu_{js}=\mu_{sj}
	\end{equation}
	the we have
	\begin{equation}
	\mathbf{S}\mathbf w_j=\sum_s \mu_{js}  \mathbf w_s,\hspace{1cm} j=1,2,...,k.
	\end{equation}

	So the subspace $\mathbb F=\text{span}\{\mathbf w_1,...,\mathbf w_k \} $ is the invariant subspace of $\mathbf S$, then we can obtain that there are $k$ unit eigenvectors $\mathbf u_1,...,\mathbf u_k$ of $\mathbf S$ s.t. $\text{span}\{\mathbf u_1,...,\mathbf u_k \}=\mathbb F$.
	Let $\lambda_i$ is the eigenvalue of $\mathbf u_i$ and $\mathbf U=(\mathbf u_1,...,\mathbf u_k)$.

	Assume $\mathbf w_j=\sum_s a_{js} \mathbf u_s$ and $\mathbf A=(a_{js})$, then $\mathbf W=\mathbf U\mathbf A$. By $\mathbf W^T\mathbf W=\mathbf I$ and $\mathbf U^T\mathbf U=\mathbf I$, we can obtain $$\mathbf A^T\mathbf A=\mathbf A^T\mathbf U^T\mathbf U\mathbf A=\mathbf W^T\mathbf W=\mathbf I,$$ so $\mathbf A$ is unit orthogonal matrix.

	\begin{align}
	f(\mathbf W)&=\sum_j (\sum_s a_{js}\mathbf u_s)\mathbf S (\sum_t a_{jt}\mathbf u_t)\\
	 &=\sum_j (\sum_s a_{js}\mathbf u_s) (\sum_t a_{jt}\lambda_t\mathbf u_t)\\
	 &=\sum_j \sum_s \lambda_s a_{js}^2\\
	 &=\sum_s \lambda_s (\sum_j a_{js}^2 ).\\
	 &=\sum_s^k \lambda_s
	\end{align}

	So $\lambda_s$ is the largest $k$ eigenvalues of $\mathbf S$(Suppose $\lambda_1 \geq \lambda_2 \geq ...\geq \lambda_k \geq...\geq \lambda_d$ are the eigenvalues of S).

We can also get the optimal $\mathbf W$ without using Lagrangian multiplier.

	\begin{equation}
	\mathbf W=\arg\min_{\mathbf W^T\mathbf W=\mathbf I}\sum_i^N ||y_i-x_i||_2^2
	\end{equation}

	\begin{align}
	||y_i-x_i||_2^2&=||\mathbf W\mathbf W^T x_i-x_i||_2^2\\\notag
	&=(\mathbf W\mathbf W^T x_i, \mathbf W\mathbf W^T x_i)-2(\mathbf W\mathbf W^T x_i,x_i)+||x_i||_2^2  \\\notag
	&=||x_i||_2^2-(\mathbf W\mathbf W^T x_i,x_i)  \\\notag
	&=||x_i||_2^2-x_i^T\mathbf W\mathbf W^T x_i
	\end{align}

	\begin{align}
	\sum_i^N x_i^T\mathbf W\mathbf W^T x_i &=  \sum_i^N \sum_j^k (\mathbf w_j^T x_i)^2 \\\notag
	&=  \sum_i^N \sum_j^k \mathbf w_j^T x_i x_i^T \mathbf w_j\\\notag
	&=  \sum_j^k \mathbf w_j^T XX^T \mathbf w_j
	\end{align}

	\begin{equation}
	\mathbf W=\arg\max_{\mathbf W^T\mathbf W=\mathbf I}f(\mathbf W)=\arg\max_{\mathbf W^T\mathbf W=\mathbf I}\sum_j^k \mathbf w_j^T XX^T \mathbf w_j
	\end{equation}

Denote the singular value decomposition of $XX^T$ as $XX^T=U\Sigma U^T$. Suppose $\mathbf w_i=U \alpha_i$, where $|\alpha_i|=1$, then
	\begin{align}
	f(\mathbf W)&=\sum_j^k \mathbf w_j^T XX^T \mathbf w_j\\\notag
	&=\sum_j^k \mathbf w_j^T U\Sigma U^T \mathbf w_j\\\notag
    &=\sum_j^k \mathbf \alpha_j^T \Sigma \alpha_j\\\notag
    &=\sum_j \sum_i \mathbf \alpha_{j,i}^2 \sigma_i^2 \\\notag
    &=\sum_i \sigma_i^2 \sum_j \mathbf \alpha_{j,i}^2  \\\notag
    &\leq\sum_i^k \sigma_i^2,
	\end{align}
	where the equality holds only when $\alpha_{j,i}=\delta_{i,j}$. In this case, $\mathbf W$ is the matrix composed with the first $k$ columns of $U$.

	We can get a theorem:
	\begin{theorem}\label{theorem:PCA}
		If $\text{dim}\mathbb F = k$, then the optimal $\mathbb F$ of PCA is $\mathbb F= \text{span}\{\mathbf u_1,...,\mathbf u_k \}$, where $\mathbf u_i$ is the corresponding eigenvector of the $i$-th big eigenvalue of $\mathbf S=\mathbf X\mathbf X^T$, i.e. the corresponding left singular of the $i$-th big value of $\mathbf X$.
	\end{theorem}

	\subsection{The case of $\sum_i^n x_i\neq 0$}
	While $\sum_i^n x_i\neq 0$, and the $\mathbb F$ is affine set of $\mathbb R^d$, suppose $\mathbf w_1,...\mathbf w_k$ is a unit orthonormal base of $\mathbb F$ and $\mathbf x_0$ is a point in $\mathbb F$ such that $\mathbf W^T\mathbf x_0 =\mathbf 0$. Write $\mathbf{W}=(\mathbf{w}_1,...,\mathbf{w}_k)$, obviously $\mathbf W^T\mathbf W=\mathbf I$. The the projection of $\mathbf x_i$ in $\mathbb F$ is
	\begin{equation}
		\mathbf y_i=\mathbf W(\mathbf W^T\mathbf x_i)+\mathbf x_0
	\end{equation}

	\begin{align*}
		f(\mathbf W,\mathbf x_0)&=\sum_i^n\|\mathbf y_i-\mathbf x_i\|_2^2\\
		&=\sum_i^n\|\mathbf W(\mathbf W^T\mathbf x_i)+\mathbf x_0-\mathbf x_i\|_2^2\\
		&=\sum_i^n\|(\mathbf W\mathbf W^T-\mathbf I)\mathbf x_i+\mathbf x_0\|_2^2\\
	\end{align*}

	\begin{align*}
	\dfrac{\partial f}{\partial \mathbf x_0}&=2\sum_i^n((\mathbf W\mathbf W^T-\mathbf I)\mathbf x_i+\mathbf x_0)\\
	&=2n((\mathbf W\mathbf W^T-\mathbf I)\bar{\mathbf x}+\mathbf x_0)
	\end{align*}
	where $\tilde{\mathbf x}=\dfrac 1n\sum_i^n \mathbf x_i$.

	Thus $\mathbf x_0=(\mathbf I-\mathbf W\mathbf W^T)\bar{\mathbf x}$, and $f$ can be write as
	\begin{equation}
	f(\mathbf W,\mathbf x_0)
	=\sum_i^n\|(\mathbf W\mathbf W^T-\mathbf I)(\mathbf x_i-\bar{\mathbf x})\|_2^2
	\end{equation}
	If we use $\hat{\mathbf x}_i=\mathbf x_i -\bar{\mathbf x}$ to replace $\mathbf x_i$ in the case of $\sum_i^n x_i=0$, the next steps are the same as the case of $\sum_i^n x_i= 0$. Then we can get a theorem:
	\begin{theorem}
		If $\mathbb F$ is the affine set of $\mathbb R^d$, and $\text{dim}\mathbb F = k$, then the optimal $\mathbb F$ of PCA is $\mathbb F= \text{span}\{\tilde{\mathbf u}_1,...,\tilde{\mathbf u}_k \}+\bar{\mathbf x}$, where $\tilde{\mathbf u}_i$ is the corresponding eigenvector of the $i$-th big eigenvalue of $\mathbf S=\tilde{\mathbf X}\tilde{\mathbf X}^T$, i.e. the corresponding left singular of the $i$-th big value of $\tilde{\mathbf X}$, where $\tilde{\mathbf X}=(\mathbf x_1-\bar{\mathbf x},...,\mathbf x_n-\bar{\mathbf x})$.
	\end{theorem}

\section{Optimal pairing of encoding and decoding}
\subsection{Linear decoding}
Suppose $\mathbf x_i\in \mathbb R^d$ and we want to use a function to
compress them into $\mathbb R^{d_c}$, we call this function encoder
(restriction, compress) function $f:\mathbb R^d\rightarrow \mathbb
R^{d_c}$. Besides, after compression, we want to uncompress the data
by a decoder (prolongation, uncompress) function and let the distance
between the origin data and uncompressed data be as small as possible,
i.e. minimize the loss function
\begin{equation}
  \label{encode-decode-loss}
	L=\sum_i^n||g\circ f(\mathbf x_i)-\mathbf x_i ||_2^2.
\end{equation}
	If we fix the decoder as a linear function, i.e.
        \begin{equation}
          \label{linear-encoding}
	g(\mathbf y)=\mathbf W\mathbf y+\mathbf b.
        \end{equation}
	Then the loss function
	\begin{align*}
	L&=\sum_i^n||\mathbf W f(\mathbf x_i)+\mathbf b-\mathbf x_i ||_2^2
	\end{align*}
	because
	\begin{align*}
	&\min_{\mathbf W,\mathbf b,f} \sum_i^n||\mathbf W f(\mathbf x_i)+\mathbf b-\mathbf x_i ||_2^2\\
	&\geq \min_{\mathbf W,\mathbf b,f(\mathbf x_i)} \sum_i^n||\mathbf W f(\mathbf x_i)+\mathbf b-\mathbf x_i ||_2^2\\
	&=\min_{\mathbf W,\mathbf b,\bm\beta_1,...,\bm\beta_n} \sum_i^n||\mathbf W \bm \beta_i+\mathbf b-\mathbf x_i ||_2^2
	\end{align*}

	Write $ I(\mathbf W,\mathbf b,\bm\beta_1,...,\bm\beta_n)=\sum_i^n||\mathbf W \bm \beta_i+\mathbf b-\mathbf x_i ||_2^2$, because of nonzero sum of $\bm \beta_i$ can be represented by $\mathbf b$, we can suppose $\sum_i^n \bm \beta_i = 0$. We calculate its derivative:

	\begin{align*}
	\dfrac{\partial I}{\partial \mathbf b}=-2\sum_i^n (\mathbf W \bm \beta_i+\mathbf b-\mathbf x_i )=0 \Rightarrow \mathbf b = \dfrac 1n \sum_i^n \mathbf x_i=\bar {\mathbf x}\\
	\dfrac{\partial I}{\partial \bm \beta_i}=-2 (\mathbf W \bm \beta_i+\mathbf b-\mathbf x_i )^T\mathbf W=0\Rightarrow \bm \beta_i =\mathbf W^T(\mathbf x_i -\mathbf b)
	\end{align*}

This means if $g$ is a linear function, the best $f$ is also linear function, and then the steps are same as PCA.

In summary, we have
\begin{theorem}
Given data $\mathbf x_i\in \mathbb R^d$ for $i=1:n$ and any encoding
function
$f:\mathbb R^d\rightarrow \mathbb R^{d_c}$,
if the encoding function $g:\mathbb R^{d_c}\mapsto \mathbb R^d$ is
chosen to the linear as given
\eqref{linear-encoding}
Then the best encoder $f$ should also be linear.  Furthermore
\begin{equation}
  \label{encoder-decoder}
f(x)=W^T(x-b), g(y)=Wy+b
\end{equation}
with
\begin{equation}
\label{b}
\mathbf b = \dfrac 1n \sum_i^n \mathbf x_i
\end{equation}
``Optimal'' $W$ may be find by
\begin{equation}
  \label{eq:1}
  \min_W L
=  \min_W \sum_{i=1}^n\|g\circ f(x_i)-x_i\|^2
=  \min_W \sum_{i=1}^n\|WW^T\hat x_i-\hat x_i\|^2
\end{equation}
where $\hat x_i=x_i-b$.
 \end{theorem}

It follows that
$$
L_i=\|WW^T\hat x_i-\hat x_i\|^2 =
\hat x_i^TWW^TWW^T\hat x_i-2\hat x_i^TWW^T\hat x_i
+\|\hat x_i\|^2
$$
\paragraph{Orthogonal case:} If $W^TW=I$, then
$$
L_i=-\hat x_i^TWW^T\hat x_i
+\|\hat x_i\|^2
$$
If we write
$$
W=(w_1,\ldots,w_{d_c})
$$
\begin{small}
$$
\sum_{i=1}^n\hat x_i^TWW^T\hat x_i
=\sum_j\sum_{i=1}^n\hat x_i^Tw_jw_j^T\hat x_i
=\sum_j\sum_{i=1}^nw_j^T\hat x_i \hat x_i^Tw_j
=\sum_jw_j^T\hat X \hat X^Tw_j
={\rm trace}(W^T\hat X \hat X^TW)
$$
\end{small}
which leads to PCA:
\begin{equation}
  \label{PCA}
W=\arg\max_{W^TW=I}  {\rm trace}(W^T\hat X \hat X^TW)
\end{equation}
The solution is given by the eigenvectors corresponding the $d_c$
largest eigenvalues of $XX^T$.

\subsubsection{General case}
\begin{tiny}
$$
\sum_ix_i^TWW^TWW^Tx_i=\sum_{i, k,l}x_i^Tw_k w_k^T w_l w_l^Tx_i
=\sum_{i, k,l}w_k^T w_l w_l^Tx_ix_i^Tw_k
=\sum_{l}w_l^TXX^TWW^T w_l
$$
\end{tiny}
$$
L=\sum_{l}w_l^TXX^T(WW^T-2) w_l +\|X\|_F^2
$$
\subsection{Nonlinear PCA}

	Now we have a linear  $\psi(\mathbf x)=\mathbf W_1^T(\mathbf x-\bar{\mathbf x})$ and a linear prolongation (uncompress, decoder) $\phi(\mathbf x)=\mathbf W_2\mathbf y+\bar{\mathbf x}$. The goal of PCA is to minimize the loss function
	$$L=\|\psi\circ \phi(\mathbf X)-\mathbf X \|_F^2$$.

	For nonlinear case, we use an entrywise activation $\sigma(x)$ on $\psi$ and $\phi$,
	$$
	f=\sigma\circ \psi, g=\sigma\circ\phi
	$$
	It becomes $$f(x)=\sigma(W_1^T x +b), g(y)=\sigma (W_2 y + b')$$
	The goal of nonlinear PCA is to minimize
	$$L=\|g\circ f(\mathbf X)-\mathbf X \|_F$$.

\section{A classical application of PCA}
In this section, we will introduce the application of PCA on face recognition. The Dataset of Labeled Faces in the Wild\footnote{Learned-Miller, Erik, et al. "Labeled faces in the wild: A survey." Advances in face detection and facial image analysis. Springer, Cham, 2016. 189-248.} is used.
There are  in total $N=13233$ face images for 5749 people  with $d=64\times 64$ resolution, $X = [x_1, \cdots, x_{N}]\in \mathbb{R}^{d\times N}$.
Some example pictures are shown below.
\begin{center}
\includegraphics[width=0.8\textwidth]{1000Face}
\end{center}

\subsection{Task of face recognition}
The task of face recognition is to identify a new image $x$ of a face whether the person is in the dataset $X$ and classify who that is if the person is in $X$.

A natural idea is to find $i$ such that $$||x-x_i||_2=\min_{1\le j\le N} ||x-x_j||_2$$ or $$i=\arg\min_{1\le j\le N} ||x-x_j||_2.$$ But it doesn't work since different pictures of the same person can have a very large $\ell^2$-distance (between pixels). The solution is projecting the data to a lower dimensional space.
Then we can find some approximation of the data in a lower dimensional hyperplane for better classification.  To minimize the total distance between the original data and the uncompressed data, we can use PCA for application.

\subsection{PCA}
Without loss of generality, we assume $\sum x_i=0$. Then we can apply Theorem \ref{theorem:PCA} to find the best projection directions.

	We use the same notation. Suppose  $\text{dim}\mathbb F = k$, then the optimal projection matrix is $\mathbf{W}=(\mathbf{w}_1,...,\mathbf{w}_k)$, where $\mathbf{w}_i$ is the corresponding eigenvector of the $i$-th big eigenvalue of $\mathbf S=\mathbf X\mathbf X^T$, and $\mathbb F=\text{span}\{\mathbf{w}_1,...,\mathbf{w}_k\}$ ($\mathbf{w}_j^T\mathbf{w}_j=1,\mathbf{w}_i^T\mathbf{w}_j=0, i\neq j$), then the uncompressed data of $\mathbf x_i$ is
	\begin{equation}
		\mathbf{y}_i=\sum_{j=1}^{k}(\mathbf{w}_j^T \mathbf{x}_i)\mathbf{w}_j=\mathbf W\mathbf W^T\mathbf x_i=\mathbf W \tilde{\mathbf x}_i.
	\end{equation}

Moreover, we call the eigenvector of $\mathbf S=\mathbf X\mathbf X^T$, $\mathbf{w}_i$, as EigenFace.  We show the first 16 EigenFaces below.
\begin{figure}
\begin{center}
\includegraphics[width=0.5\textwidth]{15EigenFace}
\caption{Examples of EigenFaces}
\end{center}
\end{figure}

\subsection{Encoding and decoding}
Similarly, we can divide the uncompressed data of $\mathbf x_i$ with two steps, encoding and decoding, where $\tilde{\mathbf x}_i=W^T\mathbf x_i$ is the encoding and $\mathbf{y}_i=\mathbf W \tilde{\mathbf x}_i$ is the decoding part. In detail, the encoding part projects the data to a $k$-dimensional space and we get a lower dimensional coefficient vector $\tilde{\mathbf x}_i=W^T\mathbf x_i$. Thus we can classify faces in this lower dimensional space. Then we can get the the uncompressed data with the decoding part by $\mathbf{y}_i=\mathbf W \tilde{\mathbf x}_i$.

We use the picture of one face data as an example.
We have an original image: $\mathbf x\in \mathbb{R}^{d}$.
\begin{figure}
\begin{center}
\includegraphics[width=0.4\textwidth]{1000Face-100ori-2}
\caption{Original image}
\end{center}
\end{figure}
\newpage
After encoding, we get a lower dimensional coefficient vector $\tilde{\mathbf x}=W^T\mathbf x$.
Then after decoding, we get the the uncompressed data
$\mathbf{y}=\mathbf W \tilde{\mathbf x} \in \mathbb{R}^{d}$.
\begin{figure}
\begin{center}
\includegraphics[width=0.4\textwidth]{1000Face-100-2}
\caption{Uncompressed image}
\end{center}
\end{figure}

\subsection{Classification accuracy}
In the task of face recognition in the dataset of Labeled Faces in the Wild,
PCA extracts features with eigenvectors and achieves 85\% test accuracy in average.
If we use CNN on this dataset, CNN extracts features with convolutional filters and can achieve 99.85\% test accuracy in average.


\subsection{Linear finite element cannot be recovered by ${\rm DNN}_1$ for $d\ge2$}
In view of  Theorem~\ref{thm:1dLFEMDNN} and the fact that ${\rm{DNN}_J}
\subseteq {\rm{DNN}_{J+1}} $, it is natural to ask that how many
layers are needed at least to recover all linear finite element
functions in $\mathbb{R}^d$ for $d\ge2$.  In this section, we will show that
\begin{equation}\label{key}
J_d \ge 2, \quad \text{if} \quad d\ge 2,
\end{equation}
where $J_d$ is the minimal $J$ such that all linear finite element
functions in $\mathbb R^d$ can be recovered by ${\rm DNN}_J$.

In particular, we will show the following theorem~\cite{he2020relu}.
\begin{theorem}\label{lowerbound}
	If $\Omega\subset \mathbb R^d$ is either
	a bounded domain or $\Omega=\mathbb{R}^d$,
	${\rm DNN}_1$ can not be used to recover all linear finite element
	functions on $\Omega$.
\end{theorem}
\begin{proof}
	We prove it by contradiction. Let us assume that for any continuous
	piecewise linear function $f: \Omega \to \mathbb{R} $, we can find
	finite $N \in \mathbb{N}$, $w_i \in \mathbb{R}^{1,d}$ as row vector
	and $\alpha_i, b_i, \beta \in \mathbb{R}$ such that
	$$
	f =  \sum_{i=1}^N \alpha_i {\rm ReLU}(w_i\cdot  x +b_i) + \beta,
	$$
	with $f_i = \alpha_i {\rm ReLU}(w_i\cdot  x +b_i)$, $\alpha_i \neq 0$ and $w_i
	\neq 0$.  Consider the finite element functions, if this one hidden
	layer ReLU DNN can recover any basis function of FEM, then it can
	recover the finite element space.  Thus let us assume $f$ is a locally
	supported basis function for FEM.
	Furthermore, if $\Omega$ is a bounded domain, we assume that
	\begin{equation}\label{distcondi}
	d({\rm supp}(f), \partial \Omega) > 0,
	\end{equation}with
	$$
	d(A, B) = \inf_{x\in A, y\in B} \|x-y\|,
	$$
	as the distance of two closed sets.

	A more important observation is that $\nabla f: \Omega \to
	\mathbb{R}^d$ is a piecewise constant vector function. The key
	point is to consider the discontinuous points for
	$$g := \nabla
	f = \sum_{i=1}^N \nabla f_i.$$
	For more general case, we can define the set of discontinuous points of a function by
	$$
	D_{g} := \{x \in \Omega~|~ x ~ \text{is a discontinuous point of} ~ g\}.
	$$
	Because of the property that
	\begin{equation}\label{eq:disfun}
	D_{f+g} \supseteq D_{f} \cup D_{g} \backslash (D_{f} \cap D_{g}),
	\end{equation}
	we have
	\begin{equation}\label{eq:dis_fn}
	D_{\sum_{i=1}^N g_i} \supseteq \bigcup_{i=1}^N D_{g_i} \backslash \bigcup_{i\neq j}\left( D_{g_i}\cap D_{g_j} \right).
	\end{equation}
	Note that
	\begin{equation}\label{eq:def_gi}
	g_i = \nabla f_i(x) =  \nabla \left( \alpha_i {\rm ReLU}(w_i\cdot   x +b_i)  \right) =\left(\alpha_iH(w_i \cdot  x +b_i)\right)w_i \in \mathbb{R}^d,
	\end{equation}
	for $i=1:N$ with $H$ be the Heaviside function defined as:
	$$
	H(x) = \begin{cases}
	0 &\text{if} ~ x \le 0, \\
	1 &\text{if} ~ x > 0.
	\end{cases}
	$$
	This means that
	\begin{equation}\label{eq: D_gi}
	D_{g_i} = \{ x ~|~ w_i\cdot   x + b_i = 0\}
	\end{equation}
	is a $d-1$ dimensional affine space in $\mathbb{R}^d$.

	Without loss of generality, we can assume that
	\begin{equation}\label{eq:assumD_gi}
	D_{g_i} \neq D_{g_j}.
	\end{equation}
	When the other case occurs, i.e. $D_{g_{\ell_1}} = D_{g_{\ell_2}} = \cdots= D_{g_{\ell_k}}$, by the definition of $g_i$ in \eqref{eq:def_gi} and $D_{g_i}$ in \eqref{eq: D_gi} ,
	this happens if and only if there is a row vector $(w, b)$ such that
	\begin{equation}\label{eq:Dfcondition}
	c_{\ell_i}\begin{pmatrix}
	w &
	b
	\end{pmatrix} =
	\begin{pmatrix}
	w_{\ell_i} &
	b_{\ell_i}
	\end{pmatrix},
	\end{equation}
	with some $c_{\ell_i} \neq 0$ for $i = 1:k$.  We combine those $g_{\ell_i}$ as
	\begin{equation*}
	\begin{aligned}\label{mergeH}
	%	\begin{split}
	\tilde g_{\ell} &= \sum_{i=1}^k g_{\ell_i} = \sum_{i=1}^k \alpha_{\ell_i} H(w_{\ell_i} \cdot  x + b_{\ell_i}) w_{\ell_i}, \\
	&= \sum_{i=1}^k \left( c_{\ell_i}\alpha_{\ell_i} H\left(c_{\ell_i}(w\cdot   x + b)\right) \right) w, \\
	&=\begin{cases}
	\displaystyle \left(\sum_{i=1}^k  c_{\ell_i}\alpha_{\ell_i} H(c_{\ell_i}) \right) w  \quad &\text{if} \quad w x + b > 0,\\
	\displaystyle \left(\sum_{i=1}^k  c_{\ell_i}\alpha_{\ell_i} H(-c_{\ell_i}) \right) w  \quad &\text{if} \quad w x + b \le 0.\\
	\end{cases}
	%	\end{split}
	\end{aligned}
	\end{equation*}
	Thus, if
	$$
	\left(\sum_{i=1}^k  c_{\ell_i}\alpha_{\ell_i} H(c_{\ell_i}) \right)  = \left(\sum_{i=1}^k  c_{\ell_i}\alpha_{\ell_i} H(-c_{\ell_i}) \right),
	$$
	$\tilde g_\ell$ is a constant vector function, that is to say $D_{\sum_{i=1}^k g_{\ell_i}} = D_{\tilde g_\ell} = \emptyset$.
	Otherwise, $\tilde g_\ell$ is a piecewise constant vector function with the property that
	$$
	D_{\sum_{i=1}^k g_{\ell_i}} = D_{\tilde g_\ell} = D_{g_{\ell_i}} = \{ x ~|~ w\cdot  x + b = 0\}.
	$$
	This means that we can use condition \eqref{eq:Dfcondition} as an equivalence relation and split $\{g_i\}_{i=1}^N$ into some groups, and we can combine those $g_{\ell_i}$ in each group as what we do above. After that, we have
	$$
	\sum_{i=1}^N g_i = \sum_{\ell=1}^{\tilde N} \tilde g_{\ell},
	$$
	with $D_{\tilde g_s} \neq D_{\tilde g_t}$.
	Finally, we can have that $D_{\tilde g_s} \cap D_{\tilde g_t}$ is an empty set or a $d-2$ dimensional affine space in $\mathbb{R}^d$.
	Since
	$\tilde N \le N$ is a finite number,
	$$
	D := \bigcup_{i=1}^N D_{\tilde g_\ell} \backslash \bigcup_{s\neq t}\left( D_{\tilde g_s}\cap D_{\tilde g_t} \right)
	$$
	is an unbounded set.
	\begin{itemize}
		\item If $\Omega = \mathbb{R}^d$,
		$$
		{\rm supp(f)} \supseteq D_{g} = D_{\sum_{i=1}^N g_i} = D_{ \sum_{\ell=1}^{\tilde N} \tilde g_{\ell}} \supseteq D,
		$$ is contradictory to the assumption that $f$ is locally supported.
		\item If $\Omega$ is a bounded domain,
		$$
		d(D, \partial \Omega) =
		\begin{cases}
		s > 0 \quad &\text{if}\quad  D_{\tilde g_i} \cap \Omega = \emptyset, \forall i\\
		0 \quad &\text{otherwise}.
		\end{cases}
		$$
		Note again that all $D_{\tilde g_i}$'s are $d-1$ dimensional affine spaces, while $D_{\tilde g_i} \cap D_{\tilde g_j}$ is either an empty set or a d-2 dimensional affine space.
		If $d(D, \partial \Omega) > 0$, this implies that $\nabla f$ is continuous in $\Omega$, which contradicts the  assumption that $f$ is a basis function in FEM.
		If $d(D, \partial \Omega) = 0$, this contradicts the previous assumption in \eqref{distcondi}.
	\end{itemize}
	Hence ${\rm DNN}_1$ cannot recover any piecewise linear function in $\Omega$ for $d \ge 2$.
\end{proof}

Following the proof above, we have the following theorem~\cite{he2020relu}.
\begin{theorem}\label{linearindep}
	$\{{\rm ReLU}(w_i\cdot x+b_i)\}_{i=1}^m$ are linearly independent if $(w_i,
	b_i)$ and $(w_j, b_j)$ are linearly independent in
	$\mathbb{R}^{1\times (d+1)} $ for any $i \neq j$.
\end{theorem}


\chapter{Bit Manipulation}
\section{Concepts}
\subsection{Basics}
\begin{enumerate}
\item Bit value: bit0, bit1.
\item BitSet/Bits
\item Bit position (bit interchangeably)
\item 32-bit signed range: $[-2^{31}, 2^{31}-1]$. $0$ is like positive number without complement.
\end{enumerate}
\begin{python}
MAX = 0x7FFFFFFF
MIN = 0x80000000
MSK = 0xFFFFFFFF
\end{python}
\subsection{Operations}
\runinhead{Mask.}
\begin{enumerate}
\item Masking to 1: to mask a single bit position, $bit\OR 1$
\item Masking to 0: to mask a single bit position, $bit\AND 0$
\item Querying a bit position value: to query a single bit position, $bit\AND 0010$
\item Toggling bit values: to toggle a single bit position, $bit\XOR 1$
\end{enumerate}
This can be extended to do masking operations on multiple bits.

\runinhead{2's complement}
$$
-i = \NOT i+1
$$

\runinhead{Check 2's power}
$$x\AND(x-1)$$

\runinhead{Rightmost bit set.} To get the rightmost bit, with the help of 2's complement:
\begin{enumerate}
\item Left extended with 1's:
$$x \XOR (-x)$$
\item Left extended with 0's:
$$x \AND (-x)$$
\end{enumerate}

\runinhead{Negation and index} We can use tilde notation for the index accessing a string or an array
\begin{lstlisting}
i  ~i
0  -1
1  -2
2  -3
3  -4
4  -5
5  -6
\end{lstlisting}
$$
\NOT i = -i+1
$$
To determine whether a string is palindrome:
\begin{python}
def is_palindrome(s):
  return all(s[i] == s[~i] for i in xrange(len(s)/2))
\end{python}

\subsection{Python}
Python int is larger than 32 bit.
If 32bit signed int, in python, we may need to mask the int:
\begin{enumerate}
\item Mask to 32bit: \pyinline{x & MSK},
\item Left extended with 1's: \pyinline{\~(x ^ MSK)}
\end{enumerate}

, where \pyinline{MSK = 0xFFFFFFFF}

\section{Radix}
\runinhead{Convert to hexadecimal with 2's complement.} Very easy to convert positive number of hex, but need to pay more attention to negative number when thinking in the decimal representation.

Everything easy to convert the number even under 2's complement if thinking in the binary representation.
\begin{enumerate}
\item current digit we need: \pyinline{num & 0xf}
\item next significant number: \pyinline{num >>= 4}
\end{enumerate}
\section{Circuit}
It is under 32-bit assumption, for Python, we need additional masking in the previous section.
\subsection{Full-adder}
\rih{Plus.} Handle carry: only 1 + 1 needs carry, thus \pyinline{a & b} determines carry.
\begin{python}
def plus(a, b):
  carry = (a & b) << 1
  out = a ^ b
  if carry != 0:
    return plus(out, carry)
  else:
    return out
\end{python}
\rih{Half Adder}. One bit \pyinline{a, b}:
\begin{python}
def half_add(a, b):
  carry = a & b
  out   = a ^ b
  return out, carry
\end{python}
\rih{Full Adder}. One bit \pyinline{a, b, cin}. \pyinline{out = a ^ b ^ cin}. and \pyinline{cout = a & b | cin & a ^ b}
\begin{python}
def full_add(a, b, cin):
  out, c1 = half_add(a, b)
  out, c2 = half_add(out, cin)
  cout    = c1 | c2  # ^ possible
  return out, cout
\end{python}

\subsection{Full-substractor}
\rih{Substract.} Handle borrow: only 0 - 1 needs borrow, thus \pyinline{\~a & b} determines borrow.
\begin{python}
def sub(a, b):
  borrow = (~a & b) << 1
  out = a ^ b
  if borrow != 0:
    return sub(out, borrow)
  else:
    return out
\end{python}
\rih{Half Substractor.} One bit \pyinline{a, b}:
\begin{python}
def half_sub(a, b):
  borrow = (~a & b)
  out = a ^ b
  return out, borrow
\end{python}
Notice negation can be done in xor.
\begin{python}
~a == 1 ^ a
\end{python}
\rih{Full Substractor}. One bit \pyinline{a, b, bin}. \pyinline{out = a ^ b ^ bin}.
\begin{python}
def full_sub(a, b, bin):
  out, b1 = half_sub(a, b)
  out, b2 = half_sub(out, bin)
  bout = b1 | b2
  return out, bout
\end{python}

\subsection{Multipler}

\section{Single Number}
\subsection{Three-time appearance}
Given an array of integers, every element appears three times except for one. Find that single one.

\rih{Using list.} Consider 4-bit numbers:
\begin{eqnarray*}
&& 0000 \\
&& 0001 \\
&& 0010 \\
&& ... \\
&& 1111
\end{eqnarray*}

Add (not $\&$) the bit values \textbf{vertically}, then result would be $abcd$ where $a, b, c, d$ can be any number, not just binary. $a, b, c, d$ can be divided by 3 if the all element appears three times. Until here, you can use a list to hold $a, b, c, d$. By mod 3, the single one that does not appear 3 times is found.

To generalize to 32-bit \pythoninline{int}, use a list of length 32.

\rih{Using bits.}
To further optimize the space, use bits (bit set) instead of list.
\begin{itemize}
\item Since all except one appears 3 times, we are only interested in $0, 1, 2$ (mod 3) count of bit1 appearances in a bit position.
\item We create 3 bit sets to represent $0, 1, 2$ appearances of all positions of bits.
\item For a bit, there is one and only one bit set containing bit1 in that bit position.
\item Transition among the 3 bit sets for every number:
$$
bitSet^{(i)} = (bitSet^{(i-1)}\AND num)\OR(bitSet^{(i)}\AND \NOT num)
$$
\end{itemize}

For $i$ appearances, the first part is the bit set \textbf{transited from} $(i-1)$ appearances, and the second part is the bit set \textbf{transited out} from itself.

Consider each single bit separately. For the $j$-th bit in $num$, if $num_j=1$, the first part indicates $bitSet^{(i-1)}$ will transit in (since transition); the 2nd part is always 0 (since transition out or initially 0). If $num_j=0$, the 1st part is always 0 (since no transition); the 2nd part indicates $bitSet^{(i)}$ will remain the same (since no transition).

\subsection{Two Numbers}
Given an array of numbers nums, in which exactly two elements appear only once and all the other elements appear exactly twice. Find the two elements that appear only once.

\begin{itemize}
\item Easily get: $x = a \XOR b$.
\item $a \neq b$; thus there are at least one 1-bit in $x$ is different.
\item Take an arbitrary 1 bit set in $x$, and such bit set can classify the elements in the array into two separate groups.
\end{itemize}

\section{Bitwise operators}
\runinhead{Comparison.} Write a method which finds the maximum of two numbers $a, b$. You should not use if- else or any other comparison operator
\\
Clues:
\begin{enumerate}
\item check the sign bit $s$ of $a-b$.
\item return $a-s*(a-b)$
\end{enumerate}
Codes:
\begin{java}
int getMax(int a, int b) {
    int c = a - b;
    int k = (c >> 31) & 0x1;
    int max = a - k * c;
    return max;
}

\end{java}
If consider overflow, it raises another level of difficulty.


\documentclass[../main.tex]{subfiles}

\begin{document}
        \section{Torsion \& The Frenet-Serret Formulas}
                We have previously defined some properties of curves, and some associated vectors, namely the unit tangent vector \(\vec{T}\), the curvature \(\kappa\), the principal normal vector \(\vec{N}\), and the binormal vector \(\vec{B}\). We have also defined the osculating planes and circles associated with curves, which function as approximations to curves different from the typical linear ones. Another property which we may wish to consider is torsion, which encodes the rate of change of the binormal vector.
                \begin{definition}{Torsion}{}
                        Consider
                        \[
                        \frac{d\vec{B}}{ds}
                        \]
                        which is the rate of change of the binormal per unit length. This vector is perpendicular to both \(\vec{B}\) and \(\vec{T}\). The fact that it is perpendicular to \(\vec{B}\) follows from the fact that the derivative of a vector of constant magnitude (\(\vec{B}\) having constant unit magnitude) is always perpendicular to that vector. For \(\vec{T}\), the relevant fact is that
                        \[
                        \frac{d\vec{B}}{ds}=\frac{d\vec{T}}{ds}\times\vec{N}+\vec{T}\times\frac{d\vec{N}}{ds}
                        \]
                        but
                        \[
                        \frac{d\vec{T}}{ds}=\kappa\vec{N}
                        \]
                        so the first term is zero, and thus
                        \[
                        \frac{d\vec{B}}{ds}=\vec{T}\times\frac{d\vec{N}}{ds}
                        \]
                        and therefore \(\vec{B}\) must be perpendicular to \(\vec{T}\), and we have
                        \[
                        \frac{d\vec{B}}{ds}=-\tau\vec{N}
                        \]
                        where \(\tau\) is defined to be the \emph{torsion} of the curve.
                \end{definition}
                \begin{example}{}{}
                        Consider the helix parameterized by \(\vec{r}=a\cos{t}\i+a\sin{t}\j+bt\k\). Find the binormal vector and the torsion.
                        \tcblower
                        We already know
                        \[
                        v(t)=\sqrt{a^2+b^2}=\frac{ds}{dt}
                        \]
                        \[
                        \vec{T}=\frac{1}{\sqrt{a^2+b^2}}(-a\sin{t}\i+a\cos{t}\j+b\k)
                        \]
                        \[
                        \vec{N}=-\cos{t}\i-\sin{t}\j
                        \]
                        \[
                        \kappa=\frac{a}{a^2+b^2}
                        \]
                        so to find \(\vec{B}\), we must simply compute \(\vec{T}\times\vec{N}\), which yields
                        \[
                        \vec{B}=\frac{1}{\sqrt{a^2+b^2}}(b\sin{t}\i-b\cos{t}\j+a\k)
                        \]
                        the torsion is then obtained by differentiating \(\vec{B}\) in \(s\). This may be done using the chain rule, noting that
                        \[
                        \frac{d\vec{B}}{dt}=\frac{1}{\sqrt{a^2+b^2}}(b\cos{t}\i+b\sin{t}\j)
                        \]
                        and using the previously calculated value for \(\frac{ds}{dt}\) we obtain
                        \[
                        \frac{d\vec{B}}{ds}=-\frac{b}{a^2+b^2}(-\cos{t}\i-\sin{t}\j)=-\frac{b}{a^2+b^2}\vec{N}
                        \]
                        and therefore we have
                        \[
                        \tau=\frac{b}{a^2+b^2}
                        \]
                \end{example}
                It may be noted that the helix has constant curvature and torsion. As such, the helix is among the simplest 3D curves, having curvature and torsion which are constant, but non-zero.
                \begin{definition}{Frenet-Serret Formulas}{}
                        The \emph{Frenet-Serret formulas} define the derivatives of \(\vec{T}\), \(\vec{B}\), and \(\vec{N}\). We have previously found
                        \begin{align*}
                                \frac{d\vec{T}}{ds}&=\kappa\vec{N}\\
                                \frac{d\vec{B}}{ds}&=-\tau\vec{N}
                        \end{align*}
                        but we require a formula for the derivative of \(\vec{N}\). We know that \(\vec{B}=\vec{T}\times\vec{N}\), but this implies \(\vec{N}=\vec{B}\times\vec{T}\), and therefore
                        \begin{align*}
                                \frac{d\vec{N}}{ds}&=\frac{d}{ds}\vec{B}\times\vec{T}\\
                                                   &=\frac{d\vec{B}}{ds}\times\vec{T}+\vec{B}\times\frac{d\vec{T}}{ds}\\
                                                   &=-\tau\vec{N}\times\vec{T}+\vec{B}\times\kappa\vec{N}\\
                                                   &=\tau(\vec{T}\times\vec{N})-\kappa(\vec{N}\times\vec{B})\\
                                                   &=\tau\vec{B}-\kappa\vec{T}
                        \end{align*}

                        This completes the Frenet-Serret formulas, which may also be written in a matrix form as
                        \[
                        \frac{d}{ds}
                        \begin{bmatrix}
                                \vec{T}\\
                                \vec{N}\\
                                \vec{B}
                        \end{bmatrix}=
                        \begin{bmatrix}
                                0&\kappa&0\\
                                -\kappa&0&\tau\\
                                0&-\tau&0
                        \end{bmatrix}
                        \begin{bmatrix}
                                \vec{T}\\
                                \vec{N}\\
                                \vec{B}
                        \end{bmatrix}
                        \]
                \end{definition}
\end{document}


\subsubsection{Parabolic Cylinders}
\noindent
A paraboloid cylinder look like a parabola that has been extruded into 3D space. One of its forms is
\begin{equation*}
	z = cy^2,
\end{equation*}
where $c \in \R$.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.33\textwidth]{./Images/differentialMultivariableCalculus/parabolid_cylinder.png}
	\caption{A parabolic cylinder}
\end{figure}


\section{Indefinite Integrals}\label{sec:IndefInt}

In this section we focus on computing indefinite integrals.
The process of finding the indefinite integral is called \dfont{integration} (or \dfont{integrating $f(x)$}).

\begin{example}{Indefinite Integral}{IndefiniteIntegral}
Evaluate the following indefinite integral:
$$\int x^5+3x-2\,dx.$$
\vspace{-0.5cm}
\end{example}

\begin{solution}
Since this is asking for the most general anti-derivative we have:
$$\int x^5+3x-2\,dx=\frac{x^6}{6}+\frac{3x^2}{2}-2x+C$$
where $C$ is a constant.
\end{solution}

{\bf Common mistakes:}
One habit students make with integrals is to \ifont{drop the dx} at the end of the integral.
This is required! Think of the integral as a set of parenthesis.
Both are required so it is clear where the integrand ends and what variable you are integrating with respect to.

Another common mistake is to \ifont{forget the +C} for indefinite integrals.

Note that we don't have properties to deal with products or quotients of functions, that is,
%Just like with derivatives, the following will \dfont{NOT} work:
$$\int f(x)\cdot g(x)\,dx\neq \int f(x)\,dx\int g(x)\,dx.$$
$$\int \frac{f(x)}{g(x)}\,dx\neq \frac{\int f(x)\,dx}{\int g(x)\,dx}.$$
With derivatives, we had the product and quotient rules to deal with these cases.
For integrals, we have no such rules, but we will learn a variety of different techniques to deal with these cases.

The following integral rules can be proved by taking the derivative of the functions on the right side.

\begin{formulabox}[Integral Rules]
Some properties and rules to know:\\
$$\mbox{Constant Rule:}\quad\int k\,dx=kx+C.$$
$$\mbox{Constant Multiple Rule:}\quad\int kf(x)\,dx=k\int f(x)\,dx,\quad\mbox{$k$ is constant}.$$
$$\mbox{Sum/Difference Rule:}\quad\int f(x)\pm g(x)\,dx=\int f(x)\,dx\pm\int g(x)\,dx.$$
$$\mbox{Power Rule:}\quad\int x^n\,dx=\frac{x^{n+1}}{n+1}+C,\quad n\neq -1.$$
$$\mbox{Log Rule:}\quad\int \frac{1}{x}\,dx=\ln|x|+C,\quad x\neq 0.$$
$$\mbox{Exponent Rule:}\quad\int a^{kx}=\frac{a^{kx}}{k\ln a}+C,\quad x\neq 0.$$
$$\mbox{Sine Rule:}\quad\int \sin x\,dx=-\cos x+C.$$
$$\mbox{Cosine Rule:}\quad\int \cos x\,dx=\sin x+C.$$
\end{formulabox}

\begin{example}{Indefinite Integral}{IndefiniteIntegral2}
If $f'(x)=x^4+2x-8\sin x$ then what is $f(x)$?
\end{example}

\begin{solution}
The answer is:
$$\begin{array}{rcl}
\ds{f(x)=\int f'(x)\,dx}&=&\ds{\int \left(x^4+2x-8\sin x\right)\,dx}\\
\\
&=&\ds{ \int x^4 \,dx + 2\int x\,dx -8 \int \sin x\,dx }\\
\\
&=&\ds{\frac{x^5}{5}+x^2+8\cos x+C,}\\
\end{array}$$
where $C$ is a constant.
\end{solution}

\begin{example}{Indefinite Integral}{IndefiniteIntegral3}
Find the general indefinite integral of $\ds\int 3x^2\,dx$.
\end{example}
\begin{solution}
	$$\begin{array}{rcl}
	\ds{\int 3x^2\,dx} & = & \ds{3\int x^2\,dx}\\
	\\
	&=&\ds{3\frac{x^3}{3}+C}\\
	\\
	&=&x^3+C\\
	\end{array} $$
\end{solution}

\begin{example}{Indefinite Integral}{IndefiniteIntegral4}
Find the general indefinite integral of $\ds\int \frac{2}{\sqrt x}\,dx$.
\end{example}
\begin{solution}
	$$\begin{array}{rcl}
	\ds{\int \frac{2}{\sqrt x}\,dx} & = & \ds{2\int x^{-\frac{1}{2}}\,dx}\\
	\\
	&=&\ds{2\frac{x^{-\frac{1}{2}+1}}{-\frac{1}{2}+1}+C}\\
	\\
	&=&4\sqrt x+C\\
	\end{array}$$
\end{solution}

\begin{example}{Indefinite Integral}{IndefiniteIntegral5}
Find the general indefinite integral of $\ds\int \left(\frac{1}{x}+e^{7x}+x^\pi+7\right)\,dx$.
\end{example}
\begin{solution}
	$$\begin{array}{rcl}
	\ds{\int \left(\frac{1}{x}+e^{7x}+x^\pi+7\right)\,dx} & = & \ds{\int \frac{1}{x}\,dx+\int e^{7x}\,dx+\int x^\pi\,dx+\int 7\,dx}\\
	\\
	&=& \ds{\ln|x|+\frac{1}{7}e^{7x}+\frac{x^{\pi+1}}{\pi+1}+7x+C}\\
	\end{array}$$
\end{solution}

\subsection*{Differential Equations}
An equation involving derivatives where we want to solve for the original function is called a \dfont{differential equation}.
For example, $f'(x)=2x$ is a differential equation with general solution $f(x)=x^2+C$.
Some solutions (i.e., particular values of $C$) are shown below.
$$\includegraphics[width=3in]{images2/integral-curves}$$
As seen with integral curves, we may have an infinite family of solutions satisfying the differential equation.
However, if we were given a point (called an \ifont{initial value}) on the curve then we could determine $f(x)$ completely.
Such a problem is known as an \ifont{initial value problem}.

\begin{example}{Initial Value Problem}{Initial Value Problem}
If $f'(x)=2x$ and $f(0)=2$ then determine $f(x)$.
\end{example}

\begin{solution}
As previously stated, we have a solution of:
$$f(x)=x^2+C.$$
But $f(0)=2$ implies:
$$2=0^2+C\quad\to\quad C=2.$$
Therefore, $f(x)=x^2+2$ is the solution to the initial value problem.
\end{solution}

%
%\dfont{Rectlinear Motion:} We can use the concept of anti-derivatives to find position functions when given the velocity function (or velocity functions when given the acceleration).
%In particular, since $a(t)=v'(t)$ and $v(t)=s'(t)$, where $s(t)$, $v(t)$ and $a(t)$ are the position, velocity and acceleration functions respectively, we have,
%$$v(t)=\int a(t) \,dt,\quad s(t)=\int v(t) \,dt.$$
%
%\begin{example}
%A particle moves along a straight line with acceleration $a(t)=6t+4$.
%Its initial velocity is $v(0)=-6$ cm/s and initial position is $s(0)=9$ cm.
%Find $s(t)$.
%\end{example}
%
%\begin{solution}
%Antidifferentiation gives (since $v(t)=\int a(t) \,dt$):
%\[ v(t) = 6 \frac{t^2}{2}+ 4t +C = 3t^2+4t+C.\]
%But $v(0)=-6$, so $C=-6$, and
%\[ v(t) =  3t^2+4t-6 \]
%Now antidifferentiation again gives  (since $s(t)=\int v(t) \,dt$):
%\[ s(t) = 3 \frac{t^3}{3} + 4 \frac{t^2}{2} -6t +D = t^3+2t^2-6t+D.\]
%But $s(0)=9$, so $D=9$, and we finally get:
%\[ s(t) = t^3+2t^2-6t+9.\]
%\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for Section \ref{sec:IndefInt}}

\begin{enumialphparenastyle}

Find the antiderivatives of the functions:

%%%%%%%%%%
\begin{ex}
 $\ds 8\sqrt{x}$
\begin{sol}
 $\ds (16/3)x^{3/2}+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds 3t^2+1$
\begin{sol}
 $\ds t^3+t+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds 4/\sqrt{x}$
\begin{sol}
 $\ds 8\sqrt{x}+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds 2/z^2$
\begin{sol}
 $-2/z+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds 7s^{-1}$
\begin{sol}
 $7\ln s+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds (5x+1)^2$
\begin{sol}
 $\ds (5x+1)^3/15+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds (x-6)^2$
\begin{sol}
 $\ds (x-6)^3/3+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds x^{3/2}$
\begin{sol}
 $\ds 2x^{5/2}/5+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds {2\over x\sqrt x}$
\begin{sol}
 $\ds -4/\sqrt{x}+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds |2t-4|$
\begin{sol}
 $\ds 4t-t^2+C$, $t<2$; $\ds t^2-4t+8+C$, $t\ge 2$
\end{sol}
\end{ex}

\end{enumialphparenastyle}


% Project for Integration Workshop. Dept. Mathematics. UArizona
% Compute and plot lagrange interpolating polynomials. Explore when they do well and when they do poorly. `Discovery' of chebychev points.

\section{Lagrange Interpolation for Function Approximation}
Let $\{(x_k,y_k) ,\,\text{for}\, k = 1, \dots, N\}$ be $N$ points in $\mathbb{R}^2$ that satisfy $x_k \neq x_j$ whenever $k \neq j$. The \textit{Lagrange interpolating polynomial} is defined as
\begin{equation}
L(x) = \sum_{k = 1}^N \left(\prod_{j \neq k}\frac{x-x_j}{x_k-x_j} \right) y_k.
\end{equation}
In this project, you will determine how well a Lagrange interpolating polynomial can approximate the functions
\begin{equation}
f(x) = (x-.9)(x-.4)(x+.1)(x+.7)(x+.8)
\end{equation}
and
\begin{equation}
g(x) = \frac{1}{1+10x^2} \quad \quad \text{for } -1 \leq x \leq 1
\end{equation}
\begin{enumerate}[(a)]
    \item Observe that the Lagrange interpolating polynomial is a linear combination of $N$ terms in the form
    \begin{equation}
    T_k(x): =\left( \prod_{j \neq k}\frac{x-x_j}{x_k-x_j}  \right)
    \end{equation}
    Let $x_1= -1$, $x_2 = 0$, and $x_3 = 1$. Plot the functions $T_k(x)$ for $k = 1,\dots,3$, and describe what you see.
    \item
    \begin{enumerate}[i.]
      \item Sample $f(x)$ at the points $x_1 = -1$, $x_2 = 0$, and $x_3= 1$. Compute and plot the Lagrange interpolating polynomial given by these 3 equi-spaced points.
      \item Repeat part (b)(i) with 5 equi-spaced points, and again with 9 equi-spaced points. Plot your results and describe whether your approximation improves with more points.
    \end{enumerate}
    \item Repeat part (b) for the function $g(x)$.
    \item There is a well-known rule of thumb for improving numerical approximations: Increase the sampling density in the regions where the error is greatest. Does this rule of thumb seem to work with the functions $f$ and $g$?
\end{enumerate}


% !TEX root = Main.tex
\section{Gaussian Mixture Models (GMM)}
let $\boldsymbol{\theta}_k = (\boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k)$, $p(\mathbf{x};\theta_k) = \mathcal{N}(\mathbf{x} | \boldsymbol{\mu}_k, \Sigma_k)$\\
\textbf{Mixture Models:} $p_\theta(\mathbf{x}) = \sum_{k=1}^K \pi_k p(\mathbf{x};\theta_k)$\\
1. sample cluster index $j \sim Categorical(\pi)$\\
2. given $j$, sample data $x \sim \text{Normal}(\mu_j, \Sigma_j)$ \\
\textbf{Latent variables:} data point $x_i$ belongs to cluster $z_i$. $p(z_i=j)=\pi_j$ \\
\textbf{Max. Likelihood Estimation (MLE):}\\
$\argmax_\theta\sum_{n=1}^N \log \left( \sum_{k=1}^K \pi_k p(\mathbf{x}_n;\theta_k)\right)\\
\ge \sum_{n=1}^N \sum_{k=1}^K{q_{k,n}[\log p(\mathbf{x}_n;\theta_k) + \log \pi_k - \log q_{k,n}]}$\\

\subsection*{Expectation-Maximization (EM) for GMM}
\textbf{E-Step: (posterior over latent variables)}\\
$q_{k,n}^* = \operatorname{Pr}(z_n=k|x_n) = \frac{1}{Z}p(z_n=k)p(x_n|z_n=k) = \frac{\boldsymbol{\pi}_k p(x_n;\theta_k)}{\sum_{l=1}^K \boldsymbol{\pi}_l p(x_n;\theta_l)}$ \\
\textbf{M-Step: } $\boldsymbol{\mu}_k^* := \frac{\sum_{n=1}^N q_{k,n} \mathbf{x}_n}{\sum_{n=1}^N q_{k,n}}$
$, \boldsymbol{\pi}_k^* := \frac{1}{N} \sum_{n=1}^N q_{k,n}$\\
$\Sigma_k^* = \frac{\sum_{n=1}^N q_{k, n} (\mathbf{x}_n - \boldsymbol{\mu}_k)(\mathbf{x}_n - \boldsymbol{\mu}_k)^\top}{\sum_{n=1}^N q_{k,n}}$

\subsection*{Gaussian distribution}
Standard deviation $\sigma$, Mean $\mu$ \\
$p(x;\mu, \sigma) = \frac{1}{\sqrt{2\pi \sigma^2}} e^{- \frac{1}{2} \frac{(x-\mu)^2}{\sigma^2}}$\\
Covariance matrix $\Sigma$, Mean $\mu$ \\
$p(x;\mu, \Sigma) = \frac{1}{(2\pi)^{D/2} |\Sigma|^{1/2}} e^{- \frac{1}{2} (x-\mu)^T \Sigma^{-1} (x-\mu)}$

\subsection*{Model Order Selection (AIC / BIC for GMM)}
Trade-off between data fit (i.e. likelihood $p(\mathbf{X} | \theta)$) and complexity (i.e. \# of free parameters $\kappa(\cdot)$). For choosing $K$:\\
\textbf{Akaike Information Criterion}: $\operatorname{AIC}(\theta | \mathbf{X}) = -\log p_\theta(\mathbf{X}) + \kappa(\theta)$\\
\textbf{Bayesian Information Criterion}: $\operatorname{BIC}(\theta | \mathbf{X}) = -\log p_\theta(\mathbf{X}) + \frac{1}{2} \kappa(\theta) \log N$\\
\# of free params for GMM: $\kappa(\theta) = KD + K\frac{D(D+1)}{2} + (K - 1)$.\\


\section{Information Criteria}
In this section we review the concepts behind
Akaike's Information Criterion (AIC).

Akaike's original work is for
IID data, however it is extended to a regression type setting in a
straight forward way. Suppose that the conditional distribution of $Y$
given $\bx$ is know except for a $P$-dimensional parameter $\bb$. In
this case,  the probability density function of $\bY =
(Y_1,\dots,Y_n)$ can be written as
\begin{equation}
\label{parsimodel}
f_{\bY}(\by;\bX,\bb) \equiv \prod_{i=1}^n f(y_i;\bx_i,\bb)
\end{equation}
with $\bX$ the design matrix with rows $\bx_i$.

Assume that
there exists a true parameter vector $\bb^*$ defining a true
probability density denoted by
$f_{\bY}(\by;\bX,\bb^*)$. Given these assumptions, we wish to select
$\bb$, from one of the models
defined as in (\ref{modelpdef}), ``nearest'' to the true
parameter $\bb^*$ based on the observed data $\by$.
The principle behind Akaike's criterion is to
define ``nearest'' as the model that minimizes the Kullback-Leibler
Information Quantity
\begin{equation}
\label{KLdef}
\Delta(\bb^*;\bX,\bb) = \int \left\{\log
  f_\bY(\by;\bX,\bb^*) - \log
  f_\bY(\by;\bX,\bb)\right\} f_{\bY}(\by;\bX,\bb^*) \, d\by.
\end{equation}

The analytical properties of the Kullback-Leibler Information Quantity
are discussed in detail by Kullback (1959) \nocite{kull:1959}. Two
important properties
for Akaike's criterion are
\begin{enumerate}
\item $\Delta(\bb^*;\bX,\bb) > 0$ if
$f_{\bY}(\by;\bX,\bb^*) \neq f_{\bY}(\by;\bX,\bb)$
\item
$\Delta(\bb^*;\bX,\bb) = 0$  if and only if $f_{\bY}(\by;\bX,\bb^*) =
f_{\bY}(\by;\bX,\bb)$
\end{enumerate}
almost everywhere on the range of $\bY$.
The properties
mentioned suggest that finding the model that
minimizes the Kullback-Leibler
Information Quantity is an appropriate way to choose the ``nearest''
model.

Since the first term on the
right hand side of
(\ref{KLdef}) is constant over all models we consider, we may instead
maximize
\begin{eqnarray}
\nonumber
H(\bb) &=& \int \log
  f_\bY(\by;\bX,\bb) f_{\bY}(\by;\bX,\bb^*) \, d\by\\
\label{Hdef}
&=& \sum_{i=1}^n \int \log f(y_i;\bX,\bb) \, f(y_i;\bx_i,\bb^*) \, dy_i.
\end{eqnarray}

Let $\bbh_p$ be the maximum likelihood estimate under
Model(p). Akaike's procedure for model selection is based on choosing
the model which produces the estimate that maximizes
$\E_{\bb^*}\left[H(\bbh_p)\right]$ amongst all
competing models. Akaike then derives a criterion by constructing an
asymptotically unbiased estimate of $\E_{\bb^*}\left[H(\bbh_p)\right]$ based
on the observed data.

Notice that $H(\bbh_p)$ is a function,
defined by (\ref{Hdef}),
of the maximum likelihood estimate $\bbh_p$, which is a
random variable obtained from the observed data.
A natural estimator of
its expected value (under the true distribution of the data) is
obtained by substituting the empirical distribution of the data into
(\ref{Hdef})
resulting in the log likelihood equation evaluated at the maximum
likelihood estimate under model(p)
\[
l(\bbh_p) = \sum_{i=1}^n \log f(y_i;\bx_i,\bbh_p).
\]
Akaike noticed that in general  $l(\bbh_p)$ will
overestimate $\E_{\bb^*}\left[H(\bbh)\right]$.  In particular Akaike found
that under some regularity conditions
\[
E_{\bb^*}\left[l(\bbh_p) - H(\bbh_p)\right] \approx p .
\]
This suggests that larger
values of $p$ will result in smaller values
of $l(\bbh_p)$, which may be
incorrectly interpreted as a ``better''
fit, regardless of the true model. We need to ``penalize'' for larger
values of $p$ in order to obtain an unbiased estimate of
the ``closeness'' of the model.
This fact leads to the Akaike Information Criteria which is a
bias-corrected estimate given by
\begin{equation}
\label{AIC}
\mbox{AIC}(p) = - 2 l(\bbh_{p}) + 2p .
\end{equation}
See, for example, Akaike (1973) and Bozdogan (1987)  for
the details.


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}
\usepackage{booktabs}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\logit}{logit}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\cov}{cov}
\DeclareMathOperator*{\expec}{E}
\DeclareMathOperator*{\deriv}{d}
\DeclareMathOperator*{\const}{constant}

\begin{document}
\title{\textsf{Homework 6 for Bayesian Data Analysis}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 10.4a}
{
    Let the identity function $I$ indicate whether a sample $x$ is accepted or not. It follows that
    $$p(I=1) = \int{ p(I=1 | x) g(x) \deriv{x} }$$
    $$= \int{ \frac{f(x)}{c g(x)} g(x) \deriv{x} }$$
    $$= \int{ \frac{f(x)}{c} \deriv{x} } = \frac{1}{c}.$$

    Then, we have
    $$p(x | I=1) = p(x, I=1) / p(I=1)$$
    $$= g(x) \frac{f(x)}{c g(x)} / \frac{1}{c} = f(x).$$
}

\section*{Question 10.4b}
{
    To make the integral above valid, it requires 1) the denominator $g(x)>0$ for any $x$ that satisfies $f(x)>0$, and 2) the probability $f(x) / c g(x) \leq 1$ for any $x$. These two requirements cannot be met unless $f(x)/g(x)$ is bounded.
}

\section*{Question 10.6}
{
    Without loss of generality, suppose the posterior distribution is normal $N(0, 3)$, which matches, in mode and curvature (variance), to the standard $t_3$ distribution.
    \begin{table}[H]
        \centering
        \caption{Estimated posterior expectation and variance}
        \begin{tabular}{@{}clc@{}clc@{}}
        \toprule
        sample size & \multicolumn{1}{c}{E (estimated)} & \multicolumn{1}{c}{E (true)} & \multicolumn{1}{c}{Var (estimated)} & \multicolumn{1}{c}{Var (true)}  \\ \midrule
        100 & 0.076218 & 0 & 3.043008 & 3   \\
        10000 & -0.035262 & 0 & 3.051913 & 3  \\ \bottomrule
        \end{tabular}
    \end{table}
    The estimation is accurate even for a small sample size ($S=100$).
    Some importance weights are extremely small, but no weights are extremely large. This implies well behaved importance weights.
    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.6\linewidth]{1_weight_100.png}
        \includegraphics[width = 0.6\linewidth]{1_log_weight_100.png}
        \caption{Sample size = 100}
    \end{figure}
    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.6\linewidth]{1_weight_10000.png}
        \includegraphics[width = 0.6\linewidth]{1_log_weight_10000.png}
        \caption{Sample size = 10000}
    \end{figure}

}

\section*{Question 10.7}
{
    Without loss of generality, suppose the posterior distribution is standard $t_3$, which matches, in mode and curvature (variance), to the normal $N(0, 3)$ distribution.
    \begin{table}[H]
        \centering
        \caption{Estimated posterior expectation and variance}
        \begin{tabular}{@{}clc@{}clc@{}}
        \toprule
        sample size & \multicolumn{1}{c}{E (estimated)} & \multicolumn{1}{c}{E (true)} & \multicolumn{1}{c}{Var (estimated)} & \multicolumn{1}{c}{Var (true)}  \\ \midrule
        100 & 0.084248 & 0 & 1.648969 & 1   \\
        10000 & -0.008052 & 0 & 2.193678 & 1  \\ \bottomrule
        \end{tabular}
    \end{table}
    The estimation is not accurate even for a large sample size ($S=10000$).
    Some importance weights are extremely large, especially for large sample size. This implies the importance weights are too variate, and therefore, explains why our estimation is even worse when we have larger sample size:

    Since the $t$ distribution has heavier tails than the normal distribution does, we can hardly draw a sample at the tail from simulation based on normal distribution. With a large sample size, it is more likely that such a tail sample is drawn, leading to a large importance weight. This would increase the variance of our estimation, as the weights are not so uniform. According to the textbook, the estimation is sensitive to an outlier of extremely large importance weight (when we try to estimate heavy tailed distribution with a light tailed one), while it is not when we try to estimate in the reversed way.
    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.6\linewidth]{2_weight_100.png}
        \includegraphics[width = 0.6\linewidth]{2_log_weight_100.png}
        \caption{Sample size = 100}
    \end{figure}
    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.6\linewidth]{2_weight_10000.png}
        \includegraphics[width = 0.6\linewidth]{2_log_weight_10000.png}
        \caption{Sample size = 10000}
    \end{figure}
}

\section*{Source Code in R}
{
    \begin{lstlisting}[language=R]
        set.seed(2018)

        weight <- function(x) {
          return (dnorm(x, sd = sqrt(3)) / dt(x, 3))
        }

        for (S in c(100, 10000)) {
          t = rt(S, 3)
          w = unlist(lapply(t, weight))

          png(sprintf("1_weight_%d.png", S), width = 640, height = 480)
          hist(w, breaks = 100)
          dev.off()

          png(sprintf("1_log_weight_%d.png", S), width = 640, height = 480)
          hist(log(w), breaks = 100)
          dev.off()

          first.order = mean(t * w) / mean(w)
          second.order = mean(t^2 * w) / mean(w)

          expectation = first.order
          variance = second.order - first.order^2

          print(sprintf("1: size = %d, expectation = %f,
                        variance = %f", S, expectation, variance))
        }

        weight <- function(x) {
          return (dt(x, 3) / dnorm(x, sd = sqrt(3)))
        }

        for (S in c(100, 10000)) {
          n = rnorm(S, sd = sqrt(3))
          w = unlist(lapply(n, weight))

          png(sprintf("2_weight_%d.png", S), width = 640, height = 480)
          hist(w, breaks = 100)
          dev.off()

          png(sprintf("2_log_weight_%d.png", S), width = 640, height = 480)
          hist(log(w), breaks = 100)
          dev.off()

          first.order = mean(n * w) / mean(w)
          second.order = mean(n^2 * w) / mean(w)

          expectation = first.order
          variance = second.order - first.order^2

          print(sprintf("2: size = %d, expectation = %f,
                        variance = %f", S, expectation, variance))
        }
    \end{lstlisting}
}

\clearpage
\end{document}


 \subsection{Model (what is $\theta$)}

 In our project, we analyze the time-series, i.e. the sequence of given data
 \begin{equation}
  \label{eq:timeseries}
	x_0, x_1, \dots, x_{T-1},
 \end{equation}
 where $x_t \in \mathbb{R}^{\mathrm{xdim}}$ and $\mathrm{xdim} \in \mathbb{N}$ is the number of values (measurements) in each time step.
 We try to understand the inner mechanism (dynamics) of the sequence. \newline

 As the first step of analyze of given data, we can use classical tools and easily compute "static"\footnote{I decided to use this term, because following properties does not take into account the time} statistics, like average, variance, deviation and other moments or central moments.
 These values provide us the basic properties of the given set of data, however they does not take into account the fact, that we are not working with only set, but we analyze the sequence.

 One of the most typical way how to analyze sequences is to appoximate the given data by much more simplier function. Afterwards, analyzing this approximating function provides us the basic knowledge of sequence behaviour with respect to time.
 The approximation is performed to be "as good as possible", i.e. in a such way, that the error of the approximation is as small as possible.

 The most simplest approximation function is linear\footnote{in fact, the simplest approximation function is constant function; however the constant function is in this case the average of given set of data; constant function is constant with respect to time and therefore it cannot reflect the dynamics of the sequence}.
 In this case, we are talking about \emph{linear regression}.

 For the simplicity, in the following we will suppose one dimensional data $\mathrm{xmem} = 1$, i.e. $x(t) \in \mathbb{R}$.
 Provided observations could be easily generalized to more dimensions.

 Suppose that all given data \eqref{eq:timeseries} linearly depends on the time, i.e. each data point could be written in form $x(t) = \alpha_0 + \alpha_1 t$, where $\alpha_0,\alpha_1 \in \mathbb{R}$ are unknown time-independent parameters of this linear \emph{model}.
 However, general sequence is not generated by linear function, therefore in each time-step we make an error. To be more exact, we rather write
 \begin{equation}
  \label{eq:linear}
  x(t) = \alpha_0 + \alpha_1 t + \varepsilon_t, ~~t = 0,\dots,T,
 \end{equation}
 where $\varepsilon_t$ is an error of approximation (hopefully small for all $t$).
 If we denote
 \begin{displaymath}
  \begin{array}{rcl}
   x & = & [x_0, \dots, x_{T-1}]^T \in \mathbb{R}^T, \\[5mm]
   Z & = & \left[
    \begin{array}{ccccc}
     1 & 1 & 1 & \dots & 1 \\
     0 & 1 & 2 & \dots & T-1
    \end{array}
     \right] \in \mathbb{R}^{2,T}, \\[5mm]
   y & = & [\alpha_0,\alpha_1]^T \in \mathbb{R}^2, \\[5mm]
   \varepsilon & = & [\varepsilon_0,\dots,\varepsilon_{T-1}] \in \mathbb{R}^T,
  \end{array}
 \end{displaymath}
 then the system of equations \eqref{eq:linear} could be written in form
 \begin{equation}
  \label{eq:linear2}
  x = Z^T y + \epsilon.
 \end{equation}
 We want to perform the approximation (i.e. find $y$) in the best way as possible. We minimize the size of the error $\Vert \varepsilon \Vert \rightarrow \min\limits_{y}$.
 Using \eqref{eq:linear2}, we can substitute and we get optimization problem
 \begin{equation}
  \label{eq:linear_opt}
  \hat{y} = \arg\min\limits_{y} \Vert \varepsilon \Vert  = \arg\min\limits_{y} \Vert Z^T y - x \Vert = \arg\min\limits_{y} \underbrace{\frac{1}{2}\Vert Z^T y - x \Vert^2}_{ = \Psi(y)}
 \end{equation}
 Please, notice that $\Psi(y)$ is quadratic function
 \begin{displaymath}
  \Psi(y) = \frac{1}{2}\Vert Z^T y - x \Vert^2 = \frac{1}{2}\langle Z^T y - x, Z^T y - x \rangle = \frac{1}{2}y^T Z Z^T y - y^T Z x + \frac{1}{2}x^T x,
 \end{displaymath}
 therefore the gradient is given by
 \begin{displaymath}
  \nabla \Psi(y) = ZZ^T y - Z x
 \end{displaymath}
 and the necessary optimality condition of \eqref{eq:linear_opt} is given by the system of linear equations\footnote{yes, this is a least-square solution of our first naiive approach $x(t) = \alpha_0 + \alpha_1 t$}
 \begin{displaymath}
  ZZ^T y = Zx .
 \end{displaymath}

% \begin{example}
%  Let us consider data
%  \begin{displaymath}
%   x_0 =
%  \end{displaymath}
% \end{example}

 Using a simple generalization idea, we are able to extend the linear regression model to polynomial models
 \begin{displaymath}
  x(t) = p(t) + \varepsilon_t, p \in \mathcal{P}_n,
 \end{displaymath}
 where $\mathcal{P}_n$ is a vector space of polynomial function of degree $n \in \mathbb{N}$.

 All parameters of the model will be denoted by a $\Theta$.

 \subsection{Non-stationarity (what is $\gamma$)}