\documentclass[a4paper,man,natbib]{apa6}
\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{setspace}

%% Images
% \usepackage{graphicx}
% 	\graphicspath{ {./images/} }
% 	\usepackage{caption}

% Minted
	\usepackage[cache=false]{minted}
	\usepackage{xcolor}
		\definecolor{light-gray}{gray}{0.97}
	\usemintedstyle{vs}

% Title
	\title{Intro to Computational Statistics: Homework 6}
	\shorttitle{HW06}
	\author{Author: Brandon Hosley}
	\affiliation{Instructor: Liang Kong, Ph.D}

%\abstract{}

\usepackage{pdfpages}

\begin{document}
\maketitle
\singlespacing

\subsection*{Q1: Use SAS data set students.sas7bdat (from homework 1) to obtain a regression model to predict the Score of Final Exam by using Intelligence Quotient, Score of Mathematics part of SAT, and Score of Pilot Examination as regressors. What is the estimate model? }
\inputminted[breaklines,bgcolor=light-gray]{sas}{./Snippets/DAT502-HW6-1.sas}
Estimate Model: \\
FEXAM = -0.18062 + 0.40527*IQ + 0.001816*MSAT + 0.37603*PEXAM

\includepdf[pages=-]{./Snippets/SAS Output HW6-1.pdf}

\subsection*{Q2: What is the estimate model of the above question with all the coefficients are significant at 0.05 level of significance. }
\inputminted[breaklines,bgcolor=light-gray]{sas}{./Snippets/DAT502-HW6-2.sas}
Estimate Model: \\
FEXAM = -0.28666 + 0.47674*IQ + 0.38137*PEXAM

\includepdf[pages=-]{./Snippets/SAS Output HW6-2.pdf}

\subsection*{Q3: Assuming Intelligence Quotient is the only information available, construct a 98\% prediction interval for the Score of Final Exam giving a score of 110 for Intelligence Quotient. }
\inputminted[breaklines,bgcolor=light-gray]{sas}{./Snippets/DAT502-HW6-3.sas}

$ 110 * 0.81561 - 13.47104 \pm 2.33 * 11.82143 $ \\
$ 89.7171 \pm 27.543919 $ \\
There is a 98\% confidence that 110 IQ will result in between 62.173181 and 117.26; though it appears that 100 is a likely upper bound.

\includepdf[pages=-]{./Snippets/SAS Output HW6-3.pdf}

\subsection*{Q4: Use SAS data set regp3.sas7bdat (located at "Data set" folder) to fit a simple linear regression model : $Y = \beta_0 + \beta_1X + \epsilon$. What is the 90\% confidence interval for $\beta_1$?
 }
\inputminted[breaklines,bgcolor=light-gray]{sas}{./Snippets/DAT502-HW6-4.sas}

$\beta_1$ has a 90\% confidence interval of 4.03634 and 4.24609

\includepdf[pages=-]{./Snippets/SAS Output HW6-4.pdf}

\subsection*{Q5: Use SAS data set regp3.sas7bdat (located at "Data set" folder) to fit the "best" linear regression model : $Y = \beta_0 + f(X; \beta_1, \beta_2, \beta_3, \ldots) + \epsilon$. For example, $f(X; \beta_1, \beta_2, \beta_3, \ldots)$ could be $\beta_1X^{0.5} + \beta_2 X^{1.5} + \beta_3 X_3$ or $\beta_1X_0 + \beta_2 X_2 \ldots$ . What is the 97\% confidence interval for the average values of $Y$ given $X=200$? }
\inputminted[breaklines,bgcolor=light-gray]{sas}{./Snippets/DAT502-HW6-5.sas}

For a $\beta_1X + \beta_2 X^2 + \beta_3$ model there is a 97\% confidence interval of $Y$ between $228.2901$ and $234.7784$ for the value of $X=200$

\includegraphics[width=0.8\linewidth]{./images/image5.png}
\includepdf[pages=-]{./Snippets/SAS Output HW6-5.pdf}

\end{document}

2021-Spr Intro to Computational Stats/HW06/bhosl2-HW06.tex


\chapter{Steepest Descent and Least-Mean-Square Adaptive Filters}
\label{ch:SD_LMS}

\section{Review of the Basics}
\subsection{The Eigenvalue Decomposition}
If a square $M\times M$ matrix $\vect{A}$ has an eigenvalue decomposition (EVD), it can be written as
\bmath
  \vect{A} = \vect{X\Lambda X}^{-1}\qquad\iff\qquad\vect{AX}=\vect{X\Lambda}
\emath
where $\vect{X}$ contains the $M$ linearly independent eigenvectors of $\vect{A}$, and the diagonal matrix $\vect{\Lambda}$ contains the $M$ eigenvalues $\{\lambda_m\}_{m=1}^M$ of $\vect{A}$. If $\vect{A}$ does not have an eigenvalue decomposition, it is said to be \textit{defective}.
\subsubsection{Matrix Powers}
Let $\vect{A}$ have an EVD. Then
\begin{align}
  \vect{A} & = \vect{X\Lambda X}^{-1}\\
  \vect{A}^2 & \underset{\vdots}{=} \vect{X\Lambda X}^{-1}\vect{X\Lambda X}^{-1}=\vect{X\Lambda}^2\vect{X}^{-1}\\
%  &\ \,\vdots\notag\\
  \vect{A}^n &= \vect{X\Lambda}^n\vect{X}^{-1}\ .
\end{align}
\subsubsection{EVD of Special Matrices}
Let $\vect{A}$ be a symmetric matrix. Then
\begin{itemize}
  \item[] $\vect{A}$ has an EVD,
  \item[] $\vect{A}=\vect{X\Lambda X}^T$ ($\vect{X}$ is an orthogonal matrix), and
  \item[] $\lambda_m\in\mathbb{R}$ for $m=1,\ldots,M$
\end{itemize}
Let $\vect{A}$ be a p.d. matrix. Then
\begin{itemize}
  \item[] $\lambda_m\in\mathbb{R}^+$ for $m=1,\ldots,M$.
\end{itemize}
where $\mathbb{R}^+$ is the set of all positive real numbers.
\subsubsection{Matrix Trace}
The trace of $\vect{A}$ is
\bmath
  	\tr{\vect{A}} = \sum_{m=1}^M a_{mm} = \sum_{m=1}^M \lambda_m = \tr{\vect{\Lambda}}\ .
\emath

\subsubsection{Condition Number of a Normal Matrix}
If $\vect{A}$ is a normal matrix (i.e., $\vect{A}^T\vect{A}=\vect{A}\vect{A}^T$), then the condition number of $\vect{A}$ is
\bmath
  \kappa(\vect{A}) = \left|\frac{\lambda_\text{max}}{\lambda_\text{min}}\right| = \chi(\vect{A})
\emath
where $\chi(\vect{A})$ is the eigenvalue spread of $\vect{A}$.

\section{The Wiener-Hopf Equations}
The adaptive filtering problem with time-varying filter coefficients is shown in Fig.~\ref{fig:nonwss_block_diagram1}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/nonwss_block_diagram}
  \caption{Block diagram of adaptive filtering in a non-WSS environment.}
  \label{fig:nonwss_block_diagram1}
\end{figure}
From the figure, we have that
\begin{itemize}
  \item[$u(n)$:] zero-mean input signal
  \item[$w_m(n)$:] $M$-tap FIR-filter with impulse response $w_0(n),w_1(n),\ldots,w_{M-1}(n)$
  \item[$y(n)$:] output signal given by $y_n = \sum_{m=0}^{M-1}w_m(n) u(n-m)$
  \item[$d(n)$:] zero-mean desired signal
  \item[$e(n)$:] error signal
\end{itemize}
\subsubsection{WSS Signals}
When $u(n)$ and $d(n)$ are jointly WSS, the filter coefficients are not time-varying. Define
\begin{align}
  \vect{w} &= \bbmtx w_0 & w_{1} & \cdots & w_{M-1}\ebmtx^T\\
  \vect{u}(n) &= \bbmtx u(n) & u(n-1) & \cdots & u(n-M+1)\ebmtx^T\ .
\end{align}
Then
\bmath
  e(n) = d(n) - y(n) = d(n)-\vect{u}^T(n)\vect{w}\ .
\emath
We wish to minimise
\begin{align}
  J_1(\vect{w}) &= E[e(n)^2] = E[d(n)^2] + \vect{w}^T E[\vect{u}(n)\vect{u}^T(n)] \vect{w}-2\vect{w}^T E[\vect{u}(n) d(n)]\notag\\
  &= \sigma_d^2+\vect{w}^T\vect{R}_u\vect{w}-2\vect{w}^T\vect{r}_{ud}
\end{align}
w.r.t. $\vect{w}$. The minimiser
\bmath
  \vect{w}_o = \vect{R}_u^{-1}\vect{r}_{ud}
\emath
is the unique solution to the Wiener-Hopf equations
\bmath
  \vect{R}_u\vect{w} = \vect{r}_{ud}\ ,
\emath
provided that $\vect{R}_u$ is p.d.
\subsubsection{Non-WSS Signals}
If $u(n)$ and $d(n)$ are \textit{not} jointly WSS, the optimal filter coefficients are time-varying, and we have that
\bmath
  \vect{R}_u(n)\vect{w}(n) = \vect{r}_{ud}(n)
\emath
where
\begin{align}
  \vect{R}_u(n) &=
  \bbmtx
    r_u(n,n) & \cdots & r_u(n,n-M+1)\\
    \vdots & \ddots & \vdots\\
    r_u(n-M+1,n) & \cdots & r_u(n-M+1,n-M+1)\\
  \ebmtx\\
  \vect{r}_{ud}(n) &= \bbmtx r_{ud}(n,n) & \cdots & r_{ud}(n-M+1,n)\ebmtx^T\ .
\end{align}
We could find the optimal solution by calculating
\bmath
  \vect{w}_o(n) = \vect{R}_u^{-1}(n)\vect{r}_{ud}(n)
\emath
for every time index $n$. However, this approach may suffer from problems such as that
\begin{enumerate}
  \item the computational complexity is high, and
  \item the statistics is unknown.
\end{enumerate}
The steepest descent algorithm solves the first problem, and the least-mean-square (LMS) adaptive filter solves both problems.

\section{The Method of Steepest Descent}
\subsection{Basic Idea}
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/steepest_descent_idea_2d}
  \caption{The first three iterations of the steepest descent algorithm for a one-dimensional filter vector.}
  \label{fig:steepest_descent_idea_2d}
\end{figure}
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/steepest_descent_idea_3d}
  \caption{The first three iterations of the steepest descent algorithm for a two-dimensional filter vector.}
  \label{fig:steepest_descent_idea_3d}
\end{figure}
The basic idea of the steepest descent (SD) algorithm is to find the unique solution $\vect{w}_o$ of the Wiener-Hopf equations through a series of steps, starting from some point $\vect{w}(0)$. The steps are taken in the opposite direction of the gradient $\vect{g}(\vect{w}(n))$. This idea is illustrated in Fig.~\ref{fig:steepest_descent_idea_2d} and Fig.~\ref{fig:steepest_descent_idea_3d}. We update the filter coefficients by a recursive update equation given by
\bmath
  \boxed{\vect{w}(n+1) = \vect{w}(n)-\frac{\mu}{2}\vect{g}(\vect{w}(n))}
\emath
where $\mu$ is the step-size and $\vect{g}(\vect{w}(n))$ is the gradient of the cost function $J_1(\vect{w}(n))$. The gradient is
\bmath
  \vect{g}(\vect{w}(n)) = 2\vect{R}_u(n)\vect{w}(n) - 2\vect{r}_{ud}(n)\ .
\emath
Since the gradient is a deterministic function, the evolution of the filter coefficient vector is also a deterministic function.

Define the weight error
\bmath
  \Delta\vect{w}(n) = \vect{w}_o-\vect{w}(n)\ .
\emath
In order for the SD algorithm to converge to the solution $\vect{w}_o$, we must require that the step-size $\mu$ is selected such that
\bmath
  \lim_{n\to\infty} \Delta\vect{w}(n) = 0
  \label{eq:sd_stability_req}
\emath
when $u(n)$ and $d(n)$ are jointly WSS. If this is true, the SD algorithm is said to be \textit{stable}. Moreover, we would like to select $\mu$ such that the $\Delta\vect{w}(n)$ becomes small as fast as possible.

\subsection{Transient Analysis}
Assume that $u(n)$ and $d(n)$ are jointly WSS. We then have that
\begin{align}
  \Delta\vect{w}(n) &= \vect{w}_o - \left(\vect{w}(n-1)-\frac{\mu}{2}\vect{g}(\vect{w}(n-1))\right)\\
  &= \Delta\vect{w}(n-1) + \mu(\vect{R}_u\vect{w}(n-1)-\vect{R}_u\vect{w}_o)\\
  &= (\vect{I}-\mu\vect{R}_u)\Delta\vect{w}(n-1)\\
  &= (\vect{I}-\mu\vect{R}_u)(\vect{I}-\mu\vect{R}_u)\Delta\vect{w}(n-2)\\
  &= (\vect{I}-\mu\vect{R}_u)^n\Delta\vect{w}(0)\ .
\end{align}
Since the correlation matrix $\vect{R}_u$ is symmetric, it has an eigenvalue decomposition $\vect{R}_u=\vect{X\Lambda X}^T$ where $\vect{X}$ is an orthogonal matrix. That is, we may write
\bmath
  \Delta\vect{w}(n) = \vect{X}(\vect{I}-\mu\vect{\Lambda})^n\vect{X}^T\Delta\vect{w}(0)
\emath
where $\vect{I}-\mu\vect{\Lambda}$ is a diagonal matrix with the $m$'th diagonal given by the mode $1-\mu\lambda_\textup{m}$. For \eq{eq:sd_stability_req} to be fulfilled from any starting point $\vect{w}(0)$, we must therefore require that
\bmath
  \lim_{n\to\infty} (1-\mu\lambda_\textup{m})^n = 0\ ,\quad\text{for }m=1,2,\ldots,M\ .
  \label{eq:sd_mode}
\emath
Thus
\begin{alignat}{2}
  && -1 < 1-&\mu\lambda_\textup{m} < 1\ ,\quad\text{for }m=1,2,\ldots,M\\
  \ArrowBetweenLines
  && -1 < 1-&\mu\lambda_\textup{max} < 1
\end{alignat}
Solving for $\mu$ leads to that the SD algorithm is stable if
\bmath
  \boxed{0 < \mu < \frac{2}{\lambda_\textup{max}}}\ .
\emath
From \eq{eq:sd_stability_req}, we see that
\begin{alignat*}{2}
  &|1-\mu\lambda_m|\text{ is close to 0} &\qquad &\iff\qquad\text{ mode with fast convergence}\\
  &|1-\mu\lambda_m|\text{ is close to 1} &\qquad &\iff\qquad\text{ mode with slow convergence}
\end{alignat*}
The optimal step-size $\mu_o$ is therefore given by
\begin{alignat}{2}
  \mu_o&= \argmin_{\mu} \max_{\lambda_m} & |1-\mu\lambda_m|&\\
  &\text{subject to }             & |1-\mu\lambda_m|&<1\ ,\text{ for } m=1,\ldots,M\ .
\end{alignat}
That is, $\mu_o$ minimises the value of the largest (slowest) mode. We solve this optimisation problem by the use of Fig~\ref{fig:optimal_step_sd}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/optimal_step_sd}
  \caption{Finding the optimal step-size.}
  \label{fig:optimal_step_sd}
\end{figure}
From Fig~\ref{fig:optimal_step_sd}, we see that the optimal step-size $\mu_o$ satisfies
\begin{alignat}{2}
   && 1-\mu_o\lambda_\textup{min} &= -(1-\mu_o\lambda_\textup{max})\\
   \ArrowBetweenLines
   && \alignedbox{\mu_o}{=\frac{2}{\lambda_\textup{max}+\lambda_\textup{min}}}\ .
\end{alignat}
For the optimal step-size, the slowest modes $\pm(1-\mu_o\lambda_\textup{max})$ and $\pm(1-\mu_o\lambda_\textup{min})$ are therefore given by
\bmath
  \pm\frac{\lambda_\textup{max}-\lambda_\textup{min}}{\lambda_\textup{max}+\lambda_\textup{min}} = \pm\frac{\kappa(\vect{R}_u)-1}{\kappa(\vect{R}_u)+1}
\emath
where $\kappa(\vect{R}_u)$ is the condition number of the correlation matrix $\vect{R}_u$. Thus, if the condition number is large, the slowest modes are close to one, and the convergence of the SD algorithm is slow. Conversely, if the condition number is close to one, the slowest modes are close to zero, and the convergence of the SD algorithm is fast.

\subsubsection{Learning Curve}
The cost function can be written as
\begin{align}
  J_1(\vect{w}(n)) &= J_1(\vect{w}_o) + \Delta\vect{w}^T(n)\vect{R}_u\Delta\vect{w}(n)\\
  &= J_1(\vect{w}_o) + \Delta\vect{w}^T(0)\vect{X}(\vect{I}-\mu\vect{\Lambda})^n\vect{\Lambda}(\vect{I}-\mu\vect{\Lambda})^n\vect{X}^T\Delta\vect{w}(0)\\
  &= J_1(\vect{w}_o)+\sum_{m=1}^M\lambda_m(1-\mu\lambda_m)^{2n}\left(\vect{x}_m^T\Delta\vect{w}(0)\right)^2\ .
\end{align}
The plot of $J_1(\vect{w}(n))$ as a function of $n$ is called the learning curve, and a sketch of it is shown in Fig.~\ref{eq:sd_learning_curve}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/learning_curve_sd}
  \caption{The learning curve of the steepest descent algorithm.}
  \label{eq:sd_learning_curve}
\end{figure}

\section{Least-Mean-Square Adaptive Filters}
\subsection{Basic Idea}
Recall, that the gradient of the mean-squared error cost function $J_1(\vect{w}(n))$ is
\bmath
  \vect{g}(\vect{w}(n)) = \frac{\partial E[e^2(n)]}{\partial\vect{w}(n)} = 2E\left[\frac{\partial e(n)}{\partial\vect{w}(n)}e(n)\right] = -2E[\vect{u}(n)e(n)]\ .
\emath
Often, we do not know the gradient, and we therefore have to estimate it. A simple estimate is
\bmath
  \hat{\vect{g}}(\vect{w}(n)) = -2\vect{u}(n)e(n)\ .
\emath
If we replace the gradient in the steepest descent algorithm with this estimate, we obtain
\bmath
  \boxed{\vect{w}(n+1) = \vect{w}(n)+\mu\vect{u}(n)e(n)}
\emath
which is called the least-mean-square (LMS) algorithm. A block diagram of the LMS filter is depicted in Fig.~\ref{eq:block_diagram_w_feedback}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/block_diagram_w_feedback}
  \caption{Typical block diagram of an adaptive filter in the case where the statistics is unknown.}
  \label{eq:block_diagram_w_feedback}
\end{figure}
The LMS algorithm is the simplest \textit{stochastic gradient method} (SGM). The naming of the SGM refers to that the gradient is a stochastic process. This has the following consequences.
\begin{enumerate}
  \item The filter coefficient vector $\vect{w}(n)$ is also a stochastic process. This makes the analysis of the SGMs difficult. To see this, consider the cost function
\begin{align}
  J_1(\vect{w}(n)) &= E[e^2(n)]\notag\\
  &= \sigma_d^2(n) + E[\vect{w}^T(n)\vect{u}(n)\vect{u}^T(n)\vect{w}(n)]-2E[\vect{w}^T(n)\vect{u}(n)d(n)]\ .
\end{align}
For the stochastic gradient methods, it is not easy to evaluate the expectations in the cost function since $\vect{w}(n)$ and $\vect{u}(n)$ are not independent (unless $M=1$ and $u(n)$ is a white process).
  \item The weight error $\Delta\vect{w}(n)=\vect{w}_o-\vect{w}(n)$ never goes permanently to zero. That is, in steady-state ($n\to\infty$), the filter coefficient vector $\vect{w}(\infty)$ fluctuates randomly around the optimum $\vect{w}_o$. Consequently, the mean-square error (MSE) $J_1(\vect{w}(\infty))$ in steady-state is larger than $J_\textup{min} = J_1(\vect{w}_o)$ by an amount referred to as the \textit{excess mean-square} error (EMSE)
  \bmath
    J_\textup{ex} = J_1(\vect{w}(\infty)) - J_\textup{min}\ .
  \emath
  The ratio of the EMSE to the MSE is called the \textit{misadjustment}
  \bmath
    \mathcal{M} = \frac{J_\textup{ex}}{J_\textup{min}} = \frac{J_1(\vect{w}(\infty))}{J_\textup{min}}-1\ .
  \emath
\end{enumerate}
In Lecture~\ref{ch:NLMS_APA}, we say more about these parameters and the analysis of adaptive filters.

\subsection{Transient Analysis}
Assume that $d(n)$ is given by\footnote{This model is quite popular for the analysis of adaptive filters. We say more about this model in Sec.~\ref{ssec:analysis_model}.}
\bmath
  d(n) = \vect{u}^T(n)\vect{w}_o + v(n)
  \label{eq:desired_model}
\emath
where $v(n)$ is white Gaussian noise with variance $\sigma_v^2$ and uncorrelated with $u(n)$.  Moreover, assume that the random variables of the stochastic process $\vect{u}(n)$ are independent and identically distributed (IID) with a Gaussian distribution and correlation matrix $\vect{R}_u$. This assumption is infeasible unless $M=1$. Nevertheless, we make this assumption anyway in order to make the transient analysis as simple as possible. We are concerned with
\begin{alignat*}{2}
  \lim_{n\to\infty} E[\Delta\vect{w}(n)] &= 0 &\qquad &\text{(Convergence in mean)}\\
  \lim_{n\to\infty} E[\|\Delta\vect{w}(n)\|^2] &= c < \infty &\qquad &\text{(Convergence in mean-square)}
\end{alignat*}
where $\|\Delta\vect{w}(n)\|^2=\Delta\vect{w}^T(n)\Delta\vect{w}(n)$ is the vector 2-norm, and $c$ in some positive constant\footnote{Apparently, the definition of convergence in mean-square used for adaptive filtering analysis is different from the usual definition where $c=0$.}.

\subsubsection{Convergence in the Mean}
By subtracting the LMS recursion from the optimal filter coefficients, we obtain
\begin{align}
  \Delta\vect{w}(n) &= \vect{w}_o - \left[\vect{w}(n-1)+\mu\vect{u}(n-1)e(n-1)\right]\\
  &= \Delta\vect{w}(n-1) - \mu\vect{u}(n-1)\left[d(n-1)-\vect{u}^T(n-1)\vect{w}(n-1)\right]\\
  &= \left[\vect{I}-\mu\vect{u}(n-1)\vect{u}^T(n-1)\right]\Delta\vect{w}(n-1)-\mu\vect{u}(n-1)v(n-1)
  \label{eq:recur_weight_err}
\end{align}
Taking the expectation of both sides and using the above assumptions, we obtain
\begin{align}
  E[\Delta\vect{w}(n)] &= (\vect{I}-\mu\vect{R}_u)E[\Delta\vect{w}(n-1)]\\
  &= \vect{X}(\vect{I}-\mu\vect{\Lambda})^n\vect{X}^T E[\Delta\vect{w}(0)]\ .
\end{align}
This recursion is the same as in the transient analysis of the steepest descent algorithm. We therefore have that the LMS algorithm is stable in the mean if
\bmath
  \boxed{0 < \mu < \frac{2}{\lambda_\textup{max}}}\ .
\emath
The bound given above ensures convergence in the mean, but places no constraint on the variance of $\Delta\vect{w}(n)$. Furthermore, since $\vect{R}_u$ is unknown, $\lambda_\textup{max}$ is unknown, and we have to estimate or upper bound it. We have that
\bmath
  \lambda_\textup{max} \leq \tr{\vect{R}_u} = \sum_{m=1}^M r_u(0) = M r_u(0) = M E[u^2(n)]\ .
\emath
The expected power $E[u^2(n)]$ can be estimated as
\bmath
  \hat{E}[u^2(n)] = \frac{1}{M}\vect{u}^T(n)\vect{u}(n)\ .
\emath
Note that the estimation accuracy increases with the filter length $M$. From Parseval's Theorem, we also have that
\bmath
  E[u^2(n)] = \frac{1}{2\pi}\int_{-\pi}^\pi S_u(\omega)d\omega \leq \frac{1}{2\pi}\int_{-\pi}^\pi S_\textup{max}d\omega = S_\textup{max}
\emath
where $S_u(\omega)$ is the power spectral density of $u(n)$. Thus, we have that
\bmath
  \boxed{\frac{2}{M S_\textup{max}}\leq \frac{2}{\tr{\vect{R}_u}} = \frac{2}{M E[u^2(n)]} \leq \frac{2}{\lambda_\textup{max}}}\ ,
\emath
provided that the desired signal model is given by \eq{eq:desired_model} and that $\vect{u}(n)$ is an IID random process.

%Under the assumption that the correlation matrix of the input signal is white
%\bmath
%  \vect{R}_u = \sigma_u^2\vect{I}
%\emath
%where $\sigma_u^2$ is the variance of the input signal, $\lambda_\textup{min}$ and $\lambda_\textup{max}$ are both equal to $\sigma_u^2$. Optimising the step size in this case does NOT lead to the NLMS!

\subsubsection{Convergence in the Mean-Square}
From \eq{eq:recur_weight_err}, we have that
\begin{align}
  \|\Delta\vect{w}(n)\|^2={}& \|\left[\vect{I}-\mu\vect{u}(n-1)\vect{u}^T(n-1)\right]\Delta\vect{w}(n-1)-\mu\vect{u}(n-1)v(n-1)\|^2\notag\\
  ={}& \|\left[\vect{I}-\mu\vect{u}(n-1)\vect{u}^T(n-1)\right]\Delta\vect{w}(n-1)\|^2\notag\\
  &{}+\mu^2v^2(n-1)\|\vect{u}(n-1)\|^2\notag\\
  &{}-2\mu v(n) \vect{u}^T(n-1)\left[\vect{I}-\mu\vect{u}(n-1)\vect{u}^T(n-1)\right]\Delta\vect{w}(n-1)
\end{align}
Taking the expectation on both sides and using the above assumptions, we obtain
\bmath
  E[\|\Delta\vect{w}(n)\|^2] = E\left[\|\left[\vect{I}-\mu\vect{u}(n-1)\vect{u}^T(n-1)\right]\Delta\vect{w}(n-1)\|^2\right]+\mu^2J_\textup{min}\tr{\vect{\Lambda}}
\emath
where $J_\textup{min}=\sigma_v^2$. Evaluating the expected value of the first term and finding the values of the step-size $\mu$ for which $E[\|\Delta\vect{w}(n)\|^2]$ converges in the mean-square require a lot of work. In Appendix~\ref{app:lms_analysis}, we show how this can be done. Alternatively, a derivation can also be found in \cite[pp.~452--465]{Sayed2003}. The final result is that the LMS filter is mean-square stable (and thus stable in the mean) if and only if the step-size satisfies \cite[pp.~462]{Sayed2003}
\bmath
  \boxed{f(\mu) = \frac{\mu}{2}\sum_{m=1}^M \frac{\lambda_m}{1-\mu\lambda_m} = \frac{1}{2}\tr{\vect{\Lambda}(\mu^{-1}\vect{I}-\vect{\Lambda})^{-1}} < 1}\ .
\emath
For small step-sizes $\mu \ll 1/\lambda_\textup{max}$, we have that
\bmath
  f(\mu) \approx \frac{\mu}{2}\sum_{m=1}^M \lambda_m = \frac{\mu}{2}\tr{\vect{\Lambda}} < 1\ .
\emath
which leads to that the LMS algorithm is mean-square stable if
\bmath
  \boxed{ 0 < \mu < \frac{2}{\tr{\vect{R}_u}}}\ .
\emath

\subsubsection{Learning Curve}
The learning curve is also difficult to find. In Appendix~\ref{app:lms_analysis}, we show that the learning curve is given by
\bmath
    \boxed{J_1(\vect{w}(n)) = \vect{\Delta w}^T(0)\vect{X}\vect{S}(n)\vect{X}^T\vect{\Delta w}(0)+\mu^2J_\textup{min}\sum_{i=0}^{n-1}\tr{\vect{S}(i)\vect{\Lambda}}+J_\textup{min}}
    \label{eq:lms_learning_curve}
\emath
where
\bmath
  \vect{S}(n) = \vect{S}(n-1) - 2\mu\vect{\Lambda}\vect{S}(n-1)+\mu^2[\vect{\Lambda}\tr{\vect{S}(n-1)\vect{\Lambda}}+2\vect{\Lambda}\vect{S}(n-1)\vect{\Lambda}]
\emath
with $\vect{S}(0)=\vect{\Lambda}$.

\subsection{Steady-State Analysis}
When the LMS algorithm is operating in steady-state and is mean-square stable, the EMSE, the mean-square deviation (MSD), and the misadjustment are \cite[pp.~465]{Sayed2003}
\begin{align}
  J_\textup{ex} &= J_\textup{min}\frac{f(\mu)}{1-f(\mu)}\\
  \mathcal{M} &= \frac{f(\mu)}{1-f(\mu)}\\
  E[\|\vect{\Delta}\vect{w}(\infty)\|^2] &= \frac{J_\textup{min}}{1-f(\mu)}\frac{\mu}{2}\sum_{m=1}^M \frac{1}{1-\mu\lambda_m}\ .
\end{align}
These results are also derived in Appendix~\ref{app:lms_analysis}. For small step-sizes $\mu \ll 1/\lambda_\textup{max}$, the EMSE, MSD, and misadjustment simplify to \cite[pp.~465]{Sayed2003}
\begin{align}
  J_\textup{ex}&\approx \mu J_\textup{min}\frac{\tr{\vect{R}_u}}{2-\mu\tr{\vect{R}_u}}\approx \frac{\mu}{2}J_\textup{min}\tr{\vect{R}_u}\\
  \mathcal{M} & \approx \mu \frac{\tr{\vect{R}_u}}{2-\mu\tr{\vect{R}_u}} \approx \frac{\mu}{2}\tr{\vect{R}_u} = \frac{\mu}{2}ME[u^2(n)]\\
  E[\|\Delta\vect{w}(\infty)\|^2] &\approx \mu J_\textup{min}\frac{M}{2-\mu\tr{\vect{R}_u}} \approx \frac{\mu}{2}J_\textup{min}M\ .
\end{align}
The approximated expression for the misadjustment shows that misadjustment is approximately proportional to the input power. This is undesirable and referred to as \textit{gradient noise amplification}. In Lecture~\ref{ch:NLMS_APA}, we consider the normalised LMS algorithm which solves this problem.

\subsection{Computational Cost}
Table~\ref{tab:comp_cost_lms} shows the computational cost of the LMS algorithm in terms of the number of multiplications and additions or subtractions. From the table, we see that the total number of flops is $4M+1$. Thus, the LMS algorithm has a linear complexity $\mathcal{O}(M)$ in the filter length $M$.
\begin{table}[htbp]
  \centering
  \begin{tabular}{l c c c}
    \toprule
    Term & $\times$ & $+$ or $-$ \\
    \midrule
    $\vect{u}^T(n)\vect{w}(n)$ & $M$ & $M-1$ \\
    $e(n)=d(n)-\vect{u}^T(n)\vect{w}(n)$ & & $1$ \\
    $\mu e(n)$ & $1$ & \\
    $\mu \vect{u}(n)e(n)$ & $M$ & \\
    $\vect{w}(n)+\mu \vect{u}(n)e(n)$ &  & $M$ \\
    \midrule
    Total & $2M+1$ & $2M$ \\
    \bottomrule
  \end{tabular}
  \caption{Computational cost of the LMS algorithm.}
  \label{tab:comp_cost_lms}
\end{table}


\lab{Data Visualization}{Data Visualization}
\label{lab:DataVis}
\objective{
This lab demonstrates how to communicate information through clean, concise, and honest data visualization.
We recommend completing the exercises in a Jupyter Notebook.
}

\section*{The Importance of Visualizations} % =================================

Visualizations of data can reveal insights that are not immediately obvious from simple statistics.
The data set in the following exercise is known as \emph{Anscombe's quartet}.
It is famous for demonstrating the importance of data visualization.

\begin{problem} % Describe Anscombe's Quartet
The file \texttt{anscombe.npy} contains the quartet of data points shown in the table below.
For each section of the quartet,
\begin{itemize}
	\setlength\itemsep{0em}
	\item Plot the data as a scatter plot on the box $[0,20]\times[0,13]$.
    \item Use \li{scipy.stats.linregress()} to calculate the slope and intercept of the least squares regression line for the data and its correlation coefficient (the first three return values).
	\item Plot the least squares regression line over the scatter plot on the domain $x \in [0,20]$.
	\item Report the mean and variance in $x$ and $y$, the slope and intercept of the regression line, and the correlation coefficient.
    Compare these statistics to those of the other sections.
	\item Describe how the section is similar to the others and how it is different.
\end{itemize}
\begin{table}[H]
\scriptsize{
\begin{tabular}{rr|rr|rr|rr}
    \multicolumn{2}{c|}{I}    & \multicolumn{2}{|c|}{II} &
    \multicolumn{2}{|c|}{III} & \multicolumn{2}{|c}{IV} \\
    \multicolumn{1}{c}{$x$}   & \multicolumn{1}{c|}{$y$} &
    \multicolumn{1}{|c}{$x$}  & \multicolumn{1}{c|}{$y$} &
    \multicolumn{1}{|c}{$x$}  & \multicolumn{1}{c|}{$y$} &
    \multicolumn{1}{|c}{$x$}  & \multicolumn{1}{c}{$y$} \\
    \hline
    10.0 & 8.04  & 10.0 & 9.14 & 10.0 & 7.46  & 8.0  & 6.58  \\
    8.0  & 6.95  & 8.0  & 8.14 & 8.0  & 6.77  & 8.0  & 5.76  \\
    13.0 & 7.58  & 13.0 & 8.74 & 13.0 & 12.74 & 8.0  & 7.71  \\
    9.0  & 8.81  & 9.0  & 8.77 & 9.0  & 7.11  & 8.0  & 8.84  \\
    11.0 & 8.33  & 11.0 & 9.26 & 11.0 & 7.81  & 8.0  & 8.47  \\
    14.0 & 9.96  & 14.0 & 8.10 & 14.0 & 8.84  & 8.0  & 7.04  \\
    6.0  & 7.24  & 6.0  & 6.13 & 6.0  & 6.08  & 8.0  & 5.25  \\
    4.0  & 4.26  & 4.0  & 3.10 & 4.0  & 5.39  & 19.0 & 12.50 \\
    12.0 & 10.84 & 12.0 & 9.13 & 12.0 & 8.15  & 8.0  & 5.56  \\
    7.0  & 4.82  & 7.0  & 7.26 & 7.0  & 6.42  & 8.0  & 7.91  \\
    5.0  & 5.68  & 5.0  & 4.74 & 5.0  & 5.73  & 8.0  & 6.89  \\
\end{tabular}}
\end{table}
\label{prob:anscombes-quartet}
\end{problem}

\section*{Improving Specific Types of Visualizations} % =======================

Effective data visualizations show specific comparisons and relationships in the data.
Before designing a visualization, decide what to look for or what needs to be communicated.
Then choose the visual scheme that makes sense for the data.
The following sections demonstrate how to improve commonly used plots to communicate information visually.

\subsection*{Line Plots} % ----------------------------------------------------

\begin{figure}[H] % Horizontal vs. vertical bar chart.
    \centering
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/chebyshev_bad.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.52\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/chebyshev_good.pdf}
    \end{subfigure}
    \caption{Line plots can be used to visualize and compare mathematical functions. For example, this figure shows the first nine Chebyshev polynomials in one plot (left) and small multiples (right). Using small multiples makes comparison easy and shows how each polynomial changes as $n$ increases.}
    \label{fig:chebyshev}
\end{figure}

\begin{lstlisting}
>>> import numpy as np
>>> from matplotlib import pyplot as plt

# Plot the first 9 Chebyshev polynomials in the same plot.
>>> T = np.polynomial.Chebyshev.basis
>>> x = np.linspace(-1, 1, 200)
>>> for n in range(9):
...     plt.plot(x, T(n)(x), label="n = "+str(n))
...
>>> plt.axis([-1.1, 1.1, -1.1, 1.1])        # Set the window limits.
>>> plt.legend(loc="right")
\end{lstlisting}

A line plot connects ordered $(x,y)$ points with straight lines, and is best for visualizing one or two ordered arrays, such as functional outputs over an ordered domain or a sequence of values over time.
Sometimes, plotting multiple lines on the same plot helps the viewer compare two different data sets.
However, plotting several lines on top of each other makes the visualization difficult to read, even with a legend.
For example, Figure \ref{fig:chebyshev} shows the first nine \emph{Chebyshev polynomials}, a family of orthogonal polynomials that satisfies the recursive relation
\[
T_0(x) = 1, \qquad T_1(x) = x, \qquad T_{n+1} = 2xT_n(x) - T_{n-1}(x).
\]
The plot on the right makes comparison easier by using \emph{small multiples}. %, a method made famous by Edward Tufte.
Instead of using a legend, the figure makes a separate subplot with a title for each polynomial.
Adjusting the figure size and the line thickness also makes the information easier to read.

%NumPy's \li{polynomial} module has a convenient tool for constructing these and other important polynomials.%
%\footnote{\li{numpy.polynomial} also has tools for computing other important polynomial families, including the Legendre, Hermite, and Laguerre polynomials.}

\begin{info} % LaTex with Matplotlib text.
Matplotlib titles and annotations can be formatted with \LaTeX, a system for creating technical documents.%
\footnote{See \url{http://www.latex-project.org/} for more information.}
% (this lab manual, for example, is written in \LaTeX).
To do so, use an \li{r} before the string quotation mark and surround the text with dollar signs.
For example, add the following line of code to the loop from the previous example.

\begin{lstlisting}
...     plt.title(r"$T_{}(x)$".<<format>>(n))
\end{lstlisting}

The \li{<<format>>()} method inserts the input $n$ at the curly braces.
The title of the sixth subplot, instead of being ``n = 5,'' will then be ``$T_5(x)$.''
\end{info}

% TODO: Make sure the Bernstein polynomial notation is consistent with the book (it is consistent with Wikipedia currently)

\begin{problem} % Plot the Bernstein polynomials.
The $n+1$ Bernstein basis polynomials of degree $n$ are defined as follows:
\[b_{v,n}(x) = {{n} \choose {v}} x^v (1-x)^{n-v},\qquad v = 0,\ 1,\ \ldots,\ n\]

Plot the first $10$ Bernstein basis polynomials ($n = 0,\ 1,\ 2,\ 3$) as small multiples on the domain $[0,1] \times [0,1]$.
Label the subplots for clarity, adjust tick marks and labels for simplicity, and set the window limits of each plot to be the same.
Consider arranging the subplots so that the rows correspond with $n$ and the columns with $v$.
\\Hint: The constant ${{n} \choose {v}} = \frac{n!}{v!(n-v)!}$ is called the \emph{binomial coefficient} and can be efficiently computed with \li{scipy.special.comb()}.
\end{problem}

\subsection*{Bar Charts} % ----------------------------------------------------

\begin{figure}[H] % Horizontal vs. vertical bar chart.
    \centering
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/bar_1.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.52\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/bar_2.pdf}
    \end{subfigure}
    \caption{Bar charts are used to compare quantities between categorical variables. The labels on the vertical bar chart (left) are more difficult to read than the  labels on the horizontal bar chart (right). Although the labels can be rotated, horizontal text is much easier to read than vertical text.}
\end{figure}

\begin{lstlisting}
>>> labels = ["Lobster Thermador", "Baked Beans", "Crispy Bacon",
...             "Smoked Sausage", "Hannibal Ham", "Eggs", "Spam"]
>>> values = [10, 11, 18, 19, 20, 21, 22]
>>> positions = np.arange(len(labels))

>>> plt.bar(positions, values, align="center")  # Vertical bar chart.
>>> plt.xticks(positions, labels)
>>> plt.show()

>>> plt.barh(positions, values, align="center") # Horizontal bar char (better).
>>> plt.yticks(positions, labels)
>>> plt.tight_layout()
>>> plt.show()
\end{lstlisting}

A bar chart plots categorical data in a sequence of bars.
They are best for small, discrete, one-dimensional data sets.
In Matplotlib, \li{plt.bar()} creates a vertical bar chart or \li{plt.barh()} creates a horizontal bar chart.
These functions receive the locations of each bar followed by the height of each bar (as lists or arrays).
In most situations, horizontal bar charts are preferable to vertical bar charts because horizontal labels are easier to read than vertical labels.
Data in a bar chart should also be sorted in a logical way, such as alphabetically, by size, or by importance.

\subsection*{Histograms} % ----------------------------------------------------

\begin{figure}[H] % Cleaning up a histogram.
    \centering
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/hist1.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/hist2.pdf}
    \end{subfigure}
    \caption{Histograms are used to show the distribution of one-dimensional data. Experimenting with different values for the bin size is important when plotting a histogram. Using only 10 bins (left) doesn't give a good sense for how the randomly generated data is distributed. However, using 35 bins (right) reveals the shape of a normal distribution.}
\end{figure}

\begin{lstlisting}
>>> data = np.random.normal(size=10000)
>>> fig, ax = plt.subplots(1, 2)
>>> ax[0].hist(data, bins=10)
>>> ax[1].hist(data, bins=35)
\end{lstlisting}

A histogram partitions an interval into a number of bins and counts the number of values that fall into each bin.
Histograms are ideal for visualizing how unordered data in a single array is distributed over an interval.
For example, if data are drawn from a probability distribution, a histogram approximates the distribution's probability density function.
Use \li{plt.hist()} to create a histogram.
The arguments \li{bins} and \li{<<range>>} specify the number of bins to draw and over what domain.
A histogram with too few or too many bins will not give a clear view of the distribution.

\subsection*{Scatter Plots} % -------------------------------------------------

\begin{figure}[H] % Heat maps and contour plots.
    \centering
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/alpha1.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/alpha2.pdf}
    \end{subfigure}
    \caption{Scatter plots show correlation between variables by plotting markers at coordinate points. The figure above displays randomly perturbed data visualized using two scatter plots with \li{alpha=.5} and \li{edgecolor='none'}. The default (left) makes it harder to see correlation and pattern whereas making the axes equal better reveals the oscillatory behavior in the perturbed sine wave.}
    \label{fig:scatter_scales}
\end{figure}

\begin{lstlisting}
>>> np.random.seed(0)
>>> x = np.linspace(0,10*np.pi,200) + np.random.normal(size=200)
>>> y = np.sin(x) + np.random.normal(size=200)

>>> plt.scatter(x, y, alpha=.5, edgecolor='none')
>>> plt.show()

>>> plt.scatter(x, y, alpha=.5, edgecolor='none')
>>> plt.axis('equal')
>>> plt.show()
\end{lstlisting}

A scatter plot draws $(x,y)$ points without connecting them.
Scatter plots are best for displaying data sets without a natural order, or where each point is a distinct, individual instance.
They are frequently used to show correlation between variables in a data set.
Use \li{plt.scatter()} to create a scatter plot.%
\footnote{Scatter plots can also be drawn with with \lif{plt.plot()} by specifying a point marker such as \lif{'.'}, \lif{','}, \lif{'o'}, or \lif{'+'}.
The keywords \lif{s} and \lif{c} can be used to change the marker size and marker color, respectively.}

Similar data points in a scatter plot may overlap, as in Figure \ref{fig:scatter_scales}.
Specifying an \emph{alpha value} reveals overlapping data by making the markers transparent (see Figure \ref{fig:scatter_hexbin} for an example).
The keyword \li{alpha} accepts values between 0 (completely transparent) and 1 (completely opaque).
When plotting lots of overlapping points, the outlines on the markers can make the visualization look cluttered.
Setting the edgecolor keyword to zero removes the outline and improves the visualization.

\begin{problem} % Scatter plots with visualizations.
The file \texttt{MLB.npy} contains measurements from over 1,000 recent Major League Baseball players, compiled by UCLA.\footnote{See \url{http://wiki.stat.ucla.edu/socr/index.php/SOCR_Data_MLB_HeightsWeights}.}
Each row in the array represents a player; the columns are the player's height (in inches), weight (in pounds), and age (in years), in that order.

Create several visualizations to show the correlations between height, weight, and age in the MLB data set.
Use at least one scatter plot.
Adjust the marker size, plot a regression line, change the window limits, and use small multiples where appropriate.
\end{problem}

\begin{problem} % Earthquake data
The file \texttt{earthquakes.npy} contains data from over 17,000 earthquakes between 2000 and 2010 that were at least a 5 on the Richter scale.%
\footnote{See \url{http://earthquake.usgs.gov/earthquakes/search/}.}
Each row in the array represents an earthquake;
the columns are the earthquake's date (as a fraction of the year), magnitude (on the Richter scale), longitude, and latitude, in that order.

Because each earthquake is a distinct event, a good way to start visualizing this data might be a scatter plot of the years versus the magnitudes of each earthquake.

\begin{lstlisting}
>>> year, magnitude, longitude, latitude = np.load("earthquakes.npy").T
>>> plt.plot(year, magnitude, '.')
>>> plt.xlabel("Year")
>>> plt.ylabel("Magnitude")
\end{lstlisting}

\begin{figure}[H] % Bad visualization: earthquake data, year vs magnitude.
    \centering
    \includegraphics[width=.7\textwidth]{figures/earthquake.pdf}
\end{figure}

Unfortunately, this plot communicates very little information because the data is so cluttered.
Describe the data with at least two better visualizations, including line plots, scatter plots, and histograms as appropriate.
Your plots should answer the following questions:
\begin{enumerate}
    \item How many earthquakes happened every year?
    \item How often do stronger earthquakes happen compared to weaker ones?
    \item Where do earthquakes happen? Where do the strongest earthquakes happen?
    \\(Hint: Use \li{plt.axis("equal")} or \li{ax.set_aspect("equal")} to fix the aspect ratio, which may improve comparisons between longitude and latitude.)
\end{enumerate}
\end{problem}

\subsection*{Hexbins} % -------------------------------------------------------

\begin{figure}[H] % Hexbins
    \centering
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/hexbin_1.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/hexbin_2.pdf}
    \end{subfigure}
    %
    \caption{Hexbins can be used instead of using a three-dimensional histogram to show the distribution of two-dimensional data. Choosing the right gridsize will give a better picture of the distribution. The figure above shows random data plotted as hexbins with a gridsize of 10 (left) and 25 (right). Hexbins use color to show height via a colormap and both histograms above use the \li{'inferno'} colormap.}
    \label{fig:scatter_hexbin}
\end{figure}

\begin{lstlisting}
# Add random draws from various distributions in two dimensions.
>>> a = np.random.exponential(size=1000) + np.random.normal(size=1000) + 5
>>> b = np.random.exponential(size=1000) + 2*np.random.normal(size=1000)
>>> x = np.hstack((a, b, 2*np.random.normal(size=1000)))
>>> y = np.hstack((b, a,   np.random.normal(size=1000)))

# Plot the samples with hexbins of gridsize 10 and 25.
>>> fig, axes = plt.subplots(1, 2)
>>> window = [x.min(), x.max(), y.min(), y.max()]
>>> for ax, size in zip(axes, [10, 25]):
...     ax.hexbin(x, y, gridsize=size, cmap='inferno')
...     ax.axis(window)
...     ax.set_aspect("equal")
...
>>> plt.show()
\end{lstlisting}

A \emph{hexbin} is a way of representing the frequency of ocurrances in a two-dimensional plane.
Similar to a histogram, which sorts one-dimensional data into bins, a hexbin sorts two-dimensional data into hexagonal bins arranged in a grid and uses color instead of height to show frequency.
Creating an effective hexbin relies on choosing an appropriate \li{gridsize} and colormap.
The \emph{colormap} is a function that assigns data points to an ordering of colors.
Use \li{plt.hexbin()} to create a hexbin and use the \li{cmap} keyword to specify the colormap.

\subsection*{Heat Maps and Contour Plots} % -----------------------------------

\begin{figure}[H] % Heat maps and contour plots.
    \centering
    \begin{subfigure}{.495\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/heatmap_1.png}
    \end{subfigure}
    %
    \begin{subfigure}{.495\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/contour_1.pdf}
    \end{subfigure}
    \\
    \begin{subfigure}{.495\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/contour_2.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.495\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/heatmap_2.png}
    \end{subfigure}
    \caption{Heat maps visualize three-dimensional functions or surfaces by using color to represent the value in one dimension. With continuous data, it can be hard to identify regions of interest. Contour plots solve this problem by visualizing the level curves of the surface. Top left: heat map. Top right: contour plot. Bottom left: heat map. Bottom right: contours plotted on a heat map.}
    \label{fig:heatcontour}
\end{figure}

\begin{lstlisting}
# Construct a 2-D domain with np.meshgrid() and calculate f on the domain.
>>> x = np.linspace(-1.5, 1.5, 200)
>>> X, Y = np.meshgrid(x, x)
>>> Z = Y**2 - X**3 + X**2

# Plot f using a heat map, a contour map, and a filled contour map.
>>> fig, ax = plt.subplots(2,2)
>>> ax[0,0].pcolormesh(X, Y, Z, cmap="viridis")     # Heat map.
>>> ax[0,1].contour(X, Y, Z, 6, cmap="viridis")     # Contour map.
>>> ax[1,0].contourf(X, Y, Z, 12, cmap="magma")   # Filled contour map.

# Plot specific level curves and a heat map with a colorbar.
>>> ax[1,1].contour(X, Y, Z, [-1, -.25, 0, .25, 1, 4], colors="white")
>>> cax = ax[1,1].pcolormesh(X, Y, Z, cmap="magma")
>>> fig.colorbar(cax, ax=ax[1,1])

>>> plt.show()
\end{lstlisting}

Let $f:\mathbb{R}^2\rightarrow\mathbb{R}$ be a scalar-valued function on a 2-dimensional domain.
A \emph{heat map} of $f$ assigns a color to each $(x,y)$ point in the domain based on the value of $f(x,y)$, while a contour plot is a drawing of the \emph{level curves} of $f$.
The level curve corresponding to the constant $c$ is the set $\left\{(x,y)\mid c = f(x,y)\right\}$.
A filled contour plot colors in the sections between the level curves and is a discretized version of a heat map.
The values of $c$ corresponding to the level curves are automatically chosen to be evenly spaced over the range of values of $f$ on the domain.
However, it is sometimes better to strategically specify the curves by providing a list of $c$ constants.

Consider the function $f(x,y) = y^2 - x^3 + x^2$ on the domain $[-\frac{3}{2}, \frac{3}{2}] \times [-\frac{3}{2}, \frac{3}{2}]$.
A heat map of $f$ reveals that it has a large basin around the origin.
Since $f(0,0) = 0$, choosing several level curves close to $0$ more closely describes the topography of the basin.
The fourth subplot in \ref{fig:heatcontour} uses the curves with $c = -1,\ -\frac{1}{4},\ 0,\ \frac{1}{4},\ 1,$ and $4$.

When plotting hexbins, heat maps, and contour plots, be sure to choose a colormap that best represents the data.
Avoid using spectral or rainbow colormaps like \li{"jet"} because they are not \emph{perceptually uniform}, meaning that the rate of change in color is not constant.
Because of this, data points may appear to be closer together or farther apart than they actually are.
This creates visual false positives or false negatives in the visualization and can affect the interpretation of the data.
As a default, we recommend using the sequential colormaps \li{"viridis"} or \li{"inferno"} because they are designed to be perceptually uniform and colorblind friendly.
For the complete list of Matplotlib color maps, see \url{http://matplotlib.org/examples/color/colormaps_reference.html}.

\begin{problem} % Rosenbrock
The \emph{Rosenbrock function} is defined as
\[f(x,y) = (1-x)^2 + 100(y-x^2)^2.\]
The minimum value of $f$ is $0$, which occurs at the point $(1,1)$ at the bottom of a steep, banana-shaped valley of the function.

Use a heat map and a contour plot to visualize the Rosenbrock function.
Also plot the minimizer $(1,1)$.
Use a different sequential colormap for each visualization.
\end{problem}

\section*{Best Practices} % ===============================================

Good scientific visualizations make comparison easy and clear.
The eye is very good at detecting variation in one dimension and poor in two or more dimensions.
For example, consider Figure \ref{fig:piebar}.
Despite the difficulty, most people can probably guess which slice of a pie chart is the largest or smallest.
However, it's almost impossible to confidently answer the question \emph{by how much?} The bar charts may not be as aesthetically pleasing but they make it much easier to precisely compare the data.
Avoid using pie charts as well as other visualizations that make accurate comparison difficult, such as radar charts, bubble charts, and stacked bar charts.

\begin{figure}[H] % Pie chart --> Bar chart
    \centering
    \begin{subfigure}{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/piechart1.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/piebar1.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/piechart2.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/piebar2.pdf}
    \end{subfigure}
    %
    \begin{subfigure}{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/piechart3.pdf}
    \end{subfigure}
    \begin{subfigure}{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/piebar3.pdf}
    \end{subfigure}
    %
    \caption{The pie charts on the left may be more colorful but it's extremely difficult to quantify the difference between each slice. Instead, the horizontal bar charts on the right make it very easy to see the difference between each variable.}
    \label{fig:piebar}
\end{figure}

No visualization perfectly represents data, but some are better than others.
Finding the best visualization for a data set is an iterative process.
Experiment with different visualizations by adjusting their parameters: color, scale, size, shape, position, and length.
It may be necessary to use a data transformation or visualize various subsets of the data.
As you iterate, keep in mind the saying attributed to George Box: ``All models are wrong, but some are useful.''
Do whatever is needed to make the visualization useful and effective.

\begin{figure}[H] % BAR CHART MONSTROSITY!
    \centering
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/chartjunk1.jpg}
    \end{subfigure}
    %
    \begin{subfigure}{.47\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/chartjunk_improved.pdf}
    \end{subfigure}
    \caption{Chartjunk refers to anything that does not communicate data. In the image on the left, the cartoon monster distorts the bar chart and manipulates the feelings of the viewer to think negatively about the results. The image on the right shows the same data without chartjunk, making it simple and very easy to interpret the data objectively.}
    \label{fig:chartjunk}
\end{figure}

Good visualizations are as simple as possible and no simpler.
Edward Tufte coined the term \emph{chartjunk} to mean anything (pictures, icons, colors, and text) that does not represent data or is distracting.
Though chartjunk might appear to make data graphics more memorable than plain visualizations, \textbf{it is more important to be clear and precise in order to prevent misinterpretation}.
The physicist Richard Feynman said, ``For a successful technology, reality must take precedence over public relations, for Nature cannot be fooled.'' Remove chartjunk and anything that prevents the viewer from objectively interpreting the data.

\begin{figure}[H] % Heat maps and contour plots.
    \centering
    \begin{subfigure}{.495\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/plannedparenthood}
    \end{subfigure}
    %
    \begin{subfigure}{.495\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/planparent_corrected}
    \end{subfigure}
    \caption{The chart on the left is an example of a dishonest graphic shown at a United States congressional hearing in 2015. The chart on the right shows a more accurate representation of the data by showing the y-axis and revealing the missing data from 2008. Source: PolitiFact.}
    \label{fig:planparent}
\end{figure}

Visualizations should be honest.
Figure \ref{fig:planparent} shows how visualizations can be dishonest.
The misleading graphic on the left was used as evidence in a United States congressional hearing in 2015.
With the $y$-axis completely removed, it is easy to miss that each line is shown on a different $y$-axis even though they are measured in the same units.
Furthermore, the chart fails to indicate that data is missing from the year 2008.
The graphic on the right shows a more accurate representation of the data.\footnote{For more information about this graphic, visit \url{http://www.politifact.com/truth-o-meter/statements/2015/oct/01/jason-chaffetz/chart-shown-planned-parenthood-hearing-misleading-/}.}

Never use data visualizations to deceive or manipulate.
Always present information on who created it, where the data came from, how it was collected, whether it was cleaned or transformed, and whether there are conflicts of interest or possible biases present.
Use specific titles and axis labels, and include units of measure.
Choose an appropriate window size and use a legend or other annotations where appropriate.

\begin{problem}
The file \texttt{countries.npy} contains information from 20 different countries.
Each row in the array represents a different country; the columns are the 2015 population (in millions of people), the 2015 GDP (in billions of US dollars), the average male height (in centimeters), and the average female height (in centimeters), in that order.%
\footnote{
See \url{https://en.wikipedia.org/wiki/List_of_countries_by_GDP_(nominal)},
\\ \url{https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population}, and
\\ \url{http://www.averageheight.co/}.
}

The countries corresponding are listed below in order.

\begin{lstlisting}
countries = ["Austria", "Bolivia", "Brazil", "China",
            "Finland", "Germany", "Hungary", "India",
            "Japan", "North Korea", "Montenegro", "Norway",
            "Peru", "South Korea", "Sri Lanka", "Switzerland",
            "Turkey", "United Kingdom", "United States", "Vietnam"]
\end{lstlisting}

Visualize this data set with at least four plots, using at least one scatter plot, one histogram, and one bar chart.
List the major insights that your visualizations reveal.
\\(Hint: consider using \li{np.argsort()} and fancy indexing to sort the data for the bar chart.)
\end{problem}

For more about data visualization, we recommend the following books and websites.
\begin{itemize}
    \item \emph{How to Lie with Statistics} by Darrell Huff (1954).
    \item \emph{The Visual Display of Quantitative Information} by Edward Tufte (2nd edition).
    \item \emph{Visual Explanations} by Edward Tufte.
    \item \emph{Envisioning Information} by Edward Tufte.
    \item \emph{Beautiful Evidence} by Edward Tufte.
    \item \emph{The Functional Art} by Alberto Cairo.
    \item \emph{Visualization Analysis and Design} by Tamara Munzner.
    \item \emph{Designing New Default Colormaps}: \url{https://bids.github.io/colormap/}.
\end{itemize}


\subsection{Congruence}

\(5\) and \(11\) are congrument \(\mod 3\)

If \(a \mod(n)=b mod(n)\) then \(a\) and \(b\) are congruent mod \(n\).


\documentclass[a4paper,11pt]{scrartcl}

\usepackage{amsmath,amssymb}
\usepackage[utf8]{inputenc}

\title{Derivatives of the likelihood for a beta-psychometric function}
\author{Ingo Fründ}
\date{\today}

\def\rd{\mathrm{d}}
\def\p{\partial}
\def\ddlnu{\frac{\p^2\ell_i}{\p\nu^2}}
\def\ddlf{\frac{\p^2\ell_i}{\p f_i^2}}
\def\ddlfnu{\frac{\p^2\ell_i}{\p f_i\p\nu}}
\def\dfdt{\frac{\p f_i}{\p\theta}}

\begin{document}

\maketitle
% Your text here

\section{Log likelihood}

The log likelihood for a beta-psychometric function can be written as
%
$$
\ell = \sum_{i=1}^m \ell_i,
$$
%
where $m$ is the number of blocks in the experiment and
%
$$
\ell_i = \log\Gamma(\nu n_i) - \log\Gamma( f_i\nu n_i ) - \log\Gamma( (1-f_i) \nu n_i ) + ( f_i \nu n_i - 1 ) \log p_i + ( (1-f_i) \nu n_i - 1 ) \log (1-p_i).
$$
%
Here, $f_i = \gamma + (1-\gamma-\lambda) F(g(x_i,\alpha,\beta))$, i.e. the predicted value of the psychometric function, and
$p_i = k_i/n_i$, i.e. the observed probability of a correct response.

\section{Derivatives of the log likelihood function}

Derivatives of the log likelihood function involve the $\psi$ function,
%
$$
\psi = \frac{\rd}{\rd x} \log\Gamma.
$$
%
They read:
%
$$
\frac{\p \ell_i}{\p\nu} = n_i\psi(\nu n_i) - f_i n_i \psi ( f_i\nu n_i) - (1-f_i) n_i \psi ( (1-f_i) \nu n_i) + f_i n_i \log(p_i) + (1-f_i) n_i \log(1-p_i),
$$
%
and
%
$$
\frac{\p \ell_i}{\p f_i} = \nu n_i \Big( \log\frac{p_i}{1-p_i} + \psi ( (1-f_i) \nu n_i) - \psi ( f_i \nu n_i) \Big).
$$
%
In order to obtain derivatives of the log likelihood with respect to parameters $\alpha,\beta,\gamma,\delta$, we use the definition of $f_i$ to arrive at
%
$$
\frac{\p \ell_i}{\p \theta} = \frac{\p\ell_i}{\p f_i} \frac{\p f_i}{\p \theta}, \quad \theta\in\{\alpha,\beta,\gamma,\delta\}.
$$
%
These are given by:
%
\begin{align*}
    \frac{\p f_i}{\p\alpha} &= (1-\gamma-\lambda) F' ( g(x_i,\alpha,\beta) ) \frac{\p g}{\p \alpha} ( x_i, \alpha, \beta ),\\
    \frac{\p f_i}{\p\beta} &= (1-\gamma-\lambda) F' ( g(x_i,\alpha,\beta) ) \frac{\p g}{\p \beta} ( x_i, \alpha, \beta ),\\
    \frac{\p f_i}{\p\lambda} &= -F(g(x_i,\alpha,\beta)),\\
    \frac{\p f_i}{\p \gamma} &= 1-F(g(x_i,\alpha,\beta)).
\end{align*}
%

\section{Second derivatives of the log likelihood function}

The second derivatives of the log likelihood function involve derivatives of the $\psi$ function.
These are called the trigamma function and are notated by $\psi'$.
%
$$
\ddlnu = \psi' (\nu n_i) n_i^2 - f_i^2 n_i^2\psi' (f_i\nu n_i) - (1-f_i)^2n_i^2 \psi' ( (1-f_i)\nu n),
$$
%
and
%
$$
\ddlf = - (\nu n)^2 \Big(\psi' ( f_i \nu n) + \psi'( (1-f_i)\nu n) \Big).
$$
Finally, we have
%
$$
\ddlfnu = - n_i \psi ( f_i \nu n_i ) - f_i n_i^2\nu \psi' ( f_i\nu n_i ) + n_i\psi( (1-f_i)\nu n_i) + (1-f)n_i^2\nu\psi'( (1-f_i)\nu n_i) + n_i \log\frac{p_i}{1-p_i}.
$$
%

We can use these identities to derive expressions for derivatives of the log likelihood function with respect to parameters of the psychometric function:
%
$$
\frac{\p^2 \ell_i}{\p \lambda^2} = \frac{\p^2\ell_i}{\p \gamma^2} = 0,
$$
%
$$
\frac{\p^2\ell_i}{\p\alpha^2} = (1-\gamma-\lambda) \ddlf \cdot \Big( F''(g(x_i,\alpha,\beta))\frac{\p g}{\p\alpha}(x_i,\alpha,\beta) + F'(g(x_i,\alpha,\beta))\frac{\p^2g}{\p\alpha^2}(x_i,\alpha,\beta)\Big),
$$
%
$$
\frac{\p^2\ell_i}{\p\beta^2} = (1-\gamma-\lambda) \ddlf \cdot \Big( F''(g(x_i,\alpha,\beta))\frac{\p g}{\p\beta}(x_i,\alpha,\beta) + F'(g(x_i,\alpha,\beta))\frac{\p^2g}{\p\beta^2}(x_i,\alpha,\beta)\Big).
$$
%

The crossterms are
%
\begin{align*}
\frac{\p^2\ell_i}{\p\alpha\p\beta} &= (1-\gamma-\lambda)\ddlf\cdot \Big( F''(g(x_i,\alpha,\beta))\frac{\p g}{\p\alpha}\frac{\p g}{\p\beta} + F'(g(x_i,\alpha,\beta))\frac{\p^2}{\p\alpha\p\beta}\Big), \\
\frac{\p^2\ell_i}{\p\alpha\p\gamma} &= - \ddlf F'(g(x_i,\alpha,\beta)) \frac{\p g}{\p\alpha},
\end{align*}
%
and an analogous expression for $\frac{\p^2\ell_i}{\p\beta\p\gamma}$.
Further
%
$$
\frac{\p^2\ell_i}{\p\alpha\p\gamma} = \ddlf (1-F'(g(x_i,\alpha,\beta)) \frac{\p g}{\p\alpha}),
$$
%
with an analogous expression for $\frac{\p^2\ell_i}{\p\beta\p\lambda}$.

We now come the the final crossterm
%
\begin{align*}
\frac{\p^2\ell_i}{\p\theta\p\nu} &=
    - n_i\dfdt \Big ( \psi (f_i \nu n_i) + f_i\nu n_i \psi'(f_i\nu n_i)\Big)\\
    &\quad+ n_i\dfdt \Big ( \psi ( (1-f_i)\nu n_i) + (1-f_i)\nu n_i\psi'( (1-f_i)\nu n_i) \Big)\\
    &\quad+ n_i\dfdt \log\frac{p_i}{1-p_i},
\end{align*}
%
where $\theta \in\{\alpha,\beta,\lambda,\gamma\}$.

\end{document}


\section{Approximation Rates for Cosine Networks}

To begin, we remark that throughout this manuscript, we use the following convention for the Fourier transform
\begin{equation}
 \hat{f}(\xi) = \int_{\mathbb{R}^d} f(x)e^{-2\pi {\mathrm{i}\mkern1mu} \xi\cdot x}dx,
\end{equation}
for which the inverse transform is given by
\begin{equation}
 f(x) = \int_{\mathbb{R}^d} \hat{f}(\xi)e^{2\pi {\mathrm{i}\mkern1mu}    \xi\cdot x}d\xi.
\end{equation}
We find that this convention results in the cleanest arguments, avoiding the necessity to keep track of normalizing constants.

In this section, we analyze the approximation properties of networks with a cosine activation function on the spectral Barron space $\mathcal{B}^s(\Omega)$. Specifically, consider approximating a function $f\in \mathcal{B}^s(\Omega)$ by a superposition of finitely many complex complex exponentials with coefficients that are bounded in $\ell^1$, i.e. by an element of the set
\begin{equation}
 \Sigma_{n,M} = \left\{\sum_{j=1}^n a_je^{2\pi {\mathrm{i}\mkern1mu}  \theta_j\cdot x}:~\theta_j\in \mathbb{R}^d,~a_j\in\mathbb{C},~\sum_{i=1}^n|a_i|\leq M\right\}.
\end{equation}

Alternatively, one can view this as the set of neural networks with a single hidden layer containing $n$ neurons with activation function $\sigma(x) = e^{2\pi {\mathrm{i}\mkern1mu}  x}$, whose weights are bounded in $\ell^1$.

Equivalently, we can consider approximation by networks with a cosine activation function
\begin{equation}
  \Sigma^{\cos}_{n,M} = \left\{\sum_{i=1}^n a_i\cos(2\pi\theta_i\cdot x + b_i):~\theta_i\in \mathbb{R}^d,~b_i\in \mathbb{R},~\sum_{i=1}^n|a_i| \leq M\right\}.
 \end{equation}
This is because
\begin{equation}
e^{2\pi {\mathrm{i}\mkern1mu}  \theta_i\cdot x} = \cos(2\pi\theta_i\cdot x) + i\cos\left(2\pi\theta_i\cdot x - \frac{\pi}{2}\right)\in \Sigma^{\cos}_{2,2}
\end{equation}
and
\begin{equation}
 \cos(2\pi\theta_i\cdot x) = \frac{1}{2}e^{2\pi {\mathrm{i}\mkern1mu}  \theta_i\cdot x} + \frac{1}{2}e^{-2\pi {\mathrm{i}\mkern1mu}  \theta_i\cdot x}\in \Sigma^d_{2,1}.
\end{equation}
Thus we have $\Sigma_{n,M}\subset \Sigma^{\cos}_{2n,2M}$ and $\Sigma^{\cos}_{n,M} \subset \Sigma_{2n,M}$ and so the rates obtained for both sets will be the same. In what follows, we consider $\Sigma_{n,M}$ for convenience in dealing with the Fourier transform.

We begin with a key lemma showing that we only need frequencies lying on a lattice to represent functions $f$ with decaying Fourier transform on a bounded set.
\begin{lemma}\label{fourier-representation-lemma-general}
 Let $\Omega=[0,1]^d$ and $\mu:\mathbb{R}^d\rightarrow \mathbb{R}_+$ be a continuous weight function. Suppose that $\mu$ satisfies the following conditions
 \begin{itemize}
  \item $\mu(\xi + \omega) \leq \mu(\xi)\mu(\omega)$
  \item There exists a $0 < \beta < 1$ and a $c > 0$ such that $\mu(\xi) \lesssim e^{c|\xi|^{\beta}}$.
 \end{itemize}

 Suppose that $f$ satisfies
 \begin{equation}
  \int_{\mathbb{R}^d} \mu(\xi)|\hat{f}(\xi)|d\xi = C_f < \infty.
 \end{equation}

 Then for any $L > 1$, there exists an $a\in L^{-1}[0,1]^d$ (which may depend on $f$) and coefficients $c_\xi$, such that for $x\in \Omega$
 \begin{equation}
  f(x) = \sum_{\xi\in L^{-1}\mathbb{Z}^d}c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}
 \end{equation}
 and
 \begin{equation}
  \sum_{\xi\in L^{-1}\mathbb{Z}^d} \mu(a+\xi)|c_\xi| \lesssim C_f.
 \end{equation}

\end{lemma}

Note that the suppressed constant in the above lemma only depends upon $d,\mu$ and $L$, but not on $f$ or $a$. Furthmore, we note that from the proof below it follows that the suppressed constant depends exponentially on the dimension $d$.

\begin{proof}
 Since by assumption $L > 1$, there exists an $\epsilon$ such that $\Omega\subset [0,L-2\epsilon]^d$. We begin by constructing a cutoff function $\phi_\Omega$, which is identically $1$ on $\Omega$ and $0$ outside of $[-\epsilon, L-\epsilon]^d$. It will be important that the Fourier transform $\hat{\phi}_\Omega$ has sufficiently fast decay, so that
 \begin{equation}
  \int_{\mathbb{R}^d}\mu(\xi)|\hat{\phi}_\Omega(\xi)| < \infty.
 \end{equation}

 To construct this function, we follow closely the calculation made in \cite{johnson2015saddle}. Choose $\alpha > 1$ such that $\beta < 1-\alpha^{-1} < 1$ and consider the smooth one-dimensional bump function $g$ by \eqref{alpha-g}.
 Let $g_d$ denote the $n$-dimensional function
 \begin{equation}
  g_d(x) = \frac{1}{C}\prod_{i=1}^d g(x_i),
 \end{equation}
 where the normalization constant $C$ is chosen so that $\int_{\mathbb{R}^d} g_d(x) = 1$. Then by \eqref{eq_181} we see that
 \begin{equation}
  |\hat{g}_d(\xi)|\lesssim e^{-c_\alpha\sum_{i=1}^d|\xi_i|^{1-\alpha^{-1}}} \lesssim e^{-c_{\alpha,d}\alpha|\xi|^{1-\alpha^{-1}}},
 \end{equation}
 for a new constant $c_{\alpha,d}$.

 Finally, let $\Omega^\prime = [-\frac{\epsilon}{2}, L - \frac{3\epsilon}{2}]$ and define
 \begin{equation}
  \phi_\Omega = \left(4^d\epsilon^{-d}g_d\left(4\epsilon^{-1}x\right)\right)*\chi_{\Omega^\prime}(x).
 \end{equation}
 The compact support and normalization of $g_d$  implies that $\phi_\Omega|_\Omega = 1$ and $\phi_\Omega = 0$ outside of $[-\epsilon, L-\epsilon]^d$. Furthermore, we calculate
 \begin{equation}
  |\hat{\phi}_\Omega(\xi)| = \left|\hat{g}_d\left(\frac{\epsilon}{4}\xi\right)\widehat{\chi}_{\Omega^\prime}\right| \lesssim e^{-c_{\alpha,\Omega}|\xi|^{1-\alpha^{-1}}}
 \end{equation}
 for a constant $c_{\alpha,\Omega}$, since $\widehat{\chi}_{\Omega^\prime}$ is bounded. The growth condition on $\mu$, combined with $\beta < 1-\alpha^{-1} < 1$ means that
 \begin{equation}\label{eq_205}
  \int_{\mathbb{R}^d}\mu(\xi)|\hat{\phi}_\Omega(\xi)| < \infty.
 \end{equation}

 Now consider the function $h_f = \phi_\Omega f$. Evidently $h_f = f$ on $\Omega$ and $h_f$ is supported on $[-\epsilon,L-\epsilon]^d$. Notice further that $\hat{h}_f = \hat{\phi}_\Omega * \hat{f}$ and we calculate
 \begin{equation}
  \begin{split}
  \int_{\mathbb{R}^d} \mu(\xi)|\hat{h}_f(\xi)|d\xi &\leq \int_{\mathbb{R}^d} \int_{\mathbb{R}^d}\mu(\xi) |\hat{\phi}_\Omega(\xi - \omega)||\hat{f}(\omega)| d\omega d\xi \\
  &=\int_{\mathbb{R}^d} \left(\int_{\mathbb{R}^d}\mu(\xi + \omega) |\hat{\phi}_\Omega(\xi)|d\xi\right)|\hat{f}_e(\omega)| d\omega.
  \end{split}
 \end{equation}
 Now $\mu(\xi + \omega) \leq \mu(\xi)\mu(\omega)$, so that we get
 \begin{equation}\label{eq_369}
  \int_{\mathbb{R}^d} \mu(\xi)|\hat{h}_f(\xi)|d\xi \leq \left(\int_{\mathbb{R}^d}\mu(\xi) |\hat{\phi}_\Omega(\xi)|d\xi\right)\left(\int_{\mathbb{R}^d}\mu(\omega) |\hat{f}(\omega)|d\omega\right) \lesssim C_f,
 \end{equation}
 where the implied constant depends the value of the integral in \eqref{eq_205}.

 We now rewrite the integral in \eqref{eq_369} as
 \begin{equation}
  \int_{\mathbb{R}^d} \mu(\xi)|\hat{h}_f(\xi)|d\xi = \int_{[0,L^{-1}]^d} \left(\sum_{\xi\in L^{-1}\mathbb{Z}^d} \mu(a+\xi)|\hat{h}_f(a+\xi)|\right)da \lesssim C_f.
 \end{equation}
 Certainly this means that there must exist an $a\in [0,L^{-1}]^d$ (depending on $f$) such that
 \begin{equation}
  \left(\sum_{\xi\in L^{-1}\mathbb{Z}^d} \mu(a+\xi)|\hat{h}_f(a+\xi)|\right) \lesssim C_f,
 \end{equation}
 where the implied constant only depends upon $L$ and $d$.

 We proceed to apply the Poisson summation formula and the fact that $h_f$ is supported in $[-\epsilon,L-\epsilon]^d$ to conclude that for a.e. $x\in \Omega \subset [-\epsilon,L-\epsilon]^d$ we have
 \begin{equation}
  f(x) = h_f(x) = \sum_{\nu \in L\mathbb{Z}^d} h_f(x+\nu)e^{2\pi {\mathrm{i}\mkern1mu}  a\cdot \nu} = \sum_{\xi\in L^{-1}\mathbb{Z}^d}\hat{h}_f(a+\xi) e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}.
 \end{equation}
 Here we have applied the Poisson summation formula to the function $g(\nu) = h_f(x+\nu)e^{2\pi {\mathrm{i}\mkern1mu}  a\cdot \nu}$, whose Fourier transform is easily seen to be $\hat{g}(\xi) = \hat{h}_f(a+\xi) e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}$.

 Setting $c_\xi = \hat{h}_f(a+\xi)$ we obtain the desired result.
\end{proof}

We now apply Lemma \ref{fourier-representation-lemma-general} with $\mu(\xi) = (1 + |\xi|)^s$ to obtain the following corollary concerning the spectral Barron space $\mathcal{B}^s(\Omega)$.
\begin{corollary}\label{fourier-representation-lemma}
 Let $\Omega = [0,1]^d$ and $s \geq 0$. Let $f\in \mathcal{B}^s(\Omega)$. Then for any $L > 1$, there exists an $a\in L^{-1}[0,1]^d$ (potentially depending upon $f$) and coefficients $c_\xi$ such that
 for $x\in \Omega$
 \begin{equation}
  f(x) = \sum_{\xi\in L^{-1}\mathbb{Z}^d}c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}
 \end{equation}
 and
 \begin{equation}
  \sum_{\xi\in L^{-1}\mathbb{Z}^d} (1+|a+\xi|)^s|c_\xi| \lesssim \|f\|_{\mathcal{B}^s(\Omega)}.
 \end{equation}

\end{corollary}
\begin{proof}
 This follows immediately from Lemma \ref{fourier-representation-lemma-general} given the characterization of $\mathcal{B}^s(\Omega)$ and the elementary fact that $(1 + |\xi + \omega|)\leq (1 + |\xi| + |\omega|) \leq (1 + |\xi|)(1 + |\omega|)$.
\end{proof}

Corollary \ref{fourier-representation-lemma} can be used to improve upon the $O(n^{-\frac{1}{2}})$ approximation rate of cosine networks obtained in \cite{jones1992simple} when $f\in \mathcal{B}^s(\Omega)$ for $s > 0$.

\begin{theorem}\label{approximation-rate-theorem}
 Let $\Omega = [0,1]^d$, $0\leq m\leq s$, and $f\in \mathcal{B}^s(\Omega)$. Then there is an $M\lesssim \|f\|_{\mathcal{B}^s(\Omega)}$ such that
 \begin{equation}
  \inf_{f_n\in \Sigma_{n,M}} \|f-f_n\|_{H^m(\Omega)} \lesssim \|f\|_{\mathcal{B}^s(\Omega)}n^{-\frac{1}{2} - \frac{s-m}{d}}.
 \end{equation}

\end{theorem}

Note that the implied constant in the above theorem depends only upon $s,m$, and $d$, but not on $f$. Comparing this with the results in \cite{jones1992simple}, we obtain a dimension dependent improvement similar to what can be obtained using stratified sampling \cite{klusowski2018approximation} for rectified linear networks. However, the improvement in Theorem \ref{approximation-rate-theorem}, which is obtained via an entirely different argument, is greater and holds for cosine networks. We will consider rectified linear networks in the next section. Also, we note that for the Sobolev spaces $H^{\frac{d}{2} + s}(\Omega)$, this result already appears in \cite{petrushev1998approximation} . However, our results apply to the spectral Barron space $\mathcal{B}^s(\Omega)$, which is not quite comparable, but we do have $H^{\frac{d}{2} + s + \epsilon}(\Omega)\subset \mathcal{B}^s(\Omega)$ (see \cite{CiCP-28-1707} Lemma 2.5, for instance). Finally, as shown in Theorem \ref{fourier-lower-bound}, the rate in Theorem \ref{approximation-rate-theorem} is actually sharp.

\begin{proof}
 Choose $L > 1$. Note that all of the implied constants in what follows depend only upon $s,m,d$ and $L$, but not upon $f$.

 By Corollary \ref{fourier-representation-lemma}, there exists an $a\in L^{-1}[0,1]^d$ and coefficients $c_\xi$ such that
  \begin{equation}\label{eq-580}
  f(x) = \sum_{\xi\in L^{-1}\mathbb{Z}^d}c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x},
 \end{equation}
 and (here the first estimate follows since $|a|\leq L^{-d}\sqrt{d}$)
 \begin{equation}\label{eq_613}
  \sum_{\xi\in L^{-1}\mathbb{Z}^d} (1+|\xi|)^s|c_\xi| \eqsim \sum_{\xi\in L^{-1}\mathbb{Z}^d} (1+|a+\xi|)^s|c_\xi|  \lesssim \|f\|_{\mathcal{B}^s(\Omega)}.
 \end{equation}
 Consider the slightly enlarged set $\Omega^\prime = [0,L]^d \supset \Omega$. On this larger set, we have for $\xi\neq\nu\in L^{-1}\mathbb{Z}^d$
 \begin{equation}\label{orthogonality-condition}
  \langle e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}, e^{2\pi {\mathrm{i}\mkern1mu}  (a+\nu)\cdot x}\rangle_{H^k(\Omega^\prime)} = 0,
 \end{equation}
 so that the frequencies in the expansion \eqref{eq-580} form an orthogonal basis in $H^k(\Omega^\prime)$. Moreover, their lengths satisfy
 \begin{equation}\label{length-estimate}
  \|e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}\|_{H^m(\Omega^\prime)} \lesssim (1+|a+\xi|)^m \eqsim (1+|\xi|)^m.
 \end{equation}
 Order the frequencies $\xi\in L^{-1}\mathbb{Z}^d$ such that
 \begin{equation}
   (1+|\xi_1|)^{2m-s}|c_{\xi_1}| \geq (1+|\xi_2|)^{2m-s}|c_{\xi_2}| \geq (1+|\xi_3|)^{2m-s}|c_{\xi_3}| \geq \cdots.
 \end{equation}

 For $n \geq 1$, let $S_n = \{\xi_1,\xi_2,...,\xi_n\}$ and set
 \begin{equation}
  f_n = \sum_{\xi\in S_n}c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x} \in \Sigma_{n,M}
 \end{equation}
 for $M\lesssim \|f\|_{\mathcal{B}^s}$ by \eqref{eq_613}.

 We now estimate, using \eqref{orthogonality-condition} and \eqref{length-estimate},
 \begin{equation}\label{eq_634}
  \|f - f_n\|^2_{H^m(\Omega^\prime)} = \left\|\sum_{\xi\in S_n^c} c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x} \right\|^2_{H^m(\Omega^\prime)} = \sum_{\xi\in S_n^c} |c_\xi|^2\|e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}\|_{H^m(\Omega^\prime)}^2 \lesssim \sum_{\xi\in S_n^c} |c_\xi|^2(1+|\xi|)^{2m}.
 \end{equation}
 Using Hoelder's inequality, we get
 \begin{equation}\label{eq_638}
  \sum_{\xi\in S_n^c} |c_\xi|^2(1+|\xi|)^{2m} \leq \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m-s} \right)\left(\sum_{\xi\in S_n^c}|c_\xi|(1+|\xi|)^{s}\right)
 \end{equation}
 By \eqref{eq_613}, the second term above is $\lesssim \|f\|_{\mathcal{B}^s(\Omega)}$. For the first term, we note that \eqref{eq_613} implies that
 \begin{equation}\label{eq_642}
  \sum_{\nu\in S_n} |c_\nu|(1+|\nu|)^{2m-s}(1+|\nu|)^{2(s-m)}\lesssim \|f\|_{\mathcal{B}^s(\Omega)}.
 \end{equation}
 Now, by the definition of $S_n$, we have for every $\nu\in S_n$
 \begin{equation}
  \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m-s}\right) \leq |c_\nu|(1+|\nu|)^{2m-s},
 \end{equation}
 so that
 \begin{equation}
  \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m-s}\right)\sum_{\nu\in S_n}(1+|\nu|)^{2(s-m)} \leq \sum_{\nu\in S_n} |c_\nu|(1+|\nu|)^{2m-s}(1+|\nu|)^{2(s-m)}.
 \end{equation}
 By \eqref{eq_642}, we thus have
 \begin{equation}
  \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m-s}\right) \lesssim \|f\|_{\mathcal{B}^s(\Omega)}\left(\sum_{\nu\in S_n}(1+|\nu|)^{2(s-m)}\right)^{-1}.
 \end{equation}
 The sum $\sum_{\nu\in S_n}(1+|\nu|)^{2(s-m)}$ is over $n$ elements of the lattice $L^{-1}\mathbb{Z}^d$, from which it easily follows by comparison with an integral (see, for instance, \cite{erdos1989lattice}) that
 \begin{equation}
  \sum_{\nu\in S_n}(1+|\nu|)^{2(s-m)} \gtrsim n^{1+\frac{2(s-m)}{d}},
 \end{equation}
 and we obtain
 \begin{equation}
  \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m-s}\right) \lesssim \|f\|_{\mathcal{B}^s(\Omega)}n^{-1-\frac{2(s-m)}{d}}.
 \end{equation}
 Combining this with \eqref{eq_634} and \eqref{eq_638}, we get
 \begin{equation}
  \|f - f_n\|^2_{H^m(\Omega^\prime)} \lesssim \|f\|^2_{\mathcal{B}^s(\Omega)}n^{-1-\frac{2(s-m)}{d}}.
 \end{equation}
 Finally, since $\Omega^\prime \supset \Omega$, we get
 \begin{equation}
  \|f - f_n\|_{H^m(\Omega)} \leq \|f - f_n\|_{H^m(\Omega^\prime)} \lesssim \|f\|_{\mathcal{B}^s(\Omega)}n^{-\frac{1}{2}-\frac{(s-m)}{d}},
 \end{equation}
 which completes the proof.
\end{proof}

In Theorem \ref{approximation-rate-theorem}, we obtained arbitrarily high polynomial rates of convergence for sufficiently smooth functions. Next we generalize this by showing that if the Fourier transform decays at a superpolynomial rate, then we can obtain spectral (i.e. superpolynomial) convergence as well. We begin by introducing an exponential version of the spectral Barron spaces.
\begin{definition}
Let $\Omega\subset \mathbb{R}^d$ be a bounded domain and let $0 < \beta < 1$ and $c > 0$. The exponential spectral Barron space with parameters $\beta$ and $c$ is defined by
\begin{equation}
 \mathcal{B}_{\beta,c}(\Omega):=\left\{f:\Omega\rightarrow\mathbb{R}:\|f\|_{\mathcal{B}_{\beta,c}(\Omega)}:=\inf_{f_e|\Omega = f}\int_{\mathbb{R}^d}e^{c|\xi|^\beta}|\hat{f}_e(\xi)|d\xi < \infty\right\},
\end{equation}
where the infemum is taken over all extension $f_e\in L^1(\mathbb{R}^d)$.
\end{definition}
 The space $\mathcal{B}_{\beta,c}(\Omega)$ is quite restrictive, however there it still contains a relatively large class of functions. For example, it contains satisfied by any linear combination of Gaussians or any band-limited function, i.e. any function whose Fourier transform is compactly supported.

For elements of $\mathcal{B}_{\beta,c}(\Omega)$, we can prove a superpolynomial convergence rate.
\begin{theorem}\label{spectral-convergence-theorem}
 Let $\Omega = [0,1]^d$, $0 < \beta < 1$, and $c > 0$.
 Then for any $m \geq 0$, there exists a $c^\prime > 0$ such that for $f\in \mathcal{B}_{\beta,c}(\Omega)$ and $M\lesssim \|f\|_{\mathcal{B}_{\beta,c}(\Omega)}$ we have
  \begin{equation}
  \inf_{f_n\in \Sigma_{n,M}} \|f-f_n\|_{H^m(\Omega)} \lesssim \|f\|_{\mathcal{B}_{\beta,c}(\Omega)}e^{-c^\prime n^{d^{-1}\beta}}.
 \end{equation}
\end{theorem}
Note that in this theorem the implied constant and the constant $c^\prime$ only depend upon $\beta,c,d$ and $m$, but not on $f$ or $n$.
\begin{proof}
 We use a similar argument to the proof of Theorem \ref{approximation-rate-theorem}. First, we apply Lemma \ref{fourier-representation-lemma-general} to the weight $\mu(\xi) = e^{c|\xi|^{\beta}}$, to obtain an $a\in L^{-1}[0,1]^d$ and coefficients $c_\xi$ such that
  \begin{equation}
  f(x) = \sum_{\xi\in L^{-1}\mathbb{Z}^d}c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}
 \end{equation}
 and
 \begin{equation}\label{eq_361}
  \sum_{\xi\in L^{-1}\mathbb{Z}^d}  e^{c|a+\xi|^{\beta}}|c_\xi| \lesssim \|f\|_{\mathcal{B}_{\beta,c}(\Omega)}.
 \end{equation}
 As in the proof of Theorem \ref{approximation-rate-theorem}, we note that the frequencies $e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x}$ are orthogonal on the enlarger set $\Omega^\prime = [0,L]^d$ and their norms are bounded by \eqref{length-estimate}.

 This time, we order the frequencies $\xi\in L^{-1}\mathbb{Z}^d$ such that
 \begin{equation}
  (1+|\xi_1|)^{2k}e^{-c|a+\xi_1|^{\beta}}|c_{\xi_1}| \geq  (1+|\xi_2|)^{2k}e^{-c|a+\xi_2|^{\beta}}|c_{\xi_2}| \geq  (1+|\xi_3|)^{2k}e^{-c|a+\xi_3|^{\beta}}|c_{\xi_3}| \geq \cdots.
 \end{equation}
 Choosing $S_n = \{\xi_1,...,\xi_n\}$ and setting
 \begin{equation}
  f_n(x) = \sum_{\xi\in S_n}c_\xi e^{2\pi {\mathrm{i}\mkern1mu}  (a+\xi)\cdot x} \in \Sigma_{n,M},
 \end{equation}
 with $M\lesssim \|f\|_{\mathcal{B}_{\beta,c}(\Omega)}$, we obtain, using the argument between equations \eqref{eq_634} and \eqref{eq_638}, that
 \begin{equation}\label{eq_375}
   \|f - f_n\|^2_{H^m(\Omega^\prime)} \leq \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m}e^{-c|a+\xi|^{\beta}} \right)\left(\sum_{\xi\in S_n^c}|c_\xi|e^{c|a+\xi|^{\beta}}\right).
 \end{equation}
 By \eqref{eq_361}, the second factor is $\lesssim \|f\|_{\mathcal{B}_{\beta,c}(\Omega)}$.

 For the first factor, the argument between equations \eqref{eq_613} and \eqref{eq_642} implies that
 \begin{equation}\label{eq_381}
  \left(\sup_{\xi\in S_n^c} |c_\xi|(1+|\xi|)^{2m}e^{-c|a+\xi|^{\beta}} \right) \lesssim C_f\left(\sum_{\nu\in S_n} e^{2c|a+\nu|^{\beta}}(1 + |a + \nu|)^{-2m}\right)^{-1}.
 \end{equation}
 We now proceed to lower bound the sum on the right by considering its largest term. Since the sum is over $n$ elements of the lattice $a + L^{-1}\mathbb{Z}^d$, the longest vector, i.e. the largest length $|a+\xi|$ which occurs in the sum, must be $\gtrsim n^{\frac{1}{d}}$. In addition $(1 + |a + \xi|)^{2m} \lesssim e^{2\epsilon|a+\xi|^{\beta}}$ for any $\epsilon > 0$, so we see that there must exist a $c^\prime > 0$ such that
 \begin{equation}
  \left(\sum_{\nu\in S_n} e^{2c|a+\nu|^{\beta}}(1 + |a + \nu|)^{-2m}\right) \gtrsim e^{2c^\prime n^{\frac{\beta}{d}}}.
 \end{equation}
 Plugging this into \eqref{eq_381} and \eqref{eq_375} and using the fact that $\Omega^\prime \subset \Omega$, we get
 \begin{equation}
  \inf_{f_n\in \Sigma_{n,M}} \|f-f_n\|_{H^m(\Omega)} \lesssim \|f\|_{\mathcal{B}_{\beta,c}(\Omega)}e^{-c^\prime n^{\frac{\beta}{d}}},
 \end{equation}
 as desired.

\end{proof}


\subsection{1D Heat Rod}\label{ex:heat-set-sample}

We demonstrate that the same principles illustrated in Example\ref{ex:decay-set-sample} apply to another nonlinear problem.
The set-- and sample--based methods are functionally equivalent solution methods which leverage the same foundational measure--theoretic principles.
Here, the quantification of uncertainty is for the thermal conductivity properties of a material based on temperature data collected after heating it in an experiment.
Consider the one-dimensional heat equation with homogeneous Neumann boundary conditions on the unit interval:

\begin{equation}
\begin{split}
\rho c \frac{\partial T}{\partial t} = \nabla \cdot ( \kappa \nabla T) + f(x), \quad & x\in (0,1), t\in (0,1) \\
f(x) = A e^\frac{- (x-0.5)^2}{w} \Chi_{[0,0.5]}(t)
\end{split}
\end{equation}
\emph{Alternative setup: }

\begin{equation}
\begin{cases}
\rho c \frac{\partial T}{\partial t} = \nabla \cdot ( \kappa \nabla T) + f(x,t), & \text{if } x\in \Omega \\
\frac{\partial T}{\partial \vec{n}} = 0 & \text{if } x\in \partial \Omega
\end{cases}
\end{equation}
where $\Omega = (0,1)\times (0,1)$ is the space-time interior and $f(x,t) = A e^\frac{- (x-0.5)^2}{w} \Chi_{[0,0.5]}(t)$.

Here, we interpret the following problem as heating the middle of an infinitesimally thin unit-length rod for half a second with a heat-source modeled by a Gaussian curve with amplitude $A=50$ and variance of $w=0.05$.
This heat source is turned on at the beginning of the experiment and turned off halfway through the 1-second duration.

The rod is subdivided in two, and each half has an uncertain thermal diffusivity $\kappa \in [0.01, 0.2]$.
This yields a two-dimensional parameter space $\param = (\param_1, \param_2) \in [0.01, 0.2]^2$, where $\param_1$ represents the thermal diffusion on the left-half and $\param_2$ is the $\kappa$ for the right half.

Two measurement locations along the rod are used for taking a measurement
We assume a uniform density for each observation with a side--length of $0.1$ and attempt to identify a set which contains our true reference value of $\paramref = (0.15, 0.05)$.
In Figure~\ref{fig:heatrod-sol-ex} we show the solution to this SIP for the set-- and sample--based methods for $N=50$ parameter samples used to explore $\pspace$.

\begin{figure}[h]
\begin{minipage}{.475\textwidth}
\includegraphics[width=\linewidth]{examples/fig_heatrod_q1/HeatrodModel--set_N50_em.png}
\end{minipage}
\begin{minipage}{.475\textwidth}
\includegraphics[width=\linewidth]{examples/fig_heatrod_q1/HeatrodModel--sample_N50_mc.png}
\end{minipage}
\caption{The inverse image of the reference measure for set-based (left) and sample-based (right) solutions for $\nsamps=50$ parameter samples.}
\label{fig:heatrod-sol-ex}
\end{figure}

The experimental results in \ref{fig:heatrod-sol-ex} both show images which identify $\paramref$ as a feasible solution.
Both solutions again agree on the support of the inverse image.
We will look a the impacts of the sample size $\nsamps$ in the next chapter and study convergence towards a high-fidelity solution for different choices of measurement locations and values of $\paramref$.

[ TK - I don't know, there's not a whole lot to say. At this point I'm just establishing that both these methods are more or less equivalent despite having quite different algorithmic approaches.]


\subsection{Prime ideals}\label{subsec:prime_ideals}

\begin{definition}\label{def:prime_ring_ideal}\mcite[384]{Knapp2016BasicAlgebra}
  An ideal \( P \) is called \term{prime} if it is proper and satisfies any of the equivalent conditions:
  \begin{thmenum}
    \thmitem{def:prime_ring_ideal/direct} If \( x, y \in R \) are such that \( xy \in P \), then either \( x \in P \) or \( y \in P \).
    \thmitem{def:prime_ring_ideal/ideals} If \( I, J \subseteq R \) are ideals such that \( IJ \subseteq P \), then either \( I \subseteq P \) or \( J \subseteq P \).
    \thmitem{def:prime_ring_ideal/quotient} The quotient \( R / P \) is an integral domain.
  \end{thmenum}

  An element \( r \in R \) is called \term{prime} if the ideal \( \braket r \) is prime.
\end{definition}
\begin{proof}
  \ImplicationSubProof{def:prime_ring_ideal/direct}{def:prime_ring_ideal/ideals} Fix ideals \( I, J \) of \( R \) such that \( IJ \subseteq P \).

  Assume that neither \( I \not\subseteq P \) nor \( J \not\subseteq P \). Take \( x \in I \setminus P \) and \( y \in J \setminus P \). It follows that \( xy \in P \) and either \( x \in P \) or \( y \in P \). This contradicts our assumption.

  The obtained contradiction proves that either \( I \subseteq P \) or \( J \subseteq P \).

  \ImplicationSubProof{def:prime_ring_ideal/ideals}{def:prime_ring_ideal/quotient} Fix an ideal \( P \) such that if \( I, J \subseteq R \) are ideals and \( IJ \subseteq P \), then either \( I \subseteq P \) or \( J \subseteq P \).

  We will prove that \( R / P \) is an integral domain. If \( R \) is an integral domain, this is obvious. If not, we fix nonzero \( x, y \in R \), so that \( xy = 0 \). Thus, \( [x][y] = (x + P)(y + P) = xy + P = P = [0] \). We will show that either \( x = 0 \) or \( y = 0 \).

  Consider the ideals
  \begin{balign*}
    \braket{x} & = xR, \\
    \braket{y} & = yR.
  \end{balign*}

  By \fullref{thm:product_of_principal_ideals}, we have \( \braket{x} \braket{y} = \braket{xy} = \braket{0} = \{ 0 \} \).

  Since \( \braket{x} \braket{y} \subseteq P \), then either \( \braket{x} \subseteq P \) or \( \braket{y} \subseteq P \). That is, either \( [x] = 0 \) or \( [y] = 0 \).

  Thus, \( R / P \) is an integral domain.

  \ImplicationSubProof{def:prime_ring_ideal/quotient}{def:prime_ring_ideal/direct} Suppose that \( R / P \) is an integral domain. Fix \( x, y \in R \), so that \( xy \in P \). If \( x = 0 \), obviously \( x = 0 \in R \) and similarly for \( y \). Suppose that both \( x \) and \( y \) are nonzero. We will show that either \( x \in P \) or \( y \in P \).

  We have
  \begin{equation*}
    [x][y] = [xy] = xy + P = P = [0].
  \end{equation*}

  Since \( R / P \) is an integral domain, either \( [x] = [0] \) or \( [y] = [0] \). That is, either \( x \in P \) or \( y \in P \).
\end{proof}

\begin{proposition}\label{thm:prime_ideal_iff_prime_quotient_ideal}
  If \( J \subseteq I \) are ideals of \( R \), then \( I \) is a \hyperref[def:prime_ring_ideal]{prime ideal} in \( R \) if and only if \( I / J \) is a prime ideal in  \( R / J \).
\end{proposition}

\begin{definition}\label{def:irreducible_ring_element}
  A nonzero element \( r \in R \) of an integral domain is called \term{reducible} if there exist non-invertible elements \( r_1, r_2 \in R \) such that
  \begin{equation*}
    r = r_1 r_2.
  \end{equation*}

  If \( r \) is not reducible, we say that it is \term{irreducible}.
\end{definition}

\begin{definition}\label{def:coprime_ring_ideals}
  Two ring ideals \( I \subseteq R \) and \( J \subseteq R \) are said to be \term{coprime} if \( I + J = R \).
\end{definition}

\begin{proposition}\label{thm:prime_implies_irreducible}\mcite[389]{Knapp2016BasicAlgebra}
  All \hyperref[def:prime_ring_ideal]{prime} elements in an integral domain are \hyperref[def:irreducible_ring_element]{irreducible}.
\end{proposition}
\begin{proof}
  Let \( p \) be prime. Assume that \( p \) is reducible, that is, there exist non-invertible elements \( r_1, r_2 \in R \) such that
  \begin{equation*}
    p = r_1 r_2.
  \end{equation*}

  Since \( p \) is prime, it must divide either \( r_1 \) or \( r_2 \). Without loss of generality, assume that \( p | r_1 \) and \( r_1 = pc \) for some \( c \in R \).

  Then \( p = r_1 r_2 = pc r_2 \). By \fullref{thm:def:semiring/properties/cancellable_iff_not_zero_divisor}, \( 1 = c r_2 \), which implies that \( r_2 \) is invertible with inverse \( c \). This contradicts our assumption that both \( r_1 \) and \( r_2 \) are invertible.

  The obtained contradiction proves that \( p \) is irreducible.
\end{proof}

\begin{definition}\label{def:maximal_ring_ideal}
  A two-sided ideal \( M \) is called \term{maximal} if it is proper and satisfies any of the equivalent conditions:
  \begin{thmenum}
    \thmitem{def:maximal_ring_ideal/maximality} \( M \) is maximal with respect to set inclusion among proper two-sided ideals.
    \thmitem{def:maximal_ring_ideal/quotient} The quotient \( R / M \) is a field.
  \end{thmenum}
\end{definition}
\begin{proof}
  \ImplicationSubProof{def:maximal_ring_ideal/maximality}{def:maximal_ring_ideal/quotient} Suppose that \( M \) is maximal among proper ideals. We will prove that every nonzero element of \( R / M \) is invertible.

  Fix \( x \not\in M \), so that \( [x] = x + M \neq M = [0] \). Define the set
  \begin{equation*}
    I \coloneqq Rx + M.
  \end{equation*}

  It is a ideal since both \( Rx \) and \( M \) are ideals. Furthermore, it contains \( M \) strictly because \( M \subseteq I \) and \( x \in I \). Since \( M \) is maximal, we have that \( I = R \).

  Hence, there exists \( y \in R \) such that \( 1 = yx + M \). Hence, \( [y] = y + M \) is an inverse of \( [x] \) in \( R / M \).

  Since \( [x] \in R / M \) was an arbitrary nonzero element, we conclude that \( R / M \) is a field.

  \ImplicationSubProof{def:maximal_ring_ideal/quotient}{def:maximal_ring_ideal/maximality} Suppose that \( R / M \) is a field. Assume that \( M \) is not maximal. Then there exists a proper ideal \( I \supsetneq M \).

  Assume that \( I \neq M \) and take \( x \in I \setminus M \). Then \( x \not\in M \) and hence \( [x] \neq [0] \) and is invertible in \( R / M \). Denote by \( y \) any representative of this inverse. Thus, \( [xy] - [1] = [0] \), that is, \( xy - 1 \in M \).

  Note that \( xy \in I \) because \( x \in I \) and \( y \in R \). Since \( I \) is closed under addition, it follows that \( 1 \in I \) and hence \( I = R \). But this contradicts our assumption that \( I \) is proper.

  The obtained contradiction proves that \( M \) is maximal.
\end{proof}

\begin{theorem}[Krull's theorem]\label{thm:krulls_theorem}\mcite{Hodges1979}
  Every nontrivial \hyperref[def:semiring/commutative_unital_ring]{commutative unital ring} has a \hyperref[def:maximal_ring_ideal]{maximal ideal}.

  In \hyperref[def:zfc]{\logic{ZF}} this theorem is equivalent to the \hyperref[def:zfc/choice]{axiom of choice} --- see \fullref{thm:axiom_of_choice_equivalences/krull}.
\end{theorem}

\begin{proposition}\label{thm:maximal_ideals_are_prime}
  Maximal ring \hyperref[def:maximal_ring_ideal]{ideals} are \hyperref[def:prime_ring_ideal]{prime}.
\end{proposition}
\begin{proof}
  If \( M \) is a maximal ideal of \( R \), by \fullref{def:maximal_ring_ideal/quotient} \( R / M \) is a field. Thus, \( R / M \) is an integral domain, which by \fullref{def:prime_ring_ideal/quotient} means that \( M \) is a prime ideal.
\end{proof}

\begin{proposition}\label{thm:field_maximal_ideal_representation}\mcite[exer. 8.1]{КоцевСидеров2016}
  If \( r_1, \ldots, r_n \) are elements of the field \( \BbbK \), then \( \braket{X_1 - r_1, \ldots, X_n - r_n} \) is a maximal ideal of \( \BbbK[X_1, \ldots, X_n] \).
\end{proposition}

\begin{proposition}\label{thm:ufd_prime_iff_irreducible}
  An element in a unique factorization domain is \hyperref[def:prime_ring_ideal]{prime} if and only if it is \hyperref[def:irreducible_ring_element]{irreducible}.
\end{proposition}
\begin{proof}
  \SufficiencySubProof Follows from \fullref{thm:prime_implies_irreducible}.

  \NecessitySubProof Let \( r \) be an irreducible element and let \( p_1 p_2 \in \braket{r} \). We will show that either  \( p_1 \in \braket{r} \) or \( p_2 \in \braket{r} \).

  Since \( \braket{r} \) is an ideal, there exists an element \( q \in R \) such that \( qr = p_1 p_2 \). Because of unique factorization, there exists a unit \( u \in R \) such that \( uqr = p_1 p_2  \).

  Therefor either
  \begin{itemize}
    \item \( p_1 = 1 \), in which case \( p_2 = uqr \in \braket{r} \).
    \item \( p_1 = u \), in which case \( p_2 = qr \in \braket{r} \).
    \item \( p_1 = uq \), in which case \( p_2 = r \in \braket{r} \).
    \item \( p_1 = ur \in \braket{r} \).
    \item \( p_1 = qr \in \braket{r} \).
    \item \( p_1 = uqr \in \braket{r} \).
  \end{itemize}
\end{proof}

\begin{proposition}\label{thm:prime_ideals_are_maximal_in_pid}
  Prime ring \hyperref[def:prime_ring_ideal]{ideals} in a principal ideal domain are \hyperref[def:maximal_ring_ideal]{maximal}.
\end{proposition}
\begin{proof}
  Let \( P \) be a prime ideal of \( R \) and let \( I \supsetneq P \) be an ideal strictly containing \( P \). We will show that \( I = R \).

  Since \( R \) is a principal ideal domain, both \( P \) and \( I \) are principal. Let \( p \) and \( i \) be their respective generators. Since \( I \) contains \( P \), there exists \( r \in R \) such that
  \begin{equation*}
    p = ir.
  \end{equation*}

  But \( p \) is prime, and thus irreducible by \fullref{thm:ufd_prime_iff_irreducible}, and hence either \( i \) or \( r \) must be a unit. If \( r \) is a unit, then \( \braket i = \braket {ir} = \braket p \), which contradicts our choice of \( I \supsetneq P \). It remains for \( i \) to be a unit.

  Therefore, \( I = \braket i = \braket 1 = R \). This proves that \( P \) is maximal with respect to inclusion of ideals.
\end{proof}

\begin{definition}\label{def:krull_dimension}\mcite[67]{КоцевСидеров2016}
  Consider chains
  \begin{equation*}
    P_0 \subsetneq P_1 \subsetneq \cdots
  \end{equation*}
  of prime \hyperref[def:prime_ring_ideal]{ideals} in \( R \) under strict inclusion. The length of this chain is defined as the zero-based index of its last element and is allowed to be infinite. Zero-based means that a chain with only one ideal has length zero.

  We call the supremum of the lengths of these chains the \term{Krull dimension} of the ring \( R \) and denote it by \( \dim R \).
\end{definition}

\begin{proposition}\label{thm:def:krull_dimension/properties}
  The Krull \hyperref[def:krull_dimension]{dimension} of a ring \( R \) has the following basic properties:
  \begin{thmenum}
    \thmitem{thm:def:krull_dimension/properties/monotone} If \( R = T / S \) is the quotient of some rings \( S \subseteq T \), \( \dim R \leq \dim T \).
    \thmitem{thm:def:krull_dimension/properties/pid} If \( R \) is a principal ideal domain, \( \dim R \in \{ 0, 1 \} \).
    \thmitem{thm:def:krull_dimension/properties/field} If \( R \) is a \hyperref[def:field]{field}, \( \dim R = 0 \).
    \thmitem{thm:def:krull_dimension/properties/polynomials_over_field}\cite[exercise 8.19]{КоцевСидеров2016} If \( R = \BbbK[X_1, \ldots, X_n] \) for some \hyperref[def:field]{field} \( \BbbK \), \( \dim R = n \).
  \end{thmenum}
\end{proposition}

\begin{corollary}\label{thm:multivariate_polynomial_rings_are_not_pid}
  Multivariate polynomial rings are not principal ideal domains.
\end{corollary}
\begin{proof}
  Follows from \fullref{thm:def:krull_dimension/properties/pid} and \fullref{thm:def:krull_dimension/properties/polynomials_over_field}.
\end{proof}

\begin{definition}\label{def:radical_ideal}\mcite[15]{КоцевСидеров2016}
  We define the \term{radical ideal} of the ideal \( I \) of \( R \) as
  \begin{equation*}
    \sqrt I \coloneqq \{ x \in R \colon \exists n: x^n \in I \}.
  \end{equation*}
\end{definition}
\begin{proof}
  We verify that the set \( \sqrt I \) is an ideal of \( R \).

  It is closed under addition because if \( x^n \in I \) and \( y^m \in I \), then by \fullref{thm:binomial_theorem},
  \begin{equation*}
    (x + y)^{n+m}
    =
    \sum_{k=0}^{n+m} \binom n k x^k y^{n+m-k}
  \end{equation*}

  In this sum, either \( k \geq n \) and thus the product \( x^k y^{n+m-k} \in I \), or \( k < n \), in which case \( n + m - k > n + m - n = m \) and the same holds. Thus,
  \begin{equation*}
    (x + y)^{n+m} \in I
  \end{equation*}
  and \( x + y \in \sqrt I \).

  The radical is also closed under multiplication with \( R \) since if \( x^n \in I \), then \( (rx)^n = r^n x^n \in I \).

  Therefore, it is an ideal of \( R \).
\end{proof}

\begin{definition}\label{def:primary_ring_ideal}\mcite[74]{КоцевСидеров2016}
  We call the proper ideal \( P \) of \( R \) \term{primary} if \( xy \in P \) implies that \( x \in P \) or \( y \in \sqrt P \).
\end{definition}

\begin{theorem}[Chinese remainder theorem]\label{thm:chinese_remained_theorem}\mcite[thm. 8.27]{Knapp2016BasicAlgebra}
  Let \( I_1, \ldots, I_n \) be pairwise \hyperref[def:coprime_ring_ideals]{coprime} ideals. Then
  \begin{equation*}
    R / \bigcap_{i=1}^n I_n \cong R / I_1 \times \cdots \times R / I_n.
  \end{equation*}
\end{theorem}

\begin{definition}\label{def:spectrum_of_ring}
  The set of all prime ideals on a ring \( R \) is called the \term{spectrum} of \( R \) denoted by \( \op{Spec}(R) \).
\end{definition}


\subsubsection*{Examples of Embedded Submanifolds }

\begin{lemma}[8.6] (\textbf{Graphs as Submanifolds}).
If open $U \subset \mathbb{R}^n$, smooth $F: U \to \mathbb{R}^k$, \\
then graph of $F$ is an embedded $n$-dim. submanifold of $\mathbb{R}^{n+k}$
\end{lemma}

\begin{proof}
  Define $\begin{aligned}
    & \quad \\
    & \varphi: U \times \mathbb{R}^k \to U \times \mathbb{R}^k \\
    & \varphi(x,y) = (x,y - F(x)) \end{aligned}$

$\varphi$ clearly smooth. \\
$\varphi$ diffeomorphism because its inverse can be written explicitly
\[
\varphi^{-1}(u,v) = (u ,  v + F(u))
\]

$\varphi( \Gamma(F))$ is the slice $\lbrace (u,v) : v = 0 \rbrace $ of $U\times \mathbb{R}^k$, so graph $\Gamma(F)$ is an embedded submanifold.

\end{proof}

\subsubsection*{Level Sets}

\subsection*{Immersed Submanifolds}

\begin{definition}
  \textbf{immersed submanifold } of $M$, $S$, is $S\subseteq M$, with topology (not necessarily subspace topology) with respect to which it's a topological manifold (without boundary), and \\
\phantom{ \quad \quad \, }   smooth structure with respect to inclusion map $i : S \hookrightarrow M$ is smooth immersion (recall $Di \equiv i_*$ injective $\Longleftrightarrow \text{rank}Di = \text{dim}S$ \\

$\text{codim}S = \text{dim}M - \text{dim}S$ \\

\textbf{smooth hypersurface} is immersed submanifold of codimension $1$.
\end{definition}