This section provides some additional examples for the features listed
in the paper.

\subsection{Simplification}
\begin{itemize}
\item \texttt{expand}:
\begin{verbatim}
>>> expand((x + y)**3)
x**3 + 3*x**2*y + 3*x*y**2 + y**3
\end{verbatim}

\item \texttt{factor}:
\begin{verbatim}
>>> factor(x**3 + 3*x**2*y + 3*x*y**2 + y**3)
(x + y)**3
\end{verbatim}

\item \texttt{collect}:
\begin{verbatim}
>>> collect(y*x**2 + 3*x**2 - x*y + x - 1, x)
x**2*(y + 3) + x*(-y + 1) - 1
\end{verbatim}

\item \texttt{cancel}:
\begin{verbatim}
>>> cancel((x**2 + 2*x + 1)/(x**2 - 1))
(x + 1)/(x - 1)
\end{verbatim}

\item \texttt{apart}:
\begin{verbatim}
>>> apart((x**3 + 4*x - 1)/(x**2 - 1))
x + 3/(x + 1) + 2/(x - 1)
\end{verbatim}

\item \texttt{trigsimp}:
\begin{verbatim}
>>> trigsimp(cos(x)**2*tan(x) - sin(2*x))
-sin(2*x)/2
\end{verbatim}

\item \texttt{hyperexpand} (showing ${{}_{2}F_{1}\left(\begin{matrix} 1, 1 \\
        2 \end{matrix}\middle| {- x} \right)} = \frac{\log{\left (x + 1 \right )}}{x}$):
\begin{verbatim}
>>> hyperexpand(hyper([1, 1], [2], -x))
log(x + 1)/x
\end{verbatim}
\end{itemize}

\subsection{Polynomials}
%% TODO - explain why these matter
\begin{itemize}
\item Factorization:
\begin{verbatim}
>>> t = symbols('t')
>>> f = (2115*x**4*y + 45*x**3*z**3*t**2 - 45*x**3*t**2 -
...      423*x*y**4 - 47*x*y**3 + 141*x*y*z**3 + 94*x*y*z*t -
...      9*y**3*z**3*t**2 + 9*y**3*t**2 - y**2*z**3*t**2 +
...      y**2*t**2 + 3*z**6*t**2 +  2*z**4*t**3 - 3*z**3*t**2 -
...      2*z*t**3)
>>> factor(f)
(t**2*z**3 - t**2 + 47*x*y)*(2*t*z + 45*x**3 - 9*y**3 - y**2 +
 3*z**3)
\end{verbatim}

\item Gr\"{o}bner bases:
\begin{verbatim}
>>> x0, x1, x2 = symbols('x0 x1 x2')
>>> I = [x0 + 2*x1 + 2*x2 - 1,
...      x0**2 + 2*x1**2 + 2*x2**2 - x0,
...      2*x0*x1 + 2*x1*x2 - x1]
>>> groebner(I, order='lex')
GroebnerBasis([7*x0 - 420*x2**3 + 158*x2**2 + 8*x2 - 7,
7*x1 + 210*x2**3 - 79*x2**2 + 3*x2,
84*x2**4 - 40*x2**3 + x2**2 + x2], x0, x1, x2, domain='ZZ',
order='lex')
\end{verbatim}

\item Root isolation:
\begin{verbatim}
>>> f = 7*z**4 - 19*z**3 + 20*z**2 + 17*z + 20
>>> intervals(f, all=True, eps=0.001)
([],
 [((-425/1024 - 625*I/1024, -1485/3584 - 2185*I/3584), 1),
  ((-425/1024 + 2185*I/3584, -1485/3584 + 625*I/1024), 1),
  ((3175/1792 - 2605*I/1792, 1815/1024 - 10415*I/7168), 1),
  ((3175/1792 + 10415*I/7168, 1815/1024 + 2605*I/1792), 1)])
\end{verbatim}
\end{itemize}
\subsection{Solvers}

\begin{itemize}
\item Single solution:
\begin{verbatim}
>>> solveset(x - 1, x)
{1}
\end{verbatim}

\item Finite solution set, quadratic equation:
\begin{verbatim}
>>> solveset(x**2 - pi**2, x)
{-pi, pi}
\end{verbatim}

\item No solution:
\begin{verbatim}
>>> solveset(1, x)
EmptySet()
\end{verbatim}

\item Interval solution:
\begin{verbatim}
>>> solveset(x**2 - 3 > 0, x, domain=S.Reals)
(-oo, -sqrt(3)) U (sqrt(3), oo)
\end{verbatim}

\item Infinitely many solutions:
% SymPy 1.0 sstr() prints S.Complexes incorrectly
% no-doctest
\begin{verbatim}
>>> solveset(x - x, x, domain=S.Reals)
(-oo, oo)
>>> solveset(x - x, x, domain=S.Complexes)
S.Complexes
\end{verbatim}

\item Linear systems (\texttt{linsolve})
\begin{verbatim}
>>> A = Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 10]])
>>> b = Matrix([3, 6, 9])
>>> linsolve((A, b), x, y, z)
{(-1, 2, 0)}
>>> linsolve(Matrix(([1, 1, 1, 1], [1, 1, 2, 3])), (x, y, z))
{(-y - 1, y, 2)}
\end{verbatim}
\end{itemize}

Below are examples of \texttt{solve} applied to problems not yet handled by \texttt{solveset}.

\begin{itemize}
\item Nonlinear (multivariate) system of equations (the intersection of a circle
and a parabola):
\begin{verbatim}
>>> solve([x**2 + y**2 - 16, 4*x - y**2 + 6], x, y)
[(-2 + sqrt(14), -sqrt(-2 + 4*sqrt(14))),
 (-2 + sqrt(14), sqrt(-2 + 4*sqrt(14))),
 (-sqrt(14) - 2, -I*sqrt(2 + 4*sqrt(14))),
 (-sqrt(14) - 2, I*sqrt(2 + 4*sqrt(14)))]
\end{verbatim}

\item Transcendental equations:
\begin{verbatim}
>>> solve((x + log(x))**2 - 5*(x + log(x)) + 6, x)
[LambertW(exp(2)), LambertW(exp(3))]
>>> solve(x**3 + exp(x))
[-3*LambertW((-1)**(2/3)/3)]
\end{verbatim}
\end{itemize}

\subsection{Matrices}

\begin{itemize}
\item Matrix expressions
\begin{verbatim}
>>> m, n, p = symbols('m n p', integer=True)
>>> R = MatrixSymbol('R', m, n)
>>> S = MatrixSymbol('S', n, p)
>>> T = MatrixSymbol('T', m, p)
>>> U = R*S + 2*T
>>> U.shape
(m, p)
>>> U[0, 1]
2*T[0, 1] + Sum(R[0, _k]*S[_k, 1], (_k, 0, n - 1))
\end{verbatim}

\item Block Matrices
\begin{verbatim}
>>> n, m, l = symbols('n m l')
>>> X = MatrixSymbol('X', n, n)
>>> Y = MatrixSymbol('Y', m ,m)
>>> Z = MatrixSymbol('Z', n, m)
>>> B = BlockMatrix([[X, Z], [ZeroMatrix(m, n), Y]])
>>> B
Matrix([
[X, Z],
[0, Y]])
>>> B[0, 0]
X[0, 0]
>>> B.shape
(m + n, m + n)
\end{verbatim}
\end{itemize}


\lab{Gaussian Quadrature}{Gaussian Quadrature}
\label{Lab:GaussQuad}

\objective{
Numerical quadrature is an important numerical integration technique.
The popular Newton-Cotes quadrature uses uniformly spaced points to approximate the integral, but Gibbs phenomenon prevents Newton-Cotes from being effective for many functions.
The Gaussian Quadrature method uses carefully chosen points and weights to mitigate this problem.
}

\section*{Shifting the Interval of Integration}

As with all quadrature methods, we begin by choosing a set of points $x_i$ and weights $w_i$ to approximate an integral.
\[
\int_{a}^b f(x) dx \approx \sum_{i=1}^n w_if(x_i).
\]

When we do guassian quadrature, we are required to choose a weight function $W(x)$.
This function determines both the $x_i's$ and the $w_i's$.
Theoretically, the weight function determines a set of orthogonal polynomials to approximate the function $f$.

The weight function also determines the interval over which the integration will occur.
For example, we choose the weight function as $W(x) = 1$ over $[-1,1]$ to integrate functions on $[-1,1]$.
To calculate the definite integrate over any interval, we perform a u-substitution.
This results in the following formula.
\[
\int_a^b f(x) dx = \frac{b-a}{2}\int_{-1}^1 f(\frac{b-a}{2}z + \frac{a+b}{2})dz.
\]

Once we have changed the interval, we may apply quadrature to the integral from $-1$ to $1$ and then scale it appropriately to get the answer we want.
\[
\int_a^b f(x) dx \approx \frac{b - a}{2} \sum_i w_if(\frac{(b-a)}{2}x_i + \frac{(b+a)}{2})
\]

% Problem 1: shift the integral for a particular function.
\begin{problem}
Let $f(x) = x^2$ on $[1,4]$. Then $g(x)$ will be the interval-adjusted version of $f$ on $[-1,1]$, with $W(x)=1$, $a=1$, and $b=4$. So,
\begin{align*}
g(x) &= f(\frac{b-a}{2}x + \frac{b+a}{2}) \\
&= \frac{9}{4} x^2 + \frac{15 x}{2} + \frac{25}{4}
\end{align*}

and the interval-adjusted integral of $f(x)$ will be

\begin{align*}
G(x) &= \frac{b - a}{2} \int f(\frac{b - a}{2} x + \frac{b + a}{2})dx \\
&= \frac{9}{8} x^3 + \frac{45}{8} x^2 + \frac{75}{8} x
\end{align*}

Verify that evaluating $G(1) - G(-1) = \int_1^4 f(x)dx$.
\end{problem}

% Problem 2: general integral shift.
\begin{problem}
Write a function that will accept a function $f$ and an interval $[a,b]$ and return a function $g$ on $[-1,1]$ that has the same integral (scaled by a constant) as $f$.

Use your function to plot $f(x) = x^2$ on $[1, 4]$ and the corresponding function $\frac{\left(b - a\right)}{2}g$ on $[-1,1]$.
Note that the functions will not look the same plotted, since they are defined over intervals with different lengths, but they integrate to the same value.
\end{problem}

\section*{Integrating with Given Weights and Points}

We now give an example of quadrature with known weights and points.
We use the constant weight function $W(x) = 1$ from $-1$ to $1$ (this weight function corresponds to the Legendre polynomials) to calculate the integral of $f(x) = sin(x)$ from $-\pi$ to $\pi$, with $5$ interpolation points.

First, we change the interval from $[-\pi,\pi]$ to $[-1,1]$.

\begin{lstlisting}
>>> import numpy as np
>>> a, b = - np.pi, np.pi

# f is the function to integrate.
>>> f = np.sin

# g is the function with the interval changed.
>>> g = lambda x: f((b - a) / 2 * x + (a + b) / 2)
\end{lstlisting}

\begin{table}[h!]
\begin{center}
\begin{tabular}{|c|c|}
\hline
point $x_i$ & weight $w_i$ \\
\hline
$-\frac{1}{3}\sqrt{5 + 2\sqrt{\frac{10}{7}}}$ &  $\frac{322-13\sqrt{70}}{900}$ \\
\hline
$-\frac{1}{3}\sqrt{5 - 2\sqrt{\frac{10}{7}}}$ & $\frac{322+13\sqrt{70}}{900}$ \\
\hline
$0$ & $\frac{128}{225}$ \\
\hline
$\frac{1}{3}\sqrt{5 - 2\sqrt{\frac{10}{7}}}$ & $\frac{322+13\sqrt{70}}{900}$ \\
\hline
$\frac{1}{3}\sqrt{5 + 2\sqrt{\frac{10}{7}}}$ & $\frac{322-13\sqrt{70}}{900}$ \\
\hline
\end{tabular}
\end{center}
\caption{Quadrature points and weights on $\left[-1, 1\right]$.}
\label{intro_table}
\end{table}

The weights($w_i$) and points at which $f$ is evaluated ($x_i$) are given in order in Table \ref{intro_table}.
We put them into an array here.

%TODO: fix these thingies.
\begin{lstlisting}
>>> from math import sqrt
>>> points = np.array([- sqrt(5 + 2 * sqrt(10. / 7)) / 3,
                       - sqrt(5 - 2 * sqrt(10. / 7)) / 3,
                       0,
                       sqrt(5 - 2 * sqrt(10. / 7)) / 3,
                       sqrt(5 + 2 * sqrt(10. / 7)) / 3])
>>> weights = np.array([(322 - 13 * sqrt(70)) / 900,
                        (322 + 13 * sqrt(70)) / 900,
                        128. / 225,
                        (322 + 13 * sqrt(70)) / 900,
                        (322 - 13 * sqrt(70)) / 900])
\end{lstlisting}

We now calculate the integral

\begin{lstlisting}
>>> integral = (b - a)/2 * np.inner(weights, g(points))
\end{lstlisting}

% Problem 3: Integrating with Weights.
\begin{problem}
Write a function that accepts a function f, an array of points, an array of weights, and limits of integration and returns the integral.
Don't forget to adjust the interval as in the above example.
\end{problem}

\section*{Calculating Weights and Points}

Calculating an integral when the weights and points are given is straightforward.
But, how are these weights and points found?
There are many publications that will give tables of points for various weight functions.
We will demonstrate how to find such a list using the Golub-Welsch algorithm.

\subsection*{The Golub-Welsch Algorithm}

This Golub-Welsch algorithm builds a tri-diagonal matrix and finds its eigenvalues.
These eigenvalues are the points at which a function is evaluated for Guassian quadrature.
The weights are the length of $\left[a, b\right]$ times the first coordinate of each eigenvector squared.
We note that finding eigenvalues for a tridiagonal matrix is a well conditioned, relatively painless problem.
Using a good eigenvalue solver gives the Golub-Welsch algorithm a complexity of $O(n^2)$.
A full treatment of the Golub-Welsch algorithm may be found at \url{http://gubner.ece.wisc.edu/gaussquad.pdf}.

We mentioned that the choice of weight function corresponds to a class of orthogonal polyomials.
An important fact about orthogonal polynomials is that any set of orthogonal polynomials $\{u_i\}_{i=1}^{N}$ satisfies a three term recurrence relation
\[
u_{i}(x) = (\gamma_{i-1}x-\alpha_i)u_{i-1}(x) - \beta_iu_{i-2}(x)
\]
where $u_{-1}(x) = 0$ and $u_0(x) = 1$.
The coefficients $\{\gamma_k, \alpha_i, \beta_i\}$ have been calculated for several classes of orthogonal polynomials, and may be determined for an arbitrary class using the procedure found in ``Calculation of Gauss Quadrature Rules'' by Golub and Welsch.
Using these coefficients we may create a tri-diagonal matrix

\[
J = \begin{bmatrix}

a_1 & b_1 & 0 & 0 & ... & 0 \\
b_1 & a_2 & b_2 & 0 & ... & 0 \\
0 & b_2 & a_3 & b_3 & ... & 0 \\
\vdots & & & & & \vdots \\
\vdots & & & & & \vdots \\
0 & ... & & & & b_{N-1} \\
0 & ... & & & b_{N-1} & a_N

\end{bmatrix}
\]

Where $a_i = \frac{-\beta_i}{\alpha_i}$ and $b_i = (\frac{\gamma_{i+1}}{\alpha_i \alpha_{i+1}})^{\frac{1}{2}}$.
This matrix is called the Jacobi matrix.
The eigenvalues of this matrix give us the points $x_i$ and the length of $\left[a, b\right]$ times the squares of the first entries of the corresponding eigenvectors gives the weights.

% Problem 4: Construct the Jacobi matrix.
\begin{problem}
Write a function that will accept three arrays representing the coefficients $\{\gamma_i, \alpha_i, \beta_i\}$ from the recurrence relation above and return the Jacobi matrix.
\end{problem}

\begin{problem}
The coefficients of the Legendre polynomials (which correspond to the weight function $W(x) = 1$ on $[-1,1]$ are given by
\begin{align}\nonumber
\alpha_i = \frac{2i - 1}{i} && \beta_i = 0 && \gamma_i = \frac{i-1}{i}
\end{align}

Write a function that accepts an integer $n$ representing the number of points to use in the quadrature. Calculate $\alpha$, $\beta$, and $\gamma$ as above, calculate the Jacobi matrix, then use it to find the points $x_i$ and weights $w_i$ that correspond to this weight function.
When $n=5$, do they match the ones given in the first part of this lab?

\end{problem}

% Problem: integrate!
\begin{problem}
Write a new function that accepts a function $f$, bounds $a$ and $b$, and $n$ for the number of points to use.
Use the previously defined functions to estimate $\int_a^b f(x)dx$ using the coefficients of the Legendre polynomials.

This completes our implementation of the Gaussian Quadrature for a particular set orthogonal polynomials.
\end{problem}

\section*{scipy.integrate}

There are other techniques for finding the weights and points for a given weighting function.
This is, in fact, not even the fastest method.
In general practice, we use \li{scipy.integrate} to calculate integrals.
\li{scipy.integrate.quadrature} offers a reasonably fast Gaussian quadrature implementation.

Another common hallmark of quadrature is that it can be used adaptively.
It is common in practice to refine the points of a quadrature estimate on an interval where a function is observed to be changing rapidly.
This allows for more accurate computation at a relatively low computational cost.
This is the approach used by the function \li{scipy.integrate.quad}.

\begin{problem}
The standard normal distribution is an important object of study in probability and statistic.
It is defined by the probability density function $p(x) = \frac{1}{\sqrt{2 \pi}} e^{-x^2/2}$ (here we are assuming a mean of $0$ and a variance of $1$).
This is a function that cannot be integrated symbollically.

The probability that a normally distributed random variable $X$ will take on a value less than (or equal to) a given value $x$ is
\[P(X \le x) = \int_{-\infty}^x \frac{1}{\sqrt{2 \pi}} e^{-t^2/2} dt\]
This function is essentially zero for values of $x$ that lie reasonably far from the mean, so we can estimate this probability by integrating from $-5$ to $x$ instead of from $-\infty$ to $x$.

Write a function that uses \li{scipy.integrate.quad} to estimate the probability that this normally distributed random variable will take a value less than a given number $x$ that lies relatively close to the mean.
You can test your result at $x = 1$ by comparing it with the following code:
\begin{lstlisting}
from scipy.stats import norm
N = norm()
N.cdf(1)
\end{lstlisting}
\end{problem}

\begin{comment}
%Another exercise could be Chebyshev polynomials.  See wikipedia for the weight function.
\section*{Additional Material}

\begin{problem}
Gauss Quadrature with Chebyshev polynomials.
\end{problem}
\end{comment}


\documentclass{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{geometry}
\usepackage[utf8]{inputenc}
\usepackage{enumitem}
\usepackage{physics}
\setlength{\parindent}{0pt}
\setlength{\parskip}{1em}
\relpenalty=10000
\binoppenalty=10000

\begin{document}
\section*{Inequalities}
	\begin{enumerate}
	\item \textbf{General mean inequality.}
	The mean of order $p$ of positive real numbers $x_1,\dots,x_n$ is defined as:
	$$M_p=
	\begin{cases}
	\left(\frac{x_1^p+ \dots + x_n^p}{n}\right)^{1/p} &\text{for } p \ne 0 \\
	\sqrt[n]{x_1 \dots x_n}                           &\text{for } p=0
	\end{cases}
	$$
	In particular
	\begin{center}
		\begin{tabular}{lcl}
			Smallest element & $\min\{x_i\}$ & $M_{-\infty}$ \\
			Harmonic mean & HM & $M_{-1}$ \\
			Geometric mean & GM & $M_0$ \\
			Arithmetic mean & AM & $M_1$ \\
			Quadratic mean & QM & $M_2$ \\
			Largest element & $\max\{x_i\}$ & $M_{\infty}$
		\end{tabular}
	\end{center}
	Then for any real $p$ and $q$
	$$M_p \leq M_q \iff p \leq q $$

	\item \textbf{Cauchy inequality.}
	For real numbers $x_1, \dots , x_n, y_1, \dots , y_n$

	$$\left(\sum_{i=1}^{n} x_i y_i\right)^2 \leq \sum_{i=1}^{n} x_i^2 \sum_{i=1}^{n} y_i^2 $$

	\item \textbf{Chebyshev inequality.}
	For real numbers $x_1 \geq \dots \geq x_n$ and $y_1 \geq \dots \geq y_n$
	$$\frac{1}{n} \sum_{i=1}^{n} x_iy_i
	\geq
	\left(\frac{1}{n}\sum_{i=1}^{n}x_i\right)
	\left(\frac{1}{n}\sum_{i=1}^{n}y_i\right)
	\geq
	\frac{1}{n} \sum_{i=1}^{n} x_iy_{n+1-i} $$

	\item \textbf{Jensen inequality.}
	Given positive real numbers $\lambda_1,\hdots,\lambda_n$ for which $\lambda_1+\hdots+\lambda_n=1$  and a convex function $f(x)$ the following holds:
	$$f(\lambda_1 x_1 + \hdots + \lambda_n x_n) \leq \lambda_1 f(x_1) + \hdots + \lambda_n f(x_n)$$
	Similarly, when $f(x)$ is a concave function, then
	$$f(\lambda_1 x_1 + \hdots + \lambda_n x_n) \geq \lambda_1 f(x_1) + \hdots + \lambda_n f(x_n)$$

	\end{enumerate}

	\newpage
\section*{Problems}
	\begin{enumerate}
	\item % vorrat 97
	Let $a_1,\dots,a_n$ be positive real numbers such that $a_1\dots a_n =1$ Prove that
	$$(1+a_1)\dots (1+a_n) \geq 2^n$$
	\item % vorrat 100
	For real numbers $x_1, \dots , x_n, y_1, \dots , y_n$ the following holds
	$$x_1+\dots+x_n \geq x_1y_1 + \dots + x_ny_n$$
	Prove that
	$$x_1+\dots+x_n \leq \frac{x_1}{y_1} + \dots + \frac{x_n}{y_n}$$

	\item % vorrat 99
	Let $n$ be an integer ($n\geq 2$) and $a_1,\dots,a_n$ be positive real numbers such that $a_1+\dots+a_n=1$. Prove the following inequality for any positive real numbers $x_1,\dots,x_n$ for which $x_1+\dots+x_n=1$
	$$2\sum_{i<j} x_ix_j \leq \frac{n-2}{n-1} + \sum_{i=1}^n \frac{a_ix_i^2}{1-a_i}$$
	When does the equality hold?

	\item % vorrat 102
	Prove that for any positive real numbers $a_1,\dots,a_n$
	$$\frac{1}{\frac{1}{1+a_1}+\dots+\frac{1}{1+a_n}} - \frac{1}{\frac{1}{a_1}+\dots+\frac{1}{a_n}} \geq n$$

	\item % vorrat 103
	Let $x_1,\dots,x_n$ be positive real numbers such that
	$$\frac{1}{1+x_1} + \dots + \frac{1}{1+x_n}=1$$
	Prove that
	$$x_1 \dots x_n \geq (n-1)^n$$

	\item % vorrat 123
	Prove for real numbers $x_1,\dots,x_5$
	$$x_1^2 + x_2^2 + x_3^2 + x_4^2 + x_5^2 \geq \frac{2}{\sqrt{3}}(x_1x_2 + x_2x_3 + x_3x_4 + x_4x_5) $$

	\item % vorrat 106
	Let $a,b,c$ be positive real numbers. Prove that
	$$\left(1+\frac{a}{b}\right) \left(1+\frac{b}{c}\right)  \left(1+\frac{c}{a}\right) \geq
	2\left(1+\frac{a+b+c}{\sqrt[3]{abc}}\right)$$

	\item % vorrat 104
	Given positive real numbers $x_1,\dots,x_n$ for which $x_1^2+\dots+x_n^2=1$, find the minimal value of the expression
	$$\frac{x_1^5}{x_2+x_3+\dots+x_n} + \frac{x_2^5}{x_1+x_3+\dots+x_n} + \dots +\frac{x_n^5}{x_1+x_2+\dots+x_{n-1}}$$

	\item % vorrat 101
	Let $x_1,\dots,x_n$ be positive real numbers for which $x_1+\dots+x_n=1$. Prove that
	$$\frac{x_1}{\sqrt{1-x_1}} + \dots + \frac{x_n}{\sqrt{1-x_n}}
	\geq
	\frac{\sqrt{x_1}+\dots+\sqrt{x_n}}{\sqrt{n-1}} $$

	\item % http://artofproblemsolving.com/wiki/index.php?title=2004_USAMO_Problems/Problem_5
	Let $a,b,c$ be positive real numbers. Prove that
	$$(a^5-a^2+3)(b^5-b^2+3)(c^5-c^2+3)\geq (a+b+c)^3$$

\end{enumerate}
\end{document}


\textbf{Modify Program 7 so that you can verify that the data in the first curve of Output 7 match the prediction of Theorem 4(a). Verify also that the third and fourth curves match the predictions of parts (c) and (d).}
\newline

In this problem we verify the convergence rates of Theorem 4. For all cases let $w$ be the $v$th spectral derivative on the grid $h\mathbb{Z}$. Since we are going to be working with the first derivative, $v=1$.

First, we focus on the function $u=|sin(x)|^3$, which has $2=p-1$ continuous derivatives in $L^2(\mathbb{R})$ and a $p=3$rd derivative of bounded variation. Hence,
\begin{align*}
|w_j - u^{(v)}(x_j)| = \mathcal{O}(h^{p-v}) = \mathcal{O}(h^{3-1}) = \mathcal{O}(h^{2})~\text{as}~h \rightarrow 0,
\end{align*}
which we can verify in the next figure.

\begin{figure}[H]
\centering
\includegraphics[scale=.7]{P4_1.eps}\caption{Error convergence for $u=|sin(x)|^3$.}
\end{figure}

Second, the function $ u = \frac{1}{1+\sin^2(x/2)}$ is analytic in a strip in the complex plane. We find this strip as follows
\begin{align*}
1 + \sin^2(z/2) = 0 ~\Rightarrow ~a = \text{Imag}(2 \arcsin(i))~\Rightarrow~ a \approx 1.76~.
\end{align*}
Then, according to the theorem,
\begin{align*}
|w_j - u^{(v)}(x_j)| = \mathcal{O}\left(e^{-(a-\varepsilon)/h)}\right)~\text{as}~h \rightarrow 0~,
\end{align*}
for every $\varepsilon>0$ (we have chosen $10^{-4}$). We can verify this result in the following figure.
\begin{figure}[H]
\centering
\includegraphics[scale=.7]{P4_2.eps}\caption{Error convergence for $ u = \frac{1}{1+\sin^2(x/2)}$.}
\end{figure}

Lastly, $u = \sin(10x)$ is band-limited. It is possible to find $a >0$ such that $u$ can be extended to an entire function and for $z \in \mathbb{C}$, $|u(z)| = o(e^{a|z|})$ as $z \rightarrow \infty$. Note that

\begin{align*}
|\sin(10z)| = \biggl| \frac{e^{i10z}-e^{-i10z}}{2i} \biggr| = \biggl| \frac{e^{i10x}e^{-10y} - e^{-i10x}e^{10y}}{2i} \biggr| \leq \biggl| \frac{e^{-10y}-e^{10y}}{2} \biggr| \leq e^{10|z|}~,
\end{align*}
where we have made $z=x+iy$. Further,
\begin{align*}
\lim_{z \rightarrow \infty} \frac{|\sin(10z)|}{|e^{a|z|}|} \leq \lim_{z \rightarrow \infty} \frac{e^{10|z|}}{e^{a|z|}} = 0 ~\Rightarrow ~a > 10.
\end{align*}
According to the theorem,
\begin{align*}
w_j = u^{(v)}(x_j),
\end{align*}
as long as $h =2\pi/N \leq \pi/a$. Hence,
\begin{align*}
w_j = u^{(v)}(x_j),
\end{align*}
for $N>2a=20$, which is clearly seen in the next figure, where for $N>20$ the error is machine precision.

\begin{figure}[H]
\centering
\includegraphics[scale=.7]{P4_3.eps}\caption{Error convergence for $u = \sin(10x)$.}
\end{figure}

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 4
close all
figformat='epsc';
linewidth=2;
% Compute derivatives for various values of N:
Nmax = 50; E = zeros(3,Nmax/2-2);
for N = 6:2:Nmax
    h = 2*pi/N; x = h*(1:N)';
    column = [0 .5*(-1).^(1:N-1).*cot((1:N-1)*h/2)]';
    D = toeplitz(column,column([1 N:-1:2]));
    v = abs(sin(x)).^3;                     % 3rd deriv in BV
    vprime = 3*sin(x).*cos(x).*abs(sin(x));
    E(1,N/2-2) = norm(D*v-vprime,inf);
    v = exp(-sin(x/2).^(-2));               % C-infinity
    vprime = .5*v.*sin(x)./sin(x/2).^4;
    E(2,N/2-2) = norm(D*v-vprime,inf);
    v = 1./(1+sin(x/2).^2);                 % analytic in a strip
    vprime = -sin(x/2).*cos(x/2).*v.^2;
    E(3,N/2-2) = norm(D*v-vprime,inf);
    v = sin(10*x); vprime = 10*cos(10*x);   % band-limited
    E(4,N/2-2) = norm(D*v-vprime,inf);
end
Nvector = 6:2:Nmax;
hvector = 2*pi./Nvector;

p = 3;
v = 1;

figure
semilogy(Nvector,E(1,:),'*')
hold on
semilogy(Nvector,hvector.^(p-v),'linewidth',linewidth)
grid on
xlabel('$N$','interpreter','latex')
ylabel('$|w_j-u^{(v)(x_j)}|$','interpreter','latex')
set(gca,'fontsize',14)
txt='Latex/FIGURES/P4_1';
saveas(gcf,txt,figformat)

a = 1.76;
e = 1e-4;

figure
semilogy(Nvector,E(3,:),'*')
hold on
semilogy(Nvector,exp(-pi*(a-e)./hvector),'linewidth',linewidth)
grid on
xlabel('$N$','interpreter','latex')
ylabel('$|w_j-u^{(v)(x_j)}|$','interpreter','latex')
set(gca,'fontsize',14)
txt='Latex/FIGURES/P4_2';
saveas(gcf,txt,figformat)

figure
semilogy(Nvector,E(4,:),'*')
hold on
plot([20 20],[1e-16 10],'r--','linewidth',linewidth)
axis([5 51 1e-16 2e1])
grid on
xlabel('$N$','interpreter','latex')
ylabel('$|w_j-u^{(v)(x_j)}|$','interpreter','latex')
set(gca,'fontsize',14)
txt='Latex/FIGURES/P4_3';
saveas(gcf,txt,figformat)
\end{verbatim}


\section{Data Approximation and Neville's Method}
\begin{defn}
The Lagrange Polynomial that agrees with $f(x)$ at the $k$ distinct points $x_{m_1},x_{m_2},\cdots,x_{m_k}$ is denoted $P_{m_1,m_2,\cdots m_k}(x)$.
\end{defn}
\begin{theo}
Let $f$ be defined at $x_0,x_1,\cdots,x_k$, then
\[
P(x)=\frac{(x-x_j)P_{0,\cdots,j-1,j+1,\cdots,k}(x)-(x-x_i)P_{0,\cdots,i-1,i+1,\cdots,k}(x)}{x_i-x_j}
\]
is the $k$th Lagrange polynomial that interpolates $f$ at the $k+1$ points.
\end{theo}

\subsection{Neville's Method}
To avoid the multiple subscripts, we let $Q_{i,j}\,(0\leq j\leq i)$ denote the interpolating polynomial of degree $j$ on the $(j+1)$ numbers $x_{i-j},\cdots,x_{i}$.
\[
Q_{i,j} = P_{i-j,i-j+1,\cdots,i-1,i}
\]
then for $i=1:n$, $j=1:i$,
\[
Q_{i,j}=\frac{(x-x_{i-j})Q_{i,j-1}-(x-x_i)Q_{i-1,j-1}}{x_i-x_{i-j}}
\]


\subsection{Back propagation}

\subsubsection{Adapting the delta rule}

To arrive at the delta rule we considered the cost function:

\(E=\sum_j\dfrac{1}{2}(y_j-a_j)^2\)

And used the chain rule:

\(\dfrac{\delta E}{\delta \theta_i }=\dfrac{\delta E}{\delta a_j}\dfrac{\delta a_j}{\delta z_j}\dfrac{\delta z_j}{\delta \theta_i}\)

This gave us:

\(\Delta \theta_i=\alpha \sum_j(y_j-a_j)a'(z_j)x_{ij}\)

Or, setting \(\delta_i=-\dfrac{\delta E}{\delta z_j}=\sum_j(y_j-a_j)a'(z_j)\)

\(\Delta \theta_i=\alpha \delta_j x_{ij}\)

Let's update the rule for multiple layers:

\(\dfrac{\delta E}{\delta \theta_{li}}=\dfrac{\delta E}{\delta a_{lj}}\dfrac{\delta a_{lj}}{\delta z_{lj}}\dfrac{\delta z_{lj}}{\delta \theta_{li}}\)

Previously \(\dfrac{\delta z_{lj}}{\delta \theta_{li}}=x_i\). We now use the more general \(a_{li}\). For the first layer, these will be the same.

We can then instead write:

\(\Delta \theta_i=\alpha \delta_{lj} a_{li}\)

\subsubsection{Calculating delta values}

Now we need a way of calculating the value of \(\delta_{lj}\) for all neurons.

\(\delta_i=-\dfrac{\delta E}{\delta z_{lj}}\)

If this is an output node, then this is simply \(\sum_j(y_j-a_j)a'(z_j)\)

If this is not an output node, then the impact of change in the parameter will affect the results through all intermediate neurons.

In this case:

\(\dfrac{\delta E}{\delta z_{lj}}=\sum_{k\in succ{l}}\dfrac{\delta E}{\delta z_{k}}\dfrac{\delta z_{k}}{\delta z_{lj}}\)

\(\dfrac{\delta E}{\delta z_{lj}}=\sum_{k\in succ{l}}-\delta_{k}\dfrac{\delta z_{k}}{\delta a_{kj}}\dfrac{\delta a_{kj}}{\delta z_{lj}}\)

\(\dfrac{\delta E}{\delta z_{lj}}=\sum_{k\in succ{l}}-\delta_{k}\theta_{kj}a'_{kj}\)

\(\delta_i=a'_{kj}\sum_{k\in succ{l}}\delta_{k}\theta_{kj}\)

For each layer there is a matrix, where the columns and rows represent the \(theta \) between the current layer and the next layer. We have a matrix for each layer in the network.


\chapter{Groups}
A group is one of the most basic structures in higher mathematics.
In this chapter I will tell you only the bare minimum:
what a group is, and when two groups are the same.

\section{Definition and examples of groups}
\prototype{The additive group of integers $(\ZZ,+)$ and the cyclic group $\Zc m$.
Just don't let yourself forget that most groups are non-commutative.}

A group consists of two pieces of data: a set $G$,
and an associative binary operation $\star$ with some properties.
Before I write down the definition of a group, let me give two examples.

\begin{example}[Additive integers]
	The pair $(\ZZ, +)$ is a group:
	$\ZZ = \left\{ \dots,-2,-1,0,1,2,\dots \right\}$ is the set
	and the associative operation is \emph{addition}.
	Note that
	\begin{itemize}
		\ii The element $0 \in \ZZ$ is an \emph{identity}:
		$a+0=0+a = a$ for any $a$.
		\ii Every element $a \in \ZZ$ has an additive \emph{inverse}: $a + (-a) = (-a) + a = 0$.
	\end{itemize}
	We call this group $\ZZ$.
\end{example}
\begin{example}[Nonzero rationals]
	Let $\QQ^\times$ be the set of \emph{nonzero rational numbers}.
	The pair $(\QQ^\times, \cdot)$ is a group:
	the set is $\QQ^\times$
	and the associative operation is \emph{multiplication}.

	Again we see the same two nice properties.
	\begin{itemize}
		\ii The element $1 \in \QQ^\times$ is an \emph{identity}:
		for any rational number, $a \cdot 1 = 1 \cdot a = a$.
		\ii For any rational number $x \in \QQ^\times$,
		we have an inverse $x\inv$, such that
		\[ x \cdot x\inv = x\inv \cdot x = 1. \]
	\end{itemize}
\end{example}

From this you might already have a guess what the definition of a group is.
\begin{definition}
	A \vocab{group} is a pair $G = (G, \star)$
	consisting of a set of elements $G$, and a binary operation $\star$ on $G$, such that:
	\begin{itemize}
		\ii $G$ has an \vocab{identity element}, usually denoted $1_G$
		or just $1$, with the property that
		\[ 1_G \star g = g \star 1_G = g \text{ for all $g \in G$}. \]
		\ii The operation is \vocab{associative}, meaning
		$(a \star b) \star c = a \star (b \star c)$
		for any $a,b,c \in G$.
		Consequently we generally don't write the parentheses.
		\ii Each element $g \in G$ has an \vocab{inverse}, that is, an element $h \in G$ such that \[ g \star h = h \star g = 1_G. \]
	\end{itemize}
	\label{def:group}
\end{definition}
\begin{remark}
	[Unimportant pedantic point]
	Some authors like to add a ``closure'' axiom,
	i.e.\ to say explicitly that $g \star h \in G$.
	This is implied already by the fact that $\star$
	is a binary operation on $G$,
	but is worth keeping in mind for the examples below.
\end{remark}

\begin{remark}
	It is not required that $\star$ is commutative ($a \star b = b \star a$).
	So we say that a group is \vocab{abelian} if the operation is
	commutative and \vocab{non-abelian} otherwise.
\end{remark}

% Now that I've made clear what the criteria of a group are,
% let us write down some non-examples of groups.

\begin{example}[Non-Examples of groups]
	\listhack
	\begin{itemize}
		\ii The pair $(\QQ, \cdot)$ is NOT a group.
		(Here $\QQ$ is rational numbers.)
		While there is an identity element, the element $0 \in \QQ$
		does not have an inverse.
		\ii The pair $(\ZZ, \cdot)$ is also NOT a group. (Why?)
		\ii Let $\Mat_{2 \times 2}(\RR)$ be the set of $2 \times 2$ real matrices.
		Then $(\Mat_{2 \times 2}(\RR), \cdot)$
		(where $\cdot$ is matrix multiplication) is NOT a group.
		Indeed, even though we have an identity matrix
		\[
			\begin{bmatrix}
				1 & 0 \\ 0 & 1
			\end{bmatrix}
		\]
		we still run into the same issue as before:
		the zero matrix does not have a multiplicative inverse.

		(Even if we delete the zero matrix from the set,
		the resulting structure is still not a group:
		those of you that know some linear algebra
		might recall that any matrix with determinant zero
		cannot have an inverse.)
	\end{itemize}
\end{example}

Let's resume writing down examples.
Here are some more \textbf{abelian examples} of groups:
\begin{example}
	[Complex unit circle]
	Let $S^1$ denote the set of complex numbers $z$ with absolute value one; that is
	\[ S^1 \defeq \left\{ z \in \CC \mid \left\lvert z \right\rvert = 1 \right\}. \]
	Then $(S^1, \times)$ is a group because
	\begin{itemize}
		\ii The complex number $1 \in S^1$ serves as the identity, and
		\ii Each complex number $z \in S^1$ has an inverse $\frac 1z$ which is also in $S^1$, since $\left\lvert z\inv \right\rvert = \left\lvert z \right\rvert\inv = 1$.
	\end{itemize}
	There is one thing I ought to also check: that $z_1 \times z_2$ is actually still in $S^1$.
	But this follows from the fact that $\left\lvert z_1z_2 \right\rvert = \left\lvert z_1 \right\rvert \left\lvert z_2 \right\rvert = 1$.
\end{example}

\begin{example}
	[Addition mod $n$]
	Here is an example from number theory:
	Let $n > 1$ be an integer,
	and consider the residues (remainders) modulo $n$.
	These form a group under addition.
	We call this the \vocab{cyclic group of order $n$},
	and denote it as $\Zc n$, with elements $\ol 0, \ol 1, \dots$.
	The identity is $\ol 0$.
	\label{def:cyclic_group}
\end{example}
\begin{example}
	[Multiplication mod $p$]
	Let $p$ be a prime.
	Consider the \emph{nonzero residues modulo $p$},
	which we denote by $\Zm p$.
	Then $\left( \Zm p, \times \right)$ is a group.
	\label{def:mult_mod_p}
\end{example}
\begin{ques}
	Why do we need the fact that $p$ is prime?
\end{ques}
(Digression: the notation $\Zc n$ and $\Zm p$ may seem strange
but will make sense when we talk about rings and ideals.
Set aside your worry for now.)

Here are some \textbf{non-abelian examples}:
\begin{example}
	[General linear group]
	Let $n$ be a positive integer.
	Then $\GL_n(\RR)$ is defined as the set of $n \times n$ real matrices
	which have nonzero determinant.
	It turns out that with this condition,
	every matrix does indeed have an inverse,
	so $(\GL_n(\RR), \times)$ is a group, called the
	\vocab{general linear group}.

	(The fact that $\GL_n(\RR)$ is closed under $\times$ follows
	from the linear algebra fact that $\det (AB) = \det A \det B$,
	proved in later chapters.)
\end{example}
\begin{example}
	[Special linear group]
	Following the example above, let $\SL_n(\RR)$ denote
	the set of $n \times n$ matrices whose determinant is actually $1$.
	Again, for linear algebra reasons
	it turns out that $(\SL_n(\RR), \times)$ is also a group,
	called the \vocab{special linear group}.
\end{example}

\begin{example}
	[Symmetric groups]
	Let $S_n$ be the set of permutations of $\left\{ 1,\dots,n \right\}$.
	By viewing these permutations as functions from $\left\{ 1,\dots,n \right\}$ to itself, we can consider \emph{compositions} of permutations.
	Then the pair $(S_n, \circ)$ (here $\circ$ is function composition)
	is also a group, because
	\begin{itemize}
		\ii There is an identity permutation, and
		\ii Each permutation has an inverse.
	\end{itemize}
	The group $S_n$ is called the \vocab{symmetric group} on $n$ elements.
\end{example}
\begin{example}
	[Dihedral group]
	The \vocab{dihedral group of order $2n$}, denoted $D_{2n}$,
	is the group of symmetries of a regular $n$-gon $A_1A_2 \dots A_n$,
	which includes rotations and reflections.
	It consists of the $2n$ elements
	\[ \left\{ 1, r, r^2, \dots, r^{n-1}, s, sr, sr^2, \dots, sr^{n-1} \right\}. \]
	The element $r$ corresponds to rotating the $n$-gon by $\frac{2\pi}{n}$,
	while $s$ corresponds to reflecting it across the line $OA_1$
	(here $O$ is the center of the polygon).
	So $rs$ mean ``reflect then rotate''
	(like with function composition, we read from right to left).

	In particular, $r^n = s^2 = 1$. You can also see that $r^k s = sr^{-k}$.
\end{example}

Here is a picture of some elements of $D_{10}$.
\begin{center}
	\begin{asy}
		size(12cm);
		picture aoeu(string a, string b, string c, string d, string e,
					string x) {
			draw(dir(0)--dir(72)--dir(144)--dir(216)--dir(288)--cycle);
			MP(a, dir(0), dir(0));
			MP(b, dir(72), dir(72));
			MP(c, dir(144), dir(144));
			MP(d, dir(216), dir(216));
			MP(e, dir(288), dir(288));
			MP(x, origin, origin);
			return CC();
		}
		picture one = aoeu("1", "2", "3", "4", "5", "1");
		picture r = aoeu("5", "1", "2", "3", "4", "r");
		picture s = aoeu("1", "5", "4", "3", "2", "s");
		picture sr = aoeu("5", "4", "3", "2", "1", "sr");
		picture rs = aoeu("2", "1", "5", "4", "3", "rs");
		add(shift( (0,0) ) * one);
		add(shift( (3,0) ) * r);
		add(shift( (6,0) ) * s);
		add(shift( (9,0) ) * sr);
		add(shift( (12,0) ) * rs);
	\end{asy}
\end{center}
Trivia: the dihedral group $D_{12}$ is my favorite example of a non-abelian group,
and is the first group I try for any exam question of the form ``find an example\dots''.

More examples:
\begin{example}
	[Products of groups]
	Let $(G, \star)$ and $(H, \ast)$ be groups.
	We can define a \vocab{product group} $(G \times H, {\cdot})$, as follows.
	The elements of the group will be ordered pairs $(g,h) \in G \times H$.
	Then
	\[ (g_1, h_1) \cdot (g_2, h_2) = (g_1 \star g_2, h_1 \ast h_2) \in G \times H
		\]
	is the group operation.
	\label{def:product_group}
\end{example}
\begin{ques}
	What are the identity and inverses of the product group?
\end{ques}

\begin{example}
	[Trivial group]
	The \vocab{trivial group}, often denoted $0$ or $1$,
	is the group with only an identity element.
	I will use the notation $\{1\}$.
\end{example}

\begin{exercise}
	Which of these are groups?
	\begin{enumerate}[(a)]
		\ii Rational numbers with odd denominators (in simplest form), where the operation is addition.
		(This includes integers, written as $n/1$, and $0 = 0/1$).
		\ii The set of rational numbers with denominator at most $2$, where the operation is addition.
		\ii The set of rational numbers with denominator at most $2$, where the operation is multiplication.
		\ii The set of nonnegative integers, where the operation is addition.
	\end{enumerate}
\end{exercise}

\section{Properties of groups}
\prototype{$\Zm p$ is possibly best.}
\begin{abuse}
	From now on, we'll often refer to a group $(G, \star)$ by just $G$.
	Moreover, we'll abbreviate $a \star b$ to just $ab$.
	Also, because the operation $\star$ is associative,
	we will omit unnecessary parentheses: $(ab)c = a(bc) = abc$.
\end{abuse}
\begin{abuse}
	From now on, for any $g \in G$ and $n \in \NN$ we abbreviate
	\[ g^n
		=
		\underbrace{g \star \dots \star g}_{\text{$n$ times}}.\]
	Moreover, we let $g\inv$ denote the inverse of $g$,
	and $g^{-n} = (g\inv)^n$.
\end{abuse}

In mathematics, a common theme is to require
that objects satisfy certain minimalistic properties,
with certain examples in mind,
but then ignore the examples on paper
and try to deduce as much as you can just from the properties alone.
(Math olympiad veterans are likely familiar with
``functional equations''
in which knowing a single property about a function
is enough to determine the entire function.)
Let's try to do this here,
and see what we can conclude just from knowing \Cref{def:group}.

It is a law in Guam and 37 other states that
I now state the following proposition.
\begin{fact}
	Let $G$ be a group.
	\begin{enumerate}[(a)]
		\ii The identity of a group is unique.
		\ii The inverse of any element is unique.
		\ii For any $g \in G$, $(g\inv)\inv = g$.
	\end{enumerate}
\end{fact}
\begin{proof}
	This is mostly just some formal manipulations,
	and you needn't feel bad skipping it on a first read.
	\begin{enumerate}[(a)]
		\ii If $1$ and $1'$ are identities, then $1 = 1 \star 1' = 1'$.
		\ii If $h$ and $h'$ are inverses to $g$, then $1_G = g \star h
		\implies h' = (h' \star g) \star h = 1_G \star h = h$.
		\ii Trivial; omitted. \qedhere
	\end{enumerate}
\end{proof}

Now we state a slightly more useful proposition.
\begin{proposition}[Inverse of products]
	Let $G$ be a group, and $a,b \in G$.
	Then $(ab)\inv = b\inv a\inv$.
\end{proposition}
\begin{proof}
	Direct computation. We have
	\[ (ab)(b\inv a\inv)
		= a (bb\inv) a\inv = aa\inv = 1_G. \]
	Hence $(ab)\inv = b\inv a\inv$.
	Similarly, $(b\inv a\inv)(ab) = 1_G$ as well.
\end{proof}

Finally, we state a very important lemma about groups,
which highlights why having an inverse is so valuable.
\begin{lemma}[Left multiplication is a bijection]
	Let $G$ be a group, and pick a $g \in G$.
	Then the map $G \to G$ given by $x \mapsto gx$ is a bijection.
	\label{lem:group_mult_biject}
\end{lemma}
\begin{exercise}
	Check this by showing injectivity and surjectivity directly.
	(If you don't know what these words mean,
	consult \Cref{ch:sets_functions}.)
\end{exercise}
\begin{example}
	Let $G = \Zm 7$ (as in \Cref{def:mult_mod_p}) and pick $g=3$.
	The above lemma states that the map $x \mapsto 3 \cdot x$ is a bijection, and we can see this explicitly:
	\begin{align*}
		1 &\overset{\times 3}{\longmapsto} 3 \pmod 7 \\
		2 &\overset{\times 3}{\longmapsto} 6 \pmod 7 \\
		3 &\overset{\times 3}{\longmapsto} 2 \pmod 7 \\
		4 &\overset{\times 3}{\longmapsto} 5 \pmod 7 \\
		5 &\overset{\times 3}{\longmapsto} 1 \pmod 7 \\
		6 &\overset{\times 3}{\longmapsto} 4 \pmod 7.
	\end{align*}
\end{example}
The fact that the map is injective is often called the \vocab{cancellation law}.
(Why do you think so?)

\begin{abuse}
	[Later on, sometimes the identity is denoted $0$ instead of $1$]
	You don't need to worry about this for a few chapters,
	but I'll bring it up now anyways.
	In most of our examples up until now the operation $\star$
	was thought of like multiplication of some sort,
	which is why $1 = 1_G$ was a natural notation for the identity element.

	But there are groups like $\ZZ = (\ZZ,+)$
	where the operation $\star$ is thought of as addition,
	in which case the notation $0 = 0_G$ might make more sense instead.
	(In general, whenever an operation is denoted $+$,
	the operation is almost certainly commutative.)
	We will eventually start doing so too
	when we discuss rings and linear algebra.
\end{abuse}

\section{Isomorphisms}
\prototype{$\ZZ \cong 10\ZZ$.}
First, let me talk about what it means for groups to be isomorphic.
Consider the two groups
\begin{itemize}
	\ii $\ZZ = (\left\{ \dots,-2,-1,0,1,2,\dots \right\}, +)$.
	\ii $10\ZZ = (\left\{ \dots, -20, -10, 0, 10, 20, \dots \right\}, +)$.
\end{itemize}
These groups are ``different'', but only superficially so -- you might even say they only differ in the names of the elements.
Think about what this might mean formally for a moment.

Specifically the map
\[ \phi : \ZZ \to 10 \ZZ  \text{ by } x \mapsto 10 x \]
is a bijection of the underlying sets which respects the group action.
In symbols,
\[ \phi(x + y) = \phi(x) + \phi(y). \]
In other words, $\phi$ is a way of re-assigning names of the elements
without changing the structure of the group.
That's all just formalism for
capturing the obvious fact that $(\ZZ,+)$
and $(10 \ZZ, +)$ are the same thing.

Now, let's do the general definition.
\begin{definition}
	Let $G = (G, \star)$ and $H = (H, \ast)$ be groups.
	A bijection $\phi : G \to H$ is called an \vocab{isomorphism} if
	\[ \phi(g_1 \star g_2) = \phi(g_1) \ast \phi(g_2) \quad
		\text{for all $g_1, g_2 \in G$}. \]
	If there exists an isomorphism from $G$ to $H$,
	then we say $G$ and $H$ are \vocab{isomorphic} and write $G \cong H$.
\end{definition}
Note that in this definition, the left-hand side
$\phi(g_1 \star g_2)$ uses the operation of $G$
while the right-hand side $\phi(g_1) \ast \phi(g_2)$
uses the operation of $H$.

\begin{example}
	[Examples of isomorphisms]
	Let $G$ and $H$ be groups. We have the following isomorphisms.
	\begin{enumerate}[(a)]
		\ii $\ZZ \cong 10 \ZZ$, as above.
		\ii There is an isomorphism
		\[ G \times H \cong H \times G\]
		by the map $(g,h) \mapsto (h,g)$.
		\ii The identity map $\id : G \to G$
		is an isomorphism, hence $G \cong G$.
		\ii There is another isomorphism of $\ZZ$ to itself: send every $x$ to $-x$.
	\end{enumerate}
\end{example}
\begin{example}
	[Primitive roots modulo $7$]
	As a nontrivial example, we claim that $\Zc 6 \cong \Zm 7$.
	The bijection is
	\[ \phi(\text{$a$ mod $6$}) = \text{$3^a$ mod $7$}. \]
	To check that this is an isomorphism, we need to verify several things.
	\begin{itemize}
		\ii First, we need to check this map actually makes sense:
		why is it the case that if $a \equiv b \pmod 6$, then $3^a \equiv 3^b \pmod 7$?
		The reason is that Fermat's little theorem guarantees that $3^6 \equiv 1 \pmod 7$.
		\ii Next, we need to check that this map is a bijection.
		You can do this explicitly:
		\[ (3^1, 3^2, 3^3, 3^4, 3^5, 3^6)
			\equiv (3,2,6,4,5,1) \pmod 7. \]
		\ii Finally, we need to verify that this map respects the group action.
		In other words, we want to see that
		$\phi(a+b) = \phi(a) \phi(b)$
		since the operation of $\Zc 6$ is addition
		while the operation of $\Zm 7$ is multiplication.
		That's just saying that $3^{a+b} \equiv 3^a 3^b \pmod 7$,
		which is true.
	\end{itemize}
\end{example}
\begin{example}
	[Primitive roots]
	More generally, for any prime $p$, there exists
	an element $g \in \Zm p$ called a \vocab{primitive root} modulo $p$
	such that $1, g, g^2, \dots, g^{p-2}$ are all different modulo $p$.
	One can show by copying the above proof that
	\[ \Zcc{p-1} \cong \Zm p \text{ for all primes $p$}. \]
	The example above was the special case $p=7$ and $g=3$.
\end{example}
\begin{exercise}
	Assuming the existence of primitive roots,
	establish the isomorphism $\Zcc{p-1} \cong \Zm p$ as above.
\end{exercise}

It's not hard to see that $\cong$ is an equivalence relation (why?).
Moreover, because we really only care about the structure of groups,
we'll usually consider two groups to be the same when they are isomorphic.
So phrases such as ``find all groups'' really mean
``find all groups up to isomorphism''.

\section{Orders of groups, and Lagrange's theorem}
\prototype{$\Zm p$.}

As is typical in math, we use the word ``order'' for way too many things.
In groups, there are two notions of order.
\begin{definition}
	The \vocab{order of a group} $G$ is the number of elements of $G$.
	We denote this by $\left\lvert G \right\rvert$.
	Note that the order may not be finite, as in $\ZZ$.
	We say $G$ is a \vocab{finite group} just to mean that $\left\lvert G \right\rvert$ is finite.
\end{definition}
\begin{example}[Orders of groups]
	For a prime $p$, $\left\lvert \Zm p \right\rvert = p-1$.
	In other words, the order of $\Zm p$ is $p-1$.
	As another example,
	the order of the symmetric group $S_n$ is $\left\lvert S_n \right\rvert = n!$
	and the order of the dihedral group $D_{2n}$ is $2n$.
\end{example}

\begin{definition}
	The \vocab{order of an element} $g \in G$ is the smallest positive integer $n$
	such that $g^n = 1_G$, or $\infty$ if no such $n$ exists.
	We denote this by $\ord g$.
\end{definition}
\begin{example}[Examples of orders]
	The order of $-1$ in $\QQ^\times$ is $2$,
	while the order of $1$ in $\ZZ$ is infinite.
\end{example}
\begin{ques}
	Find the order of each of the six elements of $\Zc 6$,
	the cyclic group on six elements.
	(See \Cref{def:cyclic_group} if you've forgotten what $\Zc 6$ means.)
\end{ques}
\begin{example}[Primitive roots]
	If you know olympiad number theory, this coincides with the definition of an order of a residue mod $p$.
	That's why we use the term ``order'' there as well.
	In particular, a primitive root is precisely an element $g \in \Zm p$
	such that $\ord g = p-1$.
\end{example}
You might also know that if $x^n \equiv 1 \pmod p$,
then the order of $x \pmod p$ must divide $n$.
The same is true in a general group for exactly the same reason.
\begin{fact}
	If $g^n = 1_G$ then $\ord g$ divides $n$.
\end{fact}
Also, you can show that any element
of a finite group has a finite order.
The proof is just an olympiad-style pigeonhole argument.
Consider the infinite sequence $1_G, g, g^2, \dots$,
and find two elements that are the same.
\begin{fact}
	Let $G$ be a finite group.
	For any $g \in G$, $\ord g$ is finite.
\end{fact}

What's the last property of $\Zm p$ that you know from olympiad math?
We have Fermat's little theorem: for any $a \in \Zm p$,
we have $a^{p-1} \equiv 1 \pmod p$.
This is no coincidence:
exactly the same thing is true in a more general setting.

\begin{theorem}
	[Lagrange's theorem for orders]
	Let $G$ be any finite group.
	Then $x^{\left\lvert G \right\rvert} = 1_G$ for any $x \in G$.
\end{theorem}
Keep this result in mind! We'll prove it later in
the generality of \Cref{thm:lagrange_grp}.

\section{Subgroups}
\prototype{$\SL_n(\RR)$ is a subgroup of $\GL_n(\RR)$.}
Earlier we saw that $\GL_n(\RR)$, the $n \times n$ matrices with nonzero determinant, formed a group under matrix multiplication.
But we also saw that a subset of $\GL_n(\RR)$, namely $\SL_n(\RR)$, also formed a group with the same operation.
For that reason we say that $\SL_n(\RR)$ is a subgroup of $\GL_n(\RR)$.
And this definition generalizes in exactly the way you expect.

\begin{definition}
	Let $G = (G, \star)$ be a group.
	A \vocab{subgroup} of $G$ is exactly what you would expect it to be:
	a group $H = (H, \star)$ where $H$ is a subset of $G$.
	It's a \vocab{proper subgroup} if $H \neq G$.
\end{definition}

\begin{remark}
	To specify a group $G$, I needed to tell you both what the set $G$ was and the operation $\star$ was.
	But to specify a subgroup $H$ of a given group $G$, I only need to tell you who its elements are: the operation of $H$ is just inherited from the operation of $G$.
\end{remark}

\begin{example}
	[Examples of subgroups]
	\listhack
	\begin{enumerate}[(a) ]
		\ii $2\ZZ$ is a subgroup of $\ZZ$, which is isomorphic to $\ZZ$ itself!
		\ii Consider again $S_n$, the symmetric group on $n$ elements.
		Let $T$ be the set of permutations $\tau : \{1, \dots, n\} \to \{1, \dots, n\}$
		for which $\tau(n) = n$.  Then $T$ is a subgroup of $S_n$;
		in fact, it is isomorphic to $S_{n-1}$.
		\ii Consider the group $G \times H$ (\Cref{def:product_group})
		and the elements $ \left\{ (g, 1_H) \mid g \in G \right\} $.
		This is a subgroup of $G \times H$ (why?).
		In fact, it is isomorphic to $G$
		by the isomorphism $(g,1_H) \mapsto g$.
	\end{enumerate}
\end{example}
\begin{example}
	[Stupid examples of subgroups]
	For any group $G$, the trivial group $\{1_G\}$
	and the entire group $G$ are subgroups of $G$.
\end{example}
%\begin{example}
%	[Center of a Group]
%	Let $G$ be a group.
%	Its \vocab{center}, denoted $Z(G)$, is the set $x \in G$ such that
%	$gx = xg$ for every $g \in G$; in other words, it is the set of
%	$x \in G$ which commute with every element of $G$.
%\end{example}
%You can check the center is indeed a group (some boring details\dots).

Next is an especially important example that we'll talk about more in later chapters.
\begin{example}[Subgroup generated by an element]
	Let $x$ be an element of a group $G$.
	Consider the set
	\[ \left<x\right> = \left\{ \dots, x^{-2}, x^{-1}, 1, x, x^2, \dots \right\}. \]
	This is also a subgroup of $G$, called the subgroup generated by $x$.
\end{example}
\begin{exercise}
	If $\ord x = 2015$, what is the above subgroup equal to?
	What if $\ord x = \infty$?
\end{exercise}

Finally, we present some non-examples of subgroups.
\begin{example}[Non-examples of subgroups]
	Consider the group $\ZZ = (\ZZ, +)$.
	\begin{enumerate}[(a)]
		\ii The set $\left\{ 0,1,2,\dots \right\}$ is
		not a subgroup of $\ZZ$ because it does not contain inverses.
		\ii The set $\{ n^3 \mid n \in \ZZ \}
		= \{ \dots, -8, -1, 0, 1, 8, \dots \}$ is not a subgroup
		because it is not closed under addition;
		the sum of two cubes is not in general a cube.
		\ii The empty set $\varnothing$ is not a subgroup
		of $\ZZ$ because it lacks an identity element.
	\end{enumerate}
\end{example}

\section{Groups of small orders}
Just for fun, here is a list of all groups of order less than or equal to ten
(up to isomorphism, of course).
\begin{enumerate}
	\ii The only group of order $1$ is the trivial group.
	\ii The only group of order $2$ is $\Zc 2$.
	\ii The only group of order $3$ is $\Zc 3$.
	\ii The only groups of order $4$ are
	\begin{itemize}
		\ii $\Zc 4$, the cyclic group on four elements,
		\ii $\Zc 2 \times \Zc 2$, called the Klein Four Group.
	\end{itemize}
	\ii The only group of order $5$ is $\Zc 5$.
	\ii The groups of order six are
	\begin{itemize}
		\ii $\Zc 6$, the cyclic group on six elements.
		\ii $S_3$, the permutation group of three elements.
		This is the first non-abelian group.
	\end{itemize}
	Some of you might wonder where $\Zc 2 \times \Zc 3$ is.
	All I have to say is: Chinese remainder theorem!

	You might wonder where $D_6$ is in this list.
	It's actually isomorphic to $S_3$.
	\ii The only group of order $7$ is $\Zc 7$.
	\ii The groups of order eight are more numerous.
	\begin{itemize}
		\ii $\Zc 8$, the cyclic group on eight elements.
		\ii $\Zc 4 \times \Zc 2$.
		\ii $\Zc 2 \times \Zc 2 \times \Zc 2$.
		\ii $D_8$, the dihedral group with eight elements, which is not abelian.
		\ii A non-abelian group $Q_8$, called the \emph{quaternion group}.
		It consists of eight elements $\pm 1$, $\pm i$, $\pm j$, $\pm k$
		with $i^2=j^2=k^2=ijk=-1$.
	\end{itemize}
	\ii The groups of order nine are
	\begin{itemize}
		\ii $\Zc 9$, the cyclic group on nine elements.
		\ii $\Zc 3 \times \Zc 3$.
	\end{itemize}
	\ii The groups of order $10$ are
	\begin{itemize}
		\ii $\Zc{10} \cong \Zc5 \times \Zc2$ (again Chinese remainder theorem).
		\ii $D_{10}$, the dihedral group with $10$ elements.
		This group is non-abelian.
	\end{itemize}
\end{enumerate}

\section{Unimportant long digression}
A common question is: why these axioms?
For example, why associative but not commutative?
This answer will likely not make sense until later,
but here are some comments that may help.

One general heuristic is:
Whenever you define a new type of general object,
there's always a balancing act going on.
On the one hand, you want to include enough constraints that your
objects are ``nice''.
On the other hand, if you include too many constraints,
then your definition applies to too few objects.

So, for example, we include ``associative''
because that makes our lives easier
and most operations we run into are associative.
In particular, associativity is required for the inverse
of an element to necessarily be unique.
However we don't include ``commutative'', because examples below
show that there are lots of non-abelian groups we care about.
(But we introduce another name ``abelian''
because we still want to keep track of it.)

Another comment: a good motivation for the inverse axioms
is that you get a large amount of \emph{symmetry}.
The set of positive integers with addition is not a group,
for example, because you can't subtract $6$ from $3$:
some elements are ``larger'' than others.
By requiring an inverse element to exist, you get rid of this issue.
(You also need identity for this;
it's hard to define inverses without it.)

Even more abstruse comment:
\Cref{thm:cayley_theorem} shows that groups are actually shadows of
the so-called symmetric groups (defined later, also called permutation groups).
This makes rigorous the notion that ``groups are very symmetric''.

\section{\problemhead}

\begin{problem}
	What is the joke in the following figure? (Source: \cite{img:snsd}.)
	\begin{center}
		\includegraphics[height=8cm]{media/love-proper-isomorphic-subgroup.jpg}
		%\caption{$\heartsuit$ is a group, $G \subsetneq \heartsuit$ a subgroup and $G \cong \heartsuit$.}
	\end{center}
	\begin{hint}
		Orders.
	\end{hint}
	\begin{sol}
		The point is that $\heartsuit$ is a group, $G \subsetneq \heartsuit$ a subgroup and $G \cong \heartsuit$.
		This can only occur if $\left\lvert \heartsuit \right\rvert = \infty$;
		otherwise, a proper subgroup would have strictly smaller size than the original.
	\end{sol}
\end{problem}

\begin{problem}
	Prove Lagrange's theorem for orders in the special case
	that $G$ is a finite abelian group.
	\begin{hint}
		Copy the proof of Fermat's little theorem, using
		\Cref{lem:group_mult_biject}.
	\end{hint}
	\begin{sol}
		Let $\{g_1, g_2, \dots, g_n\}$ denote the elements of $G$.
		For any $g \in G$, this is the same as the set $\{gg_1, \dots, gg_n\}$.
		Taking the entire product and exploiting commutativity gives
		$g^n \cdot g_1g_2 \dots g_n = g_1g_2 \dots g_n$, hence $g^n=1$.
	\end{sol}
\end{problem}

\begin{problem}
	Show that $D_6 \cong S_3$ but $D_{24} \not\cong S_4$.
	\begin{hint}
		For the former,
		decide where the isomorphism should send $r$ and $s$,
		and the rest will follow through.
		For the latter, look at orders.
	\end{hint}
	\begin{sol}
		One can check manually that $D_6 \cong S_3$,
		using the map $r \mapsto (1 \; 2 \; 3)$ and $s \mapsto (1 \; 2)$.
		(The right-hand sides are in ``cycle notation'',
		as mentioned in \Cref{subsec:cycle_notation}.)
		On the other hand $D_{24}$ contains an element of order $12$
		while $S_4$ does not.
	\end{sol}
\end{problem}

\begin{sproblem}
	Let $p$ be a prime.
	Show that the only group of order $p$ is $\Zc p$.
	\begin{hint}
		Generated groups.
	\end{hint}
	\begin{sol}
		Let $G$ be a group of order $p$, and $1 \neq g \in G$.
		Look at the group $H$ generated by $g$ and use Lagrange's theorem.
	\end{sol}
\end{sproblem}

\begin{problem}
	[A hint for Cayley's theorem]
	Find a subgroup $H$ of $S_8$
	which is isomorphic to $D_8$,
	and write the isomorphism explicitly.
\end{problem}

\begin{dproblem}
	\gim
	Let $G$ be a finite group.\footnote{In other words,
		permutation groups can be arbitrarily weird.
		I remember being highly unsettled
		by this theorem when I first heard of it,
		but in hindsight it is not so surprising.}
	Show that there exists a positive integer $n$ such that
	\begin{enumerate}[(a)]
		\ii (Cayley's theorem) $G$ is isomorphic to some subgroup of the symmetric group $S_n$.
		\ii (Representation Theory) $G$ is isomorphic to some subgroup of
		the general linear group $\GL_n(\RR)$.
		(This is the group of invertible $n \times n$ matrices.)
	\end{enumerate}
	\label{thm:cayley_theorem}
	\begin{hint}
		Use $n = \left\lvert G \right\rvert$.
	\end{hint}
	\begin{sol}
		The idea is that each element $g \in G$ can be thought of as a permutation
		$G \to G$ by $x \mapsto gx$.
	\end{sol}
\end{dproblem}

\begin{problem}
	[IMO SL 2005 C5] \gim
	There are $n$ markers, each with one side white and the other side black.
	In the beginning, these $n$ markers are aligned in a row so that their white sides are all up.
	In each step, if possible, we choose a marker whose white side is up
	(but not one of the outermost markers),
	remove it, and reverse the closest marker to the left of it
	and also reverse the closest marker to the right of it.

	Prove that if $n \equiv 1 \pmod 3$ it's impossible to reach a state
	with only two markers remaining.
	(In fact the converse is true as well.)
	% http://www.artofproblemsolving.com/Forum/viewtopic.php?f=41&t=90046&p=3573800
	\begin{hint}
		Draw inspiration from $D_6$.
	\end{hint}
	\begin{sol}
		We have $www = bb$, $bww = wb$, $wwb = bw$, $bwb = ww$.
		Interpret these as elements of $D_6$.
	\end{sol}
\end{problem}

\begin{problem}
	\gim
	Let $p$ be a prime and $F_1 = F_2 = 1$, $F_{n+2} = F_{n+1} + F_n$
	be the Fibonacci sequence.
	Show that $F_{2p(p^2-1)}$ is divisible by $p$.
	\begin{hint}
		Look at the group of $2 \times 2$ matrices mod $p$
		with determinant $\pm 1$.
	\end{hint}
	\begin{sol}
		Look at the group $G$ of $2 \times 2$ matrices mod $p$
		with determinant $\pm 1$ (whose entries are the integers mod $p$).
		Let $g = \begin{bmatrix} 1 & 1 \\ 1 & 0 \end{bmatrix}$
		and then use $g^{\left\lvert G \right\rvert} = 1_G$.
	\end{sol}
\end{problem}

%\begin{problem}[Hard]
%	Exhibit two groups $G$ and $H$ which are not isomorphic with the property that
%	for every positive integer $n$,
%	the number of elements $g \in G$ with $\ord g = n$
%	equals the number of elements $h \in H$ with $\ord h = n$.
%\end{problem}


\subsection{Circles}
\noindent
You should already recognize $x^2 + y^2 = r^2$ as the equation of a circle with radius $r$ centered at the origin.
A circle as a VVF in $\R^2$ is $\vec{r}(t) = \langle r\cos{t}, r\sin{t} \rangle$, which is identical to the parametric form of a circle.
In $\mathbb{R}^3$, the z-component is some constant that tells us which plane, $z=c$, the circle is in.
We can also have circles parallel to $x=0$ and $y=0$ planes by changing the positions of the $\sin$, $\cos$, and $c$ terms.
For example, $\vec{r}(t) = \langle \cos{t}, c, \sin{t} \rangle$ is a circle in the $y=c$ plane.


\section{The Least Mean Square (LMS) Algorithm}

\begin{enumerate}[label=\alph*), leftmargin=*]

%% a)
\item
%

A general AR(2) process $x(n)$ with parameters $a_{1}, a_{2}$ satisfies the difference equation:

\begin{equation}
    x(n) = a_{1} x(n - 1) + a_{2} x(n - 2) + \eta(n)
\label{eq:ar_2}
\end{equation}

where $\eta(n) \sim \mathcal{N}(0, \sigma_{\eta}^{2})$. The Least Mean Square (LMS) algorithm is used to approximate
the autoregressive parameters $a_{1}, a_{2}$ from data, treating $\mathbf{x}(n) = [x(n-1), x(n-2)]^{T}$ and
$y(n) = x(n)$ as the input (features) vector and the output (target), respectively.

The correlation matrix $\mathbf{R}_{xx}$ of the input vector $\mathbf{x}(n)$ is given by:

\begin{equation}
    \mathbf{R}_{xx} = \E[\mathbf{x}(n) \mathbf{x}^{T}(n)] = \E
    \begin{bmatrix}
        x(n-1) x(n-1) & x(n-1) x(n-2) \\
        x(n-1) x(n-2) & x(n-2) x(n-2)
    \end{bmatrix} =
    \begin{bmatrix}
        r_{xx}(0) & r_{xx}(1) \\
        r_{xx}(1) & r_{xx}(0)
    \end{bmatrix}
\label{eq:cov_ar_2}
\end{equation}

where $r_{xx}(k)$ the autocorrelation function (ACF) of $x(n)$. To obtain the ACF of the AR(2) process, multiply equation (\ref{eq:ar_2}) by $x(n-k)$ and take expectations:

\begin{align}
    r_{xx}(k) = \E[ x(n) x(n-k) ]   &= \E \bigg[ a_{1} x(n - 1) x(n-k) + a_{2} x(n - 2) x(n-k) + \eta(n) x(n-k) \bigg] \nonumber\\
    r_{xx}(k) = \E[ x(n) x(n-k) ]   &= a_{1} \E \big[ x(n - 1) x(n-k) \big] + a_{2} \E \big[ x(n - 2) x(n-k) \big] + \E \big[ \eta(n) x(n-k) \big]
\end{align}

Notice that $\E [\eta(n) x(n-k)]$ vanishes when $k > 0$, then:

\begin{align}
    r_{xx}(0)   &= a_{1} r_{xx}(1) + a_{2} r_{xx}(2) + \sigma_{\eta}^{2} \\
    r_{xx}(k)   &= a_{1} r_{xx}(k-1) + a_{2} r_{xx}(k-2), k > 0
\label{eq:acf_ar_2}
\end{align}

Note that ACF is an even function or equivalently $r_{xx}(-k) = r_{xx}(k), \forall k \in \sZ$, thus using the true process parameters $a_{1} = 0.1$, $a_{2} = 0.8$
and $\sigma_{\eta}^{2} = 0.25$, we obtain the three simultaneous equations with three unknowns:

\begin{align}
    r_{xx}(0)   &= a_{1} r_{xx}(1) + a_{2} r_{xx}(2) + \sigma_{\eta}^{2} \\
    r_{xx}(1)   &= a_{1} r_{xx}(0) + a_{2} r_{xx}(-1) = a_{1} r_{xx}(0) + a_{2} r_{xx}(1) \\
    r_{xx}(2)   &= a_{1} r_{xx}(1) + a_{2} r_{xx}(0)
\end{align}

with the unique solution $r_{xx}(k) = [\frac{25}{27}, \frac{25}{54}, \frac{85}{108}]$ for $k=1, 2, 3$.

Hence by substitution in equation (\ref{eq:cov_ar_2}) we obtain the  correlation matrix of the input vector $\mathbf{x}(n)$:

\begin{equation}
    \mathbf{R}_{xx} =
    \frac{25}{54}
    \begin{bmatrix}
        2 & 1 \\
        1 & 2
    \end{bmatrix}
\label{eq:cov_ar_2_result}
\end{equation}

Convergence of the LMS algorithm depends on the step size $\mu$, which should satisfy:

\begin{equation}
    0 < \mu < \frac{2}{\lambda_{max}}
\label{cond:mu_max}
\end{equation}

where $\lambda_{max}$ the largest eigenvalue of the correlation matrix $\mathbf{R}_{xx}$. Performing the eigendecomposition of $\mathbf{R}_{xx}$, we obtain
$\lambda_{1} = 0.4630$ and $\lambda_{2} = \lambda_{max} = 1.3889$ and as a result, convergence to the Wiener optimal solution is guaranteed for:

\begin{equation}
    0 < \mu < 1.44
\label{cond:mu_max_val}
\end{equation}

%% b)
\item
%

The LMS algorithm is implemented and tested on $N=1000$ samples of 100 different realisations of $x(n)$ process. The squared prediction error over time
for one and the average of the 100 realisations is provided at figure \ref{fig:3_1_b} for two different step-sizes $\mu_{1} = 0.01$ and $\mu_{2} = 0.05$.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/b/squared_prediction_error_1}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/b/squared_prediction_error_ens}
    \end{subfigure}
    \caption{LMS: squared prediction error over time and $\mu$ convergence rate.}
    \label{fig:3_1_b}
\end{figure}

Concentrating on the averaged of 100 realisation prediction error (reduced variance of estimate), we notice that the larger step-size $\mu_{2} = 0.05$ converges
within 50 timesteps, while $\mu_{1}$ converges in 200 timesteps, verifying the theoretical argument that larger $\mu$ values allow steeper decent of the error surface,
as long as the condition (\ref{cond:mu_max_val}) is satisfied. Nonetheless, fast convergence is traded with larger oscillations around the true parameter values,
motivating the use of an adaptive step-size, which decays over time.

%% c)
\item
%

The theoretical Misadjustment, $\mathcal{M}_{LMS}$, of the LMS algorithm is obtained by the approximation formula:

\begin{equation}
    \mathcal{M}_{LMS} \approx \frac{\mu}{2} Tr\{ \mathbf{R}_{xx} \} = \frac{\mu}{2} 1.8519
\end{equation}

where $\mathbf{R}_{xx}$ from (\ref{eq:cov_ar_2_result}) is used.

According to figure \ref{fig:3_1_b} the squared prediction error plateaus for both $\mu$ values after $t > 200$. To guarantee that a steady-state has been reached,
we time average the squared prediction error for $t_{0} > 500$, in order to obtain a Mean Squared Error (MSE) estimate. The empirical misadjustment, $\mathcal{M}_{emp}$,
is then obtained using the formula:

\begin{equation}
    \mathcal{M}_{emp} = \frac{\mathtt{MSE}}{\sigma_{\eta}^{2}} - 1
\end{equation}

Table \ref{tab:3_1_c} summarises the empirical and theoretical misadjustments of the simple LMS algorithm for the $x(n)$ process.

\begin{table}[h]
\centering
\begin{tabular}{|c|c|c|}
\hline
$\boldsymbol{\mu}$ & $\mathcal{M}_{LMS}$ & $\mathcal{M}_{emp}$ \\
\hline
\hline
$0.01$ & $0.0093$ & $0.00766$ \\
\hline
$0.05$ & $0.0463$ & $0.04912$ \\
\hline
\end{tabular}
\caption{LMS: empirical and theoretical approximation of misadjustment comparison matrix.}
\label{tab:3_1_c}
\end{table}

%% d)
\item
%

The evolution of the LMS filter coefficients over time is shown in figure \ref{fig:3_1_d}, along with the true AR(2) process parameters, for step-sizes $\mu_{1} = 0.01$ and $\mu_{2} = 0.05$.
The illustrated curves represent the average of the coefficients of 100 independently trained LMS filters on different realisations of the same $x(n)$ stochastic process.

Clearly, for both $\mu$ values, $\hat{a}_{1}$ coefficient converges to the true $a_{1}$ parameter, while the second coefficient $\hat{a}_{2}$ has a negative offset
($3.75\%$ for $\mu_{1} = 0.01$ and $10\%$ for $\mu_{2} = 0.05$). Moreover, we note that a larger step-size (i.e $\mu_{2}$) compromises oscillations around the true values and a greater offset
for faster convergence. In more detail, within 200 timesteps $\mu_{1}$ reaches the $10\%$ error bounds, while $\mu_{2}$ needs 350 timesteps.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/d/weights_evolution-mu_0.01}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/d/weights_evolution-mu_0.05}.pdf}
    \end{subfigure}
    \caption{LMS: steady state values of the adaptive filter coefficients for different step-sizes $\mu$.}
    \label{fig:3_1_d}
\end{figure}

%% e)
\item
%

Let the objective function $\mathcal{J}_{2}$, such that:

\begin{equation}
    \mathcal{J}_{2}(n) = \frac{1}{2} \bigg( e^{2}(n) + \gamma \| \vw(n) \|_{2}^{2} \bigg)
\label{eq:J_2}
\end{equation}

Expressing the error term, $e(n)$, in terms of the input vector $\mathbf{x}_{n}$ and the target value $y_{n}$:

\begin{align}
    \mathcal{J}_{2}(n)  &= \frac{1}{2} \bigg( \| y(n) - \vw(n)^{T} \mathbf{x}(n) \|_{2}^{2} + \gamma \| \vw(n) \|_{2}^{2} \bigg) \\
    \mathcal{J}_{2}(n)  &= \frac{1}{2} \bigg( \big( y(n) - \vw(n)^{T} \mathbf{x}(n))^{T} (y(n) - \vw(n)^{T} \mathbf{x}(n) \big) + \gamma \vw(n)^{T} \vw(n) \bigg)
\end{align}

Calculating the gradient of the objective function, $\nabla_{\vw} \mathcal{J}_{2}$, with respect to the weights $\vw$:

\begin{align}
    \nabla_{\vw} \mathcal{J}_{2}(n) &= - ( y(n) - \vw(n)^{T} \mathbf{x}(n) ) \mathbf{x}(n) + \gamma \vw(n) \\
    \nabla_{\vw} \mathcal{J}_{2}(n) &= - e(n) \mathbf{x}(n) + \gamma \vw(n)
\end{align}

Applying the gradient descent update to weights $\vw$ to minimise the objective function $\mathcal{J}_{2}$:

\begin{align}
    \vw(n + 1)  &= \vw(n) - \mu \nabla_{\vw} \mathcal{J}_{2}(n) \\
    \vw(n + 1)  &= \vw(n) - \mu \big( - e(n) \mathbf{x}(n) + \gamma \vw(n) \big) \\
    \vw(n + 1)  &= (1 - \mu \gamma) \vw(n) + \mu e(n) \mathbf{x}(n)
\label{eq:leaky_lms}
\end{align}

Hence we proved that the Leaky LMS algorithm following the update rule in (\ref{eq:leaky_lms}) is equivalent to the minimisation of the objective function $\mathcal{J}_{2}$
defined in (\ref{eq:J_2}).

%% f)
\item
%

In figure \ref{fig:3_1_f} the Leaky LMS filter coefficients for different leakage coefficient $\gamma$ and step-sizes $\mu$
are provided, against the true AR(2) process parameters.

The predicted filter coefficients do not converge to the true parameters, and increasing $\gamma$ values introduce
a greater bias between the estimates and the process autoregressive parameters.

The offline, non-adaptive but optimal solution to this linear system is the Wiener filter, which has access to all the
observed samples and relies on the construction of the autocorrelation matrix, $\mathbf{R} = \E [ \mathbf{X} \mathbf{X}^{T} ]$,
and the crosscorrelation vector (between desired output value and input vector), $\mathbf{p}$, then the optimal filter weights,
$\vw_{*}$, are given by:

\begin{equation}
    \vw_{*} = \mathbf{R}^{-1} \vp
\end{equation}

where $\mathbf{R}$ needs to be invertible. The LMS algorithm can be shown to converge to this solution if (\ref{cond:mu_max})
is true. Invertibility of the autocorrelation matrix is not guaranteed, since it is positive semi-definite and not strictly
positive definite. The Leaky LMS algorithm can be shown to converge to the solution:

\begin{equation}
    \vw_{reg} = (\mathbf{R} + \gamma \mathbf{I})^{-1} \vp
\end{equation}

where the leakage coefficient $\gamma > 0$ acts as a regularization parameter ensuring invertibility of the "modified"
autocorrelation matrix $(\mathbf{R} + \gamma \mathbf{I})$. Nonetheless, this is a common example of bias-variance trade-off
since larger values of $\gamma$ introduce bias but penalise variance of estimates
(due to the $L_{2}$ norm term in $\mathcal{J}_{2}$).

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/f/weights_evolution-mu_0.01-gamma_0.2}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/f/weights_evolution-mu_0.05-gamma_0.2}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/f/weights_evolution-mu_0.01-gamma_0.6}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/f/weights_evolution-mu_0.05-gamma_0.6}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/f/weights_evolution-mu_0.01-gamma_1.0}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/adaptive-signal-processing/the-least-mean-square-algorithm/assets/f/weights_evolution-mu_0.05-gamma_1.0}.pdf}
    \end{subfigure}
    \caption{Leaky LMS: filter weights evolution for different leakage coefficient $\gamma$.}
    \label{fig:3_1_f}
\end{figure}

%
\end{enumerate}