\documentclass{article}
\usepackage[a4paper]{geometry}
\usepackage{amsmath}
\geometry{verbose,tmargin=2cm,bmargin=2cm,lmargin=2cm,rmargin=2cm}

\begin{document}

\title{Nonlinear stochastic differential equation generating $1/f$ noise}

\author{Julius Ruseckas}

\date{\today{}}

\maketitle

\section{Nonlinear stochastic differential equations with multiplicative noise}

In Refs.~\cite{kaulakys2004,kaulakys2006} nonlinear stochastic differential
equations (SDEs) of the form
\begin{equation}
dx_{t}=\sigma^{2}\left(\eta-\frac{\lambda}{2}\right)x_{t}^{2\eta-1}dt
+\sigma x_{t}^{\eta}dW_{t}\label{eq:sde-1}
\end{equation}
have been proposed. Here $W_{t}$ is a standard Wiener process (the
Brownian motion), $\eta$ is the power-law exponent of multiplicative
noise and $\sigma$ is the amplitude of the noise. In order to avoid
the divergence of the steady-state probability density function (PDF),
Eq.~(\ref{eq:sde-1}) should be considered together with appropriate
restriction of the diffusion of the stochastic variable $x$. Here
we investigate the SDE with the exponential restriction of diffusion
at $x=x_{\mathrm{min}}$
\begin{equation}
dx_{t}=\sigma^{2}\left(\eta-\frac{\lambda}{2}+\frac{m}{2}
\left(\frac{x_{\mathrm{min}}}{x}\right)^{m}\right)x_{t}^{2\eta-1}dt
+\sigma x_{t}^{\eta}dW_{t}\,.
\label{eq:SDE-our}
\end{equation}
Equation (\ref{eq:SDE-our}) has stationary probability distribution
function (PDF) of the form
\begin{equation}
P_{0}(x)\sim x^{-\lambda}
\exp\left(-\left(\frac{x_{\mathrm{min}}}{x}\right)^{m}\right)\,.
\end{equation}
Based on scaling consideration \cite{ruseckas2014} it is predicted
that the power spectral density (PSD) of the signal $x_{t}$ has $f^{-\beta}$
behaviour in a wide range of frequencies, with
\begin{equation}
\beta=1+\frac{\lambda-3}{2(\eta-1)}\,.
\end{equation}
Using the parameters $\lambda=2\eta$ and $m=2\eta-2$, equation
(\ref{eq:SDE-our}) takes the form of the \emph{Constant Elasticity of Variance}
(CEV) process:
\begin{equation}
dx_{t}=\sigma^{2}(\eta-1)x_{\mathrm{min}}^{2(\eta-1)}x_{t}dt
+\sigma x_{t}^{\eta}dW_{t}\,.
\end{equation}
If $\lambda=3$ then SDE (\ref{eq:SDE-our}) gives $1/f$ spectrum.
Thus the CEV process has $1/f$ spectrum when $\eta=\frac{3}{2}$:
\begin{equation}
dx_{t}=\mu x_{t}dt+\sigma x_{t}^{\frac{3}{2}}dW_{t}\,.\label{eq:SDE-CEV}
\end{equation}
Here
\begin{equation}
\mu=\frac{1}{2}\sigma^{2}x_{\mathrm{min}}\,.
\end{equation}

\subsection{Derivation of the analytical expression for the power spectral density}

The analytical expression for the transition probability $P_{x}(x',t|x,0)$
(the conditional probability that at time $t$ the signal has value
$x$ with the condition that at time $t=0$ the signal had the value
$x_{0}$) of the CEV process is \cite{kazakevicius2016}
\begin{equation}
P_{x}(x,t|x_{0},0)=\frac{x_{\mathrm{min}}}{(1-e^{-\mu t})}
\sqrt{\frac{x_{0}}{x^{5}}}\exp\left(\frac{1}{2}\mu t
-\frac{x_{\mathrm{min}}}{(1-e^{-\mu t})}\left(\frac{1}{x}
+\frac{1}{x_{0}}e^{-\mu t}\right)\right)
I_{1}\left(\frac{x_{\mathrm{min}}}{\sinh\left(\frac{1}{2}\mu t\right)}
\frac{1}{\sqrt{x_{0}x}}\right)\,.
\label{eq:trans}
\end{equation}
Here $I_{z}$ is the modified Bessel function with index $z$. The
steady-state PDF has the form
\begin{equation}
P_{0}(x)=\frac{x_{\mathrm{min}}^{2}}{x^{3}}
\exp\left(-\frac{x_{\mathrm{min}}}{x}\right)\,.
\end{equation}
The average of the signal is
\begin{equation}
\bar{x}=\int_{0}^{\infty}xP_{0}(x)dx=x_{\mathrm{min}}\,.
\end{equation}
The autocorrelation function can be calculated using the expression
\begin{equation}
C(t)=\int dx\int dx'\,(x-\bar{x})(x'-\bar{x})P_{0}(x)P_{x}(x',t|x,0)\,.
\end{equation}
Using Eq.~(\ref{eq:trans}) and performing the integration we obtain
the autocorrelation function
\begin{equation}
C(t)=x_{\mathrm{min}}^{2}\left[-e^{\mu t}
\ln\left(1-e^{-\mu t}\right)-1\right]\,.
\label{eq:autocorr}
\end{equation}
When $\mu t\ll1$ we get
\begin{equation}
C(t)\approx-x_{\mathrm{min}}^{2}-x_{\mathrm{min}}^{2}\ln(\mu t)\,.
\end{equation}
Similar expansion has been obtained for the autocorrelation function
in the case of $1/f$ spectrum.

According to Wiener-Khintchine relations, the power spectral density
is connected with the autocorrelation function via the transformation
\begin{equation}
S(f)=2\int_{-\infty}^{\infty}C(t)e^{i\omega t}dt=4\int_{0}^{\infty}C(t)
\cos(\omega t)dt\,,
\label{eq:wk}
\end{equation}
where $\omega=2\pi f$ . Using Eq.~(\ref{eq:autocorr}) for the autocorrelation
function we get the following expression for the power spectral density:
\begin{equation}
S(f)=2x_{\mathrm{min}}^{2}\left[\frac{-\gamma
-\psi\left(-i\frac{\omega}{\mu}\right)}{\mu+i\omega}
+\frac{-\gamma-\psi\left(i\frac{\omega}{\mu}\right)}{\mu-i\omega}\right]\,,
\end{equation}
where $\gamma\approx0.577216$ is the Euler's constant and
$\psi(z)=\Gamma^{\prime}(z)/\Gamma(z)$ is the digamma function. When
$\omega\gg\mu$ then the power spectral density is
\begin{equation}
S(f)\approx\frac{2\pi x_{\mathrm{min}}^{2}}{\omega}\,.
\end{equation}

\section{Method of numerical solution}

Method of numerical solution with a variable time step is described
in Ref.~\cite{ruseckas2016}. For the numerical solution, we use
Euler-Marujama approximation, transforming differential equations
to difference equations. If the time step is $\Delta t=h$ then the
difference equations, corresponding to Eq.~(\ref{eq:SDE-CEV}) are
\begin{align}
x_{k+1} = & x_{k}+\mu x_{k}h+\sigma x_{k}^{\frac{3}{2}}\sqrt{h}\varepsilon_{k}\\
t_{k+1} = & t_{k}+h
\end{align}
Here $\varepsilon_{k}$ are normally distributed uncorrelated random
variables with a zero expectation and unit variance. Variable time
step of integration
\[
h_{k}=\frac{\kappa^{2}}{\sigma^{2}x_{k}}
\]
leads to the equations
\begin{align}
x_{k+1} = & x_{k}+\frac{1}{2}\kappa^{2}x_{\mathrm{min}}+\kappa x_{k}\varepsilon_{k}\\
t_{k+1} = & t_{k}+\frac{\kappa^{2}}{\sigma^{2}x_{k}}
\end{align}
Here $\kappa\ll1$ is a small parameter.

\begin{thebibliography}{1}
\bibitem{kaulakys2004}B.~Kaulakys and J.~Ruseckas, \textit{Stochastic
nonlinear differential equation generating 1/f noise}, Phys.~Rev.~E
\textbf{70}, 020101 (2004).

\bibitem{kaulakys2006}B.~Kaulakys, J.~Ruseckas, V.~Gontis and
M.~Alaburda, \textit{Nonlinear stochastic models of 1/f noise and power-law
distributions}, Physica A \textbf{365}, 217\textendash 221 (2006).

\bibitem{ruseckas2014}J.~Ruseckas and B.~Kaulakys, \textit{Scaling
properties of signals as origin of 1/f noise}, J.~Stat.~Mech.\ \textbf{2014},
P06005 (2014).

\bibitem{kazakevicius2016}R.~Kazakevi\v{c}ius and J.~Ruseckas,
\textit{Influence of external potentials on heterogeneous diffusion
processes}, Phys.~Rev.~E \textbf{94}, 032109 (2016).

\bibitem{ruseckas2016}J.~Ruseckas, R.~Kazakevi\v{c}ius and B.~Kaulakys,
\textit{1/f noise from point process and time-subordinated Langevin
equations}, J.~Stat.~Mech.\ \textbf{2016}, 054022 (2016).
\end{thebibliography}

\end{document}


%---- MATH -> Functions
\section{Functions}
%%% Figure: Function exp(x)
\begin{marginfigure}[-5\baselineskip]
  %\includegraphics[]{helix}
  \begin{tikzpicture}
     \begin{axis}[axis lines = left, xlabel = $x$, ylabel = {$f(x)=\exp{(x)}$}]
%Below the red parabola is defined
     \addplot [domain=0:3, samples=100]{exp(x)};
     %\addlegendentry{$f(x)=\exp(x)$}
     \end{axis}
  \end{tikzpicture}
  \caption{Function as a graph.}
  \setfloatalignment{b}
\end{marginfigure}
%%--- Functions
Functions are the backbone of calculus and is typically viewed in terms of their graphs of $x$ paired with $f(x)$. It can be viewed more generally as a {\it mechanism} of producing an output $f(x)$ for an {\it given} input $x$. With this mechanical view in mind, certain properties of functions can be defined more naturally. The {\it domain} of a function consists of all possible values of inputs and the {\it range} is all possible values of outputs. For single-variable calculus, the domain and ranges are going to be simple for example the real number line $R=(-\infty ~\infty)$ or certain sub-intervals of that $[a,b], [0, \infty], etc.$.
Certain operations on functions are critical, perhaps the most important being \textit{composition}, \textit{$f$ composed with $g$}, which takes it's input as $x$ and produces the output $(f\cdot g)(x)=f(g(x))$ which can be visualized as a chain operation, with proper order.
\marginnote[-2\baselineskip]{$\sqrt{x^2+2}$ can be decomposed to $(f\cdot g)(x)$ where, $g(x)=x^2+2$ and $f(x)=\sqrt{x}$}
%
\marginnote[\baselineskip]{Note, $f^{-1}(x) \neq 1/f(x)$}
Another important operation of functions is that of \textit{inverse} of a function which takes it's input as $x$ and produces an output $f^{-1}(x)$ such that, when the output is fed to the input of the function $f(x)$ it's output is $x$. In other words, $f^{-1}(x)$ \textit{undoes} whatever $f(x)$ \textit{does}.


\documentclass{article}

\usepackage[a4paper, margin=3cm]{geometry}
\usepackage{float}
\usepackage{algorithm2e}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{multicol}

\title{Lesson 2 : Problem Solving Paradigms}
\author{CPC UTEC - Lecturers Notes}

\begin{document}

\maketitle

\setlength{\parindent}{0mm}

\textbf{Scope \& Objectives:}
\begin{itemize}
	\item Have a formal understanding of Big-Oh notation.
	\item Learn about the popular problem solving paradigms.
	\item Understand Complete Search and brute force.
\end{itemize}

\setlength{\parskip}{4mm}

\section{Algorithmic Analysis}

\textit{Big-Oh} notation is a tool we can use to measure how the complexity of our solution scales as the input of our program grows. This is done by measuring how our complexity compares to other functions as the input size goes to infinity. For this reason this is know as \textit{Asymptotic Analysis}.

Formally, we can define Big-Oh notation as:
$$f(n) = O(g(n)) \longleftrightarrow f(n) \leq kg(n)$$
$$\text{Where } n > n_0, \; k > 0 \text{ and } n_0 \in \mathbb{R}$$

This definition might seem complex but is very simple to understand. We can use the aid of the following graph:

\begin{figure}[H]
	\centering
	\includegraphics[width=0.5\linewidth]{images/bigoh}
	\caption{Graphical understanding of Big-Oh}
\end{figure}

First of all we must understand that $O(g(n))$ is not a function but a family (or set) of functions. For this reasons many believe that the `$\in$' symbol would be more appropriate than the `=' symbol. Therefore, it is a good idea to read the `=' sign as \textit{belongs} or \textit{is in}.

\section{Problem Solving Paradigms}

\subsection{Paradigm vs Algorithm}

\begin{multicols}{2}
	\raggedcolumns
	\centering \textbf{Algorithm}
	\begin{itemize}
		\item Solves a specific problem.
		\item Gives steps and instructions.
	\end{itemize}

	\columnbreak

	\centering \textbf{Paradigm}
	\begin{itemize}
		\item Solves a family of problems.
		\item Gives ideas and methodologies.
		\item Is a conceptual framework for problem solving.
	\end{itemize}
\end{multicols}

\subsection{Popular Paradigms}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\linewidth]{images/programming-paradigms}
	\caption{Popular problem solving paradigms.}
\end{figure}

\section{Complete Search}

Most competitive programming problems (and most problems in general) can be solved by listing all the possible solutions and checking each of them until we find the correct one. This method of problems solving in which we exhaustively check all possible solutions is called \textit{complete search}.

This paradigm is extremely important as we can almost always find a complete search solutions to any given problem (even though this solution will probably violate the time constrains). Being good at designing complete search solutions is extremely important to learn the other problems solving paradigms and understanding the importance of optimizations.

\subsection{Key Terms}

\begin{itemize}
	\item \textbf{Solution-Space:} All the candidate or possible solutions there are.
	\item \textbf{Brute Force:} Iterative complete search.
	\item \textbf{Fixing:} Assuming a \textit{variable} has a constant value.
\end{itemize}

\subsection{Example: Number of solutions}

Lets consider the following problem. Given the positive integers $a$, $b$, $c$ and $d$, find how many solutions the equation $x + y + z = d$ has, given that $x, y, z \in \mathbb{N}^+$, $x \leq a$, $y \leq b$ and $z \leq c$.

A first approach for solving this problem is to fix the values of $x$, $y$ and $z$. We can then find the value of $x + y + z$ and check if it equals $d$.

\begin{algorithm}
	\SetAlgoNoEnd
	$cnt \gets 0$\;
	\For{$x \gets 1$ \KwTo $a$}{
		\For{$y \gets 1$ \KwTo $b$}{
			\For{$z \gets 1$ \KwTo $c$}{
				\If{$x + y + z = d$}{
					$cnt \gets cnt + 1$\;
				}
			}
		}
	}
	\Return $cnt$\;
\end{algorithm}

It is simple to see that this solution has a complexity of $O(n^3)$. In general, for complete search our time complexity can be estimates as:

$$T(n) = |\text{Solution Space}| \times \text{time to check a solution}$$

In this example we take $O(1)$ to check each candidate solution, but we visit $O(n^3)$ possible solutions, giving the total complexity of $O(n^3)$. Can we improve this solution?

One important observation is to realize that if we fix the values of $a$ and $b$, then we can re-write the equation as: $z = d - x - y$. This means that for every pair of values $(x, y)$, there exists a single possible value of $z$ that can satisfy the equation. Now we all have to do is verify that the value of $z$ is valid.

\begin{algorithm}
	\SetAlgoNoEnd
	$cnt \gets 0$\;
	\For{$x \gets 1$ \KwTo $a$}{
		\For{$y \gets 1$ \KwTo $b$}{
			$z \gets d - x - y$\;
			\If{$1 \leq z \leq c$}{
				$cnt \gets cnt + 1$\;
			}
		}
	}
	\Return $cnt$\;
\end{algorithm}

Our time taken to verify each solution is still $O(1)$, however the size of our solution space is now $O(n^2)$. This drastically improves the solutions complexity, taking it from $O(n^3)$ to $O(n^2)$. Further improvements can be done to make in order to find the answer in $O(n)$ time or faster.

The most important takeaway of this example should be that a brute force algorithm is not necessarily a dumb or easy algorithm, many time brute force solutions need to be tightly optimized in order to yield a correct solution that works within the time constrains.

\end{document}


% This chapter was modified on 1/12/05.
%\setcounter{chapter}{7}
\chapter{Law of Large Numbers}\label{chp 8}

\section[Discrete Random Variables]{Law of Large Numbers for Discrete Random Variables}
\label{sec 8.1}
We are now in a position to prove our first fundamental theorem of probability.
We have seen that an intuitive way to view the probability of a certain outcome is as the
frequency with which that outcome occurs in the long run, when the experiment is repeated a large
number of times.  We have also defined probability mathematically as a value of a distribution
function for the random variable representing the experiment.   The Law of Large Numbers, which is
a theorem proved about the mathematical model of probability, shows that this model is consistent
with the frequency interpretation of probability.  This theorem is sometimes called the \emx {law
of averages.}  To find out what would happen if this law were not true, see the article  by
Robert M. Coates.\index{COATES, R. M.}\footnote{R.~M.~Coates, ``The Law," \emx {The World of
Mathematics,} ed. James R. Newman (New York: Simon and Schuster, 1956.}

\subsection*{Chebyshev Inequality}
To discuss the Law of Large Numbers, we first need an important inequality
called the \emx {Chebyshev Inequality.}

\begin{theorem}{\bf (Chebyshev Inequality)}\index{Chebyshev Inequality}
Let $X$ be a discrete random variable with expected value $\mu = E(X)$, and let $\epsilon
> 0$ be any positive real number.  Then
$$
P(|X - \mu| \geq \epsilon) \leq \frac {V(X)}{\epsilon^2}\ .
$$
\proof
Let $m(x)$ denote the distribution function of $X$.  Then the probability that $X$
differs from $\mu$ by at least $\epsilon$ is given by
$$P(|X - \mu| \geq \epsilon) = \sum_{|x - \mu| \geq \epsilon} m(x)\ .$$
We know that
$$V(X) = \sum_x (x - \mu)^2 m(x)\ ,$$
and this is clearly at least as large as
$$\sum_{|x - \mu| \geq \epsilon} (x - \mu)^2 m(x)\ ,$$
since all the summands are positive and we have restricted the range of summation in the
second sum.  But this last sum is at least
\begin{eqnarray*}
\sum_{|x - \mu| \geq \epsilon} \epsilon^2 m(x) &=&
\epsilon^2 \sum_{|x - \mu| \geq \epsilon} m(x) \\
&=& \epsilon^2 P(|X - \mu| \geq \epsilon)\ .\\
\end{eqnarray*}
So,
$$ P(|X - \mu| \geq \epsilon) \leq \frac {V(X)}{\epsilon^2}\ .
$$
\end{theorem}
Note that $X$ in the above theorem can be any discrete random variable, and
$\epsilon$ any positive number.

\begin{example}
Let $X$ by any random variable with $E(X) = \mu$ and $V(X) = \sigma^2$.  Then,
if $\epsilon = k\sigma$, Chebyshev's Inequality states that
$$
P(|X - \mu| \geq k\sigma) \leq \frac {\sigma^2}{k^2\sigma^2} = \frac 1{k^2}\ .
$$
Thus, for any random variable, the probability of a deviation from the mean of
more than~$k$ standard deviations is ${} \leq 1/k^2$.  If, for example,
$k = 5$, $1/k^2 = .04$.
\end{example}

Chebyshev's Inequality is the best possible inequality in the sense that, for
any $\epsilon > 0$, it is possible to give an example of a random variable for
which Chebyshev's Inequality is in fact an equality.  To see this, given
$\epsilon > 0$, choose $X$ with distribution
$$
p_X = \pmatrix{
-\epsilon & +\epsilon \cr
1/2 & 1/2 \cr}\ .
$$
Then $E(X) = 0$, $V(X) = \epsilon^2$, and
$$
P(|X - \mu| \geq \epsilon) = \frac {V(X)}{\epsilon^2} = 1\ .
$$

We are now prepared to state and prove the Law of Large Numbers.

\subsection*{Law of Large Numbers}
\begin{theorem}{\bf (Law of Large Numbers)}\index{Law of Large Numbers}
Let $X_1$,~$X_2$, \dots,~$X_n$ be an independent trials process, with
finite expected value $\mu = E(X_j)$ and finite variance $\sigma^2 =
V(X_j)$.  Let $S_n = X_1 + X_2 +\cdots+ X_n$.  Then for
any $\epsilon > 0$,
$$ P\left( \left| \frac {S_n}n - \mu \right| \geq \epsilon
\right) \to 0
$$
as $n \rightarrow \infty$.
Equivalently,
$$
P\left( \left| \frac {S_n}n - \mu \right| < \epsilon \right) \to 1
$$
as $n \rightarrow \infty$.
\proof
Since $X_1$,~$X_2$, \dots,~$X_n$ are independent and have the same distributions,
we can apply Theorem~\ref{thm 6.9}.  We obtain
$$
V(S_n) = n\sigma^2\ ,
$$
and
$$
V (\frac {S_n}n) = \frac {\sigma^2}n\ .
$$
Also we know that
$$
E (\frac {S_n}n) = \mu\ .
$$
By Chebyshev's Inequality, for any $\epsilon > 0$,
$$
P\left( \left| \frac {S_n}n - \mu \right| \geq \epsilon \right) \leq \frac
{\sigma^2}{n\epsilon^2}\ .
$$
Thus, for fixed $\epsilon$,
$$
P\left( \left| \frac {S_n}n - \mu \right| \geq \epsilon \right) \to 0
$$
as $n \rightarrow \infty$, or equivalently,
$$
P\left( \left| \frac {S_n}n - \mu \right| < \epsilon \right) \to 1
$$
as $n \rightarrow \infty$.
\end{theorem}

\subsection*{Law of Averages}
Note that $S_n/n$ is an average of the individual outcomes, and one often
calls the Law of Large Numbers the ``law of averages." It is a striking fact
that we can start with a random experiment about which little can be predicted
and, by taking averages, obtain an experiment in which the outcome can be
predicted with a high degree of certainty.  The Law of Large Numbers, as we
have stated it, is often called the ``Weak Law of Large Numbers" to
distinguish it from the ``Strong Law of Large Numbers" described in
Exercise~\ref{exer 8.1.16}.

Consider the important special case of Bernoulli trials with probability~$p$
for success.  Let $X_j = 1$ if the $j$th outcome is a success and~0 if it is a
failure.  Then $S_n = X_1 + X_2 +\cdots+ X_n$ is the number of successes in $n$
trials and $\mu = E(X_1) = p$.  The Law of Large Numbers states that for any
$\epsilon > 0$
$$
P\left( \left| \frac {S_n}n - p \right| < \epsilon \right) \to 1
$$
as $n \rightarrow \infty$.  The above statement says that, in a large number of repetitions
of a Bernoulli experiment, we can expect the proportion of times the event will
occur to be near $p$.  This shows that our mathematical model of probability
agrees with our frequency interpretation of probability.

\subsection*{Coin Tossing}
Let us consider the special case of tossing a coin $n$ times with $S_n$ the
number of heads that turn up.  Then the random variable $S_n/n$ represents the
fraction of times heads turns up and will have values between 0~and~1.  The Law
of Large Numbers predicts that the outcomes for this random variable will, for
large~$n$, be near 1/2.

In Figure~\ref{fig 8.1}, we have plotted the distribution for this example for increasing values of
~$n$. We have marked the outcomes between .45~and~.55 by dots at the top of the
spikes.  We see that as
$n$ increases the distribution gets more and more concentrated around~.5 and a larger
and larger percentage of the total area is contained within the interval
$(.45,.55)$, as predicted by the Law of Large Numbers.

\putfig{5.0truein}{PSfig8-1}{Bernoulli trials distributions.}{fig 8.1}

\subsection*{Die Rolling}
\begin{example}
Consider $n$ rolls of a die.  Let $X_j$ be the outcome of the $j$th roll.  Then
$S_n = X_1 + X_2 +\cdots+ X_n$ is the sum of the first $n$ rolls.  This is an
independent trials process with $E(X_j) = 7/2$.  Thus, by the Law of Large
Numbers, for any $\epsilon > 0$
$$
P\left( \left| \frac {S_n}n - \frac 72 \right| \geq \epsilon \right) \to 0
$$
as $n \rightarrow \infty$.  An equivalent way to state this is that, for any $\epsilon > 0$,
$$
P\left( \left| \frac {S_n}n - \frac 72 \right| < \epsilon \right) \to 1
$$
as $n \rightarrow \infty$.
\end{example}

\subsection*{Numerical Comparisons}
It should be emphasized that, although Chebyshev's Inequality proves the Law of Large
Numbers, it is actually a very crude inequality for the probabilities involved.  However,
its strength lies in the fact that it is true for any random variable at all, and it allows
us to prove a very powerful theorem.
\par
In the following example, we compare the estimates given by Chebyshev's Inequality with the
actual values.
\begin{example}
Let $X_1$,~$X_2$, \dots,~$X_n$ be a Bernoulli trials process with
probability~.3 for success and~.7 for failure.  Let $X_j = 1$ if the $j$th
outcome is a success and~0 otherwise.  Then, $E(X_j) = .3$ and $V(X_j) =
(.3)(.7) = .21$.  If
$$
A_n = \frac {S_n}n = \frac {X_1 + X_2 +\cdots+ X_n}n
$$
is the \emx {average} of the $X_i$, then $E(A_n) = .3$ and $V(A_n) =
V(S_n)/n^2 = .21/n$.  Chebyshev's Inequality states that if, for example,
$\epsilon = .1$,
$$
P(|A_n - .3| \geq .1) \leq \frac {.21}{n(.1)^2} = \frac {21}n\ .
$$
Thus, if $n = 100$,
$$
P(|A_{100} - .3| \geq .1) \leq .21\ ,
$$
or if $n = 1000$,
$$
P(|A_{1000} - .3| \geq .1) \leq .021\ .
$$
These can be rewritten as
 \begin{eqnarray*}
P(.2 < A_{100} < .4) &\geq& .79\ , \\
P(.2 < A_{1000} < .4) &\geq& .979\ .
\end{eqnarray*}
These values should be compared with the actual values, which are (to six decimal
places)
\begin{eqnarray*}
P(.2 < A_{100} < .4) &\approx& .962549 \\
P(.2 < A_{1000} < .4) &\approx& 1\ .\\
\end{eqnarray*}
The program {\bf Law}\index{Law (program)} can be used to carry out the above calculations in a
systematic way.
\end{example}

\subsection*{Historical Remarks}
The Law of Large Numbers was first proved by the Swiss mathematician James
Bernoulli\index{BERNOULLI, J.|(} in the fourth part of his work \emx {Ars Conjectandi} published
posthumously in~1713.\footnote{J. Bernoulli, \emx {The Art of Conjecturing
IV,} trans.~Bing Sung, Technical Report No.~2, Dept.\ of Statistics, Harvard
Univ., 1966}  As often happens with a first proof, Bernoulli's proof was much
more difficult than the proof we have presented using Chebyshev's inequality.
Chebyshev developed his inequality to prove a general form of the Law of Large
Numbers (see Exercise~\ref{exer 8.1.13}).  The inequality itself appeared
much earlier in a work by Bienaym\'e,\index{BIENAYM\'E, I.} and in discussing its history
Maistrov\index{MAISTROV, L.} remarks that it was referred to as the Bienaym\'e-Chebyshev
Inequality for a long time.\footnote{L. E. Maistrov, \emx {Probability Theory: A Historical
Approach,} trans.\ and ed.~Samual Kotz, (New York: Academic Press, 1974),
p.~202}

In \emx {Ars Conjectandi} Bernoulli provides his reader with a long discussion
of the meaning of his theorem with lots of examples.  In modern notation he has
an event that occurs with probability~$p$ but he does not know $p$.  He wants
to estimate $p$ by the fraction $\bar{p}$ of the times the event occurs
when the experiment is repeated a number of times.  He discusses in detail the
problem of estimating, by this method, the proportion of white balls in an urn
that contains an unknown number of white and black balls.  He would do this by
drawing a sequence of balls from the urn, replacing the ball drawn after each
draw, and estimating the unknown proportion of white balls in the urn by the
proportion of the balls drawn that are white.  He shows that, by choosing $n$
large enough he can obtain any desired accuracy and reliability for the
estimate.  He also provides a lively discussion of the applicability of his
theorem to estimating the probability of dying of a particular disease, of
different kinds of weather occurring, and so forth.

In speaking of the number of trials necessary for making a judgement, Bernoulli
observes that the ``man on the street" believes the ``law of averages."

\begin{quote}
Further, it cannot escape anyone that for judging in this way about any event
at all, it is not enough to use one or two trials, but rather a great number of
trials is required.  And sometimes the stupidest man---by some instinct of
nature \emx {per se} and by no previous instruction (this is truly amazing)---
knows for sure that the more observations of this sort that are taken, the less
the danger will be of straying from the mark.\footnote{Bernoulli, op.\ cit., p.~38.}
\end{quote}

\noindent But he goes on to say that he must contemplate another possibility.

\begin{quote}
Something futher must be contemplated here which perhaps no one has thought
about till now.  It certainly remains to be inquired whether after the number
of observations has been increased, the probability is increased of attaining
the true ratio between the number of cases in which some event can happen and
in which it cannot happen, so that this probability finally exceeds any given
degree of certainty; or whether the problem has, so to speak, its own
asymptote---that is, whether some degree of certainty is given which one can
never exceed.\footnote{ibid., p.~39.}
\end{quote}

\noindent Bernoulli recognized the importance of this theorem, writing:

\begin{quote}
Therefore, this is the problem which I now set forth and make known after I
have already pondered over it for twenty years.  Both its novelty and its very
great usefullness, coupled with its just as great difficulty, can exceed in
weight and value all the remaining chapters of this thesis.\footnote{ibid.,
p.~42.}
\end{quote}

\noindent Bernoulli concludes his long proof with the remark:

\begin{quote}
Whence, finally, this one thing seems to follow: that if observations of all
events were to be continued throughout all eternity, (and hence the ultimate
probability would tend toward perfect certainty), everything in the world would
be perceived to happen in fixed ratios and according to a constant law of
alternation, so that even in the most accidental and fortuitous occurrences we
would be bound to recognize, as it were, a certain necessity and, so to speak,
a certain fate.

I do now know whether Plato wished to aim at this in his doctrine of the
universal return of things, according to which he predicted that all things
will return to their original state after countless ages have
past.\footnote{ibid., pp.~65--66.}
\end{quote}\index{BERNOULLI, J.|)}

\exercises
\begin{LJSItem}

\i\label{exer 8.1.1} A fair coin is tossed 100 times.  The expected number of
heads is~50, and the standard deviation for the number of heads is $(100 \cdot
1/2 \cdot 1/2)^{1/2} = 5$.  What does Chebyshev's Inequality tell you about the
probability that the number of heads that turn up deviates from the expected
number 50 by three or more standard deviations (i.e., by at least 15)?

\i\label{exer 8.1.100} Write a program that uses the function
$\mbox {binomial}(n,p,x)$ to compute the exact probability that you estimated in
Exercise~\ref{exer 8.1.1}.  Compare the two results.

\i\label{exer 8.1.101} Write a program to toss a coin 10{,}000 times.  Let $S_n$ be the
number of heads in the first $n$ tosses.  Have your program print out, after every
1000 tosses, $S_n - n/2$.  On the basis of this simulation, is it correct to
say that you can expect heads about half of the time when you toss a coin a
large number of times?

\i\label{exer 8.1.102} A 1-dollar bet on craps has an expected winning of $-.0141$.  What
does the Law of Large Numbers say about your winnings if you make a large number of
1-dollar bets at the craps table?  Does it assure you that your losses will be
small?  Does it assure you that if $n$ is very large you will lose?

\i\label{exer 8.1.103} Let $X$ be a random variable with $E(X) =0$ and $V(X) = 1$.  What
integer value~$k$ will assure us that $P(|X| \geq k) \leq .01$?

\i\label{exer 8.1.6} Let $S_n$ be the number of successes in $n$ Bernoulli
trials with probability~$p$ for success on each trial.  Show, using Chebyshev's
Inequality, that for any $\epsilon > 0$
$$
P\left( \left| \frac {S_n}n - p \right| \geq \epsilon \right) \leq \frac {p(1 -
p)}{n\epsilon^2}\ .
$$

\i\label{exer 8.1.7} Find the maximum possible value for $p(1 - p)$ if $0 < p
< 1$.  Using this result and Exercise~\ref{exer 8.1.6}, show that the estimate
$$
P\left( \left| \frac {S_n}n - p \right| \geq \epsilon \right) \leq \frac
1{4n\epsilon^2}
$$
is valid for any $p$.

\i\label{exer 8.1.104} A fair coin is tossed a large number of times.  Does the Law of Large
Numbers assure us that, if $n$ is large enough, with $\mbox {probability} >
.99$ the number of heads that turn up will not deviate from $n/2$ by more than
100?

\i\label{exer 8.1.105} In Exercise~\ref{sec 6.2}.\ref{exer 6.2.16}, you showed that, for the
hat check problem, the number $S_n$ of people who get their own hats back has $E(S_n) =
V(S_n) = 1$.  Using Chebyshev's Inequality, show that $P(S_n \geq 11) \leq .01$
for any $n \geq 11$.

\i\label{exer 8.1.106} Let $X$ by any random variable which takes on values 0,~1,~2,
\dots,~$n$ and has $E(X) = V(X) = 1$.  Show that, for any positive integer $k$,
$$
P(X \geq k + 1) \leq \frac 1{k^2}\ .
$$

\i\label{exer 8.1.107} We have two coins: one is a fair coin and the other is a coin that
produces heads with probability 3/4.  One of the two coins is picked at random,
and this coin is tossed $n$ times.  Let $S_n$ be the number of heads that turns
up in these $n$ tosses.  Does the Law of Large Numbers allow us to predict the
proportion of heads that will turn up in the long run?  After we have observed
a large number of tosses, can we tell which coin was chosen?  How many tosses
suffice to make us 95~percent sure?

\i\label{exer 8.1.13} (Chebyshev\index{CHEBYSHEV, P. L.}\footnote{P. L. Chebyshev, ``On Mean
Values," \emx {J.\ Math.\ Pure.\ Appl.,} vol.~12 (1867), pp.~177--184.})  Assume
that $X_1$,~$X_2$, \dots,~$X_n$ are independent random variables with possibly
different distributions and let $S_n$ be their sum.  Let $m_k = E(X_k)$,
$\sigma_k^2 = V(X_k)$, and $M_n = m_1 + m_2 +\cdots+ m_n$.  Assume that
$\sigma_k^2 < R$ for all~$k$.  Prove that, for any $\epsilon > 0$,
$$
P\left( \left| \frac {S_n}n - \frac {M_n}n \right| < \epsilon \right) \to 1
$$
as $n \rightarrow \infty$.

\i\label{exer 8.1.108}  A fair coin is tossed repeatedly.  Before each toss, you are allowed
to decide whether to bet on the outcome.  Can you describe a betting system with infinitely
many bets which will enable you, in the long run, to win more than half of your bets?
(Note that we are disallowing a betting system that says to bet until you are ahead, then
quit.) Write a computer program that implements this betting system.  As stated above, your
program must decide whether to bet on a particular outcome before that outcome is determined.
For example, you might select only outcomes that come after
there have been three tails in a row.  See if you can get more than 50\% heads
by your ``system."

\istar\label{exer 8.1.109} Prove the following analogue of Chebyshev's Inequality:
$$
P(|X - E(X)| \geq \epsilon) \leq \frac 1\epsilon E(|X - E(X)|)\ .
$$

\istar\label{exer 8.1.16} We have proved a theorem often called the ``Weak Law of
Large Numbers."  Most people's intuition and our computer simulations suggest
that, if we toss a coin a sequence of times, the proportion of heads will really
approach 1/2; that is, if $S_n$ is the number of heads in $n$ times, then we
will have
$$
A_n = \frac {S_n}n \to \frac 12
$$
as $n \to \infty$.  Of course, we cannot be sure of this since we are not able
to toss the coin an infinite number of times, and, if we could, the coin could
come up heads every time.  However, the ``Strong Law of Large Numbers,"\index{Strong Law of
Large\\ Numbers} proved in more advanced courses, states that
$$
P\left( \frac {S_n}n \to \frac 12 \right) = 1\ .
$$
Describe a sample space $\Omega$ that would make it possible for us to talk
about the event
$$
E = \left\{\, \omega : \frac {S_n}n \to \frac 12\, \right\}\ .
$$
Could we assign the equiprobable measure to this space?
\choice{}{(See
Example~\ref{exam 2.2.12}.)}

\istar\label{exer 8.1.16.5}
In this exercise, we shall construct an example of a sequence of random
variables that satisfies the weak law of large numbers, but not the strong
law. The distribution of $X_i$ will have to depend on $i$, because
otherwise both laws would be satisfied.  (This problem was communicated to us
by David Maslen.)
\vskip .1in
Suppose we have an infinite sequence of mutually independent events $A_1,
A_2, \ldots$. Let $a_i = P(A_i)$, and let $r$ be a positive integer.

\begin{enumerate}
\item Find an expression of the probability that none of the $A_i$ with
$i>r$ occur.

\item Use the fact that $x-1 \leq e^{-x}$ to show that
$$
P(\mbox{No\ $A_i$\ with\ $i > r$\ occurs}) \leq e^{-\sum_{i=r}^{\infty}
a_i}
$$
\item (The first Borel-Cantelli lemma)  Prove that if $\sum_{i=1}^{\infty} a_i$ diverges, then
$$
P(\mbox{infinitely\ many\ $A_i$\ occur}) = 1.
$$
\vskip .1in
\noindent
Now, let $X_i$ be a sequence of mutually independent random variables such
that for each positive integer $i \geq 2$,
$$
P(X_i = i) = \frac{1}{2i\log i}, \quad P(X_i = -i) =
\frac{1}{2i\log i}, \quad P(X_i =0) = 1 - \frac{1}{i \log i}.
$$
When $i=1$ we let $X_i=0$ with probability $1$.  As usual we let $S_n = X_1
+ \cdots + X_n$.  Note that the mean of each $X_i$ is $0$.
\item Find the variance of $S_n$.
\item Show that the sequence $\langle X_i \rangle$ satisfies the Weak
Law of Large Numbers, i.e. prove that for any $\epsilon > 0$
$$
P\biggl(\biggl|{\frac{S_n}{n}}\biggr| \geq \epsilon\biggr) \rightarrow 0\ ,
$$
as $n$ tends to infinity.
\vskip .1in
\noindent
We now show that $\{ X_i \}$ does not satisfy the Strong Law of
Large Numbers.  Suppose that $S_n / n \rightarrow 0$. Then because
$$
\frac{X_n}{n} = \frac{S_n}{n} - \frac{n-1}{n} \frac{S_{n-1}}{n-1}\ ,
$$
we know that $X_n / n \rightarrow 0$. From the definition of limits, we
conclude that the inequality $|X_i| \geq \frac{1}{2} i$ can only be
true for finitely many $i$.
\item Let $A_i$ be the event $|X_i| \geq \frac{1}{2} i$. Find
$P(A_i)$. Show that $\sum_{i=1}^{\infty} P(A_i)$ diverges (use the
Integral Test).
\item Prove that $A_i$ occurs for infinitely many $i$.
\item Prove that
$$
P\biggl(\frac{S_n}{n} \rightarrow 0\biggr) = 0,
$$
and hence that the Strong Law of Large Numbers fails for the sequence
$\{ X_i \}$.

\end{enumerate}
\istar\label{exer 8.1.110} Let us toss a biased coin that comes up heads with probability~$p$
and assume the validity of the Strong Law of Large Numbers as described
in Exercise~\ref{exer 8.1.16}.  Then, with probability~1, $$
\frac {S_n}n \to p
$$
as $n \to \infty$.  If $f(x)$ is a continuous function on the unit interval,
then we also have
$$
f\left( \frac {S_n}n \right) \to f(p)\ .
$$

Finally, we could hope that
$$
E\left(f\left( \frac {S_n}n \right)\right) \to E(f(p)) = f(p)\ .
$$
Show that, if all this is correct, as in fact it is, we would have proven that
any continuous function on the unit interval is a limit of polynomial
functions.  This is a sketch of a probabilistic proof of an important theorem
in mathematics called the \emx {Weierstrass approximation theorem.}\index{Weierstrass
Approximation Theorem}
\end{LJSItem}

\choice{}{\section[Continuous Random Variables]{Law of Large Numbers for Continuous Random
Variables}
\label{sec 8.2}
In the previous section we discussed in some detail the Law of Large Numbers
for discrete probability distributions.  This law has a natural analogue for
continuous probability distributions, which we consider somewhat more briefly
here.

\subsection*{\bf Chebyshev Inequality} \hfill\break\index{Chebyshev Inequality}
Just as in the discrete case, we begin our discussion with the Chebyshev
Inequality.

\begin{theorem}{\bf (Chebyshev Inequality)} \index{Chebyshev Inequality}
Let $X$ be a continuous random variable with density function $f(x)$.  Suppose $X$ has a
finite expected value $\mu = E(X)$ and finite variance $\sigma^2 = V(X)$.  Then for any
positive number $\epsilon > 0$ we have
$$
P(|X - \mu| \geq \epsilon) \leq \frac {\sigma^2}{\epsilon^2}\ .
$$
\end{theorem}

The proof is completely analogous to the proof in the discrete case, and we omit it.
\par
Note that this theorem says nothing if $\sigma^2 = V(X)$ is infinite.

\begin{example}
Let $X$ be any continuous random variable with $E(X) = \mu$ and $V(X) =
\sigma^2$.  Then, if $\epsilon = k\sigma = k$ standard deviations
for some integer~$k$, then
$$
P(|X - \mu| \geq k\sigma) \leq \frac {\sigma^2}{k^2\sigma^2} = \frac 1{k^2}\ ,
$$
just as in the discrete case.
\end{example}

\subsection*{Law of Large Numbers}
With the Chebyshev Inequality we can now state and prove the Law of Large
Numbers for the continuous case.

\begin{theorem}{\bf (Law of Large Numbers)}\index{Law of Large Numbers}
Let $X_1$,~$X_2$, \dots,~$X_n$ be an independent trials process with a
continuous density function~$f$, finite expected value~$\mu$, and finite
variance~$\sigma^2$.  Let $S_n = X_1 + X_2 +\cdots+ X_n$ be the sum of the
$X_i$.  Then for any real number $\epsilon > 0$ we have
$$
\lim_{n \to \infty} P\left( \left| \frac {S_n}n - \mu \right| \geq \epsilon
\right) = 0\ ,
$$
or equivalently,
$$
\lim_{n \to \infty} P\left( \left| \frac {S_n}n - \mu \right| < \epsilon
\right) = 1\ .
$$
\end{theorem}

Note that this theorem is not necessarily true if $\sigma^2$ is infinite
(see Example~\ref{exam 8.2.5}).

As in the discrete case, the Law of Large Numbers says that the average value
of $n$ independent trials tends to the expected value as $n \to \infty$, in the
precise sense that, given $\epsilon > 0$, the probability that the average
value and the expected value differ by more than $\epsilon$ tends to~0 as $n
\to \infty$.

Once again, we suppress the proof, as it is identical to the proof in the discrete case.
\subsection*{Uniform Case}
\begin{example}
Suppose we choose at random $n$ numbers from the interval $[0,1]$ with
uniform distribution.  Then if $X_i$ describes the $i$th choice, we have
\begin{eqnarray*}
          \mu & = & E(X_i) = \int_0^1 x\, dx = \frac 12\ , \\
     \sigma^2 & = & V(X_i) = \int_0^1 x^2\, dx - \mu^2 \\
              & = & \frac 13 - \frac 14 = \frac 1{12}\ .
\end{eqnarray*}
Hence,
\begin{eqnarray*}
E \left( \frac {S_n}n \right)  & = & \frac 12\ , \\
V \left( \frac {S_n}n \right)  & = & \frac 1{12n}\ ,
\end{eqnarray*}
and for any $\epsilon > 0$,
$$
P \left( \left| \frac {S_n}n - \frac 12 \right| \geq \epsilon \right) \leq \frac
1{12n \epsilon^2}\ .
$$

This says that if we choose $n$ numbers at random from $[0,1]$, then the
chances are better than $1 - 1/(12n\epsilon^2)$ that the difference $|S_n/n -
1/2|$ is less than~$\epsilon$.  Note that $\epsilon$ plays the role of the
amount of error we are willing to tolerate: If we choose $\epsilon = 0.1$, say,
then the chances that $|S_n/n - 1/2|$ is less than~0.1 are better than $1 -
100/(12n)$.  For $n = 100$, this is about .92, but if $n = 1000$, this is better
than .99 and if $n = 10{,}000$, this is better than .999.
\putfig{5.0truein}{PSfig8-2}{Illustration of Law of Large Numbers --- uniform case.}{fig 8.2}

We can illustrate what the Law of Large Numbers says for this example
graphically.  The density for $A_n = S_n/n$ is determined by
$$
f_{A_n}(x) = nf_{S_n}(nx)\ .
$$

We have seen in Section~\ref{sec 7.2}, that we can compute the density
$f_{S_n}(x)$ for the sum of $n$ uniform random variables.  In Figure~\ref{fig 8.2} we have
used this to plot the density for $A_n$ for various values of~$n$.  We have
shaded in the area for which $A_n$ would lie between .45~and~.55.  We see that as
we increase $n$, we obtain more and more of the total area inside the shaded
region.  The Law of Large Numbers tells us that we can obtain as much of the
total area as we please inside the shaded region by choosing $n$ large enough
(see also Figure~\ref{fig 8.1}).
\end{example}

\subsection*{Normal Case}
\begin{example}
Suppose we choose $n$ real numbers at random,
using a normal distribution with mean~0 and variance~1.  Then
\begin{eqnarray*}
         \mu &=& E(X_i) = 0\ , \\
    \sigma^2 &=& V(X_i) = 1\ .
\end{eqnarray*}
Hence,
\begin{eqnarray*}
E \left( \frac {S_n}n \right) &=& 0\ , \\
V \left( \frac {S_n}n \right) &=& \frac 1n\ ,
\end{eqnarray*}
and, for any $\epsilon > 0$,
$$
P\left( \left| \frac {S_n}n - 0 \right| \geq \epsilon \right) \leq \frac
1{n\epsilon^2}\ .
$$
In this case it is possible to compare the Chebyshev estimate for $P(|S_n/n -
\mu| \geq \epsilon)$ in the Law of Large Numbers with exact values, since we
know the density function for $S_n/n$ exactly (see Example~\ref{exam 7.12}).
The comparison is shown in Table~\ref{table 8.1}, for $\epsilon = .1$.  The data
in this table was produced by the program {\bf LawContinuous}.\index{LawContinuous (program)}
\begin{table}
\centering
\begin{tabular}{r|r|r}
$n$ & $P(|S_n/n| \ge .1)$ & Chebyshev \\
\hline
100 & .31731 & 1.00000 \\
200 & .15730 & .50000 \\
300 & .08326 & .33333 \\
400 & .04550 & .25000 \\
500 & .02535 & .20000 \\
600 & .01431 & .16667 \\
700 & .00815 & .14286 \\
800 & .00468 & .12500 \\
900 & .00270 & .11111 \\
1000 & .00157 & .10000 \\
\hline
\end{tabular}
\caption{Chebyshev estimates.}
\label{table 8.1}
\end{table}
We see here that the Chebyshev estimates are in general \emx {not} very
accurate.
\end{example}

\subsection*{Monte Carlo Method}
Here is a somewhat more interesting example.

\begin{example}
Let $g(x)$ be a continuous function defined for $x \in [0,1]$ with values in
$[0,1]$.  In Section~\ref{sec 2.1}, we showed how to estimate the area of
the region under the graph of $g(x)$ by the Monte Carlo method, that is, by
choosing a large number of random values for $x$~and~$y$ with uniform
distribution and seeing what fraction of the points $P(x,y)$ fell inside the
region under the graph (see Example~\ref{exam 2.1.2}).
\par
Here is a better way to estimate the same area (see Figure~\ref{fig 8.3}).  Let us choose a
large number of independent values $X_n$ at random from $[0,1]$ with uniform
density, set $Y_n = g(X_n)$, and find the average value of the $Y_n$.  Then
this average is our estimate for the area.  To see this, note that if the
density function for~$X_n$ is uniform,
\begin{eqnarray*}
\mu & = & E(Y_n) = \int_0^1 g(x) f(x)\, dx \\
    & = & \int_0^1 g(x)\, dx \\
    & = & \mbox {average\ value\ of\ }  g(x)\ ,
\end{eqnarray*}
while the variance is
$$
\sigma^2 = E((Y_n - \mu)^2) = \int_0^1 (g(x) - \mu)^2\, dx < 1\ ,
$$
since for all $x$ in $[0, 1]$, $g(x)$ is in $[0, 1]$, hence $\mu$ is in $[0, 1]$, and
so $|g(x) - \mu| \le 1$.  Now let $A_n = (1/n)(Y_1 + Y_2 +\cdots+ Y_n)$.  Then by Chebyshev's
Inequality, we have
$$
P(|A_n - \mu| \geq \epsilon) \leq \frac {\sigma^2}{n\epsilon^2} < \frac
1{n\epsilon^2}\ .
$$

\putfig{3truein}{PSfig8-3}{Area problem.}{fig 8.3}

This says that to get within $\epsilon$ of the true value for $\mu = \int_0^1
g(x)\, dx$ with probability at least $p$, we should choose $n$ so that
$1/n\epsilon^2 \leq 1 - p$ (i.e., so that $n \geq 1/\epsilon^2(1 - p)$).  Note
that this method tells us how large to take $n$ to get a desired accuracy.
\end{example}

The Law of Large Numbers requires that the variance $\sigma^2$ of the original
underlying density be finite: $\sigma^2 < \infty$.  In cases where this fails
to hold, the Law of Large Numbers may fail, too.  An example follows.

\subsection*{Cauchy Case}
\begin{example}\label{exam 8.2.5}
Suppose we choose $n$ numbers from $(-\infty,+\infty)$ with a Cauchy density
with parameter $a = 1$.  We know that for the Cauchy density the expected value
and variance are undefined (see Example~\ref{exam 6.23}).  In this case, the
density function for
$$
A_n = \frac {S_n}n
$$
is given by (see Example~\ref{exam 7.9})
$$
f_{A_n}(x) = \frac 1{\pi(1 + x^2)}\ ,
$$
that is, \emx {the density function for $A_n$ is the same for all $n$.}  In this
case, as $n$ increases, the density function does not change at all, and the
Law of Large Numbers does not hold.
\end{example}

\exercises
\begin{LJSItem}

\i\label{exer 8.2.1} Let $X$ be a continuous random variable with mean $\mu =
10$ and variance $\sigma^2 = 100/3$.  Using Chebyshev's Inequality, find an upper
bound for the following probabilities.
\begin{enumerate}
\item $P(|X - 10| \geq 2)$.

\item $P(|X - 10| \geq 5)$.

\item $P(|X - 10| \geq 9)$.

\item $P(|X - 10| \geq 20)$.
\end{enumerate}

\i\label{exer 8.2.2} Let $X$ be a continuous random variable with values
unformly distributed over the interval $[0,20]$.
\begin{enumerate}
\item Find the mean and variance of $X$.

\item Calculate $P(|X - 10| \geq 2)$, $P(|X - 10| \geq 5)$, $P(|X - 10| \geq
9)$, and $P(|X - 10| \geq 20)$ exactly.  How do your answers compare with those
of Exercise~\ref{exer 8.2.1}?  How good is Chebyshev's Inequality in this case?
\end{enumerate}

\i\label{exer 8.2.3} Let $X$ be the random variable of Exercise~\ref{exer
8.2.2}.
\begin{enumerate}
\item Calculate the function $f(x) = P(|X - 10| \geq x)$.

\item Now graph the function $f(x)$, and on the same axes, graph the
Chebyshev function $g(x) = 100/(3x^2)$.  Show that $f(x) \leq g(x)$ for all~$x >
0$, but that $g(x)$ is not a very good approximation for~$f(x)$.
\end{enumerate}

\i\label{exer 8.2.100} Let $X$ be a continuous random variable with values exponentially
distributed over $[0,\infty)$ with parameter $\lambda = 0.1$.
\begin{enumerate}
\item Find the mean and variance of $X$.

\item Using Chebyshev's Inequality, find an upper bound for the following
probabilities: $P(|X - 10| \geq 2)$, $P(|X - 10| \geq 5)$, $P(|X - 10| \geq
9)$, and $P(|X - 10| \geq 20)$.

\item Calculate these probabilities exactly, and compare with the bounds in
(b).
\end{enumerate}
\i\label{exer 8.2.101} Let $X$ be a continuous random variable with values normally
distributed over $(-\infty,+\infty)$ with mean $\mu = 0$ and variance $\sigma^2 = 1$.
\begin{enumerate}
\item Using Chebyshev's Inequality, find upper bounds for the following
probabilities: $P(|X| \geq 1)$, $P(|X| \geq 2)$, and $P(|X| \geq 3)$.

\item The area under the normal curve between $-1$~and~1 is .6827, between
$-2$~and~2 is .9545, and between $-3$~and~3 it is .9973 (see the table in
Appendix~A).  Compare your bounds in (a) with
these exact values.  How good is Chebyshev's Inequality in this case?
\end{enumerate}

\i\label{exer 8.2.102} If $X$ is normally distributed, with mean~$\mu$ and
variance~$\sigma^2$, find an upper bound for the following probabilities, using Chebyshev's
Inequality.
\begin{enumerate}
\item $P(|X - \mu| \geq \sigma)$.

\item $P(|X - \mu| \geq 2\sigma)$.

\item $P(|X - \mu| \geq 3\sigma)$.

\item $P(|X - \mu| \geq 4\sigma)$.
\end{enumerate}
\noindent Now find the exact value using the program {\bf NormalArea}\index{NormalArea (program)}
or the normal table in Appendix~A, and compare.

\i\label{exer 8.2.103} If $X$ is a random variable with mean~$\mu \ne 0$ and variance~$\sigma^2$,
define the \emx {relative deviation} $D$ of $X$ from its mean by
$$
D = \left| \frac {X - \mu}\mu \right|\ .
$$
\begin{enumerate}
\item Show that $P(D \geq a) \leq \sigma^2/(\mu^2a^2)$.

\item If $X$ is the random variable of Exercise~\ref{exer 8.2.1}, find an
upper bound for $P(D \geq .2)$, $P(D \geq .5)$, $P(D \geq .9)$, and $P(D \geq
2)$.
\end{enumerate}

\i\label{exer 8.2.104} Let $X$ be a continuous random variable and define the {\em
standardized version} $X^*$ of~$X$ by:
$$
X^* = \frac {X - \mu}\sigma\ .
$$
\begin{enumerate}
\item Show that $P(|X^*| \geq a) \leq 1/a^2$.

\item If $X$ is the random variable of Exercise~\ref{exer 8.2.1}, find
bounds for $P(|X^*| \geq 2)$, $P(|X^*| \geq 5)$, and $P(|X^*| \geq 9)$.
\end{enumerate}

\i\label{exer 8.2.105}
\begin{enumerate}
\item Suppose a number $X$ is chosen at random from $[0,20]$ with uniform
probability.  Find a lower bound for the probability that $X$ lies between
8~and~12, using Chebyshev's Inequality.

\item Now suppose 20 real numbers are chosen independently from $[0,20]$
with uniform probability.  Find a lower bound for the probability that their
average lies between 8~and~12.

\item Now suppose 100 real numbers are chosen independently from
$[0,20]$.  Find a lower bound for the probability that their average lies
between 8~and~12.
\end{enumerate}

\i\label{exer 8.2.106} A student's score on a particular calculus final is a random variable
with values of $[0,100]$, mean~70, and variance~25.
\begin{enumerate}
\item Find a lower bound for the probability that the student's score will
fall between 65~and~75.

\item If 100 students take the final, find a lower bound for the probability
that the class average will fall between 65~and~75.
\end{enumerate}

\i\label{exer 8.2.107} The Pilsdorff beer company runs a fleet of trucks along the 100~mile
road from Hangtown to Dry Gulch, and maintains a garage halfway in between.
Each of the trucks is apt to break down at a point $X$~miles from Hangtown,
where $X$ is a random variable uniformly distributed over $[0,100]$.
\begin{enumerate}
\item Find a lower bound for the probability $P(|X - 50| \leq 10)$.

\item Suppose that in one bad week, 20 trucks break down.  Find a lower bound
for the probability $P(|A_{20} - 50| \leq 10)$, where $A_{20}$ is the average
of the distances from Hangtown at the time of breakdown.
\end{enumerate}

\i\label{exer 8.2.12} A share of common stock in the Pilsdorff beer company
has a price $Y_n$ on the $n$th business day of the year.  Finn observes that the
price change $X_n = Y_{n + 1} - Y_n$ appears to be a random variable with mean
$\mu = 0$ and variance $\sigma^2 =1/4$.  If $Y_1 = 30$, find a lower bound for
the following probabilities, under the assumption that the $X_n$'s are mutually independent.
\begin{enumerate}
\item $P(25 \leq Y_2 \leq 35)$.

\item $P(25 \leq Y_{11} \leq 35)$.

\item $P(25 \leq Y_{101} \leq 35)$.
\end{enumerate}

\i\label{exer 8.2.108} Suppose one hundred numbers $X_1$,~$X_2$, \dots,~$X_{100}$ are chosen
independently at random from $[0,20]$.  Let $S = X_1 + X_2 +\cdots+ X_{100}$
be the sum, $A = S/100$ the average, and $S^* = (S - 1000)/(10/\sqrt3)$ the
standardized sum.  Find lower bounds for the probabilities
\begin{enumerate}
\item $P(|S - 1000| \leq 100)$.

\item $P(|A - 10| \leq 1)$.

\item $P(|S^*| \leq \sqrt3)$.
\end{enumerate}

\i\label{exer 8.2.14} Let $X$ be a continuous random variable normally
distributed on $(-\infty,+\infty)$ with mean~0 and variance~1.  Using the normal
table provided in Appendix~A, or the program {\bf NormalArea},
find values for the function $f(x) = P(|X| \geq x)$ as $x$ increases from
0~to~4.0 in steps of~.25.  Note that for $x \geq 0$ the table gives
$ NA(0,x) = P(0 \leq X \leq x)$ and thus $P(|X| \geq x) = 2(.5 - NA(0,x)$.
Plot by hand
the graph of~$f(x)$ using these values, and the graph of the Chebyshev function
$g(x) = 1/x^2$, and compare (see Exercise~\ref{exer 8.2.3}).

\i\label{exer 8.2.109} Repeat Exercise~\ref{exer 8.2.14}, but this time with mean~10 and
variance~3.  Note that the table in Appendix~A presents values for
a standard normal variable.  Find the standardized version $X^*$ for~$X$, find
values for $f^*(x) = P(|X^*| \geq x)$ as in Exercise~\ref{exer 8.2.14}, and
then rescale these values for $f(x) = P(|X -10| \geq x)$.  Graph and compare
this function with the Chebyshev function $g(x) = 3/x^2$.

\i\label{exer 8.2.110} Let $Z = X/Y$ where $X$~and~$Y$ have normal densities with mean~0 and
standard deviation~1.  Then it can be shown that $Z$ has a Cauchy density.
\begin{enumerate}
\item Write a program to illustrate this result by plotting a bar graph of
1000 samples obtained by forming the ratio of two standard normal outcomes.
Compare your bar graph with the graph of the Cauchy density.  Depending upon which
computer language you use, you may or may not need to tell the computer how to simulate a
normal random variable.  A method for doing this was described in Section~\ref{sec 5.2}.

\item We have seen that the Law of Large Numbers does not apply to the
Cauchy density (see Example~\ref{exam 8.2.5}).  Simulate a large number of
experiments with Cauchy density and compute the average of your results.  Do
these averages seem to be approaching a limit?  If so can you explain why this
might be?
\end{enumerate}

\i\label{exer 8.2.111} Show that, if $X \geq 0$, then $P(X \geq a) \leq E(X)/a$.

\i\label{exer 8.2.112} (Lamperti\footnote{Private communication.})
\index{LAMPERTI, J.} Let $X$ be a non-negative random
variable.  What is the best upper bound you can give for $P(X \geq a)$ if you know
\begin{enumerate}
\item $E(X) = 20$.

\item $E(X) = 20$ and $V(X) = 25$.

\item $E(X) = 20$, $V(X) = 25$, and $X$ is symmetric about its mean.
\end{enumerate}
\end{LJSItem}}


\documentclass[12pt]{article}
\usepackage{amsfonts}
\usepackage{graphicx}

\newcommand{\real}{{\rm Re}}
\newcommand{\imag}{{\rm Im}}

\begin{document}

This document explains the history of the DeltaPA class that is the basis for the {\tt rmfit -r}
Faraday Rotation Measure refinement algorithm.  Two versions of the algorithm were implemented:
%
\begin{itemize}
\item Mark I - computes the phase of the complex-valued cross-correlation between Stokes $Q+iU$ profiles
\item Mark II - computes the phase of the inverse-variance weighted cross-correlation
\end{itemize}

The Mark I algorithm was first implemented in September 2006.   On 26 July 2007, the Mark II algorithm
replaced it.   On 6 April 2019, the code was reverted to Mark I after discovering a fundamental problem
with the Mark II algorithm, as explained in Sections 2 and 3 of this document.

\section{Mean value of position angle change \\ Mark I }

This section documents the propagation of error when estimating the
change in position angle, $\Delta\Psi$, between two polarization
profiles using the Mark I version of the algorithm implemented by {\tt rmfit -r}.
%
Let
\begin{equation}
P_k = Q_k + i U_k = L_k \exp i2\Psi_k
\end{equation}
represent the linear polarization of the $k$th phase bin in one profile and
\begin{equation}
P^\prime_k = Q^\prime_k + i U^\prime_k = L^\prime_k \exp i2\Psi^\prime_k
\end{equation}
the same in the other profile.  To find the mean value of $\Delta\Psi$,
\begin{equation}
\bar\Psi = {1\over N}\sum_{k=1}^N \Psi^\prime_k - \Psi_k
\end{equation}
use the cross-correlation
\begin{equation}
Z = \sum_{k=1}^N P^*_k P^\prime_k
  = \sum_{k=1}^N L_k L^\prime_k \exp i2(\Psi^\prime_k - \Psi_k)
\end{equation}
such that
\begin{equation}
\bar\Psi
={1\over2}\tan^{-1}\left({\imag[Z]\over\real[Z]}\right).
\end{equation}
The variance of this estimate
\begin{equation}
\sigma_{\bar\Psi}^2 = \sum_{k=1}^N
\left({\partial\bar\Psi\over\partial Q_k}\right)^2 \sigma_Q^2 +
\left({\partial\bar\Psi\over\partial U_k}\right)^2 \sigma_U^2 +
\left({\partial\bar\Psi\over\partial Q^\prime_k}\right)^2 \sigma_Q^{\prime2} +
\left({\partial\bar\Psi\over\partial U^\prime_k}\right)^2 \sigma_U^{\prime2}.
\label{eqn:var_Psi}
\end{equation}
To compute the partial derivatives, let
\begin{equation}
C = \real[Z] = \sum_{k=1}^N Q_kQ^\prime_k + U_kU^\prime_k,
\label{eqn:real_Z}
\end{equation}
\begin{equation}
S = \imag[Z] = \sum_{k=1}^N Q_kU^\prime_k - Q^\prime_kU_k,
\label{eqn:imag_Z}
\end{equation}
and
\begin{equation}
T = {S\over C},
\end{equation}
so that
\begin{equation}
{\partial\bar\Psi\over\partial Q_k}
= {1\over1+T^2} \left( {U^\prime_k \over C} - {S Q^\prime_k\over C^2} \right)
= { C U^\prime_k - S Q^\prime_k \over C^2 + S^2 }
\end{equation}
Similarly
\begin{equation}
{\partial\bar\Psi\over\partial U_k}
= { -C Q^\prime_k - S U^\prime_k \over C^2 + S^2 }
\end{equation}
\begin{equation}
{\partial\bar\Psi\over\partial Q^\prime_k}
= { -C U_k - S Q_k \over C^2 + S^2 }
\end{equation}
\begin{equation}
{\partial\bar\Psi\over\partial U^\prime_k}
= { C Q_k - S U_k \over C^2 + S^2 }
\end{equation}

The above equations have been tested over a range of signal to noise ratios, as shown in
Figures~\ref{fig:mark1_error} and~\ref{fig:mark1_error_ratio}.  Everything looks good, except
down at low $S/N < 10$.

\begin{figure}
\centerline{\includegraphics[angle=-90,width=100mm]{plots/mark1_error.eps}}
\caption{\label{fig:mark1_error}
Standard deviation of 2500 RM estimates $\sigma_\mathrm{RM}$ as a function of signal-to-noise ratio $S/N$
using the Mark I implementation of RM refinement.  As expected, the noise scales as $(S/N)^{-1}$, as indicated
by the dashed line with a slope of -1.}
\end{figure}

\begin{figure}
\centerline{\includegraphics[angle=-90,width=100mm]{plots/mark1_error_ratio.eps}}
\caption{\label{fig:mark1_error_ratio}
Ratio between the mean estimated uncertainty $\epsilon_\mathrm{RM}$ and the standard deviation of 2500 RM estimates $\sigma_\mathrm{RM}$ as a function of signal-to-noise ratio $S/N$
using the Mark I implementation of RM refinement.  As expected, the ratio is approximately unity, as indicated
by the dashed line.}
\end{figure}

\section{Mean value of position angle change \\ Mark II}

This section documents the Mark II algorithm for estimating the change in
position angle, $\Delta\Psi$, and its uncertainty.  As before, let
\begin{equation}
P_k = Q_k + i U_k = L_k \exp i2\Psi_k
\end{equation}
represent the linear polarization of the $k$th phase bin in one profile and
\begin{equation}
P^\prime_k = Q^\prime_k + i U^\prime_k = L^\prime_k \exp i2\Psi^\prime_k
\end{equation}
the same in the other profile.

Consider the {\bf weighted} cross-correlation,
$\bar{Z}=\bar{C}+i\bar{S}$, where $\bar C$ and $\bar S$ are the
weighted mean values of the real and imaginary components of
\begin{equation}
z_k = c_k + i s_k
    = P^*_k P^\prime_k = L_k L^\prime_k \exp i2(\Psi^\prime_k - \Psi_k).
\end{equation}
That is, where
\begin{equation}
c_k = Q_kQ^\prime_k + U_kU^\prime_k,
\hspace{1cm}
s_k = Q_kU^\prime_k - U_kQ^\prime_k,
\label{eqn:ck_sk}
\end{equation}
and
\begin{eqnarray}
\sigma_{c_k}^2 & = &
 Q^{\prime2}_k\sigma_{Q_k}^2 + U^{\prime2}_k\sigma_{U_k}^2 +
 Q^2_k\sigma_{Q^\prime_k}^2 + U^2_k\sigma_{U^\prime_k}^2 \label{eqn:var_ck} \\
\sigma_{s_k}^2 & = &
 U^{\prime2}_k\sigma_{Q_k}^2 + Q^{\prime2}_k\sigma_{U_k}^2 +
 U^2_k\sigma_{Q^\prime_k}^2 + Q^2_k\sigma_{U^\prime_k}^2 \label{eqn:var_sk} \\
\sigma_{s_k c_k} & = &
 Q^\prime_k U^\prime_k (\sigma_{Q_k}^2 - \sigma_{U_k}^2) +
 Q_kU_k (\sigma_{U^\prime_k}^2 -\sigma_{Q^\prime_k}^2)
\label{eqn:covar_ck_sk}
\end{eqnarray}
the weighted means and their variances and covariance are given by
\begin{equation}
\bar C = \sigma_{\bar C}^2 \sum_{k=1}^N { c_k \over \sigma_{c_k}^2 }
\hspace{1cm}
\bar S = \sigma_{\bar S}^2 \sum_{k=1}^N { s_k \over \sigma_{s_k}^2 }
\label{eqn:barC_barS}
\end{equation}
and
\begin{equation}
\sigma_{\bar C}^2 = \left[ \sum_{k=1}^N{1\over\sigma_{c_k}^2} \right]^{-1}
\hspace{5mm}
\sigma_{\bar S}^2 = \left[ \sum_{k=1}^N{1\over\sigma_{s_k}^2} \right]^{-1}
\hspace{5mm}
\sigma_{\bar S \bar C} = \sum_{k=1}^N
\left[\sigma_{\bar S}\sigma_{\bar C}\over\sigma_{s_k}\sigma_{c_k}\right]^2
                         \sigma_{s_k c_k}
\label{eqn:var_covar_barC_barS}
\end{equation}

\noindent
The expectation value of $\Delta\Psi=\Psi^\prime_k - \Psi_k$ is given by
\begin{equation}
\langle\Delta\Psi\rangle
={1\over2}\tan^{-1}\left({\bar S\over\bar C}\right).
\end{equation}
and the variance of the expectation value
\begin{equation}
{\mathrm{var}}(\langle\Delta\Psi\rangle) =
{ \bar C^2\sigma_{\bar S}^2 + \bar S^2\sigma_{\bar C}^2
  + 2 \bar S\bar C \sigma_{\bar S \bar C} \over
  \left(\bar C^2 + \bar S^2\right)^2 }
\label{eqn:var_delta_Psi}
\end{equation}

The above equations have been tested over a range of signal to noise ratios, as shown in
Figures~\ref{fig:mark2_error} and~\ref{fig:mark2_error_ratio}.  These plots show that
the Mark II algorithm

\begin{enumerate}
\item slightly underestimates the uncertainty of $RM$ estimates; and
\item more importantly, fails to exploit $S/N$ as expected.
\end{enumerate}

\begin{figure}
\centerline{\includegraphics[angle=-90,width=100mm]{plots/mark2_error.eps}}
\caption{\label{fig:mark2_error}
Standard deviation of 2500 RM estimates $\sigma_\mathrm{RM}$ as a function of signal-to-noise ratio $S/N$
using the Mark II implementation of RM refinement.  The dashed line has a slope of -1, as expected when
the error scales as $(S/N)^{-1}$.  The Mark II algorithm clearly fails to produce estimates with the expected
uncertainty; at the highest $S/N$, the standard deviation is over ten times the predicted value.}
\end{figure}

\begin{figure}
\centerline{\includegraphics[angle=-90,width=100mm]{plots/mark2_error_ratio.eps}}
\caption{\label{fig:mark2_error_ratio}
Ratio between the mean estimated uncertainty $\epsilon_\mathrm{RM}$ and the standard deviation of 2500 RM estimates $\sigma_\mathrm{RM}$ as a function of signal-to-noise ratio $S/N$
using the Mark II implementation of RM refinement.  The dashed line indicates the expected ratio of unity;
the Mark II algorithm tends to underestimate uncertainty.}
\end{figure}

\section{The Problem With Mark II}

Consider the special case when the two profiles are identical,
i.e. $P_k^\prime = P_k$, with equal noise in all Stokes parameters,
i.e. $\sigma_{Q_k}=\sigma_{U_k}=\sigma_{Q^\prime_k}=\sigma_{U^\prime_k}=\sigma$.
In this case, referring to Equations~\ref{eqn:var_ck} to~\ref{eqn:covar_ck_sk},
$\sigma^2_{c_k}=\sigma^2_{s_k}=2\sigma^2 L_k^2$ and $\sigma_{s_k c_k}=0$.
%
Referring to Equation \ref{eqn:var_covar_barC_barS}, the variance of the weighted
mean cosine and sine,
\begin{equation}
\sigma_{\bar C}^2 = \sigma_{\bar S}^2 = 2\sigma^2 \left[ \sum_{k=1}^N{1\over L_k^2} \right]^{-1}.
\end{equation}
%
Substitution of the above into Equation~\ref{eqn:barC_barS} yields
\begin{equation}
\bar C = \sum_{k=1}^N { c_k \over L_k^2 } \left[ \sum_{k=1}^N { 1 \over L_k^2 } \right]^{-1}
\hspace{1cm}
\bar S = \sum_{k=1}^N { s_k \over L_k^2 } \left[ \sum_{k=1}^N { 1 \over L_k^2 } \right]^{-1}
\end{equation}

Already, the problem is apparent.  The above equations represent weighted averages in which each cross-correlation
$c_k$ and $s_k$ are weighted by the {\bf inverse} of the linearly polarized flux $L^2$.  Such a weighted average
gives less weight to the samples with greater linearly polarized flux!  This is opposite to what is intended,
and is a consqeuence of sticking too blindly to the rules of first-order linear error propagation.

Also, in the special case that $P_k^\prime = P_k$, then (referring to Equation~\ref{eqn:ck_sk})
$c_k=L^2$, $s_k=0$, $\bar S = 0$, and
\begin{equation}
\bar C = N \left[ \sum_{k=1}^N{1\over L_k^2} \right]^{-1}.
\end{equation}
%
Finally, plugging everything into Equation~\ref{eqn:var_delta_Psi} yields
\begin{equation}
{\mathrm{var}}(\langle\Delta\Psi\rangle) = 2 {\sigma^2 \over N^2} \sum_{k=1}^N{1\over L_k^2}.
\end{equation}
%
At first glance, it may seem reasonable that the variance of the weighted mean position angle change
should be inversely proportional to the square of the linearly polarized flux.  However, it is proportional
to the sum of terms that are inversely proportional to the square of the linearly polarized flux.
Adding more detectable phase bins increases $N$ (thereby decreasing the variance, as expected) but also adds more terms
to this sum (thereby {\bf increasing} the variance!)

By increasing the $S/N$, we increase each of the $L_k$ terms and decrease the variance of the weighted mean position angle change; however, we also increase the number of phase bins that are detected above the noise and increase the sum in the above equation.  This effect stops the standard deviation of RM estimates from scaling as one over $S/N$.

\subsubsection{Revisiting the Mark I Implementation}

As written, it is not possible to compute the sums in Equations~\ref{eqn:var_Psi}
through~\ref{eqn:imag_Z} in a single loop.  After grouping like terms,
%
\begin{equation}
\sigma_{\bar\Psi}^2 =
(C^2+S^2)^{-2} \left(C^2 \sigma_S^2 + S^2 \sigma_C^2 - 2 CS \sigma_{CS} \right)
\label{eqn:var_Psi_simplified}
\end{equation}
%
where
\begin{equation}
\sigma_C^2 = \sum_{k=1}^N \sigma_{c_k}^2
\hspace{5mm}
\sigma_S^2 = \sum_{k=1}^N \sigma_{s_k}^2
\hspace{5mm}
\sigma_{CS} = \sum_{k=1}^N \sigma_{s_k c_k}
\label{eqn:var_covar_C_S}
\end{equation}

\end{document}


%!TEX root = ./jctt.tex

\newcommand{\rell}{^\ell} % raise to ellth power
\newcommand{\relll}{^{\ell+1}} % raise to ell + 1 th power
\newcommand{\rellh}{^{\ell+1/2}} % raise to ell + 1/2 power

\newcommand{\paren}[1]{\left(#1\right)}
\newcommand{\br}[1]{\left[#1\right]}
\newcommand{\curl}[1]{\left\{#1\right\}}

\newcommand{\eddphi}[1]{\edd_{#1}\phi_{#1}}
\newcommand{\ALPHA}[2]{\frac{#1}{\sigma_{t,#2} h_{#2}}}

\section{The VEF Method}
\subsection{The Algorithm}
Here, we describe the VEF method for a planar geometry, fixed-source problem:
\change{
	\begin{equation}
		\mu \pderiv{\psi}{x} \paren{x, \mu} + \sigma_t(x) \psi(x,\mu) =
			\frac{\sigma_s(x)}{2} \phi(x) + \frac{Q(x)}{2} \,,
	\end{equation}
where $\mu$ is the $x$--axis cosine of the direction of neutron flow,  $\sigma_t(x)$ and $\sigma_s(x)$ are the total and scattering macroscopic cross sections, $Q(x)$ is the isotropic fixed-source and $\psi(x, \mu)$ is the angular flux,
and $\phi(x)$ is the scalar flux:
\begin{equation}
\phi(x) = \int_{-1}^1 \psi(x,\mu') \ud \mu' \,.
\end{equation}
}
Applying the Discrete Ordinates (\SN) angular discretization yields the following set of $N$ coupled, ordinary differential equations:
	\begin{equation} \label{eq:sn}
		\mu_n \dderiv{\psi_n}{x}(x) + \sigma_t(x) \psi_n(x) =
		\frac{\sigma_s(x)}{2} \phi(x) + \frac{Q(x)}{2} \,, \quad 1 \leq n \leq N \,,
	\end{equation}
where $\psi_n(x) = \psi(x, \mu_n)$ is the angular flux due to neutrons with directions in the cone defined by $\mu_n$.  The $\mu_n$ are given by an $N$-point Gauss quadrature rule such that the scalar flux is numerically integrated as follows:
	\begin{equation} \label{eq:phiquad}
		\phi(x) = \sum_{n=1}^N w_n \psi_n(x) \,,
	\end{equation}
where $w_n$ is the quadrature weight corresponding to $\mu_n$.

The VEF method begins by solving Eq.~\ref{eq:sn} while lagging the scattering source.  This is called a Source Iteration (SI),
and is represented as follows:
	\begin{equation} \label{eq:si}
		\mu_n \dderiv{}{x}\psi_n\rellh(x) + \sigma_t(x) \psi_n\rellh(x) =
		\frac{\sigma_s(x)}{2} \phi^\ell(x) + \frac{Q(x)}{2} \,, \quad 1 \leq n \leq N \,,
	\end{equation}
where $\ell$ is the iteration index.  The scalar flux used in the scattering source, $\phi\rell$, is assumed to be known either from the previous iteration or from the initial guess if $\ell=0$.  The use of a half-integral index indicates that SI is the first of a two-step iteration scheme.  If one is only doing SI without acceleration, the second step would simply be to set the final scalar flux iterate to the iterate after the source iteration:
	\begin{equation} \label{eq:siupdate}
		\phi(x)\relll = \phi(x)\rellh \,.
	\end{equation}
However, SI is slow to converge in optically thick and highly scattering systems. This is the motivation for accelerating
SI using the VEF method.

The second iterative step of the VEF method is to obtain a final ``accelerated'' iterate for
the scalar flux by solving the VEF drift-diffusion equation using angular flux shape information from the source iteration step:
\begin{equation} \label{eq:drift}
-\dderiv{}{x} \frac{1}{\sigma_t(x)} \dderiv{}{x} \bracket{\edd\rellh(x)\phi\relll(x)} + \sigma_a(x) \phi\relll(x) = Q(x) \,,
\end{equation}
where the Eddington factor is given by
\begin{equation} \label{eq:eddington}
		\edd\rellh(x) = \frac{\int_{-1}^1 \mu^2 \psi\rellh(x, \mu) \ud \mu}{\int_{-1}^1 \psi\rellh(x, \mu) \ud \mu} \, .
	\end{equation}
\change{While transport-consistent boundary conditions must be defined for Eq. \ref{eq:drift}, there is not a unique way to do this. Thus, we postpone discussion of the boundary conditions to Sec. 2.3.}
Note that the Eddington factor depends only upon the angular shape of the angular flux, and not its magnitude.  This drift-diffusion equation is derived by first taking the first two angular moments of Eq.~\ref{eq:sn}:
	\begin{subequations}
	\begin{equation} \label{eq:zero}
		\dderiv{}{x} J (x) + \sigma_a(x) \phi(x) = Q(x) \,,
	\end{equation}
	\begin{equation} \label{eq:first}
		\dderiv{}{x} \bracket{\edd(x) \phi (x)} + \sigma_t(x) J(x) = 0 \,,
	\end{equation}
	\end{subequations}
% added isotropic part since the MMS solution uses a non-isotropic source (removed JEM 11-18-17)
% added current definition since it was never defined.
\change{where $J(x)$ is the current:
 \begin{equation}
 J(x) = \int_{-1}^1 \mu' \psi(x,\mu')  \ud \mu' \,.
 \end{equation}}
Then Eq.~\ref{eq:first} is solved for $J(x)$, and this expression is then substituted into
Eq.~\ref{eq:zero}.
Performing a SI, computing the Eddington factor from the SI angular flux iterate, and then solving
the drift-diffusion equation to obtain a new scalar flux iterate completes one accelerated iteration. These iterations
are repeated until convergence of the scalar flux at step $\ell+1$ is achieved.  The fluxes at steps $\ell+1/2$ and $\ell+1$ will
individually converge, but not necessarily to each other unless the \SN and drift-diffusion equations are consistently
differenced or the spatial truncation error is negligible.

Acceleration occurs because the angular shape of the angular flux, and thus the Eddington factor, converges much faster than the scalar flux. In addition, the solution of the drift-diffusion equation includes scattering. This inclusion compensates for lagging
the scattering source in the SI step.

The VEF method allows the \SN equations and drift-diffusion equations to be solved with arbitrarily different spatial discretization methods. The following sections  present the application of the Lumped Linear Discontinuous Galerkin (LLDG) spatial discretization to the \SN equations and the constant-linear Mixed Finite-Element Method (MFEM) to the VEF drift-diffusion equation.

\subsection{Lumped Linear Discontinuous Galerkin \SN}
\begin{figure}
	\centering
	\input{figs/lldg.pdf_tex}
	\caption{The distribution of unknowns within an LLDG cell. The superscript $+$ and $-$ indicate the angular fluxes for $\mu_n>0$ and $\mu_n<0$, respectively. }
	\label{fig:lldg_grid}
\end{figure}
The spatial grid and distribution of unknowns for an LLDG cell are shown in Fig.~\ref{fig:lldg_grid}. We assume a computational domain of length $x_b$ discretized into $I$ cells. The indices for cell centers are integral and the indices for cell edges are half integral.
The two unknowns for discrete angle $\mu_n$ within cell $i$ are the left and right angular fluxes, $\psi_{n,i,L}\rellh$ and $\psi_{n,i,R}\rellh$.  The angular flux dependence within cells is linear and is given in cell $i$ by
\begin{equation} \label{eq:afdef}
\psi_{n,i}\rellh(x) = \psi_{n,i,L}\rellh B_{i,L}(x) + \psi_{n,i,R}\rellh B_{i,R}(x) \,, \quad x \in (x_{i-1/2},x_{i+1/2}),
\end{equation}
where
		\begin{subequations}
		\begin{equation}\label{eq:bfunL}
			B_{i,L}(x) = \begin{cases}
				\frac{x_{i+1/2} - x}{x_{i+1/2} - x_{i-1/2}} \,, & x \in [x_{i-1/2}, x_{i+1/2}] \\
				0 \,, & \text{otherwise}
			\end{cases} \,,
		\end{equation}
		\begin{equation}\label{eq:bfunR}
			B_{i,R}(x) = \begin{cases}
				\frac{x - x_{i-1/2}}{x_{i+1/2} - x_{i-1/2}} \,, & x \in [x_{i-1/2}, x_{i+1/2}] \\
				0 \,, & \text{otherwise}
			\end{cases} \,,
		\end{equation}
	\end{subequations}
are the LLDG basis functions.
The cell centered angular flux is the average of the left and right discontinuous edge fluxes:
	\begin{equation} \label{eq:lldg_i}
		\psi_{n,i}\rellh = \half\left(\psi_{n,i,L}\rellh + \psi_{n,i,R}\rellh\right) \,.
	\end{equation}
The interface or cell-edge fluxes are uniquely defined by upwinding:
	\begin{subequations}
	\begin{equation} \label{eq:downwind}
		\psi_{n,i-1/2}\rellh = \begin{cases}
			\psi_{n,i-1,R}\rellh \,, & \mu_n > 0 \\
			\psi_{n,i,L}\rellh \,, & \mu_n < 0
		\end{cases} \,,
	\end{equation}
	\begin{equation} \label{eq:upwind}
		\psi_{n,i+1/2}\rellh = \begin{cases}
			\psi_{n,i,R}\rellh \,, & \mu_n > 0 \\
			\psi_{n,i+1,L}\rellh \,, & \mu_n < 0
		\end{cases} \,.
	\end{equation}
	\end{subequations}
The fixed source is also assumed to be linear within each cell:
\begin{equation} \label{eq:Qdef}
Q_{i}(x) = Q_{i,L} B_{i,L}(x) + Q_{i,R} B_{i,R}(x) \,, \quad x \in [x_{i-1/2},x_{i+1/2}],
\end{equation}
Because there is no spatial derivative of the fixed source, there is no need to uniquely
define the fixed sources on the cell edges.

The unlumped Linear Discontinuous Galerkin discretization for Eq.~\ref{eq:si} is obtained by
substituting $\psi_{n,i}\rellh(x)$ from Eq.~\ref{eq:afdef} and $Q_{i}(x)$ from Eq.~\ref{eq:Qdef} into Eq.~\ref{eq:si},
sequentially multiplying the resultant equation by each basis function, and integrating over
each cell with integration by parts of the spatial derivative term.
The lumped discretization equations are obtained simply by performing all volumetric integrals (after formal integration by parts
of the spatial derivative term) using trapezoidal-rule quadrature.
The LLDG discretization of Eq.~\ref{eq:si} is given by:
	\begin{subequations}
	\begin{equation} \label{eq:lldg_l}
		\mu_n \left(\psi_{n,i}\rellh - \psi_{n, i-1/2}\rellh\right)
		+ \frac{\sigma_{t,i} h_i}{2} \psi_{n,i,L}\rellh
		= \frac{\sigma_{s,i} h_i}{4} \phi_{i,L}\rell + \frac{h_i}{4} Q_{i,L} \,,
		% 1 \leq n \leq N \,,
		% 1 \leq i \leq I\,,
	\end{equation}
	\begin{equation} \label{eq:lldg_r}
		\mu_n \left(\psi_{n,i+1/2}\rellh - \psi_{n,i}\rellh\right)
		+ \frac{\sigma_{t,i} h_i}{2} \psi_{n,i,R}\rellh
		= \frac{\sigma_{s,i} h_i}{4} \phi_{i,R}\rell + \frac{h_i}{4} Q_{i,R} \,,
		% 1 \leq n \leq N \,,
		% 1 \leq i \leq I\,,
	\end{equation}
	\end{subequations}
where $h_i$, $\sigma_{t,i}$, $\sigma_{s,i}$, and $Q_{i,L/R}$ are the cell width, total cross section, scattering cross section,
and fixed sources in cell $i$. The discontinuous scalar fluxes, $\phi_{i,L/R}\rell$, are assumed to be known from
the drift-diffusion step of the previous iteration or the initial guess when $\ell=0$. Equations \ref{eq:lldg_i}, \ref{eq:downwind}, \ref{eq:upwind}, \ref{eq:lldg_l}, and \ref{eq:lldg_r} can be combined and rewritten as
follows
	\begin{equation} \label{eq:sweepLR}
		\left[\begin{matrix}
			\mu_n + \sigma_{t,i} h_i & \mu_n  \\
			-\mu_n & \sigma_{t,i} + \mu_n \\
		\end{matrix}\right]
		\left[\begin{matrix}
			\psi_{n,i,L}\rellh \\ \psi_{n,i,R}\rellh
		\end{matrix}\right]
		= \left[\begin{matrix}
			\frac{\sigma_{s,i}h_i}{2} \phi_{i,L}\rell + \frac{h_i}{2} Q_{i,L} + 2\mu_n \psi_{n,i-1,R}\rellh \\
			\frac{\sigma_{s,i}h_i}{2} \phi_{i,R}\rell + \frac{h_i}{2} Q_{i,R}
		\end{matrix}\right] \,,
	\end{equation}
for sweeping from left to right ($\mu_n > 0$) and
	\begin{equation} \label{eq:sweepRL}
		\left[\begin{matrix}
			-\mu_n + \sigma_{t,i}h_i & \mu_n \\
			-\mu_n & -\mu_n + \sigma_{t,i}h_i \\
		\end{matrix} \right]
		\left[\begin{matrix}
			\psi_{n,i,L}\rellh \\ \psi_{n,i,R}\rellh
		\end{matrix} \right]
		= \left[\begin{matrix}
			\frac{\sigma_{s,i}h_i}{2} \phi_{i,L}\rell + \frac{h_i}{2} Q_{i,L} \\
			\frac{\sigma_{s,i}h_i}{2} \phi_{i,R}\rell + \frac{h_i}{2} Q_{i,R} - 2\mu_n \psi_{n,i+1,L}\rellh
		\end{matrix} \right]
		\,,
	\end{equation}
for sweeping from right to left ($\mu_n < 0$), respectively. The right hand sides of Eqs.~\ref{eq:sweepLR} and \ref{eq:sweepRL} are known
as the scalar flux from the previous iteration, the fixed source, and the angular flux entering from the upwind cell are all known. By supplying the flux entering the left side of the first cell, the solution for $\mu_n > 0$ can be propagated from left to right by solving Eq.~\ref{eq:sweepLR}. Similarly, supplying the incident flux on the right boundary allows the solution for $\mu_n < 0$ to be propagated from right to left with Eq.~\ref{eq:sweepRL}. The Variable Eddington Factors needed in the drift-diffusion acceleration step are computed at the cell edges as follows:
	\begin{equation} \label{lldg:edde}
		\edd\rellh_{i\pm 1/2} = \frac{
			\sum_{n=1}^N \mu_n^2 \psi_{n,i\pm 1/2}\rellh w_n
		}{
			\sum_{n=1}^N \psi_{n,i\pm 1/2}\rellh w_n
		} \,,
	\end{equation}
where the $\psi_{n,i\pm1/2}\rellh$ are defined by Eqs.~\ref{eq:downwind} and \ref{eq:upwind}. The Eddington factors are
computed within cell $i$ as follows:
\begin{equation} \label{lldg:eddi}
		\edd\rellh(x) = \frac{
			\sum_{n=1}^N \mu_n^2 \psi_{n}\rellh(x) w_n
		}{
			\sum_{n=1}^N \psi_{n}\rellh(x) w_n
		} \,, \quad x\in(x_{i-1/2},x_{i+1/2}),
	\end{equation}
where $\psi_{n}\rellh(x)$ is defined by Eq.~\ref{eq:afdef}.


\subsection{Real functions are vectors}

The real function space is a vector space because it is linear in multiplication and addition.

\(g(x)=cf(x)\)

\(h(x)=f(x)+k(x)\)


\subsubsection{Autoregressive Integrated Moving Averages}
\label{arima}

\cite{box1962}, \cite{box1968}, and more papers by the same authors in the
    1960s introduce a type of model where observations correlate with their
    neighbors and refer to them as autoregressive integrated moving average
    (ARIMA) models for stationary time series.
For a thorough overview, we refer to \cite{box2015} and \cite{brockwell2016}.

A time series $y_t$ is stationary if its moments are independent of the
    point in time where it is observed.
A typical example is a white noise $\epsilon_t$ series.
Therefore, a trend or seasonality implies non-stationarity.
\cite{kwiatkowski1992} provide a test to check the null hypothesis of
    stationary data.
To obtain a stationary time series, one chooses from several techniques:
First, to stabilize a changing variance (i.e., heteroscedasticity), one
    applies a Box-Cox transformation (e.g., $log$) as first suggested by
    \cite{box1964}.
Second, to factor out a trend (or seasonal) pattern, one computes differences
    of consecutive (or of lag $k$) observations or even differences thereof.
Third, it is also common to pre-process $y_t$ with one of the decomposition
    methods mentioned in Sub-section \ref{stl} below with an ARIMA model
    then trained on an adjusted $y_t$.

In the autoregressive part, observations are modeled as linear combinations of
    its predecessors.
Formally, an $AR(p)$ model is defined with a drift term $c$, coefficients
    $\phi_i$ to be estimated (where $i$ is an index with $0 < i \leq p$), and
    white noise $\epsilon_t$ like so:
$
AR(p): \ \
y_t = c + \phi_1 y_{t-1} + \phi_2 y_{t-2} + \dots + \phi_p y_{t-p}
      + \epsilon_t
$.
The moving average part considers observations to be regressing towards a
    linear combination of past forecasting errors.
Formally, a $MA(q)$ model is defined with a drift term $c$, coefficients
    $\theta_j$ to be estimated, and white noise terms $\epsilon_t$ (where $j$
    is an index with $0 < j \leq q$) as follows:
$
MA(q): \ \
y_t = c + \epsilon_t + \theta_1 \epsilon_{t-1} + \theta_2 \epsilon_{t-2}
      + \dots + \theta_q \epsilon_{t-q}
$.
Finally, an $ARIMA(p,d,q)$ model unifies both parts and adds differencing
    where $d$ is the degree of differences and the $'$ indicates differenced
    values:
$
ARIMA(p,d,q): \ \
y'_t = c + \phi_1 y'_{t-1} + \dots + \phi_p y'_{t-p} + \theta_1 \epsilon_{t-1}
       + \dots + \theta_q \epsilon_{t-q} + \epsilon_{t}
$.

$ARIMA(p,d,q)$ models are commonly fitted with maximum likelihood estimation.
To find an optimal combination of the parameters $p$, $d$, and $q$, the
    literature suggests calculating an information theoretical criterion
    (e.g., Akaike's Information Criterion) that evaluates the fit on
    historical data.
\cite{hyndman2008a} provide a step-wise heuristic to choose $p$, $d$, and $q$,
    that also decides if a Box-Cox transformation is to be applied, and if so,
    which one.
To obtain a one-step-ahead forecast, the above equation is reordered such
    that $t$ is substituted with $T+1$.
For forecasts further into the future, the actual observations are
    subsequently replaced by their forecasts.
Seasonal ARIMA variants exist; however, the high frequency $k$ in the kind of
    demand a UDP faces typically renders them impractical as too many
    coefficients must be estimated.


\section{Hilbert's nullstellensatz}

This is by no means the shortest proof of this theorem,
nor is it the most elegant.
I chose to do it this way because it's more involved and has a couple extra moving parts.
It was also a good excuse to look into Jacobson rings.

\begin{theorem}[Hilbert's nullstellensatz]
Let $k$ be an algebraically closed field,
$A$ be the polynomial ring $A = k[t_1, \ldots, t_n]$ and $\mathfrak{a}$ be an ideal of $A$.
Then
\begin{equation*}
    I(Z(\mathfrak{a})) = \sqrt{\mathfrak{a}}
\end{equation*}
\end{theorem}

We are going to do the proof in three smaller steps.
\begin{enumerate}
    \item Proving that ideals of the form $\mathfrak{m}_a = (t_1 - a_1, \ldots t_n - a_n)$ where $a\in k^n$ are the only maximal ideals in $A$.
    This is known as the weak Hilbert's nullstellensatz.
    \item Proving that the radical of an ideal $\mathfrak{b}$ in a finitely generated $k$-algebra $B$ is equal to the intersection of the maximal ideals in $B$ that contain $\mathfrak{b}$.
    This shows that all finitely generated algebras over a field is a Jacobson ring.
    \item Deduce the result.
\end{enumerate}

\subsection{Part 1}
\begin{lemma}
Let $\mathfrak{m}_a = (t_1 - a_1, \ldots t_n - a_n)$ where $a\in k^n$.
The ideals of this form are the only maximal ideals in $A$.
\end{lemma}
\begin{proof}
Let $a \in k^n$.
We define a the evaluation morphism as follows:
\begin{align*}
    e_{a}: k[t_1,\ldots, t_n]&\longrightarrow k \\
    f &\longmapsto f(a) .
\end{align*}
Note that it is a surjective $k$-algebra homomorphism and since $k$ is algebraically closed,
it has kernel $\mathfrak{m}_{a}$.
Let $\mathfrak{m}$ be a maximal ideal in $k[t_1, \ldots, t_n]$. Then $k[t_1, \ldots, t_n]/\mathfrak{m}$ is a finitely generated field extension of $k$.
By Zariski's lemma,
$k[t_1, \ldots, t_n]/\mathfrak{m}$ is in fact a finite field extension,
better known as a finite dimensional vector space.
Since $k$ is algebraically closed,
there is an isomorphism of $k$-algebras
\begin{equation*}
    k[t_1, \ldots, t_n]/\mathfrak{m}\longrightarrow k .
\end{equation*}
Now, let $a_i$ denote the image of $t_i$.
Then we get that $\mathfrak{m}_{a}\subseteq \mathfrak{m}$,
which implies $\mathfrak{m}_{a} = \mathfrak{m}$ since $\mathfrak{m}_{a}$ is a maximal ideal.
\end{proof}

\subsection{Part 2}
\begin{lemma}
Let $k$ be an algebraically closed field,
$B$ be a finitely generated $k$-algebra and $\mathfrak{b}$ be an ideal in $B$.
Then we have
\begin{equation*}
    \sqrt{\mathfrak{b}} = \bigcap_{\mathfrak{b}\subseteq\mathfrak{m}}\mathfrak{m}.
\end{equation*}
where $\mathfrak{m}$ are the maximal ideals in $B$.
\end{lemma}
\begin{proof}
First,
we note that the projection $\pi:B\rightarrow B/\mathfrak{b}$ induces bijections between the sets
\begin{itemize}
    \item prime ideals in $B/\mathfrak{b}$ and prime ideals in $B$ that contain $\mathfrak{b}$,
    \item maximal ideals in $B/\mathfrak{b}$ and maximal ideals in $B$ that contain $\mathfrak{b}$,
    \item radical ideals in $B/\mathfrak{b}$ and radical ideals in $B$ that contain $\mathfrak{b}$.
\end{itemize}
Hence we only need to prove the statement for $\mathfrak{b}=(0)$,
and since it is clear that $\sqrt{(0)}$ is contained in every maximal ideal because $\sqrt{(0)}$ consists of all nilpotent elements,
we only need to show that every element not contained in $\sqrt{(0)}$ is not contained in some maximal ideal.

Let $f\in B$ be non-nilpotent,
i.e. $f\in \sqrt{(0)}$.
This implies that $$B_f\cong B[t]/(ft-1)$$ is a non-trivial $k$-algebra,
hence it has a maximal ideal $\mathfrak{m}$.
Consider the morphism $\phi: B \longrightarrow B_f$.
This is a morphism of finitely generated $k$-algebras,
and by Zariski's lemma,
$k\subseteq B/\phi^{-1}(\mathfrak{m})\subseteq B_f/\mathfrak{m}$ is a finite extension,
and hence $k\subseteq B/\phi^{-1}(\mathfrak{m})$ is an integral extension.
Since $k$ is a field, it is a field itself.
This gives us that the inverse image of a maximal ideal is again a maximal ideal,
i.e. $\phi^{-1}(\mathfrak{m})$ is a maximal ideal of $B$.
But this ideal can't contain $f$.
Hence we have shown that every non-nilpotent element is not contained in all maximal ideals.
\end{proof}

\subsection{Part 3}
We now deduce the result.
\begin{proof}
Let $a \in k^n$.
First,
note that $a \in Z(\mathfrak{a})$ if and only if $\mathfrak{a}\subseteq \mathfrak{m}_{a}$.
Hence,
the maximal ideals containing $\mathfrak{a}$ is just the maximal ideals $\mathfrak{m}_{a}$ such that $a \in Z(\mathfrak{a})$.
In the second step we showed that the radical of an ideal was equal to the intersection of all maximal ideals containing it,
hence we have
\begin{equation*}
    \sqrt{\mathfrak{a}} = \bigcap_{a \in Z(\mathfrak{a})}\mathfrak{m}_{a}.
\end{equation*}
For the final part,
we have for $f\in k[t_1,\ldots,t_n]$ and $a\in k^n$ that $f(x) = 0$ if and only if $f \in \mathfrak{m}_{a}$.
Hence we have for subsets $V\subseteq k^n$ that $I(V) = \bigcap_{a \in V}\mathfrak{m}_{a}$.
And since $a \in Z(\mathfrak{a})$ if and only if $\mathfrak{a}\subseteq \mathfrak{m}_{a}$ we have finally

\begin{align*}
    I(Z(\mathfrak{a}))
    &= \bigcap_{a \in Z(\mathfrak{a})}\mathfrak{m}_{a} \\
    &= \sqrt{\mathfrak{a}}.
\end{align*}
And by that, we are done!
\end{proof}


\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[paper=a4paper, left=25mm, right=25mm, top=30mm, bottom=30mm]{geometry}
\usepackage{setspace}
\usepackage{hyperref}
\usepackage{graphicx}
\usepackage{fancyhdr}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{mathtools}

\pagestyle{fancy}
\renewcommand{\headrulewidth}{0pt}
\addtolength{\headheight}{11pt}
\title{\flushleft{\textbf{Problem Sheet I}}}
\date{}
\setcounter{secnumdepth}{0}

\newcommand\balancedeq{\stackrel{\mathclap{\tiny\mbox{balanced}}}{=}}

\begin{document}
\maketitle
\thispagestyle{fancy}

\subsubsection{3.1 LDA Derivation from the Least Squares Error}
We are looking for the global minimum of
\begin{equation}
\Delta: \mathbb{R}^{d+1} \rightarrow \mathbb{R} \quad (\mathbf{m},b) \mapsto \sum_{i=1}^N (\mathbf{w^T x_i} + b - y_i)^2 = \sum_{i=1}^N (\mathbf{x_i^T w} + b - y_i)^2
\end{equation}
First, we take a closer look at the summands. Let $i \in \{1,...,N\}$.\\
The function def. by $f(x):=x^2$ is in $C^\infty(\mathbb{R})$ with derivative $f'(x) = 2x$. For the function
\begin{equation}
g_i: \mathbb{R}^{d+1} \rightarrow \mathbb{R} \quad (\mathbf{m},b) \mapsto \mathbf{w^T x_i} + b - y_i
\end{equation}
holds for $k \in {1,...,d}$, $\mathbf{w} \in \mathbb{R}^d, b \in \mathbb{R}$:
\begin{equation}
\partial_{w_k} g_i(\mathbf{w}, b) = \partial_{w_k} \left( \sum_{j=1}^d x_{ij} w_j + b - y_i \right) = \sum_{j=1}^d x_{ij} \delta_{jk} = x_{ik}
\end{equation}
\begin{equation}
\partial_b g_i(\mathbf{w}, b) = 1
\end{equation}
The partial derivatives are continuous, thus $g_i \ in C^1(\mathbb{R^{d+1}}$
As a composition/sum of $C^1$ functions, $\Delta$ is a $C^1$ function as well and
\begin{align*}
D\Delta(\mathbf{w},b) = D\left(\sum_{i=1}^N f \circ g_i \right)(\mathbf{w},b) & = \sum_{i=1}^N Df(g_i(\mathbf{w}, b)) \cdot Dg_i(\mathbf{w}, b) \\
& \sum_{i=1}^N 2g_i(\mathbf{w}, b) \cdot (\nabla_{\mathbf{w}} g_i(\mathbf{w}, b)^T, \partial_b g_i(\mathbf{w},b)) \\
& \sum_{i=1}^N 2(\mathbf{x_i^Tw} + b - y_i) (\mathbf{x_i^T}, 1)\\
\end{align*}
\begin{align*}
\Rightarrow \nabla_{(\mathbf{w},b)} = 2 \sum_{i=1}^N (\mathbf{x_i^T w}+b-y_i) \left(
\begin{array}{c}
\mathbf{x_i^T}\\
1\\
\end{array}
\right) = \left(
\begin{array}{c}
2 \sum_{i=1}^N (\mathbf{x_i^T w} + b - y_i)  \mathbf{x_i^T}\\
2 \sum_{i=1}^N (\mathbf{x_i^T w} + b - y_i)\\
\end{array}
\right)
\end{align*}
Because $\Delta \in C^1(\mathbb{R}^{d+1})$ and global maxima in an open set are local maxima, it holds for the argmax $(\mathbf{\hat{w}}, \hat{b})$:
\begin{align*}
\nabla_{(\mathbf{w}, b)} \Delta(\mathbf{\hat{w}, \hat{b}}) = 0
\end{align*}
This implies
\begin{align*}
\partial_b \Delta(\mathbf{\hat{w}, \hat{b}}) = 0 & \Rightarrow 0 = \sum_{i=1}^N(\mathbf{x_i^T \hat{w}} + \hat{b} - y_i)\\
& \Rightarrow 0 = N\hat{b} + \sum_{i=1}^N(\mathbf{x_i^T \hat{w}} - y_i) \\
& \Rightarrow \hat{b} = \frac{1}{N} \sum_{i=1}^N(-\mathbf{x_i^T \hat{w}} + y_i) = \frac{-1}{N} \sum_{i=1}^N \mathbf{x_i^T \hat{w}} + \sum_{i: y_i=1} 1 - \sum_{i: y_i=-1}^N 1 \quad \balancedeq \quad -\frac{1}{N} \sum_{i=1}^N \mathbf{x_i^T \hat{w}}
\end{align*}
Furthermore $\Delta(\mathbf{\hat{w}, \hat{b}}) = 0$ implies
\begin{align*}
0 = \sum_{i=1}^N(\mathbf{x_i^T\hat{w}}+\hat{b}-y_i)\mathbf{x_i}
\end{align*}
We insert our result for $\hat{b}$ into this equation:
\begin{align*}
0 & = \sum_{i=1}^N \left[ \mathbf{x_i^T \hat{w}} - \frac{1}{N} \sum_{j=1}^N \mathbf{x_j^T \hat{w}} - y_i  \right] \mathbf{x_i}\\
\Rightarrow & \underbrace{\frac{1}{N} \sum_{i=1}^N y_i \mathbf{x_i}}_\text{a)} = \underbrace{-\frac{1}{N} \sum_{i=1}^N \frac{1}{N} \sum_{j=1}^N (\mathbf{x_j^T\hat{w}}) \mathbf{x_i}}_\text{b)} + \underbrace{\frac{1}{N} \sum_{i=1}^N (\mathbf{x_i^T \hat{w}})\mathbf{x_i}}_\text{c)}\\
\end{align*}
We will separately discuss the three terms a), b) and c):\\
\ \\
\textbf{a)}
\begin{align*}
\frac{1}{N} \sum_{i=1}^N y_i \mathbf{x_i} & = \frac{1}{N} \sum_{i:y_i=1}\mathbf{x_i} - \frac{1}{N} \sum_{i:y_i=-1}\mathbf{x_i}\\
& = \frac{1}{2} \left( \frac{1}{N/2} \sum_{i:y_i=1} \mathbf{x_i} - \frac{1}{N/2} \sum_{i:y_i=-1} \mathbf{x_i} \right)\\
& \balancedeq \quad \frac{1}{2} \left( \frac{1}{N_1} \sum_{i:y_i=1} \mathbf{x_i} - \frac{1}{N_2} \sum_{i:y_i=-1} \mathbf{x_i} \right)\\
& = (\mathbf{\mu_1} - \mathbf{\mu_ {-1}})/2\\
\end{align*}
\ \\
\textbf{b)}
\begin{align*}
-\frac{1}{N} \sum_{i=1} \frac{1}{N} \sum_{j=1}^N (\mathbf{x_j^T \hat{w}}) \mathbf{x_i} & = \left[ - \frac{1}{N} \sum_{i=1}^N \mathbf{x_i}\right] \left[ \left( \frac{1}{N} \sum_{j=1}^N \mathbf{x_j^T} \right) \mathbf{\hat{w}} \right] \\
& = -\left[ \left( \frac{1}{N} \sum_{i=1}^N \mathbf{x_i} \right) \left( \frac{1}{N} \sum_{j=1}^N \mathbf{x_j^T} \right) \right] \mathbf{\hat{w}}\\
& = -\left(  \left[  \left( \frac{1}{N} \sum_{i=1}^N \mathbf{x_i}y_i \right) + \left( \frac{2}{N} \sum_{i:y_i=1} \mathbf{x_i}y_i \right) \right] \left[  \left( \frac{1}{N} \sum_{j=1}^N \mathbf{x_j^T}y_j \right) + \left( \frac{2}{N} \sum_{j:y_j=1} \mathbf{x_j^T}y_j \right) \right] \right)\\
& = - (\frac{1}{2} (\mathbf{\mu_1} - \mathbf{\mu_{-1}}) + \mathbf{\mu_{-1}}) (\frac{1}{2} (\mathbf{\mu_1} - \mathbf{\mu_{-1}})^T + \mathbf{\mu_{-1}}^T) \mathbf{\hat{w}}\\
& = - \left[ \frac{1}{4} (\mathbf{\mu_1} - \mathbf{\mu_{-1}})(\mathbf{\mu_1} - \mathbf{\mu_{-1}})^T + (\mathbf{\mu_1} - \mathbf{\mu_{-1}}) \mathbf{\mu_{-1}}^T \right] \mathbf{\hat{w}}\\
& = - \left[ \frac{S_B}{4} + (\mathbf{\mu_1} - \mathbf{\mu_{-1}}) \mathbf{\mu_{-1}}^T \right] \mathbf{\hat{w}}
\end{align*}
\ \\
\textbf{c)}
\begin{align*}
\frac{1}{N} \sum_{i=1}^N (\mathbf{x_i^T \hat{w}}) \mathbf{x_i} &= \frac{1}{N} \sum_{i=1}^N (\mathbf{x_i x_i^T}) \mathbf{\hat{w}}\\
&= \frac{1}{N} \sum_{i=1}^N (\mathbf{x_i -\mu_{y_i} + \mu_{y_i}}) (\mathbf{x_i -\mu_{y_i} + \mu_{y_i}})^T \mathbf{\hat{w}}\\
& = \left[ \frac{1}{N} \sum_{i=1}^N (\mathbf{x_i -\mu_{y_i}}) (\mathbf{x_i -\mu_{y_i}})^T + \frac{2}{N} \sum_{i=1}^N (\mathbf{x_i -\mu_{y_i}}) \mathbf{\mu_{y_i}}^T + \frac{1}{N} \sum_{i=1}^N \mathbf{ \mu_{y_i}} \mathbf{\mu_{y_i}}^T \right] \mathbf{\hat{w}}\\
& = \left[ S_W + \frac{1}{N/2} \sum_{i=1}^N \mathbf{x_i \mu_{y_i}^T} - \frac{2}{N} \sum_{i=1}^N \mathbf{\mu_{y_i} \mu_{y_i}^T} + \frac{1}{N} \sum_{i=1}^N \mathbf{\mu_{y_i} \mu_{y_i}^T} \right]\mathbf{\hat{w}}\\
& = \left[ S_W + \underbrace{\frac{1}{N/2} \sum_{i:y_i=1} \mathbf{x_i \mu_{y_i}^T}}_{=\mathbf{\mu_1 \mu_{1}^T}} + \underbrace{\frac{1}{N/2} \sum_{i:y_i=-1} \mathbf{x_i \mu_{y_{-1}}^T}}_{=\mathbf{\mu_{-1} \mu_{-1}^T}}  - \mathbf{\mu_1 \mu_1^T} - \mathbf{\mu_{-1} \mu_{-1}^T} + \frac{1}{2}\mathbf{\mu_1 \mu_1^T} + \frac{1}{2} \mathbf{\mu_{-1} \mu_{-1}^T} \right]\mathbf{\hat{w}}\\
& = \left[ S_W + \frac{1}{2} (\mathbf{\mu_1 - \mu_{-1}})(\mathbf{\mu_1 - \mu_{-1}})^T + (\mathbf{\mu_{1}-\mu_{-1}})\mathbf{\mu_{1}}^T \right] \mathbf{\hat{w}}\\
& = \left[ S_W + \frac{S_B}{2} + (\mathbf{\mu_{1}-\mu_{-1}})\mathbf{\mu_{1}}^T \right] \mathbf{\hat{w}}\\
\end{align*}
Now we insert these results into the equation from last page.
\begin{align*}
(\mathbf{\mu_1 - \mu_{-1}})/2 = \left[ - \frac{S_B}{4} - (\mathbf{\mu_{1}-\mu_{-1}})\mathbf{\mu_{1}}^T + S_W + \frac{S_B}{2} + (\mathbf{\mu_{1}-\mu_{-1}})\mathbf{\mu_{1}}^T \right] \mathbf{\hat{w}} = \left[ S_W + \frac{S_B}{4} \right] \mathbf{\hat{w}}
\end{align*}
This is equivalent to
\begin{equation}
S_W \mathbf{\hat{w}} = \frac{\mathbf{\mu_1 - \mu_{-1}}}{2} + \frac{S_B}{4} \mathbf{\hat{w}}
\end{equation}
Because $\mathbb{R}^d$ is a finite dimensional vector space, we can choose $v_2, ...,v_d \in \mathbb{R}^d$ such that $\{(\mu_1-\mu_{-1}),v_2, ..., v_d\}$ is an orthonormal basis of $\mathbb{R}^d$.
Thus, we can write: $\mathbf{\hat{w}} = \lambda_1 (\mu_1-\mu_{-1}) + \sum_{i=2}^d \lambda_i v_i$ for $\lambda_1, ...,\lambda_d  \in \mathbb{R}$. This way we can show:
\begin{align*}
\frac{S_B}{4} \mathbf{\hat{w}} &= \frac{1}{4} (\mu_1-\mu_{-1})(\mu_1-\mu_{-1})^T \left( \lambda_1 (\mu_1-\mu_{-1}) + \sum_{i=2}^d \lambda_i v_i \right)\\
&= \frac{1}{4} \lambda_1 (\mu_1-\mu_{-1})(\mu_1-\mu_{-1})^T(\mu_1-\mu_{-1})\\
&= \frac{1}{4} \lambda_1 (\mu_1-\mu_{-1})||\mu_1-\mu_{-1}||^2
\end{align*}
The second equality holds because the scalar product of $\mu_1-\mu_{-1}$ and $v_i$ vanishes for all $i \in \{2, ...,d\}$ (ONB). Thus, we obtain with the equality from above and $\tau := \frac{1}{2} + \frac{1}{4}\lambda_1 ||\mu_1-\mu_{-1}||^2$:
\begin{align*}
\exists \tau \in \mathbb{R}: S_W\mathbf{\hat{w}} = \tau (\mu_1-\mu_{-1})
\end{align*}
Under the assumption that $S_W$ is invertible (which is true if $(x_i)$ are not located on a common ($d-1$)-dimensional hyperplane) we get:
\begin{align*}
\exists \tau \in \mathbb{R}: \mathbf{\hat{w}} = \tau S_W^{-1} (\mu_1-\mu_{-1})
\end{align*}
\end{document}