\documentclass[10pt,oneside,landscape]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{amsthm}

\newtheorem{theorem}{Theorem}[section]
\newtheorem*{problem}{Problem}

\begin{document}

\title{Approximating $\pi$ with an Infinite Series}
\author{Dave Neary}

\maketitle

\section{Introduction}

We can sum the infinite series:
\[f(t) = 1-t^2+t^4-t^6+\cdots = \frac{1}{1+t^2} \]

Integrating across from 0 to $x$:

\[\int_{t=0}^x f(t) dt = \tan^{-1}(x) = x-\frac{x^3}{3}+\frac{x^5}{5} - \cdots\]

And setting $x=1$ (which converges in the integral, but not in the original function):
$\tan^{-1}(1) = \frac{\pi}{4} = 1-\frac{1}{3}+\frac{1}{5}-\frac{1}{7} - \cdots$

\bigskip

This is a nice formula for $\pi$, but is slow to converge. You can use the tan sum formula to
find faster converging solutions:
\[\tan(A+B) = \frac{\tan A + \tan B}{1-\tan A \tan B}\]

Let $A=\tan^{-1} \frac{1}{2}, B=\tan^{-1} \frac{1}{3}$ Then apply the tangent sum forula, and take
inverse tangents on both side:

\begin{align*}
\tan(A+B) &= \frac{\frac{1}{2}+\frac{1}{3}}{1-\frac{1}{2}\cdot\frac{1}{3}} = 1 \\
\tan^{-1}(1) & = \frac{\pi}{4} = \tan^{-1}\frac{1}{2} + \tan^{-1}\frac{1}{3}
\end{align*}

\bigskip

Then:
\[\displaystyle \frac{\pi}{4} = \sum_{i=1}^{\infty} \frac{(-1)^{i+1}}{2i-1} \left( \frac{1}{2^{2i-1}} + \frac{1}{3^{2i-1}}\right)\]

which converges many times faster. You can do this for any other numbers you can find for
$A+B=1-AB$ which gives $(A+1)(B+1) = 2$ such as $\frac{1}{4}$ and $\frac{3}{5}$.
Another example, more complicated to find, is:

\[\frac{\pi}{4} = 4\tan^{-1}\left(\frac{1}{5}\right) - \tan^{-1}\left(\frac{1}{239}\right)\]

which converges to 10 digits of $\pi$ when you take the first 4 terms of the expansion of
$\tan^{-1} \frac{1}{5}$ and the first two terms of $\tan^{-1} \frac{1}{239}$.

\end{document}


% Copyright 2017 Paul Mitchener, licensed under GNU FDL v1.3
% main author:
%   Paul Mitchener
%
\section{Measure Spaces}\label{sec:measure-spaces}

\begin{definition}
Let $\Omega$ be a measurable space, equipped with a $\sigma$-algebra $\mathscr A$.
A \emph{measure} on $\Omega$ is a function $\mu \colon {\mathscr A}\rightarrow [0,\infty ]$ such that:
\begin{axiomlist}[M]
\item The function $\mu$ is \emph{$\sigma$-additive}, i.e.
\[  \mu \left( \bigcup_{n=1}^\infty A_n \right) = \sum_{n=1}^\infty \mu (A_n ) \ , \]
whenever $(A_n)_{n\in \N}$ is a sequence of disjoint mesaurable sets.
\item There is a measurable set $A$ such that $\mu (A)<\infty$.
\end{axiomlist}
The number $\mu (A)$ is called the \emph{measure} of a set $A$.  A measurable space equipped with some measure is called a
\emph{measure space}.
\end{definition}

For the above definition to make sense, we need to make a convention concerning our `number' $\infty$, namely that $a + \infty = \infty$ whenever $a\in [0,\infty ]$.

\begin{example}
Let $\Omega$ be a measurable space.  For any measurable set $E\subseteq \Omega$, let us define
$\mu (E) = |E|$, where $|E|$ denotes the number of elements of $E$.  Then $\mu$ is a measure on $\Omega$, called the {\em counting measure}.
\end{example}

\begin{example}
Let $\Omega$ be a measurable space, and let $x_0 \in \Omega$.  For any measurable set $E\subseteq \Omega$, let us define
$$\mu (E) = \left\{ \begin{array}{ll}
1 & x_0 \in E \\
0 & x_0 \not\in E \\
\end{array} \right.$$

Then $\mu$ is a measure on $\Omega$, called the {\em Dirac measure}.
\end{example}

\begin{proposition}
Let $\Omega$ be a measure space, with measure $\mu$.  Then $\mu (\emptyset ) =0$.
\end{proposition}

\begin{proof}
Choose a measurable set $A$ such that $\mu (A)<\infty$.  Then
$$\mu (A) =  \mu (A) + \mu (\emptyset ) + \mu (\emptyset ) +\cdots$$
Hence $\mu (\emptyset ) =0$.
\end{proof}

\begin{corollary}
Let $A_1 , \ldots , A_n$ be disjoint measurable sets.  Then
$$\mu (A_1 \cup \cdots \cup A_n ) = \mu (A_1 ) + \cdots + \mu (A_n)$$
\textbf{proof to be filled in!}
\end{corollary}

\begin{corollary}
Let $A$ and $B$ be measurable set where $A\subseteq B$.  Then $\mu (A) \leq \mu (B)$.
\end{corollary}

\begin{proof}
The set $B\backslash A = B\cap (\Omega \backslash A )$ is measurable, the sets $A$ and $B\backslash A$ are disjoint, and $B = A\cup B\backslash A$.  By the above corollary
$$\mu (B) = \mu (A) + \mu (B\backslash A)$$

The inequality $\mu (A)\leq \mu (B)$ follows since $\mu (B\backslash A)\geq 0$.
\end{proof}

\begin{proposition} \label{limsub}
Let $(A_n )$ be a sequence of measurable sets such that $A_n \subseteq A_{n+1}$ for all $n$.  Let $A = \bigcup_{n=1}^\infty A_n$.  Then $\lim_{n\rightarrow \infty} \mu (A_n ) = \mu (A)$.
\end{proposition}

\begin{proof}
Let $B_1 =A_1$, and $B_n = A_n \backslash A_{n-1}$ when $n\geq 2$.  Then the sets $B_n$ are measurable and disjoint.  Further
$$A_n = B_1 \cup \cdots \cup A_n \qquad A= \bigcup_{n=1}^\infty B_n$$

Hence
$$\mu (A) = \sum_{n=1}^\infty \mu (B_n) = \lim_{N\rightarrow} \sum_{n=1}^N \mu (B_n) = \lim_{N\rightarrow \infty}\mu (A_N)$$
\end{proof}

\begin{corollary}
Let $(A_n)$ be a sequence of measurable sets such that $\mu (A_1)<\infty$ and $A_{n+1}\subseteq A_n$ for all $n$.    Let $A = \bigcap_{n=1}^\infty A_n$.  Then $\lim_{n\rightarrow \infty} \mu (A_n ) = \mu (A)$.
\end{corollary}

\begin{proof}
Let $C_n =A_1\backslash A_n$.  Then the set $C_n$ is measurable, $C_n\subseteq C_{n+1}$ for all $n$, and $\bigcup_{n=1}^\infty C_n = A_1 \backslash A$.  Hence, by the above proposition
$$\lim_{n\rightarrow \infty}\mu (C_n) = \mu (A_1 \backslash A)$$

We know that the measure $\mu (A_1)$ is finite, and that we have disjoint unions$$A_1 = A_n \cup C_n \qquad A_1 = A_1\backslash A \cup A$$
Hence
$$\mu (A_1 )- \lim_{n\rightarrow \infty} \mu (A_n) = \mu (A_1 ) - \mu (A)$$
and
$$\lim_{n\rightarrow \infty} \mu (A_n ) = \mu (A)$$
\end{proof}

The above corollary is false if we omit the assumption that $\mu (A_1)<\infty$.


\chapter{Counting Principles}
\label{chapter:principles}
\marginurl{%
  Counting Principles:\\\noindent
  Introduction to Combinatorics \#2
}{youtu.be/dAoperLCjb8}
\section{The Additive Principle}
The first principle is called \emph{additive} principle and it states that if
you have two disjoint sets, then their union have size equal to the sum of their
sizes.

A simple illustration of this statement is the following. Assume you have three
pencils and two pens; how many ways to choose a writing accessory. According to
this principle the answer is $2 + 3 = 5$.
\begin{theorem}[The Additive Principle]
\label{theorem:additive-principle}
  Let $X$ and $Y$ be finite sets. If $X \cap Y = \emptyset$, then
  $\cardinality{X \cup Y} = \cardinality{X} + \cardinality{Y}$.
\end{theorem}

\begin{corollary}
\label{corollary:additive-principle}
  Let $X_1, \dots, X_n$ be some pairwise disjoint sets. Then
  $\cardinality{\bigcup_{i = 1}^n X_i} = \sum_{i = 1}^n \cardinality{X_i}$.
\end{corollary}

\begin{exercise}
  Prove Corollary~\ref{corollary:additive-principle}.
\end{exercise}

\section{The Multiplicative Principle}
The next principle is called the \emph{multiplicative} principle and it can be
illustrated as follows: imagine that you are given two postal stamps and
three envelopes, how many ways are there to pack the letters? The answer is obviously
$2 \cdot 3 = 6$.
\begin{theorem}[The Multiplicative Principle]
\label{theorem:multiplicative-principle}
  Let $X$ and $Y$ be finite sets. Then $\cardinality{X \times Y} =
  \cardinality{X} \times \cardinality{Y}$.
\end{theorem}

\begin{exercise}
  Assume you have $5$ types of envelopes and $6$ types of postal stamps.
  \begin{enumerate}
    \item How many ways to put one stamp on one envelope?
    \item How many ways to put two stamps on one envelope?
  \end{enumerate}
\end{exercise}

By analogy with unions and intersections of many sets we can define the cross
product of many sets.
Let $X_1$, \dots, $X_n$ be some sets. Then $\bigtimes_{i = 1}^1 X_i = A_1$ and
$\bigtimes_{i = 1}^{k + 1} X_i =
  \left(\bigtimes_{i = 1}^k X_i\right) \times X_{k + 1}$\footnote{%
    Note that cross product is not associative and different definitions of the
    product of several sets are not equivalent. However, the bijection
    constructed in the previous section allow us to think about these
    definitions as if they are equivalent.
}.

\begin{corollary}
\label{corollary:multiplicative-principle}
  Let $X_1, \dots, X_n$ be some finite sets. Then
  $\cardinality{\bigtimes_{i = 1}^n X_i} = \prod_{i = 1}^n \cardinality{X_i}$.
\end{corollary}

\begin{exercise}
  Prove Corollary~\ref{corollary:multiplicative-principle}.
\end{exercise}

\begin{theorem}
\label{theorem:cardinality-of-power-set}
  For any set $X$, $\cardinality{\subsets{X}} = 2^{\cardinality{X}}$.
\end{theorem}
\begin{proof}
  By Corollary~\ref{corollary:power-set-and-set-of-binary-strings},
  $\cardinality{\subsets{X}} =
  \cardinality{\functions{\cardinality{X}}{\set{0, 1}}}$, so it is enough to
  prove that $\cardinality{\set{0, 1}^{\cardinality{X}}} = \subsets{\cardinality{X}}$.
  This statement is true by \Cref{corollary:multiplicative-principle} since
  $\cardinality{\set{0, 1}^{\cardinality{X}}} =
  \prod_{i = 1}^{\cardinality{X}} \cardinality{\set{0, 1}} =
    2^{\cardinality{X}}$.
\end{proof}

Using this result and \Cref{theorem:injections-surjections-inequalities} we may
finally prove \Cref{lemma:turning-turtles-number-P}.
\begin{proof}[Proof of \Cref{lemma:turning-turtles-number-P}]
  First note that any state in turning turtles can be described by a set
  $\set{a_1, \dots, a_\ell} \subseteq \range{m}$ of positions of heads; moreover
  the position is a P-position iff
  $a_1 \bitwisexor a_2 \bitwisexor \dots \bitwisexor a_\ell = 0$.

  We may also note that for any $n \in  \range{2^r}$ there is a set $C(n) =
  \set{c_1, \dots, c_k} \subseteq \set{2^0, 2^1, 2^2, \dots, 2^{r - 1}}$ such
  that
  $n \bitwisexor c_1 \bitwisexor a_2 \bitwisexor \dots \bitwisexor c_k = 0$.
  Let $I = \range{m} \setminus \set{2^0, 2^1, 2^2, \dots, 2^{r - 1}}$.
  It is clear that for any $J \subseteq I$ there is
  $C'(J) = \set{c_1, \dots, c_k} \subseteq
    \set{2^0, 2^1, 2^2, \dots, 2^{r - 1}}$ such that $J \cup C'(J)$ describes a
  P-position in turning turtles.

  Let $I = \set{i_1, \dots, i_{m - r}}$. We can define an injection
  $f : \set{0, 1}^{\cardinality{I}} \to 2^{\range{m}}$ such that $f(x)$ is a
  P-position in turning turtles. Indeed, the function $f$ such that
  $f(x_1, \dots, x_{m - r}) = \set[x_j = 1]{i_j} \cup C'(\set[x_j = 1]{i_j})$
  is an injection and $f(x_1, \dots, x_{m - r})$ is a P-position in turning
  turtles.

  Therefore $2^{m - r} = \cardinality{\set{0, 1}^{\cardinality{I}}}$ is greater
  or equal to the number of P-positions in turning turtles.
\end{proof}

We can also use these principles to show that Hamming code is optimal.
\begin{theorem}
  Let $n = 2^{2^r - r - 1}$ and let $C : \range{n} \to \set{0, 1}^m$ be an
  error-correcting code correcting $1$ error. Then $m \ge 2^r - 1$.
\end{theorem}
\begin{proof}
  Let $B(i) = \set[{\hammingDist{C(i)}{x} \le 1}]{x \in \set{0, 1}^m}$.
  By the definition of erorr-correcting codes, $B(i)$ and $B(j)$ are disjoint
  for $i \neq j$. Note that $\cardinality{B(i)} = 1 + m$ since if
  $\hammingDist{C(i)}{x} \le 1$, then either $x = C(i)$ or they are different in
  one out of $m$ coordinates. Therefore $\cardinality{\bigcup_{i = 1}^n B(i)}
  = (1 + m) n \le 2^m$ since $\cardinality{\set{0, 1}^m} = 2^m$.

  Note that for a fixed $r$, $(1 + m) n$ and $2^m$ as functions of $m$ intersect
  only in one point and the inequality is false for $m = 0$. Moreover, the
  inequality became equality for $m = 2^r - 1$. Therefore in order for the
  equality to be true we need $m \ge 2^r - 1$.
\end{proof}

\section{The Inclusion-exclusion Principle}

The last principle we are going to discuss in this chapter is the
inclusion-exclusion principle which helps us to find the size of the union
of sets when they are not disjoint.
\begin{theorem}[The Inclusion-exclusion Principle]
\label{theorem:inclusion-exclusion-principle}
  Let $X$ and $Y$ be finite sets. Then $\cardinality{X \cup Y} = \cardinality{X}
  + \cardinality{Y} - \cardinality{X \cap Y}$.
\end{theorem}
\begin{proof}
  Note that $X \cup Y = (X \setminus Y) \cup (Y \setminus X) \cup (X \cap Y)$.
  Hence, $\cardinality{X \cup Y} = \cardinality{X \setminus Y} +
  \cardinality{Y \setminus X} + \cardinality{X \cap Y}$. But it
  is possible to note that $\cardinality{Y \setminus X} + \cardinality{X \cap Y}
  = \cardinality{Y}$ and $\cardinality{X \setminus Y} + \cardinality{X \cap Y} =
  \cardinality{X}$.
\end{proof}

\begin{corollary}
\label{corollary:inclusion-exclusion-principle}
  Let $X_1, \dots, X_n$ be some finite sets. Then
  \[
    \cardinality{\bigcup_{i = 1}^n X_i} =
    \sum_{S \subseteq \range{n} ~:~ S \neq \emptyset}
      (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} X_i}.
  \]
\end{corollary}
\begin{proof}
  As always, we prove this statement using induction by $n$. The base case for
  $n = 2$ is true by Theorem~\ref{theorem:inclusion-exclusion-principle}.

  By the induction hypothesis,
  \[
    \cardinality{\bigcup_{i = 1}^k X_i} =
    \sum_{S \subseteq \range{k} ~:~ S \neq \emptyset}
      (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} X_i}.
  \]
  In addition, by Theorem~\ref{theorem:inclusion-exclusion-principle},
  \[
    \cardinality{\bigcup_{i = 1}^{k + 1} X_i} =
    \cardinality{\bigcup_{i = 1}^k X_i} + \cardinality{X_{k + 1}} -
      \cardinality{\left(\bigcup_{i = 1}^k X_i\right) \cap X_{k + 1}}.
  \]
  We need to simplify two elements of the sum on the right of the equality.
  By the induction hypothesis,
  \[
    \cardinality{\bigcup_{i = 1}^k X_i} =
    \sum_{S \subseteq \range{k} ~:~ S \neq \emptyset}
        (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} X_i}.
  \]
  In addition, it is easy to note that
  \[
    \cardinality{\left(\bigcup_{i = 1}^k X_i\right) \cap X_{k + 1}} =
    \cardinality{\bigcup_{i = 1}^k \left(X_i \cap X_{k + 1}\right)}.
  \]
  Thus using the induction hypothesis,
  \begin{multline*}
    \cardinality{\left(\bigcup_{i = 1}^k X_i\right) \cap X_{k + 1}} = \\
    \sum_{S \subseteq \range{k} ~:~ S \neq \emptyset}
      (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} (X_i \cap X_{k + 1})} = \\
    \sum_{S \subseteq \range{k + 1} ~:~ (k + 1) \in S \text{ and } S \neq \set{k + 1}}
          (-1)^{\cardinality{S}} \cardinality{\bigcap_{i \in S} X_i}.
  \end{multline*}
  As a result,
  \[
    \cardinality{X_{k + 1}} -
      \cardinality{\left(\bigcup_{i = 1}^k X_i\right) \cap X_{k + 1}} =
    \sum_{S \subseteq \range{k + 1} ~:~ (k + 1) \in S}
          (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} X_i}.
  \]
  Which implies that
  \begin{multline*}
    \cardinality{\bigcup_{i = 1}^{k + 1} X_i} =
    \sum_{S \subseteq \range{k} ~:~ S \neq \emptyset}
      (-1)^{\cardinality{S} + 1}\cardinality{\bigcap_{i \in S} X_i} + \\
    \sum_{S \subseteq \range{k + 1} ~:~ (k + 1) \in S}
          (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} X_i} = \\
    \sum_{S \subseteq \range{k + 1} ~:~ S \neq \emptyset}
      (-1)^{\cardinality{S} + 1} \cardinality{\bigcap_{i \in S} X_i}.
  \end{multline*}
\end{proof}

\begin{chapterendexercises}
  \exercise
    Find the cardinality of the set
    \[
      \set[{x, y \in \range{9} \text{ and } x \neq y}]{(x, y)}.
    \]
  \exercise[recommended] Find the number of ordered pairs $(A, B)$ of subsets of
    $\range{n}$ such that $A \cap B = \emptyset$ (the answer is not supposed to
    contain the summation sign).
  \exercise How many functions from $\set{0, 1}^n$ to $\set{0, 1}$?
  \exercise[recommended] How many numbers from $\range{999}$ are not divisible
    neither by $3$, nor by $5$, nor by $7$.
    \begin{solution}
      Let $D_n = \set[i \text{is divisible by n}]{i \in \range{999}}$.
      Note that $D_3 \cap D_5 = D_{15}$, $D_3 \cap D_7 = D_{21}$, $D_5 \cap D_7
      = D_{35}$, and finally, $D_3 \cap D_5 \cap D_7 = D_{105}$. Additionally,
      $\cardinality{D_3} = 999 / 3 = 333$,
      $\cardinality{D_5} = \floor{999 / 5} = 199$,
      $\cardinality{D_7} = \floor{999 / 7} = 142$,
      $\cardinality{D_{15}} = \floor{999 / 15} = 66$,
      $\cardinality{D_{21}} = \floor{999 / 21} = 47$,
      $\cardinality{D_{35}} = \floor{999 / 35} = 28$, and
      $\cardinality{D_{105}} = \floor{999 / 105} = 9$.
      As a result, by the inclusion-exclusion principle, the answer is
      $999 - 333 - 199 - 142 + 66 + 47 + 28 - 9 = 457$.
    \end{solution}
  \exercise How many numbers from $1$ to $1000$ are neither square numbers nor
    cubic numbers?
    \begin{solution}
      Let us define
      $
        A_i =  \set[{\sqrt[i]{x} \in \Z}]{x \in \range{1000}}.
      $
      We need to find the value of $1000 - |A_2 \cup A_3|$. Note that
      $\cardinality{A_i} = \floor{\sqrt[i]{1000}}$. Hence,
      \begin{multline*}
        \cardinality{A_2 \cup A_3} = \cardinality{A_2} + \cardinality{A_3} -
        \cardinality{A_2 \cap A_3} = \cardinality{A_2} + \cardinality{A_3} -
        \cardinality{A_6} = \\
        \floor{\sqrt{1000}} + \floor{\sqrt[3]{1000}} - \floor{\sqrt[6]{1000}} =
        31 + 10 - 3 = 38.
      \end{multline*}
      Therefore the answer is equal to $1000 - 38 = 962$.
    \end{solution}
  \exercise How many $6$-digit numbers are there that have the same reminder
    modulo $2$ of all the digits?
    \begin{solution}
      We will give two solutions depending on one's interpretation of $6$-digit
      numbers (i.e whether or not the leading coefficient can be $0$).

      First, in the case where the leading coefficient may be $0$, there are
      $10$ choices for the first digit. Upon picking the first digit, we now
      must picks digits that have the same parity (i.e even or odd) as the first
      digit. Thus, there are $5$ choices for digits two through six, and we get
      \[
        10\cdot 5 \cdot 5 \cdot 5 \cdot 5\cdot 5 = 31250 \text{ total digits. }
      \]

      However, if you interpreted the problem as the leading coefficient cannot
      be $0$, then there would be $9$ options for the first digit and then $5$
      for each of the remaining. We would then get
      \[
        9 \cdot 5 \cdot 5 \cdot 5 \cdot 5\cdot 5 = 28125 \text{ total digits. }
      \]
    \end{solution}
  \exercise How many numbers $x$ from $1$ to $999$ such that at least one
    of the digits of $x$ is $7$?
    \begin{solution}
      First note that any number from $0$ to $999$ may be expressed using $3$
      digits (we allow to use leading $0$s to express the numbers like $1$).
      Thus we need to find the size of the set
      \[
        \set[{
          7 \in \{a_1, a_2, a_3\}
        }]{(a_1, a_2, a_3) \in \set{0, \dots, 9}^3}.
      \]

      Let $S \subseteq [3]$, it is easy to see that
      \[
        \bigcap_{i \in S}
          \set[{
            7 \in \{a_1, a_2, a_3\}
          }]{(a_1, a_2, a_3) \in \set{0, \dots, 9}^3}
      \]
      has $10^{\cardinality{S}}$ elements. As a result, the answer is
      $1000 - 3 \cdot 10^2 + 3 \cdot 10 - 1$.
    \end{solution}
  \exercise How many numbers $x$ from $1$ to $999$ such that exactly one
    of the digits of $x$ is $7$?
    \begin{solution}
      Note that any number from $0$ to $999$ can be described by three digits.
      It is easy to see that there are $9^2$ number with one digit equal to $7$
      and all others being not equal to $7$. Hence, the answer is $3 \cdot
      9^2$.
    \end{solution}
  \exercise Let $A$, $B$ be some finite sets such that $A \subseteq B$.
    Show that $\cardinality{B \setminus A} = \cardinality{B} - \cardinality{A}$.
  \exercise King Arthur invited $n$ knights for a feast. Each knight has an enemy
    (being an enemy is a mutual relation; i.e., if $A$ is an enemy of $B$, then
    $B$ is an enemy of $A$). How many ways for knights to seat around the Round
    Table so that enemies are not sitting together?
  \exercise[recommended] Let $n$ be some positive integer.
    Find the cardinality of the set
    \[
      \set[{A, B \subseteq \range{n} \text{ and } A \cap B \neq \emptyset}]{(A, B)}?
    \]
  \exercise Let $X$ and $Y$ be some finite sets, and $f : X \to Y$ be a function
    such that $\cardinality{f^{-1}(y)} = k$ for all $y \in Y$. Prove that
    $\cardinality{X} = k\cardinality{Y}$.
  \exercise[recommended] Show that if $U$ and $X_1, \dots, X_n \subseteq U$ are
    some finite sets, then
    \[
      \cardinality{\bigcap_{i = 1}^n X_i} =
      \sum_{S \subseteq \range{n}} (-1)^{\cardinality{S}}
        \cardinality{\bigcap_{i \in S} \overline{X}_i},
    \]
    where $\overline{X}_i = U \setminus X_i$ and
    $\bigcap_{i \in \emptyset} \overline{X}_i = U$.
\end{chapterendexercises}


\subsection{B-spline as activation functions}\label{sec:Bsplines}
In this section, we consider the power of ReLU as activation functions
\begin{equation}
  \label{relup}
{\rm ReLU}^k(x)=[x_+]^k.
\end{equation}

We consider the following neuron network function class
with one hidden layer:
\begin{equation}
\label{VkN}
V_N^k=\left\{\sum_{i=1}^Na_i(w_i\cdot x+b_i)_+^k, a_i, b_i\in\mathbb R^1, w_i\in \mathbb R^{1\times d}\right\}.
\end{equation}
We note that $V_N^k$ is not a linear vector space.  The definition of
neural network function class such as \eqref{VkN} can be traced back
in \cite{mcculloch1943logical} and its early mathematical analysis can be found in \cite{hornik1989multilayer,cybenko1989approximation,funahashi1989approximate}.

The functions in $V^k_N$ as defined in \eqref{VkN} will be known as finite neuron functions in this paper.
\begin{lemma}
  For any $k\ge 1$, $V_N^k$ consists of functions that are piecewise
  polynomials of degree $k$ with respect to a grid whose boundaries are
  given by intersection of the following hyperplanes
$$
w_ix + b_i=0,\quad 1\le i\le N.
$$
see Fig \ref{fig:1} and Fig \ref{fig:3}. Furthermore, if $k\ge m$,
$$
V_N^k(\Omega)\subset H^k(\Omega)\subset H^m(\Omega),
$$
where $\Omega$ is a bounded domain in $\mathbb{R}^d$.
\end{lemma}
\begin{figure}[!ht]
\begin{center}
%$(10, 10)$ $(20, 20)$ $ (40,40)$
\includegraphics[width=.3\textwidth]{6dl/figures/dnn1-50.jpg}
\end{center}
\caption{Hyperplanes with $\ell=1$, where $\ell$ is the depth of the neural network in \eqref{NNL}.}
\label{fig:1}
\end{figure}

The main goal of this section is to prove that the
following type of error estimate holds, for some $\delta\ge 0$,
\begin{equation}\label{VNerror}
\inf_{v_N\in V_N^k}\|u- v_N \|_{H^m(\Omega)} \lesssim
N^{-{1\over 2}-\delta}.
\end{equation}
We will use two different approaches to establish \eqref{VNerror}.
The first approach, presented in \S\ref{sec:Bsplines}, mainly follows
\cite{hornik1994degree} and \cite{siegel2020approximations} that gives
error estimates for a general class of activation functions.  The
second approach follows
\cite{klusowski2016uniform} that gives error estimates specifically
for ReLU activation function.

We assume that $\Omega\subset\mathbb R^d$ is a given bounded domain.
Thus,
\begin{equation}
  \label{T}
T=\max_{x\in \bar{\Omega}} \|x\|<\infty.
\end{equation}
The activation function ${\rm [ ReLU]}^k$ \eqref{relup} is related to
cardinal B-Splines.  A cardinal B-Spline of degree $k\ge 0$
denoted by $b^k$, is defined by convolution as
\begin{equation}
	b^k(x)=(b^{k-1}*b^0)(x)=\int_\mathbb{R}b^{k-1}(x-t)b^0(t)dt,
\end{equation}
where
\begin{equation}
b^0(x)=\left\{
		     \begin{array}{lr}
		    1 & x\in[0,1),\\
		    0 & \hbox{otherwise}.
		     \end{array}
	\right.
\end{equation}
\begin{figure}
\begin{center}
\includegraphics[width=0.5\textwidth]{6DL/figures/B-spline.png}
\caption{Plots of some B-spline basis}
\label{bk}
\end{center}
\end{figure}
More explicitly, see \cite{de1971subroutine}, for any
$x\in[0,k+1]$ and $k\geq 1$, we have
	\begin{equation}
	b^k(x)=\frac{x}{k}b^{k-1}(x)+\frac{k+1-x}{k}b^{k-1}(x-1),
	\end{equation}
or
	\begin{equation}\label{splinetorelu}
	b^k(x)=(k+1)\sum_{i=0}^{k+1} w_i(i-x)_+^k \hbox{~and~} w_i={\displaystyle\prod_{j=0,j\neq i}^{k+1}} \frac{1}{i-j}.
	\end{equation}
We note that all $b^k$ are locally supported and see Fig.~\ref{bk} for their plots.

For an uniform grid with mesh size $h=\frac{1}{n+1}$, we define
	\begin{equation}
	b^k_{j,h}(x)=b^{k}(\frac{x}{h}-j).
	\end{equation}
Then the cardinal B-Spline series of degree $k$ on the uniform grid is
\begin{equation}\label{Skn}
S_N^k=\Big\{v(x)=\sum_{j=-k}^{N}	c_jb^k_{j,h}(x)\Big\}.
\end{equation}
\begin{lemma} For $V_N^k$ and $S_N^k$ defined by \eqref{VkN} and
  \eqref{Skn}, we have
\begin{equation}
    \label{SV}
S_N^k\subset V_{N+k+1}^k.
  \end{equation}
As a result, for any bounded domain $\Omega\subset \mathbb R^1$, we have
\begin{equation}
  \label{SVerror}
\inf_{v\in V_{N}^k}\|u-v\|_{m,\Omega}
\le \inf_{v\in S_{N-k-1}^k}\|u-v\|_{m,\Omega} \lesssim N^{m-(k+1)} \|u\|_{k+1,\Omega}.
\end{equation}
\end{lemma}

\iffalse
Given an activation function $\sigma\in L^1(\mathbb R)$, consider its Fourier transformation:
\begin{equation}
  \label{Fsigma}
\hat \sigma(a) = \frac{1}{2\pi}\int_{\mathbb{R}} \sigma(t)e^{-iat}dt.
\end{equation}
For any $a\neq 0$ with $\hat \sigma(a)\neq 0$, by making a change of variables
$t = a^{-1}\omega\cdot x + b$ and $dt = db$, we have
 \begin{equation}
 \begin{aligned}
\hat{\sigma}(a)&=\frac{1}{2\pi}\int_{\mathbb{R}}\sigma(a^{-1}\omega\cdot x+b)e^{-ia( a^{-1}\omega\cdot x+b)}db = e^{-i\omega \cdot x}\frac{1}{2\pi}\int_{\mathbb{R}}\sigma(a^{-1} \omega\cdot x+b)e^{-iab}db.
 \end{aligned}
 \end{equation}
This implies that
\begin{equation}\label{FourierExp}
 e^{i\omega \cdot x} = \frac{1}{2\pi\hat{\sigma}(a)}\int_{\mathbb{R}}\sigma(a^{-1}\omega\cdot x+b)e^{-iab}db.
\end{equation}
We write $ \hat{u}(\omega) = e^{-i\theta(\omega)} | \hat{u}(\omega)|$
and then obtain the following integral represntation:
\begin{equation}\label{integral_representation01}
u(x) = \int_{\mathbb{R}^d} e^{i\omega\cdot x}\hat{u}(\omega)d\omega =
\int_{\mathbb{R}^d}\int_\mathbb{R}\frac{1}{2\pi\hat{\sigma}(a)}
\sigma\left(a^{-1} \omega\cdot x+b\right)|\hat{u}(\omega)|e^{-i(ab+\theta(\omega))}dbd\omega
\end{equation}
Now we consider activation function $\sigma(x)=b^k(x)$ and $\hat \sigma$ be the
Fourier transform of $\sigma(x)$. Note that, by \eqref{bk},
\begin{align}\label{splineFourier}
\hat{\sigma}(a)=\left({1-e^{-ia}\over ia}\right)^{k+1}=\left({2\over a}\sin {a\over 2}\right)^{k+1}e^{-{ia(k+1)\over 2}}.
\end{align}

We first take $a=\pi$ in \eqref{splineFourier}. Thus,
\begin{equation}
  \label{pi1}
\hat\sigma(\pi)=
\left({2\over \pi}\right)^{k+1}e^{-{i\pi (k+1)\over 2}}.
\end{equation}
Combining \eqref{integral_representation01} and \eqref{pi1}, we obtain
that
\begin{equation}
  \label{splinerep0}
u(x) =
\frac{1}{4}\left({\pi\over 2}\right)^{k}\int_{\mathbb{R}^d}\int_\mathbb{R}
\sigma\left(\pi^{-1}\omega\cdot x+b\right)|\hat{u}(\omega)|e^{-i(\pi b + {\pi (k+1)\over 2}+\theta(\omega))}dbd\omega
\end{equation}
An application of the Monte Carlo method in Lemma \ref{MC} to the integral representation \eqref{splinerep0} leads to the following estimate.
\begin{theorem}\label{splinestratify}
For any $0\le m\le k$, if $u\in {B}^{m+1}(\Omega)$, there exist $\omega_i\in \mathbb{R}^d$, $b_i\in \mathbb{R}$ such that
\begin{equation}
\left \|u - u_{N}\right\|_{H^{m}(\Omega)}\lesssim  N^{-{1\over 2}} \|u\|_{{B}^{m+1}(\Omega)}
\end{equation}
with
\begin{equation}
u_{N}(x)=\sum_{i=1}^{N} \beta_i b^k\left(\pi^{-1} \omega_i\cdot x+b_i\right).
\end{equation}
\end{theorem}

Based on the integral representation \eqref{splinerep0}, a stratified analysis similar to the one in \cite{siegel2020approximations} leads to the following result.
\begin{theorem}
For any $0\le m\le k$ and positive $\epsilon$,  if $u\in {B}^{m+1+\epsilon}(\Omega)$, , there exist $\omega_i\in \mathbb{R}^d$, $b_i\in \mathbb{R}$ such that
\begin{equation}\label{straunbdd}
\left\|u - u_{N}\right\|_{H^{m}(\Omega)}\le  N^{-{1\over 2}-{\epsilon \over (d+1)(2+\epsilon)}} \|u\|_{{B}^{m+1+\epsilon}(\Omega)}
\end{equation}
with
\begin{equation}
u_{N}(x)=\sum_{i=1}^{N} \beta_i b^k\left(\bar \omega_i\cdot x+b_i\right) .
\end{equation}
\end{theorem}
Next, we try to improve the estimate \eqref{straunbdd}. Again, we will use \eqref{integral_representation01}. Let $\displaystyle a_\omega=4\pi\lceil {\|\omega\|\over 4\pi}\rceil + \pi$ in \eqref{splineFourier} and $\displaystyle \bar\omega ={\omega\over a_\omega}$. We have
 \begin{equation}
\hat{\sigma}(a_\omega)=\left({2\over a_\omega}\right)^{k+1}, \quad \|\omega\| + \pi\le a_\omega\le \|\omega\|+5\pi,\quad \|\bar\omega\|\le 1,
 \end{equation}
which, together with \eqref{integral_representation01}, indicates that
 \begin{equation}\label{integral_representation}
 \begin{split}
  u(x) =  \int_{\mathbb{R}^d}\int_\mathbb{R}\frac{1}{2\pi}
  \sigma\left(\bar \omega\cdot x+b\right)\left({a_\omega\over 2}\right)^{k+1}\hat{u}(\omega)e^{-ia_\omega(b+{k+1\over 2})}dbd\omega.
\end{split}
 \end{equation}

\begin{theorem}
If $u\in {B}^{k+1}(\Omega)$, there exist $\|\bar \omega_i\|\le 1$, $|b_i|\le T + k+1$ such that
\begin{equation}\label{d+1}
\left\|u - u_{N}\right\|_{H^{m}(\Omega)}\lesssim  N^{-{1\over 2}-{1\over d+1}} \|u\|_{{B}^{k+1}(\Omega)}
\end{equation}
with
\begin{equation}
u_{N}(x)=\sum_{i=1}^{N} \beta_i b^k\left(\bar \omega_i\cdot x+b_i\right) .
\end{equation}
\end{theorem}
\begin{proof}
We write \eqref{integral_representation} as follows
$$
\displaystyle u(x)= \int_{\mathbb{R}^d}\int_\mathbb{R}
g(x, b, \omega)\rho(b,\omega) dbd\omega
$$
with
$$
\hat{u}(\omega) = e^{-i\theta(\omega)} | \hat{u}(\omega)|,\quad \tilde \theta(\omega)=\theta(\omega) + a_\omega(b+{k+1\over 2})
$$ and
\begin{equation}\label{eq:g}
g(x, b, \omega) = \sigma\left({\bar \omega}\cdot x+b\right)sgn(\cos \tilde\theta(\omega)),
\end{equation}
\begin{equation}\label{eq:rho}
\rho(b,\omega) = \frac{1}{(2\pi)^d}\left({a_\omega\over 2}\right)^{k+1}| \hat{u}(\omega)||\cos \tilde\theta(\omega)|.
\end{equation}
Note that
\begin{equation}
\|\bar\omega\|\le 1, \quad |b|\le T+k+1.
\end{equation}
Let
$$
G=\{(\omega, b): \omega\in \mathbb{R}^d,\ |b|\le T+k+1\}, \ \tilde G=\{(\bar \omega, b): \|\bar \omega\| \le 1,\ |b|\le T+k+1\}.
$$
For any positive integer $n$, divide $\tilde  G$ into $\tilde  M(\tilde  M\le {n\over 2})$   nonoverlapping subdomains, say
$\tilde  G=\tilde  G_1\cup \tilde  G_2\cup \cdots \cup\tilde  G_{\tilde M}$, such that
\begin{equation}
|b-b'|\lesssim n^{-{1\over d+1}},\quad |\bar\omega - \bar\omega'|\lesssim  n^{-{1\over d+1}}, \quad (\bar\omega, b),\ (\bar\omega', b')\in \tilde G_i,\ 1\le i\le \tilde M.
\end{equation}
Define $M=2\tilde  M$ and for $1\le i\le \tilde M$,
$$
G_i = \{(\omega, b): (\bar \omega, b)\in \tilde  G_i, \ \cos \tilde\theta(\omega)\ge 0\},\
G_{\tilde  M+i} = \{(\omega, b): (\bar \omega, b)\in \tilde  G_i, \ \cos \tilde\theta(\omega)\le 0\}.
$$
Thus, $G=G_1\cup G_2\cup \cdots \cup G_M$ with $\tilde  G_i\cap \tilde G_j=\varnothing$ if $i\neq j$, and
\begin{equation}
|b-b'|\lesssim n^{-{1\over d+1}},\quad |\bar\omega - \bar\omega'|\lesssim  n^{-{1\over d+1}}, \quad sgn(\cos \tilde\theta(\omega))=sgn(\cos\tilde \theta(\omega')).
\end{equation}
Let $n_i=\lceil \lambda(G_i)n\rceil$, $N=\displaystyle \sum_{i=1}^M n_i$ and
\begin{equation}
u_{N}(x)=\|\rho\|_{L^1(G)}\sum_{i=1}^{M} \frac{\lambda(G_{i})}{n_{i}} \sum_{j=1}^{n_{i}}g(x,\theta_{i,j}).
\end{equation}
It holds that
\begin{equation}\label{eq:sum}
\begin{split}
\mathbb{E}\left(\left\|u - u_{N}\right\|_{H^{m}(\Omega)}^{2}\right)\le&
\|\rho\|_{L^1(G)}\sum_{i=1}^{M}  \frac{\lambda^2(G_i)}{n_{i}}  \sup_{\theta_{i},\theta_{i}'\in G_i} \| g(x,\theta_i) - g(x,\theta_i')\|^2_{H^m(\Omega)}
 \end{split}
\end{equation}
with $\theta=(b, \omega)$.
For any $(b, \omega)\in G_i$, $1\le i\le M$, if $k\ge m+1$,
\begin{equation}
|g(x,\theta) - g(x,\theta')| \lesssim |b-b'| + |\omega - \omega'| \lesssim   n^{-{1\over d+1}}
\end{equation}
Thus,
\begin{equation}
 \sum_{i=1}^{M}  \frac{\lambda^2(G_i)}{n_{i}}  \sup_{\theta_{i},\theta_{i}'\in G_i} \| g(x,\theta_i) - g(x,\theta_i')\|^2_{H^m(\Omega)}
 \lesssim  n^{-{2\over d+1}} |\Omega|.
\end{equation}
Thus,
\begin{equation}\label{eq:}
\begin{split}
\mathbb{E}\left(\left\|u - u_{N}\right\|_{H^{m}(\Omega)}^{2}\right)\lesssim&   n^{-1-{2\over d+1}} |\Omega|\|\rho\|_{L^1(G)}.
 \end{split}
\end{equation}
Since $a\le \|\omega\|+5\pi$,
$$
\|\rho\|_{L^1(G)}\lesssim \int_G (\|\omega\| + 1)^{k+1}|\hat u(\omega)|d\omega db \lesssim \|u\|_{B^{k+1}(\Omega)}.
$$
Note that $n\le N\le 2n$. Thus, there exist $\omega_i\in \mathbb{R}^d$, $\beta_i$, $b_i\in \mathbb{R}$ such that
\begin{equation}
\left\|u - u_{N}\right\|_{H^{m}(\Omega)}\lesssim  N^{-{1\over 2}-{1\over d+1}} \|u\|_{B^{k+1}(\Omega)},
\end{equation}
which completes the proof.
\end{proof}

The above analysis can also be applied to more general activation functions with compact support.
\begin{theorem}
Suppose that $\sigma\in W^{m+1,\infty}(\mathbb{R})$ that has a compact
support. If for any $a>0$, there exists $\tilde a>0$ such that
\begin{equation}
\tilde a\gtrsim a,\quad  |\hat\sigma(\tilde a)|\gtrsim a^{-\ell},
\end{equation}
and  $u\in {B}^{\ell}(\Omega)$, then, there exist $\omega_i\in \mathbb{R}^d$ and $b_i\in \mathbb{R}$ such that
\begin{equation}
\left\|u - u_{N}\right\|_{H^{m}(\Omega)}\lesssim  N^{-{1\over 2}-{1\over d+1}} \|u\|_{B^{\ell}(\Omega)},
\end{equation}
where
\begin{equation}
u_{N}(x)=\sum_{i=1}^{N} \beta_i \sigma\left(\bar \omega_i\cdot x+b_i\right) .
\end{equation}
\end{theorem}

\fi


\section{The determinant of a triangular matrix}

\begin{outcome}
  \begin{enumerate}
  \item Calculate the determinant of an upper or lower triangular
    matrix.
  \end{enumerate}
\end{outcome}

There is a certain type of matrix for which finding the determinant is
a very simple procedure: a triangular matrix.

\begin{definition}{Triangular matrices}{triangular-matrices}
  An square matrix $A$ is \textbf{upper triangular}%
  \index{matrix!upper triangular}%
  \index{matrix!triangular}%
  \index{upper triangular matrix}%
  \index{triangular matrix} if $a_{ij}=0$ whenever $i>j$. In other
  words, a matrix is upper triangular if the entries below the main
  diagonal are $0$. Thus, an upper triangular matrix looks as follows,
  where $\ast$ refers to any non-zero number:
  \begin{equation*}
    \begin{mymatrix}{ccccc}
      \ast & \ast & \cdots & \ast & \ast \\
      0 & \ast & \cdots & \ast & \ast \\
      \vdots & \vdots & \ddots & \vdots & \vdots \\
      0 & 0 & \cdots & \ast & \ast \\
      0 & 0 & \cdots & 0 & \ast \\
    \end{mymatrix}.
  \end{equation*}
  Similarly, a square matrix is \textbf{lower triangular}%
  \index{matrix!lower triangular}%
  \index{lower triangular matrix} if all entries above the main
  diagonal%
  \index{diagonal of a matrix}%
  \index{matrix!diagonal of} are $0$.
\end{definition}

The following theorem provides a useful way to calculate the
determinant of a triangular matrix.

\begin{theorem}{Determinant of a triangular matrix}{determinant-of-triangular-matrix}
  Let $A$ be an upper or lower triangular matrix. Then $\det(A)$ is
  equal to the product of the entries on the main diagonal. Written as
  a formula, we have
  \begin{equation*}
    \det(A) = a_{11}\,a_{22}\,\cdots\,a_{nn}.
  \end{equation*}
\end{theorem}

\begin{example}{Determinant of a triangular matrix}{determinant-of-triangular-matrix}
  Compute $\det(A)$, where
  \begin{equation*}
    A=\begin{mymatrix}{rrrr}
      1 & 2 & 3 & 16 \\
      0 & 2 & 6 & -7 \\
      0 & 0 & 3 & 33 \\
      0 & 0 & 0 & -1
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  By Theorem~\ref{thm:determinant-of-triangular-matrix}, it suffices
  to take the product of the elements on the main diagonal. Thus
  \begin{equation*}
    \det (A) ~=~ 1\cdot 2\cdot 3\cdot (-1) ~=~ -6.
  \end{equation*}
\end{solution}

For comparison, let us compute the determinant without
Theorem~\ref{thm:determinant-of-triangular-matrix}, i.e., by using
cofactor expansion. If we expand the determinant along the first
column, we get:
\begin{equation*}
  \det (A)
  ~=~
  1 \begin{absmatrix}{rrr}
      2 & 6 & -7 \\
      0 & 3 & 33 \\
      0 & 0 & -1
    \end{absmatrix}
  - 0 \begin{absmatrix}{rrr}
      2 & 3 & 16 \\
      0 & 3 & 33 \\
      0 & 0 & -1
    \end{absmatrix}
  + 0 \begin{absmatrix}{rrr}
      2 & 3 & 16 \\
      2 & 6 & -7 \\
      0 & 0 & -1
    \end{absmatrix}
  - 0 \begin{absmatrix}{rrr}
      2 & 3 & 16 \\
      2 & 6 & -7 \\
      0 & 3 & 33 \\
    \end{absmatrix}.
\end{equation*}
The only non-zero term in the expansion is
\begin{equation*}
  1\begin{absmatrix}{rrr}
    2 & 6 & -7 \\
    0 & 3 & 33 \\
    0 & 0 & -1
  \end{absmatrix}.
\end{equation*}
We can in turns expand this $3\times 3$ determinant by the cofactor
method along the first column:
\begin{equation*}
  \begin{absmatrix}{rrr}
    2 & 6 & -7 \\
    0 & 3 & 33 \\
    0 & 0 & -1
  \end{absmatrix}
  ~=~ 2 \begin{absmatrix}{rr}
    3 & 33 \\
    0 & -1
  \end{absmatrix}
  - 0 \begin{absmatrix}{rr}
    6 & -7 \\
    0 & -1
  \end{absmatrix}
  + 0 \begin{absmatrix}{rr}
    6 & -7 \\
    3 & 33 \\
  \end{absmatrix}.
\end{equation*}
Again, the only non-zero term is the first term. In summary,
\begin{equation*}
  \det (A)
  ~=~
  \begin{absmatrix}{rrrr}
    1 & 2 & 3 & 16 \\
    0 & 2 & 6 & -7 \\
    0 & 0 & 3 & 33 \\
    0 & 0 & 0 & -1
  \end{absmatrix}
  ~=~ 1 \begin{absmatrix}{rrr}
    2 & 6 & -7 \\
    0 & 3 & 33 \\
    0 & 0 & -1
  \end{absmatrix}
  ~=~ 1\cdot 2
  \begin{absmatrix}{rr}
    3 & 33 \\
    0 & -1
  \end{absmatrix}
  ~=~ 1\cdot 2\cdot 3\cdot (-1)
  ~=~ -6.
\end{equation*}
Of course this is just the same as the product of the diagonal entries
of $A$, which is the point of
Theorem~\ref{thm:determinant-of-triangular-matrix}.


\section*{Exercises}

\begin{ex}
  Let $\Poly_3$ be the vector space of polynomials of degree at most
  3. Determine which of the following are bases for this vector space.

  \begin{enumerate}
  \item $\set{x^3+1,~x^2+x,~2x^3+x^2,~2x^3-x^2-3x+1}$.
  \item $\set{x+1,~x^3+x^2+2x,~x^2+x,~x^3+x^2+x}$.
  \end{enumerate}

  \begin{sol}
    \begin{enumerate}
    \item
      Yes. Suppose
      \begin{equation*}
        c_1(x^3+1) + c_2(x^2+x) + c_3(2x^3+x^2) +
        c_4(2x^3-x^2-3x+1) = 0.
      \end{equation*}
      Then collect equal powers of $x$:
      \begin{equation*}
        (c_1+2c_3+2c_4)x^3 + (c_2+c_3-c_4)x^2 +
        (c_2-3c_4)x + (c_1+c_4) = 0.
      \end{equation*}
      Does the system
      \begin{equation*}
        \begin{array}{c}
          c_1 + 2c_3 + 2c_4 = 0 \\
          c_2 + c_3 - c_4 = 0 \\
          c_2 - 3c_4 = 0 \\
          c_1 + c_4 = 0
        \end{array}
      \end{equation*}
      have a non-trivial solution? The only solution is
      \begin{equation*}
        c_1 = 0,\quad
        c_2 = 0,\quad
        c_3 = 0,\quad
        c_4 = 0,
      \end{equation*}
      and therefore, the polynomials are linearly independent. Since
      there are $4$ linearly independent polynomials in a
      $4$-dimensional space, they form a basis.
    \item Yes.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Determine whether the following is a basis for $\Poly_2$, the
  vector space of polynomials of degree at most $2$.
  \begin{equation*}
    \set{x^2+x+1,~2x^2+2x+1,~x+1}.
  \end{equation*}
\end{ex}

\begin{ex}
  Find a basis for the following subspace of $\Poly_2$:
  \begin{equation*}
    W=\sspan\set{1+x+x^2,~1+2x,~1+5x-3x^2}.
  \end{equation*}
\end{ex}

\begin{ex}
  Find a basis for the following subspace of $\Poly_3$:
  \begin{equation*}
    W = \sspan\set{
      1+x-x^2+x^3,~1+2x+3x^3,~-1+3x+5x^2+7x^3,~1+6x+4x^2+11x^3}.
  \end{equation*}
\end{ex}

\begin{ex}
  Extend the following linearly independent set of polynomials to a
  basis of $\Poly_3$:
  \begin{equation*}
    \set{x^3+x^2-x-1,~3x^3+2x^2+2x-1}.
  \end{equation*}
\end{ex}

\begin{ex}
  Let $V$ be a $5$-dimensional vector space.
  If you have $5$ linearly independent vectors in $V$, can you
  conclude that the vectors span $V$?
  \begin{sol}
    Yes, because the set of $5$ linearly independent vectors can be
    extended to a basis $B$ of $V$. But since $V$ is $5$-dimensional,
    $B$ has only $5$ elements, which must be the original $5$ vectors.
  \end{sol}
\end{ex}

\begin{ex}
  Let $V$ be a $5$-dimensional vector space.  If you have $6$ vectors
  in $V$, is it possible that they are linearly independent? Explain.
  \begin{sol}
    No. Since $V$ has a spanning set of size $5$, the $6$ vectors
    cannot be linearly independent by the Exchange Lemma.
  \end{sol}
\end{ex}

\begin{ex}
  Find a basis for the vector space of symmetric $3\times 3$-matrices%
  \index{matrix!symmetric}%
  \index{symmetric matrix}, i.e., matrices satisfying $A=A^T$.
  What is the dimension of this space?
\end{ex}

\begin{ex}
  Let $W$ be the subspace of $\Poly_3$ (over the field $\R$)
  consisting of all polynomials $p(x)$ that satisfy $p(3)=0$.
  Find a basis for $W$. What is the dimension of $W$?
\end{ex}

\begin{ex}
  Find a basis for
  $U=\set{A\in\Mat_{2,2} ~\left\vert~ A\begin{mysmallmatrix}{rr} 1 &
        0 \\ 1 & -1 \end{mysmallmatrix} = \begin{mysmallmatrix}{rr} 1
        & 1 \\ 0 & -1 \end{mysmallmatrix} A \right.}$.  What is the
  dimension of $U$?
\end{ex}

\begin{ex}
  \begin{enumerate}
  \item Let $k$ be a positive integer, let $W_k\subseteq\Seq_K$ be the
    subspace consisting of all sequences that are periodic%
    \index{periodic sequence}%
    \index{sequence!periodic} with period $k$ (see
    Exercise~\ref{ex:periodic}). Find a basis for $W_k$. What is its
    dimension?
  \item More difficult: find a basis for the infinite-dimensional
    vector space consisting of all periodic sequences of all periods.
  \end{enumerate}
  \begin{sol}
    \begin{enumerate}
    \item In case $k=3$, the following sequences form a basis for
      $W_3$:
      \begin{equation*}
        \begin{array}{l}
          (1,0,0,1,0,0,1,0,0,1,\ldots), \\
          (0,1,0,0,1,0,0,1,0,0,\ldots), \\
          (0,0,1,0,0,1,0,0,1,0,\ldots). \\
        \end{array}
      \end{equation*}
      Therefore, $W_3$ is a 3-dimensional space. For general $k$,
      the situation is analogous and the dimension of $W_k$ is $k$.
    \item Let $U$ be the set of all periodic sequences of all
      periods. We can find an (infinite) spanning set for $U$ by
      taking all the basis vectors for all of the spaces $W_k$:
      \begin{equation*}
        \begin{array}{ll}
          (1,1,1,1,1,1,1,1,1,1,\ldots) & \mbox{(period $1$)},\\[1ex]
          (1,0,1,0,1,0,1,0,1,0,\ldots) & \mbox{(period $2$)}, \\
          (0,1,0,1,0,1,0,1,0,1,\ldots) & \mbox{(period $2$)},\\[1ex]
          (1,0,0,1,0,0,1,0,0,1,\ldots) & \mbox{(period $3$)}, \\
          (0,1,0,0,1,0,0,1,0,0,\ldots) & \mbox{(period $3$)}, \\
          (0,0,1,0,0,1,0,0,1,0,\ldots) & \mbox{(period $3$)},\\[1ex]
          (1,0,0,0,1,0,0,0,1,0,\ldots) & \mbox{(period $4$)}, \\
          (0,1,0,0,0,1,0,0,0,1,\ldots) & \mbox{(period $4$)}, \\
          (0,0,1,0,0,0,1,0,0,0,\ldots) & \mbox{(period $4$)}, \\
          (0,0,0,1,0,0,0,1,0,0,\ldots) & \mbox{(period $4$)}, \\
        \end{array}
      \end{equation*}
      and so on. However, these sequences are not linearly
      independent. For example, we can obtain the sequence
      $(1,0,1,0,1,0,1,0,\ldots)$ of period $2$ as a linear combination
      of two sequences of period $4$, namely
      $(1,0,0,0,1,0,0,0,\ldots)$ and $(0,0,1,0,0,0,1,0,\ldots)$. By
      Proposition~\ref{prop:basis-from-spanning}, we know that it is
      possible to shrink the above spanning set to a basis by removing
      certain sequences. How exactly to do this is an interesting
      question. One way to construct a basis is to keep exactly those
      sequences of period $k$ that start with $\ell$ zeros, where
      $\gcd(\ell,k)=1$. Proving that this really works is an
      interesting project.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Let $K=\Q$, the field of rational numbers. Consider vectors of the
  form $a+b\sqrt{2}$ where $a,b$ are rational numbers. Show that this
  collection of vectors is a vector space over $\Q$ and give a basis
  for this vector space. What is its dimension?
  \begin{sol}
    When we add two number of the form $a+b\sqrt{2}$, we get another
    number of the same form. When we multiply a number of the form
    $a+b\sqrt{2}$ by a (rational) scalar, we get another number of the
    same form. Also, $0=0+0\sqrt{2}$ is of the required form. The 8
    axioms of a vector space are satisfied because all of them are
    laws of the arithmetic of real numbers. A basis is
    $\set{1,\sqrt{2}}$. By definition, the span of these gives the
    collection of vectors. To prove that they are linearly
    independent, assume $a+b\sqrt{2}=0$, where $a,b$ are rational
    numbers. If $b\neq 0$, then $\sqrt{2}=-\frac{a}{b}$, which cannot
    happen because $\sqrt{2}$ is irrational. If $a\neq 0$, then
    $\frac{1}{\sqrt{2}}=-\frac{b}{a}$, which again cannot happen since
    $\frac{1}{\sqrt{2}}$ is irrational. Hence both $a,b=0$. Therefore,
    $1$ and $\sqrt{2}$ are linearly independent over the rational
    numbers, and form a basis. The dimension of the space is $2$.
  \end{sol}
\end{ex}


\section{Column space, row space, and null space of a matrix}
\label{sec:null-space}

\begin{outcome}
  \begin{enumerate}
  \item Find a basis for the column space, row space, and null space
    of a matrix.
  \item Find the rank and nullity of a matrix.
  \end{enumerate}
\end{outcome}

There are three important spaces we can associate to a matrix. They
are called the column space, row space, and null space, and are
defined as follows.

\begin{definition}{Column space, row space, null space}{row-column-space}
  Let $A$ be an $m\times n$-matrix.  The \textbf{column space}%
  \index{column space}%
  \index{matrix!column space} of $A$, written $\col(A)$, is the span
  of the columns. The \textbf{row space}%
  \index{row space}%
  \index{matrix!row space} of $A$, written $\row(A)$, is the span of
  the rows.  The \textbf{null space}%
  \index{null space}%
  \index{matrix!null space} of $A$, written $\nullspace(A)$, is the
  set
  \begin{equation*}
    \nullspace(A) =\set{\vect{x} \mid A \vect{x} =\vect{0}}.
  \end{equation*}
\end{definition}

Note that the column space is a subspace of $\R^m$ and the null space
is a subspace of $\R^n$. The row space, on the other hand, is a set of
row vectors. It can be regarded as a subspace of $\R^n$, but only if
we regard $\R^n$ as the set of $n$-dimensional row vectors (and not
column vectors, as usual).

Before we give an example, recall that two matrices are called
\textbf{row equivalent}%
\index{matrix!row equivalence}%
\index{row equivalence} if one can be obtained from the other by
performing a sequence of elementary row operations. The point of
elementary row operations is that they do not affect the row space or
the null space of the matrix. (They do, however, affect the column
space). The following proposition makes this more precise.

\begin{proposition}{Effect of row operations}{row-operations-row-space}
  Let $A$ and $B$ be row equivalent matrices. Then $\row(A)=\row(B)$
  and $\nullspace(A)=\nullspace(B)$.
\end{proposition}

\begin{proof}
  The fact that elementary row operations do not change the null space
  is a special case of
  Theorem~\ref{thm:elementary-operations-and-solutions}, applied to a
  homogeneous system. To prove that they do not change the row space
  is also easy; we just need to look at each kind of elementary row
  operation. For example, adding a multiple of one row to another
  clearly does not change the span of the rows.
\end{proof}

\begin{example}{Basis of column space, row space, and null space}{basis-column-row-null}
  Find a basis for the column space, row space, and null space of the
  matrix
  \begin{equation*}
    A =
    \begin{mymatrix}{rrrrr}
      1 & 2 & 1 & 3 & 2 \\
      1 & 3 & 6 & 0 & 2 \\
      3 & 7 & 8 & 6 & 6 \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  The column space of $A$ is the span of the columns of $A$, i.e.,
  \begin{equation*}
    \col(A) = \sspan\set{
      \begin{mymatrix}{r} 1 \\ 1 \\ 3 \end{mymatrix},~
      \begin{mymatrix}{r} 2 \\ 3 \\ 7 \end{mymatrix},~
      \begin{mymatrix}{r} 1 \\ 6 \\ 8 \end{mymatrix},~
      \begin{mymatrix}{r} 3 \\ 0 \\ 6 \end{mymatrix},~
      \begin{mymatrix}{r} 2 \\ 2 \\ 6 \end{mymatrix}
    }.
  \end{equation*}
  To find a basis for the column space, we use the casting-out
  algorithm.  The {\rref} of $A$ is
  \begin{equation}\label{eqn:basis-column-row-null-1}
    \begin{mymatrix}{rrrrr}
      \circled{1} & 0 & -9 & 9 & 2 \\
      0 & \circled{1} & 5 & -3 & 0 \\
      0 & 0 & 0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation}
  Note that the first two columns of the {\rref} are pivot
  columns. Therefore, by the casting-out algorithm, the first two
  columns of $A$ form a basis for the column space. Thus, the
  following is a basis for the column space:
  \begin{equation*}
    \mbox{Basis of $\col(A)$:} \quad
    \set{
      \begin{mymatrix}{r} 1 \\ 1 \\ 3 \end{mymatrix},~
      \begin{mymatrix}{r} 2 \\ 3 \\ 7 \end{mymatrix}
    }.
  \end{equation*}
  The rows space of $A$ is the span of the rows of $A$, i.e.,
  \begin{equation*}
    \row(A) =
    \sspan \set{
      \begin{mymatrix}{rrrrr}
        1 & 2 & 1 & 3 & 2 \\
      \end{mymatrix},~
      \begin{mymatrix}{rrrrr}
      1 & 3 & 6 & 0 & 2 \\
      \end{mymatrix},~
      \begin{mymatrix}{rrrrr}
      3 & 7 & 8 & 6 & 6
      \end{mymatrix}
    }
  \end{equation*}
  We could find a basis of the row space by writing all three rows as
  column vectors and using the casting-out algorithm. However, there
  is an easier way. By
  Proposition~\ref{prop:row-operations-row-space}, the row space of
  $A$ is equal to the row space of the {\rref}
  {\eqref{eqn:basis-column-row-null-1}}. Moreover, the non-zero rows
  of the {\rref} are clearly linearly independent (no non-zero row can
  be a linear combination of other rows below it, because each
  non-zero row has a pivot entry). Therefore, the non-zero rows of the
  {\rref} form a basis of the row space.
  \begin{equation*}
    \mbox{Basis of $\row(A)$:} \quad
    \set{
      \begin{mymatrix}{rrrrr}
        1 & 0 & -9 & 9 & 2 \\
      \end{mymatrix},~
      \begin{mymatrix}{rrrrr}
        0 & 1 & 5 & -3 & 0 \\
      \end{mymatrix}
    }.
  \end{equation*}
  Finally, the null space of $A$ is just the solution space of the
  homogeneous system $A\vect{x}=\vect{0}$. Thus, finding a basis of
  the null space is the same as finding a set of basic solutions. From
  the {\rref}, we can easily find the general solution of
  $A\vect{x}=\vect{0}$, using three parameters $r$, $s$, $t$
  corresponding to the three non-pivot columns of
  {\eqref{eqn:basis-column-row-null-1}}. The general solution is:
  \begin{equation*}
    \vect{x} =
    r \begin{mymatrix}{r} 9 \\ -5 \\ 1 \\ 0 \\ 0 \end{mymatrix}
    + s \begin{mymatrix}{r} -9 \\ 3 \\ 0 \\ 1 \\ 0 \end{mymatrix}
    + t \begin{mymatrix}{r} -2 \\ 0 \\ 0 \\ 0 \\ 1 \end{mymatrix}.
  \end{equation*}
  Therefore, the following is a basis of the null space:
  \begin{equation*}
    \mbox{Basis of $\nullspace(A)$:} \quad
    \set{
    \begin{mymatrix}{r} 9 \\ -5 \\ 1 \\ 0 \\ 0 \end{mymatrix},~
    \begin{mymatrix}{r} -9 \\ 3 \\ 0 \\ 1 \\ 0 \end{mymatrix},~
    \begin{mymatrix}{r} -2 \\ 0 \\ 0 \\ 0 \\ 1 \end{mymatrix}
    }.
  \end{equation*}
\end{solution}

As the example shows, all three bases, for the column space, the row
space, and the null space of $A$, can be easily determined from the
{\rref}. In the next proposition, we use this information to determine
the dimensions of these three spaces. Recall from
Definition~\ref{def:rank} that the \textbf{rank}%
\index{rank}%
\index{matrix!rank} of a matrix is equal to the number of pivot
entries of its {\rref}.

\begin{proposition}{Dimension of column space, row space, and null space}{dimension-column-row-null}
  Let $A$ be an $m\times n$-matrix. Then the dimensions of the column
  space, row space, and null space of $A$ are as follows:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      \dim(\col(A)) &=& \rank(A), \\
      \dim(\row(A)) &=& \rank(A), \\
      \dim(\nullspace(A)) &=& n-\rank(A). \\
    \end{array}
  \end{equation*}
\end{proposition}

\begin{proof}
  Let $r=\rank(A)$.  Following the same method as in
  Example~\ref{exa:basis-column-row-null}, we can use the casting-out
  algorithm to find a basis for the column space. Since the {\rref} of
  $A$ has $r$ pivot columns, the basis has $r$ elements, and therefore
  $\dim(\col(A))=r$. Also, the {\rref} has $r$ non-zero rows (since
  each non-zero row contains exactly one pivot entry). These form a
  basis of the row space, and therefore $\dim(\row(A))=r$. Finally,
  the dimension of the null space is equal to the number of parameters
  in the general solution of the system of equations
  $A\vect{x}=\vect{0}$. There is one parameter for each non-pivot
  column, and since $A$ has $n$ columns and $r$ pivot columns, it
  follows that $\dim(\nullspace(A)) = n-r$.
\end{proof}

Among other things, the proposition states that the ``row rank'' of a
matrix (the dimension of its row space) is always equal to the
``column rank'' (the dimension of the column space). This fact is not
at all obvious when one first considers the definition of a matrix. It
is often called the \textbf{rank theorem}%
\index{rank theorem}%
\index{matrix!rank theorem} and is one of the deep and mysterious
facts of linear algebra. It means, for example, that if we do
elementary column operations instead of elementary row operations, we
end up with exactly the same number of pivots. Since the ``row rank''
and ``column rank'' are always equal, we are justified in simply
calling this quantity the ``rank'' of the matrix.

There is also a name for the dimension of the null space. It is called
the \textbf{nullity}%
\index{nullity}%
\index{matrix!nullity} of the matrix, and is written $\nullity(A)$.
The last part of Proposition~\ref{prop:dimension-column-row-null} is
also called the \textbf{rank-nullity theorem}%
\index{rank-nullity theorem}%
\index{matrix!rank-nullity theorem}, and is often written in the form
\begin{equation*}
  \rank(A) + \nullity(A) = n.
\end{equation*}

\begin{example}{Rank and nullity}{rank-nullity}
  Find the rank and nullity of the matrix
  \begin{equation*}
    A = \begin{mymatrix}{rrrrrr}
      1 & 2 & 1 & 3 & 2 \\
      1 & 3 & 6 & 0 & 2 \\
      1 & 2 & 1 & 3 & 2 \\
      1 & 3 & 2 & 4 & 0
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  The {\rref} of $A$ is
  \begin{equation*}
    \def\arraystretch{1.4}
    \begin{mymatrix}{rrrrrr}
      1 & 0 & 0 & 0 & \frac{13}{2} \\
      0 & 1 & 0 & 2 & -\frac{5}{2} \\
      0 & 0 & 1 & -1 & \frac{1}{2} \\
      0 & 0 & 0 & 0 & 0
    \end{mymatrix},
  \end{equation*}
  and so $\rank(A)=3$ and $\nullity(A) = 5-3 = 2$.
\end{solution}

We conclude this section with two useful theorems about matrices.

\begin{theorem}{}{equivalent-rank1}
  The following are equivalent for an $m\times n$-matrix $A$.
  \begin{enumerate}
  \item $\rank(A)=n$.
  \item $\row(A)=\R^n$, i.e., the rows of $A$ span $\R^n$.
  \item The columns of $A$ are linearly independent in $\R^m$.
  \item The $n\times n$-matrix $A^TA$ is invertible.
  \item $A$ is left invertible, i.e., there exists an
    $n\times m$-matrix $B$ such that $BA=I$.
  \item The system $A\vect{x}=\vect{0}$ has only the trivial solution.
  \end{enumerate}
\end{theorem}

\begin{theorem}{}{equivalent-rank2}
  The following are equivalent for an $m\times n$-matrix $A$.
  \begin{enumerate}
  \item $\rank(A)=m$.
  \item $\col(A)=\R^m$, i.e., the columns of $A$ span $\R^m$.
  \item The rows of $A$ are linearly independent in $\R^n$.
  \item The $m\times m$-matrix $AA^T$ is invertible.
  \item $A$ is right invertible, i.e., there exists an
    $n\times m$-matrix $B$ such that $AB=I$.
  \item The system $A\vect{x}=\vect{b}$ is consistent for
    every $\vect{b}\in\R^m$.
  \end{enumerate}
\end{theorem}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 4}

Show that there is an algorithm that produces the $k$ smallest elements of an unsorted set of $n$ elements in time $\mathcal{O}(n + k \log n)$.

You have to state the algorithm carefully and prove that it does what it is supposed to do.
You need also to prove that the algorithm runs in time $\mathcal{O}(n + k \log n)$.

\subsection*{Solution}

The desired algorithm \textsc{Sort-k-Min} which sorts $k$ smallest elements of array $A$ has been proposed below.

\begin{algorithm}[H]
\caption{\textsc{Sort-k-Min}}
\begin{algorithmic}[1]
\State \textsc{Build-Heap}($A$)
\For {$i \leftarrow 1$ : $k$}
\State $F[i] \leftarrow A[1]$
\State \textsc{Heap-Delete}($A$, $1$)
\EndFor
\end{algorithmic}
\end{algorithm}

As is shown, the objective is achieved by taking advantage of the heap data structure.
First, a min-heap is constructed once from all elements of the array.
This would cost a $\mathcal{O}(n)$ runtime.
To get the smallest element, each time the root element will be produced as smallest and then it is removed from the heap using a call to \textsc{Heap-Delete}($A$, 1).
Runtime of each iteration of the loop is thus $\mathcal{O}(\log n)$.
To get the first $k$ elements, the loop should be used $k$ times which results in a total runtime of $\mathcal{O}(n + k\log n)$.


	\begin{multicols}{2}

\subsection{Z-transform}
\begin{tabular}{p{7.75cm} p{4.75cm}}
$\mathcal{Z} \left\{ f_1(t) \pm f_2(t) \right\}=F_1(z)+F_2(z)$ & Addition\\
$\mathcal{Z} \left\{ af(t) \right\}= aF(z)$ & Multiplication by a Constant \\
$\mathcal{Z} \left\{ f(t-nT) \right\}=z^{-n}F(z)$ & Shifting \\
$\mathcal{Z} \left\{ f(t+kT) \right\}=z^{k}F(z)-z^{k}f(0)- \cdots - z f(kT-T)$ & Shifting (cont'd)\\
$\mathcal{Z} \left\{ e^{\mp at} f(t) \right\}=F(ze^{\pm at})$ & Complex Translation \\
$\lim_{k \rightarrow \infty} f(kT)= \lim_{z \rightarrow 0}F(z)$ & Initial Value Theorem \\ \hline
If $(1-z^{-1})F(z)$ has all singularities inside unit disk $|z|=1$, then & Final Value Theorem \\
$\lim_{k \rightarrow \infty} f(kT) = \lim_{z \rightarrow 1} (1-z^{-1})F(z)$ & \\
$\mathcal{Z} \left\{ \frac{\partial}{\partial a} f(t,a) \right\} = \frac{\partial}{\partial a} F(z,a)$& Partial differentiation
\end{tabular}

\subsection{Sampling Theory}
\vspace*{-0.25cm}
\begin{align*}
& \mathcal{Z} \left\{G_1(s)G_2(s)\right\}=G_1G_2(z)= G_2G_1(z) \quad \text{In General} \quad G_1(z)G_2(z) \neq G_1G_2(z)
\end{align*}
\vspace*{-0.175cm}
\begin{minipage}[t]{1\linewidth}
\includegraphics[width=1\linewidth]{samplerTesting-cropped.pdf}
\end{minipage}
\begin{align*}
& G(z) = \mathcal{Z} \left\{\left(\frac{1-e^{-s}}{s}\right) \left[ \frac{1}{s+1}\right] \left[\frac{1}{s}\right] \right\} \rightarrow G_1(z) = (1-z^{-1})\mathcal{Z} \left\{ G_{rest}(s) \right\}
\end{align*}
\begin{minipage}[h]{0.65\linewidth}
%\medbreak\noindent\minipage{\columnwidth}
\includegraphics[width=1.05\linewidth]{sampleEND-cropped.pdf}%\endminipage\medbreak
\end{minipage}
\begin{minipage}[h]{0.35\linewidth}
\[
G(z) = \frac{ze^{-1}+(1-ze^{-1})}{(z-1)(z-e^{-1})}
\]
\end{minipage}
$$ \text{ZOH(zero-hold-system)} \quad
f^\ast(t) = \sum_{k=-\infty}^\infty f(kT)\delta(t-kT)  \quad
G_{h}(s)= \frac{1-e^{-Ts}}{s}
$$
\subsection{Stability Test for Digital Systems}
\vspace*{-0.45cm}
\begin{align*}
& P(z) = a_0z^n+a_1z^{n-1} + \cdots + a_{n-1}z+a_n \quad G(z)= \frac{A(z)}{P(z)} \\
& \text{Stability Condition:} \quad P(z) \neq 0 \quad |z| \geq 1 \quad (\text{Draw Unit Circle to test stability})\\
& \text{Routh-Stability in Digital Domain: } s = \frac{z+1}{z-1} \quad z=\frac{s+1}{s-1}
\end{align*}
%\begin{table}
%\captionof{Table}{Jury-Marden Table}
\textbf{Jury-Marden Table Uses function P of z} \newline
%\vspace*{-0.25cm}
\begin{minipage}[t]{1\linewidth}
\begin{minipage}[h]{0.25\linewidth}
\begin{align*}
& b_k = \det \begin{bmatrix}
a_n & a_{n-1-k} \\
a_0 & a_{k+1}
\end{bmatrix} \\
& k = 0,1,  \cdots n-1 \\
& c_k = \det \begin{bmatrix}
b_{n-1} & b_{n-2-k} \\
b_0 & b_{k+1}
\end{bmatrix} \\
& k = 0,1,  \cdots n-1 \\
& q_k = \det \begin{bmatrix}
p_{3} & p_{2-k} \\
p_0 & p_{k+1}
\end{bmatrix} \\
& k =0,1,2
\end{align*}
\end{minipage}
\begin{minipage}[h]{0.75\linewidth}
\begin{tabular}{llllllll}
%\cline{1-5}
Row & $z^0$     & $z^1$     & $z^2$     &          & $z^{n-2}$ & $z^{n-1}$ & $z^n$ \\ %\cline{1-5}
1   & $a_n$     & $a_{n-1}$ & $a_{n-2}$ & $\cdots$ & $a_2$     & $a_1$     & $a_0$ \\
2   & $a_0$     & $a_1$     & $a_2$     & $\cdots$ & $a_{n-2}$ & $a_{n-1}$ & $a_n$ \\
3   & $b_{n-1}$ & $b_{n-2}$ & $b_{n-3}$ & $\cdots$ & $b_1$     & $b_0$     &       \\ %\cline{1-5}
4   & $b_0$     & $b_1$     & $b_2$     & $\cdots$ & $b_{n-2}$ & $b_{n-1}$ &       \\
5   & $c_{n-2}$ & $c_{n-3}$ & $c_{n-4}$ & $\cdots$ & $c_0$     &           &       \\
6   & $c_0$     & $c_1$     & $c_2$     & $\cdots$ & $c_{n-2}$ &           &      \\
2n-5& $p_3$ & $p_2$ & $p_1$ & $p_0$ & & \\
2n-4& $p_0$ & $p_1$ & $p_2$ & $p_3$ & & \\
2n-3& $q_2$ & $q_1$ & $q_0$ & & &
\end{tabular}
\end{minipage}
\end{minipage}

\textbf{Necessary and Sufficient Condition for Stability} \newline
\begin{minipage}[h]{0.55\linewidth}
\begin{enumerate}
\item $|a_n| < |a_0|$
\item $P(1) > 0$
\item \begin{align*}
P(-1) & > 0 \ \text{for n even } \\
& < 0 \ \text{for n odd}
\end{align*}
\item $b_{n-1}> |b_0|, |c_{n-2}|>|c_0|, \cdots |q_2| > |q_0|$
\end{enumerate}
\end{minipage}
\begin{minipage}[h]{0.5\linewidth}
\textbf{Special Case n =2} \newline
$P(z) =a_0z^2+a_1z+a_2$ \newline
\begin{tabular}{c c c}
$z^0$ & $z^1$& $z^2$ \\
$a_2$ & $a_1$ & $a_0$
\end{tabular} \newline
$P(z) \neq 0$ for $|z| \geq 1$ if and only if
\begin{enumerate}
\item $|a_2| < |a_0|$
\item $P(1) > 0$
\item $P(-1) > 0 \quad (n=2)$
\end{enumerate}
\end{minipage}

\textbf{Root Locus} presents the poles of the closed loop system when the gain K changes from zero to infinity.

\textbf{Construction of the Root Locus}

Open loop transfer function
$ \displaystyle \text{KH}\left( s \right)G\left( s \right) = K\frac{B(s)}{A(s)}$

m: the order of the \textbf{open-loop} numerator polynomial.

n: the order of the \textbf{open-loop} denominator polynomial. $q=n-m$

\textbf{Rule 1:} number of branches equals the number of poles of the
 open-loop transfer function

\textbf{Rule 2:} If the total number of poles and zeros of the open-loop
 system to the right of the s-point on the real axis is odd, then this
 point lies on the locus.

\textbf{Rule 3:} The locus starting point (K=0) are at the open-loop
poles and the locus ending points (K=$\infty$) are at the open loop zeros and
n-m branches terminate at infinity.

\textbf{Rule 4 and 5:} Slope of asymptotes of root locus as `s' approaches infinity. \newline Abscissa of the intersection between asymptotes of root locus and real-axis.
\[\sigma  = {\frac{\sum\limits_{i = 1}^n {{p_i}}  - \sum\limits_{i = 1}^m {{z_i}} }{q}} \quad \theta = \pm r{\frac{180}{q}} \quad \text{where r=1, 3, 5} \]

%\textbf{Rule 5:}

\textbf{Rule 6:} Break-away and break-in points. From the characteristic
equation

\[f\left( s \right) = A\left( s \right) + KB\left( s \right) = 0\ \ \ \ and\ \ \ \ K = - \frac{A\left( s \right)}{B\left( s \right)}\]

The break-away and break-in points can be found from

\[\frac{\text{dK}}{\text{ds}} = - \frac{A^{'}\left( s \right)B\left( s \right) - A\left( s \right)B^{'}\left( s \right)}{B^{2}\left( s \right)} = 0
\]

\textbf{Rule 7:} Angle of departure from complex poles or zeros.
Subtract from $180^o$ the sum of all angles from all other zeros and poles
of the open-loop system to the complex pole (or zero) with appropriate signs.

\[ \text{Z-transform: Definition} \quad F(z)=Z[f(t)]-Z[f(kT)]=\sum_{k=0}^{\infty}f(kT)z^{-k}\]
\end{multicols}
\newpage
\begin{multicols}{3}
\begin{align*}
& e^\ast(\infty)=\lim_{z \rightarrow 1} (1-z^{-1})E(z) \\
& K_p=\lim_{z \rightarrow 1} G(z), \quad e^\ast (\infty) = \frac{1}{1+K_p} \\
& e^\ast(\infty) = \frac{1}{K_v}, \quad K_v = \frac{1}{T} \lim_{z \rightarrow 1}(z-1) G(z)  \\
& e^\ast(\infty) = \frac{1}{K_a}, \quad K_a = \frac{1}{T^2} \lim_{z \rightarrow 1}(z-1)^2 G(z)
\end{align*}

{\bf Linear Factor Rule.}
For each factor of $Q$ of the form $(ax+b)^m$,
the partial fraction decomposition contains
the following sum of $m$ partial fractions:
\[
\frac{A_1}{ax+b} + \frac{A_2}{(ax+b)^2} + \cdots + \frac{A_m}{(ax+b)^m},
\]
where the $A_i$ are constants to be determined.

\medskip
\noindent
{\bf Quadratic Factor Rule.}
For each factor of $Q$ of the form $(ax^2+bx+c)^m$,
where $ax^2+bx+c$ is an irreducible quadratic,
the partial fraction decomposition contains
the following sum of $m$ partial fractions:
\[
\frac{A_1x+B_1}{ax^2+bx+c} + \frac{A_2x+B_2}{(ax^2+bx+c)^2} + \cdots
  + \frac{A_mx+B_m}{(ax^2+bx+c)^m},
\]
where the $A_i$ and $B_i$ are constants to be determined.
\begin{align*}
& x(k+2)-\frac{3}{2}x(k+1)+\frac{1}{2}x(k)=u(k), \text(x(0)=1,x(1)=5/2) \\
& [z^2X(z)-z^2x(0)-zx(1)]-\frac{3}{2}(zX(z)-zx(0)]+\frac{1}{2}X(z)=\frac{z}{z-1} \\
& [z^2-1.5z+0.5z]X(z)= \frac{z}{z-1}+z^2+(2.5-1.5)z \\
& X(z) = \frac{z[1+(z+1)(z-1]}{(z-1)(z-1)(z-0.5)}=\frac{z^3}{(z-1)^2(z-0.5)} \\
& \frac{X(z)}{z}=\frac{z^2}{(z-1)^2(z-0.5)}=\frac{A_{11}}{(z-1)^2}+\frac{A_{12}}{z-1}+\frac{A_{13}}{z-0.5}
\end{align*}
Geometric Sum $\sum\limits_{k = -N}^{N} {ar^{k - 1} = a\frac{1-r^{N}}{{1 - r}}} \sum_{i=0}^\infty a^i=\frac{1}{1-a}$
\textbf{Example Partial Fractions}
\begin{align*}
& \frac{20}{\left(s+3\right)\,\left(s^2+6\,s+25\right)} \rightarrow
\frac{5}{4\,\left(s+3\right)}-\frac{\frac{5\,s}{4}+\frac{15}{4}}{s^2+6\,s+25} \\
& \frac{5\,z}{4\,\left(z-{\mathrm{e}}^{-3}\right)}+\frac{5\,z\,{\mathrm{e}}^3\,\left(\cos\left(4\right)-z\,{\mathrm{e}}^3\right)}{4\,\left({\mathrm{e}}^6\,z^2-2\,\cos\left(4\right)\,{\mathrm{e}}^3\,z+1\right)} \ \text{Z-table} = \text{17}
\end{align*}
\includegraphics[width=1\linewidth]{NICEPictureRotate.png}
$\text{Ch.eqn} =\Delta P(z)=z^2+(K-4)z+0.8=0$ \newline
Inputting $z=1 \text{ and } z=-1$, $K=-0.8-1+4=2.2$, $K=(1)^2+4+0.8=5.8$, for stability $-1<K<1$.
\includegraphics[width=1\linewidth]{OgataTable.png}
% \begin{align*}
% & G_p(s)=\frac{K}{s(s+3)} \rightarrow \frac{1}{s(s+3)}
% \end{align*}
%https://www.wolframalpha.com/input/?i=Plot++(t-2)*heaviside(t-2)-+(t-5)*heaviside(t-5)
\includegraphics[width=1\linewidth]{heavsidePlot.png}
\begin{align*}
& \frac{1}{3}((t-2)u(t-2)- (t-5)u(t-5)) \rightarrow \frac{z^2 + z + 1}{3 (z - 1) z^4} \\
& =
\frac{1(z^{-3}+z^{-4}+z^{-5})}{3(1-z^{-1})} \quad \frac{1}{3}z^{-3}+\frac{2}{3}z^{-4}+z^{-5}+z^{-6}+ \cdots
\end{align*}
\end{multicols}

\newpage
\begin{sidewaysfigure}
\includegraphics[width=0.75\linewidth]{Table_z_Transforms.pdf}
\end{sidewaysfigure}
%\end{table}


\subsection{Test 2}
\begin{enumerate}
	\item Consider the function $f(x,y) = x^2 - 2x + y^2 - 4y + 7$.\\
	\begin{enumerate}[label=\alph*.]
		\item Find equations for an plot (if possible) the C-level curves of $f$ for $C = 3$ and $C = 1$.\\
		We will try to find the level curve for any $C$ and then plug in 1 and 3.
		\begin{align*}
			x^2 - 2x + y^2 - 4y + 7 &= C
			x^2 - 2x + 1 + y^2 - 4y + 4 &= C-2
			(x-1)^2 + (y-2)^2 &= C-2		.
		\end{align*}
		A circle of radius $\sqrt{C-2}$ centered at $(1,2)$.
		For $C = 1$, the level curve does not exist because the circle would have a radius of $\sqrt{2-1} = \sqrt{-1}$.
		For $C = 3$, the level curve is
		\begin{equation*}
			(x-1)^2 + (y-2)^2 = 1
		\end{equation*}

		\begin{figure}[H]
			\centering
			\includegraphics[scale=.25]{./additionalMaterials/test2_circle.png}
		\end{figure}

		\item Compute $\nabla f$.
		\begin{equation*}
			\nabla f = \langle f_x, f_y\rangle = \langle 2x-2, 2y-4 \rangle
		\end{equation*}

		\item Find the equation of the plane tangent to the surface $z = f(x,y)$ at the point $(x_0, y_0, z_0) = (2,4,7)$.
		\begin{equation*}
			\vec{n} = \langle f_x, f_y, -1\rangle = \langle 2x-2, 2y-4, -1 \rangle.
		\end{equation*}
		At $(2,4,7)$, $\vec{n} = \langle 2, 7, -1 \rangle$. So, the plane equation is
		\begin{equation*}
			\langle 2, 7, -1 \rangle \cdot \langle x-2, y-4, z-7 \rangle = 0
		\end{equation*}

		\item Perform one iteration of gradient descent on $f(x,y)$ with a learning rate $delta = 1/4$ starting from the point $(x_0,y_0) = (2,4)$.\\
		\begin{align*}
			(x_n, y_n) &= (x_{n-1},y_{n-1}) - \delta\nabla f \\
			(x_0, y_0) &= (2,4), \delta = 1/4 \text{, and } \nabla f = \langle 	2x-2, 2y-4 \rangle \\
			(x_1, y_1) &= (2,4) - \frac{1}{4} \langle 2(2)-2, 2(4)-4 \rangle \\
			&= (3/2, 3)
		\end{align*}
	\end{enumerate}

	\item Recall that for a differentiable function $f(x,y)$ and the unit vector $\hat{u} = \langle a, b \rangle$, we proved that $D_{\hat{u}}f = \nabla f \cdot \hat{u}$.
	\begin{enumerate}[label=\alph*.]
		\item Prove the statement ``The gradient is the direction of steepest ascent'' by showing that the directional derivative $D_{\hat{u}}f$ is maximized when $\hat{u}\parallel\nabla f$.
		\begin{equation*}
			D_{\hat{u}}f = \nabla f \cdot \hat{u}=\norm{\nabla f}\norm{\hat{u}}\cos{\theta} = \norm{\nabla f}\cos{\theta}.
		\end{equation*}
		This value is maximized when $\theta$ is a multiple of $2\pi$, meaning that the angle between $\nabla f$ and $\hat{u}$ is 0. This means that the maximum value of the directional derivative, the direction of steepest ascent, is in the same direction as $\nabla f$.\\

		\item State the limit definition of the directional derivative $D_{\hat{u}}f$. Starting from that definition, prove that $D_{\hat{u}}f = \nabla f\cdot\hat{u}$.\\
		\begin{align*}
			D_{\hat{u}}f &= \lim_{h \to 0}{\frac{f(x+ah, y+bh)}{h}} \text{ and} \hat{u}=\langle a, b \rangle \\
			&= \lim_{h \to 0}{\frac{f(x+ah, y+bh) - f(x+ah, y)}{h} + \frac{f(x+ah, y) - f(x,y)}{h}} \\
			&= b\lim_{h \to 0}{\frac{f(x, y+bh) - f(x,y)}{bh}} + a\lim_{h \to 0}{\frac{f(x+ah, y) - f(x,y)}{ah}} \\
			&= bf_y + af_x = \langle f_x, f_y \rangle \cdot \langle a, b \rangle = \nabla f \cdot \hat{u}
		\end{align*}
	\end{enumerate}

	\item Use the method of Lagrange Multipliers to find the maximum of the product of two numbers $x$ and $y$ given that $(x,y)$ is a coordinate pair in the 1st quadrant located on the unit circle centered at the origin. Begin by stating the objective function $f(x,y)$ and the constraint equation $g(x,y) = k$.\\
	\begin{itemize}
		\item Objective Function: $f(x,y) = xy$
		\item Constraint Equation: $g(x,y) = x^2 + y^2 = 1$, $x \geq 0$ and $y \geq 0$
	\end{itemize}
	\begin{equation*}
		F(x,y,\lambda) = xy + \lambda(1-x^2-y^2)
	\end{equation*}
	\begin{equation*}
		\frac{\partial F}{\partial x} = y-2\lambda x, \frac{\partial F}{\partial y} = x-2\lambda y \text{ and } \frac{\partial F}{\partial\lambda} = 1-x^2-y^2
	\end{equation*}
	\begin{equation*}
		\langle y-2\lambda x, x-2 \lambda y, 1-x^2-y^2 \rangle = \vec{0}
	\end{equation*}
	\begin{equation*}
		\begin{cases}
			y = 2\lambda x \\
			x = 2\lambda y \\
			x^2 + y^2 = 1
		\end{cases} \implies \begin{cases}
			\lambda = 1/2 \\
			x = 1/\sqrt{2} \\
			y = 1/\sqrt{2}
		\end{cases}.
	\end{equation*}
	So, the maximum product is $\frac{1}{\sqrt{2}} \cdot \frac{1}{\sqrt{2}} = \frac{1}{2}$ at $\left(\frac{1}{\sqrt{2}}, \frac{1}{\sqrt{2}}\right)$\\

	\item The function $p(x,y) = \frac{1}{\pi}\exp{\left(-(x-a)^2 - (y-b)^2\right)}$ is the probability density function of a bivariate normal distribution with mean $(a,b)$ and standard deviation $\frac{1}{\sqrt{2}}$. Show that the global maximum of $p(x,y)$ occurs at $(a,b)$.
	\begin{align*}
		p_x &= \frac{1}{\pi}(-2(x-a))\exp{(-((x-a)^2 + (y-b)^2))} \\
		p_y &= \frac{1}{\pi}(-2(y-b))\exp{(-((x-a)^2 + (y-b)^2))}
	\end{align*}
	$p_x = 0$ when $x=a$ and $p_y = 0$ when $y=b$
	$\implies (a,b)$ is a critical point.
	\begin{align*}
		p_{xx} &= \frac{1}{\pi}(-2(y-b))\exp{(-((x-a)^2 + (y-b)^2))} - 2\exp{(-((x-a)^2 + (y-b)^2))} \\
		&= \frac{1}{\pi}(4(x-a)^2 - 2)\exp{(-((x-a)^2 + (y-b)^2))} \\
		p_{yy} &= \frac{1}{\pi}(4(y-b)^2 - 2)\exp{(-((x-a)^2 + (y-b)^2))} \\
		p_{xy} &= p_{yx} = \frac{4}{\pi}(x-a)(y-b)\exp{(-((x-a)^2 + (y-b)^2))} \\
		p_{xx}(a,b) &= \frac{-2}{\pi}, p_{yy}(a,b) = \frac{-2}{\pi} \text{ and } p_{xy}(a,b) = p_{yx}(a,b) = 0
	\end{align*}
	\begin{equation*}
		H(a,b) = \begin{bmatrix}
		\frac{-2}{\pi} & 0 \\
		0 & \frac{-2}{\pi}
		\end{bmatrix}
	\end{equation*}
	\begin{equation*}
		\det{(H(a,b))} = \frac{4}{\pi^2}
	\end{equation*}
	$(a,b)$ is an extrema because $\det{(H(a,b))} > 0$.
	Since $f_{xx}(a,b) < 0$ and $f_{yy}(a,b) < 0$, $(a,b)$ is a maximum.
	$(a,b)$ is a global maximum because $p(x,y)$ is strictly decreasing as you move away from $(a,b)$.\\

	\item Let the C-level curve of the function $f(x,y)$ be parameterized by the VVF $\vec{r}(t) = \langle x(t), y(t) \rangle$. Use the chain rule to show that $\nabla f(\vec{r}(t))\perp\vec{r^\prime}(t)$ for all $t$.\\
	Since $\vec{r}(t)$ parameterizes a C-level curve of $f$,
	\begin{equation*}
		f\circ\vec{r}(t) = C.
	\end{equation*}
	Where $C$ is a constant.
	\begin{equation*}
		\frac{\mathrm{d}}{\mathrm{d}t}(f\circ\vec{r}(t)) = \frac{\mathrm{d}}{\mathrm{d}t}C
	\end{equation*}
	\begin{equation*}
		\frac{\partial f}{\partial x}\frac{\mathrm{d}x}{\mathrm{d}t} + \frac{\partial f}{\partial y}\frac{\mathrm{d}y}{\mathrm{d}t} = 0
	\end{equation*}
	\begin{equation*}
		\left<\frac{\partial f}{\partial x}, \frac{\partial f}{\partial y}\right> \cdot \left<\frac{\mathrm{d}x}{\mathrm{d}t}, \frac{\mathrm{d}y}{\mathrm{d}t}\right> = 0
	\end{equation*}
	\begin{align*}
		\nabla f(\vec{r}(t)) &\cdot \vec{r^\prime}(t) = 0 \\
		&\implies \nabla f(\vec{r}(t))\perp\vec{r^\prime}(t)
	\end{align*}
\end{enumerate}


\chapter{Vector and Tensor Fields}
So far, we've created a model of spacetime where each point as an associated set of cordinates $x^\mu$, and is given its own four-dimensional, real vector space $V$ with basis $\{\bm{\partial}_\mu\}$, where $\bm{\partial}_\mu$ is the partial differential operator with respect to the coordinate $x^\mu$.
If we take two points, $P$ and $Q$, and examine arbitrary vectors in their respective vector space, we get vectors $A^\mu \bm{\partial}_\mu \in V_P$ and $B^\mu \bm{\partial}_\mu \in V_Q$.
Although these vector spaces are nearly identical, they aren't the same vector space, so we have to way of comparing the two vectors we've just created.
What distinguishes them are the basis vectors; $\{\bm{\partial}_\mu\}$ represents the differential operators of functions taken at a certain point, so a more complete notation might be $A^\mu \bm{\partial}_\mu \mid_P$ and $B^\mu \bm{\partial}_\mu \mid_Q$.
Because we can no longer be confined to a single vector space, we need a way of talking about what happens in the uncountably infinite number of vector spaces which exist in our spacetime.

\section{Vector Fields}
The solution to this problem is to define what is known as a \emph{vector field}.
Since we've defined a coordinate system $(x^\mu)$ for our spacetime, we can imagine functions of those coordinates such that they are defined for every possible point.
We can then imagine taking the partial derivatives of these functions with respect to each of these coordinates.
\[ \bm{\partial}_\mu f(x^\mu) \tag{defined for every $x^\mu$} \]
We will use these differential operators $\partial_\mu$ as basis vectors in our vector field.
A \emph{vector field} is a collection of vector spaces $V_P$ defined at every point $P$ in a \emph{manifold} (like our spacetime), all of which share a common basis, and some function $f$ defined at those points.
It can be kind of hard to understand the distinction between a vector space and a vector field, so consider this example.
\begin{itemize}
    \item A vector field is like the velocity of wind at every point in a room.
    At each point, we can assign a vector which represents the speed and direction of the wind.
    While these vectors are all broadly measuring the same thing (the velocity of wind), it doesn't really make sense to add these vectors together, since they don't really live in the same space (what does it mean to add the wind at one location to the wind in another?).
    \item A vector space is where each of of those vectors live.
    This is like asking ``at a point $P$, what is every possible direction the wind could be blowing in?''.
    The vector space at point $P$ is separate from the vector space at point $Q$, since the wind at point $P$ isn't necessarily the same as the wind at point $Q$.
\end{itemize}
Mathematically, a vector field is a function
\begin{align*}
    f &: \mathcal{S} \to V_\mathcal{S} \\
      &: P \in \mathcal{S} \mapsto \vec{v} \in V_P
\end{align*}
where $S = \{p\}$ is a subset of our spacetime and $V_\mathcal{S}$ is the vector spaces defined for all point $P$ in $S$.
When we say that $\{\bm{\partial}_\mu\}$ is the basis for the vectors in our vector field, that means that $\bm{\partial}_\mu$ will be the derivative with respect to $x^\mu$ of our vector field (function) $f$ at the point $P$ with coordinates $(x^\mu)$.
In practice, we often leave the function $f$ ambiguous, and treat $\{\bm{\partial}_\mu\}$ as operators for an arbitrary vector field; this is kind of like how we know what the directions in spacetime might be $(t,x,y,z)$ but we don't know exactly \emph{what} we'll be measuring --- maybe it's wind velocity, maybe it's mass flux, maybe it's whatever vector-valued function we want --- but we know we'll be measuring it's rate of change in the directions of $(t,x,y,z)$.
In principle, this means that we can think of an arbitrary vector $A^\mu \bm{\partial}_\mu \in V_P$ in our spacetime as an operator $\mathcal{L}$.
If a function $f$ comes along, $\mathcal{L}$ can operate on it so that we evaluate the partial derivatives of $f$ at a point $P$, so
\[ \mathcal{L}f = A^\mu \bm{\partial}_\mu f \mid_P \;\in \mathbb{R}. \]
This choice of basis $\{\bm{\partial}_\mu\}$ is known as the \emph{coordinate basis}.

\section{Tensors Fields}
We have established that the basis of the vector space $V_P$ at a point $P$ is $\{\bm{\partial}_\mu\}$;
If we want to find the basis of the dual space $V^*_P$ at $P$, we need to find a map which takes a differential operator $\bm{\partial}_\mu$ and gives us a real number.
We call this basis $\{\vec{d}x^\nu\}$, so
\[ \langle \vec{d}x^\nu, \bm{\partial}_\mu \rangle = \delta^\nu_\mu. \]
Elements of $\{\vec{d}x^\nu\}$ are known as \emph{1-forms}. Just as all of the vectors with basis $\{\bm{\partial}_\mu\}$ in $V_\mathcal{S}$ formed a vector field, all of the covectors with basis $\{\vec{d}x^\mu\}$ in $V^*_\mathcal{S}$ form a covector field.
Now that we've defined a basis for the dual space, we can consider tensor product spaces which exist at points in our spacetime.
For example, a $(2,1)$ tensor $T$ would be an element of $V_P \otimes V_P \otimes V^*_P$, and so would have a basis $\bm{\partial}_\lambda \otimes \bm{\partial}_\mu \otimes \vec{d}x^\nu$.
Note that we don't really write which point we're evaluating this basis at; in practice this is contextual, not explicit, and remember that we often won't even write the basis in the first place, and  would just refer to the component $\tensor{T}{^{\lambda\mu}_\nu}$.
One way this is even useful is that we often care about the tensor evaluated at every point in spacetime, not just a particular point, so the ambiguity of the notation can help remind us of that.
Because we're describing tensors at different points in spacetime, we've now created a \emph{tensor field}; just as a vector field assigned vectors to each point in spacetime, a tensor field does the same with tensors.

\section{Vectors and the Coordinate Basis}
As a point of clarification, its helpful to go over exactly what each part of a vector $A^\mu \bm{\partial}_\mu$ in the coordinate basis actually is.
We said that the basis $\{\bm{\partial}_\mu\}$ is differential operators with respect to $x^\mu$ acting on some function $f$, but what about the components?
Well, since $A^\mu$ actually varies with $x^\mu$, we can think of $A^\mu$ as a function on the coordinates $(x^\mu)$.
Explicitly, this would make a vector
\[ A^\nu(x^\mu)\,\bm{\partial}_\nu. \]
Note that here, $(x^\mu)$ represents a point, whereas $\nu$ represents the coordinate directions, so $A^\nu$ contracts with $\partial_\nu$, not with $(x^\mu)$.
We could even replace $(x^\mu)$ with a point $(P)$ to get
\[ A^\nu (P)\, \bm{\partial}_\nu. \]
Because of this, we now have the most explicit form of a tensor yet; for a $(2,2)$ tensor $\Gamma$, we can write it as
\[ \tensor{\Gamma}{^{\alpha\beta}_{\lambda\mu}} = \tensor{\Gamma}{^{\alpha\beta}_{\lambda\mu}} (x^\nu) \,\bm{\partial}_\alpha \otimes \bm{\partial}_\beta \otimes \vec{d}x^\lambda \otimes \vec{d}x^\mu. \]
In practice, we only write out $\tensor{\Gamma}{^{\alpha\beta}_{\lambda\mu}}$, and we have to infer the rest.
Thanks, Index Gymnastics\footnote{If you haven't read any David Sedaris, I \emph{highly} recommend it.}!


\chapter{Algebra}

\section{Polynomial}
Shoeriert multiplication formulas:
$$(x + y)^2 = x^2 + 2xy + y^2$$
$$(x - y)^2 = x^2 - 2xy + y^2$$
$$(x + y + z)^2 = x^2 + y^2 + z^2 + 2xy + 2yz + 2zx$$
$$(x - y + z)^2 = x^2 + y^2 + z^2 - 2xy - 2yz + 2zx$$

Consequences of the above formulas:
$$(-x + y)^2 = (y - x)^2 = y^2 - 2xy + x^2$$
$$(-x - y)^2 = (-(x + y))^2 = (x + y)^2 = x2 + 2xy + y2$$

Formulas for 3rd degree:
$$(x + y)^3 = x^3 + y^3 + 3xy(x + y) = x^3 + 3x^2y + 3xy^2 + y^3$$
$$(x - y)^3 = x^3 - y^3 - 3xy(x - y) = x^3 - 3x^2y + 3xy^2 - y^3$$

Factor Rules:
$$x^2 - y^2 = (x - y)(x + y)$$
$$x^2 + y^2 = (x + y)^2 - 2xy = (x - y)^2 + 2xy$$
$$x^3 - y^3 = (x - y)(x^2 + xy + y^2)$$
$$x^3 + y^3 = (x + y)(x^2 - xy + y^2)$$

More Algebraic Formulas:
$$2(x^2 + y^2) = (x + y)^2 + (x - y)^2$$
$$(x - y)^2 = (x + y)^2 - 4xy$$
$$x^4 + y^4 = (x + y)(x - y)[(x + y)^2 - 2xy]$$

Factoring trinomials:
$$ ax^{2n} + bx^n + c = (ax^n + r)(x^n + \frac{s}{a}) ~~ \text{with} ~ \begin{cases}
                                                                  r + s = b \\
                                                                  r \times s = ac
                                                                 \end{cases}
$$

Polynomial remainder theorem (the remainder of the division of a polynomial $f(x)$ by a linear polynomial ${(x-a)}$ is equal to $f(a)$:
$$ f(x) = (x-a)q(x) + r(x) \implies f(a) = r(x) $$

Newton's binomial formula:
\[ (x+y)^n = \sum_{k=0}^{n}\binom{n}{k}x^ky^{n-k} \]

\section{Equations}
Quadratic equation solution:
$$ ax^2 + bx + c = 0 \rightarrow (x - x1)(x - x2) = 0 ~\text{with}~ x_{1,2}=\frac{-b\pm\sqrt{b^2-4ac}}{2a}$$

Rational root theorem (or rational root test, rational zero theorem, rational zero test or p/q theorem):
$$
\text{given } a_nx+^n+...+a_0=0 \rightarrow \text{ solutions are in the form } x=\frac{p}{q} \text{ with } \begin{cases}
p \text{ a factor of } a_0 \\
q \text{ a factor of } a_n
\end{cases}
$$

\section{Inequities TODO}


\section*{Exercises}

\begin{ex}\label{ex:orthogonal-basis-r3}
  In $\R^3$ with the usual dot product, find an orthogonal basis for
  \begin{equation*}
    \sspan\set{
      \begin{mymatrix}{c} 1 \\ 2 \\ 3 \end{mymatrix},~
      \begin{mymatrix}{c} 2 \\ 6 \\ 0 \end{mymatrix}
    }.
  \end{equation*}
  \begin{sol}
    $\vect{u}_1 = \begin{mymatrix}{c} 1 \\ 2 \\ 3 \end{mymatrix}$,
    $\vect{u}_2 = \begin{mymatrix}{c} 1 \\ 4 \\ -3 \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}
  In $\R^4$ with the usual dot product, find an orthogonal basis for
  \begin{equation*}
    \sspan\set{
      \begin{mymatrix}{c} 0 \\ 1 \\ 1 \\  0 \end{mymatrix},~
      \begin{mymatrix}{c} 3 \\ 0 \\ 2 \\ -1 \end{mymatrix}
    }.
  \end{equation*}
  \begin{sol}
    $\vect{u}_1 = \begin{mymatrix}{c} 0 \\ 1 \\ 1 \\ 0 \end{mymatrix}$,
    $\vect{u}_2 = \begin{mymatrix}{c} 3 \\ -1 \\ 1 \\ -1 \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}
  In $\R^4$ with the usual dot product, find an orthogonal basis for
  \begin{equation*}
    \sspan\set{
      \begin{mymatrix}{c} 1 \\ 0 \\ 1 \\  0 \end{mymatrix},~
      \begin{mymatrix}{c} 1 \\ 3 \\ 1 \\ -1 \end{mymatrix},~
      \begin{mymatrix}{c} 2 \\ 4 \\ 2 \\  2 \end{mymatrix}
    }.
  \end{equation*}
  \begin{sol}
    $\vect{u}_1 = \begin{mymatrix}{c} 1 \\ 0 \\ 1 \\ 0 \end{mymatrix}$,
    $\vect{u}_2 = \begin{mymatrix}{c} 0 \\ 3 \\ 0 \\ -1 \end{mymatrix}$,
    $\vect{u}_3 = \begin{mymatrix}{c} 0 \\ 1 \\ 0 \\ 3 \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{ccc}
      3 & -1 & 0 \\
      -1 & 5 & 2 \\
      0 & 2 & 3 \\
    \end{mymatrix},
  \end{equation*}
  and consider the vector space $\R^3$ with the inner product given by
  $\iprod{\vect{v},\vect{w}} = \vect{v}^T A\vect{w}$.
  Let
  \begin{equation*}
    \vect{v}_1 = \begin{mymatrix}{c} 1 \\ 0 \\ 2 \end{mymatrix},
    \quad
    \vect{v}_2 = \begin{mymatrix}{c} -1 \\ 1 \\ -5 \end{mymatrix},
    \quad\mbox{and}\quad
    \vect{v}_3 = \begin{mymatrix}{c} 2 \\ 2 \\ 3 \end{mymatrix}.
  \end{equation*}
  Apply the Gram-Schmidt procedure to
  $\vect{v}_1,\vect{v}_2,\vect{v}_3$ to find an orthogonal basis
  $\set{\vect{u}_1,\vect{u}_2,\vect{u}_3}$ for $\R^3$ with respect to
  the above inner product.
  \begin{sol}
    $\vect{u}_1 = \begin{mymatrix}{c} 1 \\ 0 \\ 2 \end{mymatrix}$,
    $\vect{u}_2 = \begin{mymatrix}{c} 1 \\ 1 \\ -1 \end{mymatrix}$,
    $\vect{u}_3 = \begin{mymatrix}{c} -1 \\ 1 \\ 0 \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{cccc}
      3 & 2 & 0 & 0 \\
      2 & 5 & 1 & 0 \\
      0 & 1 & 3 & 1 \\
      0 & 0 & 1 & 3 \\
    \end{mymatrix},
  \end{equation*}
  and consider the vector space $\R^4$ with the inner product given by
  $\iprod{\vect{v},\vect{w}} = \vect{v}^T A\vect{w}$.
  Let
  \begin{equation*}
    \vect{v}_1 = \begin{mymatrix}{c} 1 \\ -1 \\ 1 \\ 1 \end{mymatrix},
    \quad
    \vect{v}_2 = \begin{mymatrix}{c} 2 \\ 0 \\ 0 \\ 2 \end{mymatrix},
    \quad\mbox{and}\quad
    \vect{v}_3 = \begin{mymatrix}{c} 2 \\ 0 \\ 2 \\ 3 \end{mymatrix},
  \end{equation*}
  and let $W=\sspan\set{\vect{v}_1,\vect{v}_2,\vect{v}_3}$.  Apply the
  Gram-Schmidt procedure to $\vect{v}_1,\vect{v}_2,\vect{v}_3$ to find
  an orthogonal basis $\set{\vect{u}_1,\vect{u}_2,\vect{u}_3}$ for $W$
  with respect to the above inner product.
  \begin{sol}
    $\vect{u}_1 = \begin{mymatrix}{c} 1 \\ -1 \\ 1 \\ 1 \end{mymatrix}$,
    $\vect{u}_2 = \begin{mymatrix}{c} 1 \\ 1 \\ -1 \\ 1 \end{mymatrix}$,
    $\vect{u}_3 = \begin{mymatrix}{c} -1 \\ 1 \\ 1 \\ 0 \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the inner product space $C[0,2]$, with the inner product
  given by
  \begin{equation*}
    \iprod{p,q} = \int_{0}^{2} f(x)g(x)\,dx.
  \end{equation*}
  Use the Gram-Schmidt procedure to find an orthogonal basis for
  $\sspan\set{1,x,x^2}$.
  \begin{sol}
    $\vect{u}_1 = 1$, $\vect{u}_2 = x-1$, $x^2 - 2x + \frac{2}{3}$.
  \end{sol}
\end{ex}

\begin{ex}
  Find an orthonormal basis for the subspace of $\R^3$ from Exercise~\ref{ex:orthogonal-basis-r3}.
  \begin{sol}
    The orthonormal basis is $\displaystyle\set{
      \frac{\vect{u}_1}{\norm{\vect{u}_1}},
      \frac{\vect{u}_2}{\norm{\vect{u}_2}}
    } = \set{
      \frac{1}{\sqrt{14}} \begin{mymatrix}{c} 1 \\ 2 \\ 3 \end{mymatrix},
      \frac{1}{\sqrt{26}} \begin{mymatrix}{c} 1 \\ 4 \\
        -3 \end{mymatrix}
    }$.
  \end{sol}
\end{ex}


%%%%%%%%%%%%%%%
% Definitions %
%%%%%%%%%%%%%%%

\section{Definitions}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Basic Notions and Notation %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Basic Notions and Notation}

\textit{In the following, $\Omega$ is a set, $\CalF$ a \SigmaAlgebra\ on $\Omega$. If used, then $\mu$ is a measure. Otherwise, the measure is the probability measure $P$.}

\begin{definition}{1.1}{}

    Let $\mathcal{F}$ be a family of subsets of set $\Omega$. $\mathcal{F}$ is called a \emph{$\sigma$-algebra} if:

        \begin{itemize}
            \setlength{\parskip}{0em}
            \item \emph{Closed Under Complement}: $A \in \mathcal{F} \Rightarrow A^c \in \mathcal{F}$,
            \item \emph{Closed Under Arbitrary Union}: $A_n \in \mathcal{F}$ for integer $n \geq 1$ $\Rightarrow \bigcup_{n=1}^{\infty}A_n \in \mathcal{F}$,
            \item \emph{Contains Entire Set}: $\Omega \in \mathcal{F}$
        \end{itemize}

\end{definition}

\begin{definition}{1.2}{}
    Let $\mathcal{C}$ be a family of subsets of $\Omega$. There exists a $\sigma$-algebra which contains $\mathcal{C}$ \emph{and} which is contained in every $\sigma$-algebra that contains $\mathcal{C}$ (take intersection of all $\sigma$-algebras. Such $\sigma$-algebra is \emph{unique} and called \emph{smallest $\sigma$-algebra containing $\mathcal{C}$} or \emph{$\sigma$-algebra generated by $\mathcal{C}$}, denoted by $\sigma(\mathcal{C})$. Simplest example, let $A \subseteq \Omega$:

        \begin{align*}
            \sigma(A) = \{\emptyset, A, A^c, \Omega\}.
        \end{align*}
\end{definition}

\begin{definition}{}{Finite Measure Space}

    Let $(\Omega, \CalF, \mu)$ be a measure space. If $\mu(\Omega) < \infty$, then we call the measure space \emph{finite}.

\end{definition}

%%%%%%%%%%%%%%%%%%%%
% Random Variables %
%%%%%%%%%%%%%%%%%%%%

\subsection{Random Variables}

\begin{definition}{2.1.1}{}

    Let $A \subseteq \Omega$ and $\Indicator{A}$ be defined as follows:

        \begin{align*}
            \Indicator{A}(\omega) =
            \begin{cases}
                1, & \omega \in A \\
                0, & \omega \not\in A
            \end{cases}.
        \end{align*}

    Then $\Indicator{A}$ is a R.V. and called the \emph{indicator (function) of (events) A}.

\end{definition}

\begin{definition}{2.3}{Distribution Function}

    Let $X$ be a random variable. Then the function

        \begin{align*}
            F_X(x) &= P(X \leq x) = \\ &= P(X \in (-\infty, x]) = Q_X((-\infty, x]),
        \end{align*}

    for $x \in \mathbb{R}$ is called the \emph{distribution function} of $X$.

\end{definition}

%%%%%%%%%%%%%%%%%%%%%%%%%
% Expectation Integrals %
%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Expectation Integrals}

\begin{definition}{}{Indicator Integral}

    Let $A \subseteq \Omega$, then:

        \begin{align*}
            \int_{\Omega} \Indicator{A} \,d\mu = \mu(A).
        \end{align*}

\end{definition}

\begin{definition}{}{Simple Function}

    Let $f: \Omega \to \mathbb{R}$ be a \emph{simple function}, then $f$ takes finitely many values. Formally, if $I$ is a finite index set, $(A_i)_{i \in I}$ a famility of \emph{disjoint} subsets of $\Omega$ and $(c_i)_{i \in I}$ a family of real numbers, then:

        \begin{align*}
            f(\omega) = \sum_{i \in I} c_i \Indicator{A_i}(\omega).
        \end{align*}

\end{definition}

\begin{definition}{}{Lebesgue Integral for Expectation}

    Let $X$ be a random variable. Then we write:

        \begin{align*}
            EX = \int_{\Omega} X \,dP.
        \end{align*}

\end{definition}

\begin{definition}{}{Non-negative, Measurable Lebesgue Integral}

    Let $f: \Omega \to \overline{\mathbb{R}}$ be a \emph{non-negative}, measurable function and $(f_n)_{n=1}^{\infty}$ a sequence of \emph{non-negative, simple} functions sucht that $\lim_{n \to \infty} f_n = f$. Then

        \begin{align*}
            \int_{\Omega} f \, d\mu = \lim_{n \to \infty} f_n \, d\mu.
        \end{align*}

\end{definition}

\begin{definition}{}{Lebesgue Integral}

     Let $f: \Omega \to \overline{\mathbb{R}}$ be a measurable function. The \emph{Lebesgue Integral} of $f$ is defined as:

        \begin{align*}
            \int_{\Omega} f \,d\mu = \int_{\Omega} f^+ \,d\mu - \int_{\Omega} f^- \,d\mu,
        \end{align*}

    where $f^+ = \max\{f, 0\}$ and $f^-=\max\{-f, 0\}$, if at least one of the integrals on the right-hand side is finite. If both are infinite, then we say that the Lebesgue Integral of $f$ does not exist.

\end{definition}

\begin{definition}{}{Restricted Integration}

    Let $A \in \CalF$ and $f: \Omega \to \overline{\mathbb{R}}$ is a measurable function, then we define:

        \begin{align*}
            \int_A f \, d\mu = \int_{\Omega} \Indicator{A} f \, d\mu,
        \end{align*}

    when the integral of $\Indicator{A} f$ w.r.t $\mu$ exists.

\end{definition}

\begin{definition}{3.7}{Absolute Continuity}

    Let $\mu$ and $\nu$ be measures on \SigmaAlgebra\ $\CalF$ such that for some $\CalF$-measureable $g: \Omega \to \mathbb{R}$:

        \begin{align*}
            \nu(A) = \int_{\Omega} \Indicator{A} g \, d\mu = \int_{A} g \mu(dx),
        \end{align*}

    for all $A \in \CalF$. Then $\nu$ is called \emph{absolutely continuous} with respect to $\mu$ and $g$ is called the \emph{density} or \emph{Radon-Nikodym derivative} (Notation: $g = \frac{d\nu}{d\mu}$).

\end{definition}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Convergence of Measurable Functions %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Convergence of Measurable Functions}

\begin{definition}{}{$\mu$-Almost Everywhere Finite}

    Let $f: \Omega \to \overline{\mathbb{R}}$ be $\CalF$-measurable, then $f$ is said to be \emph{$\mu$-almost everywhere} ($\mu$-a.e.) finite if $\mu(|f| = \infty) = 0$.

\end{definition}

\begin{definition}{}{Almost Surely Finite}

    Let $f: \Omega \to \overline{\mathbb{R}}$ be $\CalF$-measurable, then $f$ is said to be \emph{almost surely} (a.s.) finite if $P(|f| = \infty) = 0$ $\Leftrightarrow P(|f| < \infty) = 1$.

\end{definition}

\begin{definition}{5.1}{$\mu$-Almost Everywhere Convergence}

    Let $(f_n)_{n=1}^{\infty}$ be $\CalF$-measurable functions. The $f_n$ are said to \emph{converge $\mu$-almost everywhere} to a \emph{$\mu$-a.e. finite} $f: \Omega \to \overline{\mathbb{R}}$ as $n \to \infty$ if there exists an $A \in \CalF$ s.t. $\mu(A) = 0$ and

        \begin{align*}
            \lim_{n \to \infty} f_n(\omega) = f(\omega) \in \mathbb{R}, \quad \forall \omega \in A^C.
        \end{align*}

    \emph{Notation:} $\lim_{n \to \infty} f_n = f$ ($\mu$-a.e.) or $f_n \to f$ ($\mu$-a.e.).

\end{definition}

\begin{definition}{5.1}{Almost Sure Convergence}

    Let $(f_n)_{n=1}^{\infty}$ be $\CalF$-measurable functions. The $f_n$ are said to \emph{converge almost surely} to a \emph{a.s. finite} $f: \Omega \to \overline{\mathbb{R}}$ as $n \to \infty$ if there exists an $A \in \CalF$ s.t. $P(A) = 0$ and

        \begin{align*}
            \lim_{n \to \infty} f_n(\omega) = f(\omega) \in \mathbb{R}, \quad \forall \omega \in A^C.
        \end{align*}

    \emph{Notation:} $\lim_{n \to \infty} f_n = f$ (a.s.) or $f_n \to f$ (a.s.).

\end{definition}

\begin{definition}{5.2}{Convergence in Measure}

    Let $(f_n)_{n=1}^{\infty}$ be $\CalF$-measurable functions. The $f_n$ are said to \emph{converge in measure $\mu$} to a \emph{$\mu$-a.e. finite} $f: \Omega \to \overline{\mathbb{R}}$ as $n \to \infty$ if

        \begin{align*}
            \lim_{n \to \infty} \mu(|f_n - f| \geq \varepsilon) = 0, \quad \forall \varepsilon > 0.
        \end{align*}

    \emph{Notation:} $\mu-\lim_{n \to \infty} f_n = f$.

\end{definition}

\begin{definition}{5.2}{Convergence in Probability}

    Let $(f_n)_{n=1}^{\infty}$ be $\CalF$-measurable functions. The $f_n$ are said to \emph{converge in probability} to a \emph{a.s. finite} $f: \Omega \to \overline{\mathbb{R}}$ as $n \to \infty$ if

        \begin{align*}
            \lim_{n \to \infty} P(|f_n - f| \geq \varepsilon) = 0, \quad \forall \varepsilon > 0.
        \end{align*}

\end{definition}

\begin{definition}{}{Bounded in Measure}

    Let $(f_n)_{n=1}^{\infty}$ be a sequence of measurable functions, then it is \emph{bounded in measure $\mu$} if

        \begin{align*}
            \lim_{K \to \infty} \mu(|f_n| \geq K) = 0,
        \end{align*}

    for any $n \geq 1$.

\end{definition}

\begin{definition}{}{Bounded Uniformly in Measure}

    Let $(f_n)_{n=1}^{\infty}$ be a sequence of measurable functions, then it is \emph{bounded in measure $\mu$, uniformly in $n$} if

        \begin{align*}
            \lim_{K \to \infty} \sup_{n \geq 1} \mu(|f_n| \geq K) = 0.
        \end{align*}

\end{definition}

\begin{definition}{}{Finite Second Moment}

    Let $X$ be a random variable. Then $X$ has \emph{finite second moment} if $EX^2 < \infty$.

\end{definition}

%%%%%%%%%%%%%%%%
% Independence %
%%%%%%%%%%%%%%%%

\subsection{Independence of Events and Random Variables}

\begin{definition}{6.5}{$\lambda$-system}

    Let $\Lambda$ be a family o subsets of $\Omega$. Then $\Lambda$ is a $\lambda$-system, if it satisfies all of the following properties:

        \begin{enumerate}[(i)]
            \setlength{\parskip}{0em}
            \item (Contains whole set) $\Omega \in \Lambda$;
            \item (Closed under Subset Set Subtraction) if $A, B \in \Lambda$, such that $B \subset A$, then $A \setminus B \in \Lambda$;
            \item (Closed under Disjoint Union) if $(A_n)_{n=1}^{\infty}$ is a \emph{pairwise disjoint} sequence, i.e. $A_i \cap A_j = \emptyset$ for $i \neq j$, of subsets, such that $A_i \in \Lambda$ for $i = 1,2,\hdots$, then $\bigcup_{n=1}^{\infty} \in \Lambda$.
        \end{enumerate}

\end{definition}

\begin{definition}{}{$\pi$-system}

    Let $\Pi$ be a family of subsets of $\Omega$. Then $\Pi$ is a $\pi$-system, if it is closed under finite intersections, i.e. $A,B \in \Pi$ $\Rightarrow$ $A \cap B \in \Pi$.

\end{definition}

\begin{definition}{Ws 5, 1}{$\sigma$-Finite Measure}

    Let $\mu$ be a measure, then $\mu$ is called \emph{$\sigma$-finite} if there exists an increasing sequence $(\Omega_n)_{n=1}^{\infty}$ in $\CalF$, such that $\mu(\Omega_n) < \infty$ for all $n \geq 1$ and $\bigcap_{n=1}^{\infty} \Omega_n = \Omega$.

\end{definition}

%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Conditional Expectation %
%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Conditional Expectation}

\begin{definition}{8.1}{Sub-$\sigma$-Algebra Measurable}

    Let $Y$ be a random variable and $\CalG \subset \CalF$ be a \SigmaAlgebra. Then $Y$ is $\CalG$-measurable if $Y^{-1}(F) \in \CalG$ for any $F \in \mathcal{B}(\mathbb{R})$.

\end{definition}

\begin{definition}{8.2}{Conditional Expectation}

    Let $X,Y$ be random variables such that $E|X| < \infty$ and $\CalG \subset \CalF$ be a \SigmaAlgebra. Let $Y$ satisfy the following properties:

        \begin{enumerate}[(i)]
            \setlength{\parskip}{0em}
            \item $Y$ is $\CalG$-measurable and
            \item for any $A \in \CalG$:

                    \begin{align*}
                        \int_{A} Y \, dP = \int_{A} X \, dP
                        \iff E(\Indicator{A}Y) = E(\Indicator{A}X),
                    \end{align*}
        \end{enumerate}

    then $Y$ is called the \emph{conditional expectation} with respect of $\CalG$ of $X$ and we write $Y = E(X|\CalG)$.

\end{definition}


\subsection{Eigenvectors as a basis}

If eigen vectors space space, we can write

\(v=\sum_i \alpha_i | \lambda_i\rangle \)

Under what circumstances do they span the entirity?