\documentclass[a4paper,10pt]{article}
\usepackage{mystyle}

\begin{document}

\section{Matrices and Vector Spaces}

\begin{defn}[Echelon Form]
	A matrix is in Echelon form if:
	\begin{enumerate}
		\item all non-zero rows come before zero rows
		\item below each pivot is a column of zeros
		\item each pivot is to the right of the previous pivot
	\end{enumerate}

	If each pivot is $1$ then the matrix is in row echelon form and if
	there is a column of zeros above each pivot, the matrix is in reduced
	row echelon form (RRE).
\end{defn}

\begin{defn}[Rank]
	Let $A$ be an $n \times m$ matrix. Its rank is equal to the number of
	non-zero rows in its echelon form. This is denoted by $\rho(A)$.
\end{defn}

\begin{defn}[Subspace]
	Let $U \subset \RR^n$. $U$ is a subspace if:
	\begin{enumerate}
		\item $\bm{0} \in U$
		\item $U$ is closed under addition and scalar multiplication
	\end{enumerate}
\end{defn}

\begin{defn}[Vector Space]
	$V$ is a vector space of a field $\FF$ if:
	\begin{enumerate}
		\item addition is commutative and associative
		\item neutral and inverse additive elements exist
		\item scalar multiplication is commutative and associative
		\item $1$ is the neutral scalar
		\item $(a+b)\bm{d} = a\bm{d} + b\bm{d} \quad \forall a,b \in \FF, \bm{d} \in V$
		\item $(\bm{a} + \bm{b})d = \bm{a}d + \bm{b}d \quad \forall \bm{a},\bm{b} \in V, d \in \FF$
	\end{enumerate}
\end{defn}

\begin{defn}[Null Space]
	If $A \in M_{m \times n}(\RR)$, the null space is defined as:
	\[
		N(A) = \{ \bm{x} \in \RR^n \mid A\bm{x} = \bm{0} \}
	\]
\end{defn}

\begin{prop}
	If $A \in M_{m \times n}(\RR)$, the null space $N(A)$ is a
	subspace of $\RR^n$.
\end{prop}

\begin{proof}
	It is clear that $\bm{0} \in N(A)$. It remains to be shown
	that addition and scalar multiplication are closed.

	Let $x,y \in \RR^n$. We know that $\RR^n$ is a vector space,
	hence $A(x+y) = Ax + Ay = \bm{0}$.

	Let $x \in \RR^n, \, a \in \RR$. $A(ax) = a(Ax) = a\bm{0} = \bm{0}$.
\end{proof}

\begin{defn}[Image Space]
	$Im A = \{ \bm{y} \in \RR^m \mid \bm{y} = A\bm{x}, \bm{x} \in \RR^n \}$
\end{defn}

\begin{prop}
	$Im A$ is a subspace of $\RR^m$.
\end{prop}

\begin{proof}
	We know that $\bm{0} \in \RR^n$ hence $A\bm{0} = 0 \in Im A$.

	Let $y_1 = Ax_1, y_2 = Ax_2 \in \RR^m$.
	\[
		y_1 + y_2 = Ax_1 + Ax_2 = A(x_1 + x_2)
	\]
	\[
		x_1 + x_2 \in \RR^n \Rightarrow y_1 + y_2 \in Im A
	\]

	Let $k \in \RR, y \in Im A$.
	\[
		ky = kAx = Akx
	\]
	\[
		kx \in \RR^n \Rightarrow ky \in Im A
	\]
\end{proof}

\begin{ex}
	Find $N(A)$ and $Im(A)$ where
	\[
		A =
		\begin{pmatrix}
			1  & 3 & -2 \\
			-2 & 4 & 4
		\end{pmatrix}
	\]

	First we find the RRE of $A$:
	\[
		\begin{pmatrix}
			1  & 3 & -2 \\
			-2 & 4 & 4
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 3  & -2 \\
			0 & 10 & 0
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 3 & -2 \\
			0 & 1 & 0
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 0 & -2 \\
			0 & 1 & 0
		\end{pmatrix}
	\]

	$N(A)$:
	\[
		\begin{pmatrix}
			1 & 0 & -2 \\
			0 & 1 & 0
		\end{pmatrix}
		\begin{pmatrix}
			x_1 \\
			x_2 \\
			x_3
		\end{pmatrix}
		=
		\begin{pmatrix}
			0 \\
			0 \\
			0
		\end{pmatrix}
	\]
	\[
		\Rightarrow x_2 = 0, \quad x_1 = 2x_3
	\]
	\[
		\Rightarrow N(A) = k
		\begin{pmatrix}
			2 \\
			0 \\
			1
		\end{pmatrix}
		, k \in \RR
	\]

	$Im(A)$:
	\[
		\begin{pmatrix}
			y_1 \\
			y_2 \\
			y_3
		\end{pmatrix}
		=
		\begin{pmatrix}
			1  & 3 & -2 \\
			-2 & 4 & 4
		\end{pmatrix}
		\begin{pmatrix}
			x_1 \\
			x_2 \\
			x_3
		\end{pmatrix}
		=
		\begin{pmatrix}
			x_1   + 3x_2 - 2x_3  \\
			-2x_1 + 4x_2 + 4x_3
		\end{pmatrix}
	\]
	\[
		= x_1
		\begin{pmatrix}
			1 \\
			-2
		\end{pmatrix}
		+ x_2
		\begin{pmatrix}
			3 \\
			4
		\end{pmatrix}
		+ x_3
		\begin{pmatrix}
			-2 \\
			4
		\end{pmatrix}
	\]

	Note that the vectors in this expression are simply the columns of $A$, in fact in general:
	\[ Im(A) = \{x_1 a_{i1} + \cdots + x_1 a_{in}\} \]
	Hence the image space is sometimes referred to as the column space.
\end{ex}

Another example of a vector space is the space of polynomials over a field $\FF$:
\[
	\FF[x] =
	\{a_0 + a_1 x + \cdots + a_n x^2 \mid a_i \in \FF, n \in \NN_0\}
\]
e.g. $\RR[x]$ is a vs over $\RR$.
Addition and scalar multiplication work as expected.

We can also define a vs on a set of funtions:
\[
	V = \{f:[0,1] \to \RR\}
\]
Addition:
\[
	(g+f)(x) = g(x) + f(x)
\]
Scalar multiplication:
\[
	(\alpha f)(x) = \alpha (f(x)), \alpha \in \RR
\]

% TODO: prove these examples ^^.

\begin{defn}[Transpose]
	If $A = a(i,j) \in M_{m \times n}(\RR)$, then its transpose is
	\[ A^T = a(j,i) \]
	i.e its columns are swapped with its rows.
\end{defn}
\begin{defn}[Symmetric]
	A matrix $A \in M_{n}(\RR)$ is called symmetric iff
	\[ A^T = A. \]
\end{defn}

Properties of $A^T$:
\[
	{(A^T)}^T = A
\]
\[
	{(A+B)}^T = A^T + B^T
\]
\[
	{(AC)}^T = C^T A^T
\]

\section{Spanning sets, linear independence, bases}

\begin{defn}[Linear Combination]
	Let $V$ be a vector space over $\RR$. A linear combination of vectors in $V$ is given as:
	\[
		\alpha_1 v_1 + \alpha_2 v_2 + \cdots + \alpha_r v_r, \quad
		\alpha_i \in \RR, \, v_i \in V
	\]
\end{defn}

\begin{prop}
	$U = \{
		\alpha_1 v_1 + \alpha_2 v_2 + \cdots + \alpha_r v_r \mid
		\alpha_i \in \RR, \, v_i \in V
	\}$ is a subspace.
\end{prop}

\begin{proof}
	\[
		\alpha_i = 0 \, \forall i \Rightarrow 0 + \cdots + 0 = 0
	\]
	\[
		\Rightarrow \bm{0} \in U
	\]
	\[
		a = \sum_{i=0}^r \alpha_i v_i \quad
		b = \sum_{i=0}^r \beta_i v_i \\
	\]
	\begin{align*}
		\Rightarrow a + b &= \sum (\alpha_i v_i + \beta_i v_i) \\
		&= \sum (\alpha_i + \beta_i) v_i \in U
	\end{align*}

	\[
		a = \sum \alpha_i v_i \quad \lambda \in \RR
	\]
	\[
		\lambda a = \sum \lambda \alpha_i v_i \quad
		\lambda \alpha_i \in \RR \, \forall i
		\Rightarrow \lambda a \in U
	\]
\end{proof}

\begin{defn}[Spanning Set]
	Let $V$ be a vector space and $W$ be a subspace. A set of vectors
	$\{v_1, \ldots, v_r\} \subset V$ is a spanning set for $W$ if
	\[
		W =
		\{\alpha_1 v_1 + \cdots + \alpha_r v_r \mid \alpha_i \in \RR\}
		= sp\{v_1, \ldots, v_r\}
	\]
	We say that $\{v_1, \ldots, v_r\}$ spans $W$.
\end{defn}

\begin{ex}
	If $A = \begin{pmatrix}
		1  & 3 & -2 \\
		-2 & 4 & 4
	\end{pmatrix}$ we know that
	\[
		Im(A) = x_1
		\begin{pmatrix}
			1 \\
			-2
		\end{pmatrix}
		+ x_2
		\begin{pmatrix}
			3 \\
			4
		\end{pmatrix}
		+ x_3
		\begin{pmatrix}
			-2 \\
			4
		\end{pmatrix}
	\]

	We can also say that
	\[
		Im(A) = sp\left\{
			\begin{pmatrix}
				1 \\
				-2
			\end{pmatrix}
			,
			\begin{pmatrix}
				3 \\
				4
			\end{pmatrix}
			,
			\begin{pmatrix}
				-2 \\
				4
			\end{pmatrix}
		\right\}
	\]
	since every element in $Im(A)$ is a linear combination of these vectors.
\end{ex}

\begin{ex}
	\[
		\left\{
			\begin{pmatrix}
				1 \\
				0 \\
				0
			\end{pmatrix}
			,
			\begin{pmatrix}
				0 \\
				1 \\
				0
			\end{pmatrix}
			,
			\begin{pmatrix}
				0 \\
				0 \\
				1
			\end{pmatrix}
		\right\}
	\]
	spans $\RR^3$.
\end{ex}

\begin{ex}
	Let
	\[
		W = \left\{
			\begin{pmatrix}
				a + b + 2c \\
				b + c \\
				a + c
			\end{pmatrix}
			\mid a,b,c \in \RR
		\right\}
		\subset \RR^3
	\]
	then
	\[
		w = a
		\begin{pmatrix}
			1 \\
			0 \\
			1
		\end{pmatrix}
		+ b
		\begin{pmatrix}
			1 \\
			1 \\
			0
		\end{pmatrix}
		+ c
		\begin{pmatrix}
			2 \\
			1 \\
			1
		\end{pmatrix}
		\forall w \in W
	\]
	\[
		\Rightarrow W = sp\left\{
			\begin{pmatrix}
				1 \\
				0 \\
				1
			\end{pmatrix}
			,
			\begin{pmatrix}
				1 \\
				1 \\
				0
			\end{pmatrix}
			,
			\begin{pmatrix}
				2 \\
				1 \\
				1
			\end{pmatrix}
		\right\}
	\]

	Also, since
	\[
		\begin{pmatrix}
			2 \\
			1 \\
			1
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 \\
			0 \\
			1
		\end{pmatrix}
		+
		\begin{pmatrix}
			1 \\
			1 \\
			0
		\end{pmatrix},
	\]
	\[
		W = sp\left\{
			\begin{pmatrix}
				1 \\
				0 \\
				1
			\end{pmatrix}
			,
			\begin{pmatrix}
				1 \\
				1 \\
				0
			\end{pmatrix}
		\right\}
	\]
\end{ex}

\begin{defn}[Linear Independence]
	Let $V$ be a vector space. A set of vectors
	$\{v_1, \ldots, v_r \mid v_i \in V \}$
	is linearly independent if
	\[
		\alpha_1 v_1 + \cdots + \alpha_r v_r = 0
		\implies \alpha_1 = \cdots = \alpha_r = 0.
	\]
\end{defn}

\begin{thm}
	Let $v_1, \ldots, v_r \in \RR^n$ and $A$ be the matrix whose
	columns correspond to the vectors $v_i$, then the set
	$\{v_1, \ldots, v_r\}$ is linearly independent iff
	$A\bm{x} = 0$ has the unique solution $x = 0$ iff $\rho(A) = r$.
\end{thm}

\begin{proof}
	By definition of linear independence,
	\[
		x_1 v_1 + \cdots + x_r v_r = 0
		\iff
		x_i = 0 \, \forall i
	\]
	This is clearly equivalent to saying that $\bm x$ is the
	only solution to $A\bm x = 0$.

	A matrix equation like this can have a unique solution iff
	$\rho(A) = r$, otherwise there would be free variables resulting
	in a range of solutions.
\end{proof}

We now know that any matrix whose rank is equal to the number of its
rows forms a linearly independent spanning set for a subspace of
$\RR^n$.

\begin{cor}
	Any set of $m > n$ vectors in $\RR^n$ is linearly dependent.
\end{cor}

\begin{cor}
	The columns of the indentity matrix $I_n$ form a linearly
	independent spanning set of $\RR^n$.
\end{cor}

\begin{defn}[Basis]
	A subset $B$ of a vector space $V$ is called a basis of $V$ if $B$ is linearly independent and $B$ spans $V$.
\end{defn}

\begin{rem}
	When considering $\RR^n$ or its subsets, a basis will always be
	finite; however a basis can be infinite, for example
	$\{1, x, x^2, \ldots\}$ is a basis of $\RR[x]$.
\end{rem}

\begin{ex}
	\[
		\left\{
			\begin{pmatrix}
				1 \\
				2 \\
				3
			\end{pmatrix}
			,
			\begin{pmatrix}
				1 \\
				-1 \\
				0
			\end{pmatrix}
			,
			\begin{pmatrix}
				5 \\
				4 \\
				4
			\end{pmatrix}
		\right\}
	\]
	is a basis of $\RR^3$.

	\[
		\begin{pmatrix}
			1 & 1  & 5 \\
			2 & -1 & 4 \\
			3 & 0  & 4
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 1  & 5  \\
			0 & -3 & -6 \\
			0 & -3 & -11
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 1  & 5  \\
			0 & -3 & -6 \\
			0 & 0  & -5
		\end{pmatrix}
	\]

	We can see that the rank is $3$ hence this set is linearly independent.
	\[
		\begin{pmatrix}
			1 & 1  & 5  \\
			0 & -3 & -6 \\
			0 & 0  & -5
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 1 & 5  \\
			0 & 1 & -2 \\
			0 & 0 & 1
		\end{pmatrix}
		=
		\begin{pmatrix}
			1 & 0 & 0 \\
			0 & 0 & 0 \\
			0 & 0 & 1
		\end{pmatrix}
	\]

	This shows that this set spans $\RR^3$ and is therefore a basis.
\end{ex}

\begin{thm}
	Let $V$ be a vector space over $\RR$. If $U = \{v_1, \ldots,
	v_r\}$ spans $V$ and no proper subset of $U$ spans $V$, then $U$
	is a basis for $V$.
\end{thm}

\begin{proof}
	We know that $U$ spans $V$ so we need only prove that $U$ is linearly independent. We will do so using the contrapositive method.

	If $U$ is not linearly independent then $\exists v_i \in U$ such that
	\[
		v_i = \alpha_1 v_1 + \cdots + \alpha_{i-1} v_{i-1} +
		\alpha_{i+1} v_{i+1} + \cdots + \alpha_r v_r
	\]
	\[
		\implies
		W = {v_1, \ldots, v_{i-1}, v_{i+1}, \ldots, v_r}
		\subset U
	\]
	spans $V$.

	Therefore, if there is no such $W$, $U$ must be linearly
	independent and hence a basis.
\end{proof}

\begin{thm}
	Let $V$ be a vector space over $\RR$. If $U = \{v_1, \ldots, v_r\}$
	is a linearly independent subset of $V$ and no proper, linearly
	independent  subset $W$ of $V$, where $W$ contains $U$, then $U$
	is a basis of $V$.
\end{thm}

\begin{proof}
	We know $U$ is linearly independent so we need only show that it
	spans $V$.

	Let $v \in V \setminus U$. According to our assumption that no
	linearly independent subset exists between $U$ and $V$, this
	means that $\{v_1, \ldots, v_r, v\}$ is  not linearly
	independent; hence any element in this set can be expressed as a
	linear combination of the rest.

	\[
		\implies
		v = \sum_{i=1}^r \alpha_i v_i \quad \forall v \in V
	\]

	Therefore, $U$ spans $V$ and $U$ is a basis for $V$.
\end{proof}

%TODO: look into Mac Lane generalisation of this lemma.
\begin{lemma}[Exchange (Steinitz)]
	Let $V$ be a vector space. Let $S = \{a_1, \ldots, \_s\}$ be a
	spanning set for a subspace $U \subset V$ and let $B = \{b_1,
	\ldots, b_r\}$ be a linearly independent subset of $U$. The $r
	\leq s$ and there exists a set $S_r$ containing $B$ and some
	elements from $S$ such that $U$ is spanned by $S_r$.
\end{lemma}

%TODO: be consistent with sp and span, declare them as operators:
%\DeclareMathOperator\span{span}
\begin{proof}
	\[
		b_1 \in U = sp(S)
		\implies b_1 = \sum_{i=1}^s \alpha_i a_i,
	\]
	where not all $a_i = 0$ (else $b_1 = 0$).
	\[
		\implies a_1 = \alpha_1^{-1}
		( b_1 - \alpha_2 a_2 - \cdots - \alpha_s a_s )
	\]

	Assume we have replaced $(a_1, \ldots, a_k)$ with
	$(b_1, \ldots, b_k)$ in this manner, so that
	$U$ is spanned by $\{ b_1, \ldots, b_k, a_{k+1}, \ldots, a_s \}$.

	Exchange $b_{k+1}$:
	\[ b_{k+1} \in U \]
	\[
		\implies b_{k+1} =
		\beta_1 b_1 + \cdots + \beta_k b_k
		+ \alpha_{k+1} a_{k+1} + \cdots + \alpha_s a_s
	\]
	again, not all coefficients are zero.

	At least one $\alpha_i \neq 0$ else $b_{k+1}$ would be a linear
	combination of $\{ b_1, \ldots, b_k \}$.

	\[
		\implies \alpha_{k+1} = \alpha_{k+1}^{-1}
		(
			b_{k+1} - \beta_1 b_1 - \cdots - \beta_k b_k
			- \alpha_{k+2} a_{k+2} - \cdots - \alpha_s a_s
		)
	\]

	So for all $1 \leq k \leq r$ we can replace $a_k$ with $b_k$, giving:
	\[
		U = span \{ b_1, \ldots, b_{k+1}, a_{k+2}, \ldots, a_s \}
	\]
\end{proof}

This result is significant because when we have a basis, $B$, for a
vector space, $V$, it can be used as the linearly independent set or the
spanning set. This tells us that any spanning set for $V$ has a greater
or equal cardinality compared with $B$ and that any linearly independent
subset of $V$ has a cardinality smaller than or equal to that of $B$.
The next result follows from this.

\begin{cor}
	Any two bases for a vector space have the same number elements.
\end{cor}

\begin{defn}[Dimension]
	The number of elements in a basis for a vector space is the dimension of the vector space.
\end{defn}

\begin{rem}
	Let $U$ be a vector space with dimension $n$ and $S \subset U$
	such that $|S| = n$. If $S$ spans $U$ or if $S$ is linearly
	independent, then $S$ must be a basis of $U$.
\end{rem}

\section{Rank and Nullity}

\begin{defn}[Nullity]
	The nullity of a matrix $A$ is the dimension of its null space.
\end{defn}

\begin{defn}[Rowspace]
	Let $A \in M_{m \times n}$ an $a_{ij}$ be the $i^{th}$ row of $A$.
	The rowspace of $A$ is:
	\[
		sp \{ a_{1j}, \ldots, a{nj} \}
	\]
\end{defn}

Note that the dimension of the rowspace is the number of non-zero rows
in echelon form, i.e.\ the (row) rank. We can refer to the column rank as
the dimension of the column/image space.

\begin{thm}[Row rank equals column rank]
	Let $A \in M_{m \times n}$, then $\rho(A) = \rho(A^T)$. In other
	words, the row rank and column rank are equal.
\end{thm}

\begin{proof}
	Let the column rank of $A$ be $r = \rho(A^T)$. Let $C$ be the
	matrix whose columns form a basis for the column space of $A$.

	Every column of $A$ can be expressed as a linear combination of
	the columns in $C$ hence there is a matrix $R$ such that
	$A = CR$.

	This means that every column of $A$ can be given as a linear
	combination of the rows of $R$, which implies that it is a
	spanning set for the row space of $A$.

	There are $r$ columns in $C$, hence $r$ rows in $R$ so, by the
	Exchange Lemma, we know that $\rho(A) \leq r = \rho(A^T)$.

	We can apply this same process to the matrix $A^T$, where the
	rows and columns (and hence their ranks) are swapped. From this
	we see that $\rho(A^T) \leq \rho({(A^T)}^T) = \rho(A)$.

	\[
		\implies \rho(A) = \rho(A^T)
	\]
\end{proof}

\begin{thm}
	Let $A \in M_n(\RR)$ with $Ax = b$, then the following statement
	are equivalent:
	\begin{enumerate}
		\item
			$Ax = b$ has a unique solution
		\item
			$\rho(A) = n$
		\item
			$A$ is equivalent to $I_n$
		\item
			$A$ is invertible
	\end{enumerate}
\end{thm}

\begin{proof}
	We have already seen $1 \iff 2$ and $2 \iff 3$ is obvious
	considering the RRE form of $A$.

	$3 \implies 4$ because by reducing $A$ to $I_n$, we can use the
	product of the row operations to construct the inverse matrix.

	$4 \implies 3$:
	\[ \exists A^{-1} \implies A^{-1}A = I_n \]
	$A^{-1}$ can be split into row operations that, when applied to
	$A$, yields $I_n$.
\end{proof}

\section{Determinants}

\begin{defn}
	The $(i,j)$-minor of $A \in M_n$ is the $(n-1)\times(n-1)$
	matrix obtained by removing the $i^{th}$ row and $j^{th}$
	column. It is often denoted $M_{i,j}$.
\end{defn}

\begin{defn}[Determinant]
	The determinant is defined recursively:
	\[ n=1 \implies \det [a] = a \]
	\[ n=k \implies \det A = \sum_{j=1}^k a_{1,j} C_{1,j} \]
	where
	\[ C_{i,j} = {(-1)}^{i+j} \det(M_{i,j}) \]
	is the $(i,j)$-cofactor.
\end{defn}

\subsection{Properties of the Determinant}

\begin{enumerate}
	\item
		If any row is all zeros, $\det A = 0$
	\item
		If $B$ is obtained by adding one row of $A$ to a
		different one, $\det A = \det B$
	\item
		If $B$ is obtained by swapping one row of $A$ with
		another, $\det A = -\det B$
	\item
		If $B$ is obtained by multiplying one row of $A$ by
		$\alpha$, $\alpha \det A = \det B$
	\item
		If $A$ is upper triangular, then $\det A$ is the product
		of the diagonal elements
	\item
		$\det(A^T) = \det(A)$
	\item
		$\det(AB) = \det(A) \det(B)$
\end{enumerate}

We will not prove most of these here, but property 5 can be deduced
simply enough using the others and induction.

\begin{proof}
The $n=1$ case is obvious, so we may assume the property holds for
$n=k$.

\[
	A =
	\begin{bmatrix}
		a_{1,1}   & \cdots & a_{1,k}   & a_{1,k+1} \\
		\vdots    & \ddots & \vdots    & \vdots    \\
		a_{k,1}   & \cdots & a{k,k}    & a_{k,k+1} \\
		a_{k+1,1} & \cdots & a_{k+1,k} & a_{k+1,K+1}
	\end{bmatrix}
\]

The other row properties allow us to traverse any row in order to
calculate the determinant, so we can choose the bottom row. This is all
zeros except for the element $a_{k+1,k+1}$. We can see that
\[
	\det A = a_{k+1,K+1} \det A_k
\]
which is the product of the diagonals.
\end{proof}

Some futher properties can be deduced from those we have already described:
\begin{itemize}
	\item
		If any two rows are identical then $\det A = 0$ (using 1,2,4)
	\item
		All row properties also apply to columns (due to 6)
	\item
		The determinant is commutative (due to 7)
	\item
		$\det(\alpha A) = \alpha^n \det A$ (due to 4)
	\item
		$\det I_n = 1$ (using 5)
\end{itemize}

\section{Eigenvalues and Eigenvectors}

\begin{defn}[Eigenvalues and Eigenvectors]
	Let $A \in M_n(\RR)$ and $\lambda \in \RR$. If $\exists x \neq
	0$ such that $Ax = \lambda x$, then $\lambda$ is called an
	Eigenvalue of $A$ and $x$ is called an Eigenvector of $A$.
\end{defn}

\begin{lemma}
	$\exists A^{-1} \iff |A| \neq 0$
\end{lemma}

%TODO: deduce the method for calculating the inverse (springer)
\begin{proof}
	Assume $\exists A^{-1}$
	\[ \implies |A^{-1}A| = |I| = 1 \]
	\[ \implies |A^{-1}||A| = 1 \]
	\[ \implies |A| \neq 0 \]

	Assume $|A| \neq 0$. This means we can use the standard method
	to calculate the inverse.
\end{proof}

\begin{lemma}
	$\lambda$ is an Eigenvalue $\iff |A - \lambda I_n| = 0$.
\end{lemma}

\begin{proof}
	\[ Ax = \lambda x \iff Ax = \lambda I_n x \iff (A - \lambda I_n)x = 0 \]

	By definition, $x \neq 0$ hence $(A - \lambda I_n)$ cannot have an inverse.

	\[ \iff |A - \lambda I_n| = 0 \]
\end{proof}

\begin{defn}[Characteristic Equation]
	$|A - \lambda I_n| = 0$ is called the characteristic equation of
	$A$. Solving it for $\lambda$ yields eigenvalues for $A$, which
	can be used to find eigenvectors from $(A - \lambda I_n)x = 0$.
\end{defn}

\section{Linear Transformations}

\begin{defn}[Linear Transformation]
	Let $V$ and $W$ be vector spaces over $\RR$. A function $T:V \to
	W$ is a linear transformation if $\forall v_1, v_2 \in V, \alpha
	\in \RR$ we have
	\begin{enumerate}
		\item
			$T(v_1 + v_2) = T(v_1) + T(v_2)$
		\item
			$T(\alpha v_1) = \alpha T(v_1)$
	\end{enumerate}
\end{defn}

\begin{defn}[Kernel]
	\[ \ker T = \{ v \in V \mid T(v) = 0 \} \]
\end{defn}

\begin{defn}
	\[ \im T = \{ w \in W \mid w = T(v), v \in V \} \]
\end{defn}

\begin{prop}
	$\ker T$ is a subspace of $V$ and $\im T$ is a subspace of $W$.
\end{prop}

\begin{proof}
	Let $v_1, v_2 \in \ker T$, then
	\[ T(0) = T(v_1-v_1) = T(v_1) - T(v_1) = 0 \implies 0 \in \ker T \]
	\[ T(v_1 + v_2) = T(v_1) + T(v_2) = 0 \]
	\[ T(\alpha v_1) = \alpha T(v_1) = \alpha \cdot 0 = 0 \]

	$T(v_1), T(v_2) \in \im T$
	\[ T(0) = T(v_1 - v_1) = T(v_1) - T(v_1) = 0 \]
	\[ v_1 + v_2 \in V \implies T(v_1) + T(v_2) = T(v_1 + v_2) \in \im T \]
	\[ \alpha v_1 \in V \implies \alpha T(v_1) = T(\alpha v_1) \in \im T \]
\end{proof}

\begin{prop}
	Let $T$ be a linear map. $T$ is injective $\iff \ker T = 0$.
\end{prop}

\begin{proof}
	\begin{align*}
		& &(T(x) = T(y) \implies x = y) \\
		&\iff &(T(x) - T(y) = 0 \implies x - y = 0) \\
		&\iff &(T(x-y) = 0 \implies x-y = 0) \\
		&\iff &(T(v) = 0 \implies v = 0) \\
		&\iff &\ker T = 0
	\end{align*}
\end{proof}

\begin{rem}
	It is clear that $T: V \to W$ is surjective $\iff \im T = W$.
\end{rem}

\begin{defn}[Isomorphism]
	A linear transformation $T:V \to W$ is an isomorphism if $T$ is
	bijective i.e.\ $\ker T = 0$ and $\im T = W$.
\end{defn}

\begin{defn}
	Let $T:V \to W$ be a linear transformation. Let $v_1, \ldots,
	v_n$ be a basis for $V$, then the matrix of the transformation
	is the matrix $M_T \in M_{m \times n}$ whose columns are
	$T(v_i)$.
\end{defn}

\begin{prop}
	Let $T:\RR^n \to \RR^m$ be a linear transformation, then $T(x) =
	M_T \cdot x$.
\end{prop}

\begin{proof}
	Let $\{e_I\}$ be the stadard basis for $\RR^n$ (columns of $I_n$) and $T(e_i) = f_i$.
	\begin{align*}
		T(x) &= T
		\left(
			\begin{pmatrix}
				x_1 \\
				\vdots \\
				x_n
			\end{pmatrix}
		\right) \\
		&= T(x_1 e_1 + \cdots + x_n e_n) \\
		&= x_1 T(e_1) + \cdots + x_n T(e_n) \\
		&= x_1 f_1 + \cdots + x_n f_n \\
		&=
		\begin{pmatrix}
			\uparrow   &        & \uparrow \\
			f_1        & \cdots & f_n      \\
			\downarrow &        & \downarrow
		\end{pmatrix}
		\begin{pmatrix}
			x_1 \\
			\vdots \\
			x_n
		\end{pmatrix} \\
		&= M_T \cdot x
	\end{align*}
\end{proof}

\begin{cor}
	Let $T:V \to W$ be a linear transformation and $M_T$ be its
	matrix with respect to the bases $B_V$ and $B_W$, then
	\begin{align*}
		\ker T & = N(M_T) \\
		\im T  & = \im M_T
	\end{align*}
\end{cor}

\begin{defn}[Change of basis]
	The matrix of $id:V \to W$ with respect to the bases $B_1$ and
	$B_2$ is called the change of basis matrix, from $B_1$ to $B_2$.
\end{defn}

\begin{defn}[Eigenvalues and Eigenvectors]
	Let $T:V \to W$ be a linear transformation. If $\exists \lambda
	\in \RR$ and $0 \neq v \in V$ such that $T(v) = \lambda v$, then
	$\lambda$ is and eigenvalue and $v$ is an eigenvector.
\end{defn}

\end{document}


\chapter{Statistics in Photometry}

So far we've seen how the photometry should be done. But we haven't seriously discussed how to analize the uncertainties, i.e., the error-bars of each measurement such as magnitude.

\section{Pixel-wise Uncertainty}
An object frame consist of at least bias, dark, flat, cosmic-ray, target's signal, and sky, as well as readout noise. For the $ j $-th pixel, they are denoted as:
\begin{itemize}
\item $ \tilde{N}_j $: the raw pixel value of the object frame in ADU.
\item $ B_j $: the bias in ADU.
\item $ D_j $: the dark in ADU.
\item $ F_j $: the flat in ADU (\textit{not necessarily} normalized; see \textbf{Note} at the end).
\item $ C_j $: the cosmic-ray in ADU.
\item $ I_j $: the object signal count in ADU.
\item $ S_j $: the sky signal in ADU.
\end{itemize}
In some cases, such as infrared detectors, the gain and readout noise may differ for each pixel, so I will denote gain and readout noise of the $ j $-th pixel as $ g_j \,[\mathrm{e/ADU}]$ and $ R_j \,[\mathrm{e}] $. Then we define the bias-subtracted pixel value
\begin{equation}
  N_j = \tilde{N}_j - B_j = D_j + (I_j + S_j + C_j) F_j
\end{equation}
Note that the dark is not affected by the flat value. Since this pixel value should follow a Poisson distribution, which is approximated as a Gaussian distribution,
\begin{equation}
  N_j
    \simdot \mathcal{N} \left (
      N_j ,~
      \frac{N_j}{g_j} + \left (\frac{R_j}{g_j} \right )^2
    \right ) ~.
\end{equation}

\subsection{Dark Estimation}
To estimate dark $ D_j $, you should have taken nearly tens of dark frames, and combined it. From the median value of the frames, you may have obtained the estimation of the dark $ \hat{D}_j $ (for brevity, I will just use $ D_j $ without the caret $ \hat{\cdot} $). During the combination, you can obtain the uncertainty of the dark at the $ j $-th pixel, $ \Delta D_j $, as
\begin{equation}
  \Delta D_j = \mathrm{sstd} (D_j^{i=1..N_\mathrm{Dark}})
    := \sqrt{ \frac{\sum_{i=1}^{N_\mathrm{Dark}} (D_j^{i} - \bar{D})^2}
      {N_\mathrm{Dark} - 1}}
    \equiv \mathrm{std}(D_j^{i=1..N_\mathrm{Dark}})
      \times \sqrt{\frac{N_\mathrm{Dark}}{N_\mathrm{Dark} - 1}} ~,
\end{equation}
where $ D_j^{i} $ is the dark current (ADU) of the $ j $-th pixel at the $ i $-th dark frame, $ \mathrm{sstd} $ and $ \mathrm{std} $ are the \textit{sample} standard deviation (\texttt{np.std(x, ddof=1)}) and standard deviation (\texttt{np.std(x)}) functions, respectively, and $ N_\mathrm{Dark} $ is the number of dark framses used. Here it is assumed $ D_j $ roughly follows a Gaussian distribution so that $ \mathrm{sstd} $ becomes an unbiased estimator\footnote{An unbiased estimator of a random variable $ X $, $ \hat{X} $ is defined such that the expectation value of $ \hat{X} $ is the same as the true value $ X_\mathrm{true} $.} of the true standard deviation. Note that you should \emph{not} divide it by the number of dark frames (such as Thm \ref{thm: clt}), because \textbf{what you will need is not the uncertainty of the \emph{mean} value, but the uncertainty of one pixel value that will have been added to the object frame}.
%, and $ N_\mathrm{Dark} $ is the number of dark frames. This is based on the CLT (Thm \ref{thm: clt}). Note the strong point of the CLT helps us here: CLT holds regardless of the distribution of $ D_j^{i} $! Even if we don't know anything about its distribution, we can use CLT without caring about it.

Therefore, to the first order,
\begin{equation}
\begin{aligned}
  \tilde{O}_j = N_j - D_j
    &\simdot \mathcal{N}
      \left ( \tilde{O}_j,~
        \frac{N_j}{g_j} + \left (\frac{R_j}{g_j} \right )^2 + (\Delta D_j)^2 \right ) \\
  \mathrm{or}
    &\simdot \mathcal{N}
        \left ( \tilde{O}_j,~
          \frac{\tilde{O}_j}{g_j}
          + \frac{D_j}{g_j}
          + \left (\frac{R_j}{g_j} \right )^2
          + (\Delta D_j)^2 \right ) ~.
\end{aligned}
\end{equation}
Note that we have been using $ N_j/g_j = (\tilde{O}_j + D_j)/g_j $ term in $ \Delta N_j $. This makes sence only if the Poissonity of dark is assumed.

\subsection{Flat Estimation}
Unlike the dark current, which is a probabilistic electron generation, the flat, i.e., the pixel sensitivity, is a characterestic value of the detector and optics, so it should not change at each exposure. Therefore, following the CLT (Thm \ref{thm: clt}):
\begin{equation}
  \Delta F_j
    = \frac{\mathrm{sstd} (F_j^{i=1..N_\mathrm{Flat}})} {\sqrt{N_\mathrm{Flat}}}
    := \sqrt{ \frac{1}{N_\mathrm{Flat}}
       \frac{\sum_{i=1}^{N_\mathrm{Flat}} (F_j^{i} - \bar{F})^2}{N_\mathrm{Flat} - 1}}
       \equiv \frac{\mathrm{std}(F_j^{i=1..N})}{\sqrt{N_\mathrm{Flat} - 1}} ~,
\end{equation}
where $ F_j^{i} $ is the flat frame pixel value (ADU) of the $ j $-th pixel at the $ i $-th flat frame, $ \mathrm{sstd} $ is the sample standard deviation function and $ N_\mathrm{Flat} $ is the number of the used flat frames.

Generally speaking, $ F_j $ must be about $ 10^4 $ ADU or more, so that the signal-to-noise ratio from Poisson statistics is larger than about 100. Moreover, you should have obtained, say, $ N_\mathrm{flat} = 9 $, this will increase by a factor of 3. Therefore, the uncertainty in the flat $ \Delta F_j $, is around 0.1 \% order. In some observatories, people take hundreds of flats at one time to get flat as good as roughly 0.01 \% order, assuming it won't change over certain period of time.

Although there are mathematically known ratio distribution, i.e., the pdf of $ Z := X/Y $ where $ X, Y \sim \mathcal{N}(\mu_{X, Y}, \sigma_{X, Y}^2) $ and the covariance is zero, what we are interested in is only a rough estimation of the uncertainties\footnote{If you are interested in, see FiellerEC (1932) Biometrika, 24, 428; HinkleyDV (1969) Biometrika, 56, 635;  D\'{i}az-Franc\'{e}s+RubioFJ (2013) Statistical Papers, 54, 309, as well as \href{https://en.wikipedia.org/wiki/Ratio_distribution}{Wikipedia}. For the ratio of $ N_j/F_j $, i.e., when the denominator has so high signal-to-noise ratio, the distribution is roughly a Gaussian with $ \mathcal{N}(\mu_X, \sigma_X^2) $ as we assume in this section.}. From the propagation of error,
\begin{equation}
\begin{aligned}
  O_j = \frac{N_j - D_j}{F_j} = I_j + S_j + C_j
    &\simdot \mathcal{N}
      \left ( O_j,~
        O_j^2 \left [
        \left (\frac{\Delta \tilde{O}_j}{\tilde{O}_j} \right )^2
        + \left (\frac{\Delta F_j}{F_j} \right )^2 \right ] \right ) ~.
\end{aligned}
\end{equation}
Substituting previously obtained distribution of $ \tilde{O}_j $,
\begin{equation}
  O_j
    \simdot \mathcal{N}
      \left ( O_j,~
        \frac{O_j}{g_jF_j}
        + \frac{D_j}{g_j F_j^2}
        + \left (\frac{R_j}{g_j F_j} \right )^2
        + \left  (\frac{\Delta D_j}{F_j} \right )^2
        + O_j^2 \left (\frac{\Delta F_j}{F_j} \right )^2
        \right ) ~.
\end{equation}

%Note that this variance is also obtained by the following logic: $ F_j $ has too high signal-to-noise ratio, $ \Delta F_j / F_j \ll 1 $, we can just assume $ F_j $ is a perfectly known constant without any uncertainty. Then $ O_j = \tilde{O}_j / F_j \simdot \mathcal{N}(O_j, (\Delta \tilde{O}_j)^2 / F_j^2) $, which is what we have above.

\subsection{Final Pixel-wise Uncertainty}
Many times the $ C_j $ is removed by the so-called \emph{cosmic-ray rejection} algorithms.
\begin{thm}[Pixel-wise Error]
The final, cosmic-ray removed pixel value will follow
\begin{equation} \label{eq: pix error}
\begin{aligned}
  O_j^\mathrm{cr} = \frac{N_j - D_j}{F_j} - C_j = I_j + S_j
    &\simdot \mathcal{N}
      \left ( O_j^\mathrm{cr},~
        \frac{O_j}{g_jF_j}
        + \frac{D_j}{g_j F_j^2}
        + \left (\frac{R_j}{g_j F_j} \right )^2
        + \left (\frac{\Delta D_j}{F_j} \right )^2
        + O_j^2 \left (\frac{\Delta F_j}{F_j} \right )^2
        \right ) \\
    &\simdot \mathcal{N}
      \left ( O_j^\mathrm{cr},~
        \frac{N_j}{g_jF_j^2}
        + \left (\frac{R_j}{g_j F_j} \right )^2
        + \left  (\frac{\Delta D_j}{F_j} \right )^2
        + O_j^2 \left (\frac{\Delta F_j}{F_j} \right )^2
        \right )
\end{aligned}
\end{equation}
Here, $ O_j $ is the bias, dark, and flat corrected pixel value before the cosmic-ray removal, while $ N_j $ is only bias-subtracted pixel value. Usually the uncertainty from the estimation for $ C_j $ is ignored as it is too difficult to estimate.
\end{thm}

In practical application, the following approximations may be used:
\begin{enumerate}
\item Calculating the $ \Delta D_j $ term all the time is annoying, and moreover, it is likely for most pixels $ \Delta D_j \ll \Delta O_j $, except for hot/bad pixels\footnote{Bad pixels must be masked properly prior to any sane calculation, as they are known to be ``wrong'' data. This is usually provided as a pixel mask file (e.g., \texttt{.bpm} file) by the observatory.}. Therefore, people just ignore it and set it to 0.  Theoretically, however, $ \Delta D_j > (R_j/g_j) $, because $\Delta D_j = \mathrm{sstd} (D_j^{i}) $ and $ D_j^{i} \simdot \mathcal{N} \left (D_j^{i},~ D_j^{i}/g_j + (R_j/g_j)^2 \right ) $. Therefore, a more reasonable approximation, or the lower limit of the uncertainty, would be $ R_j/g_j $.
\item Frequently, $ \Delta F_j/F_j $ is much smaller than $ \Delta \tilde{O}_j/\tilde{O}_j $ (or we can increase $ N_\mathrm{Flat} $ to force that this holds), so the flat-error term is negligible. StetsonPB, for instance, asked the user to give a constant $ \Delta F_j/F_j \equiv \sigma_F $ for all pixels, such as 0, 0.01, or 0.0075, in \texttt{DAOPHOT}.
\item Many times we use the normalized flat (for more discussion, see below) such that its mean or median is 1, so $ F_j \sim 1 $ for all pixel. Then all $ F_j $ in \cref{eq: pix error} can be just ignored.
\item The $ D_j $ term is negligible for many times. It is not negligible only for hot pixels where $ D_j \gg 1 $, but likely the observer did not put the target of interest at where hot pixels present. Moreover, state-of-the-art CCDs, such as Subaru FOCAS for instance, has too small dark current $ D_j < 0.1 \,\mathrm{e/s} $. Therefore, most important parts in the object frames will have negligible $ D_j $ compared to $ O_j^\mathrm{cr} $. Therefore, this term is also ignored many times.
\end{enumerate}

Summarizing, \cref{eq: pix error} is approximated as
\begin{equation}
\begin{aligned}
  O_j^\mathrm{cr}
    &\simdot \mathcal{N}
      \left ( O_j^\mathrm{cr},~
        \frac{O_j}{g_jF_j}
        + \frac{D_j}{g_j F_j^2}
        + \left (\frac{R_j}{g_j F_j} \right )^2
        + \left  (\frac{\Delta D_j}{F_j} \right )^2
        + O_j^2 \left (\frac{\Delta F_j}{F_j} \right )^2
        \right ) \\
    &\simdot \mathcal{N}
      \left ( O_j^\mathrm{cr},~
        \frac{O_j}{g_jF_j}
        + \frac{D_j}{g_j F_j^2}
        + 2 \left (\frac{R_j}{g_j F_j} \right )^2
        + \sigma_F O_j^2
        \right ) \\
    &\simdot \mathcal{N}
      \left ( O_j^\mathrm{cr},~
        \frac{O_j}{g_j}
        + \frac{D_j}{g_j}
        + 2 \left (\frac{R_j}{g_j} \right )^2
        + \sigma_F O_j^2
        \right ) \\
    &\simdot \mathcal{N}
      \left ( O_j^\mathrm{cr},~
        \frac{O_j}{g_j}
        + 2 \left (\frac{R_j}{g_j} \right )^2
        + \sigma_F O_j^2
        \right ) \\
\end{aligned}
\end{equation}
Most frequently people even drop the factor 2 and $ \sigma_F $, and write $ O_j^\mathrm{cr} \simdot \mathcal{N} \left ( O_j^\mathrm{cr},~ O_j/g_j + (R_j/g_j)^2 \right ) $. The first dropping ($ 2(R_j/g_j)^2 \rightarrow (R_j/g_j)^2 $) may be justified because one $ (R_j/g_j)^2 $ came from the uncertainty in dark, and we usually ignore anything from dark\footnote{Some high-spec observatories even skip taking dark frames (e.g., Subaru).}.

\subsubsection{Note}
\textbf{The flat frames are frequently assumed to be normalized}, i.e., the mean or median of $ F_j $ values of the frame is around 1. In the generalized error estimation given in this book, this normalization is not important. However, astronomers (including all tasks of IRAF) have conventionally used a very simplified version of error estimation, that is, the Poisson component of the pixel error is approximated as $ \sqrt{O_j/g_j} $: $ O_j \simdot \mathcal{N} (O_j, O_j/g_j) $. Note the factor $ F_j $ is missing! The true signal-to-noise ratio is $ O_j / \Delta O_j \approx \sqrt{O_j / (g_j F_j)} $, while in this approximation, it is $ \sqrt{O_j / g_j} $. If the flat was normalized, $ F_j \sim 1 $, so these two are similar. If it were not normalized, however, the signal-to-noise ratio in this classical approximation will be underestimated by a factor of $ \sqrt{F_j} \sim \sqrt{10^4 \,(\mathrm{ADU})} = 100 $!! You will be calculating wrong uncertainty in that case.

From STSDAS package, \texttt{stsdas/pkg/hst\_calib/wfpc/noise/fitnoise.x}: noise = R if pixel value not positive, otherwise, $ \sqrt{R^2 + N/g + (N \times \texttt{scalen}/100)^2} $ where \texttt{scalen} is maybe an uncertainty of pixel (flat uncertainty?) in percentage.

%#################################################################################
%#										#
%# NOISEFUNC --	Calculate the square-root of the variance expected at the 	#
%#		given DN level, based upon the noise model parameters.  	#
%#										#
%#	Initial code:	7/91 by RAShaw						#
%
%real procedure noisefn(dn)
%
%include	"nmcom.h"
%
%#  Calling argument
%real	dn		# Value of pixel in Data Numbers
%
%# Local variables
%real	tmp
%real	noise		# Function value
%
%begin
%	tmp = scalen / 100.
%	if (dn <= 0.)
%	    noise = readn
%	else
%	    noise = sqrt (readn * readn + dn / gain + dn * dn * tmp * tmp)
%	return (noise)
%end

\subsection{Signal-to-Noise Ratio (SNR)}
\textbf{SNR} $ := \mathrm{signal}/\mathrm{noise} $, is an estimation of the quality of the data. Simply speaking, the signal is the sky subtracted aperture summation. On the other hand, noise is the square root of the aperture summation on the \textit{variance map} (the map of $ (\Delta O_j)^2 $).

\begin{ex}[Maximum pixel-wise SNR]
Consider a classic CCD, which loses linearity at around 60,000 ADU (bias subtracted value). Say gain is $ g_j = 1.0 $, there is neither readout noise ($ R_j = 0.0 $) nor dark ($ D_j = 0.0 $), and there is no flat pattern ($ F_j = 1.0 $, $ \sigma_F = 0 $). Then only the Poisson noise term remains and thus the maximum SNR of the pixel is
\begin{equation}
  \mathrm{SNR}_\mathrm{max}^\mathrm{pixel}
    \approx \frac{60000}{\sqrt{60000}} = \sqrt{60000} = 245 ~.
\end{equation}
Even if the CCD has high bias, lose linearity earlier, high gain, etc, you should always get $ \mathrm{SNR}_\mathrm{max}^\mathrm{pixel} \sim 250 $.
\end{ex}

\begin{ex}[Maximum photometric SNR]
Consider the same CCD and now you have a simple circular Gaussian star (i.e., perfect Gaussian PSF) in the image, almost saturated but not saturated. The peak value before the saturation will be roughly 60,000 ADU after the sky subtraction. That is, this star has a radial profile $ I(r) \approx 60000 e^{-r^2/2\sigma^2} $. Depending on the seeing size and pixel scale, $ \sigma $ will change.

For the case if the $ \mathrm{FWHM} \ll 1 \,\mathrm{pix} $, $ \mathrm{apsum} \approx 60,000 $, so the maximum SNR is $ \mathrm{SNR}_\mathrm{max}^\mathrm{star} \approx \mathrm{SNR}_\mathrm{max}^\mathrm{pixel} \sim 250 $.

If the FWHM is much larger than 1 pixel (say, 5 pixel), 60,000 is roughly the peak value of the Gaussian function. The integration of such Gaussian function is $ \sqrt{2 \pi} \sigma \times 60,000 \sim 60,000 \times \mathrm{FWHM} $, where FWHM is in pixel unit (note that $ \sqrt{2\pi}\sigma = 2.51 \sigma \approx \mathrm{FWHM} $). Therefore,
\begin{equation}
 \mathrm{SNR}_\mathrm{max}^\mathrm{star}
 \approx \frac{60000 \mathrm{FWHM}}{\sqrt{60000 \mathrm{FWHM}}}
 \sim 250 \sqrt{\mathrm{FWHM}}
\end{equation}

Otherwise, $ \int_{-0.5}^{0.5}\int_{-0.5}^{0.5} I(x, y) dx dy = 60,000 $, so it is a bit complicated to calculate (you may use the fact that the integration of Gaussian function is analytically expressed using erf). But you may simply use the above rule-of-thumb to check the maximum SNR.

In reality, uncertainty in sky, dark, readout noise will increase the noise and thus decrease the SNR. Gain larger than unity will have an effect of increasing SNR by factor of $ \sqrt{g} $. The PSF usually have longer tail than Gaussian, and this will increase the SNR.
\end{ex}

The magnitude system used in astronomy uses the logarithmic scale. From Pogson's formula, $ I_1/I_2 = 10^{-0.4 \Delta m} $ for two objects having fluxes $ I_1 $ and $ I_2 $, and magnitude difference $ \Delta m $. When $ \Delta m $ is small, $ \Delta m $ is the fractional difference of the fluxes. For example, $ \Delta m = 0.01 $ means $ I_1/I_2 \approx 0.99 $ or the flux difference is 0.01 (1 \%). See \cref{fig:deltamag} for the validity of this approximation.

\begin{figure} [ht!]
  \centering
  \includegraphics[width=0.5\linewidth]{figs/delta_mag}
  \caption{The delta magnitude plot.}
  \label{fig:deltamag}
\end{figure}

Differently put, if $ \Delta m =0.01 $ was the magnitude \textit{uncertainty} of the object, the uncertainty in flux is 1 \%, or $ \mathrm{SNR} = 100 $. Therefore, SNR can be directly converted to the magnitude uncertainty and vice versa. The $ \mathrm{SNR}_\mathrm{max}^\mathrm{pixel} = 250 $ translates to $ \Delta m \sim 1/250 = 0.004 $.


\chapter{Hypothesis Testing and p-values}

\begin{ex}
  Suppose that the true value of $\theta$ is $\theta_\star\neq \theta_0$. Then
  \begin{align*}
    W
    =\frac{\thetahat-\theta_0}{\sehat}
    =\frac{\thetahat-\theta_\star+\theta_\star-\theta_0}{\sehat}
    =\frac{\thetahat-\theta_\star}{\sehat}-\frac{\theta_0-\theta_\star}{\sehat},
  \end{align*}
  and therefore
  \begin{align*}
    \P{|W|>z_{\alpha/2}}
     & =\P{\frac{\thetahat-\theta_\star}{\sehat}-\frac{\theta_0-\theta_\star}{\sehat}>z_{\alpha/2}}
    +\P{\frac{\thetahat-\theta_\star}{\sehat}-\frac{\theta_0-\theta_\star}{\sehat}<-z_{\alpha/2}}       \\
     & =\P{\frac{\thetahat-\theta_\star}{\sehat}>\frac{\theta_0-\theta_\star}{\sehat} + z_{\alpha/2}}
    +\P{\frac{\thetahat-\theta_\star}{\sehat}<\frac{\theta_0-\theta_\star}{\sehat}-z_{\alpha/2}}        \\
     & =1-\P{\frac{\thetahat-\theta_\star}{\sehat}<\frac{\theta_0-\theta_\star}{\sehat} + z_{\alpha/2}}
    +\P{\frac{\thetahat-\theta_\star}{\sehat}<\frac{\theta_0-\theta_\star}{\sehat}-z_{\alpha/2}},
  \end{align*}
  which, since $(\thetahat-\theta_\star)/\sehat \approx N(0, 1)$, implies that
  \[
    \P{|W|>z_{\alpha/2}}\approx
    1-\Phi\left(\frac{\theta_0-\theta_\star}{\sehat} + z_{\alpha/2}\right)
    +\Phi\left(\frac{\theta_0-\theta_\star}{\sehat}-z_{\alpha/2}\right).
  \]
\end{ex}

\begin{ex}
  By Theorem 10.12,
  \[
    p=\mathbb{P}_{\theta_0}{(T(X^n)\geq T(x^n))}=1-F(T(x^n)),
  \]
  where $T(X^n)\sim F$ under $H_0:\theta=\theta_0$. Therefore,
  \[
    \P{p<y}
    =\P{1-F(T(Y^n))<y},
  \]
  where $T(Y^n)\sim F$ and therefore by Exercise 2.15,
  $F(T(Y^n))\sim\text{Uniform}(0, 1)$. Finally, note that if
  $U\sim\text{Uniform}(0,1)$ then $1-U\sim\text{Uniform}(0,1)$ as well, and that
  therefore
  \[
    \P{p<y}=yI_{[0,1]}(y),
  \]
  the CDF of a $\text{Uniform}(0,1)$ distribution.
\end{ex}

\begin{ex}
  Note that $\theta_0\not\in C$, where
  \[
    C=(\thetahat-\sehat \, z_{\alpha/2}, \thetahat+\sehat \, z_{\alpha/2}),
  \]
  if and only if
  \[
    \theta_0> \thetahat+\sehat\, z_{\alpha/2}\text{, or }
    \theta_0< \thetahat-\sehat\, z_{\alpha/2}.
  \]
  This is equivalent to
  \[
    |\theta_0-\thetahat|> \sehat\, z_{\alpha/2},
  \]
  which is equivalent to
  \[
    \frac{|\thetahat-\theta_0|}{\sehat} > z_{\alpha/2},
  \]
  but this is precisely the size $\alpha$ Wald test.
\end{ex}

\begin{ex}
  Note that
  \[
    p\text{-value}
    =\inf\left\{\alpha \mid T(X^n)\geq c_\alpha\right\},
  \]
  where $\alpha=\sup_{\theta\in\Theta_0}\mathbb{P}_{\theta}(T(X^n)\geq
    c_\alpha)$ is a decreasing function of $c_\alpha$, and that therefore,
  having observed $x^n$, the smallest value of $\alpha$ will be obtained for
  the largest $c_\alpha$ such that we still reject the null, $c_\alpha=T(x^n)$.
  The size of the test for which we reject the null is then
  \[
    \sup_{\theta\in\Theta_0}\mathbb{P}_{\theta}(T(X^n)\geq T(x^n))
  \]
  and therefore
  \[
    p\text{-value}
    =\sup_{\theta\in\Theta_0}\mathbb{P}_{\theta}(T(X^n)\geq T(x^n)),
  \]
  or, in the case where $\Theta_0=\{\theta_0\}$, the supremum is over only a
  single element and therefore
  \[
    p\text{-value}
    =\mathbb{P}_{\theta_0}(T(X^n)\geq T(x^n)).
  \]
\end{ex}

% 5
\begin{ex}
  Let $X_1,\ldots,X_n\sim\text{Uniform}(0,\theta)$ and
  $Y=\max\{X_1,\ldots,X_n\}$.
  \begin{enumerate}[(a)]
    \item We have
          \begin{align*}
            \beta(\theta_0)
             & =\mathbb{P}_{\theta_0}(Y>c)                                              \\
             & =1-\mathbb{P}_{\theta_0}(Y\leq c)                                        \\
             & =1-\mathbb{P}_{\theta_0}(X_1\leq c)\cdots\mathbb{P}_{\theta'}(X_n\leq c) \\
             & =1-(c/\theta_0)^n
          \end{align*}
          for $c\in[0,\theta_0]$.
    \item Solving
          \[
            0.05=1-(c/\theta_0)^n
          \]
          for $c$ we get
          \[
            c=0.95^{1/n}\theta_0,
          \]
          or, substituting $0.5$ for $\theta_0$,
          \[
            c=0.5\cdot 0.95^{1/n}.
          \]
    \item We have
          \[
            p=1-(0.48/0.5)^{n},
          \]
          which implies that $p\approx 0.558$, which means that the test does
          not provide any evidence against $H_0$.
    \item Note that $Y=0.52$ is outside the range $[0,\theta_0]$, and therefore
          $\mathbb{P}_{\theta_0}(Y>0.52)=0$. The $p$-value is therefore $0$, and
          we can thus reject $H_0$ at any level.
  \end{enumerate}
\end{ex}

\begin{ex}
  Since we will need to construct a confidence interval, we will use the Wald
  test for a Binomial distribution. Let $H_0: p=1/2$ and $H_1:p\neq 1/2$. By
  Exercise 10.15, $\phat=X/n=922/1919$ with
  $\sehat{\phat}=\sqrt{\phat(1-\phat)/n}$. The test statistic is
  \[
    \frac{|\phat-p_0|}{\sehat(\phat)}
  \]
  and thus by Theorem 10.13 the $p$-value is given by
  \[
    \P{|Z|>\frac{|\phat-p_0|}{\sqrt{\phat(1-\phat)/n}}}
    =2\Phi\left(
    -\frac{|\phat-p_0|}{\sqrt{\phat(1-\phat)/n}}
    \right).
  \]
  \inputminted{python}{../code/10-06.py}
  \inputminted{text}{../output/10-06.txt}

  The test indicates weak evidence against $H_0$.
\end{ex}

\begin{ex}~
  \begin{enumerate}[(a)]
    \item Let $X_1$ be the distribution of the proportion of three-letter words
          in the Twain essays, and let $X_2$ be the distribution in the
          Snodgrass essays. Recall from Example 10.8 that the plug-in estimator
          for a difference of means $d=\mu_1-\mu_2$ is given by the difference
          of the plug-in estimators of the means, the sample averages. Thus,
          \[
            \widehat{d}=\Xbar_1-\Xbar_2,
          \]
          with
          \[
            \sehat(\widehat{d})
            =\sqrt{(\sehat(\muhat_1))^2+(\sehat(\muhat_2))^2},
          \]
          where
          \[
            \sehat(\muhat_j)=\frac{1}{\sqrt{n_j}}
            \sqrt{\frac{1}{n_j}\sum_{i=1}^{n_j}(X_{j,i}-\Xbar_j)^2}.
          \]

          Note that $H_0: d=0$, and therefore the size $\alpha$ Wald test for
          this hypothesis is obtained by checking whether
          \[
            \frac{\left|\Xbar_1-\Xbar_2\right|}{
              \sqrt{(\sehat(\Xbar_1))^2+(\sehat(\Xbar_2))^2}}
            > z_{\alpha/2}.
          \]

          \inputminted{python}{../code/10-07.py}
          \inputminted{text}{../output/10-07.txt}

          We may conclude that there is very strong evidence to reject the null
          hypothesis, i.e.\ that the proportion of three-letter words is
          similarly distributed in both sets of essays.
    \item We obtain the same result from the permutation test: we have very
          strong evidence that the samples comes from sets with different means.
  \end{enumerate}
\end{ex}

\begin{ex}
  Let $X_1,\ldots,X_n\sim N(\theta, 1)$. We are testing $H_0:\theta=0$ versus
  $H_1:\theta=1$, using the rejection region $R=\{x^n \mid T(x^n)>c\}$ where
  $T(x^n)=\Xbar$.
  \begin{enumerate}[(a)]
    \item Note that
          \begin{align*}
            \alpha
             & = \mathbb{P}_{\theta=0}\left(\Xbar > c \right)                 \\
             & = \mathbb{P}_{\theta=0}\left(\sqrt{n}\Xbar > \sqrt{n}c \right) \\
             & = \mathbb{P}_{\theta=0}\left(Z > \sqrt{n}c \right)             \\
             & = 1-\Phi(\sqrt{n}c),
          \end{align*}
          and that therefore $c=z_{\alpha}/\sqrt{n}$.
    \item We have
          \begin{align*}
            \beta(1)
             & = \mathbb{P}_{\theta=1}\left(\Xbar > c \right)                         \\
             & = \mathbb{P}_{\theta=1}\left(\Xbar-1 > c-1 \right)                     \\
             & = \mathbb{P}_{\theta=1}\left(\sqrt{n}(\Xbar-1) > \sqrt{n}(c-1) \right) \\
             & = \mathbb{P}_{\theta=1}\left(z > \sqrt{n}(c-1) \right)                 \\
             & = 1-\Phi(\sqrt{n}({c-1}))                                              \\
             & = 1-\Phi(z_\alpha-\sqrt{n}).
          \end{align*}
    \item Note that
          \begin{align*}
            \lim_{n\to\infty}\beta_n(1)
             & =\lim_{n\to\infty}[1-\Phi(z_\alpha-\sqrt{n})] \\
             & =1-\lim_{u\to-\infty}\Phi(u)                  \\
             & =1.
          \end{align*}
  \end{enumerate}
\end{ex}

\begin{ex}
  Note that
  \begin{align*}
    \beta(\theta_1)
     & =\mathbb{P}_{\theta_1}\left(\left|\frac{\thetahat-\theta_0}{\sehat}\right| > z_{\alpha/2} \right) \\
     & =\mathbb{P}_{\theta_1}\left(\frac{\thetahat-\theta_0}{\sehat} > z_{\alpha/2} \right)
    +\mathbb{P}_{\theta_1}\left(\frac{\thetahat-\theta_0}{\sehat} < -z_{\alpha/2} \right)                \\
     & =1
    -\mathbb{P}_{\theta_1}\left(\frac{\thetahat-\theta_0}{\sehat} < z_{\alpha/2} \right)
    +\mathbb{P}_{\theta_1}\left(\frac{\thetahat-\theta_0}{\sehat} < -z_{\alpha/2} \right)                \\
     & =1
    -\mathbb{P}_{\theta_1}
    \left(\frac{\thetahat-\theta_1}{\sehat}
    +\frac{\theta_1-\theta_0}{\sehat}
    < z_{\alpha/2} \right)
    +\mathbb{P}_{\theta_1}\left(
    \frac{\thetahat-\theta_1}{\sehat}
    +\frac{\theta_1-\theta_0}{\sehat}
    < -z_{\alpha/2} \right)                                                                              \\
     & =1
    -\mathbb{P}_{\theta_1}
    \left(\frac{\thetahat-\theta_1}{\sehat}
    < z_{\alpha/2}
    -\frac{\theta_1-\theta_0}{\sehat}
    \right)
    +\mathbb{P}_{\theta_1}\left(
    \frac{\thetahat-\theta_1}{\sehat}
    < -z_{\alpha/2}
    -\frac{\theta_1-\theta_0}{\sehat}
    \right)                                                                                \\
     & =1-\lim_{n\to\infty}\Phi\left(z_{\alpha/2}-\sqrt{nI(\thetahat)}(\theta_1-\theta_0)\right)
    +\lim_{n\to\infty}\Phi\left(-z_{\alpha/2}-\sqrt{nI(\thetahat)}(\theta_1-\theta_0)\right),
  \end{align*}
  or, since $\theta_1>\theta_0$,
  \[
    \beta(\theta_1)
    = 1 - \lim_{u\to-\infty}\Phi(u)+\lim_{u\to-\infty}\Phi(u)
    = 1.
  \]
\end{ex}

% 10
\begin{ex}
  We will use the Wald test to check whether there is a difference between the
  proportion of deaths before the Chinese Harvest Moon Festival between the two
  groups.
  \inputminted{python}{../code/10-10.py}
  \inputminted{text}{../output/10-10.txt}

  We find that there is little to no evidence against $H_0$.
\end{ex}

\begin{ex}~
  \begin{enumerate}[(a)]
    \item
          \inputminted{python}{../code/10-11.py}
          \inputminted{text}{../output/10-11.txt}
          Since only the $p$-value for Chlorpromazine is less than $0.05$, the
          only null hypothesis that we may reject is that the placebo is
          similarly effective as Chlorpromazine.
    \item Our finding remains statistically significant at the $0.05$ level
          under both the Bonferroni or the Benjamini-Hochberg multiple
          testing method corrections.
  \end{enumerate}
\end{ex}

\begin{ex}~
  \begin{enumerate}[(a)]
    \item Let $X_1,\ldots,X_n\sim\text{Poisson}(\lambda)$. We begin by computing
          the maximum likelihood estimator for $\lambda$. Recall that then
          \[
            f(x;\lambda)=e^{-\lambda}\frac{\lambda^x}{x!},
          \]
          and that therefore
          \[
            \ell_n(\lambda)=\sum_{i=1}^n-\lambda+X_i\log(\lambda)+\log(X_i!).
          \]
          Hence,
          \[
            \frac{\d\ell_n(\lambda)}{\d\lambda}
            =\sum_{i=1}^n\left[-1+\frac{X_i}{\lambda}\right]
            =\frac{n(\Xbar-\lambda)}{\lambda}
            \text{ implies }
            \lambda=\Xbar,
          \]
          and it is clear by the second derivative test that this is a maximum.
          Hence, $\widehat{\lambda}=\Xbar$, and
          $\sehat(\widehat{\lambda})=\sqrt{\Xbar/n}$.

          Let $H_0:\lambda=\lambda_0$. Then, the size $\alpha$ Wald test is
          given by rejecting $H_0$ when
          \[
            \frac{\sqrt{n}|\Xbar-\lambda_0|}{\sqrt{\Xbar}}>z_{\alpha/2}.
          \]
    \item
          \inputminted{python}{../code/10-12.py}
          \inputminted{text}{../output/10-12.txt}
  \end{enumerate}
\end{ex}

\begin{ex}
  Let $X_1,\ldots, X_n\sim N(\mu, \sigma^2)$. Recall from Example 9.11 that then
  \[
    \ell(\mu, \sigma)
    =-n\log\sigma -\frac{nS^2}{2\sigma^2}-\frac{n(\Xbar-\mu)^2}{2\sigma^2}.
  \]
  Therefore,
  \[
    \frac{\pd \ell(\mu, \sigma)}{\pd \mu}
    =\frac{n(\Xbar-\mu)}{\sigma^2}
    \text{ implies }
    \muhat = \overline{X},
  \]
  and
  \[
    \frac{\pd \ell(\Xbar, \sigma)}{\pd \sigma}=-\frac{n}{\sigma}+\frac{nS^2}{\sigma^3}
    \text{ implies }
    \sigmahat = S.
  \]
  We have $\Theta_0=\{(\mu, \sigma) \mid \mu=\mu_0\}$ and so for the likelihood
  ratio test we have
  \begin{align*}
    \lambda
     & = 2\log\left(\frac{\sup_{\theta\in\Theta}\L(\theta)}{\sup_{\theta_0\in\Theta_0}\L(\theta_0)} \right) \\
     & =2\ell(\Xbar, S)-2\ell(\mu_0, S)                                                                     \\
     & =\frac{n(\Xbar-\mu_0)^2}{S^2},
  \end{align*}
  where, under $H_0$, we expect $\lambda\rightsquigarrow \chi^2$, and
  therefore have $p$-value
  \[
    \P{\chi^2>\frac{n(\Xbar-\mu_0)^2}{S^2}}.
  \]

  Recall that $\se(\muhat)=\sigma/\sqrt{n}$, and that therefore the $p$-value
  for the Wald test is given by
  \[
    \P{|Z|>\frac{\sqrt{n}|\Xbar -\mu_0|}{S}},
  \]
  or, by squaring both sides,
  \[
    \P{Z^2>\frac{n(\Xbar -\mu_0)^2}{S^2}},
  \]
  which is equivalent to the likelihood-ratio test since $Z^2$ has a chi-squared
  distribution.
\end{ex}

\begin{ex}
  Let $X_1,\ldots, X_n\sim N(\mu, \sigma^2)$. Recall that
  \[
    \ell(\mu, \sigma)=-n\log\sigma -\frac{nS^2}{2\sigma^2}-\frac{n(\Xbar-\mu)^2}{2\sigma^2},
  \]
  and by an identical argument to the previous problem,
  \[
    \muhat=\Xbar\text{ and }\sigmahat=S.
  \]
  We then have $\Theta_0=\{(\mu, \sigma) \mid \sigma=\sigma_0\}$, and therefore
  \begin{align*}
    \lambda
     & =2\log\left(\frac{\sup_{\theta\in\Theta}\L(\theta)}{\sup_{\theta_0\in\Theta_0}\L(\theta_0)} \right) \\
     & =2\ell(\Xbar, S)-2\ell(\Xbar, \sigma_0)                                                             \\
     & = n\left[2\log\frac{\sigma_0}{S}+\frac{S^2}{\sigma_0^2}-1\right]                                    \\
     & = 2n\log\frac{\sigma_0}{S}+\frac{n(S^2-\sigma_0^2)}{\sigma_0^2},
  \end{align*}
  where, under $H_0$, we expect $\lambda\rightsquigarrow \chi^2$, and
  therefore have $p$-value
  \[
    \P{\chi^2>
      2n\log\frac{\sigma_0}{S}+\frac{n(S^2-\sigma_0^2)}{\sigma_0^2}
    }.
  \]

  Recall that $\se(\sigmahat)=\sigma/\sqrt{2n}$, and that therefore the
  $p$-value for the Wald test is given by
  \[
    \P{|Z|>\frac{\sqrt{2n}|S -\sigma_0|}{S}},
  \]
  or,
  \[
    \P{Z^2>\frac{2n(S -\sigma_0)^2}{S^2}}.
  \]
\end{ex}

% 15
\begin{ex}
  Let $X \sim \text{Binomial}(n, p)$. Note that then
  \[
    \ell(n, p)=\log\binom{n}{X}+X\log{p}+(n-X)\log(1-p),
  \]
  and therefore
  \[
    \frac{\pd\ell(n, p)}{\pd{p}}
    =\frac{X}{p}-\frac{n-X}{1-p}
    \text{ implies }
    \phat =X/n.
  \]

  We have $\Theta_0=\{p_0\}$. Therefore,
  \[
    \lambda
    =2\log\left(\frac{\sup_{\theta\in\Theta}\L(\theta)}{\sup_{\theta_0\in\Theta_0}\L(\theta_0)} \right)
    =2\ell(n, \phat)-2\ell(n, p_0)
    =2X\log\left(\frac{\phat}{p_0}\right)+2(n-X)\log\left(\frac{1-\phat}{1-p_0}\right),
  \]
  where, under $H_0$, we expect $\lambda\rightsquigarrow \chi^2$.

  Note that by Exercise 9.7, $\sehat(\phat)=\sqrt{\phat(1-\phat)/n}$, and
  therefore the $p$-value of the Wald test is given by
  \[
    \P{|Z|>\frac{\sqrt{n}|\phat-p_0|}{\sqrt{\phat(1-\phat)}}}.
  \]
\end{ex}

\begin{ex}
  Let $\ell(\theta)$ be a log-likelihood. Note that then the second degree
  Taylor polynomial expansion of $\ell$ around the MLE $\thetahat$ is given by
  \[
    \ell(\theta)
    \approx
    \ell(\thetahat)+\ell'(\thetahat)(\theta-\thetahat)
    +\frac{1}{2}\ell''(\thetahat)(\theta-\thetahat)^2,
  \]
  and therefore,
  \[
    \lambda
    =2\ell(\thetahat)-2\ell(\theta_0)
    \approx -\ell''(\thetahat)(\thetahat-\theta_0)^2
    = I(\thetahat)(\thetahat-\theta_0)^2
    = \frac{(\thetahat-\theta_0)^2}{\sehat^2(\thetahat)},
  \]
  however, this is precisely $W^2$.

  To complete the proof and show that $\frac{W^2}{\lambda}\xrightarrow{P} 1$, we
  need to be able to show that the error of our quadratic approximation to the
  log-likelihood decreases with sample size, but this is not true in general,
  and it does not look like we have any of the relevant results in the book.
\end{ex}


\subsection{Choice set}

Economic agents face options from some set. This could be consumption choices, numbers of hours to work, or how much capital to invest in at a factory.

\subsubsection{Utility functions}

Calculating choices: Unrestricted choices

The choice of an agent is the selection which corresponds to the highest value of the utility function. Consider:

\(f=2(x-1)^2-10\)

We can easily calculate that even if the agent can choose any real number \(x\), they will chose \(1\).

This approach can be used if there are not meaningful constraints, or those constraints are implicit in the utility function. For example a firm can be modelled as profit maximising, where profit is a function of revenue and costs, with a single maximising value.

Other agents, such as a consumers, may instead have utility over consumption and leisure, and a separate constraint over this. This could be solved using simultaneous equations, but such an approach is not always desirable.


\documentclass[a4paper,11pt,final]{article}
\usepackage{fancyvrb, color, graphicx, hyperref, amsmath, url}
\usepackage{palatino}
\usepackage[a4paper,text={16.5cm,25.2cm},centering]{geometry}

\hypersetup
{   pdfauthor = {Matti Pastell},
  pdftitle={FIR filter design with Python and SciPy},
  colorlinks=TRUE,
  linkcolor=black,
  citecolor=blue,
  urlcolor=blue
}

\setlength{\parindent}{0pt}
\setlength{\parskip}{1.2ex}

\title{FIR filter design with Python and SciPy}
\author{Matti Pastell \\ \url{http://mpastell.com}}
\date{15th April 2013}

\begin{document}
\maketitle

\section{Introduction}

This an example of a document that can be published using
\href{http://mpastell.com/pweave}{Pweave}. Text is written
using \LaTeX{} and code between \texttt{<<>>} and \texttt{@} is executed
and results are included in the resulting document.

You can define various options for code chunks to control code
execution and formatting (see
\href{http://mpastell.com/pweave/usage.html\#code-chunk-options}{Pweave
docs}).

\section{FIR Filter Design}

We'll implement lowpass, highpass and ' bandpass FIR filters. If you
want to read more about DSP I highly recommend
\href{http://www.dspguide.com/}{The Scientist and Engineer's Guide to
Digital Signal Processing} which is freely available online.

\subsection{Functions for frequency, phase, impulse and step response}

Let's first define functions to plot filter properties.

\begin{verbatim}
from pylab import *
import scipy.signal as signal
import warnings
warnings.simplefilter("ignore")

#Plot frequency and phase response
def mfreqz(b,a=1):
    w,h = signal.freqz(b,a)
    h_dB = 20 * log10 (abs(h))
    subplot(211)
    plot(w/max(w),h_dB)
    ylim(-150, 5)
    ylabel('Magnitude (db)')
    xlabel(r'Normalized Frequency (x$\pi$rad/sample)')
    title(r'Frequency response')
    subplot(212)
    h_Phase = unwrap(arctan2(imag(h),real(h)))
    plot(w/max(w),h_Phase)
    ylabel('Phase (radians)')
    xlabel(r'Normalized Frequency (x$\pi$rad/sample)')
    title(r'Phase response')
    subplots_adjust(hspace=0.5)

#Plot step and impulse response
def impz(b,a=1):
    l = len(b)
    impulse = repeat(0.,l); impulse[0] =1.
    x = arange(0,l)
    response = signal.lfilter(b,a,impulse)
    subplot(211)
    stem(x, response)
    ylabel('Amplitude')
    xlabel(r'n (samples)')
    title(r'Impulse response')
    subplot(212)
    step = cumsum(response)
    stem(x, step)
    ylabel('Amplitude')
    xlabel(r'n (samples)')
    title(r'Step response')
    subplots_adjust(hspace=0.5)
\end{verbatim}

\subsection{Lowpass FIR filter}

Designing a lowpass FIR filter is very simple to do with SciPy, all you
need to do is to define the window length, cut off frequency and the
window.

The Hamming window is defined as:
$w(n) = \alpha - \beta\cos\frac{2\pi n}{N-1}$, where $\alpha=0.54$ and
$\beta=0.46$

The next code chunk is executed in term mode, see the source document
for syntax. Notice also that Pweave can now catch multiple
figures/code chunk.

\begin{verbatim}
n = 61
a = signal.firwin(n, cutoff = 0.3, window = "hamming")
#Frequency and phase response
mfreqz(a)
#Impulse and step response
p = figure(2)
impz(a)
\end{verbatim}
\includegraphics[width= \linewidth]{figures/FIR_design_verb_figure2_1.pdf}
\includegraphics[width= \linewidth]{figures/FIR_design_verb_figure2_2.pdf}

\subsection{Highpass FIR Filter}

Let's define a highpass FIR filter:

\begin{verbatim}
n = 101
a = signal.firwin(n, cutoff = 0.3, window = "hanning", pass_zero=False)
mfreqz(a)
\end{verbatim}
\includegraphics[width= \linewidth]{figures/FIR_design_verb_figure3_1.pdf}

\subsection{Bandpass FIR filter}

Notice that the plot has a caption defined in code chunk options.

\begin{verbatim}
n = 1001
a = signal.firwin(n, cutoff = [0.2, 0.5], window = 'blackmanharris', pass_zero = False)
mfreqz(a)
\end{verbatim}
\begin{figure}[htpb]
\center
\includegraphics[width= \linewidth]{figures/FIR_design_verb_figure4_1.pdf}
\caption{Bandpass FIR filter.}
\label{fig:None}
\end{figure}

\end{document}


%%%%%PLEASE CONSIDER CORRECTIONS AT PLACES INDICATED%%%%%%%%
\documentclass{discussion}
%%%%%Packages Used, add more if necessary%%%%
\providecommand{\tightlist}{%
\setlength{\itemsep}{2pt}\setlength{\parskip}{0pt}}

\newenvironment{answer}
  {\vspace{-1em}\begin{proof}[Answer]}
  {\end{proof}}

\usepackage[ruled]{algorithm2e}
\begin{document}

%%%%%CHANGE HERE%%%%%%%%%
%%%%%\lecture{the ordinal number of the tutorial}{lecture title}{Instructor's name}%%%%%%%%
\lecture{6}{Ensemble Methods}{Chansoo Lee}{}

%%%%%CHANGE HERE%%%%%%%
%%%%%\section{title of the section} similarly with the rest \section{} or \subsection{} or \subsubsection{} etc

\renewcommand{\R}{\mathbb{R}}
\renewcommand{\vec}[1]{\mathbf{#1}}
\newcommand{\Dtrain}{\mathcal{D}_{\mathrm{train}}}
\newcommand{\tm}{\mathbf{\theta}_{\text{model}}}
\newcommand{\xtest}{\mathbf{\x}_{\text{test}}}
\newcommand{\test}{\mathrm{test}}
\newcommand{\pred}{\mathrm{pred}}

\newcommand{\D}{\mathcal{D}}
\newcommand{\F}{F}
\newcommand{\WL}{\mathrm{WL}}
\section{Important Concepts}
\paragraph{True vs Empirical Risk} Simply put,  \emph{risk} is the \emph{expected loss}. If we take the expectation over true data distribution, we get true risk. If we take the expectation over \emph{empirical probability density estimate} (which is uniform over a finite data, HW3 problem 2), we get empirical risk.

Now, let us define these formally. Suppose the data is i.i.d. samples from a distribution $Q$. Then, the \emph{true risk} (or simply \emph{risk}) of hypothesis $h$ is the expectation of the loss function $\ell$ over $Q$:
\[R(h) = \mathbb{E}_{(\mathbf{x},y) \sim Q}[\ell(h(\mathbf{x}), y)].\]
We have seen a very similar expression in homework 3.

The true risk cannot be computed, because $Q$ is unknown. The learning algorithm, however, can approximate the true risk using training examples $S = \{(\vec{x}_i, y_i)\}_{i=1}^{n}$:
\[R_S(h) = \frac{1}{n} \sum_{i=1}^{n} \ell(h(\mathbf{x}_i), y_i).\]
As $n$ goes to infinity, the empirical risk converges to the true risk.

For example, consider linear regression. The weight vector $\mathbf{w}$ is the model parameter, and the prediction function $h(\mathbf{x}) = \mathbf{w}^\top \mathbf{x}$ is the associated hypothesis. Define our loss function as $\ell(h(\mathbf{x}), y) = (h(\mathbf{x}) - y)^2$. The unregularized linear regression corresponds to \emph{empirical risk minimization} where your \emph{hypothesis class} is restricted to linear functions.

\paragraph{Bagging vs Boosting} Bagging focuses on aggregating outputs of multiple hypotheses independently trained on different (sub)samples of train data.

Boosting is more holistic approach, which sequentially chooses the weights on hypotheses and the weights on train examples.

\begin{exercise}
Can each hypothesis in the ensemble be trained in a parallel manner for bagging? How about for boosting?
\end{exercise}
\begin{answer}
Bagging can be, but boosting cannot be due to its sequential nature.
\end{answer}

\begin{exercise}
With access to distributed system, is bagging always computationally superior to boosting?
\end{exercise}
\begin{answer}
  Bagging reduces the variance only; its performance is essentially the average performance of the hypotheses in the ensemble. Each hypothesis in a boosted ensemble, on the other hand, needs to only satisfy weak learning hypothesis, which usually takes only a trivial amount of computing power.
\end{answer}

\section{AdaBoost}
Let $S = \{(\vec{x}_i, y_i)\}_{i=1}^{n}$ be the training set of examples. Let $\WL$ be a $\gamma$-weak learning algorithm that takes a distribution over input space (as a tuple of weights and samples) and returns a hypothesis (classifier) $f_t$ with error at most $\frac{1}{2} - \gamma$. Let $T$ be the number of iterations.

\begin{algorithm}
\caption{AdaBoost}
\KwIn{$S, \WL, T$ as defined in the text}
  initialize $\mathbf{w}^1 = (1,\ldots,1)$ \\
  \For{$t = 1,\ldots, T$}{
    normalize $\vec{w}^{t}$ \\
    invoke weak learner $f_t = \WL(\vec{w}^t, S)$ \\
    compute $r_t = \sum_{i=1}^{n} w_i^t \mathbf{1}[y_i \neq f_t(\vec{x}_i)]$ \\
    let $\displaystyle\alpha_t = \frac{1}{2} \log(\frac{1}{r_t} - 1)$ \\
    update $\displaystyle w_i^{t+1} = w_i^{t} \exp(-\alpha_t y_i f_t(\vec{x}_i))$ \\
  }
  \KwOut{Return $\F_T = \sum_{t=1}^{T} \alpha_t f_t$}
\end{algorithm}

\begin{theorem}
  The training error (empirical risk) of AdaBoost satisfies:
  \[R_S(\F_T) := \frac{1}{n} \sum_{i=1}^{n} \mathbf{1}[\F_T(\vec{x}_i) \neq y_i] \leq \exp(-2 \gamma^2 T).\]
\end{theorem}
\begin{proof} For each $t$, let
\[\textstyle \F_t = \sum_{s=1}^{t} \alpha_s f_s\]
and let $z_t$ be the normalizing constant for $\vec{w}^t$, which is (by inductive argument)
\[z_t = \frac{1}{n}\sum_{i=1}^{n} \exp(-y_i \F_t (\vec{x}_i)).\]
\emph{(Optional paragraph.)} The proof is left for exercise. Roughly speaking, we update $w_i$ in a multiplicative manner, adding $-\alpha_t y_i f_t(\mathbf{x}_i)$ to the exponent every round. So at round $t$, we have $\exp(-\sum_{s=1}^{t} \alpha_s y_i f_s(\mathbf{x}_i)) = \exp(-y_i \F_s(\mathbf{x}_i))$. The base case for $t=1$ gives the $(1/n)$.

Note that the exponential function acts as a \emph{smooth} upper bound on the loss of a hypothesis $h$, as $\mathbf{1}[h(\vec{x}) \neq y] \leq \exp(-y h(\vec{x})).$ So,
$R_S(\F_T) \leq z_T$ and it suffices to show that $z_T \leq \exp(-2\gamma^2 T)$. In other words, our normalization constant $z_t$ coincides with an upper bound on the empirical risk of $\F_t$.

Rewrite \[z_T = \frac{z_T}{z_{T-1}} \cdots \frac{z_2}{z_1}.\]
It suffices to show that for every $t$, \[\frac{z_{t+1}}{z_t} \leq \exp(-2\gamma^2)\]
This means that the \emph{upper bound} on the empirical risk decreases in a multiplicative manner (fraction of mistakes fixed as opposed to a fixed number of mistakes fixed) every round. The actual empirical risk, however, might oscillate. Higher $\gamma$ means faster decay in the upper bound, as it should.

\emph{The rest of the proof is optional:} Now,
\begin{align*}
  \frac{z_{t+1}}{z_t} &= \frac{\sum_{i=1}^{n} \exp(-y_i \F_{t+1}(\vec{x}_i))}{z_t} \\
  &= \frac{\sum_{i=1}^{n} \exp(-y_i \F_{t}(\vec{x}_i)) \exp(-y_i \alpha_{t+1} f_{t+1}(\vec{x}_i))}{z_t} \\
  &=\sum_{i=1}^{n} w_i^{t+1} \exp(-y_i \alpha_{t+1} f_{t+1}(\vec{x}_i)) \\
  &= \exp(-\alpha_{t+1})(1-r_{t+1}) + \exp(\alpha_{t+1})r_{t+1} \\
  &= \frac{1}{\sqrt{1/r_{t+1} - 1}}(1-r_{t+1}) + \sqrt{1/r_{t+1} - 1} \ r_{t+1} \\
  &= 2\sqrt{r_{t+1} (1-r_{t+1})} \leq 2 \sqrt{\left(\frac{1}{2} - \gamma\right) \left(\frac{1}{2} + \gamma\right)} = \sqrt{1 - 4\gamma^2}.
\end{align*}
To complete the proof, use the inequality $1 - a \leq e^{-a}$.
\end{proof}

Generally, minimizing a all-or-nothing (0-or-1 or 0-or-infinity) loss function is \emph{computationally} hard in the sense of NP-hardness. Hence, we use a \emph{surrogate loss} function that is continuous and convex. The \emph{hinge loss} for soft-SVM is one such example which converts a 0-or-$\infty$ loss function to a linear-and-flat function. Similarly, AdaBoost can be considered an algorithm that minimizes the exponential loss function. The key difference is that the exponential loss directly upper bounds the classification error, whereas the hinge loss does not.

\begin{exercise}
Explain the semantics of the variables used in AdaBoost: $\mathbf{w}^t$, $f_t$, $r_t$, $\alpha_t$ and $F_t$.
\end{exercise}

\begin{exercise} Answer the following questions:
\begin{itemize}
  \item Convince yourself that $r_t$ is the  risk of $f_t$ when the true distribution is $\mathbf{w}^t$.
    \[r_t = \mathbb{E}_{i \sim \mathbf{w}^t}\left[\mathbf{1}[f_t(\vec{x}_i) \neq y_i]\right].  \]
 \item Is $\alpha_t$ an increasing or decreasing function of $r_t$? Why does it make sense, considering how we construct the final output $\F_T$?
 \begin{answer}
   Decreasing. $f_t$ that had higher risk should contribute less to the final output $F_T$.
 \end{answer}
 \item The update rule for $w_i$ shows that the amount of change in $w_i^t$ is increasing or decreasing in $\alpha_t$? Why does it make sense?
  \begin{answer}
   Increasing. The weight $w_i$ should reflect how difficult it is to classify the $i$-th example correctly, so we can focus on the difficult examples. If $\alpha_t$ is large, then $f_t$ performed well overall and thus it is a ``trustworthy'' hypothesis. Its mistakes should affect our opinion on how difficult each example is, more so than a non-trustworthy hypothesis (low $\alpha_t$) would.
 \end{answer}
\end{itemize}
\end{exercise}

\begin{exercise}
Does AdaBoost guarantee a zero test error?
\end{exercise}

\begin{answer}
  No. AdaBoost guarantees that if $T$ is sufficently large, then the \emph{empirical risk} ultimately becomes 0. But it says nothing about the test error aka true risk.
\end{answer}

\section{Generalized Boosting (Optional)}

\subsection{Coordinate Descent}
Coordinate descent is a variation of gradient descent, where you change a single coordinate per iteration. For example, suppose we want to minimize
\[J(a,b) = (a - 2)^2 (b + 1)^2 \]
% with gradient
% \[\nabla J(a,b) = (2(a-2)(b+1), (a-2)^2).\]

Fix the learning rate $\eta = .1$. Initialize (arbitrarily) $a=b=0$. In the first round, we change $a$. So, we look at the partial derivative
\[\frac{\partial J}{\partial a} = 2(a-2)(b+1)^2\]
which we evaluate at $a=b=0$ and get $-4$. Now we perform the coordinate descent update and get $a=.4, b=0$. We repeat the process for $b$:
\[\frac{\partial J}{\partial b} = 2(a-2)^2(b+1)\]
which we evaluate at the current point and get $5.12$. So we perform the update and get $a=.4,b= -.512$, we repeat this process, and eventually converge to $a=2, b=-1$.

\subsection{Boosting as Coordinate Descent}
We observe that boosting is a slight variation of coordinate descent on a function of $T$ variables, where we do a single sweep through each variable. We use the notation $\hat{\ell}$ (in lecture notes, its $\phi$) to denote the surrogate loss function. Assume that $\partial{\ell}/\partial{\alpha_t}$ is always negative. We first define the objective function
\[J(\alpha_1, \ldots, \alpha_T) = \frac{1}{n} \sum_{i=1}^{n}\hat{\ell}\left(y_i \sum_{t=1}^{T} \alpha_{t}f_t (\vec{x}_i)\right) =  \frac{1}{n} \sum_{i=1}^{n}\hat{\ell}\left(y_i \sum_{t=1}^{T} \F_t (\vec{x}_i)\right).\]

We initialize $\alpha_t = 0$ for all $t$, and we do coordinate descent. At $t$-th iteration, we have
\[J(\alpha_1, \ldots, \alpha_{T}) = \frac{1}{n} \sum_{i=1}^{n}\hat{\ell}\left(y_i \sum_{s=1}^{t-1} \alpha_{s}f_s + y_i \alpha_t f_t (\vec{x}_i)\right),\]
where $\alpha_{t+1}, \ldots, \alpha_T$ terms are ommitted because they are still 0.

Hence, we update $\alpha_t$ using the partial derivative:
\[\frac{\partial J}{\partial \alpha_t} = \frac{1}{n} \sum_{i=1}^{n} \hat{\ell}'_t(y_i \F_{t-1}(\vec{x}_i) + \alpha_t y_i f_t(\vec{x}_i)) y_i f_t(\vec{x}_i)\]
where $\hat{\ell}'_t = \frac{\partial \hat{\ell}}{\partial \alpha_t}$ as a shorthand notation.

But we also have to choose $f_t$ which is not a number but a hypothesis. A smart way of choosing $f_t$ is such that the derivative is the largest, so we get the steepest coordinate descent update; that is, we choose $f_t$ to maximize the (directional) derivative:
\[\arg\max_{f_t} \frac{\partial J_t}{\partial \alpha_t}\Big|_{\alpha_t = 0} = \arg\max_{f_t} \frac{1}{n} \sum_{i=1}^{n} \hat{\ell}(y_i \F_{t-1}) y_i f_t = \arg\min_{f_t} \frac{1}{n} \sum_{i=1}^{n} \frac{\hat{\ell}'_t(y_i \F_{t-1})}{\sum_{i=1}^{n}\hat{\ell}_t'(y_i \F_{t-1})} y_i f_t.\]

The last equality is because $\ell'$ is always negative. We cannot compute the exact minimizer (if that is the case, we don't even need boosting!). Instead, we have an algorithm $\WL$ which will ``try'' to minimize the above and return a weak learning hypothesis $f_t$.

Once we fix $f_t$, it turns out that we can be very smart with how we update $\alpha_t$, instead of doing the standard coordinate descent update. In particular, we update $\alpha_t$ to be the minimizer:
\[\alpha_t = \arg\min_{\alpha} J(\alpha_{1},\ldots,\alpha_{t-1},\alpha).\]

% \paragraph{AdaBoost as Coordinate Descent}
% For AdaBoost, $\hat{\ell}$ is the exponential loss $\hat{\ell}(u) = \exp(-u)$. So,
% \begin{align*}
% \sum w_i \exp(-y_i f(\vec{x}_i) \alpha_t)
% &\leq \sum_{i} \frac{1 -y_i f(\vec{x}_i)}{2} w_i \exp(\alpha^t) + \sum_{i} \frac{1 + y_i f(\vec{x}_i)}{2} w_i \exp(- \alpha^t) \\
% &= \frac{1}{2} \exp(\alpha^{t}) - \frac{1 - r_t}{2} \exp(-\alpha_t)
% \end{align*}

\end{document}


\documentclass[10pt]{article}
\author{Alex Peyrard}
\title{equivalences}

\begin{document}
\maketitle

\section{Equivalences}
\[(1) \Leftrightarrow \forall\varepsilon>0, \lim_{n \to \infty} Pr(|X_{n}-Y|<\varepsilon)=1\]
\[(4) \Leftrightarrow \forall\varepsilon>0, \lim_{n \to \infty} Pr(|X_{n}-Y|\leq\varepsilon)=1\]
\section{Proof that $(1) \Rightarrow (4)$}
\[\lim_{n \to \infty} Pr(|X_{n}-Y|<\varepsilon) \leq \lim_{n \to \infty} Pr(|X_{n}-Y|\leq\varepsilon)\]
\[\lim_{n \to \infty} Pr(|X_{n}-Y|<\varepsilon) = 1\]
We thus have
\[1 <= \lim_{n \to \infty} Pr(|X_{n}-Y|\leq\varepsilon) <= 1\]
So \[ \lim_{n \to \infty} Pr(|X_{n}-Y|\leq\varepsilon) = 1 \]
QED
\section{Proof that $(4) \Rightarrow (1)$}
\end{document}


\subsection{Bonus: Construct the Reals from the Rationals}
The definition of convergence and Cauchy sequence makes sense in $\mathbb Q$ as well.
The matters, there are sequences in $\mathbb Q$ that ``want to converge'' (being Cauchy) but don't, like $1,1.4,1.41,1.414$.
In fact, every $x\in\mathbb R$ is a limit of a sequence in $\mathbb Q$ (by e.g. decimal expansion).
So we can define
$$\mathbb R=\{(x_n)\in\mathbb Q: (x_n)\text{ is Cauchy}\}/\sim$$
where $\sim$ is an equivalence relation defined by $(x_n)\sim (y_n)\iff x_n-y_n\to 0$.
\begin{proposition}
    This is indeed the real number with the intuitive definition of operations and order.
\end{proposition}
In particular $(x_n)\ge0=(0,0,\ldots)$ iff $\exists N\in\mathbb n,\forall n>N,x_n\ge 0$.
\begin{proof}
    Other properties are trivial, so we shall check the least-upper-bound property.
    Note that the least-upper-bound property, the monotone sequence theorem and the statement that every Cauchy sequence converges are equivalent.
    So we shall prove that every Cauchy sequence converges.
    Suppose $(x_{m,n})=((x_m)_n)$ is a Cauchy sequence in $\mathbb R$ defined the way we made it.
    Then the sequence $(y_k)=x_{k,k}$ goes to the limit we want.
\end{proof}


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\logit}{logit}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\cov}{cov}
\DeclareMathOperator*{\expec}{E}
\DeclareMathOperator*{\deriv}{d}
\DeclareMathOperator*{\const}{constant}

\begin{document}
\title{\textsf{Homework 5 for Bayesian Data Analysis}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 6.1a}
{
    Under the model assumption of identical effect for each group, we fix the hyperparameter $\tau=0$ in our normal hierarchical model. Plug it in and we obtain $\theta_j \sim N(7.7, 4.1^2)$. The posterior predictive distribution is simulated as follows: \\
    1) generate $\theta \sim N(7.7, 4.1^2)$, and \\
    2) generate $y_j^\mathrm{rep} \sim N(\theta, \sigma_j^2)$ for each $j$, followed by \\
    3) calculate the order stastistics out of $y_j^\mathrm{rep}$ for each $j$.

    The observed order statistics are approximately the order statistics calculated from the replicated simulations above.
    \begin{figure}[H]
      \centering
      \includegraphics[width = 1.0\linewidth]{data.vs.png}
      \caption{Observed order statistics v.s. Simulations order statistics}
    \end{figure}
    \begin{lstlisting}
    > mod = lm((data.simulated - data.observed) ~ data.observed)
    > summary(mod)

    Call:
    lm(formula = (data.simulated - data.observed) ~ data.observed)

    Residuals:
        Min      1Q  Median      3Q     Max
    -4.4555 -1.1794  0.3315  2.2663  2.6837

    Coefficients:
                  Estimate Std. Error t value Pr(>|t|)
    (Intercept)   -1.30976    1.33122  -0.984    0.363
    data.observed  0.07826    0.10150   0.771    0.470

    Residual standard error: 2.805 on 6 degrees of freedom
    Multiple R-squared:  0.09014,	Adjusted R-squared:  -0.06151
    F-statistic: 0.5944 on 1 and 6 DF,  p-value: 0.47
    \end{lstlisting}
    The p-value is 0.47, so the null hypothesis of unit slope cannot be rejected. We therefore conclude that the model fits the aspect of data here.
}

\section*{Question 6.1b}
{
    The model of identical effect for each group assumes that $\theta_j = \theta$ for all $j$, and that school A has better effect than school C simply \emph{by chance}. But the model assumption can be skeptical given the observation, as we wonder whether there is other factors that cause school A to perform better than schoold C. In this case, the likelihood of the assumption of identical effects is far from being convincing, especially compared to the hierarchical models.
}

\section*{Question 6.6a}
{
    We have 7 ones and 13 zeros out of the $n=20$ observations.
    \begin{itemize}
        \item \textbf{Assume stops at 20th observation:}\quad The likelihood is proportional to $\theta^{7} (1-\theta)^{13}$, according to the binomial model with number of observations fixed to 20.
        \item \textbf{Assume stops at 13th zero:}\quad The likelihood is also proportional to $\theta^{7} (1-\theta)^{13}$, \emph{up to a different scale}, according to the negative binomial model with number of failures fixed to 13 with the last trial fixed to zero.
    \end{itemize}
    There is no difference with the likelihood except for a constant factor. Therefore, the posterior distribution does not change, either.
}

\section*{Question 6.6b}
{
    Generate $\theta$ from $p(\theta) \sim \theta^{7} (1-\theta)^{13}$ for 10000 times, and generate $y^\mathrm{rep}$ under the new protocol (stops at the 13th zero) each time.
    \begin{figure}[H]
        \centering
        \includegraphics[width = 1.0\linewidth]{switches.histogram.png}
        \caption{Histogram of number of switches}
    \end{figure}

    First, the statistic T has a distribution with a heavier tail. This is because the number of observations can exceed the limit of $n=20$ under the new protocol.

    Second, we find that T is more likely to be an even number than an odd number when $T < 10$. A possible explanation is: The last few observations must be zeros when $T$ is small, since the last one must be zero and there is few switches. Note that there must be $13$ zeros in total. Therefore, the distribution of $T$ is highly related to the specific value of $T$, i.e. $p(T=k)$ is more sensitive to $k$ when $k$ is relatively small.
}

\section*{Source Code in R}
{
    \begin{lstlisting}[language=R]
    # Question 6.1
    data.observed = c(28, 18, 12, 8, 7, 1, -1, -3)
    data.simulated = c(26, 19, 14, 10, 6, 2, -3, -9)

    png("data.vs.png", width=800, height=600)
    plot(data.observed, data.simulated, main="Simulated v.s. simulated order statistics")
    dev.off()

    mod = lm((data.simulated - data.observed) ~ data.observed)
    summary(mod)

    # Question 6.6
    T0 = 3 # number of switches
    N = 10000
    T = 1:N
    theta = rbeta(N, shape1=7+1, shape2=13+1)
    for (j in 1:N) {
        count.switches = 0
        y.last = rbinom(1, size=1, prob=theta[j])
        count.zeros = 1 - y.last
        while (1) {
        y = rbinom(1, size=1, prob=theta[j])
        if (y != y.last) {
            count.switches = count.switches + 1
        }
        count.zeros = count.zeros + 1 - y.last
        if (count.zeros > 12.5) {
            break;
        }
        y.last = y
        }
        T[j] = count.switches
    }
    png("switches.histogram.png", width=800, height=600)
    hist(T, breaks=seq(-0.5, max(T)+0.5))
    dev.off()
    \end{lstlisting}
}

\clearpage
\end{document}


\subsubsection{Direction Matters}
\noindent
\begin{lemma}
	\begin{equation*}
		\int\limits_{-C}{\vec{F} \cdot \mathrm{d}\vec{r}} = -\int\limits_{C}{\vec{F}\mathrm{d}\vec{r}}
	\end{equation*}
\end{lemma}
That is, the direction in which one takes a line integral matters. You can think of this as the wind helping you sail in one direction (positive line integral) but fighting against you in the opposite direction (negative line integral).


% Created 2020-11-11 Wed 19:46
% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\author{Muaz Alhaider and Zakariya Ahmed}
\date{\today}
\title{Program 2}
\hypersetup{
 pdfauthor={Muaz Alhaider and Zakariya Ahmed},
 pdftitle={Program 2},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 27.1 (Org mode 9.3)},
 pdflang={English}}
\begin{document}

\maketitle

\section*{Modules}
\label{sec:orgeb98485}

\subsection*{Transitional Probability}
\label{sec:org7d7cbf4}
   \begin{math}
    Transitional\ Probability(Pos_i, Dir) = \\
     (Pos_{from\ left}:Drift(Left), Pos_{from\ straight}:Drift(Straight), Pos_{from\ right}:Drift(Right)): (\text{For smoothing})
\end{math}

\newline
\begin{math}
\\ \\
Transitional\ Probability(Pos_i, Dir) =\\ (Pos_{from\ left}:Drift(Left), Pos_{from\ straight}:Drift(Straight), Pos_{from\ right}:Drift(Right), Pos_{from\ behind}:Drift(Straight)): (\text{For prediction})
\\ \\
\end{math}

\emph{Left}, \emph{Right}, and \emph{Straight} are all positions defined in relation to what \emph{Dir} is: if \emph{Dir} is EAST, then \emph{Left} is SOUTH, \emph{Right} is NORTH, and Straight is EAST for example.

Here, transitional probability has two forms: one which is all paths that converge to a point, and another where they diverge from a point. Prediction (as the name implies) wants all the possible paths to a point, which is why we include \(Pos_{from\ behind}\). Smoothing however, does not use that, which is why it's not included.

\subsection*{Prediction}
\label{sec:org59a7289}
$$Prediction(Grid, Direction) = \left\{pos_i \in Grid \mid \sum_{(Pos_j, DriftProb)}^{Transition\ Probability(pos_i, direction)} DriftProb \cdot P(Pos_j) \right \}$$

\(Prediction(Grid, Direction)\) (as the name implies) attempts to predict where the agent will be given previous information. It does this by transforming the grid by the expression  \(\sum^{Transition\ Probability(pos_i, direction)}_{(Pos_j, DriftProb)} DriftProb \cdot P(Pos_j)\).This gets the probability of an agent drifting (or if direction is straight, accurately going to) a point, and what is the probability the agent would be at the point \(Pos_j\).

\subsection*{Evidence Conditional Probability}
\label{sec:org7e2fa92}
   \begin{math}
  Evidence\ Conditional\ Probability(Pos_i, Evidence)= \\
  \prod^{\text{Directions} }_{dir=W} Sense(evidence[pos_i dir], actual[pos_i+dir])
\end{math}

This is the expression we use to get the evidence conditional probability: it's the product of each the evidecne's value at a direction times what's actually in the value of the direction. So if \emph{Left} has opened, but evidence says it's closed, it's 0.2. Taking the product of all direction's sensed value and actual value, it will result in the Evidence Conditional Probability at \(Pos_i\) given \(Evidence\)

\subsection*{Filtering}
\label{sec:org2604945}

   \begin{math}
Filtering(Grid, Evidence) = \{pos_i \in Grid \mid\\
\frac{P(pos_i) \cdot Evidence\ Conditional\ Probability(pos_i, Evidence) }{\sum^{all\ positions}_{pos} P(pos_i) \cdot Evidence\ Conditional\ Probability(pos_i, evidence) } \}
\end{math}

Filtering is a transformation upon the grid: each value gets transformed by the expression \(\frac{P(pos_i) \cdot Evidence\ Conditional\ Probability(pos_i, Evidence) }{\sum^{all\ positions}_{pos} P(pos_i) \cdot Evidence\ Conditional\ Probability(pos_i, evidence) }\), which for purposes of making it easier to talk about, will be expressed as \(Filter\ Step(pos_i, Evidence)\). \(Filter\ Step\) is conditional probability of each point times what probabilty of the  point previously, and then dividing it  by the sum of all points on the grid.

\section*{Results}
\label{sec:org4a73001}
The code outputs the following:
\begin{verbatim}
julia SUBMIT.jl
\end{verbatim}

\section*{Screenshots}
\label{sec:org3466924}
\begin{center}
\includegraphics[width=.9\linewidth]{data/a9/3abd3f-f652-4b14-a3ef-d46d087ebe0c/screenshot-20201111-135323.png}
\end{center}
\begin{center}
\includegraphics[width=.9\linewidth]{data/a9/3abd3f-f652-4b14-a3ef-d46d087ebe0c/screenshot-20201111-135329.png}
\end{center}

\section*{Who did what}
\label{sec:org8a29d99}
\subsection*{Zakariya}
\label{sec:org52942fd}
\begin{itemize}
\item Transitional probability/prediction algorithm, code and report.
\end{itemize}
\subsection*{Muaz}
\label{sec:org5f24d77}
\begin{itemize}
\item Conditional Evidence probability, filtering and smoothing algorithm and code.
\end{itemize}
\end{document}


\subsection{Solving}

\begin{frame}
  \frametitle{Solving}

  Suppose that no negative cycle exists in $G(V,E)$ for $\babst{\mu}$,
  how do we find a model $\mu$ for the integer variables ?
  \vfill
  \pause
  State-of-the-art methods employ SSSP (Single Source Shortest Paths) algorithms,
  such as the {\bf Bellman-Ford} algorithm (or its variations), as follows:
  \begin{itemize}
    \item we add an artificial vertex $I$ to $V$, and add $\{ (I,x_1;0), \ldots, (I,x_n;0) \}$ to $E$
          (notice that this never creates negative cycles)
    \vfill

    \item Compute the shortest paths from source $I$ (run Bellman-Ford). Let $\pi(x)$ be the shortest path
          from $I$ to $x$, for all $x \in V$
    \vfill

    \item The model $\mu$ can be computed as $\mu(x) = -\pi(x)$ for all $x \in V$ (see later why)
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Bellman-Ford}

  $\pi(x)$: current distance of $x$ from $I$\\
  $TBV$: queue of vertexes To Be Visited\\

  \begin{tabbing}
  as \= a \= a \= a \= as \= asdfasdfasdfasdfasdfasdfasdfasdf \= \kill
  1  \> $\pi(x) = \infty$ for all $x \in V, x \not= I$ \\
  2  \> $\pi(I) = 0$ \\
  3  \> $TBV.pushBack( I )$ \\
  4  \> while ( $TBV.size(\ ) > 0$ ) \\
  5  \> \> $s = TBV.popFront(\ )$ \\
  6  \> \> foreach $(s,t;w) \in E$         \> \> \> \> // for each outgoing edge \\
  7  \> \> \> if ( $\pi(t) - \pi(s) > w$ )    \> \> \> // is too far ? \\
  8  \> \> \> \> $\pi(t) = \pi(s) + w$           \> \> // relax (decrease $\pi(t)$) \\
  9  \> \> \> \> if ( $TBV.has( t ) == false$ ) \\
  10 \> \> \> \> \> $TBV.pushBack( t )$             \> // enqueue t if not there \\
  \end{tabbing}

\end{frame}

\begin{frame}
  \frametitle{Bellman-Ford, Example}

  \scriptsize

  $\pi(x)$: current distance of $x$ from $I$\\
  $TBV$: queue of vertexes To Be Visited\\

  \begin{columns}

  \begin{column}{.45\textwidth}
    \begin{tabbing}
    as \= a \= a \= a \= as \= asdfasdfasdfasdfasdfasdfasdfasdf \= \kill
    \> \coloneat{$\pi(x) = \infty$ for all $x \in V, x \not= I$}{3|handout:0} \\
    \> \coloneat{$\pi(I) = 0$}{3|handout:0} \\
    \> \coloneat{$TBV.pushBack( I )$}{3|handout:0} \\
    \> while ( $TBV.size(\ ) > 0$ ) \\
    \> \> \coloneat{$s = TBV.popFront(\ )$}{4,7,10|handout:0} \\
    \> \> \coloneat{foreach $(s,t;w) \in E$}{5,8,11|handout:0} \\
    \> \> \> \coloneat{if ( $\pi(t) - \pi(s) > w$ )}{5,8,11|handout:0} \\
    \> \> \> \> \coloneat{$\pi(t) = \pi(s) + w$}{5,8,11|handout:0} \\
    \> \> \> \> \coloneat{if ( $TBV.has( t ) == false$ )}{5,8,11|handout:0} \\
    \> \> \> \> \> \coloneat{$TBV.pushBack( t )$}{5,8,11|handout:0} \\
    \end{tabbing}

    $TBV = $ [ \only<3|handout:0>{$I$}\only<5,6|handout:0>{$x$}\only<6|handout:0>{$,\,$}\only<6-9|handout:0>{$y,$}\only<6-12|handout:0>{$z,w,t$} ] \\
    Current vertex: \only<4,5|handout:0>{$I$}\only<7,8|handout:0>{$x$}\only<10,11|handout:0>{$y$} \\
    Current edge: \only<5|handout:0>{$(I,x;0)$}\only<11|handout:0>{$(y,z;-1)$}
  \end{column}

  \begin{column}{.55\textwidth}
    \begin{center}
    \begin{overlayarea}{.55\textwidth}{3cm}
      \only<1|handout:0>{\input{example_bf}}
      \only<2|handout:0>{\input{example_bf_1}}
      \only<3|handout:0>{\input{example_bf_2}}
      \only<4|handout:0>{\input{example_bf_3}}
      \only<5|handout:0>{\input{example_bf_4}}
      \only<6|handout:0>{\input{example_bf_5}}
      \only<7,8|handout:0>{\input{example_bf_6}}
      \only<9|handout:0>{\input{example_bf_5}}
      \only<10|handout:0>{\input{example_bf_7}}
      \only<11|handout:0>{\input{example_bf_8}}
      \only<12|handout:0>{\input{example_bf_9}}
      \only<13|handout:0>{\input{example_bf_10}}
      \only<14>{\input{example_bf_11}}
    \end{overlayarea}
    \end{center}
  \end{column}

  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Bellman-Ford, consideration}

  \scriptsize

  \begin{exampleblock}{Invariant}
  At the end of BF, $\pi(x)$ holds the shortest distance from $I$ to $x$, for all $x \in V$ \\
  \end{exampleblock}
  \vfill
  \pause
  \begin{lemma}[Shortest Path]
  At the end of BF, $\pi(y) - \pi(x) \leq c$ holds for all $(x,y;c) \in E$
  \end{lemma}
  \vfill
  \pause
  \begin{proof}
  Suppose, for the sake of contradiction, that for an edge $(x,y;c)$, we have $\pi(y) - \pi(x) > c$
    \begin{center}
      \input{arc_proof}
    \end{center}
    \vspace{-30pt}
    $\pi(x)$ is the shortest dist. from $I$ to $x$ (by Invariant). But since $\pi(y) > \pi(x) + c$,
    the shortest path from $I$ to $y$ is $\pi(x) + c$. So $\pi(y)$ is not the shortest dist. Contradiction.
  \end{proof}

\end{frame}

\begin{frame}
  \frametitle{Finding a model $\mu$}

  Because of the previous lemma we have that
  \vfill
  \begin{center}
    $\pi(y) - \pi(x) \leq c\quad\quad$ holds for all $(x,y;c) \in E$
  \end{center}
  \vfill
  So, if we take $\quad\quad\mu(x) = -\pi(x)\quad\quad$ we have that
  \vfill
  \begin{center}
    $\mu(x) - \mu(y) \leq c\quad\quad$ holds for all constraints in $\babst{\mu}$
  \end{center}
  \vfill
  and therefore $\mu$ is a model

\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 4}

Prove that all problems that are NP-complete are polynomially equivalent, in the sense that if $A$ and $B$ are NP-complete then $A \leq_P B$ and $B \leq_P A$.

\subsection*{Solution}

By definition, $A$ is NP-complete if and only if $A$ is in NP and for \textbf{any} problem $B$ in NP, $B\leq_P A$.
Proof is immediately attained by using assumptions that $A$ and $B$ are NP-complete, which means they are in NP.
Since $B$ is in NP, using the assumption that $A$ is NP-hard, we can obtain $B \leq_P A$.
Similarly, since $A$ is in NP, using the assumption that $B$ is NP-hard, we can write $A \leq_P B$.
Therefore, both NP-complete problems are equivalent.