\subsection{Cardinals}\label{subsec:cardinals}

\begin{definition}\label{def:equinumerosity}\mcite[129 \\ 145]{Enderton1977Sets}
  We say that two sets are \term{equinumerous} if there exists a \hyperref[def:function_invertibility/bijective]{bijective function} between them.

  If there exists an \hyperref[def:function_invertibility/injective]{injective} function from \( A \) to \( B \) that is not necessarily \hyperref[def:function_invertibility/surjective]{surjective}, we say that \( A \) is \term{dominated by} \( B \) or that \( B \) \term{dominates} \( A \). If \( B \) dominates \( A \), and they are not equinumerous, we say that \( B \) \term{strictly dominates} \( A \).

  Equinumerosity arises naturally outside the theory of cardinal numbers, unlike set dominance. We are usually instead interested only in injective functions that preserve some structure, i.e. \hyperref[def:first_order_homomorphism_invertibility/embedding]{embeddings}.
\end{definition}

\begin{lemma}\label{thm:three_equinumerous_sets_lemma}\mcite[prop. 6.9]{OpenLogicFull}
  If \( A \subseteq B \subseteq C \) and \( A \) is \hyperref[def:equinumerosity]{equinumerous} with \( C \), then \( B \) is equinumerous with \( C \).
\end{lemma}
\begin{proof}
  If \( B = C \), the lemma is trivial since the \hyperref[def:multi_valued_function/identity]{identity function} \( \id_B: B \to B \) is bijective. If \( B \subsetneq C \), however, the identity \( \id_B \) must be extended in order to be a bijective function between \( B \) and \( C \). It will actually be simpler for us to define a function from \( C \) to \( B \).

  Let \( f: C \to A \) be a bijective function (such a function exists by the statement of the lemma). Define the set
  \begin{equation*}
    I \coloneqq \bigcap\set{ X \subseteq C \given (C \setminus B) \subseteq X \T{and} f(X) \subseteq X }
  \end{equation*}
  of all intermediate sets between \( C \setminus B \) and \( C \) that are invariant under \( f \).

  Use \hyperref[rem:natural_number_recursion]{natural number recursion} to build the function
  \begin{equation*}
    \begin{aligned}
      &g: C \to B \\
      &g(x) \coloneqq \begin{cases}
        x,    &x \in C \setminus I \\
        f(x), &x \in I
      \end{cases}
    \end{aligned}
  \end{equation*}

  By construction, \( C \setminus B \subseteq I \) and thus \( C \setminus I \subseteq C \setminus (C \setminus B) = B \). Therefore, the range of \( g \) really is \( B \). We must show that \( g \) is injective and surjective.

  Let \( g(x_1) = g(x_2) \) for some members \( x_1 \) and \( x_2 \) of \( C \). If \( x_1 \) and \( x_2 \) both belong to either \( I \) or \( C \setminus I \), it is trivial to see that \( x_1 = x_2 \). It turns out that these are two only possible scenarios. Indeed, without loss of generality, suppose that \( x_1 \in I \) and \( x_2 \in C \setminus I \). Then \( f(x_2) = g(x_2) = g(x_1) = x_1 \). Since \( I \) is invariant under \( f \) and \( x_1 \in I \), we have \( x_2 = f(x_1) \in I \), which contradicts our choice of \( x_2 \). Therefore, \( g \) is an injective function.

  To see that \( g \) is also surjective, suppose that there exists some \( y \in B \setminus g[B] \). If \( y \in I \), then by the invariance of \( f \) we have \( g(y) = f(y) \in I \) and thus \( g(y) \not\in B \), which contradicts our definition of \( g \). If instead \( y \in C \setminus I \), then \( g(y) = y \) and thus \( y \in g[B] \), which contradicts our choice of \( y \). The obtained contradictions show that \( g \) is surjective.
\end{proof}

\begin{theorem}[Cantor-Schr\"oder-Bernstein theorem]\label{thm:cantor_schroder_bernstein_theorem}\mcite[sec. 6.5]{OpenLogicFull}
  If two sets \hyperref[def:equinumerosity]{dominate} each other, they are \hyperref[def:equinumerosity]{equinumerous}.
\end{theorem}
\begin{proof}
  Let \( f: A \to B \) and \( g: B \to A \) be injective functions. From \fullref{thm:function_composition_invertibility} it follows that \( g \bincirc f: A \to A \) is also an injective function. If we restrict its range to its image \( g[f[A]] \), it becomes bijective. Hence, \( A \) is equinumerous with \( g[f[A]] \). Since \( g[f[A]] \subseteq g[B] \subseteq A \), from \fullref{thm:three_equinumerous_sets_lemma} it follows that \( A \) is equinumerous with \( g[B] \), which is the desired result.
\end{proof}

\begin{remark}\label{rem:cardinal_definition}
  \hyperref[def:equinumerosity]{Set domination} generalizes the \hyperref[def:subset]{subset relation} between sets.

  If we take a family \( \mscrA \) of sets, then domination is a \hyperref[def:preordered_set]{preorder} rather than a true \hyperref[def:partially_ordered_set]{partial order}.
  \begin{itemize}
    \item Reflexivity follows because the \hyperref[def:multi_valued_function/identity]{identity function} for any set is injective.
    \item Transitivity is a consequence of \fullref{thm:function_composition_invertibility}.
    \item Antisymmetry fails if \( A \) dominates \( B \) and \( B \) dominates \( A \), but \( A \neq B \). For example, the map \( n \mapsto 2n \) from all natural numbers \( \BbbN \) to the even natural numbers \( 2\BbbN \) is injective and the identity map on \( 2\BbbN \) is an injective function from \( 2\BbbN \) to \( \BbbN \), however \( \BbbN \neq 2\BbbN \)
  \end{itemize}

  As a matter of fact, \hyperref[def:equinumerosity]{equinumerosity} is also a preorder and the proof for that is identical.

  If we partition \( \mscrA \) using the \hyperref[def:equinumerosity]{equinumerosity relation} if follows from \fullref{thm:preorder_to_partial_order} that the result will be a partial ordered set. The equivalence classes of this partition are then subfamilies of \( \mscrA \) such that every two sets in a single subfamily are equinumerous. For example, if \( \mscrA = \set{ \set{ A, B }, \set{ C, I }, \set{ A }, \set{ C } } \), then the corresponding equivalence classes are \( \set{ \set{ A, B }, \set{ C, I } } \) and \( \set{ \set{ A }, \set{ C } } \).

  Each of these equivalence classes consists of sets that are identical in \enquote{size} (not to be confused with \enquote{large} and \enquote{small} sets as defined in \fullref{def:large_and_small_sets}). In the above example, the corresponding equivalence classes correspond to sets of sizes \( 1 \) and \( 2 \). If we want to extend this notion of \enquote{size} to infinite sets, we must introduce a hierarchy of \enquote{sizes}. A natural candidate for such a hierarchy are the equivalence classes themselves. Unfortunately, this would mean that every family of sets has a different hierarchy. Since the entire universe is only available within the metatheory, we cannot partition the universe itself and must instead resort to finding a concrete representative of each possible equivalence class. We will call these representatives \term{cardinal numbers}.

  As explained in the proof of \fullref{thm:cardinality_existence}, it will be convenient for us to define cardinal numbers as certain \hyperref[def:ordinal]{ordinal numbers} --- see \fullref{def:cardinal}.
\end{remark}

\begin{definition}\label{def:cardinal}
  A \term{cardinal number} or simply \term{cardinal} is an \hyperref[def:ordinal]{ordinal} that is not \hyperref[def:equinumerosity]{equinumerous} with any smaller ordinal. We usually denote them using the small Greek letters \( \kappa \), \( \mu \) and \( \nu \).

  A cardinal is by definition an ordinal and this is useful. For example, the cardinals are well-ordered in the sense of \fullref{thm:ordinals_are_well_ordered}.

  We often regard cardinal numbers as abstract entities, however. It is thus accepted to call the ordinal itself the \term{initial ordinal} of the cardinal.
\end{definition}

\begin{lemma}\label{thm:natural_number_is_not_equinumerous_to_proper_subset}
  No natural number (as a member of \hyperref[thm:smallest_inductive_set_existence]{\( \omega \)}) is equinumerous to a proper subset of itself.
\end{lemma}
\begin{proof}
  We will use \fullref{thm:omega_induction} on \( n \in \omega \). The lemma holds vacuously for \( n = 0 \).

  Now suppose that \( n \) is not equinumerous to a proper subset of itself and, aiming at a contradiction, suppose that there exists a subset \( E \subseteq \op{succ}(n) \) and a bijective function \( {f: \op{succ}(n) \to E} \).
  \begin{itemize}
    \item If \( n \in E \), then \( E \setminus \set{ n } \) is a subset of \( n \) and thus the restriction \( f\restr_n: n \to (E \setminus \set{ n }) \) is a bijective function.
    \item If \( n \not\in E \), then \( E \) is a subset of \( n \) and thus \( f\restr_n: n \to E \) is a bijective function.
  \end{itemize}

  In both cases we obtain that \( n \) is equinumerous with a proper subset of itself, which is a contradiction. Hence, this also holds for \( \op{succ}(n) \).

  The induction principle allows us to conclude that the lemma holds for all natural numbers.
\end{proof}

\begin{proposition}\label{thm:natural_numbers_are_cardinals}
  The natural numbers (as members of \hyperref[thm:smallest_inductive_set_existence]{\( \omega \)}) are \hyperref[def:cardinal]{cardinals}.
\end{proposition}
\begin{proof}
  Fix a natural number \( n \in \omega \). Note that for every \( m < n \), \( m \) is a proper subset of \( n \) by \fullref{thm:ordinal_ordering_via_subsets}. From \fullref{thm:natural_number_is_not_equinumerous_to_proper_subset} it follows that no ordinal strictly smaller than \( n \) is equinumerous with \( n \) and hence \( n \) is an initial ordinal.
\end{proof}

\begin{proposition}\label{thm:omega_is_a_cardinal}
  The \hyperref[thm:smallest_inductive_set_existence]{smallest inductive set \( \omega \)} is a \hyperref[def:cardinal]{cardinal}.

  When regarded as a cardinal, we denote it by \( \aleph_0 \). This is consistent with \fullref{def:aleph_hierarchy}.
\end{proposition}
\begin{proof}
  We will use induction on \( n < \omega \) to show that no function \( f: \omega \to n \) is surjective. This is trivial for \( 0 \). Suppose that it holds for some fixed \( n \). Let \( f: \omega \to \op{succ}(n) \) be any function. Define
  \begin{equation*}
    \begin{aligned}
      &g: \omega \to n \\
      &g(m) \coloneqq \begin{cases}
        f(m), &f(m) < n \\
        0,    &f(m) = n
      \end{cases}
    \end{aligned}
  \end{equation*}

  Our inductive hypothesis states that \( g \) cannot be surjective. Hence, \( f \) also cannot be surjective.

  Therefore, \( \omega \) is an initial ordinal.
\end{proof}

\begin{proposition}\label{thm:cardinality_existence}
  Every set \( A \) is equinumerous with a unique \hyperref[def:cardinal]{cardinal}. We denote this cardinal by \( \card(A) \) and call it the \term{cardinality} of \( A \).
\end{proposition}
\begin{proof}
  By \fullref{thm:well_ordering_theorem} there exists a relation \( \prec \) that well-orders \( A \). The \hyperref[thm:well_ordered_order_type_existence]{order type} \( \ord(A, \prec) \) is an ordinal that is equinumerous with \( A \), however it may not be the smallest one. Fortunately, we can define
  \begin{equation*}
    \card(A) \coloneqq \min\set{ \beta \leq \ord(A, \prec) \given \beta \T{is equinumerous with} A }.
  \end{equation*}
\end{proof}

\begin{proposition}\label{thm:cardinality_order_compatibility}
  The set \( A \) is dominated by \( B \) if and only if \( \card(A) \leq \card(B) \).
\end{proposition}
\begin{proof}
  \SufficiencySubProof First suppose that \( \card(A) \leq \card(B) \). By \fullref{thm:ordinal_ordering_via_subsets}, we have \( \card(A) \subseteq \card(B) \) and thus the identity function \( \id_{\card(A)} \) is an injective function from \( \card(A) \) to \( \card(B) \). Since \( A \) is equinumerous with \( \card(A) \) and \( B \) is equinumerous with \( \card(B) \), by \fullref{thm:function_composition_invertibility} we obtain that there is an injective function from \( A \) to \( B \) and hence \( B \) dominates \( A \).

  \NecessitySubProof Conversely, let \( f: A \to B \) be an injective function. We again use \fullref{thm:function_composition_invertibility} to conclude that \( \card(B) \) dominates \( \card(A) \).

  We will show that \( \card(A) > \card(B) \) leads to a contradiction, which by the trichotomy of cardinals will entail that \( \card(A) \leq \card(B) \). If we suppose that \( \card(A) > \card(B) \), then \( \card(B) \subseteq \card(A) \) and hence the identity on \( \card(B) \) is an injective function. Thus, \( \card(A) \) dominates \( \card(B) \) and vice versa, which by \fullref{thm:cantor_schroder_bernstein_theorem} implies that \( \card(A) \) is equinumerous with \( \card(B) \). It follows that \( \card(A) = \card(B) \), which contradicts our assumption that \( \card(A) > \card(B) \).

  Therefore, \( \card(A) \leq \card(B) \).
\end{proof}

\begin{corollary}\label{thm:set_domination_relation_trichotomy}
  Any two sets are either equinumerous or one strictly dominates the other.

  See also \fullref{def:pigeonhole_principle}.
\end{corollary}
\begin{proof}
  Follows from cardinal trichotomy and \fullref{thm:cardinality_order_compatibility}.
\end{proof}

\begin{theorem}[Cantor's power set theorem]\label{thm:cantor_power_set_theorem}\mcite[thm. 6B]{Enderton1977Sets}
  The power set of any set \( A \) \hyperref[def:equinumerosity]{strictly dominates} \( A \). That is,
  \begin{equation*}
    \card(A) < \card(\pow(A)).
  \end{equation*}
\end{theorem}
\begin{proof}
  The function \( x \mapsto \set{ x } \) is clearly an injective function from \( A \) to \( \pow(A) \), therefore \( \pow(A) \) dominates \( A \). The converse is not true, however.

  Indeed, fix some function \( f: A \to \pow(A) \) and define the set
  \begin{equation*}
    B \coloneqq \set{ x \in A \colon x \not\in f(x) }.
  \end{equation*}

  Note that \( B \subseteq A \) and thus \( B \in \pow(A) \), however \( B \) is not in the \hyperref[def:multi_valued_function/image]{image} of \( f \) and thus \( f \) is not \hyperref[def:function_invertibility/surjective]{surjective}.

  Since \( f \) was arbitrary, we conclude that no function from \( A \) to \( \pow(A) \) is surjective.
\end{proof}

\begin{definition}\label{def:set_finiteness}
  We say that the set \( A \) is \term{finite} if any of the following equivalent conditions hold:
  \begin{thmenum}
    \thmitem{def:set_finiteness/cardinality} The \hyperref[thm:cardinality_existence]{cardinality} of \( A \) is a natural number. That is, we have \( \card(A) < \aleph_0 \).

    \thmitem{def:set_finiteness/dedekind} The set \( A \) is not \hyperref[def:equinumerosity]{equinumerous} all of its proper subsets. That is, if \( B \) is a proper subset of \( A \), then no function from \( A \) to \( B \) is injective.
  \end{thmenum}

  If a set is not finite, we say that it is \term{infinite}. If a set does not satisfy \fullref{def:set_finiteness/dedekind}, we say that it is \term{Dedekind infinite}.
\end{definition}
\begin{proof}
  \ImplicationSubProof{def:set_finiteness/cardinality}{def:set_finiteness/dedekind} We will use \fullref{thm:omega_induction} on \( n < \aleph_0 \) to prove that all sets of cardinality \( n \) are not Dedekind infinite. The case \( n = 0 \) is vacuous. Suppose that all sets of cardinality \( n \) are not Dedekind infinite and suppose that \( A \) of cardinality \( \op{succ}(n) \) is Dedekind infinite.

  Then there exists a proper subset \( B \) of \( A \) that is equinumerous with \( \op{succ}(n) \). Since \( \op{succ}(n) \) is the cardinality of \( A \), there exists a bijective function \( f: A \to \op{succ}(n) \). Then \( f[B] \subseteq \op{succ}(n) \). Furthermore, the inequality is strict because otherwise any member of \( A \setminus B \) would contradict \fullref{def:pigeonhole_principle}. Therefore, \( f[B] \) is a proper subset of \( \op{succ}(n) \) that is equinumerous with it. But this contradicts \fullref{thm:natural_number_is_not_equinumerous_to_proper_subset}.

  The obtained contradiction shows that \( A \) is also not Dedekind infinite.

  \ImplicationSubProof{def:set_finiteness/dedekind}{def:set_finiteness/cardinality} Let \( A \) be a Dedekind infinite set and let \( f: A \to B \) be a bijective function into some proper subset \( B \) of \( A \). We will construct an injective function from \( \omega \) to \( A \). Fix some member \( x_0 \in A \setminus B \) and recursively define
  \begin{equation*}
    \begin{aligned}
      &g: \omega \to A \\
      &g(n) \coloneqq \begin{cases}
        x_0,         &n = 0 \\
        f(g(n - 1)), &n > 0
      \end{cases}
    \end{aligned}
  \end{equation*}

  We now use induction on \( m \) to prove that \( g(n) = g(m) \) implies \( n = m \). Since \( g(0) \not\in B \) and \( g(n) \in B \) for any \( n > 0 \), the base case holds. Suppose that the inductive hypothesis holds for \( m \) and that for some \( n \) we have \( g(n) = g(m + 1) \). It is clear that \( g(0) \neq g(m + 1) \), so necessarily \( n > 0 \). We have
  \begin{equation*}
    f(g(n - 1))
    =
    g(n)
    =
    g(m + 1)
    =
    f(g(m)),
  \end{equation*}
  which by the inductive hypothesis implies that \( m = n - 1 \). Thus, \( g(m + 1) = g(n) \) and the inductive step is proved.

  Therefore, \( g \) is injective and thus \( A \) dominates \( \omega \). From \fullref{thm:cardinality_order_compatibility} it follows that \( \card(A) \geq \aleph_0 \) and thus \( \card(A) \) is not a natural number.
\end{proof}

\begin{proposition}\label{thm:cardinal_is_finite_iff_successor_ordinal}
  A nonzero cardinal is \hyperref[def:set_finiteness]{finite} if and only if it is a \hyperref[def:successor_and_limit_ordinal]{successor ordinal}.
\end{proposition}
\begin{proof}
  Finite cardinals are natural numbers by definition and all nonzero natural numbers are successor ordinals.

  Conversely, suppose that \( \kappa = \op{succ}(\alpha) \) is a successor ordinal that is a cardinal. That is, \( \kappa \) is not equinumerous with any smaller ordinal and in particular with \( \alpha \). From \fullref{thm:cardinality_order_compatibility} it follows that \( \card(\alpha) < \kappa \).

  Let \( A \subseteq \kappa \) be a proper subset of \( \kappa \). We want to show that \( \card(A) < \kappa \).

  If \( \alpha \not\in A \), define \( B \coloneqq A \). Otherwise, pick some member \( x_0 \in \kappa \setminus A \) and define
  \begin{equation*}
    B \coloneqq (A \cup \set{ x_0 }) \setminus \set{ \alpha }.
  \end{equation*}

  In both cases we have \( \card(A) = \card(B) \), but unlike \( A \), \( B \) is always a subset of \( \alpha \) since \( \kappa = \alpha \cup \set{ \alpha } \).

  Therefore,
  \begin{equation*}
    \card(A) = \card(B) \leq \card(\alpha) < \kappa.
  \end{equation*}

  Hence, \( \kappa \) dominates every proper subset, which by \fullref{def:set_finiteness/dedekind} means that \( \kappa \) is a finite cardinal.
\end{proof}

\begin{corollary}\label{thm:cardinal_is_infinite_iff_limit_ordinal}
  A cardinal is \hyperref[def:set_finiteness]{infinite} if and only if it is a \hyperref[def:successor_and_limit_ordinal]{limit ordinal}.
\end{corollary}
\begin{proof}
  This is the contraposition to \fullref{thm:cardinal_is_finite_iff_successor_ordinal} excluding the zero cardinal.
\end{proof}

\begin{proposition}\label{thm:power_set_finiteness}
  A set is finite if and only if its \hyperref[def:basic_set_operations/power_set]{power set} is finite.
\end{proposition}

\begin{proposition}\label{thm:finite_unions_and_products_are_finite}
  All finite \hyperref[def:basic_set_operations/union]{unions} and \hyperref[def:cartesian_product]{Cartesian products} of finite sets are finite.
\end{proposition}

\begin{definition}\label{def:successor_and_limit_cardinal}
  \begin{thmenum}
    \thmitem{def:successor_and_limit_cardinal/successor} If \( \kappa \) is the smallest cardinal such that \( \mu < \kappa \) for some other cardinal \( \mu \), we say that \( \kappa \) is the \term{successor} of \( \mu \) and that \( \kappa \) is itself a \term{successor cardinal}.

    The existence of \( \kappa \) is guaranteed by \fullref{thm:successor_cardinal_existence}, but it is natural to ask whether \( \kappa \) can be constructed from \( \mu \) similarly to how the \hyperref[def:ordinal_successor]{ordinal successor operator} gives us a successor ordinal. This turns out to be a deep question --- see \fullref{hyp:generalized_continuum_hypothesis}.

    \thmitem{def:successor_and_limit_cardinal/weak_limit} If \( \kappa > 0 \) is not the successor cardinal of any other cardinal, we say that it is a \term{weak limit cardinal}.

    See \fullref{thm:weak_limit_cardinal_equivalences} for some equivalent conditions.

    \thmitem{def:successor_and_limit_cardinal/strong_limit} We say that \( \kappa \) is a \term{strong limit cardinal} if \( \mu < \kappa \) implies that \( \card(\pow(\mu)) < \kappa \).

    We can benefit from using forward references to \fullref{subsec:transfinite_arithmetic}, more precisely \fullref{thm:cardinal_exponentiation_power_set}, which justifies using \hyperref[def:cardinal_arithmetic/exponentiation]{cardinal exponentiation} to rewrite the condition for \( \kappa \) being a strong limit cardinal as
    \begin{equation*}
      \mu < \kappa \T{implies} 2^\mu < \kappa.
    \end{equation*}

    Every strong limit cardinal is a weak limit cardinal as shown in \fullref{thm:strong_limit_cardinal_is_weak_limit}, however the converse is only true assuming \fullref{hyp:generalized_continuum_hypothesis} --- see \fullref{thm:limit_cardinals_and_gch}.

    Strong limit cardinals are further motivated by the usage of \hyperref[rem:strongly_inaccessible_cardinal]{regular strong limit cardinals} in \fullref{thm:strong_regular_cardinal_stages}.
  \end{thmenum}

  These notions should not be confused with \hyperref[def:successor_and_limit_ordinal]{successor and limit ordinals}.
\end{definition}

\begin{proposition}\label{thm:successor_cardinal_existence}
  For any cardinal there exists a successor cardinal.
\end{proposition}
\begin{proof}
  Fix a cardinal \( \kappa \). By \fullref{thm:hartogs_lemma}, there exists a smallest ordinal \( \alpha \) such that \( \kappa \) does not dominate \( \alpha \). Thus, \( \alpha \) is the initial ordinal of a cardinal \( \mu \) because it is not equinumerous with any smaller ordinal.

  \Fullref{thm:set_domination_relation_trichotomy} implies that \( \kappa < \mu \).

  Furthermore, every cardinal smaller than \( \mu \) does not dominate \( \kappa \), i.e. if \( \nu < \mu \), then \( \nu \geq \kappa \).

  Therefore, \( \mu \) is the successor cardinal of \( \kappa \).
\end{proof}

\begin{proposition}\label{thm:union_of_set_of_cardinals}\mcite[prop. 67.13]{OpenLogicFull}
  If \( A \) is a set of cardinals, then \( \bigcup A \) is a cardinal. Furthermore, \( \bigcup A \) is the supremum of \( A \) with respect to cardinal ordering.

  See a more thorough discussion of a similar issue in \fullref{thm:union_of_set_of_ordinals}.
\end{proposition}
\begin{proof}
  From \fullref{thm:union_of_set_of_ordinals} it follows that \( \bigcup A \) is an ordinal. Then there exists some cardinal \( \kappa \in A \) such that \( \alpha \in \kappa \).

  We have \( \kappa \subseteq \bigcup A \). Thus, with regards to ordinal ordering, \( \alpha < \kappa \leq \bigcup A \). But since \( \kappa \) is a cardinal, it is not equinumerous with \( \alpha \) and hence \( \bigcup A \) is also not equinumerous with \( \alpha \).

  Therefore, \( \bigcup A \) is a cardinal. It follows from \fullref{thm:ordinal_ordering_via_subsets} that it is also the supremum of \( A \).
\end{proof}

\begin{definition}\label{def:aleph_hierarchy}\mcite[def. 68.17]{OpenLogicFull}
  We use transfinite recursion to define, for each ordinal \( \alpha \), the cardinal
  \begin{equation}\label{eq:def:aleph_hierarchy}
    \aleph_\alpha \coloneqq \begin{cases}
      \omega,                                        &\alpha = 0 \\
      \T{successor cardinal of} \beta,               &\alpha = \op{succ}(\beta) \\
      \sup\set{ \aleph_\beta \given \beta < \alpha } &\alpha \T{is a limit ordinal}.
    \end{cases}
  \end{equation}

  We denote the initial ordinal of \( \aleph_\alpha \) by \( \omega_\alpha \). In particular, \( \omega_0 = \omega \) and \( \omega_1 \) is the first \hyperref[def:set_countability/uncountable]{uncountable ordinal}.

  Note that \( \aleph_\lambda \) exists and is a cardinal for every limit ordinal \( \lambda \) as a consequence of \fullref{thm:union_of_set_of_cardinals}.

  See \fullref{rem:unbounded_transfinite_recursion} for some technical details.

  This hierarchy is important because it describes all infinite cardinals as shown in \fullref{thm:infinite_cardinal_is_aleph}. It is intimately connected to the simpler \hyperref[def:beth_hierarchy]{\( \beth \) hierarchy} via \fullref{hyp:generalized_continuum_hypothesis}.
\end{definition}

\begin{remark}[Unbounded transfinite recursion]\label{rem:unbounded_transfinite_recursion}
  Although we cannot formally do unbounded transfinite recursion, there is an easy way to circumvent this.

  Formally, in \fullref{def:aleph_hierarchy}, for every ordinal \( \alpha \) we use \fullref{thm:bounded_transfinite_recursion} define a \( \alpha \)-indexed transfinite sequence \( \aleph_0, \aleph_1, \ldots, \aleph_\omega, \ldots \) and then use the sequence to define \( \aleph_\alpha \). The definition does not depend on any particular ordinal \( \alpha \), however, and thus all ways to obtain \( \aleph_\alpha \) are equivalent.
\end{remark}

\begin{proposition}\label{thm:aleph_hierarchy_is_strictly_monotone}
  If \( \alpha < \beta \), then \( \aleph_\alpha < \aleph_\beta \).
\end{proposition}
\begin{proof}
  We will use \fullref{rem:transfinite_induction} on \( \beta \).
  \begin{itemize}
    \item The condition \( \alpha < \beta \) is vacuously false for the base case \( \beta = 0 \), hence by \eqref{eq:def:intuitionistic_propositional_deductive_systems/rules/efq} the statement vacuously holds.

    \item Suppose that \( \alpha < \beta \) and \( \aleph_\alpha < \aleph_\beta \). We then have \( \alpha < \op{succ}(\beta) \) and, since, \( \aleph_{\op{succ}(\beta)} > \aleph_\beta \), also \( \aleph_\alpha < \aleph_{\op{succ}(\beta)} \).

    \item Let \( \lambda \) be a limit ordinal and suppose that the proposition holds for all \( \beta < \lambda \) and for arbitrary \( \alpha \). Then \( \aleph_\beta \subseteq \aleph_\lambda \) for every \( \beta < \lambda \), hence \( \aleph_\beta \leq \aleph_\lambda \) by \fullref{thm:ordinal_ordering_via_subsets}.

    Suppose that \( \alpha < \lambda \).
    \begin{itemize}
      \item If there exists some \( \beta_0 < \lambda \) such that \( \alpha < \beta_0 \), clearly \( \aleph_\alpha < \aleph_{\beta_0} \leq \aleph_\lambda \).
      \item If \( \alpha > \beta \) for all \( \beta < \lambda \), then \( \alpha \) is an upper bound of the set \( \lambda = \set{ \beta \given \beta < \lambda } \). Hence, \( \alpha \geq \lambda \), which contradicts our choice of \( \alpha \).
    \end{itemize}

    Therefore, \( \aleph_\alpha < \aleph_\lambda \).
  \end{itemize}
\end{proof}

\begin{remark}[Cardinal recursion and induction]\label{rem:cardinal_transfinite_recursion_and_induction}
  Just like we have (bounded and unbounded) transfinite recursion and induction on ordinals, we also have transfinite recursion and induction on cardinals.
  \begin{itemize}
    \item We only consider cardinals rather than arbitrary ordinals.
    \item In its structured form presented in \fullref{rem:transfinite_induction}, rather than considering successor ordinals and limit ordinals, we consider successor cardinals and weak limit cardinals.
  \end{itemize}

  Thus, recursion and induction on cardinals is formally quite different from the equivalent statements for ordinals. The usage of the two is analogous, however.

  See \fullref{thm:infinite_cardinal_is_aleph} for how this principles is used.
\end{remark}

\begin{proposition}\label{thm:infinite_cardinal_is_aleph}\mcite[prop. 68.19]{OpenLogicFull}
  For every \hyperref[def:set_finiteness]{infinite cardinal} \( \kappa \) there exists an ordinal \( \alpha \) such that \( \kappa = \aleph_\alpha \).
\end{proposition}
\begin{proof}
  We will use \fullref{rem:cardinal_transfinite_recursion_and_induction} on \( \kappa \).

  \begin{itemize}
    \item The base case \( \kappa = 0 \) vacuously holds because \( 0 \) is not an infinite cardinal. The actual base case is \( \kappa = \omega = \aleph_0 \), which holds by definition. This case may not seem formally necessary, however we need to consider it separately from the limit case and calling it the \enquote{base case} seems most appropriate.

    \item If \( \kappa = \aleph_\alpha \) and \( \mu \) is the successor cardinal of \( \kappa \), then by definition \( \kappa = \aleph_{\op{succ}(\alpha)} \).

    \item Finally, let \( \kappa \) be a limit cardinal and let \( \mu = \aleph_{\alpha_\mu} \) for every infinite cardinal \( \mu < \kappa \). Define
    \begin{equation*}
      \alpha \coloneqq \bigcup\set{ \alpha_\mu \given \mu < \kappa }.
    \end{equation*}

    We have
    \begin{align*}
      \kappa
      &\reloset {\eqref{eq:def:aleph_hierarchy}} =
      \bigcup\set{ \aleph_{\alpha_\mu} \given \mu < \kappa }
      \reloset {\eqref{eq:thm:ordinal_addition_is_monotone/right}} \leq \\ &\leq
      \bigcup\set[\Big]{ \aleph_\beta \given \beta < \sup\set{ \alpha_\mu \given \mu < \kappa } }
      = \\ &=
      \bigcup\set{ \aleph_\beta \given \beta < \alpha }
      \reloset {\ref{thm:ordinal_is_set_of_smaller_ordinals}} = \\ &=
      \aleph_\alpha.
    \end{align*}

    If we suppose that \( \kappa < \aleph_\alpha \), then similarly to \fullref{thm:ordinal_ordering_via_addition} there exists some ordinal \( \beta_0 < \alpha \) such that \( \aleph_{\beta_0} > \aleph_{\alpha_\mu} \) for every \( \mu < \kappa \). In particular, \fullref{thm:aleph_hierarchy_is_strictly_monotone} implies that \( \beta_0 > \alpha_\mu \) for every \( \mu < \kappa \). Thus,
    \begin{equation*}
      \underbrace{\bigcup\set{ \alpha_\mu \given \mu < \kappa }}_{\alpha} \leq \beta_0 < \alpha,
    \end{equation*}
    which is a contradiction. Therefore, \( \kappa = \aleph_\alpha \).
  \end{itemize}
\end{proof}

\begin{corollary}\label{thm:weak_limit_cardinal_equivalences}.
  The cardinal \( \kappa = \aleph_\alpha \) is a weak limit cardinal if and only if \( \alpha \) is a limit ordinal.

  In particular, \( \kappa > 0 \) is a weak limit cardinal if and only if \( \mu < \kappa \) implies that \( \nu < \kappa \), where \( \nu \) is the successor cardinal of \( \mu \).
\end{corollary}
\begin{proof}
  Clear from \fullref{def:aleph_hierarchy}.
\end{proof}

\begin{definition}\label{def:set_countability}
  We will introduce the notion of \term{countability}, which generalizes \hyperref[def:set_finiteness]{finiteness}.

  \begin{thmenum}
    \thmitem{def:set_countability/countably_infinite}\mcite[159]{Enderton1977Sets} The smallest infinite cardinal is \hyperref[thm:omega_is_a_cardinal]{\( \aleph_0 \)}. Every set with cardinality \( \aleph_0 \) is called \term{countably infinite}. The countably infinite sets are precisely those that can be ordered into a \hyperref[def:sequence]{sequence}.

    \thmitem{def:set_countability/at_most_countable} A set that is either finite or countably infinite is called \term{at most countable}.

    \thmitem{def:set_countability/uncountable} Any set that \hyperref[def:equinumerosity]{strictly dominates} \( \aleph_0 \) is called \term{uncountable}. The smallest uncountable cardinal is the successor cardinal \( \aleph_1 \) of \( \aleph_0 \).

    \thmitem{def:set_countability/continuum} The cardinality of \( \pow(\aleph_0) \) has a special name --- the \term{cardinality of the continuum}. It is sometimes denoted by \( c \). See \fullref{hyp:continuum_hypothesis} for its relation to \( \aleph_1 \).
  \end{thmenum}

  See \fullref{rem:countability_etymology} for additional terminology that is potentially more ambiguous.
\end{definition}

\begin{remark}\label{rem:countability_etymology}
  Some authors, for example \cite[159]{Enderton1977Sets}, use the shorted term \term{countable}, however other authors use \enquote{countable} to mean \enquote{countably infinite}. The terms \term{denumerable} and \term{enumerable} are also used for \enquote{countably infinite} and \enquote{at most countable} respectively. This is done in \cite[def. 4.4]{OpenLogicFull}, for example. These terms are also ambiguous unfortunately.
\end{remark}

\begin{conjecture}[Continuum hypothesis]\label{hyp:continuum_hypothesis}
  The \hyperref[def:set_countability/continuum]{cardinality of the continuum} \( c \) is the \hyperref[def:set_countability/uncountable]{first uncountable cardinal} \( \aleph_1 \).

  Compare this to \fullref{hyp:generalized_continuum_hypothesis}.
\end{conjecture}

\begin{remark}\label{rem:continuum_hypothesis}\mcite[165]{Enderton1977Sets}
  \Fullref{hyp:continuum_hypothesis} has been shown by G\"odel not to be disprovable in \hyperref[def:set]{\logic{ZFC}} and by Cohen not to be provable in \logic{ZFC}.
\end{remark}

\begin{proposition}\label{thm:omega_equinumerous_with_omega_squared}
  The smallest inductive set \( \omega \) is equinumerous with \( \omega \times \omega \).
\end{proposition}
\begin{proof}
  We can give a short proof using \fullref{thm:cantor_schroder_bernstein_theorem} using the injective functions \( (n, m) \mapsto 2^n 3^m \) in one direction and \( n \mapsto (n, 0) \) in the other direction. Proving the injectivity of \( f \), however, requires \fullref{thm:fundamental_theorem_of_arithmetic}, and we prove the latter using machinery from \fullref{sec:commutative_algebra}. We will instead give a direct proof with an explicit construction. We will construct a bijective function from \( \omega \times \omega \) to \( \omega \) --- the function visualized in \cref{fig:thm:omega_equinumerous_with_omega_squared}.

  We begin by defining the diagonal in \cref{fig:thm:omega_equinumerous_with_omega_squared}. For each natural number \( k \), define the set of pairs that sum to \( k \):
  \begin{equation*}
    A_k \coloneqq \set{ (n, m) \in \omega \times \omega \given n + m = k }.
  \end{equation*}

  We can use induction to show that \( \card(A_k) = k + 1 \). That is, \( A_k \) can \enquote{fit} \( k + 1 \) numbers. We can now define the function
  \begin{equation*}
    \begin{aligned}
      &d: \omega \to \omega \\
      &d(n) \coloneqq \sum_{k=0}^n \card(A_n)
    \end{aligned}
  \end{equation*}
  that gives us how many numbers we have already \enquote{fit} in the first \( n \) diagonals.

  It is clear that the point \( (n, m) \) lies in \( A_{n + m} \). We want to know how many numbers we have \enquote{fit} in the diagonal prior to that, for which we can use \( d(n + m - 1) \). This leads us to the definition
  \begin{equation*}
    \begin{aligned}
      &f: \omega \times \omega \to \omega \\
      &f(n, m) \coloneqq \begin{cases}
        0,                &n + m = 0 \\
        d(n + m - 1) + n, &n + m > 0. \\
      \end{cases}
    \end{aligned}
  \end{equation*}

  We will first show that \( f \) is injective using induction on \( n + m \) (that is, on the diagonals). Suppose that \( f(n_1, m_1) = f(n_2, m_2) \) implies \( n_1 = n_2 \) and \( m_1 = m_2 \) for all pairs with a sum less than \( l \). Let \( (n_1, m_1) \) and \( (n_2, m_2) \) be two points in \( A_l \) such that \( f(n_1, m_1) = f(n_2, m_2) \). The cases \( l <= 1 \) are trivial, so suppose that \( l > 1 \). Then
  \begin{equation*}
    f(n_1, m_1)
    =
    d(n_1 + m_1 - 1) + n_1
    =
    l - 1 + d(n_1 + m_1 - 2) + n_1
    =
    l + f(n_1 - 1, m_1).
  \end{equation*}

  We can now apply the inductive hypothesis and obtain that \( n_1 = n_2 \) and \( m_1 = m_2 \). This proves injectivity.

  To see that \( f \) is surjective, we will use induction on \( k \in \omega \). The base case is again trivial. Now suppose that \( n + m > 0 \) and
  \begin{equation*}
    f(n, m) = d(n + m - 1) + n = k.
  \end{equation*}

  We have two cases:
  \begin{itemize}
    \item If \( m = 0 \), then
    \begin{equation*}
      f(0, n + m + 1) = d(n + m) + 0 = d(n + m - 1) + n + m = f(n, m) + 1.
    \end{equation*}

    \item If \( m > 0 \), then
    \begin{equation*}
      f(n + 1, m - 1) = d(n + m - 1) + n + 1 = f(n, m) + 1.
    \end{equation*}
  \end{itemize}

  In both cases we have shown that \( k + 1 = f(n, m) + 1 \) is in the image of \( f \), which concludes the proof of surjectivity (and hence bijectivity).

  Lastly, although it is not necessary for the proof, we can expand the definition of \( d \) to see that \( f \) is actually a \hyperref[def:polynomial]{polynomial}:
  \begin{equation*}
    f(n, m)
    =
    \sum_{k=0}^{n + m - 1} (k + 1) + n
    =
    \sum_{k=1}^{n + m} k + n
    \reloset {\eqref{eq:thm:arithmetic_progression_partial_sums}} =
    \frac {(n + m) (n + m + 1)} 2 + n.
  \end{equation*}

  As an added benefit, this polynomial also handles the case \( n = m = 0 \).

  \begin{figure}
    \hfill
    \includegraphics[page=1]{output/thm__omega_equinumerous_with_omega_squared.pdf}
    \hfill
    \includegraphics[page=2]{output/thm__omega_equinumerous_with_omega_squared.pdf}
    \hfill\hfill
    \caption{Visualization on an integer coordinate grid of the diagonal sets \( A_k \) and of the bijective function defined in \fullref{thm:omega_equinumerous_with_omega_squared}.}\label{fig:thm:omega_equinumerous_with_omega_squared}
  \end{figure}
\end{proof}

\begin{corollary}\label{thm:countable_product_of_countable_sets}
  A finite \hyperref[def:cartesian_product]{Cartesian product} of at most countable sets is at most countable.
\end{corollary}
\begin{proof}
  Let \( A_1, \ldots, A_n \) be a finite family of at most countable sets.

  Suppose that the inductive hypothesis holds for \( n \). Countability ensures that there exist injective functions \( g: A_1 \times \cdots \times A_n \times \omega \) and \( h: A_{n+1} \to \omega \). Denote by \( f \) the bijective function from \( \omega \) to \( \omega \times \omega \) obtained in \fullref{thm:omega_equinumerous_with_omega_squared} and define
  \begin{equation*}
    \begin{aligned}
      &F: A_1 \times \cdots \times A_n \times A_{n+1} \to \omega \\
      &F(a_1, \ldots, a_n, a_{n+1}) \coloneqq f(g(a_1, \ldots, a_n), h(a_{n+1}))
    \end{aligned}
  \end{equation*}

  By \fullref{thm:function_superposition_invertibility}, the function \( F \) is injective as a \hyperref[def:multi_valued_function/superposition]{superposition} of injective functions.

  Therefore, \( \omega \) dominates the product \( A_1 \times \cdots \times A_n \), i.e. the product is countable.
\end{proof}

\begin{proposition}\label{thm:countably_infinite_union_of_countably_infinite_sets}\mcite[thm. 6Q]{Enderton1977Sets}
  A \hyperref[def:set_countability/countably_infinite]{countably infinite} union of countably infinite sets is countably infinite.
\end{proposition}
\begin{proof}
  Let \( \seq{ A_k }_{k \in \omega} \) be a countably infinite family of countably infinite sets. Define instead the disjoint family
  \begin{equation*}
    B_k \coloneqq \set{ (k, a) \given a \in A_k }.
  \end{equation*}

  Denote the union of the former family by \( A \) and of the latter family by \( B \). Since each \( A_k \) is countably infinite, so is \( A \). Furthermore, there exists an obvious injective function from \( B \) to \( A \), thus
  \begin{equation}\label{thm:countably_infinite_union_of_countably_infinite_sets/union_card_inequality}
    \aleph_0 \leq \card(A) \leq \card(B).
  \end{equation}

  Define the multi-valued mapping
  \begin{equation*}
    \begin{aligned}
      &\mscrG: \omega \to \fun(\omega, B) \\
      &\mscrG(k) \coloneqq \set{ g: \omega \to B_k \given g \T{is bijective} }.
    \end{aligned}
  \end{equation*}

  This is a total multi-valued function because we have assumed that \( B_k \) is countable for every \( k \in \mscrK \). \Fullref{thm:existence_of_multi_valued_function_selection} gives us a single-valued function \( G: \omega \to \fun(\omega, B) \). Since the family \( \seq{ B_k }_{k \in \omega} \) is disjoint, \( G \) is injective. We can thus \hyperref[def:function/currying]{uncurry} \( G \) to obtain a function \( g \) from \( \omega \times \omega \) to \( B \).

  To prove that \( g \) is injective, suppose that \( g(n_1, m_1) = g(n_2, m_2) \). Then
  \begin{equation*}
    G(n_1)(m_1) = g(n_1, m_1) = g(n_2, m_2) = G(n_2)(m_2).
  \end{equation*}

  Note that \( n_1 \neq n_2 \) would lead to a contradiction because  \( \seq{ B_k }_{k \in \omega} \) is a disjoint family. So \( n_1 = n_2 \) and we obtain \( m_1 = m_2 \) since the function \( G(n_1): \omega \to B_k \) is injective. Therefore, \( g \) itself is also injective.

  It is also surjective because for every \( a \in B \) there exists some \( k \in \omega \) such that
  \begin{equation*}
    a \in B_k = \img(G(k)).
  \end{equation*}

  Denote by \( f \) the bijective function from \( \omega \) to \( \omega \times \omega \) obtained in \fullref{thm:omega_equinumerous_with_omega_squared}. Then the function \( f \bincirc g: \omega \to B \) is bijective by \fullref{thm:function_composition_invertibility}. Therefore, the union \( B \) is countably infinite. From \eqref{thm:countably_infinite_union_of_countably_infinite_sets/union_card_inequality} it follows that \( A \) is also countably infinite.
\end{proof}

\begin{corollary}\label{thm:at_most_countable_union_of_at_most_countable_sets}
  An \hyperref[def:set_countability/at_most_countable]{at most countable} union of at most countable sets is at most countable.
\end{corollary}
\begin{proof}
  Let \( \seq{ A_k }_{k \in \mscrK} \) be an at most countable family of at most countable sets. Denote their union by \( A \). For every \( A_k \) let \( g_k: A_k \to \omega \) be an injective function and define the \hyperref[def:disjoint_union]{disjoint union}
  \begin{equation*}
    B_k \coloneqq A_k \amalg \set{ n \in \omega \given n \not\in \img(g_k) }
  \end{equation*}
  and the bijective function
  \begin{equation*}
    \begin{aligned}
      &h_k: B_k \to \omega \\
      &h_k(x) \coloneqq \begin{cases}
        (0, g_k(x)), &x \in A_k \\
        (1, x),      &\T{otherwise.}
      \end{cases}
    \end{aligned}
  \end{equation*}

  For every \( k \in \omega \setminus \mscrK \) instead define
  \begin{equation*}
    B_k \coloneqq \set{ (k, n) \given n \in \omega }
  \end{equation*}
  and let \( h_k: B_k \to \omega \) be the obvious bijective function.

  Then
  \begin{equation*}
    A
    =
    \bigcup_{k \in \mscrK} A_k
    \subseteq
    \bigcup_{k \in \mscrK} B_k
    \subseteq
    \bigcup_{k \in \omega} B_k
  \end{equation*}
  and the latter is countably infinite by \fullref{thm:countably_infinite_union_of_countably_infinite_sets}. Therefore, \( A \) is at most countable.
\end{proof}


\title{Probabilistic Models}

\subsection{Probabilistic Models}

A probabilistic model asserts how observations from a natural phenomenon arise.
The model is a \emph{joint distribution}
\begin{align*}
  p(\mathbf{x}, \mathbf{z})
\end{align*}
of observed variables $\mathbf{x}$ corresponding to data, and latent
variables $\mathbf{z}$ that provide the hidden structure to generate
from $\mathbf{x}$. The joint distribution factorizes into two
components.

The \emph{likelihood}
\begin{align*}
  p(\mathbf{x} \mid \mathbf{z})
\end{align*}
is a probability distribution that describes how any data $\mathbf{x}$
depend on the latent variables $\mathbf{z}$. The likelihood posits a
data generating process, where the data $\mathbf{x}$ are assumed drawn
from the likelihood conditioned on a particular hidden pattern
described by $\mathbf{z}$.

The \emph{prior}
\begin{align*}
  p(\mathbf{z})
\end{align*}
is a probability distribution that describes the latent variables
present in the data. It posits a generating process of the hidden structure.

For details on how to specify a model in Edward, see the
\href{/api/model}{model API}. We describe several examples in detail
in the \href{/tutorials/}{tutorials}.


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}
\usepackage{booktabs}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\logit}{logit}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\cov}{cov}
\DeclareMathOperator*{\expec}{E}
\DeclareMathOperator*{\deriv}{d}
\DeclareMathOperator*{\const}{constant}

\begin{document}
\title{\textsf{Homework 9 for Bayesian Data Analysis}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 14.3}
{
    Assuming uniform prior distribution for $\beta|\sigma$, we have
    $$p(\beta|\sigma, y) \propto p(y | \beta, \sigma) p(\beta | \sigma)$$
    $$\propto p(y | \beta, \sigma) \propto \exp{\left( -\frac{1}{2\sigma^2} (y-X\beta)^T (y-X\beta) \right)}.$$

    Note the fact that $$(y-X\beta)^T (y-X\beta) = (\beta-\hat{\beta})^T X^T X (\beta-\hat{\beta}) + \const,$$ we have
    $$p(\beta|\sigma, y) \propto \exp{\left( -\frac{1}{2\sigma^2} (y-X\beta)^T (y-X\beta) \right)}$$
    $$\propto \exp{\left( -\frac{1}{2\sigma^2} (\beta-\hat{\beta})^T X^T X (\beta-\hat{\beta}) \right)}$$
    $$= \exp{\left( -\frac{1}{2} (\beta-\hat{\beta})^T ((X^T X)^{-1} \sigma^2)^{-1} (\beta-\hat{\beta}) \right)},$$
    which implies that $$\beta | \sigma, y \sim N(\hat{\beta}, (X^T X)^{-1} \sigma^2).$$
}

\section*{Question 14.4}
{
    Assuming the noninformative prior $p(\beta, \log{\sigma}) \propto 1$, or $p(\beta, \sigma^2) \propto \sigma^{-2}$, we obtain
    $$p(\sigma | y) = p(\beta, \sigma^2 | y) / p(\beta | \sigma^2, y) \propto p(\beta, \sigma^2) p(y | \beta, \sigma^2) / p(\beta | \sigma^2, y)$$
    $$\propto \frac{\sigma^{-2} \cdot \sigma^{-n} \exp{\left( -\frac{1}{2\sigma^2} (y-X\beta)^T (y-X\beta) \right)}}{[\det{((X^T X)^{-1} \sigma^2)]^{-1/2} \cdot \exp{\left( -\frac{1}{2\sigma^2} (\beta-\hat{\beta})^T X^T X (\beta-\hat{\beta}) \right)}}}$$

    Fix $\beta = \hat{\beta}$ in the formula above, and it follows that
    $$p(\sigma | y) \propto \frac{\sigma^{-2} \cdot \sigma^{-n} \exp{\left( -\frac{1}{2\sigma^2} (y-X\hat{\beta})^T (y-X\hat{\beta}) \right)}}{[\det{((X^T X)^{-1} \sigma^2)]^{-1/2}}}$$
    $$= \frac{\sigma^{-2} \cdot \sigma^{-n} \exp{\left( -\frac{1}{2\sigma^2} (y-X\hat{\beta})^T (y-X\hat{\beta}) \right)}}{[\sigma^{2k} \cdot \det{((X^T X)^{-1})]^{-1/2}}}$$
    $$\propto \sigma^{-2-n+k} \cdot \exp{\left( -\frac{1}{2\sigma^2} (y-X\hat{\beta})^T (y-X\hat{\beta}) \right)}.$$

    Compare this expression to the Inverse-$\chi^2$ distribution\footnote{https://en.wikipedia.org/wiki/Inverse-chi-squared\_distribution}, we find that
    $$s^2 = \frac{1}{n-k} (y-X\hat{\beta})^T (y-X\hat{\beta}).$$
}

\section*{Question 14.7}
{
    $\widetilde{y}$ conforms a normal distribution, as $\widetilde{y}$ is a linear combination of $\beta$, and $p(\beta | \sigma, y)$ is normal.
}

\section*{Longley data 1}
{
    The mean of Inverse-$\mathrm{\chi^2}(n-k, s^2)$ is\footnote{https://en.wikipedia.org/wiki/Scaled\_inverse\_chi-squared\_distribution}
    $$\frac{(n-k)s^2}{n-k-2}.$$
    The calculated result is
    \begin{lstlisting}
        [1] 6.003243
    \end{lstlisting}
}

\section*{Longley data 2}
{
    The posterior mean of $\beta$ under a conjugate prior $(\beta_0, \beta_1) \sim N(0, I)$ is
    \begin{lstlisting}
    (Intercept)           X
     -76.759663    1.519031
    \end{lstlisting}
}

\section*{Longley data 3}
{
    \begin{lstlisting}
    > mod.full = BayesReg(dat$GNP.deflator, dat[, -1], g=nrow(dat))

    PostMean PostStError Log10bf EvidAgaH0
    Intercept 101.6813      0.7431
    x1         23.8697     25.1230 -0.3966
    x2          3.1068      6.6053 -0.5603
    x3          0.7078      2.5134 -0.5954
    x4        -11.0111     10.9543 -0.3714
    x5         -6.1556     32.7640 -0.6064
    x6          0.7402     10.7025  -0.614

    Posterior Mean of Sigma2: 8.8342
    Posterior StError of Sigma2: 13.0037

    > mod = BayesReg(dat$GNP.deflator, dat[, c(-1, -3)], g=nrow(dat))

    PostMean PostStError Log10bf EvidAgaH0
    Intercept 101.6813      0.7494
    x1         14.6884     15.9493 -0.4094
    x2         -0.3300      1.2139 -0.5968
    x3         -9.9863     10.8264 -0.4087
    x4          8.6301      9.3120 -0.4068
    x5         -3.5398      5.6814 -0.5194

    Posterior Mean of Sigma2: 8.9846
    Posterior StError of Sigma2: 13.2249

    > bf.full / bf
    Bayes factor analysis
    --------------
    [1] GNP + Unemployed + Armed.Forces + Population + Year + Employed : 0.1443632 ±0%

    Against denominator:
    GNP.deflator ~ GNP + Armed.Forces + Population + Year + Employed
    ---
    Bayes factor type: BFlinearModel, JZS
    \end{lstlisting}

    We find that the reduced model has similar coefficients to the full model. And the ratio of Bayes factor is much less than 1, which means the reduced model is more likely.
}

\clearpage
\end{document}


\section{Mathematical functions and Python functions}\label{sec:functions}
A mathematical function is a rule that associates a single output value with each input value in a set \cite{rudin}. Examples are:
\begin{eqnarray*}
    f(x) = x^2 & \textup{for each value $x$ in the real numbers}  \\
    f(x) = \sin(x)& \textup{for each value $x$ in the real numbers}\\
    f(x,y) = x^2 - (y-1)^2 & \textup{for each pair of values $x,y$, each in the real numbers.}
\end{eqnarray*}
Note the important distinction between the actual function $f$ and the evaluation $f(x)$ of the function at a particular point. Each evaluation of a function is just a single output value, but a function itself is an (often infinite) table of values. If you `look up' an input value in the table by evaluating the function, you'll find the corresponding output value.

One convenient way to visualize a function is by graphing it. The graphs of the first two functions above are curves. Their input and output values are single numbers, so the functions themselves are infinite tables of ordered pairs. The graph of the third function, on the other hand, is a surface. Its input value is a two-vector and its output value is a number, so the function itself is an infinite table of ordered triples. Ordered triples have to be plotted in three-dimensional space, so the graph ends up being a surface.

Python functions can emulate mathematical functions. Python representations of the mathematical functions above are:
\begin{verbatim}
def f(x):
    return x ** 2

def f(x):
    return sin(x)

def f(x,y):
    return x ** 2 - (y-1) ** 2
\end{verbatim}
These Python functions act like very large tables of values. If you `look up' an input value by passing it in as an argument, the function will tell you the corresponding output value. It would be very inefficient to store output values corresponding to each input value on a computer, so the functions have to figure out the output values corresponding to arbitrary input values on demand.

\section{A first look at Gaussian processes}\label{sec:firstlook}

Gaussian processes are probability distributions for mathematical functions. The statement `random function $f$ has a Gaussian process distribution with mean $M$ and covariance $C$' is usually written as follows:
\begin{equation}
    f\sim\textup{GP}(M,C).
\end{equation}
Gaussian processes have two parameters, which are analogous to the parameters of the normal distribution:
\begin{itemize}
    \item $M$ is the mean function. Like the mean parameter of the normal distribution, $M$ gives the central tendency for $f$. In Bayesian statistics, $M$ is usually considered a prior guess for $f$.
    \item $C$ is the covariance function. $C$ takes twice as many arguments as $f$; if $f$ is a function of one variable, $C$ is a function of two variables. $C(x,y)$ gives the covariance of $f(x)$ and $f(y)$, and $C(x,x)$ gives the variance of $f(x)$. Its role is harder to understand than that of the mean function, but among other things it regulates:
    \begin{itemize}
        \item the amount by which $f$ may deviate from $M$ at any input value $x$,
        \item the roughnesss of $f$,
        \item the typical lengthscale of changes in $f$.
    \end{itemize}
\end{itemize}
Section \ref{sec:cov} will look at covariance functions in more depth; for the time being don't worry about them too much.

As with any probability distribution, random values can be drawn from a Gaussian process. However, these values (called `realizations') are actually mathematical functions rather than the usual numbers or vectors. On the computer, the random values we draw will essentially be Python functions, with a few extra features.

\subsection{What are Gaussian processes good for?}\label{sub:applications}
Mathematical functions are ubiquitous in science. A very short list of examples:
\begin{itemize}
    \item Functional responses in predator-prey dynamics \cite{mathecol}. These functions associate a value for rate of prey capture with each value of predator population size.
    \item Transmission functions in epidemiology \cite{andersonmay}. These functions associate a value for rate of new infections with each value of infected and uninfected population size.
    \item Transfer functions in engineering \cite{duffy}. These functions associate a value for a ratio of Laplace transformed input and output signals with each value of the Laplace input variable $s$.
    \item Utility functions in microeconomics \cite{microecon}. These functions associate a value for a person's satisfaction with each portfolio of goods.
    \item Action potentials in neuroscience \cite{neuro}. These functions associate a value for transmembrane potential with each value of time since depolarization.
    \item The annual mean temperature at each point on the earth's surface in earth and atmospheric sciences.
\end{itemize}

The problem of estimating functions is equally widespread, because researchers frequently want to find out what each of the above functions is. In some cases, the phenomena underlying a function are simple and well-understood enough that the function can be derived up to a handful of parameters. For example, in Newtonian mechanics the height of a rock is known to be a parabolic function of the time since it was thrown, and the problem of inferring its trajectory reduces to the problem of inferring its initial height, its initial velocity and the acceleration due to gravity.

In many other cases it's not possible to deduce nearly as much about a function a priori. In some cases several candidate forms exist, but it's not possible to rule all of them out or even to ascertain that they are the only possibilities. As flexible and convenient probability distributions on function spaces, Gaussian processes are useful for Bayesian inference of functions without the need for reducing the problem to inference of a set of parameters.

\subsection{Creating a Gaussian process}\label{sub:inst}

In the following subsections we will create objects representing a covariance function, a mean function, and finally several random functions drawn from the Gaussian process distribution defined by those objects.

\subsubsection{Creating a mean function}\label{subsub:mean}

\begin{figure}
    \centering
        \epsfig{file=figs/mean.pdf,width=10cm}
    \caption{The mean function generated by {\sffamily `examples/mean.py'}.}
    \label{fig:mean}
\end{figure}

The first component we will create is a mean function, represented by class \class{Mean}. The mean function of a univariate GP can be interpreted as a prior guess for the GP, so it's a univariate function also. The \class{Mean} class is a wrapper for an ordinary Python function. We will use the parabolic function
\begin{equation}
    M(x) = ax^2 + bx + c.
\end{equation}

The following code will produce an instance of class \class{Mean} called M:
\verbatiminput{../../examples/gp/mean.py}

The first argument to \class{Mean}'s init method is the underlying Python function, in this case \function{quadfun}. The extra arguments \code{a}, \code{b}  and \code{c} will be memorized and passed to \function{quadfun} whenever M is called; the call \texttt{M(x)} in the plotting portion of the script doesn't need to pass them in.

Mean functions broadcast over their arguments in the same way as \citetitle[http://docs.scipy.org/doc/numpy/reference/ufuncs.html]{NumPy universal functions} \cite{numpybook}, which means that \texttt{M(x)} will return the vector
\begin{eqnarray*}
    \texttt{[M(x[0]),\ldots, M(x[N-1])]}.
\end{eqnarray*}

The last part of the code plots \texttt{M(x)} on $-1<\texttt{x}<1$, and its output is shown in figure \ref{fig:mean}. As expected, the plot is a parabola.

\subsubsection{Creating a covariance function}\label{subsub:cov}
\begin{figure}
    \centering
        \epsfig{file=figs/cov.pdf,width=15cm}
    \caption{The covariance function generated by {\sffamily `examples/cov.py'}. On the left is the covariance function $C(x,y)$ evaluated over a square: $-1\le x\le 1,\ -1\le y\le 1$. On the right is a slice of the covariance: $C(x,0)$ for $0\le x \le 1$}
    \label{fig:cov}
\end{figure}

GP covariance functions are represented by the class \class{Covariance}, which like \class{Mean} is essentially a wrapper for ordinary Python functions. In this example we will use the popular Mat\`ern covariance function \cite{banerjee}, which is provided in module \module{cov_funs}. In addition to the two arguments \texttt{x} and \texttt{y}, this function takes three tunable parameters: \code{amp} controls the amount by which realizations may deviate from their mean, \code{diff_degree} controls the roughness of realizations (the degree of differentiability), and \code{scale} controls the lengthscale over which realizations change.

You're free to write your own functions to wrap in \class{Covariance} objects. See section \ref{cha:usercov} for more information.

The code in \file{examples/cov.py} will produce an instance of class \class{Covariance} called C.
\verbatiminput{../../examples/gp/cov.py}

The first argument to \class{Covariance}'s init method, \function{eval_fun}, gives the Python function from which the covariance function will be made. In this case, \function{eval_fun} is \function{matern.euclidean}. The extra arguments \code{diff_degree, amp} and \code{scale} will be passed to \function{matern.euclidean} every time C is called.

At this stage, the covariance function C exposes a very simple user interface. In fact, it behaves a lot like the ordinary Python function \texttt{matern.euclidean} that it wraps, except that like \texttt{Mean} it `memorizes' the parameters \code{diff_degree, amp} and \code{scale} so that you don't need to pass them in when you call it. Covariance functions' calling conventions are slightly different than ordinary numpy universal functions' \cite{numpybook}:
\begin{enumerate}
    \item Broadcasting works differently. If C were a numpy universal function, \texttt{C(x,y)} would return the following array:
    \begin{eqnarray*}
        \begin{array}{ccc}
            \texttt{[C(x[0],y[0])}& \ldots& \texttt{C(x[N-1],y[N-1])]},
        \end{array}
    \end{eqnarray*}
    where \texttt{x} and \texttt{y} would need to be vectors of the same length. In fact \texttt{C(x,y)} returns a matrix:
    \begin{eqnarray*}
        \left[\begin{array}{ccc}
            \texttt{C(x[0],y[0])}& \ldots& \texttt{C(x[0],y[Ny-1])}\\
            \vdots&\ddots&\vdots\\
            \texttt{C(x[Nx-1],y[0])}& \ldots& \texttt{C(x[Nx-1],y[Ny-1])}
        \end{array}\right],
    \end{eqnarray*}
    and input arguments \texttt{x} and \texttt{y} don't need to be the same length.
    \item You can call covariance functions with just one argument. \texttt{C(x)} returns
    \begin{eqnarray*}
         \texttt{[C(x[0],x[0])}& \ldots& \texttt{C(x[Nx-1],x[Nx-1])]} = \texttt{diag(C(x,x))},
    \end{eqnarray*}
    but is computed much faster than \texttt{diag(C(x,x))} would be.
\end{enumerate}

Most of the code in \file{examples/cov.py} is devoted to output, which is shown in figure \ref{fig:cov}. It plots the covariance function \texttt{C(x,x)} evaluated over a square, and also the `slice' \texttt{C(x,0)} over an interval. You'll notice that the graph of the full covariance function resembles a rounded A-frame tent.

\subsubsection{Drawing realizations}\label{subsub:realizations}
\begin{figure}
    \centering
        \epsfig{file=figs/realizations.pdf,width=10cm}
    \caption{Three realizations from a Gaussian process displayed with mean $\pm$ 1 sd envelope. Generated by {\sffamily `examples/realizations.py'}.}
    \label{fig:realizations}
\end{figure}

Finally, let's generate some realizations (draws) from the Gaussian process defined by M and C and take a look at them. The following code will generate a list of \class{Realization} objects:
\verbatiminput{../../examples/gp/realizations.py}

    The init method of \class{Realization} takes only two required arguments, a \class{Mean} object and a \class{Covariance} object. Each element of \code{f_list} is a Gaussian process realization, which is essentially a randomly-generated Python function. Like \class{Mean} objects, \class{Realization} objects use the same broadcasting rules as numpy universal functions. Typing \texttt{f(x)} will return the vector
\begin{eqnarray*}
    [\texttt{f(x[0])}\ldots \texttt{f(x[N-1])}].
\end{eqnarray*}

The plotting portion of the code calls the function \function{plot_envelope}, which summarizes some aspects of the distribution. The dashdot black line in the middle is $M$, and the gray band is the $\pm 1$ standard deviation envelope for $f$, generated by $C$. Each of the three realizations in \texttt{f_list} is a callable function, and they are plotted superimposed on the envelope. The plot output is shown in figure \ref{fig:realizations}.

\section{The role of the covariance function}\label{sec:cov}
The following covariance functions for Euclidean coordinates are included in the module \module{cov_funs}:
\begin{itemize}
    \item \texttt{matern.euclidean}
    \item \texttt{sphere.euclidean}
    \item \texttt{pow_exp.euclidean}
    \item \texttt{gaussian.euclidean}
    \item \texttt{quadratic.euclidean}
\end{itemize}
See section 2.1.3 of \citetitle[http://www.statsnetbase.com/ejournals/books/book_summary/summary.asp?id=1285]{Banerjee et al.} \cite{banerjee} for more information on each of these. Each covariance function takes at least two parameters, called \texttt{amp} and \texttt{scale}. The following covariance functions take extra parameters:
\begin{description}
    \item[\texttt{matern}:] \texttt{diff_degree}
    \item[\texttt{pow_exp}:] \texttt{pow}
    \item[\texttt{quadratic}:] \texttt{phi}.
\end{description}

In this section we'll focus on the Mat\`ern family, as it is generally considered the state of the art. Its popularity is due to the fact that it has three parameters, each of which clearly controls one of three important properties of realizations: roughness, lengthscale of changes and amplitude.

In this section we'll set the mean function to zero (more precisely, a function whose output value is zero regardless of input) in order to focus on the covariance. This:
\begin{eqnarray*}
    f\sim\textup{GP}(M,C)
\end{eqnarray*}
is equivalent to this:
\begin{eqnarray*}
    f = M + g, \\g\sim \textup{GP}(0,C),
\end{eqnarray*}
so it's not difficult to adapt the intuition we gain in this section to GPs with nontrivial mean functions.

The covariance functions listed above are \emph{stationary} and \emph{isotropic}. Intuitively, that means our a priori expectation of how $f$ will deviate from its mean doesn't vary with location or with the direction in which we look (for functions of several variables). Section \ref{cha:usercov} describes how these restrictions can be relaxed.

All the figures in this section were produced using the file \file{examples/cov_params.py}. You can follow along by editing the line of that file which reads
\begin{verbatim}
C = Covariance(eval_fun = matern.euclidean, diff_degree = 1.4, amp = 1., scale = 1.)
\end{verbatim}

The actual formulas for the covariance functions are given in section \ref{cha:usercov}, but the Mat\`ern formula is fairly inscrutable (though its Fourier transform is much more readable \cite{stein}). This section will try to help you understand it graphically.

\subsection*{The \texttt{amp} and \texttt{scale} parameters}\label{sub:ampscale}

These parameters are common to all covariance functions provided by this package, not just \texttt{matern.euclidean}. Please see section \ref{cha:usercov} if you're planning on writing your own covariance functions, as this package provides utilities that will endow them with these parameters.

As mentioned in section \ref{subsub:cov}, the covariance function plotted in figure \ref{fig:cov} resembles a rounded A-frame tent. The width of this tent controls how tightly nearby evaluations of a realization $f$ will be coupled to each other. If the tent is wide, $f(x)$ and $f(y)$ will tend to have similar values when $x$ and $y$ are close to one another. If the tent is narrower, $f(x)$ and $f(y)$ won't be as tightly correlated. The height of the tent controls the overall amplitude of $f$'s deviation from its mean.

The mathematical definition of the covariance function is as follows:
\begin{equation}
    \label{covdef}
    C(x,y)=\textup{cov}(f(x), f(y)).
\end{equation}
That implies the following:
\begin{eqnarray*}
    \textup{var}(f(x))=C(x,x).
\end{eqnarray*}
By the definition of variance, for any real number $a$
\begin{eqnarray*}
    \textup{var}( a f(x))=a^2 C(x,x).
\end{eqnarray*}

The covariance function is multiplied by \texttt{amp}$^\texttt{2}$, and this effectively multiplies realizations by \texttt{amp}. In other words, a larger \texttt{amp} parameter means that realizations will deviate further from their mean. The effects of changing the \texttt{amp} parameter are illustrated in figure \ref{fig:amp}.
\begin{figure}
    \centering
        \epsfig{file=figs/d14a1s1.pdf, width=8cm}
        \epsfig{file=figs/d14a4s1.pdf, width=8cm}
    \caption{Mat\`ern covariances with \texttt{diff_degree=1.4}, \texttt{scale=1}, and \texttt{amp=1} (left) and 4 (right), and corresponding realizations. The amplitude of realizations tends to scale with \texttt{amp}, and the amplitude of the covariance function scales with \texttt{amp}$^2$. Note the scales on the plots of the covariance functions.}
    \label{fig:amp}
\end{figure}

Say $C$'s value can be computed from the difference in its arguments, $x-y$, rather than the arguments themselves. In that case, substituting $(x-y)/s$ for $x-y$, where $s$ is greater than 1, will make the input points `appear' closer together.

Whenever a call $C(x,y)$ is made, the distances between elements of the input arrays $x$ and $y$ are divided by the \texttt{scale} parameter before being passed to the underlying covariance function. In our one-dimensional example this effectively stretches the realizations in the $x$ direction. If \texttt{scale} is large, the function will be correlated over a larger distance and will not `wiggle' as quickly. Figure \ref{fig:scale} illustrates the effects of the scale parameter.
\begin{figure}
    \centering
        \epsfig{file=figs/d14a1s1close.pdf, width=5cm}
        \epsfig{file=figs/d14a1s4.pdf, width=5cm}
        \epsfig{file=figs/d14a1s4far.pdf, width=5cm}
    \caption{Mat\`ern covariances with \texttt{diff_degree=1.4}, \texttt{amp=1} and \texttt{scale=1} (left) and \texttt{scale=4} (center, right), and corresponding realizations. A larger value of \texttt{scale} stretches both covariance functions and realizations (center), so that realizations don't wiggle as rapidly. When the stretched functions are plotted on commensurately stretched axes (right), they look like the unstretched functions again.}
    \label{fig:scale}
\end{figure}

\subsection{The \texttt{diff_degree} parameter}\label{sub:diffdegree}
The \texttt{diff_degree} parameter, usually denoted $\nu$, is unique (amongst the covariance functions provided by this package) to the Mat\`ern family of covariance functions. It controls the sharpness of the ridge of the covariance function, which controls the roughness/smoothness of realizations.

More specifically, look at the slices $C(x,0)$ that are shown in the upper right-hand panels of the subfigures in figures \ref{fig:amp} and \ref{fig:scale}. If \texttt{diff_degree} is greater than an integer $n$, this slice is $2n$ times differentiable at $x=0$. It turns out that this means realizations will be $n$ times differentiable.

It's natural to ask what happens when \texttt{diff_degree} isn't an integer. There is such a thing as \citetitle[http://en.wikipedia.org/wiki/Fractional_calculus]{fractional calculus}, which deals with things like taking half a derivative. Stein \cite{stein} discusses the connection briefly. See also Miller and Ross \cite{fraccalc}.

For our purposes, it's safe to say that \texttt{diff_degree} is a roughness index that can be interpreted as a degree of differentiability when it is an integer. Figure \ref{fig:diffdegree} illustrates the effects of changing this parameter:
\begin{description}
    \item[0 (not shown):] $C(x,y)=1$ if $y=x$, $0$ if $\texttt{x}\ne \texttt{y}$. Not only are realizations not differentiable, they're not even continuous. $f(x)$ is an independent normal random variable for each value of $x$.
    \item[.2:] $C(x,0)$ is not even one time differentiable at its peak, where $x=0$. Realizations are very rough, but continuous.
    \item[.5:] If \texttt{diff_degree} is just larger than $.5$, $C(x,0)$ is differentiable at $x=0$. Realizations, however, aren't differentiable; their roughness is comparable to trajectories of Brownian particles. When \texttt{diff_degree} is equal to \texttt{.5}, \texttt{matern} is equivalent to another covariance function, \texttt{pow_exp}, with extra argument \texttt{pow=1}.
    \item[1:] If \texttt{diff_degree} is just larger than $1$, $C(x,0)$ is twice differentiable at $x=0$, and realizations are differentiable.
    \item[1.4:] The value from figures \ref{fig:amp}, \ref{fig:cov} and \ref{fig:scale} is shown for comparison.
    \item[2:] If \texttt{diff_degree} is just larger than $2$, $C(x,0)$ is four times differentiable at $x=0$, and realizations are twice differentiable.
    \item[10:] Realizations are very smooth. As \texttt{diff_degree} approaches infinity, \texttt{matern} gets closer to \texttt{gaussian}. Realizations from GPs with Gaussian covariances are infinitely differentiable. In fact, if \texttt{diff_degree} is larger than 10 \texttt{matern} simply calls \texttt{gaussian}, because it's much faster.
\end{description}

\begin{figure}
    \centering
        \epsfig{file=figs/d2a1s1.pdf,width=8cm}
        % \epsfig{file=figs/d4a1s1.pdf,width=4cm}
        \epsfig{file=figs/d5a1s1.pdf,width=8cm}
        % \epsfig{file=figs/d8a1s1.pdf,width=4cm}
        \epsfig{file=figs/d10a1s1.pdf,width=8cm}
        \epsfig{file=figs/d14a1s1close.pdf,width=8cm}
        \epsfig{file=figs/d20a1s1.pdf,width=8cm}
        \epsfig{file=figs/d100a1s1.pdf,width=8cm}
    \caption{Matern draws for various \texttt{diff_degree} parameters. In increasing orders of smoothness: \texttt{.2}, \texttt{.5}, (equivalent to \texttt{pow_exp} with \texttt{pow=1}, roughness similar to Brownian motion), \texttt{1}, \texttt{1.4} (the examples given so far), \texttt{2}, \texttt{10} (nearly equivalent to \texttt{gaussian}).}
    \label{fig:diffdegree}
\end{figure}

\subsection{Suggestions for further experimentation}
\label{sec:experiment}

\begin{itemize}
    \item Change the parameters of the Mat\`ern covariance function, and try to guess how realizations will look.
    \item Differentiate realizations numerically. Recall that
    \begin{eqnarray*}
        \frac{df}{dx}=\lim_{h\rightarrow 0} \frac{f(x+h)-f(x)}{h},
    \end{eqnarray*}
    so you can take an approximate `numerical derivative' by evaluating the fraction on the right hand side with $h$ equal to a small number.
    \item Replace \texttt{matern.euclidean} in \file{examples/cov_params.py} with the Euclidean version of one of the other covariance functions and experiment with the parameters. See Banerjee \cite{banerjee} for their interesting properties.
    \item Replace \function{zero_fun} in \file{cov_params.py} with a nontrivial mean function and repeat the preceding.
\end{itemize}

\section{Nonparametric regression: observing Gaussian processes}\label{sec:observing}

\begin{figure}
    \centering
        \epsfig{file=figs/obs.pdf,width=8cm}
        \epsfig{file=figs/cond.pdf,width=8cm}
    \caption{The output of {\sffamily `examples/observations.py'}: the observed GP with \texttt{obs_V = .002} (left) and \texttt{obs_V = 0} (right). Note that in the conditioned case, the $\pm$ 1 SD envelope shrinks to zero at the points where the observations were made, and all realizations pass through the observed values. Compare these plots to those in figure \ref{fig:realizations}.}
    \label{fig:obs}
\end{figure}

Consider the following common statistical situation: You decide on a GP prior for an unknown function $f$, then you observe the value of $f$ at $N$ input points $[o_0\ldots o_{N-1}]$, possibly with uncertainty. If the observation error is normally distributed, it turns out that $f$'s posterior distribution given the new information is another Gaussian process, with new mean and covariance functions.

The probability model that represents this situation is as follows:
\begin{equation}
    \label{regprior}
    \left.\begin{array}{l}
        \textup{data}_i \stackrel{\tiny{\textup{ind}}}{\sim} \textup{N}(f(o_i), V_i)\\
        f \sim \textup{GP}(M,C)\\
    \end{array}\right\}\Rightarrow f|\textup{data} \sim \textup{GP}(M_o, C_o).
\end{equation}
This package provides a function called \function{observe} that imposes normally-distributed observations on Gaussian process distributions. This function converts $f$'s prior to its posterior by transforming $M$ and $C$ in equation \ref{regprior} to $M_o$ and $C_o$:

The following code (from \file{observation.py}) imposes the observations
\begin{eqnarray*}
    f(-.5) = 3.1\\
    f(.5) = 2.9
\end{eqnarray*}
with observation variance $V=.002$ on the GP distribution defined in \file{mean.py} and \file{cov.py}:
\verbatiminput{../../examples/gp/observation.py}

The function \function{observe} takes a covariance $C$ and a mean $M$ as arguments, and essentially tells them that their realizations' values on \code{obs_mesh} have been observed to be \code{obs_vals} with variance \code{obs_V}. If \code{obs_V} is \code{None}, \function{observe} assumes that the observation precision was infinite; that is, that the realizations' values on \code{obs_mesh} were observed with no uncertainty. Making (or pretending to make) observations with infinite precision is sometimes called \emph{conditioning}, and can be a valuable tool for modifying GP priors; for example, if a rate function is known to be zero when a population's size is zero.

The output of the code is shown in figure \ref{fig:obs}, along with the output with \code{obs_V=None}. Compare these to the analogous figure for the unobserved GP, figure \ref{fig:realizations}. The covariance after observation is visualized in figure \ref{fig:obscov}. The covariance `tent' has been pressed down at points where $x\approx \pm .5$ and/or $y\approx\pm .5$, which are the values where the observations were made.

\begin{figure}
    \centering
        \epsfig{file=figs/obscov.pdf,width=10cm}
    \caption{The covariance function from {\sffamily `observation.py'} after observation. Compare this with the covariance function before observation, visualized in figure \ref{fig:cov} }
    \label{fig:obscov}
\end{figure}

\subsection{Example: Salmonid stock-recruitment functions}\label{sub:MMKregression}
Munch, Kottas and Mangel \cite{mmk} use Gaussian process priors to infer various \emph{stock-recruitment (SR) functions}. An important concept in fishery science, SR functions relate the size of a fish stock to the number or biomass of recruits to the fishery each year. In other words, they relate population size or biomass to number or biomass of new fish produced. The authors argue that model uncertainty is endemic in stock-recruitment theory, and that in this situation GP priors are a sensible alternative to particular functional forms.

We don't have the tools yet to fully duplicate Munch, Kottas and Mangel' results; that will have to wait for chapter \ref{cha:PyMC}. However, we can fit a simpler version of their model now by using the \function{observe} function. Specifically, we'll fit the data in figure 6 of their paper \cite{mmk}, which is for three salmonids: chum (\emph{Onchorhynchus keta}), pink (\emph{Onchorhynchus gorbuscha}) and sockeye (\emph{Onchorhynchus nerka}).

The code is in the script \file{examples/gp/more_examples/MMKsalmon/regression.py}. The script begins by importing the \class{salmon} class from \file{salmon.py} in the same directory.
The \class{salmon} class does the following:
\begin{itemize}
    \item Reads data from a csv file.
    \item Creates a GP prior with a Mat\`ern covariance function and a linear mean function. The parameters are chosen fairly arbitrarily at this stage. In chapter \ref{cha:PyMC}, we'll look at how to place priors on these parameters and infer them along with the unknown function itself.

    Note also that I've specified the prior using the data; for instance, the \texttt{scale} parameter depends on the maximum observed abundance. Some would consider this cheating.
    \item `Observes' the unknown function's value to be zero at the origin with no uncertainty. No matter what the data are, every draw from the posterior will have $f(0) = 0$. This isn't really an observation, it's just a convenient way to incorporate the knowledge that if there is no stock, there will be no recruitment.
    \item Provides a \method{plot} method, which just plots the posterior envelope, data and three realizations from the posterior.
\end{itemize}
% The code in \file{examples/more_examples/MKMsalmon/salmon.py} is shown here:
% \verbatiminput{../../examples/gp/more_examples/MKMsalmon/salmon.py}

\begin{figure}
    \centering
        \epsfig{file=figs/MMKchumreg.pdf,width=10cm}
        \epsfig{file=figs/MMKsockeyereg.pdf,width=10cm}
        \epsfig{file=figs/MMKpinkreg.pdf,width=10cm}
    \caption{Fits to the stock-recruitment data in Munch, Kottas and Mangel' \cite{mmk} Figure 6 using a simple nonparametric regression.}
    \label{fig:MMKregression}
\end{figure}

The main script, \file{examples/more_examples/MKMsalmon/salmon.py}, creates three \class{salmon} instances called \texttt{chum}, \texttt{pink} and \texttt{sockeye}, then imposes the data for each species on its prior to obtain its posterior. The observation variance I used was chosen fairly arbitrarily like the prior parameters, and we'll look at inferring it in chapter \ref{cha:PyMC} also. Finally, each species' \method{plot} method is called. Output is shown in figure \ref{fig:MMKregression}.
% The code in \file{examples/more_examples/MKMsalmon/regression.py} is shown here:
% \verbatiminput{../../examples/gp/more_examples/MKMsalmon/regression.py}

To reiterate, there are some major drawbacks to this simple model. The observation variance may not actually be known, and we may not be comfortable specifying a single value for each of the prior parameters. Because of these considerations, Munch, Kottas and Mangel opt for a more sophisticated statistical model that has to be fit using MCMC. We will follow them in section \ref{sub:MMKMCMC}.

\section{Higher-dimensional GPs}\label{sec:highdim}

In addition to functions of one variable such as $f(x)$, this package supports Gaussian process priors for functions of many variables such as $f(\mathbf{x})$, where $\mathbf{x}=[x_0\ldots x_{n-1}]$. This is useful for modeling dynamical or biological functions of many variables as well as for spatial statistics.

Any time you pass an array into a \texttt{Mean}, \texttt{Covariance} or \texttt{Realization}'s init method or evaluate one of these objects on an array, the convention is that the array's last index iterates over spatial dimension. To evaluate a covariance C on the ordered pairs \texttt{(0,1)}, \texttt{(2,3)}, \texttt{(4,5)} and \texttt{(6,7)}, you could pass in the following two-dimensional array:
\begin{verbatim}
[[0,1]
 [2,3]
 [4,5]
 [6,7]]
\end{verbatim}
or the following three-dimensional array:
\begin{verbatim}
[[[0,1]
  [2,3]],

  [4,5]
  [6,7]]]
\end{verbatim}
Either is fine, since in both the last index iterates over elements of the ordered pairs.

The exception to this rule is one-dimensional input arrays. The array
\begin{verbatim}
[0, 1, 2, 3, 4, 5, 6, 7]
\end{verbatim}
is interpreted as an array of eight one-dimensional values, whereas the array
\begin{verbatim}
[[0, 1, 2, 3, 4, 5, 6, 7]]
\end{verbatim}
is interpreted as a single eight-dimensional value according to the convention above.

Means and covariances learn their spatial dimension the first time they are called or observed. Some covariances, such as those specified in geographic coordinates, have an intrinsic spatial dimension. Realizations inherit their spatial dimension from their means and covariances when possible, otherwise they learn it the first time they are called. If one of these objects is subsequently called with an input of a different dimension, it raises an error.

\subsection{Covariance function bundles and coordinate systems}
The examples so far, starting with \file{examples/cov.py}, have used the covariance function \texttt{matern.euclidean}. This function is an attribute of the \texttt{matern} object, which is an instance of class \class{covariance_function_bundle}.

Instances of \class{covariance_function_bundle} have three attributes, \texttt{euclidean}, \texttt{geo_deg} and \texttt{geo_rad}, which correspond to standard coordinate systems:
\begin{itemize}
    \item \texttt{euclidean}: $n$-dimensional Euclidean coordinates.
    \item \texttt{geo_deg}: Geographic coordinates (longitude, latitude) in degrees, with radius 1.
    \item \texttt{geo_rad}: Geographic coordinates (longitude, latitude) in radians, with radius 1.
\end{itemize}
Note that you can effectively change the radius of the geographic coordinate systems using the \texttt{scale} parameter.

Covariance function bundles are described in more detail in section \ref{cha:usercov}.

\subsection{Multithreading GP operations}
This package can use multi-core systems to speed up two kinds of computations:
\begin{itemize}
	\item filling in covariance matrices,
	\item linear algebra for observing GP's or drawing realizations.
\end{itemize}
If you've built NumPy against multithreaded linear algebra libraries, all the linear algebra will be parallelized automatically. The functions contained in covariance function bundles (section \ref{cha:usercov}), which include all the covariance functions distributed with this package, are multithreaded. The number of threads they use is controlled by the environment variable \code{OMP_NUM_THREADS}.

On a quad-core system, evaluating a covariance function on large input vectors when \texttt{OMP_NUM_THREADS} is equal to 4 will use all the cores. Note that there's no point setting \texttt{OMP_TUM_THREADS} to 5 on a quad-core system, because only four cores are available.

\section{Basis covariances}\label{sec:basis}

\begin{figure}[htbp]
    \centering
        \epsfig{file=figs/basiscov.pdf,width=10cm}
        \caption{Three realizations of an observed Gaussian process whose covariance is an instance of \class{BasisCovariance}. The basis in this case is function \function{fourier_basis} from module \module{cov_funs}. 25 basis functions are used.}
    \label{fig:basiscov}
\end{figure}

It's possible to create random functions from linear combinations of finite sets of basis functions $\{e\}$ with random coefficients $\{c\}$:
\begin{eqnarray*}
    f(x) = M(x) + \sum_{i_0=0}^{n_0-1}\ldots \sum_{i_{N-1}=0}^{n_{N-1}-1} c_{i_1\ldots i_{N-1}} e_{i_1\ldots i_{N-1}}(x), &
    \{c\}\sim \textup{some distribution}.
\end{eqnarray*}
If the distribution is multivariate normal with mean zero, $f$ is a Gaussian process with mean $M$ and covariance defined by
\begin{eqnarray*}
    C(x,y)=\sum_{i_0=0}^{n_0-1}\ldots \sum_{i_{N-1}=0}^{n_{N-1}-1} \sum_{j_0=0}^{n_0-1}\ldots \sum_{j_{N-1}=0}^{n_{N-1}-1} e_{i_0\ldots i_{N-1}}(x) e_{j_1\ldots j_{N-1}}(x) K_{i_0\ldots i_{N-1}, j_1\ldots j_{N-1}},
\end{eqnarray*}
where $K$ is the covariance of the coefficients $c$.

Particularly successful applications of this general idea (shown in one dimension) are:
\begin{description}
    \item[Random Fourier series:] $e_i(x) = \sin(i\pi x/L)$ or $\cos(i\pi x/L)$, for instance \cite{spanos}.
    \item[Gaussian process convolutions:] $e_i(x) = \exp(-(x-\mu_n)^2)$, for instance \cite{convolution}.
    \item[B-splines:] $e_i(x) = $ a polynomial times an interval indicator. See \citetitle[http://en.wikipedia.org/wiki/Basis_B-spline]{Wikipedia}'s article.
\end{description}
Such representations can be very efficient when there are many observations in a low-dimensional space, but are relatively inflexible in that they generally produce realizations that are infinitely differentiable. In some applications, this tradeoff makes sense.

This package supports basis representations via the \class{BasisCovariance} class:
\begin{verbatim}
    C = BasisCovariance(basis, cov, **basis_params)
\end{verbatim}
The arguments are:
\begin{description}
    \item[\texttt{basis}:] Must be an array of functions, of any shape. Each basis function will be evaluated at \texttt{x} with the extra parameters. The basis functions should obey the same calling conventions as mean functions: return values should have shape \code{x.shape[:-1]} unless \texttt{x} is one-dimensional, in which case return values should be of the same shape as \texttt{x}. Note that each function should take the entire input array as an argument.
    \item[\texttt{cov}:] An array whose shape is either:
        \begin{itemize}
            \item Of the same shape as \texttt{basis}. In this case the coefficients are assumed independent, and \texttt{cov[i[0],...,i[N-1]]} (an $N$-dimensional index) simply gives the prior variance of the corresponding coefficient.
            \item Of shape \texttt{basis.shape * 2}, using Python's convention for tuple multiplication. In this case \texttt{cov[i[0],...,i[N-1], j[0],...,j[N-1]]} (a $2N$-dimensional index) gives the covariance of $c_{i_0\ldots i_{N-1}}$ and $c_{j_1\ldots j_{N-1}}$.
        \end{itemize}
        Internally, the basis array is ravelled and this covariance tensor is reshaped into a matrix; I have made the input convention this way because it seems easier to keep track of which covariance value corresponds to which coefficients. The covariance tensor must be symmetric (\texttt{cov[i[0],...,i[N-1], j[0],...,j[N-1]]} $=$ \texttt{cov[j[0],...,j[N-1], i[0],...,i[N-1]]}), and positive semidefinite when reshaped to a matrix.
    \item[\texttt{basis_params}:] Any extra parameters required by the basis functions.
\end{description}

\section{Separable bases}

Many bases, such as Fourier series, can be decomposed into products of functions as follows:
\begin{eqnarray*}
    e_{i_0\ldots i_{N-1}}(x) = e^0_{i_0}(x)\ldots e^{N-1}_{i_{N-1}}(x)
\end{eqnarray*}
Basis covariances constructed using such bases can be represented more efficiently using \texttt{SeparableBasisCovariance} objects. These objects are constructed just like \texttt{BasisCovariance} objects, but instead of an $n_0\times \ldots \times n_{N-1}$ array of basis functions they take a nested lists of functions as follows:
\begin{verbatim}
    basis = [ [e[0][0], ... ,e[0][n[0]-1]]
                       ...
              [e[N-1][0], ... ,e[N-1][n[N-1]-1]] ].
\end{verbatim}
For an $N$-dimensional Fourier basis, each of the \texttt{e}'s would be a sine or cosine; frequency would increase with the second index. As with \texttt{BasisCovariance}, each basis needs to take the entire input array \texttt{x} and \texttt{basis_params} as arguments. See \texttt{fourier_basis} in \texttt{examples/gp/basiscov.py} for an example.

\section{Example}

Once created, a \class{BasisCovariance} or \class{SeparableBasisCovariance} object behaves just like a \class{Covariance} object, but it and any \texttt{Mean} and \texttt{Realization} objects associated with it will take advantage of the efficient basis representation in their internal computations. An example of \class{SeparableBasisCovariance} usage is given in \file{examples/basis_cov.py}, shown below. Compare its output in figure \ref{fig:basiscov} to that of \file{examples/observation.py}.
\verbatiminput{../../examples/gp/basiscov.py}


\documentclass[letterpaper]{article}

\usepackage[utf8]{luainputenc}
\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{23 février, 2015}
\maketitle
Theorem: if $f(x)=\sum\limits{a_nx^n}$ has radius of convergence $R$ then so does $\sum\limits{na_nx^{n-1}}$ and further $f'(x)=\sum\limits{na_nx^{n-1}}$ and $\int_0^x{f(t)\;\mathrm{d}t}=\sum\limits{\frac{a_n}{n+1}x^{n+1}}$

\subsection*{examples}
$\sum\limits_{n=1}^\infty{\frac{2^n}{n5^n}}=\sum\limits_{n=1}^u{\left(\frac{2}{5}\right)^n\frac{1}{n}}$

$\sum\limits_{n=1}^\infty{\frac{x^n}{n}}=\int_{0}^x{\sum\limits_{n=0}{t^n}}=\int{\frac{x^{n+1}}{n+1}}=\int_0^x{\frac{1}{1-t}}=-\ln(1-t)|_0^x=-\ln(1-x)=-\ln(1-\frac{2}{5})=\ln 5-\ln 3$

$\sum\limits_{n=0}{(n+1)x^n}=\sum\limits_{n=1}{nx^n}$
antiderivative is $\sum\limits_{n=0}{x^n}=\frac{1}{1-x}$ and our answer is $\frac{1}{(1-x)^2}$

$\sum\limits{\frac{n+1}{2^n}}=\frac{1}{1-\frac{1}{2}}^2=\frac{1}{\frac{1}{2^2}}=4$

\subsubsection*{approximate pi}
$\tan^{-1} x=\int_0^x{\frac{1}{1+t^2}\;\mathrm{d}t}$

$\sum\limits{(-t^2)^n}=\frac{1}{1+t^2}=\frac{1}{1-(-t^2)}$ as long as $-1<-t^2<1\Leftrightarrow-1<t<1$

and so $\tan^{-1} x=\int_0^x{\sum\limits_{n=0}^\infty{(-1)^nt^2n}\;\mathrm{d}t}$

we can choose $\pi/4$ or $\pi/6$. now $\frac{\pi}{4}=\tan^{-1}1$ but $-1<x<1$

and so $\pi/6=\tan^{-1}\frac{\sqrt{3}}{3}=\sum\limits_{n=0}^\infty{(-1)^n\dots}$

\end{document}


\chapter{Terminology on sets and functions}
\label{ch:sets_functions}
This appendix will cover some notions on sets and functions
such as ``bijections'', ``equivalence classes'', and so on.

Remark for experts: I am not dealing with foundational issues in this chapter.
See \Cref{ch:zfc} (and onwards) if that's what you're interested in.
Consequently I will not prove most assertions.

\section{Sets}
A \vocab{set} for us will just be a collection of elements (whatever they may be).
For example, the set $\NN = \{1, 2, 3, 4, \dots\}$ is the positive integers,
and $\ZZ = \{ \dots, -2, -1, 0, 1, 2, \dots\}$ is the set of all integers.
As another example, we have a set of humans:
\[ H = \left\{ x \mid \text{$x$ is a featherless biped} \right\}. \]
(Here the ``$\mid$'' means ``such that''.)

There's also a set with no elements, which we call the \vocab{empty set}.
It's denoted by $\varnothing$.

It's conventional to use capital letters for sets (like $H$),
and lowercase letters for elements of sets (like $x$).

\begin{definition}
We write $x \in S$ to mean ``$x$ is in $S$'', for example $3 \in \NN$.
\end{definition}

\begin{definition}
	If every element of a set $A$ is also in a set $B$,
	then we say $A$ is a \vocab{subset} of $B$,
	and denote this by $A \subseteq B$.
	If moreover $A \neq B$, we say $A$ is a \vocab{proper subset}
	and write $A \subsetneq B$.
	(This is analogous to $\le$ and $<$.)

	Given a set $A$, the set of all subsets is denoted $2^A$
	or $\PP(A)$ and called the \vocab{power set} of $A$.
\end{definition}
\begin{example}
	[Examples of subsets]
	\listhack
	\begin{enumerate}[(a)]
		\ii $\{1,2,3\} \subseteq \NN \subseteq \ZZ$.
		\ii $\varnothing \subseteq A$ for any set $A$. (Why?)
		\ii $A \subseteq A$ for any set $A$.
		\ii If $A = \{1,2\}$ then $2^A =
		\left\{ \varnothing, \{1\}, \{2\}, \{1,2\} \right\}$.
	\end{enumerate}
\end{example}

\begin{definition}
	We write
	\begin{itemize}
		\ii $A \cup B$ for the set of elements in
		\emph{either} $A$ or $B$ (possibly both),
		called the \vocab{union} of $A$ and $B$.
		\ii $A \cap B$ for the set of elements in \emph{both} $A$ and $B$, and
		called the \vocab{intersection} of $A$ and $B$.
		\ii $A \setminus B$ for the set of elements in $A$ but \emph{not} in $B$.
	\end{itemize}
\end{definition}

\begin{example}
	[Examples of set operations]
	Let $A = \{1,2,3\}$ and $B = \{3,4,5\}$. Then
	\begin{align*}
		A \cup B &= \{1,2,3,4,5\} \\
		A \cap B &= \{3\} \\
		A \setminus B &= \{1,2\}.
	\end{align*}
\end{example}

\begin{exercise}
	Convince yourself: for any sets $A$ and $B$,
	we have $A \cap B \subseteq A \subseteq A \cup B$.
\end{exercise}

Here are some commonly recurring sets:
\begin{itemize}
	\ii $\CC$ is the set of complex numbers, like $3.2 + \sqrt 2 i$.
	\ii $\RR$ is the set of real numbers, like $\sqrt 2$ or $\pi$.
	\ii $\NN$ is the set of positive integers, like $5$ or $9$.
	\ii $\QQ$ is the set of rational numbers, like $7/3$.
	\ii $\ZZ$ is the set of integers, like $-2$ or $8$.
\end{itemize}
(These are pronounced in the way you would expect:
``see'', ``are'', ``en'', ``cue'', ``zed''.)

\section{Functions}
Given two sets $A$ and $B$, a \vocab{function} $f$ from $A$ to $B$
is a mapping of every element of $A$ to some element of $B$.

We call $A$ the \vocab{domain} of $f$, and $B$ the \vocab{codomain}.
We write this as $f \colon A \to B$ or $A \taking f B$.
\begin{abuse}
	If the name $f$ is not important, we will often just write $A \to B$.
\end{abuse}
We write $f(a) = b$ or $a \mapsto b$ to signal that $f$ takes $a$ to $b$.

If $B$ has $0$ as an element and $f(a) = 0$,
we often say $a$ is a \vocab{root} or \vocab{zero} of $f$,
and that $f$ \vocab{vanishes} at $a$.

\subsection{Injective / surjective / bijective functions}

\begin{definition}
	A function $f \colon A \to B$ is \vocab{injective}
	if it is ``one-to-one'' in the following sense:
	if $f(a) = f(a')$ then $a = a'$.
	In other words, for any $b \in B$,
	there is \emph{at most} one $a \in A$ such that $f(a) = b$.

	Often, we will write $f \colon A \injto B$ to emphasize this.
\end{definition}
\begin{definition}
	A function $f \colon A \to B$ is \vocab{surjective}
	if it is ``onto'' in the following sense:
	for any $b \in B$ there is \emph{at least} one $a \in A$
	such that $f(a) = b$.

	Often, we will write $f \colon A \surjto B$ to emphasize this.
\end{definition}
\begin{definition}
	A function $f \colon A \to B$ is \vocab{bijective}
	if it is both injective and surjective.
	In other words, for each $b \in B$,
	there is \emph{exactly} one $a \in A$ such that $f(a) = b$.
\end{definition}

\begin{example}
	[Examples of functions]
	By ``human'' I mean ``living featherless biped''.
	\begin{enumerate}[(a)]
		\ii There's a function taking every human to their
		age in years (rounded to the nearest integer).
		This function is \textbf{not injective},
		because for example there are many people with age $20$.
		This function is also \textbf{not surjective}: no one has age $10000$.

		\ii There's a function taking every
		USA citizen to their social security number.
		This is also \textbf{not surjective} (no one has SSN equal to $3$),
		but at least it \textbf{is injective} (no two people have the same SSN).
	\end{enumerate}
\end{example}

\begin{example}
	[Examples of bijections]
	\listhack
	\begin{enumerate}[(a)]
		\ii Let $A = \{1,2,3,4,5\}$ and $B = \{6,7,8,9,10\}$.
		Then the function $f \colon A \to B$ by $a \mapsto a+5$ is a bijection.
		\ii In a classroom with $30$ seats,
		there is exactly one student in every seat.
		Thus the function taking each student to the seat they're in
		is a bijection; in particular, there are exactly $30$ students.
	\end{enumerate}
\end{example}

\begin{remark}
	Assume for convenience that $A$ and $B$ are finite sets.
	Then:
	\begin{itemize}
		\ii If $f \colon A \injto B$ is injective,
		then the size of $A$ is at most the size of $B$.
		\ii If $f \colon A \surjto B$ is surjective,
		then the size of $A$ is at least the size of $B$.
		\ii If $f \colon A \to B$ is a bijection,
		then the size of $A$ equals the size of $B$.
	\end{itemize}
\end{remark}

Now, notice that if $f \colon A \to B$ is a bijection,
then we can ``apply $f$ backwards'':
(for example, rather than mapping each student to the seat they're in,
we map each seat to the student sitting in it).
This is called an \vocab{inverse function};
we denote it $f\inv \colon B \to A$.

\subsection{Images and pre-images}
Let $X \taking f Y$ be a function.

\begin{definition}
	Suppose $T \subseteq Y$.
	The \vocab{pre-image} $f\pre(T)$ is the set of all
	$x \in X$ such that $f(x) \in T$.
	Thus, $f\pre(T)$ is a subset of $X$.
\end{definition}
\begin{example}
	[Examples of pre-image]
	Let $f \colon H \to \ZZ$ be the age function from earlier.
	Then
	\begin{enumerate}[(a)]
		\ii $f\pre(\{13, 14, 15, 16, 17, 18, 19\})$ is the set of teenagers.
		\ii $f\pre(\{0\})$ is the set of newborns.
		\ii $f\pre(\{1000, 1001, 1002, \dots \}) = \varnothing$,
		as I don't think anyone is that old.
	\end{enumerate}
\end{example}

\begin{abuse}
	By abuse of notation, we may abbreviate $f\pre(\{y\})$ to $f\pre(y)$.
	So for example, $f\pre(\{0\})$ above becomes shortened to $f\pre(0)$.
\end{abuse}

The dual notion is:
\begin{definition}
	Suppose $S \subseteq X$.
	The \vocab{image} $f\im(S)$ is the set of
	all things of the form $f(s)$.
\end{definition}
\begin{example}
	[Examples of images]
	Let $A = \{1,2,3,4,5\}$ and $B = \ZZ$.
	Consider a function $f \colon A \to B$ given by
	\[
		f(1) = 17 \quad
		f(2) = 17 \quad
		f(3) = 19 \quad
		f(4) = 30 \quad
		f(5) = 234.
	\]
	\begin{enumerate}[(a)]
		\ii The image $f\im(\{1,2,3\})$ is the set $\{17, 19\}$.
		\ii The image $f\im(A)$ is the set $\{17, 19, 30, 234\}$.
	\end{enumerate}
\end{example}
\begin{ques}
	Suppose $f \colon A \surjto B$ is surjective.
	What is $f\im(A)$?
\end{ques}

\section{Equivalence relations}
Let $X$ be a fixed set now.
A binary relation $\sim$ on $X$ assigns a truth value ``true''
or ``false'' to $x \sim y$ for each $x$ or $y$.
Now an \vocab{equivalence relation} $\sim$ on $X$ is a binary relation
which satisfies the following axioms:
\begin{itemize}
	\ii Reflexive: we have $x \sim x$.
	\ii Symmetric: if $x \sim y$ then $y \sim x$
	\ii Transitive: if $x \sim y$ and $y \sim z$ then $x \sim z$.
\end{itemize}
An \vocab{equivalence class} is then a
set of all things equivalent to each other.
One can show that $X$ becomes partitioned by these equivalence classes:

\begin{example}
	[Example of an equivalence relation]
	Let $\NN$ denote the set of positive integers.
	Then suppose we declare $a \sim b$ if $a$ and $b$ have the same last digit,
	for example $131 \sim 211$, $45 \sim 125$, and so on.

	Then $\sim$ is an equivalence relation.
	It partitions $\NN$ into ten equivalence classes,
	one for each trailing digit.
\end{example}

Often, the set of equivalence classes will be denoted $X/{\sim}$
(pronounced ``$X$ mod sim'').