\subsection{Identifying primes}

different to factorising. We don't care what the actual factors are, just see if it's prime

\subsection{Fermat's primality test}

\subsubsection{Fermat's little theorem recap}

\subsubsection{Fermat's primality test}

From Fermat's little theorem we know

\(a^{n-1}=1 mod(n)\)

Where \(a\) is an integer and \(n\) is prime.


\subsection{Topologies}

\subsection{Topologies on sets}

\(T\) is a topology on set \(X\) if:

\begin{itemize}
\item \(X\in T\)
\item \(\varnothing \in T\)
\item Unions of \(T\) are in \(T\)
\item Intersections of \(T\) are in \(T\)
\end{itemize}

\subsection{Examples of topologies: The trivial topology}

The trivial topology contains only the underlying set and the empty set.

\subsection{Examples of topologies: The discrete toplogy}

The discrete toplogy contains all subsets of the underlying set (is this the power set?)


%!TEX root = RBM.tex

\subsubsection{Thouless-Anderson-Palmer Sampling\protect\footnote{Available at \protect\url{https://github.com/lzhbrian/MCMC/blob/master/src/partition/TAP.m} in Matlab}}

\para{Algorithm}
Thouless-Anderson-Palmer Sampling(TAP)\cite{gabrie2015training} is a very efficient and easy-to-practice iterative procedure based on an improved mean field method from statistical physics called Thouless-Anderson-Palmer approach.

The main idea of this method is to iteratively compute the magnetization vector $m^{v}$,$m^{h}$, and then input the values into the Legendre transform of the free energy $F=log(Z(\theta))$ to compute it.

The iteration of $m^{v}$ and $m^{h}$ follows:
\begin{equation}
\begin{aligned}
m^{h}_{j}{[t+1]} & = sigmoid\Bigg[b_{j} + \sum_{i}W_{ij}m^{v}_{i}{[t]} - \\
& W^{2}_{ij} \Bigg(m^{h}_{j}{[t] - \frac{1}{2}} \Bigg) \Big(m^{v}_{i}{[t]}-(m^{v}_{i}{[t]})^{2} \Big) \Bigg] \\
m^{h}_{i}{[t+1]} & = sigmoid\Bigg[a_{i} + \sum_{j}W_{ij}m^{h}_{j}{[t+1]} - \\
& W^{2}_{ij} \Bigg(m^{v}_{i}{[t] - \frac{1}{2}} \Bigg) \Big(m^{h}_{j}{[t+1]}-(m^{h}_{j}{[t+1]})^{2} \Big) \Bigg]
\end{aligned}
\end{equation}

The Legendre transform of $F$ to the second order is:
\begin{equation}
\begin{split}
\Gamma(\mathbf m^{v},\mathbf m^{h}) &\approx - S(\mathbf m^{v},\mathbf m^{h}) - \sum_{i} a_{i}m^{v}_{i} - \sum_{j} b_{j}m^{h}_{j} \\
& - \sum_{i,j} \Bigg( W_{i,j}m^{v}_{i}m^{h}_{j} \\
& - 0.5W_{ij}\Big(m^{v}_{i}-(m^{v}_{i})^2\Big)\Big(m^{h}_{j}-(m^{h}_{j})^2\Big) \Bigg)
\end{split}
\end{equation}
where $S(\mathbf m^{v},\mathbf m^{h})$ indicates the entropy:
\begin{equation}
\begin{split}
S(\mathbf m^{v},\mathbf m^{h}) &= - \sum_{i} \Bigg(m^{v}_{i}logm^{v}_{i} + (1-m^{v}_{i})log(1-m^{v}_{i}) \Bigg) \\
&- \sum_{j} \Bigg(m^{h}_{j}logm^{h}_{j} + (1-m^{h}_{j})log(1-m^{h}_{j}) \Bigg)
\end{split}
\end{equation}

\para{Practice}
For real practice, in the next subsection, we see TAP method can obtain a converged result in a very short time, but has less accuracy. And sometimes, the converged results are periodic, which is not what we want by us.

	\begin{algorithm}
        \caption{Thouless-Anderson-Palmer Sampling}
        \begin{algorithmic}
        	\Require Required Iteration time $N$,
        	\State Initialize $m^{v}$, $m^{h}$
            \For{$t = 1 \to N$}
            	\State $m^{h}{[t+1]} \gets f(m^{h}{[t]})$
            	\State $m^{v}{[t+1]} \gets f(m^{v}{[t]},m^{v}{[t+1]})$
			\EndFor
			\State $logZ(\theta) \gets \Gamma(\mathbf m^{v},\mathbf m^{h})$
        \end{algorithmic}
    \end{algorithm}


\section*{Statistics}
\begin{itemize}
\item [mean:] $\mu = (1/N) \sum_{i=1}^N x_i$
\item [variance:] $\sigma^2 = (1/N) \sum_{i=1}^N (x_i-\mu)^2$
\item [standard deviation:] $\sigma = \sqrt{\sigma^2}$
\end{itemize}

\subsection*{Probability}
Probability of
\begin{itemize}
\item [$A$:] $P(A) \in [0,1]$
\item [$\neg A$:] $P(\neg A) = 1 - P(A)$
\item [$A$ given $B$:] $P(A\vert B) $
\item [$A$ or $B$:] $P(A\cup B) = P(A) + P(B) - P(A\cap B)$
\begin{itemize}
  \item[if $A$ is indep. of $B$] $P(A\cup B) = P(A) + P(B)$
\end{itemize}
\item [$A$ and $B$:] $P(A\cap B) = P(A\vert B)P(B)$
\begin{itemize}
  \item[if $A$ is indep. of $B$] $P(A\cap B) = P(A)P(B)$
\end{itemize}
\item $P(x) := Pr[X = x] := \sum_{y \in Y} P(x, y)$
\item $P(x|y) := Pr[X = x | Y = y] := \frac{P(x,y)}{P(y)},\quad \text{if } P(y) > 0$
\item $\forall y \in Y: \sum_{x \in X} P(x|y) = 1$ (property for any fixed $y$)
\item $P(x, y) = P(x|y) P(y)$
\item $P(x|y) = \frac{P(y|x)P(x)}{P(y)}$ (Bayes' rule)
\item $P(x|y) = P(x) \Leftrightarrow P(y|x) = P(y)$ (iff $X$, $Y$ independent)
\item $P(x_1, \ldots, x_n) = \prod_{i=1}^n P(x_i)$ (iff IID)
\end{itemize}


\chapter{Support Vector Machines}
Super Vegetable Me
\newpage

\section{Max Margin Classifiers}
    The goal is to find an \emph{optimal} separating hyperplane $w^Tx+b = 0$ such that the margin between different classes is \emph{maximized}. In the following section we assume the data are linear separable.

    \subsection{Notations}
        \begin{itemize}
            \item $x^{(i)}$: $i$-th sample
            \item $y^{(i)} \in \{1, -1\}$: $i$-th label
        \end{itemize}

    \subsection{Basic Problem Formulation}
        \begin{itemize}
            \item $w^Tx + b = 0$ is the separating hyperplane
            \item For $x$ on the hyperplane, $w^Tx + b = 0$
            \item For $x$ on the upper boarder of the margin, $w^Tx+b = 1$, this can always be satisfied by scaling $w$ and $b$
            \item For $x$ on the lower boarder of the margin , $w^Tx+b = -1$
            \item For $x$ with label $y=1$, we want $w^Tx+b\ge1$
            \item For $x$ with label $y=-1$, we want $w^Tx+b\le1$
        \end{itemize}
        Let $h(x) = w^Tx + b$. For $x$ with $y=1$, we want $h(x) \ge 1$, and conversely for $x$ with $y=-1$, we want $h(x) \le -1$. Or in a compact way, for all $x$, we want
        \[ y(w^Tx + b) \ge 1 \]

    \subsection{Formulation into an Optimization Problem}
        Let $w_o^Tx+b_o=0$ be the optimal separating hyperplane, with normal vector $\frac{w_o}{\|w_o\|}$.
        \begin{enumerate}
            \item For any $x$, let $x_p$ be the point on $w_o^Tx+b=0$ s.t. $x - x_p$ is perpendicular to $w_o^Tx+b_o=0$
            \item $x = x_p + r\left(\frac{w_o}{\|w_o\|}\right)$, where $r$ is a scaling factor
            \item $h(x_p) = w_o^Tx_p + b_o = 0$
            \item $w_o^Tx_p = -b_o$
            \item $h(x) = w_o^Tx + b_o = w_o^T\left(x_p + r\left(\frac{w_o}{\|w_o\|}\right)\right) + b_o = w_o^Tx_p + r\left(\frac{w_o^Tw_o}{\|w_o\|}\right) + b_o = r\|w_o\|$
            \item For a support vector $x^{(s)}$, $|r| = h(x^{(s)})/\|w_o\| = 1/\|w_o\|$
            \item So the margin of separation is $2r = 2/\|w_o\|$
            \item Maximizing $1/\|w\|$ is equivalent to minimizing $\|w\|^2 = w^Tw$
        \end{enumerate}
        Therefore
        \begin{align*}
            \min \quad &\frac{1}{2}w^Tw\\
            \text{s.t.} \quad & y^{(i)}(w^Tx^{(i)}+b) \ge 1
        \end{align*}

\section{Dual Formulation of SVM}
    \subsection{Lagrangian and KKT Conditions of Hard-Margin SVM}
        We consider the minimization problem:
        \begin{align*}
             \min \quad &\frac{1}{2}w^Tw\\
             \text{s.t.} \quad & y^{(i)}(w^Tx^{(i)}+b) \ge 1
        \end{align*}
        Let $\alpha_i$ be the Lagrangian Multipliers, then the Lagrangian is
        \[ \mathcal{L}(w,b,\alpha) = \frac{1}{2}w^Tw - \sum_{i=1}^N \alpha_i (y^{(i)}(w^Tx^{(i)} + b) - 1) \]
        The KKT Conditions are
        \[
        \begin{cases}
            \nabla_w \mathcal{L} = w - \sum_{i=1}^N \alpha_i y^{(i)}x^{(i)} = 0\\
            \nabla_b \mathcal{L} = \sum_{i=1}^N \alpha_i y^{(i)} = 0\\
            y^{(i)}(w^Tx^{(i)} + b) \ge 1\\
            \alpha_i \ge 0\\
            \alpha_i(y^{(i)}(w^Tx^{(i)} + b) - 1) =0
        \end{cases}
        \]
        \begin{remark}~{}
            \begin{itemize}
                \item If $\alpha_i > 0$, $x^{(i)}$ lies on the decision margin and corresponds to a support vector.
                \item If $\alpha_i = 0$, $x^{(i)}$ does not lie on the decision margin.
            \end{itemize}
        \end{remark}

    \subsection{Dual Formulation of SVM}
        Notice that $w$ can be represented by $\alpha_i$ and $x^{(i)}$
        \[ w = \sum_{i=1}^N \alpha_i y^{(i)}x^{(i)} \]
        We can derive the dual problem of SVM from the Lagrangian,
        \begin{align*}
            \mathcal{L}(w,b,\alpha) &= \frac{1}{2}w^Tw - \sum_{i=1}^N \alpha_i y^{(i)}(w^Tx^{(i)} + b - 1)\\
            &= \frac{1}{2}w^Tw - \sum_{i=1}^N \alpha_i y^{(i)}w^Tx^{(i)} -b\sum_{i=1}^N \alpha_i y^{(i)} + \sum_{i=1}^N \alpha_i
        \end{align*}
        Using the KKT conditions and substitute $w$, we can replace the term $w^Tw$ by
        \[ w^Tw = \left(\sum_{i=1}^N \alpha_i y^{(i)}x^{(i)}\right)^T\left(\sum_{i=1}^N \alpha_i y^{(i)}x^{(i)}\right) = \sum_{i=1}^N\sum_{j=1}^N \alpha_i\alpha_j y^{(i)}y^{(j)} x^{(i)T}x^{(j)} \]
        and similarly the second term can be replaced by
        \[ -\sum_{i=1}^N \alpha_iy^{(i)}w^Tx^{(i)} = -\sum_{i=1}^N\sum_{j=1}^N \alpha_i\alpha_j y^{(i)}y^{(j)} x^{(i)T}x^{(j)} \]
        Therefore the Lagrangian dual function is
        \[ \phi(\alpha) = \sum_{i=1}^N\alpha_i - \frac{1}{2}\sum_{i=1}^N\sum_{j=1}^N \alpha_i\alpha_j y^{(i)}y^{(j)} x^{(i)T}x^{(j)} \]
        So we derived the dual problem of hard-margin SVM
        \begin{align*}
            \max \quad &\phi(\alpha) = \sum_{i=1}^N\alpha_i - \frac{1}{2}\sum_{i=1}^N\sum_{j=1}^N \alpha_i\alpha_j y^{(i)}y^{(j)} x^{(i)T}x^{(j)}\\
            \text{s.t.} \quad & \sum_{i=1}^N \alpha_i y^{(i)} = 0\\
            \quad & \alpha_i \ge 0
        \end{align*}
        \begin{remark}
            This problem an be solved efficiently (by computers) using the SMO algorithm.
        \end{remark}

\section{Soft Margin SVM}
    \subsection{Slac Variables}
        If the data is not linear separable, we can introduce slack variables $\xi_i$.

        So the primal problem is
        \begin{align*}
            \min \quad & \frac{1}{2}w^Tw + C\sum_{i=1}^N\xi_i\\
            \text{s.t.} \quad &y^{(i)}(w^Tx^{(i)} + b) \ge 1 - \xi_i\\
            \quad & \xi_i \ge 0
        \end{align*}
        where $C$ is a user-defined hyperparameter.
        \begin{remark}
            $\xi_i$ implies the location of $x^{(i)}$
            \begin{itemize}
                \item $\xi_i = 0$: Data point outside region of separation and correcty separated.
                \item $0 < \xi_i \le 1$: Data point inside region of separation but on the correct side of hyperplane.
                \item $\xi_i > 1$: Data point inside region of separation and on the wrong side of hyperplane.
            \end{itemize}
            The choice of $C$ is  related to the performance of the model
            \begin{itemize}
                \item Large $C$: hard to train, but likely to have high performance
                \item Small $C$: easy to train, but likely to have poor performance
            \end{itemize}
        \end{remark}

    \subsection{Dual Formulation}
    The Lagrangian is
    \[ \mathcal{L}(w,b,\xi,\alpha,\beta) = \frac{1}{2}w^Tw + C\sum_{i}\xi_i - \sum_i \alpha_i(y^{(i)}(w^Tx^{(i)})-1+\xi_i) - \sum_i \beta_i\xi_i \]
    and the KKT Conditions are
    \[
    \begin{cases}
        \nabla_w\mathcal{L} = 0, \quad \nabla_b\mathcal{L} = 0, \quad \nabla_{\xi_i}\mathcal{L} = 0\\
        (y^{(i)}(w^Tx^{(i)})-1+\xi_i) \ge 1-\xi_i, \quad \xi_i \ge 0\\
        \alpha_i(y^{(i)}(w^Tx^{(i)})-1+\xi_i) = 0, \quad \beta_i\xi_i = 0
    \end{cases}
    \]
    Notice that the introduction of $\xi_i$ doesn't affect the expression of $\nabla_w\mathcal{L}$ and $\nabla_b\mathcal{L}$, and therefore
    \[ w = \sum_i \alpha_iy^{(i)}x^{(i)} \]
    \[ \sum_i \alpha_i y^{(i)} = 0 \]
    still hold.

    In addition we now have
    \[ \nabla_{\xi_i}\mathcal{L} = C - \alpha_i - \beta_i = 0 \]

    After some algebra
    \begin{align*}
        \phi(\alpha, \beta) &= \inf\mathcal{L} = \frac{1}{2}w^Tw - \sum_i \alpha_i(y^{(i)}(w^Tx^{(i)})-1)
    \end{align*}

    Notice that $\alpha_i + \beta_i = C$ can be replaced by $\alpha_i \le C$, therefore
    \begin{align*}
        \max \quad &\phi(\alpha) = \sum_{i=1}^N\alpha_i - \frac{1}{2}\sum_{i=1}^N\sum_{j=1}^N \alpha_i\alpha_j y^{(i)}y^{(j)} x^{(i)T}x^{(j)}\\
        \text{s.t.} \quad & \sum_{i=1}^N \alpha_i y^{(i)} = 0\\
        \quad & 0 \le \alpha_i \le C
    \end{align*}

\section{Kernel Tricks}
    If the data is not linear separable, we can transform data into higher dimension for separation. As long as the mapping is non-linear and the target dimension is high enough, we can always make the data linear separable.

    Suppose we want to find a function $\varphi(x)$ that maps $x$ into a higher dimension. So the decision hyperplane becomes
    \[ y = w^T\varphi(x) + b \]
    where $\varphi(\cdot)$ is called the kernel function.
    The mathematical framework does not change, and all we have to do is to replace $x$ with $\varphi(x)$ in all previous derivations.
    \[ w = \sum_i \alpha_i y^{(i)}\varphi(x^{(i)}) \]
    \[ \phi(\alpha) = \sum_{i=1}^N\alpha_i - \frac{1}{2}\sum_{i=1}^N\sum_{j=1}^N \alpha_i\alpha_j y^{(i)}y^{(j)} \varphi^T(x^{(i)})\varphi(x^{(j)}) \]

    While it is sometimes hard to determin $\varphi(\cdot)$, it is usually easier to define ``result'' of the inner product $\varphi^T(x)\varphi(x)$, denoted by $K(x_i, x_j)$ For example
    \[ K(x_i, x_j) = \varphi^T(x)\varphi(x) = \exp\left\{ -\frac{\|x_i-x_j\|^2}{\sigma^2} \right\} \]

    Notice that in the dual problem, all terms related with $x$ comes in pairs $(x_i,x_j)$, so we can always replace the pair with $K(x_i,x_j)$

    \begin{remark}
        Although
        \[ w = \sum_i \alpha_i y^{(i)}\varphi(x^{(i)}) \]
        notice that $w$ is used to compute $w^T \varphi(x)$, and therefore
        \[ y = w^T\varphi(x) + b = \sum_i \alpha_i y^{(i)}\varphi^T(x^{(i)})\varphi(x) = \sum_i \alpha_i y^{(i)} K(x^{(i)}, x) + b \]
        However this operation increases complexity. \textbf{EXPENSIVE!}
    \end{remark}

    \begin{remark}
        Not all functions can be used as kernels, a kernel matrix $K$ must satisfy the \textbf{Mercer's Condition}: $K$ is positive semi-definite.
    \end{remark}


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}
\usepackage{bm}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\expec}{E}

\begin{document}
\title{\textsf{Homework 3 for Pattern Recognition}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 1}
{
    \[
    \begin{split}
        &   \frac{\partial}{\partial \hat{\theta}} R(\hat{\theta} | \bm{X}) \\
        =&  \frac{\partial}{\partial \hat{\theta}} \int_{\theta} {(\hat{\theta}-\theta)^2 p(\theta | \bm{X}) \mathrm{d}\theta} \\
        =&  \int_{\theta} {\frac{\partial}{\partial \hat{\theta}} (\hat{\theta}-\theta)^2 p(\theta | \bm{X}) \mathrm{d}\theta} \\
        =&  \int_{\theta} {2 (\hat{\theta}-\theta) p(\theta | \bm{X}) \mathrm{d}\theta} \\
        =&  \int_{\theta} {2 \hat{\theta} p(\theta | \bm{X}) \mathrm{d}\theta} - \int_{\theta} {2 \theta p(\theta | \bm{X}) \mathrm{d}\theta} \\
        =&  \hat{\theta} \cdot \int_{\theta} {2 p(\theta | \bm{X}) \mathrm{d}\theta} - \int_{\theta} {2 \theta p(\theta | \bm{X}) \mathrm{d}\theta} \\
        =&  \hat{\theta} \cdot 2 - \int_{\theta} {2 \theta p(\theta | \bm{X}) \mathrm{d}\theta} = 0.
    \end{split}
    \]

    The solution to the equation above is $$\hat{\theta} = \int_{\theta} {\theta p(\theta | \bm{X}) \mathrm{d}\theta}.$$

    Note that $$\frac{\partial^2}{\partial \hat{\theta}^2} R(\hat{\theta} | \bm{X}) = 2 > 0,$$ which means the solution above is the global minimum for $\hat{\theta}$. Thus, we have $$\theta^{*} = \int_{\theta} {\theta p(\theta | \bm{X}) \mathrm{d}\theta}.$$

}

\section*{Question 2}
{
    \subsection*{(1)}
    {
        To put it simply, the MLE (maximum likelihood estimation) is a specific branch of Bayesian estimation. Let's review each of them before reaching this conclusion.

        \begin{itemize}
            \item \textbf{Bayesian estimate} \quad A prior distribution for the parameter $\theta$ is required, and we can get the posterior distribution for $\theta$, rather than the estimator alone. This means it provides us with more knowledge about the posterior distribution of the paramter. We can use multiple ways to make the point estimate, such as using the mean, median, or the mode of the posterior distribution as an estimator for the parameter. However, the prior distribution is a bit subjective, and we have to use some carefully designed priors (such as \emph{conjugate prior family} and \emph{noninformative priors}) to prevent arbitrariness.

            \item \textbf{the Maximum likelihood estimate (MLE)} \quad It maximizes the likelihood only, which means we have a prior distribution proportional to constant in the Bayesian formula $p(\theta|x) \propto p(\theta)p(x|\theta)$. Instead of estimating the posterior distribution for $\theta$, we obtain only one estimate of it, i.e. the maximum likelihood estimate. It is equivalent to the Bayesian method if the prior distribution is uniform, and we use the mode of the posterior in Bayesian method.

        \end{itemize}
    }

    \subsection*{(2)}
    {
        \[
        \begin{split}
                &   \frac{\partial}{\partial \mu} \log{(p(\mu | \sigma^2, X_1, \cdots, X_n))} \\
                =&  \frac{\partial}{\partial \mu} \left[ -\frac{n}{2} \log{(2\pi)} - \frac{n}{2} \log{(\sigma^2)} - \sum_{i=1}^{n} {\frac{(X_i - \mu)^2}{2\sigma^2}} \right] \\
                =&  \frac{1}{2\sigma^2} \sum_{i=1}^{n} {\frac{\partial}{\partial \mu} (X_i - \mu)^2} = \frac{1}{2\sigma^2} \sum_{i=1}^{n} {\left[ 2 (X_i - \mu) \right]} \\
                =&  \frac{1}{\sigma^2} \left[ \sum_{i=1}^{n} {X_i} - n \mu \right] = 0.
        \end{split}
        \]

        The solution is $$\mu = \frac{1}{n} \sum_{i=1}^{n} X_i = \bar{X}.$$

        Note that $$\frac{\partial^2}{\partial \mu^2} \log{(p(\mu | \sigma^2, X_1, \cdots, X_n))} = - \frac{n}{\sigma^2} < 0,$$ we have $\mu = \bar{X}$ is the global maximizer of the log likelihood of $\mu$ given $\sigma^2$ and $X_1, \cdots, X_n$.

        Given that $\mu > \mu_0$, the maximum likelihood estimate of $\mu$ is $\max{(\mu_0, \bar{X})}$.
    }

    \subsection*{(3)}
    {
        From $$\frac{\partial}{\partial \sigma^2} \log{(p(\mu | \sigma^2, X_1, \cdots, X_n))} = \frac{\partial}{\partial \sigma^2} \left[ -\frac{n}{2} \log{(2\pi)} - \frac{n}{2} \log{(\sigma^2)} - \sum_{i=1}^{n} {\frac{(X_i - \mu)^2}{2\sigma^2}} \right] $$$$= -\frac{n}{2\sigma^2} + \frac{1}{2(\sigma^2)^2} \sum_{i=1}^{n} {(X_i -\mu)^2} = 0,$$
        and $$\frac{\partial^2}{\partial (\sigma^2)^2} \log{(p(\mu | \sigma^2, X_1, \cdots, X_n))} = \frac{n}{2\sigma^6}(\sigma^2 - 2 S^2) < 0,$$ we get the maximum likelihood estimate for $\sigma^2$, that is
        $$\hat{\sigma}^2 = S^2 = \frac{1}{n} \sum_{i=1}^{n} {(X_i - \bar{X})^2}.$$

        Below are the density plots with estimated parameters:

        \begin{figure}[H]
            \centering
            \includegraphics[width = 0.6\linewidth]{normal_10.png}
            \includegraphics[width = 0.6\linewidth]{normal_100.png}
            \includegraphics[width = 0.6\linewidth]{normal_1000.png}
            \caption{Density plots with estimated parameters for normal samples}
        \end{figure}
    }

    \subsection*{(4)}
    {
        Below are the density plot with estimated parameters for uniform samples:

        \begin{figure}[H]
            \centering
            \includegraphics[width = 0.6\linewidth]{uniform_100.png}
            \caption{Density plot with estimated parameters for uniform samples}
        \end{figure}
    }

    \subsection*{(5)}
    {
        \begin{itemize}
            \item \textbf{Model selection} \quad A good prior model is essential to the MLE. The selected model should be of the distribution family that is similar to that of the data samples. By contrast, we see the normal model is not successful in subsection (4), where the sample is drawn from a distribution different from our normal model.

            \item \textbf{Sample size} \quad We find in subsection (3) that the larger the sample size, the more accurate the estimated distribution. From the Bayesian perspective, a larger sample size means more weight put on the observed data, than that on the prior knowledge. Although it is not exactly precise an interpretation here (as we do not have a prior distribution for the normal likelihood), it is obvious that more observations contribute to the model accuracy, if the model is reasonable.

        \end{itemize}
    }
}

\section*{Source Code}
{
    Please download the souece code from http://39.106.23.58/files/PR3\_2015011506.7z
}

\clearpage
\end{document}


\chapter{Common Polynomial Products}

In math and physics, you will run into certain kinds of polynomials
over and over again. In this chapter, I am going to cover some
patterns that you will want to start to recognize.

\section{Difference of squares}

Watch \textbf{Polynomial special products: difference of squares} from Khan Academy at \url{https://youtu.be/uNweU6I4Icw}.

If you are asked what is $(3x - 7)(3x + 7)$, you would use the
distributive property to expand that to $(3x)(3x) + (3x)(7) + (-7)(3x) + (-7)(7)$.
Two of the terms cancel each other, so this is $(3x)^2 - (7)^2$. This would simplify to $9x^2 - 49$

You will see this pattern a lot. Anytime you see $(a + b)(a - b)$, you should immediately
recognize it equals $a^2 - b^2$. (Note that the order doesn't matter: $(a - b)(a + b)$ also $a^2 - b^2$.)

Working the other way is important too: anytime you see $a^2 - b^2$,
that you should recognize that you can change that into the product
$(a + b)(a - b)$. Making something into a product like this is known as
\emph{factoring}. You probably have done prime factorization of
numbers like $42 = 2 \times 3 \times 7$. In the next couple of
chapters you will learn to factorize polynomials.

\begin{Exercise}[title={Difference of Squares}, label=diffsquares]
  Simply the following products
  \Question{$(2x - 3)(2x + 3)$}
  \Question{$(7 + 5x^3)(7 - 5x^3)$}
  \Question{$(x - a)(x + a)$}
  \Question{$(3 - \pi)(3 + \pi)$}
  \Question{$(-4x^3 + 10)(-4x^3 - 10)$}
  \Question{$(x + \sqrt{7})(x - \sqrt{7})$}
  Factor the following polynomials:
    \Question{$x^2 - 9$}
    \Question{$49 - 16x^6$}
    \Question{$\pi^2 - 25x^8$}
    \Question{$x^2 - 5$}
\end{Exercise}
\begin{Answer}[ref=diffsquares]
  $(2x - 3)(2x + 3) = 4x^2 - 9$

  $(7 + 5x^3)(7 - 5x^3) = 49 - 25x^6$

  $(x - a)(x + a) = x^2 - a^2$

  $(3 - \pi)(3 + \pi) = 9 - \pi^2$

  $(-4x^3 + 10)(-4x^3 - 10) = 16x^6 - 100$

  $(x + \sqrt{7})(x - \sqrt{7}) = x^2 - 7$

  $x^2 - 9 = (x + 3)(x - 3)$

  $49 - 16x^6 = (7 + 4x^3)(7 + 4^3)$

  $\pi^2 - 25x^8 = (\pi + 5x^4)(\pi - 5x^4)$

  $x^2 - 5 = (x + \sqrt{5})(x - \sqrt{5})$

\end{Answer}

We are often interested in the roots of a polynomial, that is we want
to know ``For what values of $x$ does the polynomial evaluate to
zer?'' For example, when you deal with falling bodies, the first
question you might ask would be ``How many seconds before the hammer
hits the ground?''  Once you have factored a polynomial into
binomials, you can easily find the roots.

For example, what are the roots of $x^2 - 5$? You just factored it
into $(x + \sqrt{5})(x - \sqrt{5})$ This product is zero if and only
if one of the factors is zero. The first factor is only zero when $x$
is $-\sqrt{5}$. The second factor is zero only when $x$ is
$\sqrt{5}$. Those are the only two roots of this
polynomial.

Let's check that result. $\sqrt{5}$ is a little more than 2.2.  Using
your Python code, you can graph the polynomial:
\begin{Verbatim}
import poly.py
import matplotlib.pyplot as plt

# x**2 - 5
pn = [-5.0, 0.0, 1.0]

# These lists will hold our x and y values
x_list = []
y_list = []

# Start at x=-3
current_x =-3.0

# End at x=3.0
while current_x < 3.0:
    current_y = poly.evaluate_polynomial(pn, current_x)

    # Add x and y to respective lists
    x_list.append(current_x)
    y_list.append(current_y)

    # Move x forward
    current_x += 0.1

# Plot the curve
plt.plot(x_list, y_list)
plt.grid(True)
plt.show()
\end{Verbatim}

You should get a plot like this:

\includegraphics[width=\textwidth]{sqrt5.png}

It does, indeed, seem to cross the x-axis near -2.2 and 2.2.

\section{Powers of binomials}

You can raise whole polynomials to exponents. For example,
\begin{multline*}
  (3x^3 + 5)^2 = (3x^3 + 5)(3x^3 + 5) \\ = 9x^6 + 15x^3 + 15x^3 + 25 = 9x^6 + 30x^3 + 25
\end{multline*}

A polynomial with two terms is called a \emph{binomial}. $5x^9 - 2x^4$,
for example, is a binomial. In this section, we are going to
develop some handy techniques for raising a binomial to some power.

Looking at the previous example, you can see that for any monomials $a$ and $b$, $(a + b)^2 = a^2 + 2ab + b^2$.
So, for example, $(7x^3 + \pi)^2 = 49x^6 + 14\pi x^3 + \pi^2$

\begin{Exercise}[title={Squaring binomials}, label=squaringbinomials]
  Simply the following
  \Question{$(x + 1)^2$}
  \Question{$(3x^5 + 5)^2$}
  \Question{$(x^3 - 1)^2$}
  \Question{$(x - \sqrt{7})^2$}

\end{Exercise}
\begin{Answer}[ref=squaringbinomials]
  $(x+1)^2 = x^2 + 2x + 1$

  $(3x^5 + 5)^2 = 9x^10 + 30x^5 + 25$

  $(x^3 - 1)^2 = x^6 - 2x^3 + 1$

  $(x - \sqrt{7})^2 = x^2 - 2x\sqrt{7} + 7$
\end{Answer}

What about $(x + 2)^3$? You can do it as two separate multiplications:
\begin{multline*}
  (x+2)^3 = (x+2)(x+2)(x+2) \\
  = (x + 2)(x^2 + 4x + 4) = x^3 + 4x^2 + 4x + 2x^2 + 8x + 8 \\
  = x^3 + 6x^2 + 12x + 8
\end{multline*}
And, in general, we can say that for any monomials $a$ and $b$, $(a + b)^3 = a^3 + 3a^2b + 3ab^2 + b^3$.

What about higher powers? $(a + b)^4$, for example? You could use the
distributive property four times, but it starts to get pretty tedious.

Here is a trick. This is known as \emph{Pascal's triangle}
\begin{equation*}
\begin{array}{c}
 1 \\
 1 \quad 1 \\
 1 \quad 2 \quad 1 \\
 1 \quad 3 \quad 3 \quad 1 \\
 1 \quad 4 \quad 6 \quad 4 \quad 1 \\
 1 \quad 5 \quad 10 \quad 10 \quad 5 \quad 1 \\
 1 \quad 6 \quad 15 \quad 20 \quad 15 \quad 6 \quad 1 \\
 1 \quad 7 \quad 21 \quad 35 \quad 35 \quad 21 \quad 7 \quad 1 \\
 \ldots
\end{array}
\end{equation*}
Each entry is the sum of the two above it.

The coefficients of each term are given by the entries in Pascal's triangle:
\begin{equation*}
(a + b)^4 = 1a^4 + 4a^3b + 6a^2 b^2 + 4 a b^3 + 1 b^4
\end{equation*}

\begin{Exercise}[title={Using Pascal's Triangle}, label=pascalbinomial]
    \Question{What is $(x + \pi)^5$?}
\end{Exercise}
\begin{Answer}[ref=pascalbinomial]
  $(x + \pi)^5 = x^5 + 5\pi x^4 + 10\pi^2 x^3 + 10 \pi^3 + x^2 + 5 \pi^2 x + \pi^5$
\end{Answer}


\section{Basic probability}
\section{Basic Probability Theory}
\begin{itemize}
\item Outcome --- Set of possible outcomes
\item Event  --- Subset of possible outcomes, an event is something which can happen or not happen
\item Distribution  ---  measure on the outcome space, just give the probability of each outcome
\item Independence  --- "Events that are unrelated", depends on the distribution
\end{itemize}

\subsection{Discrete Examples}
\begin{itemize}
\item Coin Flip:
        \begin{itemize}
        \item Outcomes: $\left\{ H, T\right\}$
        \item Event:  If we toss a coin once, there are two possible events: $\{H\}$ and $\{T\}$; If we toss a coin twice, the event that the first toss is heads is $A = \{HH,HT\}$;  If we toss a coin three times, the event that the second toss is heads is $B = \{HHH, THT,HHT, THT\}$.
        \item Distribution: If we toss a coin, the possibility that it is H and the possibility that it is T are the same, that is $p(\left\{H  \right\})=p(\left\{ T \right\})=0.5$.%, $p(\left\{ H \right\})=0.2, p(\left\{  T\right\})=0.8$
        \end{itemize}
\item Rolling a Die:
        \begin{itemize}
        \item Outcomes:$\left\{ 1, 2, 3, 4, 5, 6\right\}$
        \item Event: some examples\newline
        		$E_{1}=\left\{ 3\right\}$--- rolling a 3\newline
                 $E_{2}=\left\{ 1, 3, 5\right\}$ --- rolling an odd number\newline
                 $E_{3}=\left\{ 2, 4, 6\right\}$ --- rolling an even number \newline
                 $E_{4}=\left\{ 1, 2\right\} $--- rolling either a 1 or 2
        \item Distribution: $p(\left\{1 \right\})=p(\left\{ 2 \right\})=...=p(\left\{ 6 \right\})= \frac{1}{6}$(most common case)\newline
        $p(\left\{ 1\right\})=\frac{1}{2}, p(\left\{ 2 \right\})=...=p(\left\{ 6 \right\})=\frac{1}{10}$ (for some special die)
        \item Independence: Given events $E_{1}$ and $E_{2}$, they are independent if \newline
        $p(E_{1} \bigwedge E_{2})=p(E_{1})p(E_{2})$(the probability that both events appear is equal to the product of the probability that one event appears)\newline
        $p(E_{1}|E_{2})=\frac{p(E_{1} \bigwedge E_{2})}{p(E_{2})}=p(E_{1})$ (the conditional probability of $E_1$ given the event $E_{2}$ is equal to the probability of $E_1$)
        \item example of independence: The die roll 2 and 3 are independent; Events $E_2$ and $E_4$ are also independent:\newline
        $E_{2}  \bigwedge E_{4}=\left\{1,3,5\right\} \bigwedge \left\{1,2\right\} =\left\{1\right\}$\newline
        $p(E_{2} \bigwedge E_{4})=p(\left\{1\right\})=\frac{1}{6}$\newline
        $p(E_{2}) p(E_{4})=p(\left\{1,3,5\right\})p(\left\{1,2\right\})=\frac{1}{2}\frac{1}{3}=\frac{1}{6}$
        \end{itemize}
\end{itemize}

\subsection{Independent Copies}
\begin{itemize}
\item Coin Flip: 2 independent copies:\newline
$C_{1}=\left\{H,T\right\}, C_{2}=\left\{H,T\right\}\Rightarrow C_{1}\times C_{2}$\newline
\begin{tabular}{c|c|c}
	& $H^{\operatorname{P_{H}}}$ & $T^{P_{T}}$ \\
	\hline$P_{H} H$ & $H H$ & $H T$ \\
	\hline$P_{T}T$ & $T H$ & $T T$
\end{tabular}
  \begin{itemize}
      \item Event: $\left\{HT,HT\right\} \Rightarrow $ exactly one head \newline
      $\left\{HT,HH,TH\right\} \Rightarrow $at least one head
      \item Distribution: $p((H,T))=p_{1}(H)p_{2}(T)$
      \item Generalized: K-coins \textbf{independent}. $\left\{H,T\right\}^{K}=\left\{K-tuples \ of  \  H,T\right\}$ \newline
      Distribution. $p((H,...,T))=p_{1}(H)...p_{k}(T)$ Can generalize this to infinite protucts as well. (Doesn't hold for dependent events)
   \end{itemize}
\end{itemize}

\subsection{Continuous Distributions}
\begin{itemize}
	\item Ex: Uniform Distribution on $\left[ 0,1\right]$ .\newline
	Outcomes: $\left[ 0,1\right]$ \newline
	Events: Ex: the event $\left[ 0,0.5\right]$ means random number $\leq$ 0.5 \newline
	            Ex: the event $\left[ 0,0.25\right] \bigcup \left[ 0.5,0.75\right]$ means random number less than $0.025$ or between $0.5$ and $0.75$\newline
 \begin{figure}[ht!]
	\centering
	\includegraphics[width=.35\textwidth]{../figures/probability1.png}
\end{figure}

	\item Distribution: \newline
	A distribution a rule which gives the probability of any event. \newline
	Properties to satisfy: if events $E_{1}, E_{2}, ..., E_{n},...  \mbox{satisfy that}  \ E_{i} \bigcap E_{j}= \varnothing $, then
	\begin{enumerate}
	\item $ p(E_{1} \bigcup ... \bigcup E_{n} \bigcup ...) = \sum_{i=1}^{\infty}p(E_{j})$
	\item $p(E)=\int_{E} d_{x}$, "length of E"
	\item examples 1: $E_{1}=\left[ 0,0.5\right] $
	$$
	P\left(E_{1}\right)=\int_{E_{1}} d_{x}=\int_{0}^{1 / 2} d x=\frac{1}{2}.
	$$
	\item examples 2: $E_{2}=\left[ 0,0.25\right] \bigcup \left[ 0.5,0.75\right]$
	$$p\left(E_{2}\right)=\int_{E_{2}} d x=\int_{0}^{1 / 4} d x+\int_{1 / 2} ^{3/4}d x=\frac{1}{4}+\frac{1}{4}=\frac{1}{2}
	$$
	\end{enumerate}
 \begin{figure}
	\centering
	\includegraphics[width=.7\textwidth]{../figures/probability2.jpeg}
\end{figure}
\item Independence: \newline
Two events $E_{1}, E_{2}$ are independent if \newline
$$p\left(E_{1} \cap E_{2}\right)=p\left(E_{1}\right) p\left(E_{2}\right)$$
For example,
$E_{1} \cap E_{2}=[0,0.5] \cap\left(\left[0, \frac{1}{4}\right] \cup\left[\frac{1}{2}, \frac{3}{4}\right]\right)$
$$
p\left(E_{1}\bigcap E_{2}\right)=\frac{1}{4}=\frac{1}{2} \cdot \frac{1}{2}=p\left(E_{1}\right) p\left(E_{2}\right)
$$

Frist two binary digits are like two independent coin flips. Every binary digit is like an independent coin flip, so we ca think of the random number as being an infinite sequence of coin flips. In general, we'll consider distribution defined by a probability density function p(x). The probability of an event is given by	$P\left(E\right)=\int_{E_{1}} p(x)d_{x}$
\begin{itemize}
\item Outcomes: $\left[ 0,1\right]$
\item Density function: $p(x)=1$ ($\int_0^1 p(x)=1$)
\end{itemize}
\end{itemize}

\subsection{Gaussian/ Normal Distribution}
 \begin{figure}[ht!]
	\centering
	\includegraphics[width=.7\textwidth]{../figures/probability3.png}
\end{figure}
\begin{itemize}
	\item Outcomes:
	\item Events: Subsets of R
	\item Density function: $P(x)=\frac{1}{\sqrt{2\pi}} e^{-x^{2} / 2}$\newline
	         Means that for any event $E$,
	         $$
	         p\left(E\right)=\int_{E} \frac{1}{\sqrt{2 \pi}} e^{-x^{2} / 2} d x.
	         $$

	 \item Cummulative distribution Function
	 $$
	 F(x)=\int_{-\infty}^{x} p(t) d t=p(t<x)
	 $$
	 Note that $\lim_{x\rightarrow \infty} F(x)=1$.
\end{itemize}

\section{Random, Variable, Mean, Variance}
\begin{itemize}
\item Recall:
\begin{itemize}
	\item Set if outcomes:  $\Omega$
	\item Event: Subset of outcomes: $E$
	\item Probability Distribution: $p(x)$
\end{itemize}
\subsection{Random Variable}
\item Defin: A random variable X is a function $X: \Omega \rightarrow S$ Here $ S=R, R^{d}$, but could be arbitrary.
 \item Ex: Rolling a die:
 \begin{itemize}
 	\item Outcomes: $\Omega={1,2,...,6}$
 	\item Events are subsets of $\Omega$
 	\item Distribution: $p(1)=p(2)=...=p(6)=\frac{1}{6}$
 	Suppose we roll the die, then if the die comes up d times, you win d dollars, minus 1 dollar if it's even. The amount you win is a random variable. \newline
 	$\begin{array}{l}
 		X_{1}: \Omega \rightarrow R\\
 		X_{1}(1)=1 \\
 		X_{1}(2)=1 \\
 		X_{1}(3)=3 \\
 		X_{1}(4)=3 \\
 		X_{1}(5)=5 \\
 		X_{1}(6)=5
 	\end{array}$ \newline
 	Can define multiple random variables on a single outcome space $\Omega$.
 \end{itemize}
\item Ex: Rolling a die. If the die comes up d times, you get d dollars. If d is even, then you give a dollar to your friend.\newline
$X_{1} \leftarrow$ your winnings,
$X_{2}\leftarrow$ your friends winnings. \newline
$$
\begin{aligned}
	X_{2}: \Omega \rightarrow \mathbb{R}, & X_{2}(1)=0, \quad X_{2}(2)=1 \\
	& X_{2}(3)=0, \quad X_{2}(4)=1 \\
	& X_{2}(5)=0, \quad X_{2}(6)=1
\end{aligned}
$$
From here on out $\Omega$ will be fixed, we talk about different random variables on $\Omega$.
\subsection{Mean of  random variable}
\item Defin: Mean of a random variable $X$. \newline
Expectation of $X$:
$$
\mathbb{E}[X]=\sum_{\omega \in \Omega} p(\omega) X(\omega)\left(=\int_{\Omega} X(\omega) p(\omega) d \omega\right)
$$
\item Ex:
 $$\begin{aligned}
& \mathbb{E}\left[X_{1}\right]=\frac{1}{6}(1+1+3+3+5+5)=3 \\
& \mathbb{E}\left[X_{2}\right]=\frac{1}{6}(0+1+0+1+0+1)=\frac{1}{2}
\end{aligned}
$$
\subsection{Variance of  Random Variables}
\item Defin: Variance of a Random Variable $X$:
$$
V[X]=\mathbb{E}\left[X^{2}\right]-\mathbb{E}[X]^{2}=\mathbb{E}\left[(X-\mathbb{E}[X])^{2}\right]
$$
\item Ex:
$$
\begin{array}{l} V\left[X_{1}\right]=\mathbb{E}\left[X_{1}^{2}\right]-(3)^{2}\\
	= \frac{1}{6}\left(1^{2}+1^{2}+3^{2}+3^{2}+5^{2}+5^{2}\right)-9\\
	=\frac{1}{3}(1+9+25)-9\\
	=\frac{35}{3}-9=\frac{8}{3}\\
\end{array}$$ \newline
Variance measures "how much $X$ deviates from it's average".

 \begin{figure}
	\centering
	\includegraphics[width=.35\textwidth]{../figures/probability4.png}
\end{figure}

\subsection{Independenve of Random variables}
\item Defin:  Two random variables $X_{1},X_{2}$ are independent if for any $ \alpha, \beta,$
$E_{1}=\left\{w: X_{1}(\omega)<\alpha\right\}, E_{2}=\left\{w: X_{2}(\omega)<\beta\right\}$ are independent events.
\item Ex: $X_{1} , X_{2}$ are independent: if $(\alpha, \beta) \quad \alpha=4, \quad \beta=\frac{1}{2}$,
$$
\begin{array}{l}
E_{1}=\left\{w: X_{1}(u)<4\right\}=\{1,2,3,4\} \\
E_{2}=\left\{w: X_{2}(\omega)<\frac{1}{2}\right\}=\{1,3,5\}
\end{array}
$$
$$
\begin{array}{l}
p\left(E_{1}\bigcap E_{2}\right)=p\left(E_{1}\right) p\left(E_{2}\right) \\
p(\{1,3\})=p\left(E_{1}\right) p\left(E_{2}\right)
\end{array}
$$

\subsection{Properties of E, V, Independence}
\begin{itemize}
	\item If $X_{1}, X_{2}$ are random variable, then
	$$
	\left(a_{1}X_{1}+a_{2} x_{2}\right)(w)=a_{1} x_{1}(w)+a_{2} x_{2}(w)
	$$
	$$\mathbb{E}\left[a_{1}X_{1}+a_{2} X_{2}\right]=a_{1}\mathbb{E}\left[X_{1}\right]+a_{2} \mathbb{E}\left[X_{2}\right]
	$$
	\item If $X_{1}, X_{2}$ are independent random variables, then
	$$
	\mathbb{E}\left[X, X_{2}\right]=\mathbb{E}\left[X_{1}\right] \mathbb{E}\left[X_{2}\right]
	$$
	\item From this, we get if $X_{1}, X_{2}$ are independent,
	$$
	V\left[X_{1}+X_{2}\right]=V\left[X_{1}\right]+V\left[X_{2}\right]
	$$
	$$V\left[a_{1} x_{1}+a_{2} x_{2}\right]=a_{1}^{2} V\left[x_{1}\right]+a_{2}^{2} V\left[x_{2}\right]
	$$
\end{itemize}

\end{itemize}

%
%\includepdf{HandWrittenNotes/Probability1.pdf}
%\includepdf{HandWrittenNotes/Probability2.pdf}
%\includepdf{HandWrittenNotes/Probability3.pdf}
%\includepdf{HandWrittenNotes/Probability4.pdf}
%\includepdf{HandWrittenNotes/Probability5.pdf}
%\includepdf{HandWrittenNotes/Probability6.pdf}
%\includepdf{HandWrittenNotes/Probability7.pdf}
%\includepdf{HandWrittenNotes/Probability8.pdf}
%\includepdf{HandWrittenNotes/Probability9.pdf}


\subsection{Cauchy-Riemann equations}

Consider complex number z=x+iy

A function on this gives:

\(f(z)=u+iv\)

Take the total differential of :

\(df/dz=\dfrac{\delta f}{\delta z}+\dfrac{\delta f}{\delta x}\dfrac{dx}{dz}+\dfrac{\delta f}{\delta y}\dfrac{dy}{dz}\)

We know that:

\begin{itemize}
\item \(\dfrac{dx}{dz}=1\)
\item \(\dfrac{dy}{dz}=-i\)
\end{itemize}

We can see from this that

\begin{itemize}
\item \(\dfrac{du}{dx}=\dfrac{dv}{dy}\)
\item \(\dfrac{du}{dy}=-\dfrac{dv}{dx}\)
\end{itemize}

These are the Cauchy-Riemann equations


\subsection{Orthogonal diagonalization}

We begin this section by recalling some important definitions. Recall from Definition~\ref{def:orthogonal-set} that non-zero vectors are called orthogonal if their dot product equals $0$.  A set is orthonormal if it is orthogonal and each vector is a unit vector.

An orthogonal matrix $U$, from Definition~\ref{def:orthogonal-matrices}, is one in which $UU^{T} = I$. In other words, the transpose of an orthogonal matrix is equal to its inverse. A key characteristic of orthogonal matrices, which will be essential in this section, is that the columns of an orthogonal matrix form an orthonormal set.

We now recall another important definition.

\begin{definition}{Symmetric and antisymmetric matrices}{symmetric-antisymmetric-matrices}
A real $n\times n$-matrix $A$, is \textbf{symmetric }if $A^{T}=A$. If $%
A=-A^{T}$, then $A$ is called \textbf{antisymmetric. }
\index{symmetric matrix}
\end{definition}

Before proving an essential theorem, we first examine the following lemma which will be used below.

\begin{lemma}{The dot product}{dot-product-lemma}
Let $A=\mat{a_{ij} }$ be a real symmetric $n \times n$-matrix, and let $\vect{x}, \vect{y} \in \R^n$. Then
\[
A\vect{x} \dotprod \vect{y} = \vect{x} \dotprod A \vect{y}
\]
\end{lemma}

\begin{proof}
This result follows from the definition of the dot product together with properties of matrix multiplication, as follows:
\begin{eqnarray*}
A\vect{x} \dotprod \vect{y} &=& \sum_{k,l}a_{kl}x_{l}y_{k} \\
&=&\sum_{k,l} (a_{lk})^Tx_{l}y_{k} \\
&=& \vect{x}\dotprod A^{T}\vect{y} \\
&=& \vect{x}\dotprod A \vect{y}
\end{eqnarray*}

The last step follows from $A^T = A$, since $A$ is symmetric.
\end{proof}

We can now prove that the eigenvalues of a real symmetric matrix are real numbers.  Consider the following important theorem.

\begin{theorem}{Orthogonal eigenvectors}{symmetric-orthogonal-eigenvectors}
Let $A$ be a real symmetric matrix. Then the eigenvalues of $A$ are real numbers and eigenvectors corresponding to distinct eigenvalues are orthogonal.
\end{theorem}

\begin{proof}
Recall that for a complex number $a+ib$, the complex conjugate, denoted by $
\conjugate{a+ib}$ is given by $\conjugate{a+ib}=a-ib$. The
notation, $\conjugate{\vect{x}}$ will denote the vector which has every
entry replaced by its complex conjugate.

Suppose $A$ is a real symmetric matrix and $A\vect{x}=\lambda \vect{x}$.
Then
\begin{equation*}
\conjugate{\lambda }\conjugate{\vect{x}}^{T}\vect{x}=(\conjugate{A
\vect{x}}) ^{T}\vect{x}=\conjugate{\vect{x}}^{T}A^{T}\vect{x}=
\conjugate{\vect{x}}^{T}A\vect{x}=\lambda \conjugate{\vect{x}}^{T}
\vect{x}
\end{equation*}
Dividing by $\conjugate{\vect{x}}^{T}\vect{x}$ on both sides yields $
\conjugate{\lambda }=\lambda $ which says $\lambda $ is real. To do this, we need to ensure that $\conjugate{\vect{x}}^{T}\vect{x} \neq 0$. Notice that $\conjugate{\vect{x}}^{T}\vect{x} = 0$ if and only if $\vect{x} = \vect{0}$. Since we chose $\vect{x}$ such that $A\vect{x} = \lambda \vect{x}$, $\vect{x}$ is an eigenvector and therefore must be non-zero.

Now suppose $A$ is real symmetric and $A\vect{x}=\lambda \vect{x}$, $A
\vect{y}=\mu \vect{y}$ where $\mu \neq \lambda$. Then since $A$
is symmetric, it follows from Lemma~\ref{lem:dot-product-lemma} about the dot product that
\begin{equation*}
\lambda \vect{x}\dotprod \vect{y}=A\vect{x}\dotprod \vect{y}=\vect{x}\dotprod A\vect{y}=\vect{x}\dotprod \mu \vect{y}=\mu \vect{x}\dotprod \vect{y}
\end{equation*}
Hence $(\lambda -\mu) \vect{x}\dotprod \vect{y}=0$. It follows that,
since $\lambda -\mu \neq 0$, it must be that $\vect{x}\dotprod \vect{y}=0$. Therefore the eigenvectors form an orthogonal set.
\end{proof}

The following theorem is proved in a similar manner.

\begin{theorem}{Eigenvalues of antisymmetric matrix}{antisymmetric-eigenvalues}
The eigenvalues of a real antisymmetric matrix are either equal to $0$ or are pure imaginary numbers.
\end{theorem}

\begin{proof}
First, note that if $A=0$ is the zero matrix, then $A$ is antisymmetric and has eigenvalues equal to $0$.

Suppose $A=-A^{T}$ so $A$ is antisymmetric and $A\vect{x}=\lambda
\vect{x}$. Then
\begin{equation*}
\conjugate{\lambda \vect{x}}^{T}\vect{x}=(\conjugate{A
\vect{x}}) ^{T}\vect{x}=\conjugate{\vect{x}}^{T}A^{T}\vect{x}=-
\conjugate{\vect{x}}^{T}A\vect{x}=-\lambda \conjugate{\vect{x}}^{T}
\vect{x}
\end{equation*}
and so, dividing by $\conjugate{\vect{x}}^{T}\vect{x}$ as before, $
\conjugate{\lambda }=-\lambda$. Letting $\lambda =a+ib$, this means $
a-ib=-a-ib$ and so $a=0$. Thus $\lambda $ is pure imaginary.
\end{proof}

Consider the following example.

\begin{example}{Eigenvalues of an antisymmetric matrix}{antisymmetric-eigenvalues}
Let $A=\begin{mymatrix}{rr}
0 & -1 \\
1 & 0
\end{mymatrix}$.  Find its eigenvalues.
\end{example}

\begin{solution}
First notice that $A$ is antisymmetric. By Theorem~\ref{thm:antisymmetric-eigenvalues}, the eigenvalues will either equal $0$ or be pure imaginary.  The eigenvalues of $A$ are obtained by solving the usual equation
\[
\det (\eigenvar I - A ) =
\det \begin{mymatrix}{rr}
\eigenvar & 1 \\
-1 & \eigenvar
\end{mymatrix} =\eigenvar ^{2}+1=0
\]

Hence the eigenvalues are $\pm i$, pure
imaginary.
\end{solution}

Consider the following example.

\begin{example}{Eigenvalues of a symmetric matrix}{eigenvalues-symmetric}
Let $A=\begin{mymatrix}{rr}
1 & 2 \\
2 & 3
\end{mymatrix}$. Find its eigenvalues.
\end{example}

\begin{solution}
First, notice that $A$ is symmetric. By Theorem~\ref{thm:symmetric-orthogonal-eigenvectors}, the eigenvalues will all be real. The eigenvalues of $A$ are obtained by solving the usual equation
\[
\det (\eigenvar I - A) =
\det \begin{mymatrix}{rr}
\eigenvar - 1 & -2 \\
-2 & \eigenvar - 3
\end{mymatrix} = \eigenvar^2 -4\eigenvar -1=0
\]
The eigenvalues are given by $\lambda_1 =2+
\sqrt{5}$ and $\lambda_2 =2-\sqrt{5}$ which are both real.
\end{solution}

Recall that a diagonal matrix $D=\mat{d_{ij} }$ is one in which $d_{ij} = 0$ whenever $i \neq j$. In other words, all numbers not on the main diagonal are equal to zero.

Consider the following important theorem.

\begin{theorem}{Orthogonal diagonalization}{orthogonal-diagonalization}
Let $A$ be a real symmetric matrix. Then there exists an
orthogonal matrix $U$ such that
\[
U^{T}AU = D
\]
where $D$ is a diagonal matrix. Moreover,
the diagonal entries of $D$ are the eigenvalues of $A$.
\end{theorem}

We can use this theorem to diagonalize a symmetric matrix, using orthogonal matrices. Consider the following corollary.

\begin{corollary}{Orthonormal set of eigenvectors}{orthonormal-eigenvectors}
If $A$ is a real $n\times n$ symmetric matrix, then there exists an
orthonormal set of eigenvectors, $\set{\vect{u}_{1},\ldots,\vect{u}
_{n}}$.
\end{corollary}

\begin{proof}
Since $A$ is symmetric, then by Theorem~\ref{thm:orthogonal-diagonalization},
there exists an orthogonal matrix $U$ such that $U^{T}AU=D$, a diagonal
matrix whose diagonal entries are the eigenvalues of $A$. Therefore, since $
A $ is symmetric and all the matrices are real,
\begin{equation*}
\conjugate{D}=\conjugate{D^{T}}=\conjugate{U^{T}A^{T}U}=U^{T}A^{T}U=U^{T}AU=D
\end{equation*}
showing $D$ is real because each entry of $D$ equals its complex conjugate.

Now let
\begin{equation*}
U=\begin{mymatrix}{cccc}
\vect{u}_{1} & \vect{u}_{2} & \cdots & \vect{u}_{n}
\end{mymatrix}
\end{equation*}
where the $\vect{u}_{i}$ denote the columns of $U$ and
\begin{equation*}
D=\begin{mymatrix}{ccc}
\lambda _{1} &  & 0 \\
& \ddots &  \\
0 &  & \lambda _{n}
\end{mymatrix}
\end{equation*}
The equation, $U^{T}AU=D$ implies $AU = UD$ and
\begin{eqnarray*}
AU &=&\begin{mymatrix}{cccc}
A\vect{u}_{1} & A\vect{u}_{2} & \cdots & A\vect{u}_{n}%
\end{mymatrix} \\
&=&\begin{mymatrix}{cccc}
\lambda _{1}\vect{u}_{1} & \lambda _{2}\vect{u}_{2} & \cdots & \lambda
_{n}\vect{u}_{n}
\end{mymatrix} \\
&=& UD
\end{eqnarray*}
where the entries denote the columns of $AU$ and $UD$ respectively.
Therefore, $A\vect{u}_{i}=\lambda _{i}\vect{u}_{i}$.  Since the matrix $U$
is orthogonal, the $\ijth$ entry of $U^{T}U$ equals $\delta _{ij}$ and so
\begin{equation*}
\delta _{ij}=\vect{u}_{i}^{T}\vect{u}_{j}=\vect{u}_{i}\dotprod \vect{u}
_{j}
\end{equation*}
This proves the corollary because it shows the vectors $\set{\vect{u}
_{i}} $ form an orthonormal set.
\end{proof}

\begin{definition}{Principal axes}{principal-axes}
Let $A$ be an $n \times n$-matrix. Then the principal axes of $A$ is a set of orthonormal eigenvectors of $A$.
\index{principal axes}
\end{definition}

In the next example, we examine how to find such a set of orthonormal eigenvectors.

\begin{example}{Find an orthonormal set of eigenvectors}{orthonormal-basis-eigenvectors}
Find an orthonormal set of eigenvectors for the symmetric matrix
\begin{equation*}
A = \begin{mymatrix}{rrr}
17 & -2 & -2 \\
-2 & 6 & 4 \\
-2 & 4 & 6
\end{mymatrix}
\end{equation*}
\end{example}

\begin{solution}
Recall Procedure~\ref{proc:find-eigenvalues-vectors} for finding the eigenvalues and eigenvectors of a matrix. You can verify that the eigenvalues are $18,9,2$. First find the eigenvector for $18$ by solving the equation $(18I-A)X = 0$.
The appropriate augmented matrix is given by
\begin{equation*}
\begin{mymatrix}{ccc|c}
18-17 & 2 & 2 & 0 \\
2 & 18-6 & -4 & 0 \\
2 & -4 & 18-6 & 0
\end{mymatrix}
\end{equation*}
The {\rref} is
\begin{equation*}
\begin{mymatrix}{rrr|r}
1 & 0 & 4 & 0 \\
0 & 1 & -1 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
Therefore an eigenvector is
\begin{equation*}
\begin{mymatrix}{r}
-4 \\
1 \\
1
\end{mymatrix}
\end{equation*}
Next find the eigenvector for $\lambda =9$. The augmented matrix and resulting {\rref} are
\begin{equation*}
\begin{mymatrix}{ccc|c}
9-17 & 2 & 2 & 0 \\
2 & 9-6 & -4 & 0 \\
2 & -4 & 9-6 & 0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 0 & -\vspace{0.05in}\frac{1}{2} & 0 \\
0 & 1 & -1 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
Thus an eigenvector for $\lambda =9$ is
\begin{equation*}
\begin{mymatrix}{r}
1 \\
2 \\
2
\end{mymatrix}
\end{equation*}
Finally find an eigenvector for $\lambda =2$. The appropriate augmented
matrix and {\rref} are
\begin{equation*}
\begin{mymatrix}{ccc|c}
2-17 & 2 & 2 & 0 \\
2 & 2-6 & -4 & 0 \\
2 & -4 & 2-6 & 0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 0 & 0 & 0 \\
0 & 1 & 1 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
Thus an eigenvector for $\lambda =2$ is
\begin{equation*}
\begin{mymatrix}{r}
0 \\
-1 \\
1
\end{mymatrix}
\end{equation*}

The set of eigenvectors for $A$ is given by
\[
\set{
\begin{mymatrix}{r}
-4 \\
1 \\
1
\end{mymatrix},
\begin{mymatrix}{r}
1 \\
2 \\
2
\end{mymatrix},
\begin{mymatrix}{r}
0 \\
-1 \\
1
\end{mymatrix}
}
\]
You can verify that these eigenvectors form an orthogonal set. By dividing each eigenvector by its magnitude, we obtain an orthonormal set:
\begin{equation*}
\set{\frac{1}{\sqrt{18}}\begin{mymatrix}{r}
-4 \\
1 \\
1
\end{mymatrix} ,\frac{1}{3}\begin{mymatrix}{r}
1 \\
2 \\
2
\end{mymatrix} ,\frac{1}{\sqrt{2}}\begin{mymatrix}{r}
0 \\
-1 \\
1
\end{mymatrix} }
\end{equation*}
\end{solution}

Consider the following example.

\begin{example}{Repeated eigenvalues}{}
Find an orthonormal set of three eigenvectors for the matrix
\begin{equation*}
A = \begin{mymatrix}{rrr}
10 & 2 & 2 \\
2 & 13 & 4 \\
2 & 4 & 13
\end{mymatrix}
\end{equation*}
\end{example}

\begin{solution}
You can verify that the eigenvalues of $A$ are $9$ (with multiplicity two) and $18$ (with multiplicity one). Consider the
eigenvectors corresponding to $\lambda =9$. The appropriate augmented matrix
and {\rref} are given by
\begin{equation*}
\begin{mymatrix}{ccc|c}
9-10 & -2 & -2 & 0 \\
-2 & 9-13 & -4 & 0 \\
-2 & -4 & 9-13 & 0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 2 & 2 & 0 \\
0 & 0 & 0 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
and so eigenvectors are of the form
\begin{equation*}
\begin{mymatrix}{c}
-2y-2z \\
y \\
z
\end{mymatrix}
\end{equation*}
We need to find two of these which are orthogonal. Let one be given by setting $z=0$ and $y=1$, giving $
\begin{mymatrix}{r}
-2 \\
1 \\
0
\end{mymatrix}$.

In order to find an eigenvector orthogonal to this one, we need to satisfy
\begin{equation*}
\begin{mymatrix}{r}
-2 \\
1 \\
0
\end{mymatrix} \dotprod \begin{mymatrix}{c}
-2y-2z \\
y \\
z
\end{mymatrix} =5y+4z=0
\end{equation*}
The values $y=-4$ and $z=5$ satisfy this equation, giving another eigenvector
corresponding to $\lambda=9$ as
\begin{equation*}
\begin{mymatrix}{c}
-2(-4) -2(5) \\
(-4) \\
5
\end{mymatrix} =\begin{mymatrix}{r}
-2 \\
-4 \\
5
\end{mymatrix}
\end{equation*}
Next find the eigenvector for $\lambda =18$. The augmented matrix and the resulting {\rref} are given by
\begin{equation*}
\begin{mymatrix}{ccc|c}
18-10 & -2 & -2 & 0 \\
-2 & 18-13 & -4 & 0 \\
-2 & -4 & 18-13 & 0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 0 & -\vspace{0.05in}\frac{1}{2} & 0 \\
0 & 1 & -1 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
and so an eigenvector is
\begin{equation*}
\begin{mymatrix}{r}
1 \\
2 \\
2
\end{mymatrix}
\end{equation*}

Dividing each eigenvector by its length, the orthonormal set is
\begin{equation*}
\set{\frac{1}{\sqrt{5}} \begin{mymatrix}{r}
-2\\
1 \\
0
\end{mymatrix} , \frac{\sqrt{5}}{15} \begin{mymatrix}{r}
-2 \\
-4 \\
5
\end{mymatrix} , \frac{1}{3}\begin{mymatrix}{r}
1 \\
2 \\
2
\end{mymatrix} }
\end{equation*}

\end{solution}

In the above solution, the repeated eigenvalue implies that there would have been many other
orthonormal bases which could have been obtained. While we chose to
take $z=0, y=1$, we could just as easily have taken $y=0$
or even $y=z=1$. Any such change would have resulted in a different
orthonormal set.

Recall the following definition.

\begin{definition}{Diagonalizable}{non-defective}
An $n\times n$-matrix $A$ is said to be \textbf{non-defective}
\index{non-defective}or \textbf{diagonalizable}
\index{diagonalizable} if there exists an invertible matrix $P$ such that $
P^{-1}AP=D$ where $D$ is a diagonal matrix.
\end{definition}

As indicated in Theorem~\ref{thm:orthogonal-diagonalization} if $A$ is a real symmetric matrix, there exists an
orthogonal matrix $U$ such that $U^{T}AU=D$ where $D$ is a diagonal matrix. Therefore,
every symmetric matrix is diagonalizable because if $U$ is an orthogonal
matrix, it is invertible and its inverse is $U^{T}$. In this case, we say that $A$ is \textbf{orthogonally diagonalizable}\index{orthogonally diagonalizable}. Therefore every symmetric matrix is in fact orthogonally diagonalizable. The next theorem provides another way to determine if a matrix is orthogonally diagonalizable.

\begin{theorem}{Orthogonally diagonalizable}{orthogonal-diagonalizable}
Let $A$ be an $n \times n$-matrix. Then $A$ is orthogonally diagonalizable if and only if $A$ has an orthonormal set of eigenvectors.
\end{theorem}

Recall from Corollary~\ref{cor:orthonormal-eigenvectors} that every symmetric matrix has an orthonormal set of eigenvectors. In fact these three conditions are equivalent.

In the following example, the orthogonal
matrix $U$ will be found to orthogonally diagonalize a matrix.

\begin{example}{Diagonalize a symmetric matrix}{orthogonal-diagonalization}
Let $A=\begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 &
\vspace{0.05in}\frac{3}{2} & \vspace{0.05in}\frac{1}{2} \\
0 & \vspace{0.05in}\frac{1}{2} & \vspace{0.05in}\frac{3}{2}
\end{mymatrix}$. Find an orthogonal matrix $U$ such that $U^{T}AU$ is a diagonal
matrix.
\end{example}

\begin{solution}
In this case, the eigenvalues are $2$ (with multiplicity one) and $1$ (with multiplicity two). First
we will find an eigenvector for the eigenvalue $2$. The appropriate augmented matrix and resulting {\rref} are given by
\begin{equation*}
\begin{mymatrix}{ccc|c}
2-1 & 0 & 0 &  0 \\
0 & 2-\vspace{0.05in}\frac{3}{2} & -\vspace{0.05in}\frac{1}{2} & 0 \\
0 & -\vspace{0.05in}\frac{1}{2} & 2-\vspace{0.05in}\frac{3}{2} &  0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 0 & 0 & 0 \\
0 & 1 & -1 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
and so an eigenvector is
\begin{equation*}
\begin{mymatrix}{r}
0 \\
1 \\
1
\end{mymatrix}
\end{equation*}
However, it is desired that the eigenvectors be unit vectors
and so dividing this vector by its length gives
\begin{equation*}
\begin{mymatrix}{c}
0 \\
\vspace{0.05in}\frac{1}{\sqrt{2}} \\
\vspace{0.05in}\frac{1}{\sqrt{2}}
\end{mymatrix}
\end{equation*}
Next find the eigenvectors corresponding to the eigenvalue equal to $1$. The appropriate augmented matrix and resulting {\rref} are given by:
\begin{equation*}
\begin{mymatrix}{ccc|c}
1-1 & 0 & 0 & 0 \\
0 & 1-\vspace{0.05in}\frac{3}{2} & -\vspace{0.05in}\frac{1}{2} & 0 \\
0 & -\vspace{0.05in}\frac{1}{2} & 1-\vspace{0.05in}\frac{3}{2} & 0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
0 & 1 & 1 & 0 \\
0 & 0 & 0 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
Therefore, the eigenvectors are of the form
\begin{equation*}
\begin{mymatrix}{r}
s \\
-t \\
t
\end{mymatrix}
\end{equation*}
Two of these which are orthonormal are $\begin{mymatrix}{c}
1 \\
0 \\
0
\end{mymatrix}$, choosing $s=1$ and $t=0$, and $\begin{mymatrix}{c}
0 \\
-\vspace{0.05in}\frac{1}{\sqrt{2}} \\
\vspace{0.05in}\frac{1}{\sqrt{2}}
\end{mymatrix}$, letting $s=0$, $t= 1 $ and normalizing the resulting vector.

To obtain the desired orthogonal matrix, we let the orthonormal eigenvectors computed above be the columns.
\begin{equation*}
\begin{mymatrix}{rrr}
0 & 1 & 0 \\
-\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
\end{mymatrix}
\end{equation*}

To verify, compute $U^{T}AU$ as follows:
\begin{equation*}
U^{T}AU =
\begin{mymatrix}{rrr}
0 & -\vspace{0.05in}\frac{1}{\sqrt{2}} & \vspace{0.05in}\frac{1}{\sqrt{2}}
\\
1 & 0 & 0 \\
0 & \vspace{0.05in}\frac{1}{\sqrt{2}} & \vspace{0.05in}\frac{1}{\sqrt{2}}
\end{mymatrix} \begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 & \vspace{0.05in}\frac{3}{2} & \vspace{0.05in}\frac{1}{2} \\
0 & \vspace{0.05in}\frac{1}{2} & \vspace{0.05in}\frac{3}{2}
\end{mymatrix} \begin{mymatrix}{rrr}
0 & 1 & 0 \\
-\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}} \\
\frac{1}{\sqrt{2}} & 0 & \frac{1}{\sqrt{2}}
\end{mymatrix}
\end{equation*}
\begin{equation*}
=\begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 & 1 & 0 \\
0 & 0 & 2
\end{mymatrix} = D
\end{equation*}
the desired diagonal matrix. Notice that the eigenvectors, which construct the columns of $U$, are in the same order as the eigenvalues in $D$.
\end{solution}

We conclude this section with a Theorem that generalizes earlier results.

\begin{theorem}{Triangulation of a matrix}{triangulation}
Let $A$ be an $n \times n$-matrix. If $A$ has $n$ real eigenvalues, then an orthogonal matrix $U$ can be found to result in the upper triangular matrix $U^T A U$.
\end{theorem}

This Theorem provides a useful Corollary.

\begin{corollary}{Determinant and trace}{determinant-trace}
Let $A$ be an $n \times n$-matrix with eigenvalues $\lambda_1,\ldots, \lambda_n$. Then it follows that $\det(A)$ is equal to the product of the $\lambda_1$, while $\tr(A)$ is equal to the sum of the $\lambda_i$.
\end{corollary}

\begin{proof}
By Theorem~\ref{thm:triangulation}, there exists an orthogonal matrix $U$ such
that $U^TAU=P$, where $P$ is an upper triangular matrix.
Since $P$ is similar to $A$, the eigenvalues
of $P$ are $\lambda_1, \lambda_2, \ldots, \lambda_n$.
Furthermore, since $P$ is (upper) triangular, the entries on the
main diagonal of $P$ are its eigenvalues, so
$\det(P)=\lambda_1 \lambda_2 \cdots \lambda_n$ and
$\tr(P)=\lambda_1 + \lambda_2 + \ldots + \lambda_n$.
Since $P$ and $A$ are similar, $\det(A)=\det(P)$ and $\tr(A)=\tr(P)$,
and therefore the results follow.
\end{proof}


\chapter{Tensor Product Space Transformations}
Let's consider our good ol' vector space $V$ with basis $\{\vec{e}_\mu\}$ and its dual space $V^*$ with basis $\{\vec{e}^\nu\}$.
We now know that $\{\vec{e}_\mu\}$ will transform covariantly, while $\{\vec{e}^\nu\}$ will transform contravariantly.
We always choose $\{e^\nu\}$ such that $\langle \vec{e}^\nu, \vec{e}_\mu \rangle = \delta^\nu_\mu$.
Now that we've learned about change of basis transformations, let's construct alternate bases for $V$ and $V^*$, called $\{\unit{e}_\mu\}$ and $\{\unit{e}^\nu\}$, again with the property that $\langle \unit{e}_\nu, \unit{e}_\mu \rangle = \delta_\mu^\nu$.
Let's also say for the sake of argument that $\{\unit{e}_\mu\}$ transforms into $\{\vec{e}_\mu\}$ using a transformation $\Lambda$, so we then have that
\[ \unit{e}_\nu = \tensor{\Lambda}{_\nu^\mu} \vec{e}_\mu. \]
Because the bases of the dual space are contravariant, we know that the equivalent transformation for $\{\unit{e}^\nu\}$ and $\{\vec{e}^\nu\}$ would be
\[ \unit{e}^\nu = \tensor{\qty(\Lambda^{-1})}{_\mu^\nu} \vec{e}^\mu. \]

To understand how transformations affect tensor product spaces, let's construct a rank $(1,2)$ tensor,
\[ \tensor{T}{^\lambda_{\mu\nu}} \vec{e}_\lambda \otimes \vec{e}^\mu \otimes \vec{e}^\nu, \]
and transform it from the $\vec{e}$ basis into the $\unit{e}$ basis.
This goes in the direction $\vec{e} \to \unit{e}$, which is the opposite way we've gone previously, but the technique remains the exact same.
We start by defining the transformation $\Lambda$ such that
\[ \vec{e}_\mu = \tensor{\Lambda}{_\mu^\nu} \unit{e}_\nu. \]
We then rewrite our original tensor definition, replace the original bases with their hatted variants and adding the appropriate variant of $\Lambda$, so that
\[ \tensor{T}{^\lambda_{\mu\nu}}\vec{e}_\lambda \otimes \vec{e}^\mu \otimes \vec{e}^\nu = \tensor{T}{^\lambda_{\mu\nu}} \Big[\tensor{\Lambda}{_\lambda^\alpha}\unit{e}_\alpha\Big] \otimes \qty[\tensor{\qty(\Lambda^{-1})}{_\beta^\mu}\unit{e}^\beta] \otimes \qty[\tensor{\qty(\Lambda^{-1})}{_\gamma^\nu}\unit{e}^\gamma]. \]
Because tensors are multilinear products, we can factor the $\Lambda$ and $\Lambda^{-1}$ out in front of our new bases to get
\[ \tensor{T}{^\lambda_{\mu\nu}}\Lambda\Lambda^{-1}\Lambda^{-1} \unit{e}_\lambda \otimes \unit{e}^\mu \otimes \unit{e}^\nu. \]
This means that our component $\tensor{\widehat{T}}{^\lambda_{\mu\nu}} = \tensor{T}{^\lambda_{\mu\nu}}\Lambda\Lambda^{-1}\Lambda^{-1}$.
This represents the transformation of the basis of the tensor; if we want to transform the component $\tensor{T}{^\lambda_{\mu\nu}}$, we need to use the opposite transformations, since each index of the component is opposite the index on the corresponding basis vector.
This means that
\[ \tensor{T}{^\lambda_{\mu\nu}} = \tensor{\widehat{T}}{^\lambda_{\mu\nu}}\Lambda\Lambda\Lambda^{-1}. \]
Note that not only are the transformations inverted, but their order is also reversed; to understand why this is, let's examine the full transformation of our tensor from $\vec{e}$ to $\unit{e}$.
By replacing each piece of our original tensor with its transformed variant, we see that
\begin{align*}
    \tensor{T}{^\lambda_{\mu\nu}}\vec{e}_\lambda \otimes \vec{e}^\mu \otimes \vec{e}^\nu &=
    \tensor{\widehat{T}}{^\lambda_{\mu\nu}}\Lambda\Lambda\Lambda^{-1}\Lambda\Lambda^{-1}\Lambda^{-1} \unit{e}_\lambda \otimes \unit{e}^\mu \otimes \unit{e}^\nu \\
    &= \tensor{\widehat{T}}{^\lambda_{\mu\nu}}\unit{e}_\lambda \otimes \unit{e}^\mu \otimes \unit{e}^\nu.
\end{align*}
Just like how using both $\Lambda$ and $\Lambda^{-1}$ to transform an arbitrary vector causes the transformations to cancel, reversing the order of $\Lambda$ when transforming the basis and the components does the same.

\subsection{Covariant and Contravariant Tensors}
The previous example discussed the transformation of a mixed, rank $(1,2)$ tensor.
Because the basis vectors of the tensor product space included both vectors and covectors, some parts of the tensor transformed covariantly and some transformed contravariantly.
This needn't always be the case though; consider a rank $(0,3)$ tensor, whose basis would be of the form $\vec{e}^\lambda \otimes \vec{e}^\mu \otimes \vec{e}^\nu$, and whose component would be $T_{\lambda\mu\nu}$.
This tensor has a basis comprised solely of covectors.
It's basis is then completely contravariant, and its component is entirely covariant.
This is a \emph{covariant tensor}.
Note that we always identify a tensors variance by the variance of its component, not its basis.
A contravariant tensor might be a rank $(3,0)$ tensor $T^{\lambda\mu\nu}$.
\begin{itemize}
    \item Covariant tensors are identified with all lower indices, like $T_{\lambda\mu\nu}$, and are of rank $(0,n)$. Their basis is comprised entirely of covectors, and they map $n$-tuples of vectors to a scalar field.
    \item Contravariant tensors have all upper indices, like $T^{\lambda\mu\nu}$, and are of rank $(n,0)$. Their basis is entirely products of vectors, and they map $n$-tuples of covectors to a scalar field.
\end{itemize}


\lab{Algorithm}{Image Segmentation with Minimal Spanning Trees}{Image Segmentation with Minimal Spanning Trees}
\label{Ch:MSTImgSeg}

\objective{This section teaches about how to use minimum spanning trees to segmentate an image.}

\section*{Image Segmentation}

%Lab \ref{MSTImgSeg}

One application of Minimal Spaning Trees (MSTs) is in image segmentation. Kruskal's algorthm is especially good at this. Let $k$ be the number of divisions that is wanted and $n$ be the number of nodes. Kruskals algorithm is performed until $n-k$ edges is added.

There are many different ways to turn a image into a graph and weight the edges. A simple, yet effective, version is to make every pixel a node and the edges are the difference in intensities in the four cardinal direction.

This means that there is less that $4n$ edges. Other image segmentation algorithms have to use $n^2$ space. This gives the MST algorithm a critical advantage over other image segmentation algorithms.

\begin{problem}
Write a function that takes as input a black and white image and outputs a list of the edges where thre egdes are in the form $(node,node,weight)$ and a list of nodes.
\end{problem}

The provided kruskal's algorithm takes as inputs the list of nodes, the list of edges and the number of divisions desired. The number of divisions is often has to be higher than the actual number that is needed because sometimes one or two pixels form a division because the difference between them and the pixels around them is so great. Adjust the number of divisions until the desired result is found.

\begin{problem}
Perform the image segmentation algorithm on the image. Then graph the original image and the three highest divisions. (Use the Counter class from collections to find the modes.)
\end{problem}

\begin{problem}
Make a division of the image a different color.
\end{problem}

This algorithm can also be extended to 3D images. One configuration is that each pixel is a node and the edges are the four cardinal directions, the pixel directly above it and the pixel below it.

This algorithm can also be used to segment other data such as connections on facebook.


\subsection{Identifying the order of integration using Augmented Dickey-Fuller}

The Dickey-Fuller test with deterministic time trend was:

\(\Delta y_t=\alpha + \beta t + \gamma y_{t-1}+\epsilon_t \)

The Augmented Dickey-Fuller model adds lags for the differences.

\(\Delta y_t=\alpha + \beta t + \gamma y_{t-1}+\sum_i^p \delta_i \Delta y_{t-i} + \epsilon_t \)


\section*{Ex.3.6}
\subsection*{Give upper bound for probability}

We are given $\abs{A}=10^8, p=1/100, \mu=10^6$.

First note that $\sqrt{\mu} = 1,000$, we want to use Lemma~3.2 to bound $\abs*{X-\mu}\geq 10,000$. Hence we find that $q = 10$ as $q\mu= 10,000$. So
$$
P(\abs*{X-10^6}\geq 10^4)\leq \frac{1}{10^2} = \frac{1}{100}
$$
So the upper bound is 1\%.


The hyperexponential (mixture exponential) distribution may be useful
for fitting fat-tailed empirical distributions.

\subsection{Hyperexponential densities}

The hyperexponential distribution is a mixture of $K$ independent
exponentials with a common location $\mu$ and different decay
constants $\lambda_k$.

The probability density function (PDF) is:

\begin{equation}
P(X=x) = \sum_k^{K} q_k \lambda_k e^{- \lambda_k (x - \mu)}
\label{eqn:hyperexp_pdf}
\end{equation}

The cumulative distribution function (CDF) is:

\begin{equation}
P(X \leq x) = \sum_k^{K} q_k (1 - e^{- \lambda_k (x - \mu)})
\label{eqn:hyperexp_cdf}
\end{equation}

Variate $x$ ranges $\mu \leq x < \infty$.

Mixture coefficients $q_k$ specify the prior probability of each
component $k$; $0 \leq q_k \leq 1$ and $\sum_k q_k = 1$.

The single location parameter $\mu$ is unconstrained, $-\infty < \mu <
\infty$. (Exponential distributions are usually represented without an
explicit location parameter, implicitly assuming $\mu = 0$.)

The scale parameters $\lambda_k$ for each component are nonnegative,
$\lambda_k > 0$.


\section{Kullback-Leibler divergence}

Bayesian inference means we have full distribution of \(p(w)\), not just moments of a specific point estimate

\subsection{Cross entropy:}

\(H(P,Q)=E_P(I(Q))\)

So for a discrete distribution this is:

\(H(P,Q)=-\sum_x P(x)\log Q(x)\)

\(Q\) is prior

\(P\) is posterior

\subsection{Kullback-Leibler divergence}

When we move from a prior to a posterior distribution, the entropy of the probability distribution changes.

\(D_{KL}(P||Q)=H(P,Q)-H(P)\)

KL divergence is also called the information gain.

\subsection{Gibb's inequality}

\(D_{KL}(P||Q)\ge 0\)


\subsection{Translation symmetry}

The distance between two vectors is:

\((v-w)^TM(v-w)\)

So what operations can we do now?

As before, we can do the transformations which preserve \(u^TMv\), such as the orthogonal group.

But we can also do other translations

\((v-w)TM(v-w)\)

\(v^TMv+w^TMw-v^TMw-w^TMv\)

so symmetry is now \(O(3,1)\) and affine translations

\subsubsection{Translation matrix}

\([[1,x][0, 1]]\) moves vector by \(x\).


\section{Numerical Differentiation}
To approximate $f'(x)$($x_0\in(a,b)$, $f\in C^2[a,b]$), $x_1=x_0+h\in[a,b]$.
\begin{align*}
    & f(x) = P_{0,1}(x)+\frac{(x-x_0)(x-x_1)}{2!}f''(\xi(x)) \\
    & \phantom{f(x)} = \frac{f(x_0)(x-x_0-h)}{-h}+\frac{f(x_0)(x-x_0)}{h}+\frac{(x-x_0)(x-x_1)}{2}f''(\xi(x)) \\
    \Rightarrow & f'(x) = \frac{f(x_0+h)-f(x_0)}{h}+\frac{2(x-x_0)-h}{2}f''(\xi(x))+\frac{(x-x_0)(x-x_0-h)}{2}D_x\left(f''(\xi(x))\right) \\
    \Rightarrow & f'(x_0)=\frac{f(x_0+h)-f(x_0)}{h}-\frac{h}{2}f''(\xi)
\end{align*}
The above formula is known as the forward-difference formula if $h>0$, and the backward-difference formula if $h<0$.

\begin{theo}[(n+1)-point Formula]
$\{x_0,x_1,\ldots,x_n\}$ are $(n+1)$ distinct numbers in interval $I$, $f\in C^{n+1}(I)$.
\begin{align*}
    & f(x) = \sum_{k=0}^nf(x_k)L_k(x)+\prod_{k=0}^n\left(\frac{x-x_k}{k+1}\right)f^{(n+1)}(\xi(x)). \\
    \Rightarrow & f'(x) = \sum_{k=0}^nf(x_k)L_k'(x)+D_x\left[\prod_{k=0}^n\left(\frac{x-x_k}{k+1}\right)\right]f^{(n+1)}(\xi(x)) \\
    & \phantom{f(x)} + \prod_{k=0}^n\left(\frac{x-x_k}{k+1}\right) D_x\left[f^{(n+1)}(\xi(x))\right]. \\
    \Rightarrow & f'(x_j) = \sum_{k=0}^nf(x_k)L_k'(x_j)+\frac{f^{(n+1)}(\xi(x_j))}{(n+1)!}\prod_{k=0 \atop k\neq j}^n(x_j-x_k).
\end{align*}
\end{theo}

\subsection{Three-Point Formulas}
If the nodes are equally spaced, $x_1=x_0+h$, $x_2=x_0+2h$, then
\begin{itemize}
    \item Three-Point Formula
    \[
     f'(x_0) = \frac{1}{h}\left[\frac{1}{2}f(x_0)-2f(x_0+h)+\frac{3}{2}f(x_0+2h)\right] + \frac{h^2}{3}f^{(3)}(\xi_0).
    \]
    \item Three-Point Endpoint Formula
    \[
     f'(x_0) = \frac{1}{h}\left[-\frac{3}{2}f(x_0)+2f(x_0+h)-\frac{1}{2}f(x_0+2h)\right] + \frac{h^2}{3}f^{(3)}(\xi_1).
    \]
    \item Three-Point Midpoint Formula
    \[
     f'(x_0) = \frac{1}{h}\left[-\frac{1}{2}f(x_0-h)+\frac{1}{2}f(x_0+h)\right] - \frac{h^2}{6}f^{(3)}(\xi_2).
    \]
\end{itemize}

\subsection{Five-Point Formulas}
\begin{itemize}
    \item Five-Point Midpoint Formula
    \begin{align*}
    f'(x_0)&=\frac{1}{12h}\left[f(x_0-2h)-8f(x_0-h)+8f(x_0+h)\right.\\
    &\left.-f(x_0+2h)\right] + \frac{h^4}{30}f^{(5)}(\xi).
    \end{align*}
    \item Five-Point Endpoint Formula
    \begin{align*}
    f'(x_0)&=\frac{1}{12h}\left[-25f(x_0)+48f(x_0+h)-36f(x_0+2h)\right.\\
    &\left.+16f(x_0+3h-3f(x_0+4h))\right] + \frac{h^4}{5}f^{(5)}(\xi).
    \end{align*}
\end{itemize}

\subsection{Second Derivative Midpoint Formula}
\[
f''(x_0)=\frac{1}{h}\left[f(x_0-h)-2f(x_0)+f(x_0+h)\right] - \frac{h^2}{12}f^{(4)}(\xi)
\]
If $f^{(4)}$ is continuous on $[x_0-h,x_0+h$, it is also bounded, and the approximation is $O(h^2)$.

\subsection{Round-Off Error Instability}
\[
f'(x_0)=\frac{1}{2h}\left[f(x_0+h)-f(x_0-h)\right] - \frac{h^2}{6}f^{(3)}(\xi_1)
\]
Suppose that in evaluating $f(x_0+h)$ and $f(x_0-h)$ we encounter round-off errors $e(x_0+h)$ and $e(x_0-h)$.
\[
f(x_0+h)=\tilde{f}(x_0+h)+e(x_0+h)\qquad and\qquad f(x_0-h)=\tilde{f}(x_0-h)+e(x_0-h)
\]
The total error in the approximation
\begin{align*}
\abs{f'(x_0)-\frac{\tilde{f}(x_0+h)-\tilde{f}(x_0-h)}{2h}}&=\abs{\frac{e(x_0+h)-e(x_0-h)}{2h}-\frac{h^2}{6}f^{(3)}(\xi_1)} \\
& \leq \frac{\varepsilon}{h}+\frac{h^2}{6}M,
\end{align*}
where $e(x_0\pm h)$ are bounded by $\varepsilon>0$ and $f^{(3)}$ are bounded by $M>0$. There is an optimal $h$ such that the bound is small.


\section{Logarithms}\label{sec:Logarithms}
Recall the \ifont{three kinds} of exponential functions $f(x)=a^x$ depending on whether $0<a<1$, $a=1$ or $a>1$:
$$\includegraphics[width=6.5in]{images/exp3}$$
So long as $a\neq 1$, the function $f(x)=a^x$ satisfies the horizontal line test and therefore has an inverse.
We call the \ifont{inverse of $a^x$} the \dfont{logarithmic function with base a} and denote it by $\log_a$.
In particular,
$$\log_a x=y\iff a^y=x.$$
The \ifont{cancellation formulas} for logs are:
$$\log_a(a^x)=x,\mbox{\quad for every $x\in\R$},$$
$$a^{\log_a(x)}=x,\mbox{\quad for every $x>0$}.$$
Since the function $f(x)=a^x$ for $a\neq 1$ has domain $\mathbb{R}$ and range $(0,\infty)$,
the logarithmic function has domain $(0,\infty)$ and range $\mathbb{R}$.
For the most part, we only focus on logarithms with a base larger than
$1$ (i.e., $a>1$) as these are the most important.
$$\includegraphics[width=3.0in]{images/log1}$$
Notice that every logarithm passes through the point $(1,0)$ in the same way that every exponential function passes through the point $(0,1)$.

Some properties of logarithms are as follows.

\begin{formulabox}[Logarithm Properties]
Let $A,B$ be positive numbers and $b>0$ ($b\neq1$) be a base.
\begin{itemize}
	\item $\ds{\log_b(AB)=\log_b A+\log_b B}$,
	\item $\ds{\log_b\left(\frac{A}{B}\right)=\log_b A-\log_b B}$,
	\item $\ds{\log_b(A^n)=n\log_b A}$, where $n$ is any real number.
\end{itemize}
\end{formulabox}

\begin{example}{Compute Lorarithms}{ComputeLorarithms}
To compute $\log_2(24)-\log_2(3)$ we can do the following:
$$\log_2(24)-\log_2(3)=\log_2\left(\frac{24}{3}\right)=\log_2(8)=3,$$
since $2^3=8$.
\end{example}

\subsection*{The Natural Logarithm}
As mentioned earlier for exponential functions, the number $e\approx 2.71828\ldots$
is the most convenient base to use in Calculus.
For this reason we give the logarithm with base $e$ a special name: \dfont{the natural logarithm}.
We also give it special notation:
$$\log_ex=\ln x.$$

You may pronounce $\ln$ as either: ``el - en'', ``lawn'', or refer to it as ``natural log''.
The above properties of logarithms also apply to the natural logarithm.

Often we need to turn a logarithm (in a different base) into a natural logarithm.
This gives rise to the \ifont{change of base formula}.

\begin{formulabox}[Change of Base Formula]
$$\log_ax=\frac{\ln x}{\ln a}.$$
\end{formulabox}

\begin{example}{Combine Logarithms}{CombineLogarithms}
Write $\ln A+2\ln B -\ln C$ as a single logarithm.
\end{example}

\begin{solution}
Using properties of logarithms, we have,
$$\begin{array}{rcl}
\ln A+2\ln B -\ln C & = & \ln A + \ln B^2 - \ln C\\
~&=& \ln (AB^2) - \ln C\\
~&=& \ds{\ln\frac{AB^2}{C}}\\
\end{array}$$
\end{solution}

\begin{example}{Solve Exponential Equations using Logarithms}{SolveExponentialEquationsLogarithms}
If $e^{x+2}=6e^{2x}$, then solve for $x$.
\end{example}

\begin{solution}
Taking the natural logarithm of both sides and noting the cancellation formulas (along with $\ln e=1$), we have:
$$\begin{array}{rcl}
e^{x+2} & = & 6e^{2x}\\
\\
\ln e^{x+2}&=& \ln (6e^{2x})\\
\\
x+2&=& \ln 6 + \ln e^{2x}\\
\\
x+2&=& \ln 6 + 2x\\
\\
x&=& 2-\ln 6\\
\end{array}$$
\end{solution}

\begin{example}{Solve Logarithm Equations using Exponentials}{SolveLogarithmEquationsExponentials}
If $\ln(2x-1)=2\ln(x)$, then solve for $x$.
\end{example}

\begin{solution}
``Taking $e$'' of both sides and noting the cancellation formulas, we have:
$$\begin{array}{rcl}
e^{\ln(2x-1)} & = & e^{2\ln(x)}\\
\\
(2x-1) & = & e^{\ln(x^2)}\\
\\
2x-1 & = & x^2\\
\\
x^2-2x+1 & = & 0\\
\\
(x-1)^2 & = & 0\\
\end{array}$$
Therefore, the solution is $x=1$.
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:Logarithms}}

\begin{enumialphparenastyle}

%%%%%%%%%%
\begin{ex}
Expand $\ds\log_{10} ((x+45)^7 (x-2))$.
\end{ex}

%%%%%%%%%%
\begin{ex}
Expand $\ds\log_2 {\frac{x^3}{3x-5 +(7/x)}}$.
\end{ex}

%%%%%%%%%%
\begin{ex}
Write $\ds \log_2 3x + 17 \log_2 (x-2) -
2\log_2 (x^2 + 4x + 1)$ as a single logarithm.
\end{ex}

%%%%%%%%%%
\begin{ex}
Solve $\ds \log_2 (1+ \sqrt{x} ) = 6$ for  $x$.
\end{ex}

%%%%%%%%%%
\begin{ex}
Solve $\ds 2^{x^2} = 8$ for $x$.
\end{ex}

%%%%%%%%%%
\begin{ex}
Solve $\ds \log_2 (\log_3 (x) ) = 1$ for $x$.
\end{ex}

\end{enumialphparenastyle}