\documentclass[../PHYS306Notes.tex]{subfiles}

\begin{document}
\subsection{Worksheet - Generalized Coupled Oscillators}
\begin{p}
Show that for any system undergoing small oscillations, the kinetic and potential energies can be written as quadratic forms of the generalized velocities and coordinates, respectively.
\end{p}
\begin{s}
Our vector of $n$ generalized coordinates is given by:
\[\v{q} = (q_1, \cdots, q_n)\]
Which may be related to the Cartesian coordinates $\v{r}_i = \v{r}_i(q_1, \cdots, q_n)$. Previously, we solved for the kinetic energy:
\[T(\v{q}, \dot{\v{q}}) = \frac{1}{2}\sum_{j,k}A_{jk}(q) \dot{q}_j\dot{q}_k\]
Which is a quadratic form. When we actually evaluate this, we only keep the constant terms from $A_{jk}$ so we get the familiar kinetic energy:
\[T(\dot{\v{q}}) = \frac{1}{2}\sum_{j,k}M_{jk}\dot{q}_j\dot{q}_k = \frac{1}{2}\dot{\v{q}}^T\MM\dot{\v{q}}\]
Where things have been written as a matrix product in the last line (replacing the double sum). The potential energy we can do a similar way. In general, the potential energy depends on all of the positions of the particles. We can taylor expand this around an equilibrium value, where $\v{q} = \v{0}$:
\[U(\v{q}) = U(\v{0}) + \sum_j \dpd{U}{q_j}q_j + \frac{1}{2}\sum_{j,k}\frac{\partial^2 U}{\partial q_j\partial q_k}q_jq_k + \cdots\]
By the fact that we take $\v{0}$ as a minimum, the second term vanishes. We can also neglect the constant term, so:
\[U(\v{q}) = \frac{1}{2}\sum_{j, k}K_{jk}q_jq_j = \frac{1}{2}\v{q}^T\KK\v{q}\]
Hence, we have obtained quadratic forms for both the kinetic and potential energies! This is just a general description that we could do for any system.
\end{s}

\begin{p}
Prove that the equations of motion for the eigenfrequencies and eigenvectors are $\det{\KK - \omega^2\MM} = 0$ and $(\KK - \omega^2\MM)\v{a} = 0$.
\end{p}
\begin{s}
Given our system, we have $m$ Lagrange equations in general. These are given by:
\[\sum_{j=1}^mM_{ij}\ddot{q}_j = \dpd{U}{q_i} = -\sum_{j=1}^m K_{ij}q_j\]
Which we may rewrite as:
\[\MM\ddot{\v{q}} = -\KK\v{q}\]
As done previously, let $\v{q}(t) = \Re\v{z}(t)$, where $\v{z}(t) = \v{a}\exp(i\omega t)$. Plugging this in and cancelling out the time derivatives, we obtain an algebraic set of equations:
\[(\KK - \omega^2\MM)\v{a} = \v{0}\]
Which we can get the eigenvalues from:
\[\det(\KK - \omega^2\MM) = 0\]
We know that these matrices are positive semidefinite, so the eigenvalues will be positive (or perhaps zero).
\end{s}

\begin{p}
Show generally that each normal coordinate $\xi_i$ oscillates at its own normal frequency $\omega_i$, uncoupled to the other normal coordinates.
\end{p}
\begin{s}
Normal modes are given by:
\[\v{q}_i(t) = \v{a}_i\cos(\omega_i t - \delta_i)\]
I.e. each have an oscillatory solution with characterstic frequency $\omega_i$. It also satisfies the eigenvalue problem from above:
\[\KK\v{a}_i = \omega^2_i\MM\v{a}_i\]
The normal mode expansion is then given by:
\[\v{q}(t) = \sum_{i=1}^m \xi_i(t)\v{a}_i\]
Where $\xi_i(t)$ are the weights of the modes in the general solution representation. We note that $\v{a}_i$ forms a basis in which we can expand the solution, which is nice because in this basis we can solve the equations of motion for the normal coordinates trivially. We have that $\v{q}(t)$ satisfies:
\[\MM\ddot{\v{q}} = -\KK\v{q}\]
Let us plug in this normal mode expansion into the equation of motion. This yields:
\[\sum_{i=1}^m\ddot{\xi}_i(t)\MM\v{a}_i = -\sum_{i=1}^m\xi_i(t)\KK\v{a}_i\]
But $\KK\v{a}_i = \omega_i^2\MM\v{a}_i$ (eigenvalue) so:
\[\sum_{i=1}^m\ddot{\xi}_i(t)\MM\v{a}_i = -\sum_{i=1}^m\xi_i(t)(-\omega_i^2)\MM\v{a}_i\]
This must hold for the components individually, i.e. this must hold for each value of $i$. We therefore obtain the identity:
\[\ddot{\xi}_i(t) = -\omega_i^2\xi_i(t)\]
From which we recover the fact that the normal coordinates satisfy equations for simple harmonic motion.
\end{s}

\begin{p}
Check for the 2-mass-3-springs problem that the normal coordinates are the coefficients of the eigenvector expansion for the displacements $\binom{x_1}{x_2}$.
\end{p}
\begin{s}
The eigenvectors on Monday were given by:
\[\v{a}_1 = \m{1 \\ 1}, \quad \v{a}_2 = \m{1 \\ -1}\]
The normal mode expansion is then:
\[\v{q} = \xi_1\v{a}_1 + \xi_2\v{a}_2\]
So therefore:
\[\v{q} = \m{\xi_1 + \xi_2 \\ \xi_1 - \xi_2} = \m{q_1 \\ q_2}\]
From which we obtain the normal coordinates:
\[\xi_1 = \frac{q_1 + q_2}{2}, \quad \xi_2 = \frac{q_1 - q_2}{2}\]
\end{s}

\begin{p}
Using the expansion of $(x_1, x_2)$ in terms of eigenvectors,find the normal coordinates for the double pendulum.
\end{p}
\begin{s}
For the double pendulum, we had the general solution:
\[\phi_1(t) = A_1\exp(i\omega_1 t) + A_2\exp(i\omega_2 t), \quad \phi_2(t) = A_1\sqrt{2}\exp(i\omega_1 t) - A_2\sqrt{2}\exp(i\omega_2 t)\]
This corresponds to eigenvectors:
\[\v{a}_1 = \m{1 \\ \sqrt{2}}, \v{a}_2 \m{1 \\ -\sqrt{2}}\]
The normal mode expansion is given by:
\[\bm{\phi} = \xi_1\v{a}_1 + \xi_2\v{a}_2 = \m{\xi_1 + \xi_2 \\ \sqrt{2}\xi_1 - \sqrt{2}\xi_2} = \m{\phi_1 \\ \phi_2}\]
Solving for $\xi_1, \xi_2$ we then solve the system:
\[\xi_1 + \xi_2 = \phi_1, \quad \xi_1 - \xi_2 = \frac{\phi}{\sqrt{2}}\]
Adding and subtracting these equations, we get:
\[\xi_1 = \frac{\left(\phi_1 + \frac{\phi_2}{\sqrt{2}}\right)}{2}\]
\[\xi_2 = \frac{\left(\phi_1 - \frac{\phi_2}{\sqrt{2}}\right)}{2}\]

\end{s}
\end{document}


\subsection{Integers}

\subsubsection{Defining integers}

To extend the number line to negative numbers, we define:

\(\forall ab \in \mathbb{N} \exists c (a+c=b)\)

For any pair of numbers there exists a terms which can be added to one to get the other.

For \(1+x=3\) this is another natural number, however for \(3+x=1\) there is no such number.

Integers are defined as the solutions for any pair of natural numbers.

There are an infinite number of ways to write any integer. \(-1\) can be written as \(0-1\), \(1-2\) etc.

The class of these terms form an equivalence class.

\subsubsection{Integers as ordered pairs}

Integers can be defined as an ordered pair of natural numbers, where the integer is valued at: \(a-b\).

For example \(-1\) could be shown as:

\(-1= \{ \{ 0 \},\{0,1\}\}\)

\(-1= \{ \{ 5 \},\{5,6\}\}\)

\((a,b)=a-b\)

\subsubsection{Converting natural numbers to integers}

Natural numbers can be shown as integers by using:

\((n,0)\)

Natural numbers can be converted to integers:

\(\{\{a\},\{a,0\}\}\)
\subsubsection{Cardinality of integers}


\section{Rings}
\begin{definition}
    A ring is a triple $(R,+,\cdot)$ consisting of a set $R$ and two binary operations $+,\cdot:R\times R\to R$, called addition and multiplication, such that\\
    1. $(R,+)$ is an abelian group.
    Its identity is called $0=0_R$.\\
    2. $\cdot$ is associative and has an identity called $1=1_R$.\\
    3. $\forall x,y,z\in R,x\cdot(y+z)=x\cdot y+x\cdot z,(x+y)\cdot z=x\cdot z+y\cdot z$.
\end{definition}
\begin{remark}
    1. Again closure is not explicitly listed as an axiom, but it is included in the well-definedness of $+,\cdot$, so to verify something is a ring, we still have to check the closure.\\
    2. For $x\in R$, we write $-x$ for its additive inverse and abbreviate $x+(-y)=x-y$.\\
    3. $0=0\cdot x-0\cdot x=(0+0)\cdot x-0\cdot x=0\cdot x+0\cdot x-0\cdot x=0\cdot x$.
    Similarly $x\cdot 0=0$.\\
    4. $-x=0-x=0\cdot x-x=(1+(-1))\cdot x-x=x+(-1)\cdot x-x=(-1)\cdot x$.\\
    5. Using 4 and other axioms, we can deduce back the property that addition is commutative.
\end{remark}
\begin{definition}
    If $\cdot$ is commutative as well, we say $R$ is a commutative ring.
\end{definition}
In this course, we are only interested in commutative rings.
When we say ``ring'' afterwards, we will always imply commutativity.
\begin{definition}
    Given a ring $R$, a subring $S$ is a subset of $R$ such that $S$ is a ring under the same addition and multiplication restricted to $S\times S$ and the same identity elements.
\end{definition}
\begin{example}
    1. $\mathbb Z$ is a ring under the familiar operations.\\
    2. The Gaussian integers $\mathbb Z[i]=\{a+ib:a,b\in\mathbb Z\}$ is a ring and is a subring of $\mathbb C$.\\
    3. The set $\mathbb Q[\sqrt{2}]=\{p+q\sqrt 2:p,q\in\mathbb Q\}$.\\
    4. $\mathbb Z[1/p]=\{m/p^n:m\in\mathbb Z,n\in\mathbb N\}$ where $p$ is a prime gives a ring that is a subring of $\mathbb Q$.\\
    5. $\mathbb Z/n\mathbb Z$ is a ring for any natural number $n$.
\end{example}
We can also construct new rings from old.
\begin{definition}
    Given rings $R,S$, the product ring is the Cartesian product $R\times S$ with operations
    $$(r_1,s_1)+(r_2,s_2)=(r_1+r_2,s_1+s_2),(r_1,s_1)\cdot(r_2,s_2)=(r_1\cdot r_2,s_1\cdot s_2)$$
    So $0_{R\times S}=(0_R,0_S)$ and $1_{R\times S}=(1_R,1_S)$.
\end{definition}
\begin{definition}
    Let $R$ be a ring and $X$ a set, then the set of functions $X\to R$ is a ring under pointwise operations.
    So $(f+g)(x)=f(x)+g(x),(f\cdot g)(x)=f(x)\cdot g(x)$.
\end{definition}
Further interesting examples appear as subrings of it.
For example, the set of all continuous functions $\mathbb R\to\mathbb R$ is a ring.
\begin{definition}
    Let $R$ be a ring and $S$ the set of all sequences of elements $r_0,r_1,r_2,\ldots$ of $R$ that is eventually zero.
    Consider the operations
    $$(a_0,a_1,\ldots)+(b_0,b_1,\ldots)=(a_0+b_0,a_1+b_1,\ldots)$$
    and
    $$(a_0,a_1,\ldots)\cdot (b_0,b_1,\ldots)=(c_1,c_2,\ldots),c_n=\sum_{i=0}^na_ib_{n-i}$$
\end{definition}
One can verify that this is indeed a ring.
We can identify $R$ as a subring of $S$ by $r\mapsto (r,0,0,\ldots)$.
Define $X=(0,1,0,0,\ldots)$, so $X^n=(0,\ldots,0,1,0,0,\ldots)$ where the $1$ occurs as the $n^{th}$ entry (counting from $0$).
So $S$ is generated by $X$ and $R$, hence every element of $S$ can be identifies as a sum
$$\sum_{i=0}^na_iX^i,a_i\in\mathbb R$$
So this ring $S=R[X]$ is called the polynomial ring over $R$.
We define the degree of a polynomial to be the largest $i$ such that the coefficient $a_i$ is nonzero.\\
We can obviously identify each polynomial $a_0+a_1X+\cdots +a_nX^n$ as a function $x\mapsto a_0+a_1x+\cdots +a_nx^n$, however
\begin{remark}
    Let $R=\mathbb Z/p\mathbb Z$ where $p$ is a prime, and $f(X)=X^p-X$, so we can identify a function $R\to R$ by $x\mapsto x^p-x=0$.
\end{remark}
\begin{definition}
    For a ring $R$, we can define multivariate polynomial ring $R[X_1,\ldots,X_n]=R[X_1,\ldots,X_{n-1}][X_n]$ inductively.
\end{definition}
\begin{definition}
    Given a ring, we can define the power series ring $R[[X]]$ that is all power series in $X$, which are formal sequence of coefficient following the same operation as polynomial rings.
\end{definition}
\begin{definition}
    The Laurent polynomials $R[X,X^{-1}]$ is defined as functions from $\mathbb Z\to R$ taking finitely many nonzero values following yet again the same operation.
    We write the elements in the form $\cdots +a_{-2}X^{-2}+a_{-1}X^{-1}+a_0+a_1X+a_2X^2+\cdots$ where only finitely many $a_i$ is nonzero.
\end{definition}
\begin{definition}
    Given a ring $R$, an element $r\in R$ is called a unit if it has a multiplicative inverse.
    We write $r^{-1}$ for that inverse.
\end{definition}
A warning is that whether or not an element is a unit depends on the ring, for example $2$ is a unit in $\mathbb Q$ but not in $\mathbb Z$.\\
The set of all inverse in a ring forms a group $R^\times$ under multiplication.
So $\mathbb Z^\times=\{\pm 1\}$ and $\mathbb Q^\times=\mathbb Q\setminus\{0\}$.
Sometimes we write $R^\times$ as $R^\star$.
\begin{definition}
    A field is a ring with $0\neq 1$ and that every nonzero element is a unit.
\end{definition}
\begin{example}
    $\mathbb Q,\mathbb R,\mathbb C,\mathbb Z/p\mathbb Z$ ($p$ prime) are fields.
\end{example}
There is obviously a reason why we need $0\neq 1$.
\begin{remark}
    If $R$ is a ring such that $0=1$, then $\forall x\in R,x=1\cdot x=0\cdot x=0$, so every element of this ring is zero.
\end{remark}
\begin{lemma}
    Let $f,g\in R[X]$ be polynomials, and suppose that $g$ is nonconstant and that the leading coefficient of $g$ is a unit, then there exists a quotient with remainder.
    That is, $\exists q,r\in R[X]$ such that $f=qg+r$ with $\deg r<\deg g$.
\end{lemma}
\begin{proof}
    Simple induction.
\end{proof}


\chapter{Union-Find}

Consider the following problem. Given $n$ disjoint sets of 1 element each,
perform $n$ unions and then $m$ queries for what set a given element is in.
We will call these two operations $union(A,B)$ and $find(x)$.

\section{Approach 1: Linked Lists}
Our first
approach to this problem is to describe our sets as linked lists. We know
we can combine linked lists quite quickly, so this seems ideal for $union$.
All we must do is have the head of $B$ point to the tail of $A$, which can be
done in constant time.
However, linked lists are not particularly well suited for $find$. To resolve
this, at every node we shall store a $back pointer$ to the linked list
the node is part of. This allows us to perform $find$ in constant time.
However, now $union$ needs some extra work.
When we call $union(A,B)$, we will now walk through $B$ and fix all of its
back pointers to point to $A$. This will take time linear in the size of $B$.

Given $n$ sets, what is the worst possible way to $union$ them? Well, since
the run time $union$ is linear in the size of the second list, adversarial
we can
do is to $union$ every individual element with the current unioned set. For
instance, $union(D,union(C,union(A,B)))...$. Clearly this will require
$O(\summ{i=1}{n-1}i)$, which is $O(n^2)$. If we then perform $m$ $find$s, all
of which take $O(1)$ time, we will have performed $n$ $union$s and $m$ $find$s
in $O(n^2 + m)$ time.

\section{Approach 2: Better Linked Lists}
Somehow our first approach was na\"ive, which allowed us to ``game" the system
to create a very bad result for the $union$s. To get a better result, we will
make a slight modification to our $union$ algorithm. Instead of blindly
attaching $B$ to the end of $A$, we will attach the smaller set to the larger.
This fixes our adversarial approach, but is it actually better?

Consider how frequently we need to change the back pointers on an individual
node. At first, it is part of a set of size one, and will have to change its
pointer when unioned to a set of size $\geq 1$, placing it in a set
of size $\geq 2$. The next time it will be changed is when it is unioned
to a set of size $\geq 2$, then $\geq 4$ and so on. The last time will be
when it is unioned to a set of size $\geq n/2$ after which we can not find
another set of large enough size to change it again. Therefore each
back pointer needs to be changed $O(\log n)$ times at worst. Since there
are $n$ back pointers, this new approach only require $O(n \log n)$
time to perform the unions. Since the $find$ operation is not effected,
our approach now performs $n$ $union$s and $m$ $find$s
in $O(n \log n + m)$ time.

\section{Approach 3: Trees}
Having to fix back pointers is still fairly wasteful, what if we didn't have
to? Instead of implementing our sets as linked lists, we can instead
use trees. Each set will be a node with either the name of the set, or a
pointer to its parent. When we perform $union(A,B)$, we will simply
replace the name of the shorter set with a pointer to the head of the taller
set. Since we are just changing a pointer, this will only take $O(1)$ time.
Our $find$ algorithm, however, will now have to walk from the node all the way
to the root of the tree it is in to find out what list it is in. By a
similar argument from the previous section, the height of our tree will only
ever be $\log n$.
Therefore this approach can support $n$ $union$s and $m$ $find$s
in $O(n + m \log n)$ time.

\section{Approach 4: Path Compression}
Our $union$ algorithm is optimal using the tree approach, but it has
made our $find$ algorithm suffer. To fix this, we make a simple observation.
Since our find algorithm must already walk through several nodes in the tree,
once we get to the root we can, without worsening the time,
relabel all of their pointers to point directly to the root. This approach
is called $path compression$. This will make
subsequent queries on these elements and their children substantially faster.
The analysis of this algorithm is beyond the scope of this class, but
evidently it  can support $n$ $union$s and $m$ $find$s
in $O(n + m \alpha(n))$ time. Where $\alpha$ is the inverse Ackermann function.
Although $\alpha(n)$ tends towards infinity as $n$ does, for any ``practical"
$n$ it is at most $4$. It turns out that this is in fact optimal for the
union-find problem.


\lab{Applications}{Markov Chains and Graph Theory}{Markov Chains and Graph Theory}
\label{Ch:Markov}

\objective{This section teaches about two simple applications of Linear Algebra. First it teaches about Markov Chains, which in this context represent discrete random transitions. Second it teaches about Graph Theory, which can be used to represent many physical problems.}

\section*{Markov Chains}

%Lab \ref{Markov}

A Markov Chain describe a particular type of random variable. This type of random variable is characterized by the fact that all relevant information is related to its current state. We can easily model this type of random variable using matrices. We will start with a canonical example of a frog jumping from one lilypad to another.

Fredo the Frog hops around between the three lily pads $A$, $B$, and $C$.  If he's on lily pad $A$ and jumps, there is a 25\% chance that he will land back on lily pad $A$, a 25\% chance that he will land on lily pad $B$, and a 50\% chance that he will land on lily pad $C$.  In Figure `2.1, we have a transition diagram that reflects the various probabilities from which Fredo will go from one lily pad to another.

\begin{figure}[h!]
\label{markov1_fig1}
\begin{center}
\includegraphics[scale = 1]{markov1}
\end{center}
\caption{Transition diagram for Fredo the Frog}
\end{figure}

We can convert our transition diagram into a transition matrix, where the $(i,j)$-entry of the matrix corresponds to the probability that Fredo jumps from the $j^{th}$ lily pad to the $i^{th}$ lily pad (where of course $A$ is the first lily pad, $B$ is the second, and so on).  In Fredo's case, the transition matrix is
\[
A = \begin{pmatrix}
1/4 & 1/2 & 1/2\\
1/4 & 1/6 & 1/2\\
1/2 & 1/3 & 0
\end{pmatrix}
\]
Note that all of the columns add up to one.  This is important.

If Fredo is on lily pad $A$, where will he be after two jumps?  By multiplying the matrix $A$ by itself, we have (approximately)

\[
A^2 = \begin{pmatrix}
0.4375 & 0.3750 & 0.3750\\
0.3542 & 0.3194 & 0.2083\\
0.2083 & 0.3056 & 0.4167
\end{pmatrix}
\]
From this, we infer that there is a 43.75\% chance he will still be on lily pad $A$ after two jumps.  Note that he might have jumped from $A$ to $A$ to $A$, denoted $A \rightarrow A \rightarrow A$, or he could have jumped to one of the other lily pads and then back again, that is, either $A \rightarrow B \rightarrow A$ or $A \rightarrow C \rightarrow A$.  In addition, there is a 35.42\% chance he will be on lily pad $B$ and a 20.83\% chance that he will be on lily pad $C$.  Using Python, we can type in our transition matrix and see where Fredo will be after 5, 10, 20 or 100 jumps.

\begin{lstlisting}[style=python]
#Remember, the 1.'s in the numerator force floating point division
: A = sp.array([[1./4,1./2,1./2],[1./4,1./6,1./2],[1./2,1./3,0]])
: np.linalg.matrix_power(A,5)
: np.linalg.matrix_power(A,10)
: np.linalg.matrix_power(A,20)
: np.linalg.matrix_power(A,100)
\end{lstlisting}

Note that in the limit that the number of jumps goes to infinity, we get
\[
A^\infty = \begin{pmatrix}
0.4 & 0.4 & 0.4\\
0.3 & 0.3 & 0.3\\
0.3 & 0.3 & 0.3
\end{pmatrix}
\]
This means that after several jumps, the probability that we will find Fredo on a given lily pad will have nothing to do with where he started initially.

\section*{Markov Chains}

We can generalize this notion beyond that of frogs and lily pads.  Let the state of our system be represented by a probability vector
\[
\x = \begin{bmatrix}
x_1\\
x_2\\
\vdots\\
x_n
\end{bmatrix}
\]
where each entry represents the probability of being in that state.  Note that each entry is nonnegative and the sum of all the entries adds up to one.  For example, in the case of Fredo, if we know initially that he is on lily pad $A$, then we have the state vector
\[
\x_0 = \begin{bmatrix}
1\\
0\\
0
\end{bmatrix}
\]
because we know for certainty (100\%) that Fredo is in the first state.  After one jump, we have
\[
\x_1 = A \x_0 = \begin{bmatrix}
0.25\\
0.25\\
0.50
\end{bmatrix}
\]
After two jumps, we have
\[
\x_2 = A \x_1 = A^2 \x_0 = \begin{bmatrix}
0.4375\\
0.3542\\
0.2083
\end{bmatrix}
\]
After a large number of jumps $(n>>1)$, we have
\[
\x_n = A \x_{n-1} = \dots = A^n \x_0 \approx \begin{bmatrix}
0.4\\
0.3\\
0.3
\end{bmatrix}
\]
Since all of the columns are the same for $A^\infty$, then for any initial probability vector $\x_0$, we get the same limiting output, or in other words, all initial vectors converge to the same point, call it $\x_\infty$.  Moreover, we have that
\[
\x_\infty = A \x_\infty
\]
This is called a stable fixed point.  How can we check that a stable fixed point exists?  Hint: Think eigenvalues and eigenvectors.

\section*{Example}

Consider the Markov chain given by
\[
A = \begin{pmatrix}
0.5 & 0.3 & 0.4\\
0.2 & 0.2 & 0.3\\
0.3 & 0.5 & 0.3
\end{pmatrix}.
\]
We show that it has a stable fixed point by checking that it has a single eigenvalue $\lambda=1$.  We do this via Python:
\begin{lstlisting}[style=python]
: A = sp.array([[.5,.3,.4],[.2,.2,.3],[.3,.5,.3]])
: V = la.eig(A)[1]
\end{lstlisting}
Note that the entries in the $\lambda=1$ eigenvector do not generally add up to one.  Indeed, any multiple of an eigenvector is an eigenvector.  So we need to multiply it by the appropriate constant so that all of the entries add up to one.
\begin{lstlisting}[style=python]
: x = V[:,0]
: x = x/sp.sum(x);x
array([ 0.41836735,  0.23469388,  0.34693878])
\end{lstlisting}
We can check this answer by taking $A$ to a high exponent, say $A^{100}$.

\begin{problem}
Suppose a basketball player's success at shooting free throws can be
described with the following Markov chain
\[
A = \begin{pmatrix}.75&.50\\.25&.50\end{pmatrix}
\]
where the first state corresponds to success and the second state to failure.
\begin{enumerate}
\item If the player makes his first free throw, what is the probability that he also makes his third one?
\item What is the player's average free throw percentage?
\end{enumerate}
\end{problem}

\begin{problem}
Consider the Markov process given by the transition diagram in Figure 2 below:
\begin{enumerate}
\item Find the transition matrix.
\item If the Markov process is in state $A$, initially, find the probability that it is in state $B$ after 2 periods.
\item Find the stable fixed point if it exists.
\end{enumerate}
\end{problem}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale = 1]{markov2}
\end{center}
\caption{Transition diagram}
\end{figure}

\newpage

\section*{Graph Theory}
\begin{center}
\includegraphics[scale = .8]{graphExample}
\end{center}

Graph theory is an important branch of mathematics and computer science. It describes how objects are connected to one another. In a rigorous sense, a graph is composed of two sets: a set of nodes and a set of edges that connect these nodes.

A graph is directed if connections are uni-directional, and undirected if they are bi-directional. The above graphic shows an undirected graph. We can write a matrix that describes this type of graph. We let each row of our matrix represent our starting point and each column represent our destination. We put a 1 if there is a path and a 0 if there is not. For the above graph we generate the following matrix:

\[
A = \begin{pmatrix}
0 & 1 & 0 & 0 & 1 & 0\\
1 & 0 & 1 & 0 & 1 & 0\\
0 & 1 & 0 & 1 & 0 & 0\\
0 & 0 & 1 & 0 & 1 & 1\\
1 & 1 & 0 & 1 & 0 & 0\\
0 & 0 & 0 & 1 & 0 & 0
\end{pmatrix}
\]

This matrix is called an adjacency matrix. Note that this matrix is symmetric, since the graph is undirected.

What happens if we square an adjacency matrix? It turns out that raising an adjacency matrix to the n power yields the number of paths of length n between two vertices. For example by squaring the above matrix Python gives:
\begin{lstlisting}[style=python]
: np.linalg.matrix_power(A,2)
array([[2, 1, 1, 1, 1, 0],
       [1, 3, 0, 2, 1, 0],
       [1, 0, 2, 0, 2, 1],
       [1, 2, 0, 3, 0, 0],
       [1, 1, 2, 0, 3, 1],
       [0, 0, 1, 0, 1, 1]])
\end{lstlisting}

Now try to find the number of connections of length 6 from node 3 to itself. This is simple to do in Python:
\begin{lstlisting}[style=python]
: sp.matrix_power(A,6)
array([[45, 54, 38, 45, 54, 16],
       [54, 86, 29, 77, 51, 11],
       [38, 29, 55, 15, 70, 27],
       [45, 77, 15, 75, 31,  4],
       [54, 51, 70, 31, 93, 34],
       [16, 11, 27,  4, 34, 14]])
\end{lstlisting}
It turns out that there are 55 unique paths of length 6 from node 3 to itself. Imagine trying to count all of those paths by hand! It would be very easy to count incorrectly. However, this method makes it very simple to count paths without any mistakes.

The astute reader may ask now why this matters. It turns out that the study of graphs and connectivity have many important applications. For example, connections between web pages can be described as graphs. So can flights between airports or friends on social networking sites. The same ideas are applied frequently in computer chip design and in the preservation of endangered species. We will explore one surprising application to chemistry.

Remeber the \li{bucky} array we explored in chapter one? The graph that this matrix represents is a graph for a geodesic dome, which has structure almost identical to certain types of carbon atoms. Understanding the graphs of certain types of molecules allows scientists to better understand the structure of the molecule, making identification and manipulation easier.

We will manipulate the matrix the \li{bucky} to simulate the types of analysis a scientist could do on a complex carbon atom. For our purposes, each column and row of the \li{bucky} matrix represents an atom in our molecule, and connections are chemical bonds from one atom to another.

\begin{problem}
Find the number of connections between atoms in our molecule(the command \li{sp.count\_nonzero} may be useful). Then find the number of atoms that are connected by paths of length two. Three? At what path length are all of the atoms connected?
A nifty way to visualize this is the \li{plt.spy} command. Read the documentation for \li{plt.spy} and then use \li{plt.spy} to visualize how the graph is connected at path length one, two, four and ten. Remember, to load \li{``bucky.csv''} into an array use the command \lstinline{bucky = sp.loadtxt ( "bucky.csv" , delimiter = "," )}.
\end{problem}


\section{Linear Discriminant Analysis}

Linear Discriminant Analysis (LDA) can be used both for dimensionality reduction and for classification. The basic principle is that LDA finds directions within the data that best separate the classes. For two classes, this amounts to finding a line that maximizes $\frac{(\mu_1 - \mu_2)^2}{s_1^2 + s_2^2}$, where $\mu_k$ are the mean locations of the classes and $s$ is the "scatter", which is basically the variance within one class about the mean. In higher dimensions, the algorithm finds something like a center point between the classes, or, more precisely, a point that maximizes the separation between the classes:

\begin{equation}
\frac{\sum_{i\neq j | i,j \in K} (\mu_i - \mu_j)^2}{\sum_{k\in K} s_k^2}
\end{equation}

Based on this point, a coordinate system can be constructed in which the first axis creates the best separation, the second axis the second best separation, etc.. These axes can be used as a means for supervised dimensionality reduction, somewhat analogously to PCA. Classification can be performed by investigating in which sector of the separation the datapoint falls.

This was an attempt at an intuitive explanation of LDA. The true story is slightly less intuitive. LDA classification corresponds to fitting multivariate Gaussian distributions to the classes, where each class is assumed to have an identical covariance matrix, and the prior probability of observing a certain class is also taken into account. Classification then happens by looking at the Mahalanobis distance of a point to the Gaussians fitted to the various classes, and choosing the class which is closest. Quadratic Discriminant Analysis (QDA) works analogous to LDA, but allows for the different classes to have different covariance matrices. The result is a quadratic (rather than linear) decision surface. The Gaussian naive Bayes classifier turns out to be equivalent to QDA with the limitation that the covariance matrices of the classes are assumed to be diagonal. (This is in accordance with the basic naive Bayes assumption that the feature are conditionally independent on the class label.)


\subsection{Euclid's lemma}

\subsubsection{Statement}

If a prime number \(p\) divides product \(a.b\) then \(p\) must divide at least of one of \(a\) or \(b\).

\subsubsection{Proof}

From Bezout's identity we know that:

$d=px+by$

Where \(p\) and \(b\) are natural numbers and \(d\) is their greatest common denominator.

Let's choose a prime number for \(p\). There are no common divisors, other than one. As a result there are exist values for \(x\) and \(y\) such that:

$1=px+by$

Now, we are trying to prove that if \(p\) divides \(a.b\) then \(p\) must divide at least one of \(a\) and \(b\), so let's multiply this by \(a\).

$a=pax+aby$

We know that \(p\) divides \(pax\), and \(p\) divides \(ab\) by definition. As a result \(p\) can divide \(a\).


\subsection{Defining partial order operator $\leq$ through \^{} meet operator}
$a\leq b$ if and only if a \^{} b = a. We can see this in example in figure \ref{fig:semilattice_and}. Consider \{false, true\} and \{true, true\}. The meet operator logical AND gives us \{false, true\}. This satisfies above definition thus \{false, true\} $\leq$ \{true, true\}.

\subsection{Defining meet operator \^{} through partial order operator $\leq$}
a \^{} b = c if and only if c $\leq$ a, c $\leq$ b and $\nexists$ d d $\leq$ a, d $\leq$ b, c \textless  d. \\
If we are given a partial ordering of values we can always find unique value in the same set which satisfies the above.

\subsection{Semilattice diagram through  $\leq$}
$x\leq y$ indicates that there is a path from x to y and vice versa. This can be observed in figure \ref{fig:semilattice_and}. There may not be a direct edge between two nodes related by partial order operator but there will exist a path.

\subsection{Properties of partial order}
\begin{itemize}
    \item \textbf{Reflexive x $\leq$ x}
    \item \textbf{Anti-symmetric x $\leq$ y and y $\leq$ x $\rightarrow$ x = y}
    \item \textbf{Transitive x $\leq$ y and y $\leq$ z $\rightarrow$ x $\leq$ z}
\end{itemize}
Properties of the meet operator \^{} (idempotent, commutative, associative) guarantee the properties for $\leq$. This can be shown easily.

\subsection{The $<$ operator}
$a<b$ if and only if $a \leq b$ and $b \leq a$. In other words $a\leq b$ and $a\neq b$.

\subsection{Semilattice diagram}
\begin{itemize}
\item \textbf{Set of nodes}: Set of values
\item \textbf{Set of edges}: \{ (y, x) {\textbar} x \textless  y and $\nexists$ z .( x \textless  z and z \textless  y)  \}. If there is such a z then there will be a path from x to z and z to y, so we do not need to draw an edge from x to y.
\end{itemize}

\subsection{Example of \^{} and $\leq$}
\textbf{Meet operator \^{} $\triangleq$ set-union $\cup$} : ((x $\leq$ y) $\triangleq$ (x $\cup$ y = x)) $\equiv$ ( x $\supseteq$ y). Thus x $\leq$ y $\equiv$ ( x $\supseteq$ y).


\section{Matrices}
In the previous section we described linear transformations in a rather abstract way: what they are, how they behave qualitatively and how they look like in 2- and 3-dimensions. In this section we introduce a numerical method of representing linear transformations: matrices.

\subsection{Linear transformation of basis vectors}
Recall that any vector $\vec{v}\in\Rs{n}$ can be written as a linear combination of basis vectors $\vec{b}_{1}, \vec{b}_{2}, \dots, \vec{b}_{n}$:
\begin{equation}
	\vec{v} = \sum\limits_{i=1}^{n}\alpha_{i}\vec{b}_{i} = \alpha_{1}\vec{b}_{1} + \alpha_{2}\vec{b}_{2} + \cdots + \alpha_{n}\vec{b}_{n}.
\end{equation}

Applying a linear transformation $T$ on $\vec{v}$ yields, using the properties of linear transformations,
\begin{align}
	T\left(\vec{v}\right) &= T\left(\alpha_{1}\vec{b}_{1} + \alpha_{2}\vec{b}_{2} + \cdots + \alpha_{n}\vec{b}_{n}\right)\nonumber\\
	\tikz[baseline=-0.5ex]{\draw[-stealth, xred] (0,0) -- (1,0) node[pos=-0.1, anchor=east] {additivity}}
						  &= T\left(\alpha_{1}\vec{b}_{1}\right) + T\left(\alpha_{2}\vec{b}_{2}\right) + \cdots + T\left(\alpha_{n}\vec{b}_{n}\right)\nonumber\\
    \tikz[baseline=-0.5ex]{\draw[-stealth, xblue] (0,0) -- (1,0) node[pos=-0.1, anchor=east] {scalability}}
						  &= \alpha_{1}T\left(\vec{b}_{1}\right) + \alpha_{2}T\left(\vec{b}_{2}\right) + \cdots + \alpha_{n}T\left(\vec{b}_{n}\right).
	\label{eq:transfom_by_basis}
\end{align}

This result is pretty neat: it means that by knowing how a linear transformation $T$ changes the basis vectors, we know exactly how any vector is transformed by $T$. This true for any basis, and thus specifically to the standard basis, where the coefficients $\alpha_{1},\alpha_{2},\dots,\alpha_{n}$ are actually the components of the vector, i.e. $v_{1},v_{2},\dots,v_{n}$. Thus in the standard basis:
\begin{equation}
	T\left(\vec{v}\right) = v_{1}T\left(\eb{1}\right) + v_{2}T\left(\eb{2}\right) + \cdots + v_{n}T\left(\eb{n}\right).
\end{equation}

\begin{example}{Vector transformation via a basis}{}
	Applying the transformation $T:\Rs{3}\to\Rs{3}$, defined as
	\[
		T\left(\colvec{x;y;z}\right)=\colvec{x+y-2z;2x+z;-x-y-z}
	\]
	on the vector $\vec{v}=\colvec{2;-1;3}$ yields the following vector:
	\[
		T\left(\vec{v}\right) = T\left(\colvec{2;-1;3}\right) = \colvec{2+(-1)-2\cdot3;2\cdot2+3;-2-(-1)-3} = \colvec{2-1-6;4+3;-2+1-3} = \colvec{-5;7;-4}.
	\]

	Now, let us apply $T$ first to the three standard basis vectors $\hat{x},\hat{y},\hat{z}$:
	\begin{align*}
		T\left( \hat{x} \right) &= T\left(\colvec{1;0;0}\right) = \colvec{1+0-2\cdot0;2\cdot1+0;-1-0-0} = \colvec{1;2;-1},\\
		T\left( \hat{y} \right) &= T\left(\colvec{0;1;0}\right) = \colvec{0+1-2\cdot0;2\cdot0+0;-0-1-0} = \colvec{1;0;-1},\\
		T\left( \hat{z} \right) &= T\left(\colvec{0;0;1}\right) = \colvec{0+0-2\cdot1;2\cdot0+1;-0-0-1} = \colvec{-2;1;-1}.
	\end{align*}

	Taking these results and applying \autoref{eq:transfom_by_basis} yields
	\begin{align*}
		T\left( \vec{v} \right) &= 2T \left( \hat{x} \right) -T \left( \hat{y} \right) + 3T \left( \hat{z} \right)\\
								&= 2\colvec{1;2;-1} - \colvec{1;0;-1} + 3\colvec{-2;1;-1}\\
								&= \colvec{2;4;-2} - \colvec{1;0;-1} + \colvec{-6;3;-3}\\
								&= \colvec{2-1-6;4-0+3;-2-(-1)+(-3)}\\
								&= \colvec{-5;7;-4},
	\end{align*}
	which is indeed what we got when we applied $T$ directly to $\vec{v}$.
\end{example}

\subsection{From transformations to matrices}
The most general linear transformation $T:\Rs{2}\to\Rs{2}$ has the following form:
\begin{equation}
	T \left( \colvec{x;y} \right) = \colvec{ax+by;cx+dy},
	\label{eq:generic_R2_LT}
\end{equation}
where $a,b,c,d\in\mathbb{R}$. If we apply this transformation to $\hat{x}$ and $\hat{y}$ we get, respectively,
\begin{equation}
	T \left( \hat{x} \right) = \colvec{a;c},\quad T \left( \hat{y} \right) = \colvec{b;d}.
	\label{eq:}
\end{equation}
We can now collect these two vectors to form a new structure, which we call a \emph{matrix} (in this specific casr a $2\times2$ matrix):
\begin{equation}
	A = \begin{bNiceMatrix} a&b\\c&d \end{bNiceMatrix}.
	\label{eq:matrix}
\end{equation}

We then define the product of $M$ with a vector $\vec{v}=\colvec{x;y}$ to yield $T \left( \vec{v} \right)$, i.e.
\begin{equation}
	A\vec{v} = \begin{bNiceMatrix} a&b\\c&d \end{bNiceMatrix} \cdot\colvec{x;y} = \colvec{ax+by;cx+dy}.
	\label{eq:matrix_vector_product}
\end{equation}
This defintion can be re-written as following:
\begin{equation}
	A\vec{v} = \begin{bNiceMatrix} a&b\\c&d \end{bNiceMatrix} \cdot\colvec{x;y} = \colvec{A_{1}\cdot\vec{v};A_{2}\cdot\vec{v}},
	\label{eq:matrix_vector_product_as_dot_product}
\end{equation}
i.e. the $i$-th component of the resulting vector is the scalar product of the $i$-th \textbf{row} of the matrix with the vector $\vec{v}$.

\begin{example}{Matrix-vector product}{}
	Some matrix-vector products:

	\begin{align*}
		\begin{bNiceMatrix}
			1 & -2 \\
			0 & 5
		\end{bNiceMatrix}\colvec{-3;2} &= \colvec{1\cdot(-3) + (-2)\cdot2;0\cdot(-3)+5\cdot2} = \colvec{-7;10},\\[5mm]
		\begin{bNiceMatrix}
			1 & 2 \\
			1 & 2
		\end{bNiceMatrix}\colvec{5;-4} &= \colvec{1\cdot5+2\cdot(-4);1\cdot5+2\cdot(-4)} = \colvec{-3;-3},\\[5mm]
		\begin{bNiceMatrix}
			1 & 0 \\
			0 & 2
		\end{bNiceMatrix}\colvec{2;-2} &= \colvec{2\cdot2+0\cdot(-2);0\cdot2+2\cdot(-2)} = \colvec{4;-4}.
	\end{align*}
\end{example}

\begin{challenge}{Proof of linearity}{}
	Prove that the transformation $T$ in \autoref{eq:generic_R2_LT} is indeed linear.
\end{challenge}

The most general form of a linear transformation is $T:\Rs{n}\to\Rs{m}$, i.e. a transformation which takes $n$-dimensional vectors as input and returns $m$-dimensional vectors as output:
\begin{equation}
	T \left( \colvec{\tikzmark{N}x_{1};x_{2};\vdots;x_{n}} \right) = \colvec{
		\Ma{1}{1}x_{1}+\Ma{1}{2}x_{2}+\cdots+\Ma{1}{n}x_{n};
		\Ma{2}{1}x_{1}+\Ma{2}{2}x_{2}+\cdots+\Ma{2}{n}x_{n}\tikzmark{M};
		\vdots;
		\Ma{m}{1}x_{1}+\Ma{m}{2}x_{2}+\cdots+\Ma{m}{n}x_{n}
	},
	\label{eq:}
\end{equation}
\begin{tikzpicture}[overlay, remember picture]
	\node[xblue] (Ntxt) at ($(pic cs:N)+(-1.5cm,-2mm)$) {$\Rs{n}\ni$};
	\draw[-stealth, xblue] (Ntxt) to [out=45, in=90] ($(pic cs:N)+(1mm,2.5mm)$);
	\node[xred] (Mtxt) at ($(pic cs:M)+(2.5cm,5mm)$) {$\in\Rs{m}$};
	\draw[-stealth, xred] (Mtxt) to [out=180, in=0] ($(pic cs:M)+(2.5mm,0)$);
\end{tikzpicture}

where $\Ma{i}{j}\in\mathbb{R},\ \textcolor{xred}{i}=1,2,3,\dots,m$ and $\textcolor{xblue}{j}=1,2,3,\dots,n$.

\begin{challenge}{Proof of linearity}{}
	Prove that the above transformation $T$ is indeed linear.
\end{challenge}

Respectively, we define an $\textcolor{xred}{m}\times \textcolor{xblue}{n}$ matrix (\textcolor{xred}{$m$} rows by \textcolor{xblue}{$n$} columns) by collecting all the coefficients $\Ma{i}{j}$ into a single structure:
\begin{equation}
	A =
	\begin{bNiceMatrix}
		\Ma{1}{1} & \Ma{1}{2} & \cdots & \Ma{1}{n}\\
		\Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
		\vdots & \vdots & \Ddots & \vdots\\
		\Ma{m}{1} & \Ma{m}{2} & \cdots & \Ma{m}{n}
	\end{bNiceMatrix}.
	\label{eq:mxn_matrix}
\end{equation}
The product $M\vec{v}$ (where $\vec{v}\in\Rs{n}$) is then defined as
\begin{equation}
	A\vec{v} =
	\begin{bNiceMatrix}
		\Ma{1}{1} & \Ma{1}{2} & \cdots & \Ma{1}{n}\\
		\Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
		\vdots & \vdots & \Ddots & \vdots\\
		\Ma{m}{1} & \Ma{m}{2} & \cdots & \Ma{m}{n}
	\end{bNiceMatrix}\colvec{x_{1};x_{2};\vdots;x_{n}} = \colvec{
	\Ma{1}{1}x_{1}+\Ma{1}{2}x_{2}+\cdots+\Ma{1}{n}x_{n};
	\Ma{2}{1}x_{1}+\Ma{2}{2}x_{2}+\cdots+\Ma{2}{n}x_{n};
	\vdots;
	\Ma{m}{1}x_{1}+\Ma{m}{2}x_{2}+\cdots+\Ma{m}{n}x_{n}
	}.
	\label{eq:}
\end{equation}
Again, note that the $i$-th component of the resulting vector is the scalar product $A_{i}\cdot\vec{v}$.

\begin{note}{When is a matrix-vector product defined}{}
	In order for a matrix-vector product to be defined, the vector must be of the same dimension as the number of \textbf{columns} in the matrix - i.e. given an $a\times b$ matrix, a vector must be $b$-dimensional for the product to be defined.
\end{note}

\begin{example}{Some matrix-vector products}{}
	\blindtext[2]
\end{example}

The structure of an $m\times n$ matrix $A$ has a nice property: given that the transformation in represented in some basis $B=\left\{ \vec{b}_{1},\vec{b}_{2},\dots,\vec{b}_{n} \right\}$, the $i$-th column of the matrix always shows how $\vec{b}_{i}$ is transformed by the product $A\vec{b}_{n}$. This is easy to see in the case of the standard basis, which we anyway use througout this chapter:

\vspace{1cm}
\begin{equation*}
	\setlength{\arraycolsep}{2.7mm}
	A =
	\begin{bNiceMatrix}
		\tikzmark{A1}\Ma{1}{1} & \tikzmark{B1}\Ma{1}{2} & \cdots & \tikzmark{N1}\Ma{1}{n}\\
	    \Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
		\vdots & \vdots & \Ddots & \vdots\\
		\Ma{m}{1}\tikzmark{A2} & \Ma{m}{2}\tikzmark{B2} & \cdots & \Ma{m}{n}\tikzmark{N2}
	\end{bNiceMatrix}.
\end{equation*}
\tikzset{
	highlight/.style={thick, draw=#1, rounded corners, draw opacity=1, fill=#1, fill opacity=0.2},
	hltxt/.style={highlight=#1, draw opacity=1, text=black, text opacity=1, above, yshift=1.5cm},
	nrtxt/.style={black, opacity=1, text opacity=1},
	hlarrow/.style={-stealth, thick, #1},
}
\tikz[overlay, remember picture, blend mode=multiply]{
	\draw[highlight={xgreen}]  ($(pic cs:A1)+(-2pt,7pt)$) rectangle node[hltxt={xgreen} ](Atxt){$T \left( \eb{1} \right)$}($(pic cs:A2)+(2pt,-5pt)$);
	\draw[highlight={xpurple}] ($(pic cs:B1)+(-2pt,7pt)$) rectangle node[hltxt={xpurple}](Btxt){$T \left( \eb{2} \right)$}($(pic cs:B2)+(2pt,-5pt)$);
	\draw[highlight={xorange}] ($(pic cs:N1)+(-2pt,7pt)$) rectangle node[hltxt={xorange}](Ntxt){$T \left( \eb{n} \right)$}($(pic cs:N2)+(2pt,-5pt)$);

	\draw[hlarrow={xgreen}]  (Atxt.south) -- ++(0,-5mm);
	\draw[hlarrow={xpurple}] (Btxt.south) -- ++(0,-5mm);
	\draw[hlarrow={xorange}] (Ntxt.south) -- ++(0,-5mm);
}

\begin{example}{Matrices}{}
	The product of the following matrix $A$ with each of the vectors $\eb{1},\eb{2},\eb{3}$ (i.e. $\hat{x},\hat{y}$ and $\hat{z}$, respectively) returns the respective column of the matrix:
	\begin{align*}
		A\eb{1}=
		\begin{bNiceMatrix}
			 1 & 2 & 0\\
			-1 & 3 & 4\\
			 0 & 1 & 3\\
		 \end{bNiceMatrix}\colvec{1;0;0} &= \colvec{1\cdot1+2\cdot0+0\cdot0;-1\cdot1+3\cdot0+4\cdot0;0\cdot1+1\cdot0+3\cdot0} = \colvec{1;-1;0},\\[3mm]
		A\eb{2}=
		\begin{bNiceMatrix}
			 1 & 2 & 0\\
			-1 & 3 & 4\\
			 0 & 1 & 3\\
		\end{bNiceMatrix}\colvec{0;1;0} &= \colvec{1\cdot0+2\cdot1+0\cdot0;-1\cdot0+3\cdot1+4\cdot0;0\cdot0+1\cdot1+3\cdot0} = \colvec{2;3;1},\\[3mm]
		A\eb{3}=
		\begin{bNiceMatrix}
			 1 & 2 & 0\\
			-1 & 3 & 4\\
			 0 & 1 & 3\\
		\end{bNiceMatrix}\colvec{0;0;1} &= \colvec{1\cdot0+2\cdot0+0\cdot1;-1\cdot0+3\cdot0+4\cdot1;0\cdot0+1\cdot0+3\cdot1} = \colvec{0;4;3}.
	\end{align*}
\end{example}

\subsection{Matrix representation of the basic linear transformations (2D)}
We can now represent all of the basic linear transformations in $\Rs{2}$ mentioned in the previous section (\autoref{fig:basicLinearTrans}) as $2\times2$ matrices. We do this by observing how the basis vectors $\hat{x}$ and $\hat{y}$ change after the application of each transformation.

\begin{descitemize}
	\item[Identity] both basis vectors remain the same: $\colvec{1;0}\to\colvec{1;0},\ \colvec{0;1}\to\colvec{0;1}$. Therefore the matrix $I$ representing the identity transformation is
		\begin{equation}
			I = \begin{bNiceMatrix} 1&0 \\ 0&1 \end{bNiceMatrix}.
		\end{equation}

	\item[Scaling by $\bm{s}$ in the $\bm{x}$-direction] the basis vector $\hat{x}$ is streched by $s$: $\colvec{1;0}\to\colvec{s;0}$. The basis vector $\hat{y}=\colvec{0;1}$ stays the same. Therefore the matrix $S_{x}$ representing the transformation is
		\begin{equation}
			S_{x} = \begin{bNiceMatrix} s&0 \\ 0&1 \end{bNiceMatrix}.
			\label{eq:}
		\end{equation}

	\item[Scaling by $\bm{s}$ in the $\bm{y}$-direction] much like with $S_{x}$, now the basis vector $\hat{y}$ is the one getting streched, by $\beta$: $\colvec{0;1}\to\colvec{0;s}$. The basis vector $\hat{x}=\colvec{1;0}$ stays the same. Therefore the matrix $S_{y}$ representing the transformation is
		\begin{equation}
			S_{y} = \begin{bNiceMatrix} 1&0 \\ 0&s \end{bNiceMatrix}.
			\label{eq:}
		\end{equation}

	\item[Rotating by $\bm{\theta}$ counter-clockwise about the origin] \autoref{fig:rotationT} shows how do $\hat{x}$ and $\hat{y}$ transformed by the rotation. In the case of $\hat{x}$, the resulting vector is $R_{\theta} \left( \hat{x} \right)=\colvec{\cos(\theta),\sin(\theta)}$, since thiese are the respective sides of a right triangle of hypotenous $1$ and angle $\theta$. The components of $R_{\theta}\left(\hat{y}\right)$ can be calculated by rotating $\hat{x}$ by $\theta+\frac{\pi}{2}$ ($\theta+\ang{90}$): $\cos \left( \theta+\frac{\pi}{2} \right) = -\sin(\theta)$, and $\sin \left( \theta+\frac{\pi}{2} \right) = \cos \left( \theta \right)$. Therefore we get
		\begin{equation}
			\colvec{1;0} \to \colvec{\cos(\theta);\sin(\theta)},\ \colvec{0;1} \to \colvec{-\sin(\theta);\cos(\theta)}.
			\label{eq:}
		\end{equation}

		Altogether the rotation matrix $R_{\theta}$ is
		\begin{equation}
			R_{\theta} = \begin{bNiceMatrix} \cos(\theta)&-\sin(\theta) \\ \sin(\theta)&\cos(\theta) \end{bNiceMatrix}.
			\label{eq:2D_rotation_matrix}
		\end{equation}

		\begin{figure}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
					vector plane,
					width=10cm, height=10cm,
					xmin=-1.3, xmax=1.3,
					ymin=-1.3, ymax=1.3,
					xtick={-1,0,1},
					ytick={-1,0,1},
					yticklabel pos=right,
					yticklabel style={anchor=west},
				]
				\pgfmathsetmacro{\t}{30};
				\draw[black!20] (1,0) arc (0:360:1);
				\tikzset{point/.style={circle, fill=#1, inner sep=0pt, minimum size=3pt}}

				% x
				\draw[vector, xred] (0,0) -- (1,0) node[midway, below] {$\hat{x}$};
				\draw[vector, xred, dashed] (0,0) -- ({cos(\t)},{sin(\t)}) node[midway, above, rotate=\t] {$R_{\theta} \left( \hat{x} \right)$};
				\fill[xpurple, opacity=0.15] (0,0) -- (1,0) arc (0:\t:1) -- cycle;
				\draw[vector, xpurple, dashed] (1,0) arc (0:\t:1) node[point={xred}] (rx) {};
				\node[text=xpurple] at ({1.1*cos(\t/2)},{1.1*sin(\t/2)}) {$\theta$};

				% y
				\draw[vector, xblue] (0,0) -- (0,1) node[midway, right] {$\hat{y}$};
				\draw[vector, xblue, dashed] (0,0) -- ({-sin(\t)},{cos(\t)}) node[midway, below, rotate={\t-90}] {$R_{\theta} \left( \hat{y} \right)$};
				\fill[xpurple, opacity=0.15] (0,0) -- (0,1) arc (90:{90+\t}:1) -- cycle;
				\draw[vector, xpurple, dashed] (0,1) arc (90:{90+\t}:1) node[point={xblue}] (ry) {};
				\node[text=xpurple] at ({-1.1*sin(\t/2)},{1.1*cos(\t/2)}) {$\theta$};
				\end{axis}
				\node[xred, anchor=west, yshift=5pt]  at (rx) {$\left(  \Ctrig,\Strig \right)$};
				\node[xblue, anchor=east, xshift=-4pt] at (ry) {$\left( -\Strig,\Ctrig \right)$};
			\end{tikzpicture}
			\caption{Rotation of $\textcolor{xred}{\hat{x}}$ and $\textcolor{xblue}{\hat{y}}$ by an angle $\textcolor{xpurple}{\theta}$ counter-clockwise about the origin. The notations $\Ctrig,\Strig$ stand for $\cos(\textcolor{xpurple}{\theta})$ and $\sin(\textcolor{xpurple}{\theta})$, respectively.}
			\label{fig:rotationT}
		\end{figure}

	\item[Skew by $\bm{k}$ in the $\bm{x}$-direction] what differentiates this transformation from scaling in the $x$-direction is that a skew changes only $\hat{y}$ by adding to it some horizontal displacement $\vec{K}=k\hat{x}$ (see \autoref{fig:skew_in_x}). Therefore $\hat{x}$ remains the same while $\hat{y}$ is transformed as $\hat{y}\to\hat{y}+\vec{k}=\hat{y}+k\hat{x}=\colvec{0;1}+\colvec{k;0}=\colvec{k;1}$, and altogether the matrix is
		\begin{equation}
			K_{x} = \begin{bNiceMatrix} 1&k \\ 0&1 \end{bNiceMatrix}.
			\label{eq:}
		\end{equation}

		\begin{figure}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
					vector plane,
					width=10cm, height=10cm,
					xmin=-1.3, xmax=1.3,
					ymin=-1.3, ymax=1.3,
					ticks=none,
				]
				\draw[vector, xred]  (0,0) -- (1,0) node[midway, below] {$\hat{x}, K_{x}\left(\hat{x}\right)$};
				\draw[vector, xblue] (0,0) -- (0,1) node[midway, right] {$\hat{y}$};
				\draw[vector, xblue, dashed] (0,0) -- (0.5,1) node[midway, right] {$K_{x}\left( \hat{y} \right)$};
				\draw[vector, xpurple, dashed] (0,1) -- (0.5,1) node [midway, above] {$\vec{k}=k\hat{x}$};
				\end{axis}
			\end{tikzpicture}
			\caption{Skew in the $x$-direction.}
			\label{fig:skew_in_x}
		\end{figure}

	\item[Skew by $\bm{k}$ in the $\bm{y}$-direction] same idea, except the roles of the axes are reveresed:
		\[
			\colvec{1;0}\to\colvec{1;k},\ \colvec{0;1}\to\colvec{0;1}.
		\]
		Thus the matrix is
		\begin{equation}
			K_{y} = \begin{bNiceMatrix} 1&0 \\ k&1 \end{bNiceMatrix}.
			\label{eq:}
		\end{equation}

	\item[Reflections across a line going through the origin] in the case of reflections across the $x$-axis, $\hat{x}$ stays the same, while $\hat{y}$ is flipped (see \autoref{fig:ref_x_axis}), i.e. $\colvec{0;1}\to-\colvec{0;1}=\colvec{0;-1}$. Therefore the matrix is
		\begin{equation}
			\Refl_{x} = \begin{bNiceMatrix} 1&0 \\ 0&-1 \end{bNiceMatrix}.
		\end{equation}

		Similarily, a reflection across the $y$-axis flipps $\hat{x}$ while keeping $\hat{y}$ the same (see \autoref{fig:ref_y_axis}), i.e.
		\begin{equation}
			\Refl_{y} = \begin{bNiceMatrix} -1&0 \\ 0&1 \end{bNiceMatrix}.
		\end{equation}

		Another special case of these kinds of reflections is done across the line rotated by $\frac{\pi}{4}=\ang{45}$ relative to the $x$-axis, i.e the line $y=x$. In this case $\hat{x}$ and $\hat{y}$ are swapped, giving
		\begin{equation}
			\Refl_{\frac{\pi}{4}} = \begin{bNiceMatrix} 0&1 \\ 1&0 \end{bNiceMatrix}.
		\end{equation}

		The most general reflection is made across a line of angle $\theta$ relative to the $x$-axis (see \autoref{fig:ref_line}):
		\begin{equation}
			\Refl_{\theta} = \begin{bNiceMatrix} \cos \left( 2\theta \right) & \sin \left( 2\theta \right) \\ \sin \left( 2\theta \right) & -\cos \left( 2\theta \right) \end{bNiceMatrix}.
		\end{equation}
		A way to calculate this matrix will be shown later in the chapter.

		We can translate the matrix to be based on the slope $m$ of the line instead of its angle $\theta$ relative to the $x$-axis by using the relation $m=\tan\left(\theta\right)$ and the two trigonomentric identities for double angles (\autoref{eq:tan_double_angles}):
		\begin{align*}
			\begin{bNiceMatrix} \cos \left( 2\theta \right) & \sin \left( 2\theta \right) \\ \sin \left( 2\theta \right) & -\cos \left( 2\theta \right) \end{bNiceMatrix} &= \begin{bNiceMatrix} \frac{1-\tan^{2} \left( \theta \right) }{1+\tan^{2} \left( \theta \right) } & \frac{2\tan \left( \theta \right) }{1+\tan^{2} \left( \theta \right) } \\ \frac{2\tan \left( \theta \right) }{1+\tan^{2} \left( \theta \right) } & \frac{\tan^{2} \left( \theta \right)-1 }{1+\tan^{2} \left( \theta \right) } \end{bNiceMatrix}\\
														&= \frac{1}{1+\tan^{2} \left( \theta \right)}\begin{bNiceMatrix} 1-\tan^{2} \left( \theta \right) & 2\tan \left( \theta \right) \\ 2\tan \left( \theta \right) & \tan^{2} \left( \theta \right) -1  \end{bNiceMatrix}\\
														&= \frac{1}{1+m^{2}} \begin{bNiceMatrix} 1-m^{2} & 2m \\ 2m & m^{2}-1 \end{bNiceMatrix}.
		\end{align*}

		\begin{figure}
			\centering
			\begin{subfigure}[c]{0.45\textwidth}
			\begin{center}
				\begin{tikzpicture}
					\begin{axis}[
						vector plane,
						width=8cm, height=8cm,
						xmin=-1.5, xmax=1.5,
						ymin=-1.5, ymax=1.5,
						ticks=none,
					]
					\draw[vector, xred]  (0,0) -- (1,0) node[below] {$\hat{x},\ \Refl_{x}\left(\hat{x}\right)$};
					\draw[vector, xblue] (0,0) -- (0,1) node[right] {$\hat{y}$};
					\draw[vector, xblue, dashed] (0,0) -- (0,-1) node[right] {$\Refl_{x}\left(\hat{y}\right)$};
					\end{axis}
				\end{tikzpicture}
			\end{center}
			\caption{Reflection across the $x$-axis.}
			\label{fig:ref_x_axis}
			\end{subfigure}
			\hfill
			\begin{subfigure}[c]{0.45\textwidth}
			\begin{center}
				\begin{tikzpicture}
					\begin{axis}[
						vector plane,
						width=8cm, height=8cm,
						xmin=-1.5, xmax=1.5,
						ymin=-1.5, ymax=1.5,
						ticks=none,
					]
					\draw[vector, xred]  (0,0) -- (1,0) node[below] {$\hat{x}$};
					\draw[vector, xred, dashed] (0,0) -- (-1,0) node[below] {$\Refl_{y}\left(\hat{x}\right)$};
					\draw[vector, xblue] (0,0) -- (0,1) node[right] {$\hat{y},\ \Refl_{y}\left(\hat{y}\right)$};
					\end{axis}
				\end{tikzpicture}
			\end{center}
			\caption{Reflection across the $y$-axis.}
			\label{fig:ref_y_axis}
			\end{subfigure}
			\begin{subfigure}[c]{0.45\textwidth}
			\begin{center}
				\begin{tikzpicture}
					\begin{axis}[
						vector plane,
						width=8cm, height=8cm,
						xmin=-1.5, xmax=1.5,
						ymin=-1.5, ymax=1.5,
						ticks=none,
					]
					% original vectors
					\draw[vector, xred]  (0,0) -- (1,0) node[below] {$\hat{x}$};
					\draw[vector, xblue] (0,0) -- (0,1) node[right] {$\hat{y}$};

					% reflection line
					\draw[thick, black!75, dashed] (-1.5,-0.5) -- (1.5,0.5);

					% reflected vectors
					\draw[vector, xred, dashed]  (0,0) -- (0.8,0.6) node[below, anchor=west, yshift=5pt] {$\Refl_{\theta}\left(\hat{x}\right)$};
					\draw[vector, xblue, dashed, anchor=west] (0,0) -- (0.6,-0.8) node[right] {$\Refl_{\theta}\left(\hat{y}\right)$};

					% angles
					\fill[xpurple, opacity=0.2] (0,0) -- (0.5,0) arc (0:18.435:0.5) node (A) {} -- cycle;
					\fill[xgreen, opacity=0.2]  (0,0) -- (A) arc (18.435:36.87:0.5) -- (0,0);
					\fill[xpurple, opacity=0.2] (0,0) -- (0,0.5) arc (90:198.435:0.5) node (B) {} -- cycle;
					\fill[xgreen, opacity=0.2]  (0,0) -- (B) arc (198.435:306.87:0.5) -- (0,0);
					\end{axis}
				\end{tikzpicture}
			\end{center}
			\caption{Reflection across a line going through the origin. Notice how in both cases the purple and green angles are the same: this shows that both $\hat{x}$ and $\hat{y}$ are reflected across the line.}
			\label{fig:ref_line}
			\end{subfigure}
			\caption{Reflections across different lines going through the origin.}
			\label{fig:reflections}
		\end{figure}

	\item[Reflection across the origin] in this case both $\hat{x}$ and $\hat{y}$ are flipped, i.e.
		\[
			\colvec{1;0}\to\colvec{-1;0},\ \colvec{0;1}\to\colvec{0;-1},
		\]
		and the matrix is essentially a rotation by $\pi$ ($\ang{180}$) around the origin:
		\begin{equation}
			R = \begin{bNiceMatrix} -1&0 \\ 0&-1 \end{bNiceMatrix}.
			\label{eq:}
		\end{equation}
\end{descitemize}

Table \autoref{tab:matrix_basic_LTs} summarizes all the matrices of the basic linear transformations.

\begin{longtable}{lcccc}
	% !!! vvv MUST FIND WHY THESE DON'T WORK vvv !!! %
	% \caption{my caption}\\
	% \label{tab:matrix_basic_LTs}
	% !!! ^^^ THESE TWO LINES ^^^ !!! %
	\toprule
	Transformation & Trans Tapir & $T \left( \hat{x} \right)$ & $T \left( \hat{y} \right)$ & Matrix\\
	\midrule
	Identity & \tikz[baseline=-0.5ex]{\tapirTransComp{1}{0}{0}{1}{0}{0}{}} & $\colvec{1;0}$ & $\colvec{0;1}$ & $\begin{bNiceMatrix} 1&0 \\ 0&1 \end{bNiceMatrix}$\\
	Scale in $x$ & \tikz[baseline=-0.5ex]{\tapirTransComp{1.3}{0}{0}{1}{0}{0}{}} & $\colvec{s;0}$ & $\colvec{0;1}$ & $\begin{bNiceMatrix} s&0 \\ 0&1 \end{bNiceMatrix}$\\
	Scale in $y$ & \tikz[baseline=-0.5ex]{\tapirTransComp{1}{0}{0}{1.4}{0}{0}{}} & $\colvec{1;0}$ & $\colvec{0;s}$ & $\begin{bNiceMatrix} 1&0 \\ 0&s \end{bNiceMatrix}$\\
	Rotation & \tikz[baseline=-0.5ex]{\tapirTransComp{0.866}{0.5}{-0.5}{0.866}{0}{0}{}} & $\colvec{\Ctrig;-\Strig}$ & $\colvec{\Strig;\Ctrig}$ & $\begin{bNiceMatrix} \Ctrig & -\Strig \\ \Strig & \Ctrig \end{bNiceMatrix}$\\
	Skew in $x$ & \tikz[baseline=-0.5ex]{\tapirTransComp{1}{0}{0.5}{1}{0}{0}{}} & $\colvec{1;0}$ & $\colvec{k;1}$ & $\begin{bNiceMatrix} 1&0 \\ k&1 \end{bNiceMatrix}$\\
	Skew in $y$ & \tikz[baseline=-0.5ex]{\tapirTransComp{1}{0.35}{0}{1}{0}{0}{}} & $\colvec{1;k}$ & $\colvec{0;1}$ & $\begin{bNiceMatrix} 1&k \\ 0&1 \end{bNiceMatrix}$\\
	Reflection by $x$ & \tikz[baseline=-0.5ex]{\tapirTransComp{1}{0}{0}{-1}{0}{0}{}} & $\colvec{1;0}$ & $\colvec{0;-1}$ & $\begin{bNiceMatrix} 1&0 \\ 0&-1 \end{bNiceMatrix}$\\
	Reflection by $y$ & \tikz[baseline=-0.5ex]{\tapirTransComp{-1}{0}{0}{1}{0}{0}{}} & $\colvec{-1;0}$ & $\colvec{0;1}$ & $\begin{bNiceMatrix} -1&0 \\ 0&1 \end{bNiceMatrix}$\\
	Reflection by line & \tikz[baseline=-0.5ex]{\tapirTransComp{0.882}{0.471}{0.471}{-0.882}{0}{0}{};\draw[very thick, dashed, xpurple](-2,-0.5)--(2,0.5)node[pos=-0.05, anchor=east] {$y=mx$}} & $\colvec{\Ctrig_{2};\Strig_{2}}$ & $\colvec{\Strig_{2};-\Ctrig_{2}}$ & $\begin{bNiceMatrix} \Ctrig_{2}&\Strig_{2} \\ \Strig_{2}&-\Ctrig_{2} \end{bNiceMatrix}$\\
	Reflection about origin & \tikz[baseline=-0.5ex]{\tapirTransComp{-1}{0}{0}{-1}{0}{0}{}} & $\colvec{-1;0}$ & $\colvec{0;-1}$ & $\begin{bNiceMatrix} -1&0 \\ 0&-1 \end{bNiceMatrix}$\\
	\bottomrule
\end{longtable}

\subsection{Matrix representation of the basic linear transformations (3D)}
In 3-dimensions, the respective matrices are very similar. For example, the matrix for scaling by $\alpha$ in the $x$-direction, $\beta$ in the $y$-direction and $\gamma$ in the $z$-direction is
\begin{equation}
	S =
	\begin{bNiceMatrix}
		\alpha & 0 & 0\\
		0 & \beta & 0\\
		0 & 0 & \gamma
	\end{bNiceMatrix}.
	\label{eq:3d_scale_matrix}
\end{equation}

As mentioned in the previous section, in 3-dimensions there are infinitely many rotations: the axis of rotation can be any line going through the origin (i.e. any vector except $\vec{0}$ can represent an axis of rotation). Let us start with constructing rotations around the three axes $x,y$ and $z$ first. When rotating around the $x$ axis it stays stationary, while the rotation itself is done in the $yz$-plane. This means that we can take the $2\times2$ rotation matrix (\autoref{eq:2D_rotation_matrix}) and expand it such that it affects only the $yz$-plane:

\vspace{2em}
\begin{equation}
	R^{x}_{\theta} =
	\begin{bNiceMatrix}
		\tikzmark{D1}1 & 0 & 0\\
		0 & \tikzmark{E1} \cos \left( \theta \right) & -\sin \left( \theta \right) \\
		0\tikzmark{D2} & \sin \left( \theta \right) & \cos \left( \theta \right)\tikzmark{E2}
	\end{bNiceMatrix}.
	\label{eq:rotation_matrix_x}
\end{equation}
\begin{tikzpicture}[overlay, remember picture, blend mode=multiply]
	\small
	\draw[highlight={xred}] ($(pic cs:D1)+(-4pt,9pt)$) rectangle node[hltxt={xred}, above right, anchor=east, xshift=-1cm] (Atxt) {$\hat{x}$ doesn't change} ($(pic cs:D2)+(4pt,-5pt)$);
	\draw[hlarrow={xred}] (Atxt.east) to [out=0, in=90] ($(pic cs:D1) + (3pt,10pt)$);
	\draw[highlight={xgreen}] ($(pic cs:E1)+(-4pt,9pt)$) rectangle node[hltxt={xgreen}, below left, anchor=west, yshift=-3cm, xshift=9mm] (Btxt) {2D rotation matrix} ($(pic cs:E2)+(4pt,-5pt)$);
	\draw[hlarrow={xgreen}] (Btxt.west) to [out=180, in=-90] ($(pic cs:E1) + (35pt,-20pt)$);
\end{tikzpicture}

\vspace{3em}
A graphical representation of the rotation can be seen in \autoref{fig:rotation_in_yz}.

\begin{figure}
	\centering
	\def\angThe{75}
	\def\angPhi{45}
	\tdplotsetmaincoords{\angThe}{\angPhi}
	\begin{tikzpicture}[tdplot_main_coords]
		\draw[stealth-, very thick] (-3,0,0) -- (0,0,0);
		\begin{scope}[canvas is yz plane at x=0]
			\fill[fill=xgreen, fill opacity=0.2] (-2,2) -- (2,2) -- (2,-2) -- (-2,-2) -- cycle;
			\draw[step=0.5, xdarkgreen!30] (-2,-2) grid (2,2);
			\draw[vector, xgreen] (-1,0) arc (180:350:1);
			\draw[vector, xgreen] (1,0) arc (0:170:1);
		\end{scope}
		\draw[stealth-stealth, very thick] (0,-3,0) -- (0,3,0) node[pos=1.05] {$y$};
		\draw[stealth-stealth, very thick] (0,0,-3) -- (0,0,3) node[pos=1.05] {$z$};
		\begin{scope}[canvas is yz plane at x=2]
			\draw[vector, xgreen] (-0.5,0) arc (180:350:0.5);
		\end{scope}
		\draw[-stealth, very thick] (0,0,0) -- (3,0,0) node[pos=1.05] {$x$};
		\begin{scope}[canvas is yz plane at x=2]
			\draw[vector, xgreen] (0.5,0) arc (0:170:0.5);
		\end{scope}
	\end{tikzpicture}
	\caption{In $\Rs{3}$, rotation around the $x$-axis is a rotation in the $yz$-plane (i.e. $x=0$).}
	\label{fig:rotation_in_yz}
\end{figure}

The rotation matrices around the $y$- and $z$-axes follow the same structure:
\begin{align}
	R^{y}_{\varphi} &=
		\begin{bNiceMatrix}
			\cos \left( \varphi \right) & 0 & \sin \left( \varphi \right)\\
			0 & 1 & 0 \\
			-\sin \left( \varphi \right) & 0 & \cos \left( \varphi \right)
		\end{bNiceMatrix},\\
	R^{z}_{\psi} &=
		\begin{bNiceMatrix}
			\cos \left( \psi \right) & -\sin \left( \psi \right) & 0\\
			\sin \left( \psi \right) & \cos \left( \psi \right)  & 0\\
			0 & 0 & 1
		\end{bNiceMatrix}.
	\label{eq:rotation_y_z}
\end{align}

\begin{note}{Direction of the $y$-axis}{}
	The signs of $\sin \left( \varphi \right)$ in $R^{y}_{\varphi}$ are flipped compared to $R^{x}_{\theta}$ and $R^{z}_{\psi}$, for the same reason a similar thing happens in the $y$-component of the cross product: it is due to the use of a right-handed system.
\end{note}

The most general rotation in $\Rs{3}$, i.e. around an axis represented by the unit vector $\hat{u}=\colvec{u_{x};u_{y};u_{z}}$ counter-clockwise by an angle $\theta$, is given in matrix form as
\begin{equation}
	R_{\theta}=
	\begin{bNiceMatrix}
		\cos \left( \theta \right) +u_{x}^{2}\left[1-\cos \left( \theta \right) \right] & u_{x}u_{y}\left[1-\cos \left( \theta \right) \right]-u_{z}\sin \left( \theta \right)  & u_{x}u_{z}\left[1-\cos \left( \theta \right) \right]+u_{y}\sin \left( \theta \right) \\
		u_{y}u_{x}\left[1-\cos \left( \theta \right) \right]+u_{z}\sin \left( \theta \right)  & \cos \left( \theta \right) +u_{y}^{2}\left[1-\cos \left( \theta \right) \right] & u_{y}u_{z}\left[1-\cos \left( \theta \right) \right]-u_{x}\sin \left( \theta \right) \\
		u_{z}u_{x}\left[1-\cos \left( \theta \right) \right]-u_{y}\sin \left( \theta \right)  & u_{z}u_{y}\left[1-\cos \left( \theta \right) \right]+u_{x}\sin \left( \theta \right)  & \cos \left( \theta \right) +u_{z}^{2}\left[1-\cos \left( \theta \right) \right]\end{bNiceMatrix}.
	\label{eq:}
\end{equation}
For the moment the derivation of this matrix is not presented.

TBW: REFLECTIONS IN 3D.

\subsection{Matrix operations}
An important operation that can be performed on a matrix is the \emph{transpose}: this operation "rotates" all rows of the matrix to columns, and wise-versa:
\begin{equation}
	\begin{bNiceMatrix}
		\Ma{1}{1} & \Ma{1}{2} & \cdots & \Ma{1}{n}\\
		\Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
		\vdots & \vdots & \Ddots & \vdots\\
		\Ma{m}{1} & \Ma{m}{2} & \cdots & \Ma{m}{n}
	\end{bNiceMatrix}
	\xrightarrow[] {\text{transpose}}
	\begin{bNiceMatrix}
		\Ma{1}{1} & \Ma{2}{1} & \cdots & \Ma{n}{1}\\
		\Ma{1}{2} & \Ma{2}{2} & \cdots & \Ma{n}{2}\\
		\vdots & \vdots & \Ddots & \vdots\\
		\Ma{1}{m} & \Ma{2}{m} & \cdots & \Ma{n}{m}
	\end{bNiceMatrix}.
	\label{eq:transpose}
\end{equation}
Mathematically, the transpose takes any element $\Ma{i}{j}$ of the matrix and exchanges its indeces, yielding $\Ma{j}{i}$. If the original matrix has dimensions $\rhl{m}\times \bhl{n}$, then the transposed matrix has dimensions $\rhl{n}\times \bhl{m}$.

\begin{example}{Transposing matrices}{}
    \begin{align*}
		\underset{3\times3}{
			\begin{bmatrix}
				\rhl{1} & \phl{2} & \phl{3}\\
				\ghl{4} & \rhl{5} & \phl{6}\\
				\ghl{7} & \ghl{8} & \rhl{9}\\
			\end{bmatrix}
		}
	  &\xrightarrow[] {\text{transpose}}
	  \underset{3\times3}{
		  \begin{bmatrix}
			  \rhl{1} & \ghl{4} & \ghl{7}\\
			  \phl{2} & \rhl{5} & \ghl{8}\\
			  \phl{3} & \phl{6} & \rhl{9}\\
		  \end{bmatrix}
	  }
	  \\[5mm]
	  \underset{2\times3}{
		  \begin{bmatrix}
			  \ghl{0} & \ghl{1} & \ghl{-1}\\
			  \phl{2} & \phl{-3} & \phl{5}
		  \end{bmatrix}
	  }
	  &\xrightarrow[] {\text{transpose}}
	  \underset{3\times2}{
		  \begin{bmatrix}
			  \ghl{0}  & \phl{2}\\
			  \ghl{1}  & \phl{-3}\\
			  \ghl{-1} & \phl{5}
		  \end{bmatrix}
	  }
	  \\[5mm]
	  \underset{6\times1}{
		  \begin{bmatrix}
			  \phl{1} \\
			  \phl{2} \\
			  \phl{-1}\\
			  \phl{0}\\
			  \phl{7}\\
			  \phl{-4}\\
		  \end{bmatrix}
	  }
	  &\xrightarrow[] {\text{transpose}}
	  \underset{1\times6}{
		  \begin{bmatrix}
			  \phl{1} & \phl{2} & \phl{-1} & \phl{0} & \phl{7} & \phl{-4}
		  \end{bmatrix}
	  }
    \end{align*}
\end{example}

The notation for the transpose of a matrix $A$ is $A^{\top}$. Since for the main diagonal elements of a matrix the row and column have equal indeces, the transpose operation does not affect their position in the matrix, i.e. $\Ma{i}{i}\xrightarrow[] {\text{transpose}}\Ma{i}{i}$. This means that $\tr{A}=\tr{A^{\top}}$. Also, diagonal matrices are not affected by a transpose. The transpose of a transposed matrix is the original matrix, i.e. $\left(A^{\top}\right)^{\top} = A$.

Much like vectors, a matrix can be scaled by a real number, and two matrices can be added together if their dimensions are identical. The rules for scaling a matrix by a scalar and the addition of two matrices are the same as with vectors, namely everything is dome element wise:
\begin{descitemize}
	\item[Scaling] given a matrix
		\[
			A =
			\begin{bNiceMatrix}
				\Ma{1}{1} & \Ma{1}{2} & \cdots & \Ma{1}{n}\\
				\Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
				\vdots & \vdots & \Ddots & \vdots\\
				\Ma{m}{1} & \Ma{m}{2} & \cdots & \Ma{m}{n}
			\end{bNiceMatrix}
		\]
		and a scalar $\gamma\in\mathbb{R}$, their product is
		\begin{equation}
			\gamma A =
			\begin{bNiceMatrix}
				\gamma\cdot\Ma{1}{1} & \gamma\cdot\Ma{1}{2} & \cdots & \gamma\cdot\Ma{1}{n}\\
				\gamma\cdot\Ma{2}{1} & \gamma\cdot\Ma{2}{2} & \cdots & \gamma\cdot\Ma{2}{n}\\
				\vdots & \vdots & \Ddots & \vdots\\
				\gamma\cdot\Ma{m}{1} & \gamma\cdot\Ma{m}{2} & \cdots & \gamma\cdot\Ma{m}{n}
			\end{bNiceMatrix}.
			\label{eq:matrix_scaling}
		\end{equation}

	\item[Addition] given two matrices,
		\[
			A =
			\begin{bNiceMatrix}
				\Ma{1}{1} & \Ma{1}{2} & \cdots & \Ma{1}{n}\\
				\Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
				\vdots & \vdots & \Ddots & \vdots\\
				\Ma{m}{1} & \Ma{m}{2} & \cdots & \Ma{m}{n}
			\end{bNiceMatrix},\quad
			B =
			\begin{bNiceMatrix}
				\Mb{1}{1} & \Mb{1}{2} & \cdots & \Mb{1}{n}\\
				\Mb{2}{1} & \Mb{2}{2} & \cdots & \Mb{2}{n}\\
				\vdots & \vdots & \Ddots & \vdots\\
				\Mb{m}{1} & \Mb{m}{2} & \cdots & \Mb{m}{n}
			\end{bNiceMatrix},
		\]
		their sum is
		\begin{equation}
			A+B =
			\begin{bNiceMatrix}
				\Ma{1}{1}+\Mb{1}{1} & \Ma{1}{2}+\Mb{1}{2} & \cdots & \Ma{1}{n}+\Mb{1}{n}\\
				\Ma{2}{1}+\Mb{2}{1} & \Ma{2}{2}+\Mb{2}{2} & \cdots & \Ma{2}{n}+\Mb{2}{n}\\
				\vdots & \vdots & \Ddots & \vdots\\
				\Ma{m}{1}+\Mb{m}{1} & \Ma{m}{2}+\Mb{m}{2} & \cdots & \Ma{m}{n}+\Mb{m}{n}
			\end{bNiceMatrix}.
			\label{eq:}
		\end{equation}
\end{descitemize}

\subsection{Types of matrices}
Any matrix $A$ which represents a transformation of the type $\Rs{n}\to\Rs{n}$ (i.e. from a space onto itself) has the same number of rows and columns (i.e. its dimension is $n\times n$):
\begin{equation}
	A =
	\begin{bNiceMatrix}
		\Ma{1}{1} & \Ma{1}{2} & \cdots & \Ma{1}{n}\\
		\Ma{2}{1} & \Ma{2}{2} & \cdots & \Ma{2}{n}\\
		\vdots & \vdots & \Ddots & \vdots\\
		\Ma{n}{1} & \Ma{n}{2} & \cdots & \Ma{n}{n}
	\end{bNiceMatrix}.
	\label{eq:square_matrix}
\end{equation}

Due to their shape, such matrices are called \emph{square matrices}. The elements $\Ma{1}{1},\Ma{2}{2},\Ma{3}{3},\dots,\Ma{n}{n}$ of a square matrix jointly form its \emph{main diagonal} (also: \emph{principal diagonal}):

\vspace{1em}
\begin{equation}
	A =
	\begin{bNiceMatrix}
		\tikzmark{m11}\cxgreen\Ma{1}{1} & \Ma{1}{2} & \Ma{1}{3} & \cdots & \Ma{1}{n}\\
		\Ma{2}{1} & \cxgreen\Ma{2}{2} & \Ma{2}{3} & \cdots & \Ma{2}{n}\\
		\Ma{3}{1} & \Ma{3}{2} & \cxgreen\Ma{2}{3} & \cdots & \Ma{2}{n}\\
		\vdots & \vdots & \vdots & \Ddots & \vdots\\
		\Ma{n}{1} & \Ma{n}{2} & \Ma{n}{3} & \cdots & \cxgreen\Ma{n}{n}\tikzmark{mnn}
	\end{bNiceMatrix}.
	\label{eq:square_matrix_main_diag}
\end{equation}
% \tikz[overlay, remember picture, blend mode=multiply]{
% 	\draw[xgreen, line width=5mm, opacity=0.2, cap=round] ($(pic cs:m11)+(5pt,3pt)$) -- ($(pic cs:mnn)+(-5pt,2pt)$) node[pos=-0.1, anchor=east, xshift=-7mm, yshift=4mm, text=xdarkgreen, opacity=1] (txt) {main diagonal};
% 	\draw[-stealth, xdarkgreen] ($(txt.east)-(3mm,0)$) to [out=0, in=135] ($(pic cs:m11)+(-1pt,9pt)$);
% }

The sum of the main diagonal elements is called the \emph{trace} of the matrix:
\begin{equation}
	\tr \left( A \right) = \sum\limits_{i=1}^{n}\Ma{i}{i}.
	\label{eq:trace}
\end{equation}

\emph{Triangular matrices} are matrices in which the elements above or below the main diagonal are all zeros, e.g.

\begin{center}
	\def\bperc{25}
	\begin{tabular}{rcp{5mm}rc}
		$U=$ &
		$\begin{bmatrix}
			1 & 6 & 6 & -3 \\
			\textcolor{black!\bperc}{0} & 2 & 7 & 1 \\
			\textcolor{black!\bperc}{0} & \textcolor{black!\bperc}{0} & 3 & 5 \\
			\textcolor{black!\bperc}{0} & \textcolor{black!\bperc}{0} & \textcolor{black!\bperc}{0} & -4
		\end{bmatrix},$
			 & &
		$L=$ &
		$\begin{bmatrix}
			1 & \textcolor{black!\bperc}{0} & \textcolor{black!\bperc}{0} & \textcolor{black!\bperc}{0} \\
			\tikzmark{LT1} 2 & 3 & \textcolor{black!\bperc}{0} & \textcolor{black!\bperc}{0} \\
			5 & 1 & -5 & \textcolor{black!\bperc}{0} \\
			-4 & 1 & 2\tikzmark{LT2} & -3
		\end{bmatrix}.$
		\\[2.5em]
			 & upper triangular & & & lower triangular
	\end{tabular}
\end{center}

A \emph{diagonal matrix} $A$ is a matrix in which all the non-main diagonal elements, i.e. $\Ma{i}{j}$ where $\textcolor{xred}{i}\neq \textcolor{xblue}{j}$, equal zero. These matrices can be thought of as scaling matrices: each entry $\Ma{i}{i}$ tells us how the sapce is scaled in the $i$-th dimension.

\begin{example}{Diagonal matrices}{}
	Text.
\end{example}
As we saw in the cases of $\Rs{2}$ and $\Rs{3}$, diagonal matrices are \emph{scaling matrices}: each entry $a_{ii}$ tells us by how much space is scaled in the $i$-th direction.

A very important family of \textbf{square} matrices are the \emph{identity matrices}. These matrices have a unique structure: their main diagonal elements are all $1$, while the rest of the elements (the \emph{off-diagonal elements}) are all $0$:

\vspace{1em}
\begin{equation}
	I_{n} =
	\begin{bNiceMatrix}
		\tikzmark{I11} 1 & 0 & 0 & \cdots & 0\tikzmark{I1n}\\
		0 & 1 & 0 & \cdots & 0\\
		0 & 0 & 1 & \cdots & 0\\
		\vdots & \vdots & \vdots & \Ddots & \vdots\\
		\tikzmark{In1}0 & 0 & 0 & \cdots & 1\tikzmark{Inn}
	\end{bNiceMatrix}
	\label{eq:}
\end{equation}
\begin{tikzpicture}[overlay, remember picture]
	  \draw [xred, thick, decorate, decoration={brace, amplitude=3pt, raise=7pt}] (pic cs:I1n) -- (pic cs:Inn) node[midway, right, xshift=10pt]{$n$ rows};
	  \draw [xblue, thick, decorate, decoration={brace, amplitude=3pt, raise=10pt}] (pic cs:I11) -- (pic cs:I1n) node[midway, above, yshift=13pt]{$n$ columns};
\end{tikzpicture}

A common way of depicting an identity matrix is
\[
	I_{n} =
	\begin{bNiceMatrix}
	1   &       & \Block{2-3}<\huge>{0} \\
		&   1   &        &      &       \\
		&       &   1    &      &       \\
	\Block{2-3}<\huge>{0}
		&       &       & \Ddots    &   \\
		&       &       &      &   1   \\
	\end{bNiceMatrix}.
\]
In such a depiction, the off-diagonal elements are each written using a single zero. This kind of notation will come in handy in later sections. Yet another way of defining the identity matrix is by using the \emph{Kronecker delta}, which takes two integers $i,j$ and returns $1$ if they are equal, otherwise it returns $0$:
\begin{equation}
	\delta_{ij} =
	\begin{cases}
		1 & i=j,\\
		0 & i\neq j.
	\end{cases}
	\label{eq:kronecker_delta}
\end{equation}
Using the Kronecker delta, each element $a_{ij}$ of the identity matrix $I_{n}$ simply equals $\delta_{ij}$.

An identity matrix of dimension $n$ represents the identity transformation in $\Rs{n}$: each standard basis vector $\eb{i}$ is left unchanged by the transformation.

\begin{example}{Identity matrices}{}
	The identity matrices of $\Rs{2},\Rs{3},\dots,\Rs{6}$ (in each matrix the main diagonal is highlighted):

	\centering
	\begin{tabular}{ccccc}
		&
		$\begin{bmatrix}
			\cxblue1 & 0 & 0\\
			0 & \cxblue1 & 0\\
			0 & 0 & \cxblue1
		\end{bmatrix}$&
		$\begin{bmatrix}
			\cxgreen1 & 0 & 0 & 0\\
			0 & \cxgreen1 & 0 & 0\\
			0 & 0 & \cxgreen1 & 0\\
			0 & 0 & 0 & \cxgreen1\\
		\end{bmatrix}$&
		$\begin{bmatrix}
			\cxpurple1 & 0 & 0 & 0 & 0\\
			0 & \cxpurple1 & 0 & 0 & 0\\
			0 & 0 & \cxpurple1 & 0 & 0\\
			0 & 0 & 0 & \cxpurple1 & 0\\
			0 & 0 & 0 & 0 & \cxpurple1\\
		\end{bmatrix}$&
		$\begin{bmatrix}
			\cxorange1 & 0 & 0 & 0 & 0& 0\\
			0 & \cxorange1 & 0 & 0 & 0& 0\\
			0 & 0 & \cxorange1 & 0 & 0& 0\\
			0 & 0 & 0 & \cxorange1 & 0& 0\\
			0 & 0 & 0 & 0 & \cxorange1& 0\\
			0 & 0 & 0 & 0 & 0& \cxorange1\\
		\end{bmatrix}$\\
		$I_{2}$ & $I_{3}$ & $I_{4}$ & $I_{5}$ & $I_{6}$
	\end{tabular}
\end{example}
In the next section we will see the importance of the identity matrices.

Another important family of matrices are the \emph{orthogonal matrices} (also \emph{orthonormal matrices}): we say that a matrix $Q$ is an orthogonal matrix if all of its columns, when viewed as column vectors, form an orthonormal set. For example, the identity matrices are all orthogonal matrices. Another orthogonal matrix is the matrix
\begin{equation}
	B = \frac{1}{\sqrt{2}}\begin{bmatrix}1&1\\1&-1\end{bmatrix},
	\label{eq:}
\end{equation}
since both $\frac{1}{\sqrt{2}}\colvec{1;1}$ and $\frac{1}{\sqrt{2}}\colvec{1;-1}$ are unit vectors, and they are orthogonal to each other (as seen in REF).

A \emph{symmetric matrix} is a square matrix for which
\begin{equation}
	A^{\top} = A.
	\label{eq:symmetric_matrix}
\end{equation}
"Graphically", the symmetry of such matrices can be seen in respect to their main diagonal.

\begin{example}{Symmetric matrix}{}
	The following matrix $S$ is a symmetric $4\times4$ matrix, in which the elements $a_{ij},a_{ji}$ are higlighted with the same color:

	\centering
	\begin{tikzpicture}[node distance=1.6cm]
		\node (Seq) {$S=$};
		\matrix (S) [matrix of nodes, right of=Seq, left delimiter={[}, right delimiter={]}]{
			1 & 3 & 5 & 7\\
			3 & 0 & 1 & 3\\
			5 & 1 & 4 & 2\\
			7 & 3 & 2 & 6\\
		};
		\tikzset{every node/.style={inner sep=0}}
		\scoped[on background layer]{
			\foreach \k in {1,...,4}
				\node[fill=white, fit=(S-\k-\k)(S-\k-\k)] {};
			\node[fill=xred!20, fit=(S-1-2)(S-1-2)] {};
			\node[fill=xred!20, fit=(S-2-1)(S-2-1)] {};
			\node[fill=xblue!20, fit=(S-1-3)(S-1-3)] {};
			\node[fill=xblue!20, fit=(S-3-1)(S-3-1)] {};
			\node[fill=xgreen!20, fit=(S-1-4)(S-1-4)] {};
			\node[fill=xgreen!20, fit=(S-4-1)(S-4-1)] {};
			\node[fill=xpurple!20, fit=(S-2-3)(S-2-3)] {};
			\node[fill=xpurple!20, fit=(S-3-2)(S-3-2)] {};
			\node[fill=xorange!20, fit=(S-2-4)(S-2-4)] {};
			\node[fill=xorange!20, fit=(S-4-2)(S-4-2)] {};
			\node[fill=xpink!20, fit=(S-3-4)(S-3-4)] {};
			\node[fill=xpink!20, fit=(S-4-3)(S-4-3)] {};
		}
	\end{tikzpicture}
\end{example}

A rather non-interesting family of matrices are the \emph{zero matrices}: these are matrices which have only zero-elements, i.e.
\begin{equation}
	\bm{0}_{n} =
	\begin{bNiceMatrix}
		\tikzmark{z1}0 & 0 & \cdots & 0\tikzmark{z2}\\
		0 & 0 & \cdots & 0\\
		\vdots & \vdots & \Ddots & \vdots\\
		0 & 0 & \cdots & 0\tikzmark{z3}
	\end{bNiceMatrix}.
	\label{eq:}
\end{equation}
\begin{tikzpicture}[overlay, remember picture]
	  \draw [xred, thick, decorate, decoration={brace, amplitude=3pt, raise=7pt}] (pic cs:z2) -- (pic cs:z3) node[midway, right, xshift=10pt]{$m$ rows};
	  \draw [xblue, thick, decorate, decoration={brace, amplitude=3pt, raise=10pt}] (pic cs:z1) -- (pic cs:z2) node[midway, above, yshift=13pt]{$n$ columns};
\end{tikzpicture}
We will see later in the section why exactly we call these matrices "zero matrices".


\section{The Jacobi and Gauss-Siedel Iterative Techniques}
\subsection{Jacobi's Method}
The \emph{Jacobi iterative method} is obtained by solving the $i$th equation in $A\bfx=\mathbf{b}$ for $x_i$ to obtain(provided $a_{ii}\neq 0$)
\[
x_i=\sum_{\substack{j=1\\j\neq i}}^n\left(-\frac{a_{ij}x_j}{a_{ii}}\right)+\frac{b_i}{a_{ii}},\quad\text{for $i=1,1,\ldots,n$}.
\]
In general, iterative techniques for solving linear systems by converting the system $\bfA\bfx=\bf b$ into an equivalent system of the form $\bfx={\bf Tc}$ for some fixed matrix $\bf T$ and vector $\bf c$. After the initial vector $x^{(0)}$ is selected, the sequence of approximate solution vectors is generated by computing
\[
\bfx^{(k)} = {\bf T}\bfx^{(k-1)}+{\bf c}.
\]
For Jacobi method, $\bfA={\bf D}(iag)-{\bf L}(ower)-{\bf U}(pper)$. Then
\[
\bfx = {\bf D}^{-1}({\bf L+U})\bfx + {\bf D}^{-1}{\bf b}
\]

\subsection{The Gauss-Seidel Method}
\[
\bfx_i^{(k)} = \frac{1}{a_{ii}}\left[-\sum_{j=1}^{i-1}(a_{ij}x_j^{(k)})-\sum_{j=i+1}^{n}(a_{ij}x_j^{(k-1)})+b_i\right]
\]
That is
\[
\bf (D-L)x^{(k)} = Ux^{(k-1)} + b
\]
Then
\[
\bf x^{(k)} = (D-L)^{-1}Ux^{(k-1)} + (D-L)^{-1}b
\]

\subsection{General Iterative Methods}
To study the convergence of general iteration techniques, we need to analyze the formula.
\[
\bf x^{(k)} = Tx^{(k-1)}+c
\]
\begin{lemma}
If the spectral radius satisfies $\rho({\bf T})<1$, then $\bf (I-T)^{-1}$ exists, and
\[
\bf (I-T)^{-1} = I+T+T^2+\ldots=\sum_{j=0}^\infty T^j.
\]
\end{lemma}
\begin{theo}
For any $x^{(0)}\in\mathbb{R}^n$, the sequence $\left\{x^{(k)}\right\}_{k=0}^\infty$ defined by
\[
\bf x^{(k)} = Tx^{(k-1)}+c,\quad\text{for each $k\geq 1$}
\]
converges to the unique solution of $\bf x=Tx+c$ if and only if $\rho({\bf T})<1$.
\end{theo}
\begin{coro}
If $\|\bf T\|<1$ for any natural matrix
\begin{enumerate}[(i)]
    \item \[\bf \|x-x^{(k)}\|\leq\|T\|^k\|x^{(0)}-x\|\]
    \item \[\bf \|x-x^{(k)}\|\leq\frac{\|T\|^k}{1-\|T\|}\|x^{(1)}-x^{(0)}\|\]
\end{enumerate}
\end{coro}
\begin{theo}[Stein-Rosenberg]
If $a_{ij}\leq 0$, for each $i\neq j$ and $a_{ii}>0$, for each $i=1,2,\ldots$, then one and only one of the following statements holds
\begin{enumerate}[(i)]
    \item $0\leq \rho({\bf T}_g)<\rho({\bf T}_j)<1$;
    \item $1\leq \rho({\bf T}_j)<\rho({\bf T}_g)$;
    \item $\rho({\bf T}_j)=\rho({\bf T}_j)=0$;
    \item $\rho({\bf T}_j)=\rho({\bf T}_j)=1$.
\end{enumerate}
\end{theo}


\title{Maximum a Posteriori Estimation}

\subsection{Maximum a Posteriori Estimation}

Maximum a posteriori (MAP) estimation is a form of approximate
posterior inference. It uses the mode as a point estimate of the
posterior distribution,
\begin{align*}
  \mathbf{z}_\text{MAP}
  &=
  \arg \max_\mathbf{z}
  p(\mathbf{z} \mid \mathbf{x})\\
  &=
  \arg \max_\mathbf{z}
  \log p(\mathbf{z} \mid \mathbf{x}).
\end{align*}
In practice, we work with logarithms of densities to avoid numerical underflow
issues \citep{murphy2012machine}.

The MAP estimate is the most likely configuration of the hidden patterns $\mathbf{z}$
under the model. However, we cannot directly solve this optimization problem
because the posterior is typically intractable. To circumvent this, we use Bayes' rule to
optimize over the joint density,
\begin{align*}
  \mathbf{z}_\text{MAP}
  &=
  \arg \max_\mathbf{z}
  \log p(\mathbf{z} \mid \mathbf{x})\\
  &=
  \arg \max_\mathbf{z}
  \log p(\mathbf{x}, \mathbf{z}).
\end{align*}
This is valid because
\begin{align*}
  \log p(\mathbf{z} \mid \mathbf{x})
  &=
  \log p(\mathbf{x}, \mathbf{z}) - \log p(\mathbf{x})\\
  &=
  \log p(\mathbf{x}, \mathbf{z}) - \text{constant in terms of } \mathbf{z}.
\end{align*}
MAP estimation includes the common scenario of maximum
likelihood estimation as a special case,
\begin{align*}
  \mathbf{z}_\text{MAP}
  &=
  \arg \max_\mathbf{z}
  p(\mathbf{x}, \mathbf{z})\\
  &=
  \arg \max_\mathbf{z}
  p(\mathbf{x}\mid \mathbf{z}),
\end{align*}
where the prior $p(\mathbf{z})$ is flat, placing uniform probability over all
values $\mathbf{z}$ supports. Placing a nonuniform prior can be thought of as
regularizing the estimation, penalizing values away from maximizing
the likelihood, which can lead to overfitting. For example, a normal
prior or Laplace prior on $\mathbf{z}$ corresponds to $\ell_2$ penalization, also
known as ridge regression, and $\ell_1$ penalization, also
known as the LASSO.

Maximum likelihood is also known as cross entropy minimization. For a
data set $\mathbf{x}=\{x_n\}$,
\begin{align*}
  \mathbf{z}_\text{MAP}
  &=
  \arg \max_\mathbf{z}
  \log p(\mathbf{x}\mid \mathbf{z})
  \\
  &=
  \arg \max_\mathbf{z}
  \sum_{n=1}^N \log p(x_n\mid \mathbf{z})
  \\
  &=
  \arg \min_\mathbf{z}
  -\frac{1}{N}\sum_{n=1}^N \log p(x_n\mid \mathbf{z}).
\end{align*}
The last expression can be thought of as an approximation to the cross
entropy between the true data distribution and $p(\mathbf{x}\mid \mathbf{z})$,
using a set of $N$ data points.

\subsubsection{Gradient descent}

To find the MAP estimate of the latent variables $\mathbf{z}$, we use
the gradient of the log joint density
\begin{align*}
  \nabla_\mathbf{z}
  \log p(\mathbf{x}, \mathbf{z})
\end{align*}
and follow it to a (local) optima.
Edward uses TensorFlow's automatic differentiation, making this gradient
computation both simple and efficient to distribute.

Edward currently does not support MAP for discrete latent variables.
This amounts to discrete optimization, which is difficult.

For more details, see the \href{/api/}{API} as well as its
implementation in Edward's code base.

\subsubsection{References}\label{references}


\textbf{Consider the nonlinear initial value problem
\begin{align*}
u_t=u_{xx}+e^u,~~~~x\in [-1,1],~~t>0,~~u(\pm 1,t)=u(x,0)=0,
\end{align*}
for the unknown function $u(x,t)$. To at least eight digits of accuracy, what is $u(0,3.5)$, and what is the time $t_5$ such that $u(0,t_5)=5$.
}
\newline

The numerical scheme used in this problem is the following,
\begin{align*}
u^{n+1}=u^{n}+\Delta t\left[Du^{n}+e^{u^{n}}\right].
\end{align*}
We have obtained the solutions shown in figure 4 and the following results,
\begin{align*}
t_5&=3.53594879,\\
u(0,3.5)&=3.53878310.
\end{align*}

It is evident given the results that the solutions starts developing slowly but, because of the exponential non-linear term, it accelerates quickly. This can be appreciated by realizing that it takes the solution $t=3.5$ to reach the red curve, and only $t_5-3.5 = 0.03594879$ to get to the blue curve.

\begin{figure}[H]
\centering
\includegraphics[scale=0.9]{P3.png}\caption{Numerical solution of the non-linear PDE at $t=3.5$ and at $t_5$, time when $u(0)=5$.}
\end{figure}

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Homework 4, Problem 3 - Francisco Castillo'
clear all; close all; clc;
labelfontsize = 14;
linewidth = 2;

N = 50;
dt = 8e-1/N^3;
[D,x] = cheb(N); % D:(N+1)x(N+1), x:(N+1)x1
x = x(2:N);
D2=D^2;
D2 = D2(2:N,2:N);
u = zeros(size(x));
u35 = u;
u0 = u(N/2);
tobs = [3.5 100];
t = 0;
k=1;
igraph = 1000;
tstep = 0;

while u0<5
    if (t+dt>tobs(k))
        dt=tobs(k)-t;
        t=t+dt;
        k=k+1;
    else
        t=t+dt;
        dt = 8e-1/N^3;
    end

    u = u +dt*(D2*u+exp(u));
    u0 = u(N/2);
    tstep = tstep+1;
    if t == tobs(1)
        u35 = u;
    end
    if (mod(tstep,igraph)==0 || round(u0,10)>=5)
        h1 = plot([1;x;-1],[0;u;0],'b*');
        hold on
        plot([1;x;-1],[0;u;0],'b')
        if (u35~=0)
            h2 = plot([1;x;-1],[0;u35;0],'r*');
            plot([1;x;-1],[0;u35;0],'r')
        end
        grid on
        axis([-1 1 0 5])
        xlabel('$x$','interpreter','latex','fontsize',labelfontsize)
        ylabel('$u(x,t)$','interpreter','latex','fontsize',labelfontsize)
        hold off
        shg
    end
end
legend([h1 h2],'$u(x,t_5)$', '$u(x,3.5)$','interpreter','latex')
saveas(gcf,'Latex/FIGURES/P3','png')
\end{verbatim}


\section{Local Likelihood}

Suppose we have independent observation s
$\{ (\bx_1,y_1),\dots,(\bx_n,y_n) \}$ that are the
realization of a response random variable $Y$ given a
$P \times 1$ covariate vector $\bx$ which we  consider to be
known. Given the
covariate $\bx$, the  response variable $Y$
follows a parametric
distribution
$Y \sim g(y; \theta)$ where $\theta$ is a function of $\bx$.
We are interested in estimating $\theta$ using the observed data.

The log-likelihood function can be written as
\begin{equation}
\label{llf}
l(\theta_1,\dots,\theta_n) = \sum_{i=1}^n \log g(y_i; \theta_i)
\end{equation}
where $\theta_i = s(\bx_i)$. A standard modeling procedure
would assume
a parsimonious form  for
the $\theta_i$s, say $\theta_i = \bx_i'\bb$,
$\bb$ a $P \times 1$ parameter vector. In this case the log-likelihood
$l(\theta_1,\dots,\theta_n)$ would be  a function of the
parameter $\bb$ that could be estimated by maximum
likelihood, that is by finding the $\bbh$ that maximizes
$l(\theta_1,\dots,\theta_n)$.

The local likelihood approach is based on a more general assumption,
namely that  $s(\bx)$ is a ``smooth'' function of the
covariate $\bx$.
Without more restrictive assumptions, the maximum likelihood estimate of
$\bth=\{s(\bx_1),\dots,s(\bx_n)\} $ is no
longer useful because of over-fitting. Notice for example that for the
case of regression with all the $\bx_i$s distinct the maximum likelihood estimate  would simply
reproduce the data.

Suppose we are interested in estimating only
$\theta_{0} = \theta(\bx_0)$ for a fixed covariate value $\bx_0$.
The local likelihood estimation approach is to assume that
there is some neighborhood $N_0$ of covariates that are ``close''
enough to
$\bx_0$ such that the
 data $\{ (\bx_i,y_i) ; \bx_i \in N_0\}$ contain
information about $\theta_0$ through some {\sl link function} $\eta$ of the
form
\begin{eqnarray}
\label{etadef}
\theta_0 &=& s(\bx_0) \equiv \eta(\bx_0, \bb) \mbox
{ and }\\
\label{etadef2}
\theta_i &=& s(\bx_i) \approx \eta(\bx_i,\bb), \mbox
{ for } \bx_i \in N_0 .
\end{eqnarray}
Notice that we are abusing notation here since we are considering a
different $\bb$ for every $\bx_0$. Throughout the work we will be
acting as if $\theta_0$ is the only parameter of interest and
therefore not indexing variables that depend on the choice of
$\bx_0$.

The local likelihood estimate of $\theta_0$ is obtained by
assuming that, for data in $N_0$, the true distribution of the data,
$g(y_i;\theta_i)$ is approximated by
\begin{equation}
\label{approxdist}
f(y_i;\bx_i,\bb) \equiv  g(y_i; \eta(\bx_i,\bb) ),
\end{equation}
finding the
$\bbh$ maximizes the
local log-likelihood equation
\begin{equation}
\label{lle} %local likelihood equation
l_0(\bb) = \sum_{\bx_i \in N_0} w_i \, \log f(y_i; \bb),
\end{equation}
and then using Equation (\ref{etadef}) to obtain the local
likelihood estimate $\hat{\theta}_0$. Here $w_i$ is a weight
coefficient related to the ``distance'' between $\bx_0$ and
$\bx_i$. In order to obtain
a useful estimate of $\theta_0$, we need $\bb$ to be of ``small''
enough dimension so that we fit a parsimonious model within $N_0$.

Hastie and Tibshirani (1987) \nocite{hast:tibs:1987} discuss the case
where the covariate $\bx$
is a real
valued scalar and the link function is linear
\[
\eta(x_i, \bb) = \beta_0 + x_i \beta_1
\]
Notice that in this case, the assumption being made is that the
parameter function $s(x_i)$ is approximately linear within
``small'' neighborhoods of $x_0$, i.e. locally linear.

Staniswalis (1989) \nocite{stan:1989} presents a similar approach. In
this case the
covariate $\bx$ is  allowed to be a vector, and the link function
is a constant
\[
\eta(\bx_i, \beta) = \beta
\]
The assumption being made here is that the
parameter function $s(x_i)$ is locally constant.

If we assumes a density function
of the form
\begin{equation}
\label{loess}
\log g(y_i; \theta_i) = C  + (y_i - \theta_i)^2 / \phi
\end{equation}
where $K$ and $\phi$ are constants that do not depend on the
$\theta_i$s,
local regression may be considered a special case of local
likelihood estimation.

Notice that in this case the local likelihood estimate is going to be
equivalent to the estimate obtained by
minimizing a sum of squares equation. The approach in
Cleveland (1979) \nocite{clev:1979} and Cleveland and Devlin (1988)
\nocite{clev:devl:1988} is to consider a real valued covariate
and the polynomial link function
\[
\eta(\bx_i,\bb) = \sum_{j=0}^d x_i^j \beta_j.
\]

In general, the approach of local likelihood estimation, including the
three above-mentioned examples, is to assume that
for ``small'' neighborhoods around $\bx_0$, the distribution of
the data is approximated by a distribution that depends on a constant
parameter $\bb(\bx_0)$, i.e. we have locally parsimonious models. This
allows us to
use the usual estimation technique of maximum likelihood. However, in
the local version of maximum likelihood we often
have an a priori belief that points
``closer'' to $\bx_0$ contain more information about
$\theta_0$, which suggest a weighted approach.

The asymptotic theory presented in, for example,  Staniswalis (1989)
\nocite{stan:1989} and
Loader (1986) \nocite{load:1996} is developed under the assumption that
as the size (or radius) of some neighborhood of the covariate of
interest $\bx_0$ tends to 0, the difference between the true and
approximating distributions within such neighborhood becomes
negligible. Furthermore, we assume that despite the fact that the
neighborhoods become arbitrarily small, the
number of data points in the neighborhood somehow tends to $\infty$.
The idea is that, asymptotically, the behavior of the
data within a given neighborhood, is like the  one assumed in
classical asymptotic theory for non-IID data: The small window size
assure that the difference between the true and approximating models is
negligible and the large number of independent observations is
available
to estimate a parameter of fixed dimension that completely specifies
the joint distribution. This concept motivates the
approach taken in the following sections to derive a model selection
criteria.