\section{The matrix of a linear transformation}

\begin{outcome}
  \begin{enumerate}
  \item Find the matrix of a linear transformation and determine the
    action on a vector in $\R^n$.
  \end{enumerate}
\end{outcome}

In the above examples, the action of the linear transformations was to
multiply by a matrix.  It turns out that this is always the case for
linear transformations.  If $T$ is \textbf{any} linear transformation
which maps $\R^{n}$ to $\R^{m}$, there is \textbf{always} an
$m\times n$-matrix $A$ with the property that
\begin{equation}
T(\vect{x}) = A\vect{x} \label{matrix-of-transf}
\end{equation}
for all $\vect{x} \in \R^{n}$.

\begin{theorem}{Matrix of a linear transformation}{matrix-linear-transformation}
Let $T:\R^{n}\to \R^{m}$ be a linear transformation. Then we can find a matrix $A$ such that $T(\vect{x}) = A\vect{x}$.
 In this case, we say that $T$ is {\em determined} or {\em induced}
by the matrix $A$.
\end{theorem}

Here is why. Suppose $T:\R^{n}\to \R^{m}$ is a linear transformation and you want to find
the matrix defined by this linear transformation as described in {\eqref{matrix-of-transf}}.
 Note that
\begin{equation*}
\vect{x} =\begin{mymatrix}{c}
x_{1} \\
x_{2} \\
\vdots \\
x_{n}
\end{mymatrix} = x_{1}\begin{mymatrix}{c}
1 \\
0 \\
\vdots \\
0
\end{mymatrix} + x_{2}\begin{mymatrix}{c}
0 \\
1 \\
\vdots \\
0
\end{mymatrix} +\ldots + x_{n}\begin{mymatrix}{c}
0 \\
0 \\
\vdots \\
1
\end{mymatrix} = \sum_{i=1}^{n}x_{i}\vect{e}_{i}
\end{equation*}
where $\vect{e}_{i}$ is the $i\th$ column of $I_n$, that is the $n \times
1$ vector which has zeros in every slot but the $i\th$ and a 1 in
this slot.

Then since $T$ is linear,
\begin{eqnarray*}
T(\vect{x})&=&\sum_{i=1}^{n}x_{i}T(\vect{e}_{i}) \\
&=&\begin{mymatrix}{ccc}
| &  & | \\
T(\vect{e}_{1}) & \cdots & T(\vect{e}_{n}) \\
| &  & |
\end{mymatrix} \begin{mymatrix}{c}
x_{1} \\
\vdots \\
x_{n}
\end{mymatrix} \\
&=& A\begin{mymatrix}{c}
x_{1} \\
\vdots \\
x_{n}
\end{mymatrix}
\end{eqnarray*}
Therefore,  the desired matrix is obtained from constructing the $i\th$
column as $T(\vect{e}_{i})$. We state this formally as the
following theorem.

\begin{theorem}{Matrix of a linear transformation}{matrix-of-linear-transformation}
Let $T: \R^{n} \to \R^{m}$ be a linear transformation. Then the matrix $A$ satisfying $T(\vect{x})=A\vect{x}$\index{linear transformation!matrix} is given by
\begin{equation*}
A=
\begin{mymatrix}{ccc}
| &  & | \\
T(\vect{e}_{1}) & \cdots & T(\vect{e}_{n}) \\
| &  & |
\end{mymatrix}
\end{equation*}
where $\vect{e}_{i}$ is the $i\th$ column of $I_n$, and then $T(\vect{e}_{i}
)$ is the $i\th$ column of $A$.
\end{theorem}

The following Corollary is an essential result.

\begin{corollary}{Matrix and linear transformation}{matrix-linear-transformation-equivalence}
A transformation $T$ is a linear transformation if and only if it is a matrix transformation.
\end{corollary}

Consider the following example.

\begin{example}{The matrix of a linear transformation}{matrix-of-linear-transformation}
Suppose $T$ is a linear transformation, $T:\R^{3}\rightarrow \R^{2}$ where
\begin{equation*}
T\begin{mymatrix}{r}
1 \\
0 \\
0
\end{mymatrix} =\begin{mymatrix}{r}
1 \\
2
\end{mymatrix} ,\ T\begin{mymatrix}{r}
0 \\
1 \\
0
\end{mymatrix} =\begin{mymatrix}{r}
9 \\
-3
\end{mymatrix} ,\ T\begin{mymatrix}{r}
0 \\
0 \\
1
\end{mymatrix} =\begin{mymatrix}{r}
1 \\
1
\end{mymatrix}
\end{equation*}
Find the matrix $A$ of $T$ such that $T (\vect{x})=A\vect{x}$  for all $\vect{x}$.
\end{example}

\begin{solution} By Theorem~\ref{thm:matrix-of-linear-transformation} we construct $A$ as follows:
\begin{equation*}
A =
\begin{mymatrix}{ccc}
| &  & | \\
T(\vect{e}_{1}) & \cdots & T(\vect{e}_{n}) \\
| &  & |
\end{mymatrix}
\end{equation*}

In this case, $A$ will be a $2 \times 3$-matrix, so we need to find $T
(\vect{e}_1), T (\vect{e}_2)$, and $T (\vect{e}_3
)$. Luckily, we have been given these values so we can fill in
$A$ as needed, using these vectors as the columns of $A$.  Hence,
\begin{equation*}
A=\begin{mymatrix}{rrr}
1 & 9 & 1 \\
2 & -3 & 1
\end{mymatrix}
\end{equation*}
\end{solution}

In this example, we were given the resulting vectors of $T (\vect{e}_1),
T (\vect{e}_2)$, and $T (\vect{e}_3)$. Constructing the matrix $A$ was simple, as we
could simply use these vectors as the columns of $A$. The next example shows how to find $A$ when we are not given the $T (\vect{e}_i)$ so clearly.

\begin{example}{The matrix of linear transformation: inconveniently \\ defined}{2x2-inconvenient-matrix-of-linear-transformation}
Suppose $T$ is a linear transformation, $T:\R^{2}\rightarrow \R^{2}$ and
\begin{equation*}
T\begin{mymatrix}{r}
1 \\
1
\end{mymatrix} =\begin{mymatrix}{r}
1 \\
2
\end{mymatrix} ,\ T\begin{mymatrix}{r}
0 \\
-1
\end{mymatrix} =\begin{mymatrix}{r}
3 \\
2
\end{mymatrix}
\end{equation*}
Find the matrix $A$ of $T$ such that $T (\vect{x})=A\vect{x}$  for all $\vect{x}$.
\end{example}

\begin{solution} By Theorem~\ref{thm:matrix-of-linear-transformation} to find this matrix, we need to determine the action of $T$ on
$\vect{e}_{1}$ and $\vect{e}_{2}$. In Example~\ref{exa:matrix-of-linear-transformation}, we were given these resulting vectors.
However, in this example, we have been given $T$ of two different vectors. How can we find out the action
of $T$ on $\vect{e}_{1}$ and $\vect{e}_{2}$? In particular for $\vect{e}_{1}$, suppose there exist $x$ and $y$ such that
\begin{equation}
\begin{mymatrix}{r}
1 \\
0
\end{mymatrix} = x\begin{mymatrix}{r}
1\\
1
\end{mymatrix} +y\begin{mymatrix}{r}
0 \\
-1
\end{mymatrix}
\label{matrix-values}
\end{equation}

Then, since $T$ is linear,
\begin{equation*}
T\begin{mymatrix}{r}
1 \\
0
\end{mymatrix}  = x T\begin{mymatrix}{r}
1 \\
1
\end{mymatrix} +y T\begin{mymatrix}{r}
0 \\
-1
\end{mymatrix}
\end{equation*}

Substituting in values, this sum becomes
\begin{equation}
T\begin{mymatrix}{r}
1 \\
0
\end{mymatrix} =
 x\begin{mymatrix}{r}
1 \\
2
\end{mymatrix} +y\begin{mymatrix}{r}
3 \\
2
\end{mymatrix}
\label{matrix-values2}
\end{equation}

Therefore, if we know the values of $x$ and $y$ which satisfy {\eqref{matrix-values}}, we can substitute these into equation {\eqref{matrix-values2}}. By doing so,
we find $T(\vect{e}_1)$ which is the first column of the matrix $A$.

We proceed to find $x$ and $y$. We do so by solving {\eqref{matrix-values}}, which can be done by solving the system
\begin{equation*}
\begin{array}{c}
x = 1 \\
x - y = 0
\end{array}
\end{equation*}

We see that $x=1$ and $y=1$ is the solution to this system.
Substituting these values into equation {\eqref{matrix-values2}}, we have
\begin{equation*}
T\begin{mymatrix}{r}
1 \\
0
\end{mymatrix} =
 1 \begin{mymatrix}{r}
1 \\
2
\end{mymatrix} + 1 \begin{mymatrix}{r}
3 \\
2
\end{mymatrix}
=
 \begin{mymatrix}{r}
1 \\
2
\end{mymatrix} + \begin{mymatrix}{r}
3 \\
2
\end{mymatrix}
=
\begin{mymatrix}{r}
4 \\
4
\end{mymatrix}
\end{equation*}

Therefore $\begin{mymatrix}{r}
4 \\
4
\end{mymatrix}$
is the first column of $A$.

Computing the second column is done in the same way, and is left as an exercise.

The resulting matrix $A$ is given by
\begin{equation*}
A
=
\begin{mymatrix}{rr}
4 & -3 \\
4 & -2
\end{mymatrix}
\end{equation*}
\end{solution}

This example illustrates a very long procedure for finding the matrix of $A$. While this method is reliable and
will always result in the correct matrix $A$, the following procedure provides an alternative method.

\begin{procedure}{Finding the matrix of inconveniently defined linear transformation}{finding-matrix-of-linear-transformation}
Suppose $T:\R^{n}\rightarrow \R^{m}$ is a linear transformation. Suppose there exist vectors $\set{\vect{a}_{1},\ldots,\vect{a}_{n}} $ in $\R^{n}$ such that $\begin{mymatrix}{ccc}
\vect{a}_{1} & \cdots & \vect{a}_{n}
\end{mymatrix} ^{-1}$ exists, and
\begin{equation*}
T (\vect{a}_{i})=\vect{b}_{i}
\end{equation*}
Then the matrix of $T$ must be of the form
\begin{equation*}
\begin{mymatrix}{ccc}
\vect{b}_{1} & \cdots & \vect{b}_{n}
\end{mymatrix} \begin{mymatrix}{ccc}
\vect{a}_{1} & \cdots & \vect{a}_{n}
\end{mymatrix} ^{-1}
\end{equation*}
\end{procedure}

We will illustrate this procedure in the following example. You may also find it useful
to work through Example~\ref{exa:2x2-inconvenient-matrix-of-linear-transformation} using this procedure.

\begin{example}{Matrix of a linear transformation \\ given inconveniently}{inconvenient-matrix-linear-transformation}
Suppose $T:\R^{3}\rightarrow \R^{3}$ is a linear
transformation and
\begin{equation*}
T\begin{mymatrix}{r}
1 \\
3 \\
1
\end{mymatrix} =\begin{mymatrix}{r}
0 \\
1 \\
1
\end{mymatrix} ,T\begin{mymatrix}{r}
0 \\
1 \\
1
\end{mymatrix} =\begin{mymatrix}{r}
2 \\
1 \\
3
\end{mymatrix} ,T\begin{mymatrix}{r}
1 \\
1 \\
0
\end{mymatrix} =\begin{mymatrix}{r}
0 \\
0 \\
1
\end{mymatrix}
\end{equation*}
Find the matrix of this linear transformation.
\end{example}

\begin{solution}
By Procedure~\ref{proc:finding-matrix-of-linear-transformation},
$A=  \begin{mymatrix}{rrr}
1 & 0 & 1 \\
3 & 1 & 1 \\
1 & 1 & 0
\end{mymatrix} ^{-1}$ and
 $B=\begin{mymatrix}{rrr}
0 & 2 & 0 \\
1 & 1 & 0 \\
1 & 3 & 1
\end{mymatrix}$

Then, Procedure~\ref{proc:finding-matrix-of-linear-transformation} claims that the matrix of $T$ is
\begin{equation*}
C= BA^{-1}
=\begin{mymatrix}{rrr}
2 & -2 & 4 \\
0 & 0 & 1 \\
4 & -3 & 6
\end{mymatrix}
\end{equation*}

Indeed you can first verify that $T(\vect{x})=C\vect{x}$ for the 3 vectors above:

\begin{equation*}
 \begin{mymatrix}{ccc}
2 & -2 & 4 \\
0 & 0 & 1 \\
4 & -3 & 6
\end{mymatrix} \begin{mymatrix}{c}
1 \\
3 \\
1
\end{mymatrix} =\begin{mymatrix}{c}
0 \\
1 \\
1
\end{mymatrix} ,\ \begin{mymatrix}{ccc}
2 & -2 & 4 \\
0 & 0 & 1 \\
4 & -3 & 6
\end{mymatrix} \begin{mymatrix}{c}
0 \\
1 \\
1
\end{mymatrix} =\begin{mymatrix}{c}
2 \\
1 \\
3
\end{mymatrix}
\end{equation*}
\begin{equation*}
\begin{mymatrix}{ccc}
2 & -2 & 4 \\
0 & 0 & 1 \\
4 & -3 & 6
\end{mymatrix} \begin{mymatrix}{c}
1 \\
1 \\
0
\end{mymatrix} =\begin{mymatrix}{c}
0 \\
0 \\
1
\end{mymatrix}
\end{equation*}

But more generally $T(\vect{x})= C\vect{x}$ for any $\vect{x}$. To see this, let $\vect{y}=A^{-1}\vect{x}$ and then using linearity of $T$:
\[ T(\vect{x})= T(A\vect{y}) = T \paren{\sum_i \vect{y}_i\vect{a}_i} = \sum \vect{y}_i T(\vect{a}_i) \sum \vect{y}_i \vect{b}_i = B\vect{y} = BA^{-1}\vect{x} = C\vect{x}\]
\end{solution}

Recall the dot product discussed earlier. Consider the map $\vect{v}$\textbf{$\mapsto $}
$\proj_{\vect{u}}(\vect{v}) $ which takes a vector a transforms it to its projection onto a given vector $\vect{u}$. It turns out that
this map is linear, a result which follows from the properties of the
dot product. This is shown as follows.
\begin{eqnarray*}
\proj_{\vect{u}}(k \vect{v}+ p \vect{w})
&=&\paren{\vspace{0.05in}\frac{(k \vect{v}+ p \vect{w})\dotprod \vect{u}}{
\vect{u}\dotprod \vect{u}}} \vect{u} \\
&=& k  \paren{\vspace{0.05in}\frac{
\vect{v}\dotprod \vect{u}}{\vect{u}\dotprod \vect{u}}} \vect{u}+p \paren{\vspace{
0.05in}\frac{\vect{w}\dotprod \vect{u}}{\vect{u}\dotprod \vect{u}}} \vect{u} \\
&=& k \; \proj_{\vect{u}}(\vect{v}) +p \; \proj
_{\vect{u}}(\vect{w})
\end{eqnarray*}

Consider the following example.

\begin{example}{Matrix of a projection map}{projection-matrix}
Let $\vect{u} = \begin{mymatrix}{r}
1 \\
2 \\
3
\end{mymatrix}$ and let $T$ be the projection map $T: \R^3 \to \R^3$ defined by
\[
T(\vect{v}) = \proj_{\vect{u}}(\vect{v})
\]
for any $\vect{v} \in \R^3$.
\begin{enumerate}
\item Does this transformation come from
multiplication by a matrix?
\item If so, what is the matrix?
\end{enumerate}
\end{example}

\begin{solution}
\begin{enumerate}
\item
First, we have just seen that $T (\vect{v}) = \proj_{\vect{u}}(\vect{v})$ is linear. Therefore by Theorem~\ref{thm:matrix-linear-transformation}, we can find a matrix $A$ such that $T(\vect{x}) = A\vect{x}$.

\item
The columns of the matrix for $T$ are defined above as $T(\vect{e}_{i})$.
It follows that $T(\vect{e}_{i}) = \proj
_{\vect{u}}(\vect{e}_{i}) $ gives the $i\th$ column of the
desired matrix. Therefore, we need to find
\begin{equation*}
\proj_{\vect{u}}(\vect{e}_{i}) = \paren{\vspace{0.05in}
\frac{\vect{e}_{i}\dotprod \vect{u}}{\vect{u}\dotprod \vect{u}}}
\vect{u}
\end{equation*}
For the given vector $\vect{u}$ , this implies the columns of the desired
matrix are
\begin{equation*}
\vspace{0.05in}\frac{1}{14}\begin{mymatrix}{r}
1 \\
2 \\
3
\end{mymatrix} ,\vspace{0.05in}\frac{2}{14}\begin{mymatrix}{r}
1 \\
2 \\
3
\end{mymatrix} ,\vspace{0.05in}\frac{3}{14}\begin{mymatrix}{r}
1 \\
2 \\
3
\end{mymatrix}
\end{equation*}
which you can verify.
Hence the matrix of $T$ is
\begin{equation*}
\vspace{0.05in}\frac{1}{14}\begin{mymatrix}{rrr}
1 & 2 & 3 \\
2 & 4 & 6 \\
3 & 6 & 9
\end{mymatrix}
\end{equation*}
\end{enumerate}
\end{solution}


\documentclass[../searching.tex]{subfiles}
\begin{document}
To search in a sorted array or string using brute force with a for loop, it takes $O(n)$ time. Binary search is designed to reduce search time if the array or string is already sorted. It uses the divide and conquer method; each time we compare our target with the middle element of the array and with the comparison result to decide the next search region: either the left half or the right half. Therefore, each step we filter out half of the array which gives the time complexity function $T(n) = T(n/2) + O(1)$, which decrease the time complexity to $O(\log n)$.

Binary Search can be applied to different tasks:
\begin{enumerate}
    \item Find Exact target, find the first position that value>= target, find the last position that value<= target.(this is called lower\_bound, and upper\_bound.
\end{enumerate}

\subsection{Standard Binary Search and Python Module bisect}
Binary search is usually carried out on a Static sorted array or 2D matrix. There are three basic cases: (1) find the exact target that value = target; If there are duplicates, we are more likely to be asked to (2) find the first position that has value >= target; (3) find the first position that has value <= target. Here, we use two example array: one without duplicates and the other has duplicates.
\begin{lstlisting}[language=Python]
a = [2, 4, 5, 9]
b = [0, 1, 1, 1, 1, 1]
\end{lstlisting}

\paragraph{Find the Exact Target} This is the most basic application of binary search. We can set two pointers, l and r. Each time we compute the middle position, and check if it is equal to the target. If it is, return the position; if it is smaller than the target, move to the left half, otherwise, move to the right half. The Python code is given:
\begin{lstlisting}[language=Python]
def standard_binary_search(lst, target):
    l, r = 0, len(lst) - 1
    while l <= r:
        mid = l + (r - l) // 2
        if lst[mid] == target:
            return mid
        elif lst[mid] < target:
            l = mid + 1
        else:
            r = mid - 1
    return -1 # target is not found
\end{lstlisting}
Now, run the example:
\begin{lstlisting}[language=Python]
print("standard_binary_search: ", standard_binary_search(a,3), standard_binary_search(a,4), standard_binary_search(b, 1))
\end{lstlisting}
The print out is:
\begin{lstlisting}
standard_binary_search:  -1 1 2
\end{lstlisting}
From the example, we can see that multiple \textbf{duplicates} of the target exist, it can possibly return any one of them. And for the case when the target does not exist, it simply returns -1. In reality, we might need to find a position where we can potentially insert the target to keep the sorted array sorted. There are two cases: (1) the first position that we can insert, which is the first position that has value>= target (2) and the last position we can insert, which is the first position that has value > target. For example, if we try to insert 3 in a, and 1 in b, the first position should be 1 and 1 in each array, and the last position is 1 and 6 instead.  For these two cases, we have a Python built-in Module \textbf{bisect} which offers two methods: bisect\_left() and bisect\_right() for these two cases respectively.

\paragraph{Find the First Position that value >= target} This way the target position separates the array into two halves: value < target, target\_position, value>= target. In order to meet the purpose, we make sure that if value < target, we move to the right side, else, move to the left side.
\begin{lstlisting}[language=Python]
# bisect_left, no longer need to check the mid element,
# it separate the list in to two halfs: value < target, mid,  value >= target
def bisect_left_raw(lst, target):
    l, r = 0, len(lst)-1
    while l <= r:
        mid = l + (r-l)//2
        if lst[mid] < target: # move to the right half if the value < target, till
            l = mid + 1 #[mid+1, right]
        else:# move to the left half is value >= target
            r = mid - 1  #[left, mid-1]
    return l # the final position is where
\end{lstlisting}
% Now insert the value with:
% \begin{lstlisting}[language=Python]
% lst.insert(l+1, target)
% \end{lstlisting}

\paragraph{Find the First Position that value > target} This way the target position separates the array into two halves: value <= target, target\_position, value> target. Therefore, we simply change the condition of if value < target to if value <= target, then we move to the right side.
\begin{lstlisting}[language=Python]
#bisect_right: separate the list into two halfs: value<= target, mid, value > target
def bisect_right_raw(lst, target):
    l, r = 0, len(lst)-1
    while l <= r:
        mid = l + (r-l)//2
        if lst[mid] <= target:
            l = mid + 1
        else:
            r = mid -1
    return l
\end{lstlisting}
Now, run an example:
\begin{lstlisting}[language=Python]
print("bisect left raw: find 3 in a :", bisect_left_raw(a,3), 'find 1 in b: ', bisect_left_raw(b, 1))
print("bisect right raw: find 3 in a :", bisect_right_raw(a, 3), 'find 1 in b: ', bisect_right_raw(b, 1))
\end{lstlisting}
The print out is:
\begin{lstlisting}
bisect left raw: find 3 in a : 1 find 1 in b:  1
bisect right raw: find 3 in a : 1 find 1 in b:  6
\end{lstlisting}

\paragraph{Bonus} For the last two cases, if we return the position as l-1, then we get the last position that value < target, and the last position value <= target.

\paragraph{Python Built-in Module bisect} This module provides support for maintaining a list in sorted order without having to sort the list after each insertion. It offers six methods as shown in Table~\ref{tab:method_bisect}. However, only two are most commonly used: bisect\_left and bisect\_right.
\begin{table}[h]
\begin{small}
\centering
\noindent\captionof{table}{ Methods of \textbf{bisect}}
 \noindent \begin{tabular}{|p{0.25\columnwidth}|p{0.75\columnwidth}| }
  \hline
Method & Description   \\ \hline
bisect\_left(a, x, lo=0, hi=len(a)  &  The parameters lo and hi may be used to specify a subset of the list; the function is the same as bisect\_left\_raw  \\\hline
bisect\_right(a, x, lo=0, hi=len(a)  &  The parameters lo and hi may be used to specify a subset of the list; the function is the same as bisect\_right\_raw  \\\hline
bisect(a, x, lo=0, hi=len(a))  &Similar to bisect\_left(), but returns an insertion point which comes after (to the right of) any existing entries of x in a.\\ \hline
insort\_left(a, x, lo=0, hi=len(a))  &This is equivalent to a.insert(bisect.bisect\_left(a, x, lo, hi), x).\\ \hline
insort\_right(a, x, lo=0, hi=len(a)) & This is equivalent to a.insert(bisect.bisect\_right(a, x, lo, hi), x).\\ \hline
insort(a, x, lo=0, hi=len(a)) & Similar to insort\_left(), but inserting x in a after any existing entries of x.\\ \hline
\end{tabular}
  \label{tab:method_bisect}
  \end{small}
\end{table}
Let's see come examplary code:
\begin{lstlisting}[language=Python]
from bisect import bisect_left,bisect_right, bisect
print("bisect left: find 3 in a :", bisect_left(a,3), 'find 1 in b: ', bisect_left(b, 1)) # lower_bound, the first position that value>= target
print("bisect right: find 3 in a :", bisect_right(a, 3), 'find 1 in b: ', bisect_right(b, 1)) # upper_bound, the last position that value <= target
\end{lstlisting}
The print out is:
\begin{lstlisting}
bisect left: find 3 in a : 1 find 1 in b:  1
bisect right: find 3 in a : 1 find 1 in b:  6
\end{lstlisting}
\subsection{Binary Search in Rotated Sorted Array}
\label{concept_binary_search_in_array}
The extension of the standard binary search is on array that the array is ordered in its own way like rotated array.

\paragraph{Binary Search in Rotated Sorted Array } (See LeetCode problem, 33. Search in Rotated Sorted Array (medium). Suppose an array (without duplicates) sorted in ascending order is rotated at some pivot unknown to you beforehand. (i.e., 0 1 2 4 5 6 7 might become 4 5 6 7 0 1 2). You are given a target value to search. If found in the array return its index, otherwise return -1. You may assume no duplicate exists in the array.
\begin{lstlisting}[numbers=none]
Example 1:

Input: nums = [3, 4,5,6,7,0,1,2], target = 0
Output: 5

Example 2:

Input: nums = [4,5,6,7,0,1,2], target = 3
Output: -1
\end{lstlisting}

In the rotated sorted array, the array is not purely monotonic. Instead, there is one drop in the array because of the rotation, where it cuts the array into two parts. Suppose we are starting with a standard binary search with example 1, at first, we will check index 3, then we need to move to the right side?  Assuming we compare our middle item with the left item,
\begin{lstlisting}[numbers=none]
if nums[mid] > nums[l]: # the left half is sorted
elif nums[mid] < nums[l]: # the right half is sorted
else: # for case like [1,3], move to the right half
\end{lstlisting}
For a standard binary search, we simply need to compare the target with the middle item to decide which way to go. In this case, we can use objection. Check which side is sorted, because no matter where the left, right and the middle index is, there is always one side that is sorted. So if the left side is sorted, and the value is in the range of the [left, mid], then we move to the left part, else we object the left side, and move to the right side instead.
\begin{figure}[h]
    \centering
    \includegraphics[width=0.7\columnwidth]{fig/rotated_array.png}
    \caption{Example of Rotated Sorted Array}
    \label{fig:rotated_sorted_array}
\end{figure}

The code is shown:
\begin{lstlisting}[language=Python]
'''implemente the rotated binary search'''
def RotatedBinarySearch(nums, target):
    if not nums:
        return -1

    l,r = 0,len(nums)-1
    while l<=r:
        mid = l+ (r-l)//2
        if nums[mid] == target:
            return mid
         if nums[l] < nums[mid]: # if the left part is sorted
                if nums[l] <= target <= nums[mid]:
                    r = mid-1
                else:
                    l = mid+1
            elif nums[l] > nums[mid]: # if the right side is sorted
                if nums[mid] <= target <= nums[r]:
                    l = mid+1
                else:
                    r = mid-1
            else:
                l = mid + 1
    return -1
\end{lstlisting}
\begin{bclogo}[couleur = blue!30, arrondi=0.1,logo=\bccrayon,ombre=true]{What happens if there is duplicates in the rotated sorted array? } In fact, similar comparing rule applies:
\begin{lstlisting}[numbers=none]
if nums[mid] > nums[l]: # the left half is sorted
elif nums[mid] < nums[l]: # the right half is sorted
else: # for case like [1,3], or [1, 3, 1, 1, 1] or [3, 1, 2, 3, 3, 3]
   only l++
\end{lstlisting}
\end{bclogo}

%%%%%%%%%%%%%%binary search on result space%%%%%%%
\subsection{Binary Search on Result Space}
If the question gives us the context: the target is in the range [left, right], we need to search the first or last position that satisfy a condition function. We can apply the concept of standard binary search and bisect\_left and bisect\_right and its mutant. Where we use the condition function to replace the value comparison between target and element at middle position. The steps we need:
\begin{enumerate}
    \item get the result search range [l, r] which is the initial value for l and r pointers.
    \item decide the valid function to replace such as if lst[mid] < target
    \item decide which binary search we use: standard, bisect\_left/ bisect\_right or its mutant.
\end{enumerate}

For example:
\begin{examples}[resume]
\item \textbf{441. Arranging Coins (easy)}. You have a total of n coins that you want to form in a staircase shape, where every k-th row must have exactly k coins. Given n, find the total number of full staircase rows that can be formed. n is a non-negative integer and fits within the range of a 32-bit signed integer.
\begin{lstlisting}[numbers=none]
Example 1:

n = 5

The coins can form the following rows:
*
* *
* *

Because the 3rd row is incomplete, we return 2.
\end{lstlisting}

\textbf{Analysis: } Given a number n>=1, the minimum row is 1, and the maximum is n. Therefore, our possible result range is [1, n]. These can be treated as indexes of the sorted array. For a given row, we write a function to check if it is possible. We need a function $r* (r+1) // 2 <= n$. For this problem, we need to search in the range of [1, n] to find the last position that is valid. This is bisect\_left or bisect\_right, where we use the function replace the condition check:
\begin{lstlisting}[language=Python]
def arrangeCoins(self, n):
    def isValid(row):
        return (row*(row+1))//2 <= n
    # we need to find the last position that is valid (<=)
    def bisect_right():
        l, r = 1, n
        while l <= r:
            mid = l + (r-l) // 2
            if isValid(mid): # replaced compared with the standard binary search
                l = mid + 1
            else:
                r = mid - 1
        return l-1
    return bisect_right()
\end{lstlisting}
\item \textbf{278. First Bad Version.} You are a product manager and currently leading a team to develop a new product. Unfortunately, the latest version of your product fails the quality check. Since each version is developed based on the previous version, all the versions after a bad version are also bad.

Suppose you have n versions [1, 2, ..., n] and you want to find out the first bad one, which causes all the following ones to be bad.

You are given an API bool isBadVersion(version) which will return whether version is bad. Implement a function to find the first bad version. You should minimize the number of calls to the API.

Solution: we keep doing binary search until we have searched all possible areas.
\begin{lstlisting}[language = Python]
class Solution(object):
    def firstBadVersion(self, n):
        """
        :type n: int
        :rtype: int
        """
        l,r=0,n-1
        last = -1
        while l<=r:
            mid = l+(r-l)//2
            if isBadVersion(mid+1): #move to the left, mid is index, s
                r=mid-1
                last = mid+1 #to track the last bad one
            else:
                l=mid-1
        return last
\end{lstlisting}
\end{examples}
% \subsection{Bisection Method} (second edition)
% The binary search principle can be used to find the root of a function that may be difficult to compute mathematically. We have not seen any problems that require this method on LeetCode yet. Thus we define the problem as:

% Find the monthly payment for a loan: You want to buy a car using loan and want to pay in monthly installment of d d
% \subsection{Python Library}
% Python has \textbf{bisect} module for binary search.
% \begin{lstlisting}[numbers=none]
% bisect.bisect_left(a,    x):  Return the leftmost index where we can  insert x into a to maintain sorted order! Leftmost rl that satisfy: x<=a[rl]

% bisect.bisect_right(a,    x):  Return the rightmost index where we can  insert x into a to maintain sorted order! Right most rr that satisfy: x>=a[rr]
% \end{lstlisting}
% For example:
% \begin{lstlisting}[language=Python]
% from bisect import bisect_left,bisect_right
% a = [1,    2,    3,    3,    3,    4,    5]
% p1, p2= bisect_left(a,3), bisect_right(a, 3)
% print(p1, p2)
% # output
% # 2, 5
% \end{lstlisting}

\subsection{LeetCode Problems}
\begin{examples}
\item \textbf{35. Search Insert Position (easy).} Given a sorted array and a target value, return the index if the target is found. If not, return the index where it would be if it were inserted in order.

You can assume that there are no duplicates in the array.
\begin{lstlisting}[numbers=none]
Example 1:

Input: [1,3,5,6], 5
Output: 2

Example 2:
Input: [1,3,5,6], 2
Output: 1

Example 3:
Input: [1,3,5,6], 7
Output: 4

Example 4:
Input: [1,3,5,6], 0
Output: 0
\end{lstlisting}

\textbf{Solution: Standard Binary Search Implementation.} For this problem, we just standardize the Python code of binary search, which takes $O(logn)$ time complexity and O(1) space complexity without using recursion function. In the following code, we use exclusive right index with len(nums), therefore it stops if l == r; it can be as small as 0 or as large as n of the array length for numbers that are either smaller or equal to the nums[0] or larger or equal to nums[-1]. We can also make the right index inclusive.
\begin{lstlisting}[language = Python]
# exclusive version
def searchInsert(self, nums, target):
    l, r = 0, len(nums) #start from 0, end to the len (exclusive)
    while l < r:
        mid = (l+r)//2
        if nums[mid] < target: #move to the right side
            l = mid+1
        elif nums[mid] > target: #move to the left side, not mid-1
             r= mid
        else: #found the traget
            return mid
    #where the position should go
    return l
\end{lstlisting}

\begin{lstlisting}[language = Python]
# inclusive version
def searchInsert(self, nums, target):
   l = 0
    r = len(nums)-1
    while l <= r:
        m = (l+r)//2
        if target > nums[m]: #search the right half
            l = m+1
        elif target < nums[m]: # search for the left half
            r = m-1
        else:
            return m
    return l
\end{lstlisting}
\end{examples}
Standard binary search
\begin{enumerate}
    \item 611. Valid Triangle Number (medium)
    \item 704. Binary Search (easy)

\item  74. Search a 2D Matrix) Write an efficient algorithm that searches for a value in an m x n matrix. This matrix has the following properties:
\begin{enumerate}
    \item Integers in each row are sorted from left to right.
    \item The first integer of each row is greater than the last integer of the previous row.
    \end{enumerate}
\begin{lstlisting}[numbers=none]
For example,
Consider the following matrix:

[
  [1,   3,  5,  7],
  [10, 11, 16, 20],
  [23, 30, 34, 50]
]

Given target = 3, return true.
\end{lstlisting}

Solution: 2D matrix search, time complexity from $O(n^2)$ to $O(lgm+lgn)$.
\begin{lstlisting}[language = Python]
def searchMatrix(self, matrix, target):
        """
        :type matrix: List[List[int]]
        :type target: int
        :rtype: bool
        """

        if not matrix:
            return False
        row, col = len(matrix), len(matrix[0])
        if row==0 or col==0: #for [[]]
            return False
        sr, er = 0, row-1
        #fisrst search the mid row
        while sr<=er:
            mid = sr+(er-sr)//2
            if target>matrix[mid][-1]: #go to the right side
                sr=mid+1
            elif target < matrix[mid][0]: #go the the left side
                er = mid-1
            else: #value might be in this row
                #search in this row
                lc, rc = 0, col-1
                while lc<=rc:
                    midc = lc+(rc-lc)//2
                    if matrix[mid][midc]==target:
                        return True
                    elif target<matrix[mid][midc]: #go to left
                        rc=midc-1
                    else:
                        lc=midc+1
                return False
        return False
\end{lstlisting}

Also, we can treat is as one dimensional, and the time complexity is $O(lg(m*n))$, which is the same as $O(log(m)+log(n))$.
\begin{lstlisting}[language = Python]
class Solution:
    def searchMatrix(self, matrix, target):
        if not matrix or target is None:
            return False

        rows, cols = len(matrix), len(matrix[0])
        low, high = 0, rows * cols - 1

        while low <= high:
            mid = (low + high) / 2
            num = matrix[mid / cols][mid % cols]

            if num == target:
                return True
            elif num < target:
                low = mid + 1
            else:
                high = mid - 1

        return False
\end{lstlisting}
\end{enumerate}

Check \url{http://www.cnblogs.com/grandyang/p/6854825.html} to get more examples.

Search on rotated and 2d matrix:
\begin{enumerate}
    \item 81. Search in Rotated Sorted Array II (medium)
    \item 153. Find Minimum in Rotated Sorted Array (medium) The key here is to compare the mid with left side, if mid-1 has a larger value, then that is the minimum
    \item 154. Find Minimum in Rotated Sorted Array II (hard)
\end{enumerate}
Search on Result Space:
\begin{enumerate}
    \item 367. Valid Perfect Square (easy) (standard search)
    \item 363. Max Sum of Rectangle No Larger Than K (hard)
    \item 354. Russian Doll Envelopes (hard)
    \item 69. Sqrt(x) (easy)
\end{enumerate}
\end{document}


\subsection{Analytic optimisation}

\subsubsection{Convex and concave functions}

Convex functions only have one minimum, and concave functions have only one maximum.

If a function is not concave or convex, it may have multiple minima

If a function is convex, then there is only one critical point – the local minimum. We can identify this this by looking for critical points using first-order conditions.

Similarly, if a function is concave, then there is only one critical point – the local maximum.

We can identify whether a function is concave or convex by evaluating the Hessian matrix.

\subsubsection{Evaluating multiple local optima}

We can evaluate each of the local minima or maxima, and compare the sizes.

We can identify these by taking partial derivatives of the function in question and identifying where this function is equal to zero.

\(u=f(x)\)

\(u_{x_i}=\dfrac{\delta f}{\delta x_i}=0\)

We can then solve this bundle of equations to find the stationary values of \(x\).

After identifying the vector \(x\) for these points we can then  determine whether or not the points are minima or maxima by  examining the second derivative at these points. If it is positive it is a local minima, and therefore not an optimal point. Points beyond these will be higher, and may be higher than any local maxima.


\subsection{Rank and independence}

If $\{\tb{v}_1,\tb{v}_2,\cdots ,\tb{v}_m\}$ is a collection if independent vectors then

\[
\left(\begin{array}{ccccc}
\mid & \mid & \mid & & \mid \\
\tb{v}_{1} & \bar{v}_{2} & \bar{v}_{3} & \ldots & \bar{v}_{m} \\
\mid & \mid & \mid & & \mid
\end{array}\right)
\]

must have a rank of $m$. This is because row reducing the matrix corresponds to the following relation:

\[c_1\tb{v}_1+c_2\tb{v}_2+c_3\tb{v}_3+\dots +c_m\tb{v}_m=\tb{0}\]

\noindent
Also, $m\leq n$ where $n$ is the number of rows in each column vector, in order to have linear independence for this set.

\subsection{Dimension}

Considering an $xy$-plane in $\R^3$:

\[V=\left\{\begin{bmatrix}s\\t\\0\\ \end{bmatrix}\Bigg| s,t \in \mathbb{R} \right\}\]

The basis of this set contains 2 vectors (e.g. dimension of 2), with example being:

\[\mathfrak{B}=\left\{\begin{bmatrix}1\\0\\0 \end{bmatrix} , \begin{bmatrix} 0\\1\\0 \end{bmatrix} \right\}\]

\begin{framed}
    If $V$ is a subspace of $\R^n$ and $\mathfrak{B}$ and $\mathfrak{C}$ are two bases of $V$, then
    $\mathfrak{B}$ and $\mathfrak{C}$ contain the same number of vectors.
\end{framed}

\textbf{Dimension} of a subspace is number of vectors in the basis.

\subsubsection{Example}

Considering the following matrix:

\[A=\begin{bmatrix}
    1&2&0&1&2\\
    1&2&0&2&3\\
    1&2&0&3&4\\
    1&2&0&4&5\\
    \end{bmatrix}\]

By discounting the redundant vectors, a possible basis for $\mathrm{Im}(A)$:

\[\mathfrak{B}_\mathrm{image}=\left \{\begin{bmatrix} 1\\1\\1\\1 \end{bmatrix},\begin{bmatrix}1\\2\\3\\4 \end{bmatrix} \right\}\]

So the dimension of $\mathrm{Im}(A)$ is 2.
Finding a basis for $\mathrm{ker}(A)$ is the same as solving $A\tb{x}=\tb{0}$:

\[\mbox{ker}(A)=\left\{\begin{bmatrix}-2s-w\\ s\\t\\-w\\w \end{bmatrix} \Bigg| s,t,w\in\mathbb{R} \right\}=
\left\{s\begin{bmatrix}-2\\1\\0\\0\\0\\\end{bmatrix}+t\begin{bmatrix}0\\0\\1\\0\\0\\\end{bmatrix}+w\begin{bmatrix}-1\\0\\0\\-1\\1\\\end{bmatrix}\Bigg|s,t,w\in\mathbb{R}\right\}\]

So the basis for $\mathrm{ker}(A)$:

\[\mathfrak{B}_\mathrm{kernel}=\left\{\begin{bmatrix}-2\\1\\0\\0\\0\\ \end{bmatrix}, \begin{bmatrix} 0\\0\\1\\0\\0\\ \end{bmatrix}, \begin{bmatrix}-1\\0\\0\\-1\\1\\ \end{bmatrix} \right\}\]

And dimension of $\mathrm{ker}(A)$ is 3.
However, it is shown that $\mathrm{rref}(A)$ gives dimension of \textbf{image and kernel}.

\subsection{Rank-Nullity Theorem}

\begin{framed}
    \begin{itemize}
        \item If $T$ is a linear transformation,
        then \\$\mathrm{dim}(\mathrm{Im}(T))+\mathrm{dim}(\mathrm{ker}(T))=\text{dimension of domain of }T$
        \item If $A$ is a matrix, then $\mathrm{rank}(A)+\mathrm{nullity}(A)=\text{number of columns of }A$
        \item In a linear system,
        \\$\text{number of leading variables}+\text{number of free variables}=\text{total number of variables}$
    \end{itemize}
\end{framed}

Considering non-invertible matrices $A$ and $B$, let $AB$ be invertible.
It must hold true that $\mathrm{ker}(B)=\{\tb{0}\}$. If the dimensions of $B$ are $p\times n$,
$\mathrm{Im}(B)$ is a subspace of $\R^p$ has dimension $n$. This means that it is a vertically rectangular
matrix with $n\leq p$. Thus, $A$ is $n\times p$ so it is horizontally rectangular.


\section{Principle component analysis}

In principle component analysis (PCA) we assume a linear generative model of the form
shown in equation \eqref{eq:LGM} with $D = M$, i.e. a complete LGM, and independent sources
of unequal variance such that $\sigma_1^2 > \sigma_2^2 > \dots > \sigma_M^2$.
If we further assume that the matrix $\mb{A}$ is orthogonal such that
$\mb{A} \mb{A}\TT = \imat_M$ we can show that the Eigendecomposition of the data
covariance matrix $\mb{C}_x$ finds the linear transformation matrix $\mb{A}$.
We noticed earlier that the data covariance matrix for an LGM of the given form
can be decomposed such that:
\begin{align}
	\mb{C}_x = \mb{A} \mb{C}_s \mb{A}\TT
	\label{eq:Cx_dec}
\end{align}
We can compare this result to the Eigendecomposition of a matrix:
\begin{align}
	\mb{B} = \mb{U} \mb{D} \mb{U}^{-1}
\end{align}
where $\mb{B}$ is an invertible $M \times M$ matrix, $\mb{U}$ is the matrix of Eigenvectors
and $\mb{D}$ is the diagonal matrix of Eigenvalues.
We further notice that for a real-valued symmetric matrix $\mb{B}$ the Eigendecomposition
is given by
\begin{align}
	\mb{B} = \mb{U} \mb{D} \mb{U}\TT
\end{align}
and has the same functional form as equation \eqref{eq:Cx_dec}. Since a covariance matrix
is always symmetric and can therefore be diagonalized we notice that $\mb{C}_s = \mb{D}$
with $\sigma_1^2 > \sigma_2^2 > \dots > \sigma_M^2$ is a
diagonal covariance matrix for the independent sources and that $\mb{A} = \mb{U}$ and
is the matrix of Eigenvectors of $\mb{C}_x$ where $\mb{U} = \{\mb{u}_1, \mb{u}_2, \dots, \mb{u}_M\}$.

\paragraph{Whitening} If we now want to recover the independent sources we we can apply a decorrelating
transform based on the Eigenvector matrix of the data covariance matrix $\mb{C}_x$. We noticed that
$\mb{U} = \mb{A}$ and therefore we can find a backtransform such that
\begin{align}
	\mb{x} &= \mb{U s} \\
	\mb{U \TT x} &= \underbrace{\mb{U\TT U}}_{ = \imat_M} \mb{s} \\
    \mb{s} &= \mb{U\TT x}
\end{align}
where we have removed correlations between vector dimensions. If we want to reduce the dimensionality
of our data we simply use the first $K$ Eigenvectors of $\mb{U}$ which are called the $K$ principle
components of $X$. It is important to note here that bacause $\sigma_1^2 > \sigma_2^2 > \dots > \sigma_M^2$
projecting the data on the $K$ principle components leads to a projection to the $K$-dimensional subspace
of highest variance. If we want to remove the difference in variance such that
$\sigma_1^2 = \sigma_2^2 = \dots = \sigma_M^2$, we can introduce isovariance by applying the transform
\begin{align}
	\mb{\tilde{s}} = \mb{D}^{-\frac{1}{2}} \mb{s}
\end{align}
This approach is called whitening. After the whitening we can be sure that a linear transform will
not introduce correlations between dimensions (see \ref{par:cov_isovar}).

\paragraph{How to use PCA on data}
Given $\mb{X} = (\mb{x_1}, \dots, \mb{x_N})$ with $\mb{x_n}$ being a column vector of length $M$.
Each column vector represents an observation. We can calculate the empirical mean by
\begin{align}
	\hat{\greekvec{\mu}} = \frac{1}{N} \mb{X \ivec_M}
\end{align}
From now on we will assume that data matrices have zero mean, i.e.
\begin{align*}
	\mb{X} = \mb{X} - \greekvec{\mu}\ivec_M\TT
\end{align*}
and that $N > M$. The empirical covariance matrix is then given by:
\begin{align}
	\begin{split}
	\hat{\mb{C}} &= \frac{1}{N} \mb{X X \TT} \\
	             &= \frac{1}{N} \sum_{n = 1}^N \mb{x_n x_n}
	\end{split}
\end{align}

\subsection{Singular value decomposition}
In some cases it can be easier to find the singular value decomposition (SVD) of a data matrix
rather than finding the Eigendecomposition of the empirical covariance matrix. In this section
we will shortly introduce SVD and show it's relationship to PCA.\\
SVD is a generalization of the Eigendecomposition which can decompose any matrix into different
factors such that for a $M \times N$ real matrix $\mb{X}$ we find
\begin{align}
	\mb{X} = \mb{U S V\TT}
\end{align}
where $\mb{U}$ is a $M \times M$ orthogonal matrix, $\mb{S}$ is a $M \times N$  diagonal matrix
where the values on the diagonal called the singular values, and $\mb{V}$ is a $N \times N$
orthogonal matrix. The SVD of a diagonizable matrix is equivalent to the Eigendecomposition
of that matrix. A unique solution for SVD exists if all values on the sigular values are different. \\

\noindent To show the relationship between SVD and PCA we will decompose the empirical covariance matrix
into the data matrix as above and then apply SVD to further decompose the data matrices:
\begin{align}
	\begin{split}
	N \hat{\mb{C}} &= \mb{X X \TT} \\
	               &= \mb{U S V\TT} \left( \mb{U S V\TT} \right)\TT \\
   	               &= \mb{U S} \underbrace{\mb{V\TT V}}_{= \imat_N} \mb{S\TT U\TT}\\
   	               &= \mb{U S S\TT U\TT}\\
	\end{split}
\end{align}
\noindent We notice that $\mb{S S\TT} = \mb{D}$ is the matrix of Eigenvalues for $\mb{X X\TT}$ and
$\mb{U}$ is the matrix of Eigenvectors. That is the singular values of the data matrix represent
the standard deviation in the priniciple directions, whereas the Eigenvalues of $\mb{X X\TT}$
represent the variance in the principle directions.

\subsubsection{SVD of the Gram matrix}
In kernel methods we often deal with a gram matrix $\mb{G}$ of the form:
\begin{align}
	\mb{G} = \frac{1}{N} \mb{X\TT X}
\end{align}
which is represented by the scalar product between any pair of observations. If we perform
SVD on the gram matrix we find that:
\begin{align}
	\begin{split}
	N \mb{G} &= \left( \mb{U S V\TT} \right) \TT \mb{U S V\TT} \\
			 &= \mb{V S\TT} \underbrace{\mb{U\TT U}}_{= \imat_K} \mb{S V\TT}	 \\
			 &= \mb{V S\TT S V\TT}
	\end{split}
\end{align}
This decomposition is referred to as Kernel PCA. For more information on the topic see
\cite[Chapter 6 \& 12.3]{Bishop2006}

\subsection{Model comparison}
If we want to perform model comparison we can estimate the posterior probability of our
model given the data based on Bayes theorem:
\begin{align}
	P(M|X) = \frac{1}{Z} P(X|M)P(M)
\end{align}
Dependent on our preferences towards a class of models we can specify the prior probability of
a model by $P(M)$. If we do not want to make assumptions about $P(M)$ or just for convenience
we sometimes assume $P(M) = const.$ which is equivalent to a Likelihood-based model comparison.

Given we have observed i.i.d. data represented in a $M \times N$ matrix:
\begin{align}
	\mb{X} = \left( \underbrace{\mb{x_1}, \dots, \mb{x_N}}_{\text{i.i.d samples}} \right)
\end{align}
and we assume $P(M) = const.$, then $P(M|X)$ is proportional to the likelihood of the
data given a model such that:
\begin{align}
	\begin{split}
	P(M|X) \varpropto \prod_{n=1}^N \rho(\mb{x_n}|M)
	    &= \exp \left( \sum_{n=1}^N \underbrace{\log \left( \rho(\mb{x_n}|M) \right)}_{\text{Log-likelihood}} \right)
	\end{split}
\end{align}
\noindent For the Gaussian with $\rho(\mb{x_n}|M) = \mathcal{N}(\mb{x_n}|0,\mb{C})$ the likelihood becomes:
\begin{align}
	\begin{split}
	\prod_{n=1}^N \rho(\mb{x_n}|M)
	        &= \exp \left( \sum_{n=1}^N \log \left( \normpdfd \right) \right) \\
	        &= \exp \left( \sum_{n=1}^N -\frac{1}{2}\log \left( 2\pi^M |\mb{C}| \right) -\frac{1}{2} \mb{x_n \TT C^{-1} x_n} \right) \\
	        &= \exp \left( -\frac{N}{2}\log \left( 2\pi^M |\mb{C}| \right) -\frac{1}{2} \sum_{n=1}^N \mb{x_n \TT C^{-1} x_n} \right) \\
	\end{split}
\end{align}

\subsubsection{The rescaling problem}
We will now consider the rescaling problem. If we rescale a random variable using a linear transform
the probability density of the respective linear mapping is also rescaled. As a result the likelihood
function of the two random variable has a different scale.
\begin{example}[The rescaling problem] \label{ex:rescaling}
Consider a one-dimensional linear mapping of the form
\begin{align}
	y &= \lambda x
\end{align}
with
\begin{align*}
	x \sim \mathcal{N}(0,\sigma^2) \qquad \text{ and } \qquad
	y \sim \mathcal{N}(0,\lambda^2 \sigma^2)
\end{align*}
then
\begin{align}
	\begin{split}
	\rho_y(y) &= \frac{1}{\sqrt{2\pi \lambda^2 \sigma^2}} \exp \left( \frac{y^2}{2 \lambda^2 \sigma^2} \right)\\
	         &= \frac{1}{\lambda} \frac{1}{\sqrt{2\pi \sigma^2}} \exp \left( \frac{\lambda^2 x^2}{2 \lambda^2 \sigma^2} \right)\\
	         &= \frac{1}{\lambda} \rho_x(x)
	\end{split}
\end{align}
is the relationship between the two probability densities.
\end{example}
\noindent The likelihood depends on the scale of a random variable. Thus all models should be evaluated with
respect to the same scale for model comparison with the likelihood measure. Or differently, the likelihood is
a relative measure, not an absolute one.

\begin{proposition}[Change of variables]
Given an invertible nonlinear mapping of the form:
\begin{align*}
	\mb{x} &  \stackrel{f}{\longrightarrow} \mb{y} \\
	\mb{x} &  \stackrel{g = f^{-1}}{\longleftarrow} \mb{y}
\end{align*}
If the probability density function $\rho_{\mb{x}}(\mb{x})$ of a random variable
$X$ is known we can calculate the
probability density $\rho_{\mb{y}}(\mb{y})$. This is called a change of variables.
We know that the probability under the differential area is unchanged under the
change of variables:
\begin{align}
	\rho_{\mb{y}}(\mb{y}) \mathrm{d}\mb{y} = \rho_{\mb{x}}(\mb{x}) \mathrm{d}\mb{x}
\end{align}
and hence the probability density $\rho_{\mb{y}}(\mb{y})$ can be calculated by
\begin{align}
	\begin{split}
	\rho_{\mb{y}}(\mb{y})
	&= \rho_{\mb{x}}(\mb{x})  \begin{vmatrix}\frac{\mathrm{d}\mb{x}}{\mathrm{d}\mb{y}}\end{vmatrix} \\
	&= \rho_{\mb{x}}(g(\mb{y})) \begin{vmatrix}\frac{\mathrm{d}g}{\mathrm{d}\mb{y}}\end{vmatrix} \\
	&= \frac{\rho_{\mb{x}}(f^{-1}(\mb{y}))}{\begin{vmatrix}\frac{\mathrm{d}f}{\mathrm{d}\mb{x}} f^{-1}(\mb{y})\end{vmatrix}}
	\end{split}
\end{align}
where
\begin{align}
	\begin{vmatrix}\frac{\mathrm{d}\mb{y}}{\mathrm{d}\mb{x}}\end{vmatrix}
	    = |J|
	    = \begin{vmatrix}
	    	\frac{\mathrm{d}y_1}{\mathrm{d}x_1} & \cdots & \frac{\mathrm{d}y_M}{\mathrm{d}x_1} \\
	    	\vdots & \ddots & \vdots \\
	    	\frac{\mathrm{d}y_1}{\mathrm{d}x_M} & \cdots & \frac{\mathrm{d}y_M}{\mathrm{d}x_M} \\
	    \end{vmatrix}
\end{align}
is the Jacobian determinant. For more information see \cite[p.18, p.247]{Bishop2006}. In example
\ref{ex:rescaling} from above the transform can be simply read out as:
\begin{align}
	\rho_{\mb{y}}(\mb{y}) = \frac{1}{\lambda} 	\rho_{\mb{x}} \left( \frac{\mb{y}}{\lambda} \right)
\end{align}
\end{proposition}

\subsubsection{Generally for LGMs}
We will now come back to our linear generative model of the form
\begin{align*}
	f: \qquad \mb{x} = \mb{A s} = \mb{U s}
\end{align*}
where $\mb{U}$ is the Eigenvector basis of the covariance matrix $\mb{C}_x$.
If we now want to find the change of variables from $\rho_\mb{x}(\mb{x})$ to $\rho_\mb{s}(\mb{s})$:
\begin{align}
	\rho_{\mb{S}}(\mb{s})
	&= \rho_{\mb{x}}(\mb{x})  \begin{vmatrix}\frac{\mathrm{d}\mb{x}}{\mathrm{d}\mb{s}}\end{vmatrix} \nonumber \\
	&= \rho_{\mb{x}}(\mb{x})  \begin{vmatrix}\frac{\mathrm{d}f(\mb{s})}{\mathrm{d}\mb{s}}\end{vmatrix}
\end{align}
we notice that
\begin{align}
	\begin{vmatrix}\frac{\mathrm{d}f(\mb{s})}{\mathrm{d}\mb{s}}\end{vmatrix} = |\det(\mb{U})|
\end{align}
since
\begin{equation*}
	%\left. \begin{aligned}
		f(\mb{s}) = \mb{U s}  \qquad \text{and} \qquad
		\mb{s} = \mb{U\TT x}
		%\mb{s} &= g(\mb{x}) = \mb{U\TT x} \\
		%f(\mb{s}) &= g^{-1}(\mb{x}) = \mb{U s} \\
	%\end{aligned} \qquad \right\}
    %\qquad \begin{vmatrix}\frac{\mathrm{d}f}{\mathrm{d}\mb{s}}\end{vmatrix} = |\det(\mb{U})|
\end{equation*}
We further find that
\begin{align*}
	|\det(\mb{U})| = \sqrt{\det(\mb{U})^2} %\\
	               = \sqrt{\det(\mb{U}) \det(\mb{U}\TT)} %\\
	               = \sqrt{\det(\mb{U U\TT})} %\\
	               = \sqrt{\det(\imat_M)} %\\
	               = 1
\end{align*}
As a result the change of variables reduces to the form
\begin{align}
	\rho_{\mb{s}}(\mb{s}) = \rho_{\mb{x}}(\mb{U s})
\end{align}
and vice versa
\begin{align}
	\rho_{\mb{x}}(\mb{x}) = \rho_{\mb{s}}(\underbrace{\mb{U\TT x}}_{\mb{s}})
	                      = \prod_{m=1}^M \rho(s_m)
\end{align}
\begin{proposition}[Necessary condition]
For all $\mb{U}$ with the same Jacobian determinant the likelihood must be largest
for the true generating model.
\end{proposition}


\section{Approximation}{}{}\label{sec:Approximation of differential equations}
We have seen how to solve a restricted collection of differential
equations, or more  accurately, how to attempt to solve them---we still may
not be able to find the required anti-derivatives. Not surprisingly,
non-linear equations can be even more difficult to solve. Yet much is
known about solutions to some more general equations.

Suppose $\phi(t,y)$ is a function of two variables. A more general
class of first order differential equations has the form $\ds y'=\phi(t,y)$.
This is not necessarily a linear first order equation,
since $\phi$ may depend on $y$ in some complicated way; note however
that $\ds y'$ appears in a very simple form. Under suitable
conditions on the function $\phi$, it can be shown that every such
differential equation has a solution, and moreover that for each
initial condition the associated initial value problem has exactly one
solution. In practical applications this is obviously a very desirable
property.

\begin{example}{First Order Non-linear}{First Order Non-linear}\label{First Order Non-linear}
 The equation $\ds y'=t-y^2$ is a first order non-linear
equation, because $y$ appears to the second power. We will not be able
to solve this equation.
\end{example}

\begin{example}{Non-linear and Separable}{Non-linear and Separable}\label{Non-linear and Separable}
 The equation $\ds y'=y^2$ is also non-linear, but it is
separable and can be solved by separation of variables.
\end{example}

Not all differential equations that are important in practice can be
solved exactly, so techniques have been developed to approximate
solutions. We describe one such technique, \dfont{Euler's Method}, which is simple though not particularly
useful compared to some more sophisticated techniques.

Suppose we wish to approximate a solution to the initial value problem
$\ds y'=\phi(t,y)$, $\ds y(t_0)=y_0$, for $t\ge t_0$. Under
reasonable conditions on $\phi$, we know the solution exists,
represented by a curve in the $t$-$y$ plane; call this solution
$f(t)$. The point $\ds (t_0,y_0)$ is of course on this curve. We also
know the slope of the curve at this point, namely
$\ds\phi(t_0,y_0)$. If we follow the tangent line for a brief
distance, we arrive at a point that should be almost on the graph of
$f(t)$, namely $\ds(t_0+\Delta t, y_0+\phi(t_0,y_0)\Delta t)$; call
this point $\ds(t_1,y_1)$. Now we pretend, in effect, that this point
really is on the graph of $f(t)$, in which case we again know the
slope of the curve through $\ds(t_1,y_1)$, namely
$\ds\phi(t_1,y_1)$. So we can compute a new point,
$\ds(t_2,y_2)=(t_1+\Delta t, y_1+\phi(t_1,y_1)\Delta t)$ that
is a little farther along, still close to the graph of $f(t)$ but
probably not quite so close as $\ds(t_1,y_1)$. We can continue in this
way, doing a sequence of straightforward calculations, until we have
an approximation $\ds(t_n,y_n)$ for whatever time $\ds t_n$ we need.
At each step we do essentially the same calculation, namely:
$$(t_{i+1},y_{i+1})=(t_i+\Delta t, y_i+\phi(t_i,y_i)\Delta t).$$

We expect that smaller time steps $\Delta t$ will give better
approximations, but of course it will require more work to compute to
a specified time. It is possible to compute a guaranteed upper bound
on how far off the approximation might be, that is, how far $\ds y_n$
is from $f(t_n)$. Suffice it to say that the bound is not particularly
good and that there are other more complicated approximation
techniques that do better.

\begin{example}{Approximating a Solution}{Approximating a Solution}\label{Approximating a Solution}
Compute an approximation to the solution for $\ds y'=t-y^2$, $y(0)=0$, when $t=1$.
\end{example}

\begin{solution}
We will use $\Delta t=0.2$, which is
easy to do even by hand, though we should not expect the resulting
approximation to be very good. We get
\begin{eqnarray*}
(t_1,y_1)&=(0+0.2,0+(0-0^2)0.2) = (0.2,0)\cr
(t_2,y_2)&=(0.2+0.2,0+(0.2-0^2)0.2) = (0.4,0.04)\cr
(t_3,y_3)&=(0.6,0.04+(0.4-0.04^2)0.2) = (0.6,0.11968)\cr
(t_4,y_4)&=(0.8,0.11968+(0.6-0.11968^2)0.2) = (0.8,0.23681533952)\cr
(t_5,y_5)&=(1.0,0.23681533952+(0.6-0.23681533952^2)0.2) = (1.0,0.385599038513605)
\end{eqnarray*}
So $y(1)\approx 0.3856$. As it turns out, this is not accurate to
even one decimal place. Figure~\ref{fig:euler method vs rk} shows
these points connected by line segments (the lower curve) compared to
a solution obtained by a much better approximation technique. Note
that the shape is approximately correct even though the end points are
quite far apart.

\figure[H]
%\texonly
\centerline{\vbox{\beginpicture
\normalgraphs
%\sevenpoint
\setcoordinatesystem units <8truecm,8truecm>
\setplotarea x from 0 to 1.05, y from 0 to 0.55
\axis left shiftedto x=0 ticks length <2pt> numbered from 0 to 0.5 by 0.5 /
\axis bottom shiftedto y=0 ticks length <2pt> numbered from 0 to 1 by 1 /
\put {$t$} [tl] <0pt,-3pt> at 1.06 0
\put {$y$} [b] <0pt,3pt> at 0 0.56
\setlinear
\plot
0 0
0.2 0.0000
0.4 0.0400
0.6 0.1196
0.8 0.2368
1.0 0.3855 /
\setquadratic
\plot 0  0
0.025000000000000001  0.00031249951171971195
0.050000000000000003  0.0012499843752447799
0.075000000000000011 0.0028123813539146945
0.10000000000000001 0.0049995000624952881
0.125  0.0078109744935353819
0.14999999999999999  0.0112462047261264
0.17499999999999999 0.015304298951036615
0.19999999999999998  0.019984015983738772
0.22499999999999998  0.025283708473608849
0.24999999999999997 0.031201267053176683
0.27499999999999997  0.037734065705321729
0.29999999999999999  0.044878908658235558
0.32500000000000001 0.052631979147269851
0.35000000000000003  0.060988790408866761
0.37500000000000006  0.069944139294007487
0.40000000000000008 0.079492062906380986
0.4250000000000001  0.089625798683132307
0.45000000000000012  0.10033774834298456
0.47500000000000014 0.11161944612716239
0.50000000000000011  0.12346153175236174
0.52500000000000013  0.1358537284815772
0.55000000000000016 0.14878482669758133
0.57500000000000018  0.16224267333504916
0.6000000000000002  0.17621416749066984
0.62500000000000022 0.1906852624861928
0.65000000000000024  0.20564097460749087
0.67500000000000027  0.22106539868385133
0.70000000000000029 0.23694173060647841
0.72500000000000031  0.25325229681444411
0.75000000000000033  0.26997859070107927
0.77500000000000036 0.28710131581523379
0.80000000000000038  0.30460043565128503
0.8250000000000004  0.32245522974067869
0.85000000000000042 0.34064435567768192
0.87500000000000044  0.35914591663448664
0.90000000000000047  0.37793753384740725
0.92500000000000049 0.39699642348821867
0.95000000000000051  0.41629947727413563
0.97500000000000053  0.4358233461178857
1.0000000000000004 0.45554452607694396 /
\setplotsymbol ({\small.})
\plotsymbolspacing=.2pt
\arrow <4pt> [0.35, 1] from 1.05 0 to 1.06 0
\arrow <4pt> [0.35, 1] from 0 0.55 to 0 0.56
\endpicture}}
\caption{\label{fig:euler method vs rk}
Approximating a solution to $\ds y'=t-y^2$, $y(0)=0$.}
%\endtexonly
%\figrdef{fig:euler method vs rk}
%\htmlfigure{DE-approximate_solution.html}
%\begincaption
%Approximating a solution to $\dsy'=t-y^2$, $y(0)=0$.
%\endcaption
\endfigure

If you need to do Euler's method by hand, it is useful to construct a
table to keep track of the work, as shown in Figure~\ref{fig:eulers method table}.
Each row holds the computation for a single step: The starting point
$(t_i,y_i)$; the stepsize $\Delta t$; the computed slope
$\phi(t_i,y_i)$; the change in $y$, $\Delta y=\phi(t_i,y_i)\Delta t$;
and the new point, $(t_{i+1},y_{i+1})=(t_i+\Delta t,y_i+\Delta y)$. The starting point in
each row is the newly computed point from the end of the previous row.

\figure[H]
%\texonly
%\stablestyle3
%\null
$$\begin{array}{|l|l|l|l|l|}
\hline
 (t,y) &  \Delta t &  \phi(t,y) &  \Delta y=\phi(t,y)\Delta
  t &  (t+\Delta t,y+\Delta y) \\\hline
 (0,0) \hfill&  0.2  \hfill& 0 \hfill& 0 \hfill&  (0.2,0) \hfill\\\hline
 (0.2,0) \hfill&  0.2  \hfill&  0.2  \hfill&  0.04  \hfill&  (0.4,0.04) \hfill\\\hline
 (0.4,0.04) \hfill&  0.2  \hfill&  0.3984  \hfill&  0.07968  \hfill&  (0.6,0.11968) \hfill\\\hline
 (0.6,0.11968) \hfill&  0.2  \hfill&  0.58\ldots  \hfill&  0.117\ldots  \hfill&  (0.8,0.236\ldots) \hfill\\\hline
 (0.8,0.236\ldots) \hfill&  0.2  \hfill&  0.743\ldots  \hfill&  0.148\ldots   \hfill&  (1.0,0.385\ldots) \hfill\\
\hline
\end{array}$$
%\endtexonly
%\figrdef{fig:eulers method table}
%\htmlfigure{DE-eulers_method_table.html}
\caption{\label{fig:eulers method table}Computing with Euler's Method.}
%\endcaption
\endfigure
\end{solution}

Euler's method is related to another technique that can help in
understanding a differential equation in a qualitative way. Euler's
method is based on the ability to compute the slope of a solution
curve at any point in the plane, simply by computing $\phi(t,y)$. If
we compute $\phi(t,y)$ at many points, say in a grid, and plot a small
line segment with that slope at the point, we can get an idea of how
solution curves must look. Such a plot is called a
\dfont{ slope field\index{slope field}}. A slope field for
$\ds\phi=t-y^2$ is shown in Figure~\xrefn{fig:slope field}; compare
this to figure~\xrefn{fig:euler method vs rk}.
With a little practice, one can sketch reasonably accurate solution
curves based on the slope field, in essence doing Euler's method
visually.

\figure[H]
%\texonly
\hbox to \hsize{\hss
\epsfxsize6.5cm\epsfbox{images/slope_field.eps}
\hss}
%\endtexonly
%\figrdef{fig:slope field}
%\htmlfigure{DE-slope_field.html}
\caption{\label{fig:slope field}
A slope field for $\ds y'=t-y^2$.}
\endfigure

Even when a differential equation can be solved explicitly, the slope
field can help in understanding what the solutions look like with
various initial conditions. Recall the logistic equation
%from exercise~\xrefn{exer:logistic equation} in  section~\xrefn{sec:first order differential equations},
$\ds y' = ky(M-y)$: $y$ is a population at time $t$, $M$ is a measure of how
large a population the environment can support, and $k$ measures the
reproduction rate of the population.
Figure~\xrefn{fig:logistic slope field} shows a slope field
for this equation that is quite informative.
It is apparent that if the initial population is smaller than $M$ it
rises to $M$ over the long term, while if the initial population is
greater than $M$ it decreases to $M$.
%\texonly
%It is quite easy to
%generate slope fields with Sage; follow the AP link in the figure caption.
%\endtexonly

\figure[H]
%\texonly
\hbox to \hsize{\hss
\epsfxsize12cm\epsfbox{images/logistic_slope_field.eps}
\hss}
%\begincaption
%A slope field for $\dsy'=0.2y(10-y)$.
%(\expandafter\url\expandafter{\sageurl logistic_slope_field}%
%AP\endurl)
%\endcaption
%\endtexonly
%\figrdef{fig:logistic slope field}
%\htmlfigure{DE-slope_field_logistic.html}
\caption{\label{fig:logistic slope field}
A slope field for $\ds y'=0.2y(10-y)$.}
%\endcaption
\endfigure

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:Approximation of differential equations}}

\begin{enumialphparenastyle}

In the following exercises, compute the Euler approximations for the initial
value problem for $0\le t\le 1$ and $\Delta t=0.2$. If you have access
to Sage, generate the slope field first and attempt to sketch the
solution curve. Then use Sage to compute better approximations with
smaller values of $\Delta t$.

%%%%%%%%%%
\begin{ex}
 $\ds y'=t/y$, $y(0)=1$
\begin{sol}
 $y(1)\approx 1.355$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds y'=t+y^3$, $y(0)=1$
\begin{sol}
 $y(1)\approx 40.31$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds y'=\cos(t+y)$, $y(0)=1$
\begin{sol}
 $y(1)\approx 1.05$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds y'=t\ln y$, $y(0)=2$
\begin{sol}
 $y(1)\approx 2.30$
\end{sol}
\end{ex}

\end{enumialphparenastyle}