\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}
\usepackage{booktabs}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\logit}{logit}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\cov}{cov}
\DeclareMathOperator*{\expec}{E}
\DeclareMathOperator*{\deriv}{d}
\DeclareMathOperator*{\const}{constant}

\begin{document}
\title{\textsf{Homework 7 for Bayesian Data Analysis}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Importance Resampling}
{
    Suppose you want to estimate $\expec_f h(\theta)$, with $f$ being some posterior distribution $P(\theta|y)$. Further suppose that you choose a proposal distribution $g(\theta)$, and get a sample $(x_1, \cdots, x_m)$ by importance resampling. Please prove that the average of $(h(x_1), \cdots, h(x_m))$ can be used as an estimator of $\expec_f h(\theta)$.

    \textbf{Proof:}\quad Note that the probability density for $x_i$ is $w(x_i) g(x_i)$ for any $i$, which does not depend on the order in the sampling with replacement. It follows that
    $$\expec_g {(h(x_i))} = \int{ h(x_i) w(x_i) g(x_i) \deriv{x_i} }$$
    $$ = \int{ h(x_i) \frac{P(x_i|y)}{g(x_i)} g(x_i) \deriv{x_i} } = \int{ h(x_i) P(x_i|y) \deriv{x_i} } = \expec_f h(\theta)$$ for any $i$. Therefore, the average of $(h(x_1), \cdots, h(x_m))$ is an unbiased estimator of $\expec_f h(\theta)$:
    $$\expec_g {\frac{1}{m} \sum_{i=1}^{m}{(h(x_i))}} = \frac{1}{m} \sum_{i=1}^{m} {\expec_g {h(x_i)}} = \frac{1}{m} \sum_{i=1}^{m} {\expec_f h(\theta)} = \expec_f h(\theta).$$
}

\section*{Question 10.6d}
{
    $$p(\theta|y) = f(\theta) = N(0, 3) = \frac{1}{\sqrt{6\pi}} \exp{\left( -\frac{\theta^2}{3} \right)}.$$

    $$g(\theta) = t_3 = \frac{2}{\pi \sqrt{3}} \left( 1+\frac{\theta^2}{3} \right) ^{-2}.$$

    It follows that
    $$\expec_g{\left[ (\frac{f(\theta)}{g(\theta)})^2 \right]} = \int {(\frac{f(\theta)}{g(\theta)})^2 g(\theta) \deriv{\theta}}$$
    $$= \int {\frac{1}{6\pi} \exp{\left( -\frac{2\theta^2}{3} \right)} \cdot \frac{\pi \sqrt{3}}{2} \left( 1+\frac{\theta^2}{3} \right)^2 \deriv{\theta}}$$
    $$= \frac{\sqrt{3}}{12} \int { \exp{\left( -\frac{2\theta^2}{3} \right)} \cdot \left( 1+\frac{\theta^2}{3} \right)^2 \deriv{\theta}}$$
    $$= \frac{\sqrt{2}}{12} \int { \exp{\left( -t^2 \right)} \cdot \left( 1+\frac{t^2}{2} \right)^2 \deriv{t}}$$
    $$= \frac{\sqrt{2}}{12} \left[ \int { \exp{\left( -t^2 \right)} \deriv{t}} + \int { t^2 \cdot \exp{\left( -t^2 \right)} \deriv{t}} + \frac{1}{4} \int { t^4 \cdot \exp{\left( -t^2 \right)} \deriv{t}} \right] $$
    $$= \frac{\sqrt{2}}{12} \left[ \sqrt{\pi} + \frac{1}{2}\sqrt{\pi} + \frac{3}{16}\sqrt{\pi} \right] = \frac{9\sqrt{2\pi}}{64} \approx 0.3525.$$

    The effective sample size for $n=10000$ is
    $$n_\mathrm{eff} = \frac{n}{\expec_g{\left[ (\frac{f(\theta)}{g(\theta)})^2 \right]}} = 10000 / \frac{9\sqrt{2\pi}}{64} \approx 28369.$$
}

\section*{Question 11.1}
{
    \textbf{Lemma: (Detailed Balance condition)}\quad If a Markov chain with transition probability $p(\cdot|\cdot)$ that satisfies
    $$\pi(\theta_a) \cdot p(\theta_b|\theta_a) = \pi(\theta_b) \cdot p(\theta_a|\theta_b)$$ for some distribution $\pi(\cdot)$, then $\pi(\cdot)$ is the stationary distribution of this Markov chain.

    Using the lemma above, we only need to verify that the Detailed Balance condition is satisfied when $\pi(\cdot) = p(\cdot|y)$.

    Note that $r(\theta_a, \theta_b) \cdot r(\theta_b, \theta_a) = 1$, for
    $$r(\theta_a, \theta_b) = \frac{ p(\theta_b|y) \cdot g(\theta_a|\theta_b) }{ p(\theta_a|y) \cdot g(\theta_b|\theta_a) }$$ and
    $$r(\theta_b, \theta_a) = \frac{ p(\theta_a|y) \cdot g(\theta_b|\theta_a) }{ p(\theta_b|y) \cdot g(\theta_a|\theta_b) },$$
    which means it is safe to assume that $r(\theta_a, \theta_b) \geq 1$ \emph{without loss of generality}. Thus, $\theta_b$ is always accepted after generated from the previous value $\theta_a$, with the probability of $1$. On the contrary, $\theta_a$ is accepted after generated from $\theta_b$ with the probability of $r(\theta_b, \theta_a)$. It follows that
    $$p(\theta_b|\theta_a) = g(\theta_b|\theta_a) \cdot 1$$ and
    $$p(\theta_a|\theta_b) = g(\theta_a|\theta_b) \cdot r(\theta_b, \theta_a).$$

    Plug them all in, and we obtain
    $$\mathrm{LHS} = p(\theta_a|y) \cdot p(\theta_b|\theta_a) = p(\theta_a|y) \cdot g(\theta_b|\theta_a) \cdot 1$$
    $$= p(\theta_a|y) \cdot g(\theta_b|\theta_a),$$ and
    $$\mathrm{RHS} = p(\theta_b|y) \cdot p(\theta_a|\theta_b) = p(\theta_b|y) \cdot g(\theta_a|\theta_b) \cdot r(\theta_b, \theta_a)$$
    $$= p(\theta_b|y) \cdot g(\theta_a|\theta_b) \cdot \frac{ p(\theta_a|y) \cdot g(\theta_b|\theta_a) }{ p(\theta_b|y) \cdot g(\theta_a|\theta_b) }$$
    $$= p(\theta_a|y) \cdot g(\theta_b|\theta_a),$$
    which gives $\mathrm{LHS} = \mathrm{RHS}$ and proves the Detailed Balance condition. Q.E.D.
}

\clearpage
\end{document}


\section{Counting Principles}\label{mod2:section:CountingPrinciples}

\begin{defn}\label{mod2:defn:MultiplicationRule}
	If event $K_i$ can occur in $p$ ways and event $K_j$ can occur in $q$ ways, the number of ways that $K_i$ \textbf{and} $K_j$ can occur is,
	\begin{equation}
		p \times q.
	\end{equation}
	This can be extended to any number of events. This is known as the \textbf{Multiplication Rule}.
\end{defn}

\begin{defn}\label{mod2:defn:AdditionRule}
	If event $K_i$ can occur in $p$ ways and event $K_j$ can occur in $q$ ways, the number of ways that $K_i$ \textbf{or} $K_j$ can occur is,
	\begin{equation}
		p + q.
	\end{equation}
	This can be extended to any number of events. This is known as the \textbf{Addition Rule}.
\end{defn}

\begin{defn} \label{mod2:defn:Combination}
	A \textbf{combination} is a group of elements chosen from a set of objects. The order of the elements does not matter.
\end{defn}

\begin{defn} \label{mod2:defn:CombinationEqn}
	The number of combinations of $r$ elements from a set of $n$ \textbf{distinct} elements is defined as,
	\begin{equation}
		^nC_r = \frac{n!}{r!~(n-r)!} \,.
	\end{equation}
\end{defn}

\begin{defn}\label{mod2:defn:Permutation}
	A \textbf{permutation} is a group of elements chosen from a set of objects where the order of the elements \textbf{does} matter.
\end{defn}

\begin{defn}\label{mod2:defn:PermutationEqn}
	The number of permutations of $r$ elements from a set of $n$ \textbf{distinct} elements is defined as,
	\begin{equation}
		^nP_r = \frac{n!}{(n-r)!} \,.
	\end{equation}
\end{defn}

\begin{defn}\label{mod2:defn:PermutationEqn2}
	The number of permutations of $n$ elements, not all distinct, is defined as,
	\begin{equation}
		\frac{n!}{{k_1}!\,{k_2}!\,...\,{k_r}!} \,,
	\end{equation}
	where  $\sum_{i=1}^{r} k_i = n$, and $k_i$ is the number of non-distinct elements of type $i$.
\end{defn}

\begin{defn}\label{mod2:defn:PermutationEqn3}
	The number of permutations of size $r$ out of $n$ elements, with repetition, is defined as,
	\begin{equation}
		n^r
	\end{equation}
\end{defn}

\section{Probability Theory}\label{mod2:section:ProbabilityTheory}

\begin{axiom}  \label{mod2:axiom:ProbabilityAxioms}
	Consider an experiment whose sample space is $S$. The \textbf{probability of the event $E$}, denoted as $P(E)$, is a number that satisfies the following three axioms
	\begin{enumerate}[label = \bfseries (\arabic*)]
		\item 	$0 \leq P(E) \leq 1 , $ \label{mod2:axiom:ProbabilityAxiom:1}

		\item 	$	P(S) = 1, $ \label{mod2:axiom:ProbabilityAxiom:2}

		\item 	For any sequence of mutually exclusive events $E_1, E_2, ... , E_n$.

					$P\left( \cup_{i=1}^n E_i \right) = \sum_{i=1}^n P(E_i), $ \label{mod2:axiom:ProbabilityAxiom:3}

				Eg. $P(E_1 \cup E_2) = P(E_1)+P(E_2)$.

	\end{enumerate}
\end{axiom}

\begin{defn} \label{mod2:defn:Probability}
	Furthermore, we define,
	\begin{equation}
		P(\text{event E occurs}) = \frac{\text{no. of times A can occur}}{\text{total number of outcomes}}.
	\end{equation}
\end{defn}

\begin{defn} \label{mod2:defn:Complement}
	The probability of the \textbf{complement} of event $A$, denoted as $P(A^c)$ or $P(\bar{A})$ or $P(A^\prime)$, is defined as
	\begin{equation}
		P(\bar{A}) = 1 - P(A).
	\end{equation}
\end{defn}

\begin{prop} \label{mod2:prop:SetPropositions}
Some useful propositions can be easily derived using the above and drawing your own diagrams
\begin{align}
P(A \cup B) &= P(A) + P(B) - P(A \cap B), \label{mod2:eq:SetProp:1}  \\
P(A^c \cap B) &= P(B) - P(A \cap B), \label{mod2:eq:SetProp:2} \\
P(A^c \cap B^c) &=  P(A \cup B)^c ,\label{mod2:eq:SetProp:3} \\
P(A^c \cup B^c) &= P (A \cap B) ^c .\label{mod2:eq:SetProp:4} \
\end{align}
\end{prop}

\begin{defn} \label{mod2:defn:MutuallyExclusive}
Two events $A$ and $B$ are said to be \textbf{mutually exclusive} if and only if
\begin{equation}
	P ( A \cap B) = 0. \label{mod2:eq:MutuallyExclusive:1} \
\end{equation}
Applying Eq~[\ref{mod2:eq:SetProp:1}], this implies
\begin{equation}
	P(A \cup B) = P(A) + P(B).\  \label{mod2:eq:MutuallyExclusive:2}
\end{equation}
\end{defn}

\begin{defn} \label{mod2:defn:Independent}
Two events $A$ and $B$ are said to be \textbf{independent} if and only if
\begin{equation}
P(A \cap B) = P (A) \times P(B). \label{mod2:eq:Independent} \
\end{equation}
\end{defn}

\begin{defn} \label{mod2:defn:Conditional}
For events $A$ and $B$, the \textbf{conditional probability} of $A$ given $B$ has occurred, denoted by $P(A|B)$, is defined by
\begin{equation}
P(A|B) = \frac{P(A\cap B)}{P(B)}. \label{mod2:eq:ConditionalProbability} \
\end{equation}
\end{defn}

\section{Discrete Random Variables} \label{mod2:section:DiscreteRandomVariables}

\begin{defn} \label{mod2:defn:DiscreteRandomVar}
	A \textbf{discrete random variable}, $X$, can take on at most a countable number of possible values. We define its \textit{probability mass} function, $P(x)$ by
	\begin{equation}
	P(x) =P(X = x).
	\end{equation}
\end{defn}

\begin{prop}\label{mod2:prop:Discrete:Properties}
	If $X$ is a discrete random variable with probability mass function $P(x)$, then we know from Axiom~[\ref{mod2:axiom:ProbabilityAxiom:1}] that
	\begin{equation}
	0 \leq P(X=x) \leq 1 \: \forall x.
	\end{equation}
	and from Axiom~[\ref{mod2:axiom:ProbabilityAxiom:2}] and Axiom~[\ref{mod2:axiom:ProbabilityAxiom:3}] that
	\begin{equation}
	\sum_{\forall x}P(X=x) = 1 .
	\end{equation}
\end{prop}

\begin{defn} \label{mod2:defn:Discrete:Expectation}
	If $X$ is a discrete random variable, the \textbf{expectation value}, \textbf{mean} or \textbf{first moment of $X$}, $E[X]$, is defined by
	\begin{equation}
	E[X]= \sum_{\forall i} x_i P(X=x_i). \label{mod2:eq:Discrete:Expectation}\
	\end{equation}
\end{defn}

\begin{defn} \label{mod2:defn:Discrete:SecondMoment}
	If $X$ is a discrete random variable, the \textbf{second moment of $X$} is defined by
	\begin{equation}
	E[X^2] = \sum_{\forall i} x_i^2 P(X=x_i). \label{mod2:eq:Discrete:SecondMoment} \
	\end{equation}
\end{defn}

\begin{defn}\label{mod2:defn:Discrete:Variance}
	If $X$ is a random variable with mean $\mu$, then the \textbf{variance}, $Var[X]$, is defined by
	\begin{align}
	Var[X] &= E[(X-\mu)^2],  \label{mod2:eq:Discrete:Variance:1}  \\
	&= E[X^2] - (E[X])^2  = E[X^2] - \mu^2 . \label{mod2:eq:Discrete:Variance:2} \
	\end{align}
\end{defn}

\section{Discrete Uniform Distribution} \label{mo2:section:DiscreteUniform}

\begin{defn} \label{mod2:defn:DiscreteUniform}
	A discrete random variable $X$ is said to follow a \textbf{discrete uniform distribution} if $X$ can take a finite number of values which are observed with equal probabilities. Therefore, the probability mass function, $P$, of $X$ which can take $n$ possible values,  is given by
	\begin{equation}
		P(X = x_i) = \frac{1}{n} \label{mod2:eq:DiscreteUniform}.
	\end{equation}
\end{defn}

\begin{note}\label{mod2:note:DiscreteUniform}
	In the discrete case, the notation $X \sim \text{Unif}(a, b)$ or $X \sim \text{U}(a, b)$ implies that $X$ can take \textbf{integer} values $x$ such that $a \leq x \leq b$. Here, the total numer of values $X$ can take is given by
	\begin{equation}
		n = b - a + 1,
	\end{equation}
	Therefore the p.m.f of $X$ can be given by
	\begin{equation}
	P(X = x)= \frac{1}{b - a + 1}\ \quad a \leq x \leq b.
	\end{equation}
		The \textit{expected value (mean)} and \textit{variance} of $X \sim \text{U}(a,b)$ are given by:
	\begin{align}
	E[X] &= \frac{a+b}{2}, \label{mod2:eq:DiscreteUniform:Mean} \\
	Var[X] &= \frac{(b-a+1)^2 - 1}{12}. \label{mod2:eq:DiscreteUniform:Variance}
	\end{align}
\end{note}

\section{Binomial Distribution} \label{mod2:section:Binomial}
\begin{defn} \label{mod2:defn:Binomial}
	A discrete random variable $X$ is said to follow a \textbf{binomial distribution} with parameters $n$ and $p$, $X \sim \text{Bin}(n,p)$, if its probability mass function, $P$, is given by
	\begin{equation}
	P(X = x) = { n \choose x} p^x (1-p)^{n-x} \quad x \in ( 0, 1, 2, ... , n). \label{mod2:eq:BinomialDist} \
	\end{equation}
\end{defn}

\begin{note}
	The \textit{expected value (mean)} and \textit{variance} of $X$ are given by:
	\begin{align}
	E[X] &= np,  \label{mod2:eq:Binomial:Mean} \\
	Var[X] &= np(1-p), \label{mod2:eq:Binomial:Variance} \\
	&= npq  \: \text{ where } q = (1-p).\
	\end{align}
\end{note}

\begin{note} \label{mod2:note:Binomial:Conditions}
	There are four conditions that describe a binomial distribution
	\begin{enumerate}[label = (\roman*)]
		\item The experiment consist of a fixed number of trials, $n$.
		\item The trials are independent.
		\item Each trial can be classified as a success or failure.
		\item The probability of success, $p$, is constant
	\end{enumerate}
\end{note}

\section{Geometric Distribution} \label{mod2:section:Geometric}
\begin{defn}\label{mod2:defn:Geomtric}
	A discrete random variable $X$ is said to follow a \textbf{geomteric distribution} with parameter $p$, $X \sim \text{Geo}(p)$, if its probability mass function, $P$, is given by
\begin{equation}
P(X = x) = p(1-p)^{x-1} \quad x \in (1, 2, ...). \label{mod2:eq:GeometricDist} \
\end{equation}
\end{defn}

\begin{note}
	The \textit{expected value (mean)} and \textit{variance} of $X$ are given by:
	\begin{align}
	E[X] &= \frac{1}{p},  \label{mod2:eq:Geometric:Mean} \\
	Var[X] &= \frac{1-p}{p^2}, \label{mod2:eq:Geomtric:Variance} \\
	&= \frac{q}{p^2}  \: \text{ where } q = (1-p).\
	\end{align}
\end{note}

\begin{note} \label{mod2:eq:Geometric:Prop}
	The probability of $X>k$ is given as
	\begin{equation}
		P(X>k)=q^k
	\end{equation}
\end{note}
\begin{note} \label{mod2:note:Geometric:Conditions}
	There are three conditions that describe a geometric distribution
	\begin{enumerate}[label = (\roman*)]
		\item The trials are independent.
		\item Each trial can be classified as a success or failure.
		\item The probability of success, $p$, is constant
	\end{enumerate}
\end{note}

\section{Poisson Distribution}\label{mod2:section:Poisson}
\begin{defn} \label{mod2:defn:Poisson}
	A discrete random variable $X$ is said to follow a \textbf{poisson distribution} with parameter $\lambda$, $X \sim \text{Pois}(\lambda)$, if its probability mass function, $P$, is given by
\begin{equation}
P(X = x) =\frac{ \lambda ^ x e^{-\lambda}}{x!} \quad x \in (0, 1, 2, ...). \label{mod2:eq:PoissonDist} \
\end{equation}
\end{defn}

\begin{note}
	The \textit{expected value (mean)} and \textit{variance} of $X$ are given by:
	\begin{align}
	E[X] &= \lambda,  \label{mod2:eq:Poisson:Mean} \\
	Var[X] &= \lambda. \label{mod2:eq:Poisson:Variance}
	\end{align}
\end{note}

\begin{note} \label{mod2:note:Poisson:Conditions}
	The Poisson distribution is popular for modeling the number of times an event occurs in an interval of time. There are two conditions that describe a poisson distribution
	\begin{enumerate}[label = (\roman*)]
		\item Events are independent of each other.
		\item The average rate at which events occur, $\lambda$, is independent of any occurrences.
	\end{enumerate}
\end{note}

\section {Poisson Approximation to the Binomial Distribution} \label{mod2:section:PoissonApproxBinomial}

 \begin{defn} \label{mod2:defn:PoissonApproxBinomial}
The \textbf{Poisson limit theorem} states that a binomially distributed random variable $X \sim \text{Bin}\{n, p\}$ can be approximated by a poisson distribution if the following hold;
\begin{enumerate}[label = (\roman*)]
	\item $n > 50$,
	\item $np < 5$.
\end{enumerate}

Under the above conditions, $X \sim \text{Bin}(n, p)$ can be approximated by $X \sim \text{Pois}(np)$, $\lambda = E[X]$.

\end{defn}

\section{Continuous Random Varibales} \label{mod2:section:ContinuousRandomVar}

\begin{defn} \label{mod2:defn:ContinuousRandomVar}
	We say that $X$ is a \textbf{continuous random variable}, if there exists a non-negative function $f$, defined for all real $x \in (-\infty,\infty)$, having the property that, for any set of real numbers $B$,
	\begin{equation}
	P(X \in B) = \int_B f(x).dx. \
	\end{equation}
	The function $f$ is called the \textit{probability density function} of the random variable $X$.
\end{defn}

\begin{prop} \label{mod2:prop:ContinuousRV:1}
	Since $X$ must assume some value, $f$ must satisfy,
	\begin{equation}
	1 = P(X \in (-\infty,\infty)) = \int_{-\infty}^{\infty} f(x).dx.  \
	\end{equation}
\end{prop}

\begin{note} \label{mod2:note:ContinuousRV:Note1}
	If $B=[a,b]$, then
	\begin{align}
	P(X\in B) &= P(a \leq X \leq b), \\
	&= \int_a^b f(x) .dx. \
	\end{align}
\end{note}

\begin{prop} \label{mod2:prop:ContinuousRV:2}
	For any continuous random variable $X$ and real number $a$,
	\begin{equation}
	P(X=a) = 0.\
	\end{equation}
\end{prop}

\begin{note}
	This implies
	\begin{equation}
	P( X \leq a) = P(X<a). \
	\end{equation}
\end{note}

\begin{defn} \label{mod2:defn:ContinuousRV:Expectation}
	If $X$ is a continuous random variable, the \textbf{expectation (mean)} of $X$ is given by,
	\begin{equation}
	E[X] = \int_{-\infty}^{\infty}x f(x).dx. \label{mod2:eq:ContinuousRV:Expectation} \
	\end{equation}
\end{defn}

\begin{defn} \label{mod2:defn:ContinuousRV:Variance}
	If $X$ is a continuous random variable, the \textbf{variance} of $X$ is given by,
	\begin{align}
	Var[X] &= E[X^2] - {E[X]}^2, \\
	 &= \int_{-\infty}^{\infty}x^2 f(x).dx - {(\int_{-\infty}^{\infty}x f(x).dx)} ^ 2. \label{mod2:eq:ContinuousRV:Variance} \
	\end{align}
\end{defn}

\begin{law} \label{mod2:law:UnconsciousStat}
	In the above formula for variance of a continuous random variable,  the \textbf{Law of the Unconscious Statiscian (LOTUS)} is used. This law states that if $X$ is a continuous randon varibale with probability density function $f$, then for any function g,
	\begin{equation}
		E[g(X)] = \int_{-\infty}^{\infty}g(x) f(x).dx. \label{mod2:eq:LOTUS:ContinuousRV}
	\end{equation}

	The law also extends to the discrete case. If $X$ is a discrete random variable, then by \textbf{LOTUS},

	\begin{equation}
	E[g(X)] =  \sum_{\forall i} g(x_i) P(X=x_i). \label{mod2:eq:LOTUS:DiscreteRV}
	\end{equation}
\end{law}

\begin{defn} \label{mod2:defn:ContinuousRV:CDF}
	For a continuous random variable $X$ with p.d.f $f$,  the \textbf{cumulative distribution function, F,} is defined as follows,
	\begin{equation}
		F(x) = P(X \leq x), \
	\end{equation}

	Using Eq~[\ref{mod2:note:ContinuousRV:Note1}], where $B = (-\infty, x]$,
	\begin{equation}
		F(x) = \int_{-\infty}^{x} f(x).dx, \
	\end{equation}

	Using the \textbf{Fundamental Law of Calculus}, then
	\begin{equation}
	f(x) = \frac{d}{dx}F(x)
.	\end{equation}

\end{defn}

\begin{note} \label{mod2:note:ContinuousRV:CDF}
	This implies
	\begin{equation}
		P( a \leq X \leq b) = F(b) - F(a).
	\end{equation}
\end{note}

\begin{prop} \label{mod2:prop:ContinuousRV:CDF}
	The c.d.f can be used to calculate the median and quartiles of a distribution. If a continuous random variable $X$ has c.d.f $F$,  then
	\begin{align}
	F(M) &= 0.5, \\
	F(LQ) &= 0.25, \\
	F(UQ) &= 0.75.
	\end{align}
	where $M$ denotes the median, $LQ$ denotes the lower quartile and $UQ$ denotes the upper quartile.
\end{prop}

\section{Normal Distribution}\label{mod2:section:Normal}

\begin{defn}\label{mod2:defn:Normal}
	A continuous random variable $X$ is said to follow a \textbf{normal distribution} with parameters $\mu$ and $\sigma^2$, $X \sim \text{N}(\mu,\sigma^2)$, if its probability density function, $f$, is given by
	\begin{equation}
	f(x) = \frac{1}{\sqrt{2 \pi}\sigma} e^{-(x-\mu)^2/2\sigma^2} \quad -\infty < x < \infty  \label{mod2:eq:Normal}\
	\end{equation}
	The parameters $\mu$ and $\sigma^2$ represent the expected value (mean) and variance respectively.
\end{defn}

\begin{note} \label{mod2:note:Normal:Properties}
	Normal random variables have important properties
	\begin{enumerate}[label = (\roman*)]
		\item If $X$ is normally distributed with parameters $\mu$ and $\sigma^2$, then $Y = aX + b$ is normally distributed with parameters $a\mu + b$ and $a^2\sigma^2$ respectively. \label{mod2:note:Normal:Property1}
		\item The previous property implies that if $X$ is normally distributed with parameters $\mu$ and $\sigma^2$, then $Z=\frac{(X-\mu)}{\sigma}$ is normally distributed with parameters $0$ and $1$. We call $Z$ a \textit{standard} or \textit{unit} normal random variable. \label{mod2:note:Normal:Property2}
	\end{enumerate}
\end{note}

\section{Normal Approximation to the Poisson Distribution} \label{mod2:section:NormalApproxPois}

\begin{defn}\label{mod2:def:NormalApproxToPois:Definition}
	A poisson distributions $X \sim \text{Pois}(\lambda)$ where $\lambda > 15$, can be approximated by a normal distribution, $X \sim \text{N}(\lambda, \sqrt{\lambda})$.
\end{defn}

\begin{note}\label{mod2:note:NormalApproxtoPois:Continuity}
	To use this approximation, we must note that because the poisson is a discrete integer-valued random variable and the normal is a continuous random variable, we must write
	\begin{align}
	P(X=a) &= P \left( a -\frac{1}{2} < X < a + \frac{1}{2} \right), \\
	P(X \leq a) &= P \left( X < a + \frac{1}{2} \right), \\
	P(X \geq a) &= P \left( X > a - \frac{1}{2} \right), \\
	P(X < a) &= P \left( X < a - \frac{1}{2} \right), \\
	P(X > a) &= P \left( X > a + \frac{1}{2} \right), \\
	P(a \leq X < b) &= P \left( a - \frac{1}{2} < X < b - \frac{1}{2} \right), \\
	P(a < X \leq  b) &= P \left( a + \frac{1}{2} < X < b + \frac{1}{2} \right), \\
	P(a  \leq  X \leq  b) &= P \left( a - \frac{1}{2} < X < b + \frac{1}{2} \right).
	\end{align}
	This is called the \textbf{continuity correction}.
\end{note}

\section{Expectation and Variance} \label{mod2:section:ExpectationVariance}

\begin{prop}

Define  functions $g$ and $f$ with $a, b, c \in \mathbb{R}$. For \textbf{ANY} random variables (discrete or cotinuous) $X$ and $Y$, the following properties hold
\begin{align}
E[aX + b] &= aE[X] + b, \\
E[X + Y] &= E[X] + E[Y], \\
E[ag(X) + bf(Y)  + c] &= aE[g(X) ]+ bE[f(Y)] + c, \\
Var[ag(X) + b] &= a^2Var[g(X)].
\end{align}

If $X$ and $Y$ are \textbf{independent} random variables, then the following properties also hold
\begin{align}
E[XY] &= E[X] \ E[Y], \\
E[g(X)f(Y)] &= E[g(X)] \ E[f(Y)], \\
Var[X + Y] &= Var[X]+ Var[Y].
\end{align}

\end{prop}

\begin{note} \label{mod2:note:Independence:Non-implication}

	If $E[XY] = E[X] \ E[Y]$ or $Var[X + Y] = Var[X]+ Var[Y]$, this does not imply $X$ and $Y$ are independent.

\end{note}


% Document Head
\documentclass[11pt, oneside]{book}
\input{latex-classnotes-preamble.tex}

% Main Body
\title{PMATH352W18 - Complex Analysis - Topical Exercises}
\author{Johnson Ng}

\begin{document}
\hypersetup{pageanchor=false}
\maketitle
\hypersetup{pageanchor=true}
\tableofcontents

\chapter{Complex Numbers}
	\label{chapter:complex_numbers}

\section{Basic Algebraic Properties} % (fold)
\label{sec:basic_algebraic_properties}

\begin{enumerate}
	\item Verify that

	\begin{inlinelist}
		\item $(\sqrt{2} - i) - i (1 - \sqrt{2}i) = -2i$;
		\item $(2 - 3i)(-2 + i) = -1 + 8i$;
		\item $(3 + i)(3 - i)(\frac{1}{5} - \frac{1}{10} i) = 2 + i$.
	\end{inlinelist}
	\item Find the complex numbers which are complex conjugates of

	\begin{inlinelist}
		\item Their own squares;
		\item Their own cubes.
	\end{inlinelist}

	\item Caclulate the following quantities:

	\begin{inlinelist}
		\item $\frac{1 + i \tan \theta}{1 - i \tan \theta}$;
		\item $\frac{(1 + 2i)^3 - (1 - i)^3}{(3 + 2i)^3 - (2 + i)^2}$;
		\item $\frac{(1 - i)^5 - 1}{(1 + i)^5 + 1}$;
		\item $\frac{(1 + i)^9}{(1 - i)^7}$.
	\end{inlinelist}

	\item Find the points $z = x + iy$ such that

	\begin{inlinelist}
		\item $\abs{z} \leq 2$;
		\item $\im z > 0$;
		\item $\re z \leq \frac{1}{2}$;
		\item $\re (z^2) = a$;
		\item $\abs{z^2 - 1} = a$;
		\item $\abs{\frac{z-1}{z + 1}} \leq 1$;
		\item $\abs{\frac{z - \alpha}{z - \beta}} = 1$.
	\end{inlinelist}

	\item Derive the identity
	\begin{equation*}
		\left( \frac{z_1}{z_3} \right)\left( \frac{z_2}{z_4} \right) = \frac{z_1 z_2}{z_3 z_4} \quad (z_3 \neq 0, z_4 \neq 0)
	\end{equation*}

	\item Using the above identity, derive the cancellation law
	\begin{equation*}
		\frac{z_1 z}{z_2 z} = \frac{z_1}{z_2} \quad (z_2 \neq 0, z \neq 0)
	\end{equation*}

	\item Using properties of moduli that has been introduced, show that when $\abs{z_3} \neq \abs{z_4}$,
	\begin{equation*}
		\frac{\re(z_1 + z_2)}{\abs{z_3 + z_4}} \leq \frac{\abs{z_1} + \abs{z_2}}{\abs{\abs{z_3} - \abs{z_4}}}
	\end{equation*}

	\item Verify that $\sqrt{2}\abs{z} \geq \abs{\re z} + \abs{\im z}$. (Hint: Reduce this inequality to $(\abs{x} - \abs{y})^2 \geq 0$.)
\end{enumerate}

(Jump to \hyperref[sub:basic_algebraic_properties]{solutions})

% section basic_algebraic_properties (end)

\section{Polar Form} % (fold)
\label{sec:polar_form}

\begin{enumerate}
	\item Represent the following complex numbers in polar form:

	\begin{inlinelist}
		\item $1 + i$;
		\item $-1 + i$;
		\item $-1 - i$;
		\item $1 - i$;
		\item $ 1 + \sqrt{3} i$;
		\item $-1 + \sqrt{3} i$;
		\item $-1 - \sqrt{3} i$;
		\item $ 1 - \sqrt{3} i$;
		\item $2 + \sqrt{3} + i$.
	\end{inlinelist}

	\item Generalize the Triangle Inequality.
	\item Prove the identity
	\begin{equation*}
		\abs{z_1 + z_2}^2 + \abs{z_1 - z_2}^2 = 2(\abs{z_1}^2 + \abs{z_2}^2),
	\end{equation*}
	for arbitrary complex numbers $z_1, z_2, ..., z_n$.

	\item When do three points $z_1, z_2, z_3 \in \mathbb{C}$ lie on a straight line in the complex plane?

	\item Let $\sigma$ be the line segment joining two points $z_1$ and $z_2$. Find the point $z$ dividign $\sigma$ in the ratio $\lambda_1 : \lambda_2$.

	\item Four points $z_1, z_2, z_3, z_4$ satisfy the conditions
	\begin{equation*}
		z_1 + z_2 + z_3 + z_4 = 0, \quad \abs{z_1} = \abs{z_2} = \abs{z_3} = \abs{z_4} = 1.
	\end{equation*}
	Show that the points either lie ast the vertices of a square inscribed in the unit circle or else coincide in pairs.

	\item Calculate the following quantities:

	\begin{inlinelist}
		\item $(1 + i)^{25}$;
		\item $\left(\frac{1 + \sqrt{3} i}{1 - i} \right)^{30}$;
		\item $\left( 1 - \frac{\sqrt{3 - i}}{2} \right)^24$;
		\item $\frac{(-1 + \sqrt{3}i)^{15}}{(1 - i)^{30}} + \frac{(-1-\sqrt{3}i)^{15}}{(1 + i)^{20}}$.
	\end{inlinelist}

	\item Use De Moivre's theorem to express $\cos nx$ and $\sim nx$ in terms of powers of $\cos x$ and $\sin x$.

	\item Express $\tan 6x$ in terms of $\tan x$.

	\item Write $\sqrt{1 + i}$ in polar form.
\end{enumerate}

% section polar_form (end)

\section{Roots of a Complex Number} % (fold)
\label{sec:roots_of_a_complex_number}

\begin{enumerate}
	\item Find all the values of the following roots:

	\begin{inlinelist}
		\item $\sqrt[3]{1}$;
		\item $\sqrt[3]{i}$;
		\item $\sqrt[4]{-1}$;
		\item $\sqrt[6]{-8}$;
		\item $\sqrt[8]{1}$;
		\item $\sqrt{3 + 4i}$;
		\item $\sqrt[3]{-2 + 2i}$;
		\item $\sqrt[5]{-4 + 3i}$;
		\item $\sqrt[6]{\frac{1 - i}{\sqrt{3} + i}}$;
		\item $\sqrt[8]{\frac{1 + i}{\sqrt{3} - i}}$;
	\end{inlinelist}

	\item Prove that the sum of all the distinct $n$th roots of unity is zero. What geometric fact does this express?

	\item Let $\epsilon$ be any $n$th root of unity other than 1. prove that
	\begin{equation*}
		1 + 2\epsilon + 3 \epsilon^2 + \hdots + n\epsilon^{n - 1} = \frac{n}{\epsilon - 1}
	\end{equation*}

	\item Prove that every complex number $\alpha \neq -1$ of unit modulus can be represented in the form
	\begin{equation*}
		\alpha = \frac{1 + it}{1 - it},
	\end{equation*}
	where $t \in \mathbb{R}$.
\end{enumerate}

% section roots_of_a_complex_number (end)

\section{Mash up} % (fold)
\label{sec:mash_up}

\begin{enumerate}
	\item Express in the form $a + bi$:
	\begin{enumerate}
		\item $\frac{1}{6 + 2i}$
		\item $\frac{(2 + i)(3 + 2i)}{1 - i}$
		\item $\left( - \frac{1}{2} + i \frac{\sqrt{3}}{2} \right)^4$
		\item $i^2, i^3, i^4, i^5, ...$
	\end{enumerate}

	\item Solve the equation $z^2 + \sqrt{32}iz - 6i = 0$

	\item Suppose $P$ is a polynomial with real coefficients. Show that $P(z) = 0$ iff $P(\bar{z}) = 0$ (i.e. zeroes of ``real'' polynomials come in conjugate pairs).
\end{enumerate}

% section mash_up (end)

% chapter complex_numbers (end)

\chapter{Complex Functions}
	\label{chapter:complex_functions}

\section{Limits and Continuity} % (fold)
\label{sec:limits_and_continuity}

\begin{enumerate}
	\item Let $f : \Omega \subseteq \mathbb{C} \to \mathbb{C}$, $z_0 \in \Omega$. $\forall z \in \Omega$, prove that
	\begin{equation*}
		(z \to z_0 \implies f(z) \to \infty) \iff (z \to z_0 \implies \psi(z) = \frac{1}{f(z)} \to 0)
	\end{equation*}

	\item The \hlnotea{Cauchy Convergence Criterion for sequences} states that a complex sequence $z_n$ is convergent iff
	\begin{gather*}
		\forall \epsilon > 0 \; \exists N = N(\epsilon) > 0 \; \forall m, n > N \\
		\abs{z_m - z_n} < \epsilon.
	\end{gather*}
	Prove the generalization of the criterion: The function $f(z)$ approaches a limit as $z \to z_0$ iff
	\begin{gather*}
		\forall \epsilon > 0 \; \exists \delta = \delta(\epsilon) > 0 \\
		(0 < \abs{z' - z_0} < \delta \; \land \; 0 < \abs{z'' - z} < \delta) \implies \abs{f(z') - f(z'')} < \epsilon
	\end{gather*}

	\item Let $f(z)$ be a rational function, i.e., a ratio
	\begin{equation}\label{eq:1}
		f(z) = \frac{a_0 + a_1 z + \hdots a_m z^m}{b_0 + b_1z + \hdots + b_n z^n} \enspace (a_m \neq 0, b_n \neq 0)
	\end{equation}
	of two polynomials. Discuss the possible values of $\lim_{z \to \infty} f(z)$.

	\item Where is the function \cref{eq:1} continuous?

	\item Prove that if $f(z)$ is continuous in a region $\Omega$, then so is $\abs{f(z)}$.

	\item Is the function
	\begin{equation*}
		f(zz) = \frac{1}{1 - z}
	\end{equation*}
	continuous in the open disk $\abs{z} < 1$?
\end{enumerate}

% section limits_and_continuity (end)

% chapter complex_functions (end)

\chapter{Differentiation}
	\label{chapter:differentiation}

\section{Others} % (fold)
\label{sec:others}

\begin{enumerate}
	\item \begin{enumerate}
		\item Suppose $f(z)$ is real-valued and differentiable for all real $z$. Show that $f'(z)$ is also real-valued for real $z$.
		\item Suppose $f(z)$ is real-valued and differentiable for all imaginary points $z$. Show that $f'(z)$ is imaginary for all imaginary points $z$.
	\end{enumerate}
\end{enumerate}

% section others (end)

% chapter differentiation (end)

\chapter{Integration}
	\label{chapter:integration}

\section{Analyticity} % (fold)
\label{sec:analyticity}

\begin{enumerate}
	\item Prove that a nonconstant entire function cannot satisfy the two equations
		\begin{enumerate}
			\item $f(z + 1) = f(z)$
			\item $f(z + i) = f(z)$
		\end{enumerate}
		for all $z$ [\textit{Hint:} Show that a function satisfying both equalities would be bounded.]
\end{enumerate}

% section analyticity (end)

% chapter integration (end)

\chapter{Singularities}
	\label{chapter:singularities}

% chapter singularities (end)

\chapter*{Answers}\label{chapter:answers}
\addcontentsline{toc}{chapter}{Answers}

\fontsize{9}{11}\selectfont

\section*{Chapter 1} % (fold)
\label{sec:chapter_1}

\subsection*{Basic Algebraic Properties} % (fold)
\label{sub:basic_algebraic_properties}

(Jump to: \cref{sec:basic_algebraic_properties})

\begin{enumerate}
	\item
	\item \begin{enumerate}
		\item $(0, 0), (1, 0), (-\frac{1}{2} , \pm \sqrt{\frac{3}{4}})$
		\item $(0, 0), (1, 0), (-1, 0), (0, i), (0, -i)$
	\end{enumerate}
	\item \begin{enumerate}
		\item $(\cos \theta - i \sin \theta)^2$
		\item
		\item $\frac{9 - 40i}{41}$
		\item $2$
	\end{enumerate}
	\item \begin{enumerate}
		\item $\{(x, y) : x^2 + y^2 \leq 4\}$
		\item $\{(x, y) : y > 0\}$
		\item $\{(x, y) : x \leq \frac{1}{2} \}$
		\item $x^2 - y^2 = a$. Also, refer to this graph on Desmos: \url{https://www.desmos.com/calculator/buwtyobjrn}
		\item Refer to graph on Desmos: \url{https://www.desmos.com/calculator/a3pnbwueja}
	\end{enumerate}
	\item \begin{align*}
		\left( \frac{z_1}{z_3} \right) \left( \frac{z_2}{z_4} \right)
			&= z_1 \left( \frac{\bar{z_3}}{\abs{z_3}^2} \right) z_2 \left( \frac{\bar{z_4}}{\abs{z_4}^2} \right) \quad \text{since } z\bar{z} = \abs{z}^2 \\
			&= z_1 z_2 \left( \frac{\bar{z_3} \bar{z_4}}{\abs{z_3}^2 \abs{z_4}^2} \right) \\
			&= z_1 z_2 \left( \frac{\bar{z_3 z_4}}{\abs{z_3 z_4}^2} \right) \quad \text{since } \bar{z}\bar{w} = \bar{zw} \text{ and } \abs{z}\abs{w} = \abs{zw} \\
			&= \frac{z_1 z_2}{z_3 z_4} \quad \text{since } z\bar{z} = \abs{z}^2
	\end{align*}
	\item $\frac{z_1 z}{z_2 z} = \left( \frac{z_1}{z_2} \right) \left(\frac{z}{z} \right) = \frac{z_1}{z_2}$
	\item By the Triangle Inequality,
	\begin{equation*}
		\abs{z_1} + \abs{z_2} \geq \abs{z_1 + z_2} \geq \sqrt{\re(z_1 + z_2)^2 + \im(z_1 + z_2)^2} \geq \sqrt{\re(z_1 + z_2)} = \re(z_1 + z_2)
	\end{equation*}
	and by the Reversed Triangle Inequality,
	\begin{gather*}
		\abs{z_3 + z_4} \geq \abs{\abs{z_3} - \abs{z_4}} \\
		\implies \frac{1}{\abs{z_3 + z_4}} \leq \frac{1}{\abs{\abs{z_3} - \abs{z_4}}}
	\end{gather*}
	Thus
	\begin{equation*}
		\frac{\re(z_1 + z_2)}{\abs{z_3 + z_4}} \leq \frac{\abs{z_1} + \abs{z_2}}{\abs{z_3 + z_4}} \leq \frac{\abs{z_1} + \abs{z_2}}{\abs{\abs{z_3} - \abs{z_4}}}
	\end{equation*}
\end{enumerate}

% subsection basic_algebraic_properties (end)

% section chapter_1 (end)

% chapter answers (end)

\end{document}


\subsection{Exterior (Grassman) algebra}

The exterior algebra is the algebra generated by the wedge product.

The term \(u\land v\) can be interpreted as the area covered by the parallelogram generated by \(u\) and \(v\).

As \(a\mathbf u\land b\mathbf v=ab \mathbf u\land \mathbf v\), we can see that scaling the length of one of the vectors by a scalar, we also increase the exterior product by the same scalar.

\subsection{Orientation}

We can describe the exterior product of two vectors as \(\mathbf u\land \mathbf v\) or \(\mathbf v \land \mathbf u\).


\documentclass[11pt, a4paper]{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{pgfplots}
\usepackage{algorithm}
\usepackage{placeins}
\usepackage{algorithmic}
\usepackage{booktabs}

\begin{document}

\title{THE PERCEPTRON}
\date{}
\maketitle

The perceptron is a learning algorithm for binary classification of real valued vectors.

\section{Introduction}

The perceptron binary classifier can be thought of as the following function,

\begin{align*}
	f(\boldsymbol{x}) = \left\{
	\begin{array}{ll}
	1\ \text{if}\ \boldsymbol{w}^T \boldsymbol{x} > 0 \\
	0\ \text{otherwise}                               \\
	\end{array}
	\right.
\end{align*}

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]

		\draw[-latex] (-3,0) -- (3,0) node[right]{$x_1$};
		\draw[-latex] (0,-2) -- (0,2) node[left]{$x_2$};

		\draw[dashed] (-1, 2) -- (1, -2);
		\draw (1, -2) node [below right] {Separating Plane};

		\draw[very thick,->, >=stealth] (0, 0) -- (1, 0.5);
		\draw (1, 0.5) node [above right] {$\vec{\boldsymbol{w}}$};

		\draw (-0.1, -0.3) node [below left] {$f(\boldsymbol{x}) < 0$};
		\draw (0.5, 0) node [below right] {$f(\boldsymbol{x}) > 0$};
	\end{tikzpicture}
\end{figure}

One severe limitation of this formulation is that the separating hyperplane always passes through origin which might be undesirable in many cases. However, this limitation can be overcome in the following way,

\begin{align*}
	f(\boldsymbol{x}) = \left\{
	\begin{array}{ll}
	1\ \text{if}\ \boldsymbol{w}^T \boldsymbol{x} + b > 0 \\
	0\ \text{otherwise}                                   \\
	\end{array}
	\right.
\end{align*}

The bias term $b$ frees the separating hyperplane from origin. Additionally, we can employ a trick to simplify the expression as follows.

\begin{align*}
	f(\boldsymbol{x}) = \left\{
	\begin{array}{ll}
	1\ \text{if}\ \begin{pmatrix} \boldsymbol{w}^T, b \end{pmatrix} \begin{pmatrix} \boldsymbol{x} \\
	1 \end{pmatrix} > 0                                                                            \\
	0\ \text{otherwise}                                                                            \\
	\end{array}
	\right.
\end{align*}

Hence, by increasing the dimension of input by one and defaulting the intercept on the new dimension to 1 for every input vector, any separating hyperplane in the old dimensional space
becomes a hyperplane passing through origin in the new dimensional space. This simplifies the mathematics without constraining the classifier.

The vector $\vec{\boldsymbol{w}}$ is learned from training data as usual. The algorithm for learning the same is presented in the next section.

\section{Learning Algorithm}

The training data consists of $N$ pairs of $d$-dimensional real input vectors $\boldsymbol{x}_i$s and the output binary labels $y_i$s. For mathematical convenience, positive output labels are denoted by $+1$ and negative output labels by {-1}.
The learning algorithm makes the following assumptions:

\begin{itemize}
	\item It is assumed that the input vectors have already been extended to account for the bias term.
	\item The training data is assumed to be linearly separable at origin. So, there exists a unit vector $\boldsymbol{w}^*$ such that $y_i\boldsymbol{w}^{*T}\boldsymbol{x_i} > \gamma \ \ \forall i \ \ \text{where}\ \ \gamma > 0$.
	\item It is assumed that all training input vectors are finite.
\end{itemize}

The algorithm is defined as follows.

\begin{algorithm}
	\renewcommand{\thealgorithm}{}
	\caption{The Perceptron Learning Algorithm}
	\begin{algorithmic}
		\STATE $k \leftarrow 1$
		\STATE $\boldsymbol{w}_k \leftarrow 0$

		\WHILE {there exists $j \in \{1, 2, .., N\}$ such that $ y_j\boldsymbol{w}^{*T}\boldsymbol{x_j} <= 0$ }
		\STATE {pick $i \in \{1, 2, .., N\}$ such that $ y_i\boldsymbol{w}^{*T}\boldsymbol{x_i} <= 0$}
		\STATE $\boldsymbol{w}_{k+1} \leftarrow \boldsymbol{w}_k + y_i \boldsymbol{\boldsymbol{x}_i}$
		\STATE $ k \leftarrow k + 1$
		\ENDWHILE
		\RETURN $\boldsymbol{w}_k$
	\end{algorithmic}
\end{algorithm}

\section{Example}

The algorithm is illustrated on a very simple example.

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]

		\draw[-latex] (-4,0) -- (4,0) node[right]{$x_1$};
		\draw[-latex] (0,-3) -- (0,3) node[left]{$x_2$};

		\draw[help lines] (-3.5,-2.5) grid (3.5,2.5);

		\draw[point line] (-1.7,0) -- (-2.3,0);
		\draw[point line] (-0.3,-2) -- (0.3,-2);

		\draw[point line] (-1.7,2) -- (-2.3,2);
		\draw[point line] (-2,1.7) -- (-2,2.3);

		\draw[point line] (1.7,2) -- (2.3,2);
		\draw[point line] (2,1.7) -- (2,2.3);

		\draw (-2, 2) node [below right] {(-2, 2)};
		\draw (2, 2) node [below left] {(2, 2)};
		\draw (-2, 0) node [below right] {(-2, 0)};
		\draw (0, -2) node [below right] {(0, -2)};
	\end{tikzpicture}
\end{figure}

\FloatBarrier

\subsection{Iteration 1}

\begin{align*}
	\boldsymbol{w}_1 = \begin{pmatrix} 0 \\ 0 \end{pmatrix}
\end{align*}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|}
		\toprule
		$\boldsymbol{x}$    \\
		\midrule
		$\begin{pmatrix} -2 \\ 0 \end{pmatrix}$ & -1 & 0 \\
		$\begin{pmatrix} 0  \\ -2 \end{pmatrix}$ & -1 & 0 \\
		$\begin{pmatrix} -2 \\ 2 \end{pmatrix}$ & 1 & 0 \\
		$\begin{pmatrix} 2  \\ 2 \end{pmatrix}$ & 1 & 0 \\
		\hline
	\end{tabular}
\end{table}

\subsection{Iteration 2}

\begin{align*}
	\boldsymbol{w}_2 & = \begin{pmatrix} 0 \\ 0 \end{pmatrix} + (-1)  \begin{pmatrix} -2 \\ 0 \end{pmatrix}  \\
	                 & = \begin{pmatrix} 2 \\ 0 \end{pmatrix}
\end{align*}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|}
		\toprule
		$\boldsymbol{x}$    \\
		\midrule
		$\begin{pmatrix} -2 \\ 0 \end{pmatrix}$ & -1 & 4 \\
		$\begin{pmatrix} 0  \\ -2 \end{pmatrix}$ & -1 & 0 \\
		$\begin{pmatrix} -2 \\ 2 \end{pmatrix}$ & 1 & -4 \\
		$\begin{pmatrix} 2  \\ 2 \end{pmatrix}$ & 1 & 4 \\
		\hline
	\end{tabular}
\end{table}

\FloatBarrier

\subsection{Iteration 3}

\begin{align*}
	\boldsymbol{w}_3 & = \begin{pmatrix} 2 \\ 0 \end{pmatrix} + (-1)  \begin{pmatrix} 0 \\ -2 \end{pmatrix}  \\
	                 & = \begin{pmatrix} 2 \\ 2 \end{pmatrix}
\end{align*}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|}
		\toprule
		$\boldsymbol{x}$    \\
		\midrule
		$\begin{pmatrix} -2 \\ 0 \end{pmatrix}$ & -1 & 4 \\
		$\begin{pmatrix} 0  \\ -2 \end{pmatrix}$ & -1 & 4 \\
		$\begin{pmatrix} -2 \\ 2 \end{pmatrix}$ & 1 & 0 \\
		$\begin{pmatrix} 2  \\ 2 \end{pmatrix}$ & 1 & 8 \\
		\hline
	\end{tabular}
\end{table}

\FloatBarrier

\subsection{Iteration 4}

\begin{align*}
	\boldsymbol{w}_4 & = \begin{pmatrix} 2 \\ 2 \end{pmatrix} + (1)  \begin{pmatrix} -2 \\ 2 \end{pmatrix}  \\
	                 & = \begin{pmatrix} 0 \\ 4 \end{pmatrix}
\end{align*}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|}
		\toprule
		$\boldsymbol{x}$    \\
		\midrule
		$\begin{pmatrix} -2 \\ 0 \end{pmatrix}$ & -1 & 0 \\
		$\begin{pmatrix} 0  \\ -2 \end{pmatrix}$ & -1 & 8 \\
		$\begin{pmatrix} -2 \\ 2 \end{pmatrix}$ & 1 & 8 \\
		$\begin{pmatrix} 2  \\ 2 \end{pmatrix}$ & 1 & 8 \\
		\hline
	\end{tabular}
\end{table}

\FloatBarrier

\subsection{Iteration 5}

\begin{align*}
	\boldsymbol{w}_5 & = \begin{pmatrix} 0 \\ 4 \end{pmatrix} + (-1)  \begin{pmatrix} -2 \\ 0 \end{pmatrix}  \\
	                 & = \begin{pmatrix} 2 \\ 4 \end{pmatrix}
\end{align*}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|}
		\toprule
		$\boldsymbol{x}$    \\
		\midrule
		$\begin{pmatrix} -2 \\ 0 \end{pmatrix}$ & -1 & 4 \\
		$\begin{pmatrix} 0  \\ -2 \end{pmatrix}$ & -1 & 8 \\
		$\begin{pmatrix} -2 \\ 2 \end{pmatrix}$ & 1 & 4 \\
		$\begin{pmatrix} 2  \\ 2 \end{pmatrix}$ & 1 & 12 \\
		\hline
	\end{tabular}
\end{table}

\FloatBarrier

All the points are correctly classified, hence the algorithm terminates. The algorithm converged in 5 iterations. In fact, as will be proved later, the algorithm is always guaranteed to converge in finite iterations.The resulting separating hyperplanes is shown below.

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]

		\draw[-latex] (-5,0) -- (5,0) node[right]{$x_1$};
		\draw[-latex] (0,-5) -- (0,5) node[left]{$x_2$};

		\draw[help lines] (-4.5,-4.5) grid (4.5,4.5);

		\draw[point line] (-1.7,0) -- (-2.3,0);
		\draw[point line] (-0.3,-2) -- (0.3,-2);

		\draw[point line] (-1.7,2) -- (-2.3,2);
		\draw[point line] (-2,1.7) -- (-2,2.3);

		\draw[point line] (1.7,2) -- (2.3,2);
		\draw[point line] (2,1.7) -- (2,2.3);

		\draw (-2, 2) node [below right] {(-2, 2)};
		\draw (2, 2) node [below left] {(2, 2)};
		\draw (-2, 0) node [below right] {(-2, 0)};
		\draw (0, -2) node [below right] {(0, -2)};

		\draw [dashed] (-4, 2) -- (4, -2);
		\draw (4, -2) node [below right] {Separating Plane};

		\draw[very thick,->, >=stealth] (0, 0) -- (2, 4);
		\draw (2, 4) node [above right] {$\vec{\boldsymbol{w}_5}$};

	\end{tikzpicture}
\end{figure}

\FloatBarrier

\section{Proof of Convergence}

The intention is to show that the perceptron learning algorithm always converges to a perfect classifier in finite iterations given that all the assumptions that the algorithm makes are met.

Assume that after k+1 iterations, $\vec{\boldsymbol{w}_{k+1}}$ makes an angle $\theta_{k+1}$ with $\vec{\boldsymbol{w}^*}$.

\begin{align*}
	cos(\theta_{k+1}) = \frac{\vec{\boldsymbol{w}_{k+1}}.\vec{\boldsymbol{w}^*}}{|\vec{\boldsymbol{w}_{k+1}}|\ |\vec{\boldsymbol{w}^*}|}
\end{align*}

Since $\vec{\boldsymbol{w}^*}$ is a unit vector,

\begin{align*}
	cos(\theta_{k+1}) = \frac{\vec{\boldsymbol{w}_{k+1}}.\vec{\boldsymbol{w}^*}}{|\vec{\boldsymbol{w}_{k+1}}|}
\end{align*}

Now, bounds for numerator and denominator are established to proceed ahead.

\subsection{Bound for numerator}

The update rule for $\vec{\boldsymbol{w}_{k+1}}$ is,

\begin{align*}
	\vec{\boldsymbol{w}_{k+1}} = \vec{\boldsymbol{w}_k} + y_i \vec{\boldsymbol{\boldsymbol{x}_i}}
\end{align*}

Taking dot product with $\vec{\boldsymbol{w}^*}$ on both sides.

\begin{align*}
	\vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{k+1}} = \vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_k} + y_i \vec{\boldsymbol{w}^*}.\vec{\boldsymbol{\boldsymbol{x}_i}}
\end{align*}

Since $y_i \vec{\boldsymbol{w}^*}.\vec{\boldsymbol{\boldsymbol{x}_i}} > \gamma$ as per the algorithm assumption.

\begin{align*}
	\vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{k+1}} > \vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_k} + \gamma      \\
	\vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{k+1}} > \vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{k-1}} + 2\gamma \\
	\vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{k+1}} > \vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{1}} + k\gamma
\end{align*}

since $\vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{1}} = 0$.

\begin{align*}
	\vec{\boldsymbol{w}^*}.\vec{\boldsymbol{w}_{k+1}} >  k\gamma
\end{align*}

\subsection{Bound for denominator}

Again starting with the update rule for $\vec{\boldsymbol{w}_{k+1}}$,

\begin{align*}
	\vec{\boldsymbol{w}_{k+1}} = \vec{\boldsymbol{w}_k} + y_i \vec{\boldsymbol{\boldsymbol{x}_i}}
\end{align*}

Taking modulus on both sides,

\begin{align*}
	|\vec{\boldsymbol{w}_{k+1}}|^2 = |\vec{\boldsymbol{w}_k}|^2 + |y_i \vec{\boldsymbol{\boldsymbol{x}_i}}|^2 + 2y_i \vec{\boldsymbol{w}_k}.\vec{\boldsymbol{\boldsymbol{x}_i}}
\end{align*}

Since $y_i \vec{\boldsymbol{w}_k}.\vec{\boldsymbol{\boldsymbol{x}_i}} <= 0$ according to the algorithm specification.

\begin{align*}
	|\vec{\boldsymbol{w}_{k+1}}|^2 \leq |\vec{\boldsymbol{w}_k}|^2 + |y_i \vec{\boldsymbol{\boldsymbol{x}_i}}|^2
\end{align*}

Assuming that the largest input vector has length $R$.

\begin{align*}
	|\vec{\boldsymbol{w}_{k+1}}|^2 \leq |\vec{\boldsymbol{w}_k}|^2 + R^2      \\
	|\vec{\boldsymbol{w}_{k+1}}|^2 \leq |\vec{\boldsymbol{w}_{k-1}}|^2 + 2R^2 \\
	|\vec{\boldsymbol{w}_{k+1}}|^2 \leq kR^2                                  \\
	|\vec{\boldsymbol{w}_{k+1}}| \leq \sqrt{k}R
\end{align*}

\subsection{Bound for Number of Iterations}

Using both the numerator and denominator bounds,

\begin{align*}
	cos(\theta_{k+1}) > \frac{k\gamma}{\sqrt{k}R} \\
	cos(\theta_{k+1}) > \sqrt{k} \frac{\gamma}{R}
\end{align*}

This shows that after each iteration, the lower bound of the angle between $\vec{\boldsymbol{w}} $ and $\vec{\boldsymbol{w}^*}$ increases. And since $cos(\theta_{k+1})$ can never be greater than one, $\sqrt{k} \frac{\gamma}{R} \leq1$ must hold true yielding $k \leq \frac{R^2}{\gamma^2}$.

\section{Conclusion}

The perceptron was one of the first algorithms indicating that computers could learn from data. And today, after half a century, it continues to be the basis of Artificial Neural Networks which are at the forefront of Artificial Intelligence.

\end{document}


\textbf{Compute te solution to the systems for the given initial conditions using \texttt{rk4.m}. Plot your results in the phase plane.}
\vskip.00005in
\noindent\rule{\textwidth}{1pt}
\vspace{0.1in}
\begin{enumerate}
\item[1.] \textbf{Using initial data conditions with $x(0)^2+y(0)^2$ both smaller and larger than $1$, solve}
\begin{align*}
x'(t)&=-4y+x(1-x^2-y^2),\\
y'(t)&=4x+y(1-x^2-y^2),
\end{align*}
\textbf{over the interval $0\leq t\leq 10$}.
Given the system of ODEs, we see that when at the unit circle the parenthesis of the right hand side vanishes. Moreover, if we express the ODEs in polar coordinates, we see that, at the unit circle, we obtain $dr=0$ and $d\theta=4$. This implies that the orbits, once they reach the unit circle, the stay indefinitely there since $dr=0$ and moving counterclockwise since $d\theta=4$. Inside the unit circle the vector field is divergent, it points away from the origin. This implies that the orbits are gonna approach the unit circle if we place the initial condition inside it. We have the inverse situation outside the unit circle. Thus, the unit circle constitutes an $\omega-limit$ set of this system of ODEs. In the next figure we can see the behaviour described and the vector field.
\begin{figure}[H]
\centering     %%% not \center
\hspace*{\fill}
\subfigure[Vector field]{\includegraphics[scale=0.75]{P1_1field.eps}}
\hspace*{\fill}
\subfigure[Orbits]{\includegraphics[scale=0.75]{P1_1.eps}}
\hspace*{\fill}
\caption{Solutions of the system of ODEs.}
\end{figure}
\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 1a
legendfontsize=14;
axisfontsize=16;

F = @(t,V) [-4*V(2)+V(1).*(1-V(1).^2-V(2).^2);...
                4*V(1)+V(2).*(1-V(1).^2-V(2).^2)];
theta=pi/4;
R=[0.1; 2.4];
tspan=[0 10];

N=300;
a=2.5;
for i=1:length(R)
    V0 = [R(i)*cos(theta); R(i)*sin(theta)];
    [t,V] = rk4(F,tspan,V0,N);
    figure(1)
    l(1)=plot(V(:,1),V(:,2),'r');
    hold on
    l(2)=plot(V0(1),V0(2),'r*');
    axis([-a a -a a])
    grid on
end
% Plot orbits
figure(1)
hold on
l(3)=plot(cos(0:0.001:2*pi),sin(0:0.001:2*pi),'--b');
xlabel('$x(t)$','interpreter','latex')
ylabel('$y(t)$','interpreter','latex')
pbaspect([1 1 1])
set(gca,'fontsize',14)
legend([l(1) l(2) l(3)],{' Orbits',' Initial condtions',' Limit circle'},...
 'Interpreter','latex','fontsize',legendfontsize,'location','SouthEast')
txt='Latex/FIGURES/P1_1';
saveas(gcf,txt,'epsc')

% Plot vector field
step=0.2;
[x,y] = meshgrid(-a:step:a,-a:step:a);
u=-4*y+x.*(1-x.^2-y.^2);
v=4*x+y.*(1-x.^2-y.^2);
figure(2)
q=quiver(x,y,u,v,'r');
hold on
plot(cos(0:0.001:2*pi),sin(0:0.001:2*pi),'--b');
xlabel('$x(t)$','interpreter','latex')
ylabel('$y(t)$','interpreter','latex')
pbaspect([1 1 1])
axis([-a a -a a])
set(gca,'fontsize',14)
set(q,'AutoScale','on', 'AutoScaleFactor', 1.5)
txt='Latex/FIGURES/P1_1field';
saveas(gcf,txt,'epsc')
\end{verbatim}
\item[2.] \textbf{Using initial data conditions with $x(0)^2+y(0)^2$ both inside and outside of the circles of radius $1$ and $2$, solve}
\begin{align*}
x'(t)&=-4y+x(1-x^2-y^2)(4-x^2-y^2),\\
y'(t)&=4x+y(1-x^2-y^2)(4-x^2-y^2),
\end{align*}
\textbf{over the interval $0\leq t\leq 10$}.

Given the system of ODEs, we see that when at the unit circle or at the circle of radius $2$, the second term of the right hand side cancels in both ODEs. This give us the same condition as before, once the orbits reach those circles, they stay there indefinitely. However, the addition of the extra factor makes the vector field divergent outside the outer circle, and changes the behaviour of the system. If the initial condition is outside the outter circl, the solutions are unstable. We can see the vector field and the orbits of the solutions in the next figure. Note that the initial conditions outisde the circle of radius $2$ are placed just very close to the limit.
\begin{figure}[H]
\centering     %%% not \center
\hspace*{\fill}
\subfigure[Vector field]{\includegraphics[scale=0.75]{P1_2field.eps}}
\hspace*{\fill}
\subfigure[Orbits]{\includegraphics[scale=0.75]{P1_2.eps}}
\hspace*{\fill}
\caption{Solutions of the system of ODEs.}
\end{figure}
\end{enumerate}
\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 1b
close all
F = @(t,V) [-4*V(2)+V(1).*(1-V(1).^2-V(2).^2).*(4-V(1).^2-V(2).^2);...
                4*V(1)+V(2).*(1-V(1).^2-V(2).^2).*(4-V(1).^2-V(2).^2)];
theta=pi/5;
R=[0.1; 1.2; 1.8];
tspan=[0 10];
N=300;
a=3;
for i=1:length(R)
    V0 = [R(i)*cos(theta); R(i)*sin(theta)];
    [t,V] = rk4(F,tspan,V0,N);
    figure(1)
    plot(V(:,1),V(:,2),'r')
    hold on
    plot(V0(1),V0(2),'r*')
    axis([-a a -a a])
    grid on
end
Theta=0:pi/4:2*pi;
for i=1:length(Theta)
    V0 = [2*cos(Theta(i)); 2*sin(Theta(i))];
    [t,V] = rk4(F,[0 1],V0,N);
    figure(1)
    l(1)=plot(V(:,1),V(:,2),'r');
    hold on
    l(2)=plot(V0(1),V0(2),'r*');
    axis([-a a -a a])
    grid on
end

figure(1)
hold on
l(3)=plot(cos(0:0.001:2*pi),sin(0:0.001:2*pi),'--b');
plot(2*cos(0:0.001:2*pi),2*sin(0:0.001:2*pi),'--b');
xlabel('$x(t)$','interpreter','latex')
ylabel('$y(t)$','interpreter','latex')
pbaspect([1 1 1])
set(gca,'fontsize',14)
%legend([l(1) l(2) l(3)],{' Orbits',' Initial condtions',' Limit circle'},...
% 'Interpreter','latex','fontsize',legendfontsize,'location','SouthEast')
txt='Latex/FIGURES/P1_2';
saveas(gcf,txt,'epsc')

% Plot vector field
step=0.1;
b=2;
[x,y] = meshgrid(-b:step:b,-b:step:b);
u=-4*y+x.*(1-x.^2-y.^2).*(4-x.^2-y.^2);
v=4*x+y.*(1-x.^2-y.^2).*(4-x.^2-y.^2);
figure(2)
q=quiver(x,y,u,v,'r');
set(q,'AutoScale','on', 'AutoScaleFactor', 3)
hold on
% step=0.1;
% b=2.1;
% [x,y] = meshgrid(-b:step:b,-b:step:b);
% u=-4*y+x.*(1-x.^2-y.^2).*(4-x.^2-y.^2);
% v=4*x+y.*(1-x.^2-y.^2).*(4-x.^2-y.^2);
% q=quiver(x,y,u,v,'r');
% step=0.05;
% b=1.2;
% [x,y] = meshgrid(-b:step:b,-b:step:b);
% u=-4*y+x.*(1-x.^2-y.^2).*(4-x.^2-y.^2);
% v=4*x+y.*(1-x.^2-y.^2).*(4-x.^2-y.^2);
% q=quiver(x,y,u,v,'r');
% set(q,'AutoScale','on', 'AutoScaleFactor', 0.5)
plot(cos(0:0.001:2*pi),sin(0:0.001:2*pi),'--b');
plot(2*cos(0:0.001:2*pi),2*sin(0:0.001:2*pi),'--b');
xlabel('$x(t)$','interpreter','latex')
ylabel('$y(t)$','interpreter','latex')
pbaspect([1 1 1])
grid on
a=2.1;
axis([-a a -a a])
set(gca,'fontsize',14)
txt='Latex/FIGURES/P1_2field';
saveas(gcf,txt,'epsc')
\end{verbatim}


\documentclass[a4paper]{scrartcl}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{amssymb,amsmath}

\newtheorem{theorem}{Theorem}
\newenvironment{proof}{\paragraph{Proof:}}{\hfill$\square$}
\newcommand{\Prob}{\mathbb{P}}

\begin{document}
    \begin{theorem}
    Let $Y \sim \mathcal{N}(\mu, \sigma^2)$ and $X \sim e^Y$.
    Then X has the density
    \[f_X(x) = \begin{cases} \frac{1}{x \sigma \sqrt{2 \pi}}\exp{- \frac{(\log x - \mu)^2}{2 \sigma^2}} &\text{if } x > 0\\
                             0 & \text{otherwise}\end{cases}\]
    \end{theorem}

    \begin{proof}
    \begin{align}
        \Prob(X \leq t) &= \Prob(e^Y \leq t)\\
                        &= \begin{cases}\Prob(Y \leq \log(t)) &\text{if } x > 0\\
                                        0 &\text{otherwise}
                           \end{cases}
    \end{align}

    Obviously, the density $f_X(x) = 0$ for $x \leq 0$. Now continue with
    $t > 0$:

    \begin{align}
        \Prob(X \leq t) &= \Prob(Y \leq \log(t))\\
                        &= \Phi_{\mu, \sigma^2}(\log(t))\\
                        &= \Phi_{0, 1} \left (\frac{\log(t) - \mu}{\sigma} \right)\\
        f_X(x) &= \frac{\partial}{\partial x} \Phi_{0, 1} \left (\frac{\log(x) - \mu}{\sigma} \right)\\
               &= \left (\frac{\partial}{\partial x} \left (\frac{\log(x) - \mu}{\sigma} \right) \right) \cdot \varphi_{0, 1} \left (\frac{\log(x) - \mu}{\sigma} \right)\\
               &= \left (\frac{\sigma \cdot \frac{1}{x}}{\sigma^2} \right) \cdot \varphi_{0, 1} \left (\frac{\log(x) - \mu}{\sigma} \right)\\
               &= \frac{1}{x \sigma} \cdot \varphi_{0, 1} \left (\frac{\log(x) - \mu}{\sigma} \right)\\
               &= \frac{1}{x \sigma} \cdot \frac{1}{\sqrt{2\pi}} \exp \left (-\frac{1}{2} \cdot {\left(\frac{\log(x) - \mu}{\sigma} \right )}^2 \right )
    \end{align}
    \end{proof}
\end{document}


\subsubsection{Improper Fractions}
\noindent
If the degree of the numerator is greater than or equal to the degree of the denominator, we have a case of improper fractions.
In this case, we have to do polynomial long division to get a quotient and remainder and then decompose the remainder if necessary.
So,
\begin{equation*}
	\frac{P(x)}{Q(x)} = R(x) + \frac{S(x)}{Q(x)}.
\end{equation*}

\begin{example}
	Find the partial fraction decomposition of the following expression:
	\begin{equation*}
		\frac{x^3+3}{x^2-2x-3}.
	\end{equation*}
\end{example}
\noindent
First we do polynomial long division to find that
\begin{equation*}
	\frac{x^3+3}{x^2-2x-3} = x + 2 + \frac{7x+9}{x^2-2x-3}.
\end{equation*}
Now that the numerator is of a lesser degree than the denominator, we can decompose it normally.
\begin{equation*}
	x^2-2x-3 = (x-3)(x+1).
\end{equation*}
So,
\begin{equation*}
	\frac{7x+9}{x^2-2x-3} = \frac{A_1}{x-3} + \frac{A_2}{x+1}.
\end{equation*}
Multiplying each side by the denominator,
\begin{equation*}
	7x+9 = A_1(x+1) + A_2(x-3).
\end{equation*}
At $x=-1$,
\begin{equation*}
	2 = -4A_2 \implies A_2 = \frac{-1}{2}.
\end{equation*}
At $x=3$,
\begin{equation*}
	30 = 4A_1 \implies A_1 = \frac{15}{2}.
\end{equation*}
So,
\begin{equation*}
	\frac{x^3+3}{x^2-2x-3} = x + 2 + \frac{15/2}{x-3} + \frac{-1/2}{x+1}.
\end{equation*}


%!TEX root = main.tex
\section{Deliverable 1}
\label{sec:deliverable-1}

\subsection{Math Equations}
For example, one can type the following equation:
\begin{equation}
\vt^W = \MR_r^W \vt^r = \left[\begin{array}{ccc}
1 & 0 & 0 \\
0 & 1 & 0 \\
0 & 0 & 1
\end{array} \right]
\left[\begin{array}{c}
1\\
1 \\
1
\end{array} \right]
=
\left[\begin{array}{c}
1\\
1 \\
1
\end{array} \right].
\end{equation}

Later in the course, you may want to type an optimization problem:
\begin{eqnarray}
f^\star = \min_{\vxx \in \mathbb{R}^n} & f(\vxx), \label{eq:optimization}\\
\text{subject to}  & \vxx \in \calX,
\end{eqnarray}
and refer to this optimization as problem~\eqref{eq:optimization}. For mathematical symbols, it is suggested that you define shortcuts to commonly used symbols and formats.

\subsection{Lists}
You may also make your answers more organized by using \emph{bulleted list}:
\begin{itemize}
	\item Observation 1 ...
	\item Observation 2 ...
\end{itemize}
and \emph{numbered list}:
\begin{enumerate}
	\item Observation 1 ...
	\item Observation 2 ...
\end{enumerate}

\subsection{Citations}
You can make a citation to a paper by~\cite{horn87josa-registration}.

\subsection{Figures}
You can also include a plot in Fig.~\ref{fig:simplefig}.

\input{fig-simplefig}


\section{Evaluation}
\label{sec:Evaluation}
This section contains the evaluation and the graphical representation of the measurements.

\subsection{Measuring the Magnetic Surface}
\label{subsec:Measuring_the_Magnetic_Surface}
The magnetic flux density was measured over a distance of about 1 m along the z-axis. The gaussmeter was zeroed at the position $z=0.6$ m. A measurement was taken every 8 cm.
\begin{figure}[H]
	\centering
	\includegraphics[scale=1]{Magnetic_Surface}
	\caption{Magnetic Surface}
	\label{fig:Magnetic_Surface}
\end{figure}
The red line in figure \ref{fig:Magnetic_Surface} shows the 4th order polynomial fit. A polynomial fit is able to represent the missing parts accurately, because the magnetic flux density can not jump. Every measuring point has the uncertainty of 0.3 \% (gaussmeter and Hall sensor) drawn in blue. Since the uncertainty is extremely small, it is almost impossible to see it. The measured values are generally very small and are therefore not used for compensation.

The resulting parameters of the 4th order polynomial fit are shown in the following table:
\begin{table}[H]
	\centering
	\renewcommand{\arraystretch}{1.3}
	\begin{tabular}{r|c|c|c|c|c}
		& \textbf{a}$_0$ & \textbf{a}$_1$ & \textbf{a}$_2$ & \textbf{a}$_3$ & \textbf{a}$_4$ \\
		\hline\hline
		\textbf{Value} & $2.031\cdot 10^{-5}$ & $-8.534\cdot 10^{-5}$ & $1.261\cdot 10^{-4}$ & $-7.747\cdot 10^{-5}$ & $1.702\cdot 10^{-5}$ \\
	\end{tabular}
	\caption{Polynomial Parameters}
	\label{tab:Polynomial_Parameters}
\end{table}
Using the polynomial parameters from the table \ref{tab:Polynomial_Parameters} above, the magnetic flux density of the surface $B_z$ at a certain position $z$ can be calculated with the following equation:
\[
B_z(z)=a_0+a_1z+a_2z^2+a_3z^3+a_4z^4
\]

\subsection{Measuring the Central Value $B_0(I)$}
\label{subsec:Measuring_the_Central_Value}
The magnetic flux density in the center of the cylindrical coil was measured in function of the current. The current ranges from 0 A to 1 A and the step size was chosen randomly.
\subsubsection{Short Cylindrical Coil}
\label{subsubsec:Short_Cylindrical_Coil}
The measured values are listed in appendix \ref{sec:Measurements}.
\begin{figure}[H]
	\centering
	\includegraphics[scale=1]{Central_Value_Short}
	\caption{Central Value Short Cylindrical Coil}
	\label{fig:Central_Value_Short}
\end{figure}
The red fitted curve was creating by utilizing equation \ref{eq:cylindrical_coil_b0}. All parameters except the vacuum permeability $\mu_0$ were set to their according values and marked as constants. As seen in figure \ref{fig:Central_Value_Short} above, the magnetic flux density $B_0$ is proportional to the applied current $I$. The absolute error (shown in blue) of 0.3 \% is smaller than the size of a single data point and thus hard to see. The parameters of the fitted curve are listed in the following table:
\begin{table}[H]
	\centering
	\renewcommand{\arraystretch}{1.3}
	\begin{tabular}{r|l}
		& \textbf{Value} \\
		\hline\hline
		\textbf{Vacuum Permeability} $\mu_0$ & $(1.25396\pm0.00019)\cdot10^{-6}$\ $\,^\text{Vs}\!/_\text{Am}$ \\
		\textbf{Windings} $N$ & 240 (constant) \\
		\textbf{Radius} $R$ & 0.0485 m (constant) \\
		\textbf{Length} $l$ & 0.1 m (constant) \\
	\end{tabular}
	\caption{Fit Parameters (Central Value Short Cylindrical Coil)}
	\label{tab:Central_Value_Short}
\end{table}
\subsubsection{Long Cylindrical Coil}
\label{subsubsec:Long_Cylindrical_Coil}
These measurements are also listed in appendix \ref{sec:Measurements}.
\begin{figure}[H]
	\centering
	\includegraphics[scale=1]{Central_Value_Long}
	\caption{Central Value Long Cylindrical Coil}
	\label{fig:Central_Value_Long}
\end{figure}
The red fitted curve was again creating by utilizing equation \ref{eq:cylindrical_coil_b0}. The plot with the fit was created the same way as in section \ref{subsubsec:Short_Cylindrical_Coil}. The absolute error (shown in blue) of 0.3 \% is again smaller than the size of a single data point. The parameters of the fitted curve are listed in the following table:
\begin{table}[H]
	\centering
	\renewcommand{\arraystretch}{1.3}
	\begin{tabular}{r|l}
		& \textbf{Value} \\
		\hline\hline
		\textbf{Vacuum Permeability} $\mu_0$ & $(1.24853\pm0.00016)\cdot10^{-6}$\ $\,^\text{Vs}\!/_\text{Am}$ \\
		\textbf{Windings} $N$ & 240 (constant) \\
		\textbf{Radius} $R$ & 0.049 m (constant) \\
		\textbf{Length} $l$ & 0.2 m (constant) \\
	\end{tabular}
	\caption{Fit Parameters (Central Value Long Cylindrical Coil)}
	\label{tab:Central_Value_Long}
\end{table}
\newpage
\subsection{Measuring the Field Pattern $B_z(z)$}
\label{subsec:Measuring_the_Field_Pattern}
To measure the field pattern, the current was set to about 1 A. Then the cylindrical coil was moved along the z-axis and measurements were taken in dependence of how fast the magnetic flux density changed.
\subsubsection{Short Cylindrical Coil}
\label{subsubsec:Short_Cylindrical_Coil_Field}
\begin{figure}[H]
	\centering
	\includegraphics[scale=1]{Field_Pattern_Short}
	\caption{Field Pattern Short Cylindrical Coil}
	\label{fig:Field_Pattern_Short}
\end{figure}
The fitted curve was created by using equation \ref{eq:cylindrical_coil_bz}. All parameter except the vacuum permeability $\mu_0$ and an offset $o$ were defined and set as constants. The offset $o$ is essential because the position of the Hall sensor is not exactly known. Furthermore, the ruler scale used to determine the $z$ values has an uncertainty of 1 mm. By using an offset, the fit is able to correct for those errors. The absolute error is shown in blue. The following parameter were obtained:
\begin{table}[H]
	\centering
	\renewcommand{\arraystretch}{1.3}
	\begin{tabular}{r|l}
		& \textbf{Value} \\
		\hline\hline
		\textbf{Vacuum Permeability} $\mu_0$ & $(1.25017\pm0.00063)\cdot10^{-6}$\ $\,^\text{Vs}\!/_\text{Am}$ \\
		\textbf{Offset} $o$ & $(0.26\pm0.04)$ mm \\
		\textbf{Windings} $N$ & 240 (constant) \\
		\textbf{Current} $I$ & 1.002 A (constant) \\
		\textbf{Radius} $R$ & 0.0485 m (constant) \\
		\textbf{Length} $l$ & 0.1 m (constant) \\
	\end{tabular}
	\caption{Fit Parameters (Field Pattern Short Cylindrical Coil)}
	\label{tab:Field_Pattern_Short}
\end{table}
\subsubsection{Long Cylindrical Coil}
\label{subsubsec:Long_Cylindrical_Coil_Field}
All measured values are listed in appendix \ref{sec:Measurements}.
\begin{figure}[H]
	\centering
	\includegraphics[scale=1]{Field_Pattern_Long}
	\caption{Field Pattern Long Cylindrical Coil}
	\label{fig:Field_Pattern_Long}
\end{figure}
The red fitted curve was again creating by utilizing equation \ref{eq:cylindrical_coil_bz}. The plot with the fit was created the same way as in section \ref{subsubsec:Short_Cylindrical_Coil_Field}. Since the uncertainty of the gaussmeter (combined with the Hall sensor) is so small, it is hard to see the error (shown in blue). The following parameters were obtained from the fit:
\begin{table}[H]
	\centering
	\renewcommand{\arraystretch}{1.3}
	\begin{tabular}{r|l}
		& \textbf{Value} \\
		\hline\hline
		\textbf{Vacuum Permeability} $\mu_0$ & $(1.24867\pm0.00036)\cdot10^{-6}$\ $\,^\text{Vs}\!/_\text{Am}$ \\
		\textbf{Offset} $o$ & $(0.97\pm0.03)$ mm \\
		\textbf{Windings} $N$ & 240 (constant) \\
		\textbf{Current} $I$ & 0.999 A (constant) \\
		\textbf{Radius} $R$ & 0.049 m (constant) \\
		\textbf{Length} $l$ & 0.2 m (constant) \\
	\end{tabular}
	\caption{Fit Parameters (Field Pattern Long Cylindrical Coil)}
	\label{tab:Field_Pattern_Long}
\end{table}


\section{Gaussian Processes}

% ===
%\textbf{Idea:} Introduce non-linearity in lin.-reg. setting.

% ===
\subsection{Bayesian Linear Regression}

\emph{Model:}\enskip
$\bm y = \bm X^\top \beta + \epsilon$,\enskip
with $\epsilon \sim \Gauss{\epsilon\mid 0, \sigma^2 \mathbb I}$

Likelihood:\enskip
$P(\bm y\mid \bm X, \beta,\sigma) = \Gauss{\bm y\mid \bm X^\top \beta, \sigma^2 \mathbb I}$

Prior:\enskip
$P(\beta\mid\bm\Lambda) = \Gauss[d]{\beta\mid 0, \bm\Lambda^{-1}}$\\\enskip
(Ridge regr. if $\bm\Lambda = \lambda\mathbb I$ and $\sigma=1$)

Posterior:\enskip
$P(\beta\mid \bm X, \bm y, \bm\Lambda) = \Gauss{\beta\mid \bm\mu_\beta, \bm\Sigma_\beta}$\\\enskip
with $\bm\mu_\beta = (\bm X^\top \bm X + \sigma^2 \bm\Lambda)^{-1} \bm X^\top \bm y$\\\enskip
and $\bm\Sigma_\beta = \sigma^2 (\bm X^\top \bm X + \sigma^2 \bm\Lambda)^{-1}$

$\implies$ $\bm y \sim \Gauss{\bm y \mid 0, \; \bm{X\Lambda}^{-1} \bm X^\top {+} \sigma^2 \mathbb I}$
\hfill {\small using $\E[\beta,\epsilon]{}$}
%With all these assumptions, you can say that $\bm y$ is Gaussian with $\mu_y = \E[\beta,\epsilon]{y} = 0$ and $\sigma_y = \Cov[\beta,\epsilon]{y} = \bm{X\Lambda}^{-1} \bm X^\top + \sigma^2 \mathbb I$.
\\\enskip
kernel $k(x_i, x_j) \coloneqq x_i^\top \bm\Lambda^{-1} x_j$

% ===
\subsection{Gaussian Process}

$\bm y \sim \Gauss{\bm y \mid m(\bm X), K(\bm X, \bm X) + \sigma^2 \mathbb I}$

\begin{minipage}{\linewidth}
    \centering
    $\begin{bsmallmatrix} \bm y \\ y_{n+1} \end{bsmallmatrix} \sim
    \Gauss*{
        \begin{bsmallmatrix} \bm y \\ y_{n+1} \end{bsmallmatrix} \mid
        \begin{bsmallmatrix} m(\bm X) \\ m(x_{n+1}) \end{bsmallmatrix},
        \begin{bsmallmatrix} \bm C_n & \bm k \\ \bm k^\top & c \end{bsmallmatrix}
    }$
\end{minipage}

$p(y_{n+1} \mid x_{n+1}, \bm X, \bm y) = \Gauss{y_{n+1} \mid \mu_{n+1}, \sigma_{n+1}^2}$\\\enskip
with $\mu_{n+1} = m(x_{n+1}) + \bm k^\top \bm C_n^{-1} (\bm y - m(\bm X))$\\\enskip
and $\sigma_{n+1}^2 = c - \bm k^\top \bm C_n^{-1} \bm k$

where
$\bm K = k(\bm X, \bm X)$,\enskip
$\bm k = k(x_{n+1}, \bm X)$,\\
\phantom{where}
$\bm C_n = \bm K + \sigma^2 \mathbb I$,\enskip
$c = k(x_{n+1}, x_{n+1}) + \sigma^2$

% ===
\subsection{Kernels \normalfont\sffamily
\quad scalar product
\quad $K_{ij} = k(\bm x_i, \bm x_j)$}

\emph{Valid kernel:}
must be
\textbf{symmetric}
    %($\bm K^\top\! {=} \bm K$)
and \textbf{p.s.d.}
    ($\bm x^\top \bm{Kx} \geq 0 \; \forall\bm x$ \textit{\,or\,}
    pos. eigenvalues \textit{\,or\,}
    pos. principal minors).\enskip
Must have a (pot. $\infty$-dim.) \textbf{feature vector} $\phi$ s.t. $k(\bm x, \bm x') = \phi(\bm x)^\top \phi(\bm x')$.

\iffalse
    Kernel $k(\bm x, \bm x')$ must be \emph{symmetric} ($\bm K^\top {=} \bm K$) and \emph{p.s.d.} ($\bm x^\top \bm{Kx} \geq 0 \; \forall\bm x$ \textit{\;or\;} pos. eigenvalues).

    A valid kernel $k$ must have a (potent. $\infty$-dim.) feature vector $\phi(\bm x)$ s.t. $k(\bm x, \bm x') = \phi^\top \! (\bm x) \phi (\bm x')$.
\fi

\emph{Common kernels:}\\
\begin{tabular}{@{}l @{:\hfill\enskip}l}
    Linear & $\bm x^\top \bm x'$ \\
    Polynomial & $(\bm x^\top \bm x' + 1)^p$,\enskip $\color{gray} p\in\mathbb N$ \\
    RBF (Gaussian) & $\exp(-\norm{\bm x - \bm x'}_2^2 / h^2)$ \\
    Sigmoid & $\tanh(\kappa \cdot \bm x^\top \bm x' - b)$
\end{tabular}

\emph{Kernel construction:}
\enskip\textbullet~$k_1 {+} k_2$
\enskip\textbullet~$c\cdot k_1, \; \color{gray} c>0$
\enskip\textbullet~$k_1\cdot k_2$
\enskip\textbullet~$f(\bm x) k_1(\bm x, \bm x') f(\bm x')$
\\\textbullet~$k(\phi(\bm x), \phi(\bm x'))$
    \enskip with $\phi : \mathcal X \to \mathbb R^d$
\\\textbullet~$g(k_1)$
    \enskip with $g :$ exp. \textit{ or } polyn. w/ \underline{all} pos. coeff.

% ===


\subsection{Maximum A-Priori estimation (MAP) for generative models}

\subsubsection{Bayesian regression for generative models}

We know:

\(P(\theta |y,X)=\dfrac{P(y, \theta, X )}{P(y, X)}\)

\(P(\theta |y,X)=\dfrac{P(y, X |\theta )P(\theta )}{P(y, X)}\)

The bottom bit is a normalisation factor, and so we can use:

\(P(\theta |y,X)\propto P(y, X| \theta)P(\theta)\)

We have here:

\begin{itemize}
\item Our prior - \(P(\theta )\)
\item Our posterior - \(P(\theta |y,X)\)
\item Our likelihood function - \(P(y, X| \theta )\)
\end{itemize}


%% State Space Modelling of Dynamic Systems
%% Lecture 18: General Solution of a State Space System
\def\FileDate{10/02/01}
\def\FileVersion{1.0}
% ----------------------------------------------------------------
% Notes pages *********************************************************
% ----------------------------------------------------------------

In this lecture we conclude our introduction to state space systems by developing a method that can be used to solve any linear time invariant (LTI) system using the state space model. This will put on a formal mathematical footing the approach described in Lecture 15. To do this we need to derive the Taylor series for the state matrix and then use the results from the previous two lectures to formally derive the state transmission matrix $\mathbf{\phi}(t)$ from the resolvant matrix $\mathbf{\Phi}(\lambda) = |\lambda\mathbf{I}-\mathbf{A}|$. This will lead is to a general solution which we will illustrate with examples and compare with the solution developed in the last lecture and the solution obtained by inverting the system transfer function. As before, we will refer to Matlab where relevant.

Once we have completed this material, we will be in a position to move on to look at state-space methods for control system design.

\section*{The Matrix Exponential Function}

In mathematics, the Taylor series is a representation of a function as an infinite sum of terms calculated from the values of its derivatives at a single point. It is named after the English mathematician Brook Taylor. It is common practice to use a finite number of terms of the series to approximate a function. The Taylor series may be regarded as the limit of the Taylor polynomials.\footnote{Definition from `Taylor Series', Wikipedia.}

We use Taylor series to approximate general functions, and here we adapt the Taylor series to discover the form of a function of a matrix.

\input{frag1}
\ifslidesonly
\begin{slide}
	\heading{Taylor series}
   \input{frag1}
\end{slide}
\fi

\input{frag2}
\ifslidesonly
\begin{slide}
	\heading{Using the Similarity Transform}
   \input{frag2}
\end{slide}
\fi

\input{frag3}
\ifslidesonly
\begin{slide}
	\heading{Matrix Function}
   \input{frag3}
\end{slide}
\fi

\input{frag4}
\ifslidesonly
\begin{slide}
	\heading{Transition Matrix}
   \input{frag4}
\end{slide}
\fi

\input{frag5}
\ifslidesonly
\begin{slide}
	\heading{Forced Solution}
   \input{frag5}
\end{slide}
\fi

\ifslidesonly
\begin{slide}
\heading{End of Pre-Class Presentation}
In the class we will work through a worked example of this approach
and contrast it with two alternatives -- diagonalization by similarity transform
and Laplace.
\end{slide}
\fi

\subsection*{Example 1}
\ifslidesonly
\begin{slide}
\heading{Example 1}
Find the solution of:
\input{example1}
{\bf Note}: The solutions to this example and the remaining alternatives
 are available in the form of Matlab scripts and as Pencasts on Blackboard.
\end{slide}
\fi
\textbf{Problem}: Find the solution of:
\input{example1}

\textbf{SOLUTION}: The state matrix $\mathbf{A}$ is the same as Example 4 from Lecture 17, so the transformation matrix is:
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaCivaiabg2da9iabg2da9maadmaaba
% qbamqabiGaaaqaaiaaigdaaeaacaaIXaaabaGaeyOeI0IaaGymaaqa
% aiabgkHiTiaaicdacaGGUaGaaGynaaaaaiaawUfacaGLDbaaaaa!3935!
\[
{\bf{T}} = \left[ {\begin{array}{*{20}c}
   1 & 1  \\
   { - 1} & { - 0.5}  \\
\end{array}} \right]
\]
with an inverse:
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaCivamaaCaaaleqabaGaeyOeI0IaaG
% ymaaaakiabg2da9iabg2da9maadmaabaqbamqabiGaaaqaaiabgkHi
% TiaaigdaaeaacqGHsislcaaIYaaabaGaaGOmaaqaaiaaikdaaaaaca
% GLBbGaayzxaaaaaa!39A7!
\[
{\bf{T}}^{ - 1}  = \left[ {\begin{array}{*{20}c}
   { - 1} & { - 2}  \\
   2 & 2  \\
\end{array}} \right]
\]
and with corresponding eigenvalues $\lambda_1=-1$ and $\lambda_2=-2$.

Therefore the state transition matrix is:
\begin{eqnarray*}
\mathbf{\phi}(t) & = & e^{\mathbf{A}t}=\mathbf{T}e^{\mathbf{\Lambda}t}\mathbf{T}^{-1} \\
	             & = & \left[ {\begin{array}{*{20}c}
	   1 & 1  \\
	   { - 1} & { - 0.5}  \\
	\end{array}} \right]\left[ {\begin{array}{*{20}c}
	   {e^{ - t} } & 0  \\
	   0 & {e^{ - 2t} }  \\
	\end{array}} \right]\left[ {\begin{array}{*{20}c}
	   { - 1} & { - 2}  \\
	   2 & 2  \\
	\end{array}} \right] \\
	             & = & \left[ {\begin{array}{*{20}c}
	   1 & 1  \\
	   { - 1} & { - 0.5}  \\
	\end{array}} \right]\left[ {\begin{array}{*{20}c}
	   {-e^{ - t} } & {-2e^{ - t} }  \\
	   {2e^{ - 2t} } & {2e^{ - 2t} }  \\
	\end{array}} \right] \\
	             & = & \left[ {\begin{array}{*{20}c}
	   \left\{ {-e^{ - t} + 2e^{-2t}} \right\} & \left\{ {-2e^{ - t} + 2e^{-2t}} \right\}  \\
	   \left\{ {e^{ - t} - e^{-2t}} \right\} & \left\{ {2e^{ - t} - e^{-2t}} \right\}  \\
	\end{array}} \right] \\
\end{eqnarray*}

The solution is:
\[
\mathbf{x}(t) = \mathrm{term}_1+\int_0^t\mathrm{term}_2 d\tau.
\]
where $\mathrm{term}_1=\mathbf{\phi}(t)\mathbf{x}_0=e^{\mathbf{A}t}\mathbf{x}_0$.
\begin{eqnarray*}
\mathrm{term}_1 & = & \left[ {\begin{array}{*{20}c}
	   \left\{ {-e^{ - t} + 2e^{-2t}} \right\} & \left\{ {-2e^{ - t} + 2e^{-2t}} \right\}  \\
	   \left\{ {e^{ - t} - e^{-2t}} \right\} & \left\{ {2e^{ - t} - e^{-2t}} \right\}  \\
	\end{array}} \right] \left[ {\begin{array}{*{20}c}
	   1  \\
	   1  \\
	\end{array}} \right] \\
	& = & \left[ {\begin{array}{*{20}c}
	   {\left\{ {\left. { - 3e^{ - t}  + 4e^{ - 2t} } \right\}} \right.}  \\
	   {\left\{ {\left. {3e^{ - t}  - 2e^{ - 2t} } \right\}} \right.}  \\
	\end{array}} \right]
\end{eqnarray*}
and
\begin{eqnarray*}
	{\rm{term}}_2  & = & e^{{\bf{A}}(t - \tau )} {\bf{B}}u = e^{{\bf{A}}(t - \tau )} \left[ {\begin{array}{*{20}c}
	   1  \\
	   0  \\
	\end{array}} \right] \times 1 \\
 & = & \left[ {\begin{array}{*{20}c}
   {\left. {\left\{ { - e^{ - (t - \tau )}  + 2e^{ - 2(t - \tau )} } \right.} \right\}}  \\
   {\left. {\left\{ {e^{ - (t - \tau )}  - e^{ - 2(t - \tau )} } \right.} \right\}}  \\
\end{array}} \right]
\end{eqnarray*}

\begin{eqnarray*}
	\int_0^t {{\rm{term}}_2 d\tau }  & = & \left[ {\begin{array}{*{20}c}
	   {\int_0^t {\left. {\left\{ { - e^{ - (t - \tau )}  + 2e^{ - 2(t - \tau )} } \right.} \right\}d\tau } }  \\
	   {\int_0^t {\left. {\left\{ {e^{ - (t - \tau )}  - e^{ - 2(t - \tau )} } \right.} \right\}d\tau } }  \\
	\end{array}} \right] \\
	& = &
	  \left[ {\begin{array}{*{20}c}
	   {\left. {\left\{ { - e^{ - (t - \tau )}  + e^{ - 2(t - \tau )} } \right.} \right\}}  \\
	   {\left. {\left\{ {e^{ - (t - \tau )}  - 0.5e^{ - 2(t - \tau )} } \right.} \right\}}  \\
	\end{array}} \right]_{\tau  = 0}^t  \\
	 & = & \left[ {\begin{array}{*{20}c}
	   { - 1 + e^{ - t}  + 1 - e^{ - 2t} }  \\
	   { 1 - e^{ - t}  - 0.5 + 0.5e^{ - 2t} }  \\
	\end{array}} \right] \\
	 &  = & \left[ {\begin{array}{*{20}c}
	   {e^{ - t}  - e^{ - 2t} }  \\
	   {0.5 - e^{ - t}  + 0.5e^{ - 2t} }  \\
	\end{array}} \right]
\end{eqnarray*}

Combining $\mathrm{term}_1$ and $\int_0^t \mathrm{term}_2 d\tau$ we find the total, forced, response:
\[
{\bf{x}}(t) = \left[ {\begin{array}{*{20}c}
   { - 3e^{ - t}  + 4e^{ - 2t} }  \\
   {3e^{ - t}  - 2e^{ - 2t} }  \\
\end{array}} \right] + \left[ {\begin{array}{*{20}c}
   {e^{ - t}  - e^{ - 2t} }  \\
   {0.5 - e^{ - t}  + 0.5e^{ - 2t} }  \\
\end{array}} \right]
\]

Therefore:
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaCiEaiaacIcacaWG0bGaaiykaiabg2
% da9maadmaabaqbamqabiqaaaqaaiabgkHiTiaaikdacaWGLbWaaWba
% aSqabeaacqGHsislcaWG0baaaOGaey4kaSIaaG4maiaadwgadaahaa
% WcbeqaaiabgkHiTiaaikdacaWG0baaaaGcbaGaaGimaiaac6cacaaI
% 1aGaey4kaSIaaGOmaiaadwgadaahaaWcbeqaaiabgkHiTiaadshaaa
% GccqGHsislcaaIXaGaaiOlaiaaiwdacaWGLbWaaWbaaSqabeaacqGH
% sislcaaIYaGaamiDaaaaaaaakiaawUfacaGLDbaaaaa!4C2A!
\[
{\bf{x}}(t) = \left[ {\begin{array}{*{20}c}
   { - 2e^{ - t}  + 3e^{ - 2t} }  \\
   {0.5 + 2e^{ - t}  - 1.5e^{ - 2t} }  \\
\end{array}} \right]
\]

\subsection*{Alternative solution 1}
\ifslidesonly
\begin{slide}
\heading{Alternative solution 1}
Repeat Example 1 using the transformation to normal canonical form:
\[
\frac{d\mathbf{w}}{t}=\mathbf{\Lambda w}+ \mathbf{T}^{-1}\mathbf{B}u
\]
\end{slide}
\fi
Using the transformation to normal canonical form:
\[
\frac{d\mathbf{w}}{t}=\mathbf{\Lambda w}+ \mathbf{T}^{-1}\mathbf{B}u
\]
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaWaaSaaaeaacaWGKbGaaC4Daaqaaiaads
% gacaWG0baaaiabg2da9maadmaabaqbamqabiGaaaqaaiabgkHiTiaa
% igdaaeaacaaIWaaabaGaaGimaaqaaiabgkHiTiaaikdaaaaacaGLBb
% GaayzxaaGaaC4DaiabgUcaRmaadmaabaqbamqabiGaaaqaaiabgkHi
% TiaaigdaaeaacqGHsislcaaIYaaabaGaaGOmaaqaaiaaikdaaaaaca
% GLBbGaayzxaaWaamWaaeaafaWabeGabaaabaGaaGymaaqaaiaaicda
% aaaacaGLBbGaayzxaaGaamyDaiaacUdacaaMf8UaamyDaiabg2da9i
% aaigdaaaa!4BE3!
\[
\frac{{d{\bf{w}}}}{{dt}} = \left[ {\begin{array}{*{20}c}
   { - 1} & 0  \\
   0 & { - 2}  \\
\end{array}} \right]{\bf{w}} + \left[ {\begin{array}{*{20}c}
   { - 1} & { - 2}  \\
   2 & 2  \\
\end{array}} \right]\left[ {\begin{array}{*{20}c}
   1  \\
   0  \\
\end{array}} \right]u;\quad u = 1
\]
\[
\frac{dw_1}{dt}=-w_1-1\ \textrm{and}\ \frac{dw_2}{dt}=-2w_2 + 2
\]
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaC4DamaaBaaaleaacaaIWaaabeaaki
% abg2da9iaahsfadaahaaWcbeqaaiabgkHiTiaaigdaaaGccaWH4bWa
% aSbaaSqaaiaaicdaaeqaaOGaeyypa0ZaamWaaeaafaWabeGacaaaba
% GaeyOeI0IaaGymaaqaaiabgkHiTiaaikdaaeaacaaIYaaabaGaaGOm
% aaaaaiaawUfacaGLDbaadaWadaqaauaadeqaceaaaeaacaaIXaaaba
% GaaGymaaaaaiaawUfacaGLDbaacqGH9aqpdaWadaqaauaadeqaceaa
% aeaacqGHsislcaaIZaaabaGaaGinaaaaaiaawUfacaGLDbaaaaa!466E!
\[
{\bf{w}}_0  = {\bf{T}}^{ - 1} {\bf{x}}_0  = \left[ {\begin{array}{*{20}c}
   { - 1} & { - 2}  \\
   2 & 2  \\
\end{array}} \right]\left[ {\begin{array}{*{20}c}
   1  \\
   1  \\
\end{array}} \right] = \left[ {\begin{array}{*{20}c}
   { - 3}  \\
   4  \\
\end{array}} \right]
\]

\begin{eqnarray*}
	w_1 & = & w_{10}e^{-t}-\int_0^t e^{-(t-\tau)} d\tau \\
	    & = & -3e^{-t}-\left[e^{-(t-\tau)}\right]_0^t \\
	    & = & -3e^{-t}-\left[1 - e^{-t}\right] \\
	    & = & -1 -2e^{-t}
\end{eqnarray*}

\begin{eqnarray*}
	w_2 & = & w_{20}e^{-2t}+2\int_0^t e^{-2(t-\tau)} d\tau \\
	    & = & 4e^{-2t}+2\left[0.5e^{-2(t-\tau)}\right]_0^t \\
	    & = & 4e^{-2t}+\left[1 - e^{-2t}\right] \\
	    & = & 1 + 3e^{-2t}
\end{eqnarray*}

% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaCiEaiabg2da9iaahsfacaWH3bGaey
% ypa0ZaamWaaeaafaWabeGacaaabaGaaGymaaqaaiaaigdaaeaacqGH
% sislcaaIXaaabaGaeyOeI0IaaGimaiaac6cacaaI1aaaaaGaay5wai
% aaw2faamaadmaabaqbamqabiqaaaqaaiabgkHiTiaaigdacqGHsisl
% caaIYaGaamyzamaaCaaaleqabaGaeyOeI0IaamiDaaaaaOqaaiaaig
% dacqGHRaWkcaaIZaGaamyzamaaCaaaleqabaGaeyOeI0IaaGOmaiaa
% dshaaaaaaaGccaGLBbGaayzxaaaaaa!49AC!
\[
{\bf{x}} = {\bf{Tw}} = \left[ {\begin{array}{*{20}c}
   1 & 1  \\
   { - 1} & { - 0.5}  \\
\end{array}} \right]\left[ {\begin{array}{*{20}c}
   { - 1 - 2e^{ - t} }  \\
   {1 + 3e^{ - 2t} }  \\
\end{array}} \right]
\]
Therefore
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaCiEaiabg2da9maadmaabaqbamqabi
% qaaaqaaiabgkHiTiaaikdacaWGLbWaaWbaaSqabeaacqGHsislcaWG
% 0baaaOGaey4kaSIaaG4maiaadwgadaahaaWcbeqaaiabgkHiTiaaik
% dacaWG0baaaaGcbaGaaGimaiaac6cacaaI1aGaey4kaSIaaGOmaiaa
% dwgadaahaaWcbeqaaiabgkHiTiaadshaaaGccqGHsislcaaIXaGaai
% OlaiaaiwdacaWGLbWaaWbaaSqabeaacqGHsislcaaIYaGaamiDaaaa
% aaaakiaawUfacaGLDbaaaaa!49D8!
\[
{\bf{x}} = \left[ {\begin{array}{*{20}c}
   { - 2e^{ - t}  + 3e^{ - 2t} }  \\
   {0.5 + 2e^{ - t}  - 1.5e^{ - 2t} }  \\
\end{array}} \right]
\]

\subsection*{Alternative solution 2}
 \ifslidesonly
 \begin{slide}
 \heading{Alternative solution 2}
 Solve the problem of example 1 using the Laplace transform method.
 \end{slide}
 \fi
Using the Laplace transform method.

Taking Laplace transforms of the state equations, taking account of the initial conditions:
\[
s\mathbf{X}(s)-\mathbf{x}_0=\mathbf{A}\mathbf{X}(s)+\mathbf{B}U(s)
\]
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaiikaiaadohacaWHjbGaeyOeI0IaaC
% yqaiaacMcacaWHybGaaiikaiaadohacaGGPaGaeyypa0JaaCiEamaa
% BaaaleaacaaIWaaabeaakiabgUcaRiaahkeacaWGvbGaaiikaiaado
% hacaGGPaGaeyypa0ZaamWaaeaafaWabeGabaaabaGaaGymaaqaaiaa
% igdaaaaacaGLBbGaayzxaaGaey4kaSYaamWaaeaafaWabeGabaaaba
% GaaGymaaqaaiaaicdaaaaacaGLBbGaayzxaaWaaSaaaeaacaaIXaaa
% baGaam4Caaaaaaa!4885!
\[
(s{\bf{I}} - {\bf{A}}){\bf{X}}(s) = {\bf{x}}_0  + {\bf{B}}U(s) = \left[ {\begin{array}{*{20}c}
   1  \\
   1  \\
\end{array}} \right] + \left[ {\begin{array}{*{20}c}
   1  \\
   0  \\
\end{array}} \right]\frac{1}{s}
\]
Therefore
% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaCiwaiaacIcacaWGZbGaaiykaiabg2
% da9iaacIcacaWGZbGaaCysaiabgkHiTiaahgeacaGGPaWaaWbaaSqa
% beaacqGHsislcaaIXaaaaOWaamWaaeaafaWabeGabaaabaGaaGymai
% abgUcaRmaalaaabaGaaGymaaqaaiaadohaaaaabaGaaGymaaaaaiaa
% wUfacaGLDbaaaaa!3F1F!
\[
{\bf{X}}(s) = (s{\bf{I}} - {\bf{A}})^{ - 1} \left[ {\begin{array}{*{20}c}
   {1 + \frac{1}{s}}  \\
   1  \\
\end{array}} \right]
\]

% MathType!MTEF!2!1!+-
% faaagaart1ev2aaaKnaaaaWenf2ys9wBH5garuavP1wzZbqedmvETj
% 2BSbqefm0B1jxALjharqqtubsr4rNCHbGeaGqiVu0Je9sqqrpepC0x
% bbL8FesqqrFfpeea0xe9Lq-Jc9vqaqpepm0xbba9pwe9Q8fs0-yqaq
% pepae9pg0FirpepeKkFr0xfr-xfr-xb9Gqpi0dc9adbaqaaeGaciGa
% aiaabeqaamaabaabaaGcbaGaaeOtaiaab+gacaqG3bGaaeiiaiaacI
% cacaWGZbGaaCysaiabgkHiTiaahgeacaGGPaWaaWbaaSqabeaacqGH
% sislcaaIXaaaaOGaeyypa0ZaamWaaeaafaWabeGacaaabaGaam4Cai
% abgUcaRiaaiodaaeaacaaIYaaabaGaeyOeI0IaaGymaaqaaiaadoha
% aaaacaGLBbGaayzxaaWaaWbaaSqabeaacqGHsislcaaIXaaaaOGaey
% ypa0ZaaSaaaeaacaaIXaaabaGaaiikaiaadohacqGHRaWkcaaIZaGa
% aiykaiaadohacqGHRaWkcaaIYaaaamaadmaabaqbamqabiGaaaqaai
% aadohaaeaacqGHsislcaaIYaaabaGaaGymaaqaaiaadohacqGHRaWk
% caaIZaaaaaGaay5waiaaw2faaaaa!5355!
\[
{\rm{Now }}(s{\bf{I}} - {\bf{A}})^{ - 1}  = \left[ {\begin{array}{*{20}c}
   {s + 3} & 2  \\
   { - 1} & s  \\
\end{array}} \right]^{ - 1}  = \frac{1}{{(s + 3)s + 2}}\left[ {\begin{array}{*{20}c}
   s & { - 2}  \\
   1 & {s + 3}  \\
\end{array}} \right]
\]
Therefore
\begin{eqnarray*}
	{\bf{X}}(s) & = & \frac{1}{{s^2  + 3s + 2}}\left[ {\begin{array}{*{20}c}
	   s & { - 2}  \\
	   1 & {s + 3}  \\
	\end{array}} \right]\left[ {\begin{array}{*{20}c}
	   {1 + \frac{1}{s}}  \\
	   1  \\
	\end{array}} \right] \\
	            & = & \frac{1}{{(s + 1)(s + 2)}}\left[ {\begin{array}{*{20}c}
	   {s - 1}  \\
	   {\frac{{s^2  + 4s + 1}}{s}}  \\
	\end{array}} \right] \\
	&  = & \left[ {\begin{array}{*{20}c}
	   {\frac{{s - 1}}{{(s + 1)(s + 2)}}}  \\
	   {\frac{{s^2  + 4s + 1}}{{s(s + 1)(s + 2)}}}  \\
	\end{array}} \right]
\end{eqnarray*}

Taking partial fractions:
\[
{\bf{X}}(s) = \left[ {\begin{array}{*{20}c}
   {\frac{{ - 2}}{{s + 1}} + \frac{3}{{s + 2}}}  \\
   {\frac{{0.5}}{s} + \frac{2}{{s + 1}} + \frac{{ - 1.5}}{{s + 2}}}  \\
\end{array}} \right]
\]

Finally, taking inverse Laplace transforms:
\[
{\bf{x}}(t) = \left[ {\begin{array}{*{20}c}
   { - 2e^{ - t}  + 3e^{ - 2t} }  \\
   {0.5 + 2e^{ - t}  - 1.5e^{ - 2t} }  \\
\end{array}} \right]
\]

%----------------------------------------------------------------
% The end of notes
% ----------------------------------------------------------------
\endinput

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\section{The kernel and image of a linear map}

\begin{outcome}
  \begin{enumerate}
  \item Describe the kernel and image of a linear transformation.
  \item Use the kernel and image to determine if a linear
    transformation is one to one or onto.
  \end{enumerate}
\end{outcome}

Here we consider the case where the linear map is not necessarily an
isomorphism. First here is a definition of what is meant by the image and
kernel of a linear transformation.

\begin{definition}{Kernel and image}{}
Let $V$ and $W$ be vector spaces and let $T:V\rightarrow W$ be a linear transformation. Then the image of $T$
denoted as $\func{im}(T) $ is defined to be the set
\begin{equation*}
\set{T(\vect{v}):\vect{v}\in V}
\end{equation*}
In words, it consists of all vectors in $W$ which equal $T(\vect{v})$ for some $
\vect{v}\in V$. The kernel, $\ker (T)$,
consists of all $\vect{v}\in V$ such that $T(\vect{v})=\vect{0}$. That is,
\begin{equation*}
\ker (T) =\set{\vect{v}\in V:T(\vect{v})=\vect{0}}
\end{equation*}
\end{definition}

Then in fact, both $\func{im}(T) $ and $\ker (T) $
are subspaces of $W$ and $V$ respectively.

\begin{proposition}{Kernel and image as subspaces}{kernel-image-vector-spaces}
Let $V,W$ be vector spaces and let $T:V\rightarrow W$ be a linear transformation. Then $\ker (
T) \subseteq V$ and $\func{im}(T) \subseteq W$. In fact, they are both subspaces.
\end{proposition}

\begin{proof}
First consider $\ker (T)$. It is necessary to
show that if $\vect{v}_{1},\vect{v}_{2}$ are vectors in $\ker (T) $
and if $a,b$ are scalars, then $a\vect{v}_{1}+b\vect{v}_{2}$ is also in $\ker
(T)$. But
\begin{equation*}
T(a\vect{v}_{1}+b\vect{v}_{2}) =aT(\vect{v}_{1})+bT(\vect{v}_{2})=a\vect{0}+b\vect{0}=\vect{0}
\end{equation*}
Thus $\ker (T) $ is a subspace of $V$.

Next suppose $T(\vect{v}_{1}),T(\vect{v}_{2})$ are two vectors in $\func{im}(
T)$. Then if $a,b$ are scalars,
\begin{equation*}
aT(\vect{v}_{2})+bT(\vect{v}_{2})=T(a\vect{v}_{1}+b\vect{v}_{2})
\end{equation*}
and this last vector is in $\func{im}(T) $ by definition.
\end{proof}

Consider the following example.

\begin{example}{Kernel and image of a transformation}{kernel-image}
Let $T:\Poly_1\to\R$ be the linear transformation defined by
\[ T(p(x))=p(1)\mbox{ for all } p(x)\in \Poly_1.\]
Find the kernel and image of $T$.
\end{example}

\begin{solution}
We will first find the kernel of $T$. It consists of all polynomials in $\Poly_1$ that have $1$ for a root.
\begin{eqnarray*}
\func{ker}(T) & = & \set{p(x)\in \Poly_1 \mid p(1)=0} \\
& = & \set{ax+b \mid a,b\in\R \mbox{ and }a+b=0} \\
& = & \set{ax-a \mid a\in\R}
\end{eqnarray*}
Therefore a basis for $\func{ker}(T)$ is
\[
\set{x-1 }
\]
Notice that this is a subspace of $\Poly_1$.

Now consider the image. It consists of all numbers which can be obtained by evaluating all polynomials in $\Poly_1$ at $1$.
\begin{eqnarray*}
\func{im}(T) & = & \set{p(1) \mid p(x)\in \Poly_1} \\
 & = & \set{a+b \mid ax+b\in \Poly_1} \\
 & = & \set{a+b \mid a,b\in\R}\\
 & = & \R
\end{eqnarray*}
Therefore a basis for $\func{im}(T)$ is
\[
\set{1 }
\]
Notice that this is a subspace of $\R$, and in fact is the space $\R$ itself.
\end{solution}

\begin{example}{Kernel and image of a linear transformation}{finding-kernel-image}
Let $T: \Mat_{2,2} \to \R^2$ be defined by
\[
T \begin{mymatrix}{cc}
a & b \\
c & d
\end{mymatrix}
 =
\begin{mymatrix}{c}
a - b \\
c + d
\end{mymatrix}
\]
Then $T$ is a linear transformation. Find a basis for $\func{ker} (T)$ and $\func{im}(T)$.
\end{example}

\begin{solution}
You can verify that $T$ represents a linear transformation.

Now we want to find a way to describe all matrices $A$ such that $T(A) = \vect{0}$, that is the matrices in $\func{ker}(T)$.
Suppose $A = \begin{mymatrix}{cc}
a & b \\
c & d
\end{mymatrix}$ is such a matrix.
Then
\[
T \begin{mymatrix}{cc}
a & b \\
c & d
\end{mymatrix}
 =
\begin{mymatrix}{c}
a - b \\
c + d
\end{mymatrix}
 =
\begin{mymatrix}{c}
0 \\
0
\end{mymatrix}
\]
The values of $a, b, c, d$ that make this true are given by solutions to the system
\begin{eqnarray*}
a - b &=& 0 \\
c + d &=& 0
\end{eqnarray*}
The solution is $a = s, b = s, c = t, d = -t$ where $s, t$ are scalars. We can describe $\func{ker}(T)$ as follows.
\[
\func{ker}(T) =
\set{
\begin{mymatrix}{cc}
s & s \\
t & -t
\end{mymatrix}
}
=
\sspan
\set{
\begin{mymatrix}{cc}
1 & 1 \\
0 & 0
\end{mymatrix},
\begin{mymatrix}{cc}
0 & 0 \\
1 & -1
\end{mymatrix}
}
\]
It is clear that this set is linearly independent and therefore forms a basis for $\func{ker}(T)$.

We now wish to find a basis for $\func{im}(T)$. We can write the image of $T$ as
\[
\func{im}(T) = \set{
\begin{mymatrix}{c}
a - b  \\
c + d
\end{mymatrix}
}
\]
Notice that this can be written as
\[
\sspan
\set{
\begin{mymatrix}{c}
1 \\
0
\end{mymatrix},
\begin{mymatrix}{c}
-1 \\
0
\end{mymatrix},
\begin{mymatrix}{c}
0 \\
1
\end{mymatrix},
\begin{mymatrix}{c}
0 \\
1
\end{mymatrix} }
\]

However this is clearly not linearly independent. By removing vectors from the set to create an independent set gives a basis of $\func{im}(T)$.
\[
\set{
\begin{mymatrix}{c}
1 \\
0
\end{mymatrix},
\begin{mymatrix}{c}
0 \\
1
\end{mymatrix}
}
\]

Notice that these vectors have the same span as the set above but are now linearly independent.
\end{solution}

A major result is the relation between the dimension of the kernel and
dimension of the image of a linear transformation. A special case was done
earlier in the context of matrices. Recall that for an $m\times n$-matrix $%
A, $ it was the case that the dimension of the kernel of $A$ added to the
rank of $A$ equals $n$.

\begin{theorem}{Dimension of kernel and image}{}
Let $T:V\rightarrow W$ be a linear transformation where $V,W$ are vector
spaces. Suppose the dimension of $V$ is $n$.
Then $n=\dim (\ker (T)) +\dim (\func{im}
(T))$.
\end{theorem}

\begin{proof}
From Proposition~\ref{prop:kernel-image-vector-spaces}, $\func{im}(T) $
is a subspace of $W$. By Theorem~\ref{thm:basis-vector-space}, there exists a basis for $
\func{im}(T) ,\set{T(\vect{v}_{1}),\ldots,T(\vect{v}_{r})}
. $ Similarly, there is a basis for $\ker (T) ,\set{\vect{u}
_{1},\ldots,\vect{u}_{s}}$. Then if $\vect{v}\in V$, there exist
scalars $c_{i}$ such that
\begin{equation*}
T(\vect{v})=\sum_{i=1}^{r}c_{i}T(\vect{v}_{i})
\end{equation*}
Hence $T\paren{\vect{v}-\sum_{i=1}^{r}c_{i}\vect{v}_{i}} =0$. It follows
that $\vect{v}-\sum_{i=1}^{r}c_{i}\vect{v}_{i}$ is in $\ker (T)$.
Hence there are scalars $a_{i}$ such that
\begin{equation*}
\vect{v}-\sum_{i=1}^{r}c_{i}\vect{v}_{i}=\sum_{j=1}^{s}a_{j}\vect{u}_{j}
\end{equation*}
Hence $\vect{v}=\sum_{i=1}^{r}c_{i}\vect{v}_{i}+\sum_{j=1}^{s}a_{j}\vect{u}
_{j}. $ Since $\vect{v}$ is arbitrary, it follows that
\begin{equation*}
V=\sspan\set{\vect{u}_{1},\ldots,\vect{u}_{s},\vect{v}_{1},\ldots,
\vect{v}_{r}}
\end{equation*}
If the vectors $\set{\vect{u}_{1},\ldots,\vect{u}_{s},\vect{v}_{1},\ldots,
\vect{v}_{r}} $ are linearly independent, then it will follow that
this set is a basis. Suppose then that
\begin{equation*}
\sum_{i=1}^{r}c_{i}\vect{v}_{i}+\sum_{j=1}^{s}a_{j}\vect{u}_{j}=0
\end{equation*}
Apply $T$ to both sides to obtain
\begin{equation*}
\sum_{i=1}^{r}c_{i}T(\vect{v}_{i})+\sum_{j=1}^{s}a_{j}T(\vect{u}
_{j})=\sum_{i=1}^{r}c_{i}T(\vect{v}_{i})= \vect{0}
\end{equation*}
Since $\set{T(\vect{v}_{1}),\ldots,T(\vect{v}_{r})} $ is linearly
independent, it follows that each $c_{i}=0$. Hence $\sum_{j=1}^{s}a_{j}\vect{u
}_{j}=0$ and so, since the $\set{\vect{u}_{1},\ldots,\vect{u}_{s}} $
are linearly independent, it follows that each $a_{j}=0$ also. It follows
that $\set{\vect{u}_{1},\ldots,\vect{u}_{s},\vect{v}_{1},\ldots,\vect{v}
_{r}} $ is a basis for $V$ and so
\begin{equation*}
n=s+r=\dim (\ker (T)) +\dim (\func{im}(
T))
\end{equation*}
\end{proof}

Consider the following definition.

\begin{definition}{Rank of linear transformation}{}
Let $T:V\rightarrow W$ be a linear transformation and suppose $V,W$ are finite-dimensional vector spaces. Then
the rank of $T$ denoted as $\rank(T) $ is defined as the
dimension of $\func{im}(T)$. The nullity of $T$ is the
dimension of $\ker (T)$. Thus the above theorem says that $
\rank(T) +\dim (\ker (T)) =\dim
(V)$.
\end{definition}

Recall the following important result.

\begin{theorem}{Subspace of same dimension}{subspace-vector-space}
Let $V$ be a vector space of dimension $n$ and let $W$ be a
subspace. Then $W=V$ if and only if the dimension of $W$ is also $n$.
\end{theorem}

From this theorem follows the next corollary.

\begin{corollary}{One to one and onto characterization}{one-one-onto-char}
Let $T:V\rightarrow W$ be a linear map where the dimension of $V$ is $n$ and
the dimension of $W$ is $m$. Then $T$ is one to one if and only if $\ker
(T) =\set{\vect{0}} $ and $T$ is onto if and only if $
\rank(T) =m$.
\end{corollary}

\begin{proof}
The statement $\ker (T) =\set{\vect{0}} $
is equivalent to saying if $T (\vect{v})=\vect{0}$, it follows that $\vect{v}=\vect{0}$
. Thus by Lemma~\ref{lem:one-to-one-abstract} $T$ is one to one. If $T$ is onto, then $
\func{im}(T) =W$ and so $\rank(T) $ which is
defined as the dimension of $\func{im}(T) $ is $m$. If $\rank(T) =m$, then by Theorem~\ref{thm:subspace-vector-space}, since $\func{im}
(T) $ is a subspace of $W$, it follows that $\func{im}(
T) =W$.
\end{proof}

\begin{example}{One to one transformation}{one-to-one-kernel}
Let $S:\Poly_2\to\Mat_{2,2}$ be a linear transformation
defined by
\[ S(ax^2+bx+c)
=
\begin{mymatrix}{cc}
a+b & a+c \\ b-c & b+c \end{mymatrix}
\mbox{ for all }
 ax^2+bx+c\in \Poly_2.\]
Prove that $S$ is one to one but not onto.
\end{example}

\begin{solution}
You may recall this example from earlier in Example~\ref{exa:one-to-one-general}. Here we will determine that $S$ is one to one, but not onto, using the method provided in Corollary~\ref{cor:one-one-onto-char}.

By definition,
\[ \ker(S)=\set{ax^2+bx+c\in \Poly_2 \mid a+b=0, a+c=0, b-c=0, b+c=0}.\]

Suppose $p(x)=ax^2+bx+c\in\ker(S)$.
This leads to a homogeneous system of four equations in three
variables.
Putting the augmented matrix in {\rref}:

\[ \begin{mymatrix}{rrr|c}
1 & 1 & 0 & 0  \\
1 & 0 & 1 & 0  \\
0 & 1 & -1 & 0  \\
0 & 1 & 1 & 0  \end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{ccc|c}
1 & 0 & 0 & 0  \\
0 & 1 & 0 & 0  \\
0 & 0 & 1 & 0  \\
0 & 0 & 0 & 0  \end{mymatrix}. \]

Since the unique solution is $a=b=c=0$, $\ker(S)=\set{\vect{0}}$, and thus
$S$ is one-to-one by Corollary~\ref{cor:one-one-onto-char}.

Similarly, by Corollary~\ref{cor:one-one-onto-char}, if $S$ is onto it will have $\rank(S) = \dim(\Mat_{2,2}) = 4$. The image of $S$ is given by
\[
\func{im}(S) = \set{\begin{mymatrix}{cc}
a+b & a+c \\ b-c & b+c \end{mymatrix} } = \sspan \set{\begin{mymatrix}{rr}
1 & 1 \\
0 & 0 \end{mymatrix}, \begin{mymatrix}{rr}
1 & 0 \\
1 & 1 \end{mymatrix}, \begin{mymatrix}{rr}
0 & 1 \\
-1 & 1 \end{mymatrix} }
\]
These matrices are linearly independent which means this set forms a basis for $\func{im}(S)$. Therefore the dimension of $\func{im}(S)$, also called $\rank(S)$, is equal to $3$. It follows that $S$ is not onto.
\end{solution}