%!TEX root = Calculus_I.tex
\chapter{The Derivative}
\section{How do We Measure Speed?}
\textbf{\textit{Speed}} is the magnitude of \textbf{\textit{velocity}}, and is a scalar quantity. \textbf{\textit{Velocity}} is a vector quantity, meaning it has magnitude \textit{and} direction. Thus, velocity can be negative if an object is moving opposite the positive direction. If $s(t)$ is the position of an object at time $t$, then the \textbf{\textit{average velocity}} of the object over the interval $t \in \left[a, b\right]$ is:
\begin{equation}
\label{eq:DQ}
V_{avg} = \frac{\Delta s}{\Delta t} = \frac{s(b) - s(a)}{b - a}
\end{equation}

This representation does not help to understand the velocity of an object at a given instant in time, only over the interval $[a, b]$. For this problem of \textbf{\textit{instantaneous velocity}}, we need to look closer at the specified time instant, $t$.

\vspace{0.1in}
One way to converge to the instantaneous velocity at time $t$ is to use the definition of the \textbf{\textit{limit}}. As the interval $[a,b]$ gets sufficiently small and encloses the time $t$, a two-sided limit is being taken to determine the \textbf{\textit{instantaneous velocity}}; therefore, if $s(t)$ is the position at time $t$, the \textbf{\textit{instantaneous velocity}} at time $t = a$ can be defined as:
\begin{equation}
\lim_{h\rightarrow 0} \frac{s(a+h) - s(a)}{h}
\end{equation}
This approximation is valid as $h\rightarrow 0$ because most functions, $f(x)$, appear linear for small changes between $x$ values. This approximation yields the slope of the curve $f(x)$ at a point, $x = a$. Using this definition, we can redefine the \textbf{\textit{average velocity}} over any time interval $t \in [a, b]$ as:

\vspace{0.1in}
\textbf{\textit{Average Velocity}} -- the slope of the line joining the points on the graph of $s(t)$ corresponding to $t = a$ and $t = b$.

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{The Derivative at a Point}
In this section, the \textbf{\textit{difference quotient}} shown in Equation (\ref{eq:DQ}) is applied to functions that are not necessarily functions of time. We are still interested in intervals of length $[a, a+h]$ where $h$ is sufficiently small. For any function $f$, the \textbf{\textit{difference quotient}} is given by $\frac{\Delta f}{\Delta x}$.

\vspace{0.1in}
The numerator of the \textbf{\textit{difference quotient}} is the \textbf{\textit{absolute change}} of the function, whereas the full \textbf{\textit{difference quotient}} is the average rate of change. When the interval $[a, a+h]$ is sufficiently small, that is $h \rightarrow 0$, we arrive at the \textbf{\textit{instantaneous rate of change}}. This is referred to as the \textit{derivative of function f at point a}, and is denoted by $f^\prime(a)$.

\begin{equation}
\label{eq:derivative}
f^\prime(a) = \lim_{h\rightarrow 0} \frac{f(a+h) - f(a)}{h}
\end{equation}

If the \textbf{\textit{limit}} in Equation (\ref{eq:derivative}) exists, then $f$ is said to be \textbf{differentiable at a}. The \textbf{\textit{derivative}} at point $a$ can be interpreted as the slope of the curve $f(x)$ at point $a$, or the slope of the tangent line to the curve at point $a$ if $h \ll 1$.

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{The Derivative Function}
The previous section focused on the \textbf{\textit{derivative}} at a fixed point, $a$. This section will cover how the \textbf{\textit{derivative}} changes at different points because it is also a function of the independent variable. Using the definition of the \textbf{\textit{derivative}} at a point, we can arrive at the following extention to functions:
\begin{equation}
\label{eq:deriv_function}
f^\prime(x) = \lim_{h \rightarrow 0} \frac{f(x+h) - f(x)}{h}
\end{equation}
Thus, for every $x$ value that the limit in Equation (\ref{eq:deriv_function}) exists, we can state that the function $f$ is \textit{differentiable at} that $x$ value. If there are no points that $f$ is \textbf{\textit{undifferentiable}}, then $f$ is \textit{differentiable everywhere}, which is often the case.

\vspace{0.1in}
The derivative tells us how the function $f(x)$ is changing with $x$. If $f^\prime > 0$ over an interval $[a, b]$, then $f$ is \textbf{\textit{increasing}} over that interval. Conversely,  If $f^\prime < 0$ over an interval $[a, b]$, then $f$ is \textbf{\textit{decreasing}} over that interval. The magnitude of $f^\prime$ indicates how much the function increases ($|f^\prime| \gg 0 \rightarrow f$ is changing rapidly; ($|f^\prime| \ll 1 \rightarrow f$ is changing slowly).

\vspace{0.1in}
For constant functions, $f(x) = k$, the \textbf{\textit{derivative}} is equal to zero because the function values do not change with changes in $x$. For linear functions, $f(x) = mx + b$, the \textbf{\textit{derivative}} is $m$ because it is the slope of the tangent line to $f(x)$, which is linear. For $n-degree$ polynomials, $n > 1$, the derivative can be estimated numerically by function evaluations using Equation (\ref{eq:deriv_function}).

\vspace{0.1in}
For power functions, $f(x) = x^n$, the \textbf{\textit{Binomial Theorem}} can be used to show the \textbf{\textit{Power Rule of Differentiation}}, which is valid for $n \in \mathbb{R}$:
\begin{equation}
\text{If} \hphantom{-} f(x) = x^n, \hphantom{-} \text{then} \hphantom{-} f^\prime(x) = n x^{n-1}
\end{equation}

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{Interpretations of the Derivative}
Another commonly used notation for the \textbf{\textit{derivative}} is \textbf{\textit{Leibniz's Notation}}:
\begin{equation}
f^\prime(x) = \frac{dy}{dx}
\end{equation}
Here, $d$ suggests ``\textit{small difference in}." An alternative form of \textbf{\textit{Leibniz's Notation}}:
\begin{equation}
\frac{dy}{dx} = \frac{d}{dx}(y)
\end{equation}
which means ``\textit{the derivative of y with respect to x}". The terms $dy$ and $dx$ in this notation are often used a seperate entities in mathematics, representing infinitesimally small changes in $y$ and $x$, respectively. Relating this back to previous examples, velocity can be writen as:
\begin{equation}
v = \frac{ds}{dt}
\end{equation}

\vspace{0.1in}
To specify the derivative at a point, $c$, we can write:
\begin{equation}
\at{\frac{dy}{dx}}{x=c} % Manually defined command in preamble of Calculus_I.tex
\end{equation}
The units of the \textbf{\textit{derivative}} of a function depend on the units of the dependent and independent variables, but it is always given by:
\begin{equation}
\frac{Y\hphantom{-} \text{units}}{X\hphantom{-} \text{units}}
\end{equation}

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{The Second Derivative}
Because the \textbf{\textit{derivative}} is a function, we can also consider its \textbf{\textit{derivative}}. The \textbf{\textit{derivative}} of the \textbf{\textit{derivative}} results in the \textbf{\textit{second derivative}}, which is denoted by:
\begin{equation}
\label{eq:second_deriv}
f^{\prime\prime}(x) = \frac{d^2 y}{dx^2} = \frac{d}{dx}\left(\frac{dy}{dx}\right)
\end{equation}

The \textbf{\textit{second derivative}} provides the same information about the \textbf{\textit{derivative}} as the \textbf{\textit{derivative}} provides about $f(x)$, namely:
\begin{enumerate}
\item  If $f^{\prime\prime} > 0$ on an interval $[a,b]$, then $f^\prime$ is \textbf{\textit{increasing}} over $[a,b]$.
\item  If $f^{\prime\prime} < 0$ on an interval $[a,b]$, then $f^\prime$ is \textbf{\textit{decreasing}} over $[a,b]$.
\end{enumerate}
With this information, we can determine if $f(x)$ is \textbf{\textit{concave up}} or \textbf{\textit{concave down}}.
\begin{enumerate}
\item  If $f^{\prime\prime} > 0$ on an interval $[a,b]$, then $f(x)$ is \textbf{\textit{concave up}} over $[a,b]$.
\item  If $f^{\prime\prime} < 0$ on an interval $[a,b]$, then $f(x)$ is \textbf{\textit{concave down}} over $[a,b]$.
\end{enumerate}
If $f^{\prime\prime} = 0$, then this is an \textbf{\textit{inflection point}}, where the function $f(x)$ changes the direction of curvature, or a \textbf{\textit{saddle point}}, where the \textbf{\textit{derivative}} is zero, but concavity does not change.
% TODO: Verify that saddle point definition is true in general.

\vspace{0.1in}
Relating the \textbf{\textit{second derivative}} back to speed and velocity, and using the definition in Equation (\ref{eq:second_deriv}), the acceleration of an object can be given by:
\begin{equation}
a(t) = v^\prime(t) = s^{\prime\prime}(t) = \frac{d^2 s}{dt^2} = \frac{d}{dt}\left(v\right) = \frac{d}{dt}\left(\frac{ds}{dt}\right)
\end{equation}

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{Differentiability}
A function $f$ is \textbf{\textit{differentiable}} at $x$ if the following \textbf{\textit{limit}} exists:
\begin{equation}
\lim_{h \rightarrow 0} \frac{f(x+h) - f(x)}{h}
\end{equation}
Thus, the graph of $f$ has a non-vertical tangent line at $x$. The value of the \textbf{\textit{limit}} and the slope of the tangent line are the \textbf{\textit{derivative}} of $f$ at $x$.

\vspace{0.1in}
Additionally, the function must be continuous at the point $x$ and not have a \textbf{\textit{sharp corner}} at $x$. \textbf{\textit{Sharp corners}} occur when each \textbf{\textit{one-sided limit}} approaches different values, that is $L^+ \neq L^-$. An example of this type of function is:
\begin{equation}
f(x) = |x|
\end{equation}
which has a \textbf{\textit{sharp corner}} at $x = 0$.

\vspace{0.1in}
If a function $f(x)$ is \textbf{\textit{differentiable}} at $x = a$, then $f(x)$ is \textbf{\textit{continuous}} at $x = a$; however $f(x) = |x|$ is continuous at $x =0$, but undifferentiable.
\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}


\section{Monte Carlo Methods}
Monte Carlo (MC) methods in the context of this paper generally work by randomly sampling $N$ values from a distribution
and averaging the results
(unless otherwise noted values are sampled evenly from the interval $[0,1)$).
From a theoretical standpoint this is equivalent to an integration because for $N \rightarrow \infty$ the result of
MC and numerical integration via quadrature are the same (ignoring floating point error).
The only meaningful difference between MC and a quadrature is that MC samples randomly from the distribution
while a quadrature samples evenly.
The methods introduced in this section are taken from \cite{james}.

In one dimension a quadrature is clearly superior to MC.
The error of MC is $O(N^{-\frac{1}{2}})$ while the error of even a simple quadrature like the trapezoid rule is $O(N^{-2})$.
However, the error of MC notably does \textit{not} depend on the dimensionality $d$ of the problem.
Quadratures on the other hand need a number of points that increases exponentially with $d$ to achieve the same precision.
If $N$ is constant the precision instead decreases by a factor of $N^\frac{1}{d}$.
\begin{table}
	\caption{
		Comparison of the precision of numerical integration over a $d$-dimensional volume
		with a fixed number of points $N$ when using MC or a quadrature rule.
	}
	\centering
	\begin{tabular}{cc}
		Method & Precision\\
		Monte Carlo & $O(N^{-\frac{1}{2}})$\\
		Trapezoid rule & $O(N^{-\frac{2}{d}})$\\
		Simpson rule & $O(N^{-\frac{4}{d}})$\\
		Gauss rule ($m$th order) & $O(N^{-\frac{2m-1}{d}})$\\
	\end{tabular}
	\label{tab:mc_vs_quad}
\end{table}
Table \ref{tab:mc_vs_quad} compares the precision of some quadratures to MC for multi-dimensional problems.
While more sophisticated quadrature rules are more precise for $d \rightarrow \infty$ all quadratures will be less precise than MC.
Typically problems have a high dimensionality when they have many coupled degrees of freedom,
e.g. particle physics event generators, simulation of galaxies, or weather forecasting.
\subsection{Hit-And-Miss Monte Carlo}
A very basic MC technique is to randomly sample points $\bf p$ and to simply count the points fulfilling some criterion
$f : t_\mathbf{p} \rightarrow \mathbf{bool}$.
This is called \textit{hit-and-miss MC}.
Because each point has the same likelihood of being accepted the number of accepted points will follow a binomial distribution
with probabilities $p$ and $q = 1 - p$.
The standard deviation of hit-and-miss MC can then be estimated as $s = \sqrt{\frac{pq}{N}}$.

As an example, let us consider the estimation of $\pi$ by sampling points in the $xy$ plane.
Points with $x^2 + (y - 1)^2 < 1$ are accepted as being inside the unit circle around $(0, 1)$.
The unit circle has an area of $\pi$ and one fourth of it lies inside the sampled area.
With the number of accepted points $N_\mathrm{Acc}$ we can thus calculate $\pi$ and the corresponding standard deviation $s_\pi$ as:
\begin{equation}
	\pi = 4 \frac{N_\mathrm{Acc}}{N}, \quad s_\pi = 4 \frac{pq}{N}.
\end{equation}
\begin{figure*}
	\centering
	\includegraphics[width=\linewidth]{pi_hit_and_miss.png}
	\caption{
		Visualization of hit-and-miss MC.
		Randomly sampled points are accepted if they are inside the circle around the upper left corner.
	}
	\label{fig:pi_hit_and_miss}
\end{figure*}
A graphical representation of this example is shown in Figure \ref{fig:pi_hit_and_miss}.
For $N = 1000$ hit-and-miss MC yields a precision of $s_\pi = 1.7\%$.
\subsection{Crude Monte Carlo}
If we can formulate our problem as an integration we can improve upon hit-and-miss MC.
The idea is to randomly sample values $x_i$ from the area to be integrated
and to average the results of the function $f : t_x \rightarrow \mathbf{float}$.
The integral $I$ can then be estimated as:
\begin{equation}
	I = \frac{1}{N} \sum_{i=1}^N f(x_i).
\end{equation}
The uncertainty $s_I$ of the result depends on the function variance $V[f(x)]$:
\begin{equation}
	s_I(N) = \sqrt{\frac{V[f(x)]}{N}}, \quad V[f(x)] = E \left[ (f(x) - E[f(x)])^2 \right].
\end{equation}
If we again consider our example of calculating $\pi$ we find that we can rearrange our condition as:
\begin{equation}
	y = f(x) = 1 - \sqrt{1 - x^2}.
\end{equation}
With $\pi = 4 (1 - I), \: s_\pi = 4 s_I$ we then find that crude MC has a precision of $s_\pi = 0.9\%$ for $N = 1000$.
Compared to hit-and-miss MC the uncertainty is roughly cut in half.
\subsection{Importance Sampling}
Because the function variance depends on the deviations from the expectation $E[f(x)]$
the result of crude MC will converge faster for flat functions.
A trick to reduce the function variance is to use what is known as \textit{importance sampling}:
instead of sampling evenly from the interval to be integrated,
values are sampled with an uneven probability density function (PDF) $g(x)$.
In order to compensate for the uneven sampling the sampled function values are then scaled with a factor of $\frac{1}{g(x)}$.
The effective function variance then becomes $ V \left[ \frac{f(x)}{g(x)} \right] $.
To minimize the function variance $g(x) \approx f(x)$ should be chosen.

Importance sampling can be applied to our example for crude MC by choosing $g(x) = \frac{1}{3} x^2$
(see Figure \ref{fig:pi_variance_reduction}).
\begin{figure*}
	\centering
	\includegraphics[width=\linewidth]{pi_variance_reduction.png}
	\caption{
		Visualization of crude MC with variance reduction.
		The ratio of $f(x)$ and $g(x)$ is much flatter than the individual functions.
		Samples concentrate towards high $x$ values where $g(x)$ is high.
	}
	\label{fig:pi_variance_reduction}
\end{figure*}
With the same number of points $N = 1000$ this drastically reduces the uncertainty of our result to $s_\pi = 0.1\%$
- an almost tenfold improvement in precision.
\subsection{VEGAS Algorithm}
While importance sampling is a powerful technique it can be very difficult to find a suitable PDF $g(x)$,
particularly for problems with high dimensionality.
The VEGAS algorithm implements importance sampling without a known suitable PDF.
Instead it iteratively adapts a step function to the function to be integrated.
This step function is defined by a number of bin edges:
\begin{equation}
	0=x_1<x_2 < ... < x_{M+1}=1, \quad \Delta x_i = x_{i+1} - x_{i},
\end{equation}
where each bin is given the same probability content $\frac{1}{M}$.
The PDF derived from these bins is thus:
\begin{equation}
	g(x) = \frac{1}{M \Delta x_i}.
\end{equation}
In order to adapt $g(x)$ to $f(x)$ the bin edges $x_i$ are now iteratively adapted.
Initially all bins have the same size so $g(x)$ is flat.
After drawing some samples the bin edges $x_i$ are then moved in such a way that the contribution
of each bin to the integral becomes roughly equal:
regions with a large contribution to the integral receive a high density of bins and samples
while regions with a small contribution to the integral receive a low density of bins and samples.
The sampling and resizing of bins is repeated iteratively until the algorithm terminates.
A graphical representation of the VEGAS algorithm is shown in Figure \ref{fig:pi_vegas}.
\begin{figure*}
	\centering
	\includegraphics[width=\linewidth]{pi_vegas.png}
	\caption{
		Visualization of the VEGAS algorithm with 3 iterations.
		The PDF $g(x)$ is initially flat.
		In the second and third iterations the shape of $g(x)$ is much closer shape of $f(x)$.
	}
	\label{fig:pi_vegas}
\end{figure*}


\chapter{Basis and Dimension}

\begin{definition}
	Let $V$ be a vector space and $U \subset V$ a non-empty set. We call $U$ a subspace of $V$ if it is closed under linear combinations. \\
	$ \forall \lambda, \mu \in F$ and $\forall u, v \in V : \lambda u + \mu v \in U$, where $F$ is the field where scalars come from.
\end{definition}

So essentially subspace is a vector space on its own. And it is closed means that we can add two elements and multiply by scalars and still remain in that space. \\

\textbf{Examples}

\begin{itemize}
	\item $C(\chi)$ is a subspace of $\Re^\chi$ where $C(\chi)$ is the set of continuous functions on domain $\chi$ and $\Re^\chi$ is any real valued function. Sum of two continuous functions is continuous and scalar multiplication of a continous function is a continous function as well.
	\item The set $S$ of symmetric matrices of size $n \times m$ is a subspace of $\Re^{n \times m}$. Since sum of two symmetric matrices is a symmetric matrix and the same goes for scalar multiplication.
\end{itemize}

\begin{definition}
	Let $V$ be a vector space over $F$ and $u_1,....,u_n \in V$, $\lambda_1,......\lambda_n \in F$. Then $\sum_{i=1}^{n} \lambda_i u_i$ is called a linear combination. The set of all linear combinations of $u_1,....,u_n$ is called the span (linear hull) of $u_1,....,u_n$. \\
	$span(u_1,....,u_n) := \{\sum_{i=1}^{n} \lambda_i u_i | \lambda_i \in F \}$
\end{definition}

The set $U:= \{ u_1,....,u_n \}$ is the generator of $span(u_1,....,u_n)$

\begin{definition}
	A set of vectors $v_1,.....v_n$ is called linearly independent if the following holds: \\
	$ \sum_{i=1}^{n} \lambda_i v_i = 0 \implies \lambda_1 = ..... = \lambda_n = 0$ \\
\end{definition}

\textbf{Examples}

\begin{itemize}
	\item The vectors $\begin{bmatrix} 	1 \\ 0 \\ 0 \end{bmatrix}, \begin{bmatrix} 	2 \\ 1 \\ 0 \end{bmatrix}$ and $\begin{bmatrix} 3 \\ 4 \\ 1 \end{bmatrix} \in \Re^3$ are linearly independent.
	\item The functions $sin(x)$ and $cos(x) \in \Re^\Re$ are linearly independent.
	\item Any set of $d+1$ vectors in $\Re^d$ is linearly dependent.
\end{itemize}

\begin{definition}
	A subset $B$ of a vector space $V$ is called a (Hammel) basis if:
	\begin{itemize}
		\item $span(B) = V$
		\item $B$ is linearly independent.
	\end{itemize}
\end{definition}

This means that any vector in $V$ can be written as a linear combination of vectoes in $B$ but basis vectors themselves cannot be written in terms of each other since they are independent. \\

\textbf{Examples}

\begin{itemize}
	\item The canonical basis of $\Re^3$ is  $\begin{bmatrix} 	1 \\ 0 \\ 0 \end{bmatrix}, \begin{bmatrix} 	0 \\ 1 \\ 0 \end{bmatrix}$ and $\begin{bmatrix} 0 \\ 0 \\ 1 \end{bmatrix}$.
	\item Another basis of $\Re^3$ is given by $\Re^3$ is  $\begin{bmatrix} 	1 \\ 0 \\ 0 \end{bmatrix}, \begin{bmatrix} 	1 \\ 1 \\ 0 \end{bmatrix}$ and $\begin{bmatrix} 1 \\ 1 \\ 1 \end{bmatrix}$. \\
\end{itemize}

\begin{proposition}
	If $U = \{u_1,....u_n\}$ spans a vector space $V$, then the set $U$ can be reduced to a basis of $V$.
\end{proposition}

\begin{proof}
	If $U$ is already linearly independent then we are done. If $U$ is linearly dependent, $ \exists u \in U$ that is a linear combination of other vectors in $U$. We repeat this step untill we reach a point where all vectors in $U$ are linearly independent.
\end{proof}

\begin{definition}
	A vector space is called finite dimentional if it has a finite basis.
\end{definition}

\begin{proposition}
	Let $U = \{u_1,...,u_n\} \subset V$ be a set of linearly independent vectors and let $V$ be a finite dimensional vector space. Then $U$ can be extended to a basis of $V$.
\end{proposition}

\begin{proof}
	Let $w_1,...,w_m$ be a basis of $V$. Consider a set $\{u_1,...u_n,w_1,...,w_m\}$. Remove vectors from the end untill remaining vectors are linearly independent. The remaining set $spans(V)$, is linearly independent by construction and contains $U$.
\end{proof}

\begin{corollary}
	Let $V$ be a finite dimensional vector space. Then any two basis of $V$ have the same length.
\end{corollary}

\begin{definition}
	The length of the basis of a finite dimensional vector space is called its dimension.
\end{definition}

We have defined what a basis and we also know what a subspace is. Another notion which brings these two things together is known as the \textit{sum} and \textit{direct sum} of subspaces.

\begin{definition}
	Assume that we have two subspaces $U_1$ and $U_2$ of a vector space $V$. The sum of the two spaces is defined as : \\
	$U_1 + U_2 := \{ u_1 + u_2 | u_1 \in U_1, u_2 \in U_2 \}$
\end{definition}

The sum which has just seen is known as the direct sum. If each element in the sum can be written in exactly one way$(U_1\oplus U_2)$.

\begin{proposition}
	Suppose $V$ is finite-dimensional, and $U \subset V$ is a subspace, then there exists a subspace $W \subset V$ such that $U \oplus W =  V$.
\end{proposition}

\begin{proof}
	Let the set $\{u_1,...u_k\}$ be a basis of $U$. Extend it to a basis of $V$, say the resulting set is $\{u_1,...u_k, v_1,...v_m\}$. Define $W = span\{v_1,...v_m\}$
\end{proof}


\chapter{Interpolation}

\section{Introduction}

In numerical methods we often have a limited amount of information about a function, and we want to use that information to estimate, predict or compute additional quantities. For example, we may know (or be able to compute) the values of a function $f(x)$ at two points, say $x_1$ and $x_2$, and want to use this information to estimate the value of the function at a third point $x_3$.

This can be generalized to the idea of \emph{interpolation}. We assume, for example, that we know the value of a function at $N$ points $x_j$, with $j = 0, \dots, N-1$. We want to compute, or estimate, the value of the function at some other point $x$. This problem can be extended to multiple dimensions, or to vector functions component-by-component.

We will see later that interpolation is one way of approaching considerably more complicated problems. For example, we will typically perform interpolation by approximating the function that we are interested in (but do not know how to compute in general), $f$, with a different function $g$ that approximates $f$ (for example, equals $f$ at specific points) and that we can easily compute everywhere. We can then perform \emph{operations} on $g$, and treat them as giving approximately the same results as operations performed on $f$. For example, the integral of $f$ can be approximated by the integral of $g$.

\section{Polynomial interpolation}

We start with the one dimensional problem. We assume we do \emph{not} know anything about the function $f(x)$ except for its values $f_j \equiv f(x_j)$ at the \emph{nodes} $\{ x_j \}$, where $j = 0, \dots, N$. We want to interpolate $f$ to the general location $x$.

To do this we introduce an \emph{interpolating function} $g(x)$. The general form of the function $g$ will be given in advance: here it will be a polynomial,
%
\begin{align}
  g(x) &= c_0 + c_1 x + c_2 x^2 + \dots \\ &= \sum_k c_k x^k.
\end{align}
%
We will then fix the specific form -- that is, the range of the sum, and the value of the constants $\{ c_k \}$ -- by enforcing that $g(x_j) = f_j = f(x_j)$: i.e., the value of the interpolating function equals the value of the function being interpolated at the nodes. As we know $N+1$ pieces of information about the function $f$, we can fix $N+1$ coefficients of the polynomial $g$, meaning the sum should stop at $N$ (thanks to the $0$ power coefficient).

Therefore our interpolation problem becomes: given $f_j \equiv f(x_j)$ at the \emph{nodes} $\{ x_j \}$, where $j = 0, \dots, N$, find the coefficients $c_k$ such that the polynomial
%
\begin{equation}
  g(x) = \sum_{k=0}^{N} c_k x^k
\end{equation}
%
matches $f$ at the nodes, i.e., $f(x_j) = g(x_j)$.

We will use $g$ to interpolate, or approximate, $f$. Therefore we are also interested in the question of how accurately it does so: what is $g(x) - f(x)$?

\subsection{Lagrange polynomials}

The easiest polynomials to work with are the Lagrange basis polynomials. There are $N+1$ of these are polynomials, each of degree $N$, defined by their value at the nodes so that
%
\begin{equation}
  l^{(\ell)}(x_j) = \begin{cases} 1 & x_j = x_\ell \\ 0 & x_j \ne x_\ell \end{cases}.
\end{equation}
%
In words, the $\ell^{\text{th}}$ Lagrange basis polynomial has value $1$ at the $\ell^{\text{th}}$ node and vanishes at all other nodes. This immediately means that
%
\begin{equation}
  g(x) = \sum_{\ell=0}^{N} f(x_{\ell}) l^{(\ell)}(x)
\end{equation}
%
is \emph{an} interpolating polynomial: it is a polynomial of the right degree ($N$), and when evaluated at a node $x_j$ we have
%
\begin{align}
  g(x_j) &= \sum_{\ell=0}^{N-1} f(x_{\ell}) l^{(\ell)}(x) \\
  &= f_{j}
\end{align}
%
as all the basis functions vanish at all the nodes \emph{except} for the $j^{\text{th}}$ one which has value $1$.

Now that we know that the Lagrange basis functions $l^{(\ell)}(x)$ are what we want, we need an explicit way of constructing them. We know that they must be polynomials, so we can write them as products of the form $(x - a)$. We know that we want them to vanish at all the nodes (except one), so the obvious form of these individual terms would be $(x - x_j)$. Finally, we konw that at the $\ell^{th}$ node we do not want them to vanish (so there should be so term of the form $(x - x_\ell)$), but instead want the value to be one. This gives us the value of the overall scaling constant. From these considerations we see that
%
\begin{equation}
  l^{(\ell)}(x) = \prod_{m=0, m \ne \ell}^{N} \frac{x - x_m}{x_{\ell} - x_m}.
\end{equation}

\subsection{Low order examples}

Putting everything together we have that $f(x) \simeq g(x)$, where the Lagrange polynomial interpolating function is given by
%
\begin{equation}
  g(x) = \sum_{j=0}^{N-1} f(x_{j}) \prod_{m=0, m \ne j}^{N-1} \frac{x - x_m}{x_j - x_m}.
\end{equation}
%
We can explicitly check the simple examples.

\subsubsection{Linear interpolation}

To get a linear interpolating polynomial we need the interpolating polynomial to have two coefficients (the constant and the linear part), meaning we need $N=1$. Therefore we are assuming we know $f(x)$ at the two nodes $x_0$ and $x_1$.

The Lagrange basis polynomials are
%
\begin{align}
  l^{(0)}(x) &= \frac{x - x_1}{x_0 - x_1}, \\
  l^{(1)}(x) &= \frac{x - x_0}{x_1 - x_0}.
\end{align}
%
This gives the interpolating polynomial as
%
\begin{align}
  g(x) &= f(x_0) \frac{x - x_1}{x_0 - x_1} + f(x_1) \frac{x - x_0}{x_1 - x_0}, \\
  &= f(x_0) \left( 1 - \frac{x - x_0}{x_1 - x_0} \right) + f(x_1) \frac{x - x_0}{x_1 - x_0}.
\end{align}
%
The final form of the equation can be written as $g(x) = f_0 (1 - \Delta) + f_1 \Delta$, where $\Delta$ is the distance between $x$ and $x_0$ relative to the distance between $x$ and $x_1$.

\subsubsection{Quadratic interpolation}

For quadratic interpolation we have $N=2$, which requires assuming knowledge of $f(x)$ at $x_0, x_1$ and $x_2$. The Lagrange basis polynomials are
%
\begin{align}
  l^{(0)}(x) &= \frac{(x - x_1)(x - x_2)}{(x_0 - x_1)(x_0 - x_2)}, \\
  l^{(1)}(x) &= \frac{(x - x_0)(x - x_2)}{(x_1 - x_0)(x_1 - x_2)}, \\
  l^{(2)}(x) &= \frac{(x - x_0)(x - x_1)}{(x_2 - x_0)(x_2 - x_1)}.
\end{align}
%
This gives the interpolating polynomial as
%
\begin{align}
  g(x) &= f(x_0) \frac{(x - x_1)(x - x_2)}{(x_0 - x_1)(x_0 - x_2)} + f(x_1) \frac{(x - x_0)(x - x_2)}{(x_1 - x_0)(x_1 - x_2)} + f(x_2) \frac{(x - x_0)(x - x_1)}{(x_2 - x_0)(x_2 - x_1)}.
\end{align}
%
In the special case where the points are equally spaced, $x_1 - x_0 = h = x_2 - x_1$, implying that $x_2 - x_0 = 2 h$, and using the coordinate $\hat{x} = x - x_1$, we get
%
\begin{equation}
  g(x) = \frac{1}{2 h^2} \left( f_0 \hat{x} (\hat{x} - h) - 2 f_1 (\hat{x} + h) (\hat{x} - h) + f_2 (\hat{x} + h) \hat{x} \right).
\end{equation}
%
This particular form is useful for integration and differentiation. For example,
%
\begin{align}
  \eval{\dv{g}{x}}_{x = x_1} &= \eval{\dv{g}{x}}_{\hat{x} = 0} \\
  &= \left. \frac{1}{2 h^2} \left( f_0 (2 \hat{x} - h) - 2 f_1 (2 \hat{x}) + f_2 (2 \hat{x} + h) \right) \right|_{\hat{x} = 0} \\
  &= \frac{1}{2 h} \left( f_2 - f_0 \right).
\end{align}

\section{Accuracy}

We want to know how close the interpolating polynomial $g(x)$ gets to the function $f$ that it is interpolating: that is, we want to know the size of the error $e(x) = f(x) - g(x)$. As $f$ is completely general, we will never know this exactly: however, we can make some general points.

First, we have explicitly chosen $g$ such that it agrees with $f$ at all of the nodes. That means the error must have the form
%
\begin{equation}
  e(x) = f(x) - g(x) = F(x) \prod_{j=0}^N (x - x_j).
\end{equation}
%
That is, the error function vanishes at the nodes, $e(x_j) \equiv 0$.

Second, we note that at any fixed point $X$, we can use a Taylor series expansion that matches the terms in the polynomial $g$. That is (under some assumptions), there is a value $\xi$ such that
%
\begin{equation}
  e(X) = f(X) - g(X) = \frac{1}{(N+1)!} \eval{\dv[N+1]{f}{x}}_{x=\xi} \prod_{j=0}^N (X - x_j).
\end{equation}
%
The precise form matters less than the interpretation. The important point is that the error depends on (a) the magnitude of the derivative of the function being interpolated, and (b) the magnitude of the product $\prod_{j=0}^N (x - x_j)$. The first cannot be controlled except by changing $N$. The second can be controlled both by changing $N$ and by changing the location of the nodes.

We focus on the term $|\prod_{j=0}^N (x - x_j)|$. If we make the nodes equally spaced then each term $x - x_j \propto h$, where $h = x_{j+1} - x_j$. This means the error is $e(x) \propto h^{N+1}$ and we expect the error to decay rapidly if $h$ is sufficiently small. In particular this implies that the larger $N$ is, the faster the error will decay as we make $h$ smaller. Whilst this is usually true, by ignoring the other terms in the error we may be ignoring larger errors, and even ones that grow: the Runge phenomenon is one example of this.

We also note (without proof!) that we can reduce the error by \emph{not} choosing the nodes to be equally spaced. For example, if we restrict the nodes to lie within $(-1, 1)$, by choosing the Chebyshev nodes
%
\begin{equation}
  x_j = \cos \left( \frac{2 j + 1}{2 (N + 1)}) \pi \right)
\end{equation}
%
the error $e(x)$ will behave as $e(x) \propto 2^{-N} / N!$. Using appropriate non-equally spaced nodes is often better in terms of accuracy, but may be more work to set up for integration and differentiation.


\subsection{Inner products}

An inner product is a sesquilinear form with a positive-definite Hermitian matrix.

\( \langle u, v \rangle =u^*Hv\)

If we are using the real field this is the same as:

\( \langle u, v \rangle =u^THv\)

Where \(H\) is now a symmetric real matrix.

\subsubsection{Same}

\( \langle v, v \rangle =v^*Hv\)

Always positive and real.
\subsubsection{Properties}

\(\langle u, v\rangle \langle v, u\rangle=|\langle u, v\rangle|^2\)


\chapter{Prerequisites}

This course uses many concepts from prerequisite courses, particularly those from calculus and circuits. While we assume you know this material, the following sections offer a review of the most pertinent and establish some notation. If you have trouble with any of them seek assistance -- the sooner the better.

\section{Complex Numbers}

Complex numbers are used extensively throughout the course. You need to be very adept at manipulating them.

\subsection*{The Number System}

By way of review and to motivate the discussion of complext numbers, recall the following basic facts.

\begin{itemize}
\item The \emph{Natural Numbers} $\mathbb{N}$ are the positive integers $1,2,3,4,\cdots$. Given two natual numbers $a$ and $b$ the sum $a+b$ and the product $a\,b$ are also natural numbers, that is the set of natural numbers is \emph{closed} under addition and multiplication.

\item Solving equations of the form $x + a = b$ for any natural numbers $a,b$ requires the introduction of the negative integers $\cdots, -4, -3, -2 -1$ and $0$. These plus the natural numbers give the \emph{integers}  $\mathbb{Z}$. Note $\mathbb{N} \subset \mathbb{Z}$. Zero ($0$) is called the identity element with respect to addition, while $1$ is the identity with respect to multiplication, that is $a+0 = a$ and $ a \cdot 1 = a$. The \emph{inverse} of an integer $a$ is $-a$, such that thier sum gives the identity for addition, i.e. $a + -a = 0$.

\item The \emph{rational numbers} $\mathbb{Q}$ are of the form $\frac{b}{a}$ for integers $a,b$ with $a \neq 0$. They solve problems of the form $ax=b$ and provide the inverse for multiplication since $\frac{1}{a} \cdot a = 1$. Note $\mathbb{Z} \subset \mathbb{Q}$

\item The \emph{irrational numbers} are those that cannot be written as a rational number, for example $\sqrt{2} = 1.414\ldots$ and $\pi = 3.14159\ldots$

\item The union of the rational and irrational numbers give the \emph{real numbers} denoted $\mathbb{R}$.
\end{itemize}

Graphically the numbers and thier ordering can be expressed using the number line:

\begin{center}
  \includegraphics[scale=1]{graphics/number-line.pdf}
\end{center}

\subsection*{Complex numbers as extension of reals}

Continuing the pattern of the basic number system we can ask what are solutions of equations of the form $x^2+a = 0$ or $x^2 + 2ax +a^2 + b^2 = 0$ for $a,b \in \mathbb{R}$ ? As above, finding such solutions requires moving to a larger set of numbers, the \emph{complex numbers} denoted $\mathbb{C}$.

A complex variable $z\in\mathbb{C}$ can be written as $z = a + j\, b$ for $a,b\in\mathbb{R}$, where $j$ is the imaginary unit and $j^2 = -1$. Note in mathematics the imaginary unit is denoted $i$; this difference is purely historical. Some basic definitions:

\begin{itemize}
\item the \emph{real part} $\Re(z) = \text{Re}(z) = a$
\item the \emph{imaginary part} $\Im(z) = \text{Im}(z) = b$
\item two complex numbers $z_1, z_2\in \mathbb{C}$ are equal if $\text{Re}(z_1) = \text{Re}(z_2)$ and $\text{Im}(z_1) = \text{Im}(z_2)$
\item $\mathbb{R} \subset \mathbb{C}$, when $b = 0$ and we say that the number is purely real
\item if $a = 0$ we say the number is purely imaginary
\item the \emph{complex conjugate} of $z = a + jb$ is $z^* = a - jb$.
\end{itemize}

\subsection*{Operations on complex numbers}

Arithmetic operations on complex numbers are defined using the algebra of real numbers, replacing $j^2 = -1$. Given complex numbers $a + j\, b$ and $c + j\, d$

\begin{description}
\item[addition] $(a + jb) + (c +jd) = (a+c) + j(b+d)$
\item[subtraction] $(a + jb) - (c +jd) = (a-c) + j(b-d)$
\item[multiplication] $(a + jb)\cdot(c +jd) = ac + jbc + jad + j^2 bd = (ac-bd) + j(bc+ad)$
\item[division] $\frac{(a + jb)}{(c +jd)} = \frac{ac+jbc-jad-j^2 bd}{c^2 -j^2 d^2} = \frac{(ac+bd) + j(bc-ad)}{c^2 + d^2} = \frac{ac+bd}{c^2 + d^2} + j \frac{bc-ad}{c^2 + d^2}$
\end{description}

\subsection*{Basic properties of complex numbers}

Let $z_1, z_2, z_3 \in \mathbb{C}$, then:

\begin{description}
\item[closure property] $z_1 + z_2 \in \mathbb{C}$ and  $z_1 \cdot z_2 \in \mathbb{C}$
\item[communative property] $z_1 + z_2 = z_2 + z_1$ and $z_1 \cdot z_2 = z_2 \cdot z_1$
\item[associative property] $z_1 + (z_2 + z_3) = (z_1 + z_2) + z_3$ and $z_1 \cdot (z_2 \cdot z_3) = (z_1 \cdot z_2) \cdot z_3$
\item[identity elements] $0 = (0 + j0) \in \mathbb{C}$ is the identity element for addition since $z_1 + 0 = z_1$ and $1 = 1 + j0 \in \mathbb{C}$ is the identity element for multiplication since $z_1\cdot 1 = z_1$
\item[inverse elements] for any $z_1$ there exists an inverse $z_2 = -z_1$ such that $z_1 + z_2 = 0$, and for any $z_1 \neq 0$ there exists an inverse $z_2 = z_1^{-1} = \tfrac{1}{z_1}$ such that $z_1 \cdot z_2 = 1$
\end{description}

\subsection*{Absolute Value (Magnitude) of complex numbers}

The absolute value or \textit{magnitude} of a complex number $z = a + jb$ is denoted $|z| = |a+jb|$ and is given by
\[
|a + jb| = \sqrt{a*2 + b^2}
\]
For complex numbers $z_1, z_2, \ldots, z_n$, the following useful properties hold
\begin{itemize}
\item $|z_1\cdot z_2 \cdots z_N| = |a_1|\cdot |z_2|\cdots|z_N|$
\item $\left| \frac{z_1}{z_2}\right| = \frac{|z_1|}{|z_2|}$ for $z_2 \neq 0$
\end{itemize}

\subsection*{Argument (Angle) of complex numbers}
The argument or \textit{angle} of a complex number $z = a + jb$ is denoted $\angle z = \angle(a+jb)$ and is given by
\[
\angle(a + jb) = \arctan\frac{b}{a}
\]
Take care when computing this number on your calculator (or in a programming language) so that it produces and angle in radians and in the correct quadrant. For example $\angle(-1-j1) = \arctan\frac{-1}{-1} = \frac{5\pi}{4} = -\frac{3\pi}{4}$ is different from $\angle(-1-j1) = \arctan\frac{-1}{-1} = \arctan\frac{1}{1} = \frac{\pi}{4}$, the later being incorrect.

\subsection*{Cartesian and Polar representation of complex numbers}

A complex number $z$ can be represented in Cartesian form as a pair of numbers in the \textit{Complex Plane}, $(\Re{z},\Im{z})$. The same $z$ can be represented in polar form as $z = |z|\cdot e^{j\angle z}$. We can convert between the representations using  $\Re{z} = |z| \cos(\angle z)$ and $\Im{z} = |z| \sin(\angle z)$. The following relations hold

\begin{itemize}
\item Multiplication by $j$ is equivalent to rotation by $\tfrac{\pi}{2}$
\[
j\cdot z = e^{j\tfrac{\pi}{2}}\cdot |z|\cdot e^{j\angle z} = |z|\cdot e^{j(\angle z + \tfrac{\pi}{2})}
\]
\item Division by $j$ is equivalent to rotation by $- \tfrac{\pi}{2}$
\[
\frac{1}{j}\cdot z = e^{-j\tfrac{\pi}{2}}\cdot |z|\cdot e^{j\angle z} = |z|\cdot e^{j(\angle z - \tfrac{\pi}{2})}
\]
\end{itemize}

A related expression that will be very useful to us is \textit{Eulers formula}: $e^{j\theta} = \cos(\theta) + j\sin(\theta)$. From this we can derive the relations:
\[
\cos(\theta) = \frac{1}{2} e^{j\theta} + \frac{1}{2} e^{-j\theta}
\]
\[
\sin(\theta) = \frac{1}{2j} e^{j\theta} - \frac{1}{2j} e^{-j\theta}
\]

These representations and relations can be visualized as follows
\begin{center}
  \includegraphics[scale=1]{graphics/complex-viz-prereq.pdf}
\end{center}

\subsection*{Complex numbers as roots of polynomial equations}

Recall our original motivation for complex numbers, as solutions to polynomials. Consider the $N^\text{th}$ order polynomial
\[
z^N + a_N z^{N-1} + \cdots + a_2 z + a_1
\]
where in cases of interest to us in this course the $N$ coefficients $a_{N}, \cdots, a_1$ are real. In such cases the polynomial can be factored into
\[
z^N + a_{N} z^{N-1} + \cdots + a_2 z + a_1 = (z-z_1)\cdot(z-z_2)\cdots(z-z_N)
\]
where the $z_i$ are the $N$ \textit{roots} of the polynomial. These are complex numbers in general with two cases:

\begin{itemize}
\item the root is real
\item the root is complex or purely imaginary, in which case they come in congugate pairs
\end{itemize}

Note: the \texttt{roots} function in Matlab can be used to find the roots of any order polynomial given a vector of coefficients.

\section{Functions}

As we will see in the first few lectures, signals are modelled as functions. Recall a \textit{function} is a mapping between sets
\[
f: A \rightarrow B
\]
where $A$ is a set called the {\it domain} and $B$ is a set called the {\it co-domain}. In this course we are primarily concerned with four kinds of functions

\begin{itemize}
\item the real-valued functions of an integer variable $f:\mathbb{Z}\mapsto\mathbb{R}$
\item the complex-valued functions of an integer variable $f:\mathbb{Z}\mapsto\mathbb{C}$
\item the real-valued functions of a real variable $f:\mathbb{R}\mapsto\mathbb{R}$
\item the complex-valued functions of a real variable $f:\mathbb{R}\mapsto\mathbb{C}$
\end{itemize}
We will also briefly discuss the the complex-valued functions of a complex variable $f:\mathbb{C}\mapsto\mathbb{C}$.

Functions can be defined using an expression operating on the \textit{independent variable} representing a value from the domain. For example a function $f:\mathbb{R}\mapsto\mathbb{R}$ might be defined by the expression
\[
f(t) = 2t^2
\]
where $t\in\mathbb{R}$ is the independent variable. Common operations are sums, difference, products, quotients, powers, and application of trigonometric and transendental functions. Functions with different expressions for different intervals of the domain are called \textit{piecewise}. For example
\[
f(t) = \left\{ \begin{array}{cc}
   e^{t} & t < 0\\
   e^{-t} & t \geq 0
\end{array}
\right.
\]

Functions can also be defined using composition. Given two functions $f:\mathbb{R}\mapsto\mathbb{R}$ and $g:\mathbb{R}\mapsto\mathbb{R}$ we can define a new function
\[
(f \circ g)(t) = f(g(t))
\]
where we first apply $g$ then use it's value as the input to $f$. This will be used to define several transformations of signals.

\subsection*{Visualizing Functions}

You are certainly familiar with the graph of functions $f:\mathbb{R}\mapsto\mathbb{R}$. To graph a complex-valued function of a single variable we need to plot two functions. Consider a function $z(t) \in \mathbb{C}$ for $t \in \mathbb{R}$ expressed in Cartesian form:
\[
z(t) = z_r(t) + j z_i(t)
\]
where $z_r(t) = \Re(z(t))$ and $z_r(t) = \Im(z(t))$ are the real and imaginary parts of the complex value at a given $t$. We can plot these two real-valued functions to visualize the complex function. Similarly consider a function $z(t) \in \mathbb{C}$ for $t \in \mathbb{R}$ expressed in polar form:
\[
z(t) = z_m(t) e^{jz_a(t)}
\]
where $z_m(t) = |z(t)|$ and $z_a(t) = \angle z(t)$ are the magnitude and angle of the complex value at a given $t$. We can plot these two real-valued functions to visualize the complex function.

Another approach to visualizing a complex number is to plot it as the tip of a vector that moves as a function of the independent variable.

\begin{example} Consider the function $z(t) = e^{-|t| + j2t}$. Lets convert it to polar and Cartesian form
  \begin{align*}
    z(t) &=  e^{-|t| + j2t}\\
    &=  \underbrace{e^{-|t|}}_{z_m(t)} e^{j\overbrace{2t}^{z_a(t)}}\\
    &=  e^{-|t|} (\cos(2t) + j\sin(2t))\\
    &=  \underbrace{e^{-|t|}\cos(2t)}_{z_r(t)} + j \underbrace{e^{-|t|}\sin(2t)}_{z_i(t)}
  \end{align*}
  We can then visualize the function as plots of the real and imaginary functions,

  \begin{tabular}{cc}
  \begin{tikzpicture}
    \begin{axis}[xmin=-5, xmax=5, ymin = -1, ymax=1, samples=1000, xlabel=$t$, ylabel=$z_r(t)$]
      \addplot[blue, thick] {exp(-abs(x))*cos(2*x*180/pi)};
      \addplot[mark=none, black] coordinates {(0,-1) (0,1)};
      \addplot[mark=none, black] coordinates {(-5,0) (5,0)};
    \end{axis}
  \end{tikzpicture}
  &
  \begin{tikzpicture}
    \begin{axis}[xmin=-5, xmax=5, ymin = -1, ymax=1, samples=1000, xlabel=$t$, ylabel=$z_i(t)$]
      \addplot[blue, thick] {exp(-abs(x))*sin(2*x*180/pi)};
      \addplot[mark=none, black] coordinates {(0,-10) (0,10)};
      \addplot[mark=none, black] coordinates {(-5,0) (5,0)};
    \end{axis}
  \end{tikzpicture}
  \end{tabular}\\
  or the mangitude and angle functions,\\
  \begin{tabular}{cc}
  \begin{tikzpicture}
    \begin{axis}[xmin=-5, xmax=5, ymin = 0, ymax=1, samples=1000, xlabel=$t$, ylabel=$z_m(t)$]
      \addplot[blue, thick] {exp(-abs(x))};
      \addplot[mark=none, black] coordinates {(0,0) (0,1)};
      \addplot[mark=none, black] coordinates {(-5,1) (5,1)};
    \end{axis}
  \end{tikzpicture}
  &
  \begin{tikzpicture}
    \begin{axis}[xmin=-5, xmax=5, ymin = -10, ymax=10, samples=10, xlabel=$t$, ylabel=$z_a(t)$]
      \addplot[blue, thick] {2*x};
      \addplot[mark=none, black] coordinates {(0,-10) (0,10)};
      \addplot[mark=none, black] coordinates {(-5,0) (5,0)};
    \end{axis}
  \end{tikzpicture}
  \end{tabular}
\end{example}

\section{Calculus}

Calculus is used heavily in the course. Here we remind ourselves of some basic facts. Consult your calculus text for more details.

\subsection*{Limits}

The \textit{limit} $L$ of a function $f(t)$ is the function value as the independent variable approaches a constant $t\rightarrow c$, written as
\[
\lim_{t\rightarrow c} f(t) = L
\]
The one-sided limit is defined as the limit as the independent variable approaches the constant from below or above
\[
\lim_{t\rightarrow c^-} f(t) = L \mbox{ (limit from below) or } \lim_{t\rightarrow c^+} f(t) = L \mbox{ (limit from above)}
\]

\subsection*{Derivatives of real-valued functions}

For functions $f:\mathbb{R}\mapsto\mathbb{R}$ recall the derivative is the instantaneous rate of change in the value as a function of the independent variable, and can be defined using a limit of a difference. Consider such a function $f(t)$ for $t\in\mathbb{R}$, it's derivative is given using a limit of a forward difference:

\[
\frac{df}{dt} (t) = \lim_{h\rightarrow0^+} \frac{f(t+h)-f(t)}{h}
\]

Higher-order derivative are defined recursively. For example, the second derivative is

\[
\frac{d^2f}{dt^2} (t) = \lim_{h\rightarrow0^+} \frac{\frac{df}{dt}(t+h)-\frac{df}{dt}(t)}{h}
\]
In the general case the $N^\text{th}$ order derivative is
\[
\frac{d^Nf}{dt^N} (t) = \lim_{h\rightarrow0^+} \frac{\frac{d^{N-1}f}{dt^{N-1}}(t+h)-\frac{d^{N-1}f}{dt^{N-1}}(t)}{h}
\]

Note there are several different notations for derivatives, e.g. $\frac{df}{dt}(t) = f^\prime (t) = \dot{f}(t)$, but we will use the former (Leibnitz) in most cases. We will also use the derivative operator notation $\frac{d^Nf}{dt^N} = (D^N f)(t)$, which is convenient for higher-order derivatives.

A function with finite derivatives (in the limit) for all values of the independent variable over an interval is called \textit{continuous} over that interval. Values of the independent variable where the derivative is not finite (in the limit) are called \textit{discontinuities}. A function with a finite number of discontinuities is called \textit{piecewise continuous}. If the limit is one-sided then we say the function is piecewise continuous from the left or right (or from below/above).

\subsection*{Integrals of real-valued functions}

The \textit{indefinite integral} $F(t)$ is the anti-derivative of a function $f(t)$ if $\frac{dF}{dt}(t) = f(t)$ up to a constant term, written as
\[
F(t) + C = \int f(t)\; dt
\]
where $C$ is an arbitrary constant.

The \textit{definite integral} is the area under a function between the \textit{lower limit} $a$ and the \textit{upper limit} $b$, defined as
\[
\int\limits_{a}^{b} f(t) \; dt = F(b) - F(a)
\]
In cases where one or both of the limits is infinite, the definition changes to use limits
\[
\int\limits_{-\infty}^{b} f(t) \; dt = F(b) - \lim_{a\rightarrow -\infty} F(a)
\]
\[
\int\limits_{a}^{\infty} f(t) \; dt = \lim_{b\rightarrow \infty} F(b) - F(a)
\]
\[
\int\limits_{-\infty}^{\infty} f(t) \; dt = \lim_{b\rightarrow \infty} F(b) - \lim_{a\rightarrow -\infty} F(a)
\]

\section{Differential Equations}

This course assumes a background in basic differential equations (e.g. as taught in Math 2214). However, we use only consider linear, constant-coefficient differential equations.

A linear, constant coefficient (LCC) differential equation is of the form
\[
a_0\, y + a_1\, \frac{dy}{dt} + a_2\, \frac{d^2y}{dt^2} + \cdots + a_N\, \frac{d^Ny}{dt^N}  = b_0\, x + b_1\, \frac{dx}{dt} + b_2\, \frac{d^2x}{dt^2} + \cdots + b_M\, \frac{d^My}{dt^M}
\]
which can be written compactly as
\[
\sum\limits_{k = 0}^{N} a_k\, \frac{d^ky}{dt^k} = \sum\limits_{k = 0}^{M} b_k\, \frac{d^kx}{dt^k}
\]

It is helpful to clean up this notation using the derivative operator $D^n = \frac{d^n}{dt^n}$. For example
$D^2y = \frac{d^2y}{dt^2}$ and $D^0 y= y$. To gives the form:
\[
\sum\limits_{k = 0}^{N} a_k\, D^k y = \sum\limits_{k = 0}^{M} b_k\, D^k x
\]

We can factor out the derivative operators
\[
a_0y + a_1Dy + a_2D^2y + \cdots + a_ND^Ny  = b_0\, x + b_1\, Dx + b_2\, D^2x + \cdots + b_M\, D^M x
\]
\[
\underbrace{\left(a_0 + a_1D + a_2D^2 + \cdots + a_ND^N\right)}_{\text{Polynimial in } D, Q(D)} y = \underbrace{\left(b_0 + b_1 D + b_2 D^2 + \cdots + b_M D^M\right)}_{\text{Polynimial in } D, P(D)} x
\]
to give:

\[
Q(D)y = P(D)x
\]

You learned how to solve these in differential equations as

\[
y(t) = y_\text{h}(t) + y_\text{p}(t)
\]

The term $y_\text{h}(t)$ is the solution of the homogeneous equation
\[
Q(D)y = 0
\]
Given the $N-1$ auxillary conditions $y(t_0) = y_0$, $Dy(t_0) = y_1$, $D^2y(t_0) = y_2$, up to $D^{N-1}y(t_0) = y_{N-1}$.

The term $y_\text{p}(t)$ is the solution of the particular equation
\[
Q(D)y = P(D)x
\]
for a given $x(t)$.

Rather than recapitulate the solution to $y_\text{h}(t)$ and $y_\text{p}(t)$ in the general case we focus on the homogeneous solution $y_\text{h}(t)$ only. The reason is that we will use the homogeneous solution to find the impulse response in future lectures and take a different approach to solving the general case for an arbitrary input using the impulse response and convolution.

To solve the homogenous system:

\textbf{Step 1:} Find the \emph{characteristic equation} by replacing the derivative operators by powers of an aribrary complex variable $s$.
\[
Q(D) = a_0 + a_1D + a_2D^2 + \cdots + a_ND^N
\]
becomes
\[
Q(s) = a_0 + a_1s + a_2s^2 + \cdots + a_Ns^N
\]
a polynomial in $s$ with $N$ roots $s_i$ for $i = 1, 2, \cdots, N$ such that
\[
(s - s_1)(s-s_2)\cdots(s-s_N) = 0
\]

\textbf{Step 2:} Select the form of the solution, a sum of terms corresponding to the roots of the characteristic equation.

\begin{itemize}
\item For a real root $s_1\in \mathbb{R}$ the term is of the form
  \[
  C_1 e^{s_1 t}.
  \]
\item For a pair of complex roots (they will always be in pairs) $s_{1,2} = a \pm jb$ the term is of the form
  \[
  C_1 e^{s_1 t} + C_2 e^{s_2 t} = e^{a t}\left(C_3\cos(bt) + C_4\sin(bt)\right) = C_5 e^{a t}\cos(bt + C_6).
  \]
\item For a repeated roots $s_1$, repeated r times, the term is of the form
  \[
  e^{s_1 t} (C_0 + C_1 t + \cdots + C_{r-1} t^{r-1}).\]
\end{itemize}

\textbf{Step 3:} Solve for the unknown constants in the solution using the auxillary conditions.

We now examine two common special cases, when $N=1$ (first-order) and when $N=2$ (second-order).

\subsection*{First-Order Homogeneous LCCDE}

Consider the first order homogeneous differential equation
\[
\frac{dy}{dt}(t) + ay(t) = 0 \mbox{ for } a \in \mathbb{R}
\]
The characteristic equation is given by
\[
s + a = 0
\]
which has a single root $s_1 = -a$. The solution is of the form
\[
y(t) = Ce^{s_1 t} = Ce^{-a t}
\]
where the constant $C$ is found using the auxillary condition $y(t_0) = y_0$.

\textit{Example}: Consider the homogeneous equation
\[
\frac{dy}{dt}(t) + 3y(t) = 0 \mbox{ where } y(0) = 10
\]
The solution is
\[
y(t) = Ce^{-3 t}
\]
To find $C$ we use the auxillary condition
\[
y(0) = Ce^{-3 \cdot 0} = C = 10
\]
and the final solution is
\[
y(t) = 10e^{-3 t}
\]
\subsection*{Second-Order Homogeneous LCCDE}

Consider the second-order homogeneous differential equation
\[
\frac{d^2y}{dt^2}(t) + a\frac{dy}{dt}(t) + by(t) = 0 \mbox{ for } a,b \in \mathbb{R}
\]
The characteristic equation is given by
\[
s^2 + as + b = 0
\]

Let's look at several examples to illustrate the functional forms.

Example 1:
\[
\frac{d^2y}{dt^2}(t) + 7\frac{dy}{dt}(t) + 10y(t) = 0
\]
The characteristic equation is given by
\[
s^2 + 7s + 10 = 0
\]
which has roots $s_1 = -2$ and $s_2 = -5$. Thus the form of the solution is
\[
y(t) = C_1e^{-2t} + C_2e^{-5t}
\]

Example 2:
\[
\frac{d^2y}{dt^2}(t) + 2\frac{dy}{dt}(t) + 5y(t) = 0
\]
The characteristic equation is given by
\[
s^2 + 2s + 5 = 0
\]
which has complex roots $s_1 = -1+j2$ and $s_1 = -1-j2$. Thus the form of the solution is
\[
y(t) = e^{-t}\left(C_1\cos(2t) + C_2\sin(2t)\right)
\]

Example 3:
\[
\frac{d^2y}{dt^2}(t) + 2\frac{dy}{dt}(t) + y(t) = 0
\]
The characteristic equation is given by
\[
s^2 + 2s + 1 = 0
\]
which has a root $s_1 = -1$ repeated $r=2$ times. Thus the form of the solution is
\[
y(t) = e^{-t}\left(C_1 + C_2t\right)
\]

In each of the above cases the constants, $C_1$ and $C_2$, are found using the auxillary conditions $y(t_0)$ and $y\prime(t_0)$.

\section{Circuits}

ECE 2024 is required for knowledge of continuous signals representation as voltages and currents, and the analysis and construction of circuits containing resistors, capacitors, inductors, and operational amplifiers. We will assume you can derive the differential governing equation for simple circuits using the voltage and current relationships for the circuit elements and Kerchoffs laws.

\begin{itemize}
\item Resistor: $v_R = R \,i_R$\hspace{2em}
  \begin{center}
  \begin{circuitikz}[american voltages,scale=0.8, every node/.style={transform shape}]
    \draw
    (0,0) to[R,l=$R$,v=$v_R$,i=$i_R$] (4,0);
  \end{circuitikz}
  \end{center}
\item Capacitor: $i_C = C\, v_C^\prime$ \hspace{2em}
  \begin{center}
  \begin{circuitikz}[american voltages,scale=0.8, every node/.style={transform shape}]
    \draw
    (0,0) to[C,l=$C$,v=$v_C$,i=$i_C$] (4,0);
  \end{circuitikz}
  \end{center}
\item Inductor: $v_L = L\, i_L^\prime$ \hspace{2em}
  \begin{center}
  \begin{circuitikz}[american voltages,scale=0.8, every node/.style={transform shape}]
    \draw
    (0,0) to[L,l=$L$,v=$v_L$,i=$i_L$] (4,0);
  \end{circuitikz}
  \end{center}
\item Ideal Op-Amp: the ideal op-amp operates so as to keep $i_+ = i_- = 0$ and $v_- = v_+$.
  \begin{center}
  \begin{circuitikz}[american voltages,scale=0.8, every node/.style={transform shape}]
    \draw
    (0,0) node[op amp] (op1) {}
    (-2,0.5) to[short,i=$i_-$,o-] (op1.-)
    (-2,-0.5) to[short,i=$i_+$,o-] (op1.+)
    (op1.out) to[short,i=$i_o$,-o] (2,0)
    (-2.5,0.5) node {$v_-$}
    (-2.5,-0.5) node {$v_+$}
    (2.5,0) node {$v_o$};
  \end{circuitikz}
  \end{center}
\end{itemize}

These elements are the building blocks of most continuous-time signal processing implementations.

\subsection*{KVL}

\textit{Kerchoff's Voltage Law} (KVL) says that the sum of the voltages around any closed loop must be zero.

\subsection*{KCL}

\textit{Kerchoff's Current Law} (KVL) says that the sum of the currents into a node must be zero.

\subsection*{Ideal OpAmps}

An op-amp is a device that has two inputs, labeled the inverting ($-$) and non-inverting ($+$) input respectively, and a single output. The ideal op-amp is an approximation to simplify analysis. This approximation assumes:

\begin{enumerate}
\item the inputs have an infinite impedance such that no current flows into the inputs, $i_+ = i_- = 0$
\item the op-amp operates such that the input voltages are made equal $v_- = v_+$
\item the output can source as much current as needed
\end{enumerate}

While this is just an approximation, within the frequency regimes we are concerned with in this course, it works well.

\subsection*{Governing Equations}

Given the component descriptions above and Kerchoff's laws with some algebra and calculus we can derive the input output equation for a wide variety circuits, inlcuding those with op-amps. When the circuit contains $N$ energy storage elements (inductors or capacitors) the resulting equation will be an $N^\text{th}$ order linear, constant-coefficient differential equation. Note, the cleanest route to the governing equation is not always clear at the start and it takes some trial and error on your part.

\begin{example} Consider the following RC circuit where $v(t)$ is the time-varying source voltage and we wish to know the resulting voltage across the capacitor $v_C(t)$.
  \begin{center}
    \begin{circuitikz}[american voltages,scale=0.8, every node/.style={transform shape}]
      \draw
      (0,2) to[V,l=$v(t)$] (0,0)
      (0,2) to[R,l=$R$, i=$i(t)$] (3,2)
      (3,2) to[C,l=$C$, v=$v_c(t)$] (3,0)
      (0,0) to[short] (3,0);
  \end{circuitikz}
  \end{center}
  We can analyze it using either a KVL or a KCL. Using a KVL we note
  \[
  v(t) = R\,i(t) + v_c(t)
  \]
  and note that the curent through the resistor and capacitor is the same. We use the voltage-current relationship for the capacitor to obtain
  \[
  i(t) = C\,\frac{dv_c}{dt}\; .
  \]
  Substituting into the KVL we get
  \[
  v(t) = RC\,\frac{dv_c}{dt} + v_c(t)
  \]
  which we can rearrange into the standard form for a differential equation
  \[
  \frac{dv_c}{dt} + \frac{1}{RC}v_c(t) = \frac{1}{RC}\,v(t)
  \]
  Alternatively we could have used a KCL at the top of the capacitor
  \[
  i(t) = C\frac{dv_C(t)}{dt}
  \]
  The current through the resistor using Ohm's law is
  \[
  i(t) = \frac{v(t) - v_C(t)}{R}
  \]
  Equating the two expressions and putting into standard form we get the same governing equation as before.
\end{example}
\begin{example} We can take the previous circuit and connect to an op-amp in the voltage follower form to arrive at a circuit that isolates the circuit from other parts of a larger circuit. This strategy, where we form these \textit{stages} will be used throughout the course.
  \begin{center}
    \begin{circuitikz}[american voltages,scale=0.8, every node/.style={transform shape}]
      \draw
      (5.5,2.5) node[op amp] (op1) {}
      (0,2) to[V,l=$v(t)$] (0,0)
      (0,2) to[R,l=$R$, i=$i(t)$] (3,2)
      (3,2) to[C,l=$C$, v=$v_c(t)$] (3,0)
      (3,2) to[short] (op1.+)
      (op1.out) to (6.7,4) to (4.3,4) to (op1.-)
      (op1.out) to[short, -o] (7,2.5)
      (0,0) to[short, -o] (7,0)
      (7,2.5) to[open, v=$y(t)$] (7,0);
  \end{circuitikz}
  \end{center}
  The analysis proceeds very similar to the previous one. Using the ideal op-amp conditions, a KCL at the non-inverting input gives
  \[
  i(t) = C\frac{dv_C(t)}{dt} + 0
  \]
  and
  \[
  y(t) = v_c(t)
  \]
  Again, the current through the resistor using Ohm's law is
  \[
  i(t) = \frac{v(t) - v_C(t)}{R}
  \]
  Equating the two expressions, substituting $v_C(t) \rightarrow y(t)$, and putting into standard form we get
  \[
  \frac{dy}{dt}(t) + \frac{1}{RC}y(t) = \frac{1}{RC}\,v(t)
  \]
\end{example}

\subsection*{Building and Characterizing Circuits}

We will be building and characterizing physical circuits to better connect the course to the real world. We assume you know how to build relatively simple circuits on a protoboard, as well as use a power supply, function generator and oscilliscope (i.e. the Digilent Analog Discovery). We also assume you have taken or are currently taking the embedded course, so that toward the end of the semester you know how to interface and program with the TI MSP432 "Red Board".
(MSP-EXP432P401R Evaluation board).

\section{Programming}

ECE 2514 is required for the ability to model and simulate physical systems using computational tools, and basic programming ability.

\begin{itemize}
\item Matlab for general computation and plotting
\item C++ (a small subset) for implementing digital filters
\end{itemize}

For general computation we don't require Matlab and Mathematica, Python, or Julia work as well. Matlab is required for access to filter design functionality using the signal processing toolbox.

\subsection*{Plotting and Visualization}

We assume you can plot real and complex functions using Matlab/Python/Julia/Mathematica, label axes appropriately, and generate readable graphics for inclusion in problem set solutions and the project report.

\subsection*{Computation with C or C++}

\section{Digitial Systems}

ECE 2544 is required for for knowledge of digital signal representation and the analysis and construction of circuits containing combinatorial and sequential logic.

\subsection*{Binary Representation of Integers vs Floating Point}

\subsection*{shift registers}

\subsection*{adders and multipliers}


\documentclass[11pt,twoside]{article}

\usepackage[headings]{fullpage}
\usepackage[utopia]{mathdesign}

\pagestyle{myheadings}
\markboth{What goes around}{What goes around}

\input{../../fncextra}

\begin{document}

\begin{center}
  \bf What goes around, comes around
\end{center}

A small satellite in the Earth--moon system can be modeled as a \emph{restricted three-body problem}, in which one object has so little relative mass that the motions essentially take place in a plane. In this formulation, Newton's laws of motion are simplified by a transformation so that the Earth has mass $\mu_*$ and remains at $(0,0)$, while the moon has mass $\mu$ and is at $(1,0)$, where $\mu+\mu_*=1$. The position of the satellite is $(x(t),y(t))$ and satisfies
\begin{equation}
\begin{split}
  x'' &= x + 2y' - \mu_* \frac{x+\mu}{r} - \mu \frac{x-\mu_*}{r_*} \\
  y'' &= y - 2x' - \mu_* \frac{y}{r} - \mu \frac{y}{r_*},
\end{split}\label{eq:r3body}
\end{equation}
where
\begin{align*}
  r &= \left[ (x+\mu)^2 + y^2 \right]^{3/2} \\
	r_* &= \left[ (x-\mu_*)^2 + y^2 \right]^{3/2}.
\end{align*}
For the Earth/moon system, $\mu=0.012277471$.

The key to computing solutions using standard software is to transform the original ODE system into a first-order one. Since there are two dependent variables $x$ and $y$, and both of these appear to second order, there will be 4 variables and 4 equations in the first-order version. Define $u_1=x$, $u_2=y$, $u_3=x'$, $u_4=y'$. Two trivial equations in the new system are $u_1'=u_3$ and $u_2'=u_4$. The other two equations, for $u_3'$ and $u_4'$, come from substitution into the original system~\eqref{eq:r3body}. For example,
\begin{equation*}
  u_3' = u_1 + 2u_4 - \mu_* \frac{u_1+\mu}{r} - \mu \frac{u_1-\mu_*}{r_*},
\end{equation*}
and so on.

\subsection*{Goals}

You will explore unlikely-looking satellite orbits that have fascinated mathematicians since Poincar\'e. They are of more than academic interest, because despite their strangeness, they are energy-efficient.

\subsection*{Preparation}

Read section 6.1. Fully write out the first-order system of ODEs $\bfu'=\bff(t,\bfu)$ that is equivalent to~\eqref{eq:r3body}.

\subsection*{Procedure}

Download the script template and complete it to execute the following steps.

\begin{enumerate}
\item Set
\begin{verbatim}
opt = odeset('reltol',1e-13,'abstol',1e-13);
\end{verbatim}
This will be used below to require strict error tolerances in the ODE solutions. The solutions below will all use \texttt{ode113} with \texttt{opt} as a fourth input argument.

\item Using the expression of equation~\eqref{eq:r3body} as a first-order system, $\bfu'=\bff\,(t,\bfu)$, write the function
\begin{verbatim}
function dudt = r3body(t,u)
\end{verbatim}
in which \texttt{u} is a vector containing numerical values for the dependent variables, and \texttt{dudt} returns the components of $\bfu'$.

\item Using the initial conditions
  \begin{align*}
    x(0) &= 1.2, \; & x'(0) &= 0, \\
    y(0) &= 0, \; & y'(0) &= -1.049357510,
  \end{align*}
solve the problem on the interval $[0,6.192169331]$, using \texttt{ode113} and passing \texttt{opt} as a fourth input argument. Plot the trajectory in the $(x,y)$ phase plane. It will have two major loops, one to each side of the Earth. Add the moon and Earth positions as points in the plot.

\item Repeat step 3 for
  \begin{align*}
    x(0) &= 0.994 \; & x'(0) &= 0 \\
    y(0) &= 0 \; & y'(0) &= -2.03173262955734
  \end{align*}
  and solve over $[0,11.124340337266]$. This orbit has three loops.

\item Repeat step 3 for
      \begin{align*}
        x(0) &= 0.994 \; & x'(0) &= 0 \\
        y(0) &= 0 \; & y'(0) &= -2.00158510637908
      \end{align*}
      and solve over $[0,17.06521656015796]$. This one has four loops.

\item These orbits are very sensitive to perturbations, and while our convergence theorems apply to a fixed time interval and $h\to 0$, the case of $t\to\infty$ is quite different. Re-solve step~5 for $0\le t \le 100$. Your plot will show the path of the satellite departing from the apparently periodic orbit.
\end{enumerate}

\subsection*{Discussion}

In step 3 above, what is the maximum speed of the satellite over the whole orbit?

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\Extrachap{Notation}
%\chapter{Notation}
\label{sec:Notation}

\section*{Introduction}
It is very difficult to come up with a single, consistent notation to cover the wide variety of
data, models and algorithms that we discuss. Furthermore, conventions differ between machine
learning and statistics, and between different books and papers. Nevertheless, we have tried
to be as consistent as possible. Below we summarize most of the notation used in this book,
although individual sections may introduce new notation. Note also that the same symbol may
have different meanings depending on the context, although we try to avoid this where possible.

\section*{General math notation}

\begin{longtable}{ll}
\hline\noalign{\smallskip}
\textbf{Symbol} & \textbf{Meaning} \\
\noalign{\smallskip}\hline\noalign{\smallskip}
$\lfloor x \rfloor$ & Floor of $x$, i.e., round down to nearest integer\\
$\lceil x \rceil$ & Ceiling of $x$, i.e., round down to nearest integer\\
$\vec{x} \otimes \vec{y}$ & Convolution of $\vec{x}$ and $\vec{y}$\\
$\vec{x} \odot \vec{y}$ & Hadamard (elementwise) product of $\vec{x}$ and $\vec{y}$\\
$a \wedge b$ & logical AND\\
$a \vee b$ & logical OR\\
$\neg a$ & logical NOT\\
$\mathbb{I}(x)$ & Indicator function, $\mathbb{I}(x)=1$ if x is true, else $\mathbb{I}(x)=0$\\
$\infty$ & Infinity\\
$\rightarrow$ & Tends towards, e.g., $n \rightarrow \infty$\\
$\propto$ &Proportional to, so $y = ax$ can be written as $y \propto x$\\
$\abs{x}$ & Absolute value\\
$\abs{\mathcal{S}}$ & Size (cardinality) of a set\\
$n!$ & Factorial function\\
$\nabla$ & Vector of first derivatives\\
$\nabla^2$ & Hessian matrix of second derivatives\\
$\triangleq$ & Defined as\\
$O(\cdot)$ & Big-O: roughly means order of magnitude\\
$\mathbb{R}$ & The real numbers\\
$1:n$ & Range (Matlab convention): $1:n = {1, 2,...,n}$\\
$\approx$ & Approximately equal to\\
$\arg\max\limits_x f(x)$ & Argmax: the value $x$ that maximizes $f$\\
$B(a,b)$ & Beta function, $B(a,b)=\dfrac{\Gamma(a)\Gamma(b)}{\Gamma(a+b)}$\\
$B(\vec{\alpha})$ & Multivariate beta function, $\dfrac{\prod\limits_k \Gamma(\alpha_k)}{\Gamma(\sum\limits_k \alpha_k)}$\\
$\binom{n}{k}$ & $n$ choose $k$ , equal to $n!/(k!(n−k )!)$\\
$\delta(x)$ & Dirac delta function,$\delta(x)=\infty$ if $x=0$, else $\delta(x)=0$\\
$\exp(x)$ & Exponential function $e^x$\\
$\Gamma(x)$ & Gamma function, $\Gamma(x)=\int_0^\infty u^{x-1}e^{-u}\mathrm{d}u$\\
$\Psi(x)$ &  Digamma function,$Psi(x)=\dfrac{d}{dx}\log\Gamma(x)$\\
$\mathcal{X}$ & A set from which values are drawn (e.g.,$\mathcal{X}=\mathbb{R}^D$)\\
\noalign{\smallskip}\hline\noalign{\smallskip}
\end{longtable}

\section*{Linear algebra notation}
We use boldface lower-case to denote vectors, such as $\vec{x}$, and boldface upper-case to denote matrices, such as $\vec{X}$. We denote entries in a matrix by non-bold upper case letters, such as $X_{ij}$.

Vectors are assumed to be column vectors, unless noted otherwise. We use $(x_1,\cdots,x_D)$ to denote a column vector created by stacking $D$ scalars. If we write $\vec{X}=(\vec{x}_1,\cdots,\vec{x}_n)$, where the left hand side is a matrix, we mean to stack the $\vec{x}_i$ along the columns, creating a matrix.

\begin{longtable}{ll}
\hline\noalign{\smallskip}
\textbf{Symbol} & \textbf{Meaning} \\
\noalign{\smallskip}\hline\noalign{\smallskip}
$\vec{X} \succ 0$ & $\vec{X}$ is a positive definite matrix\\
$tr(\vec{X})$ & Trace of a matrix\\
$det(\vec{X})$ & Determinant of matrix $\vec{X}$\\
$\abs{\vec{X}}$ & Determinant of matrix $\vec{X}$\\
$\vec{X}^{-1}$ & Inverse of a matrix\\
$\vec{X}^{\dagger}$ & Pseudo-inverse of a matrix\\
$\vec{X}^T$ & Transpose of a matrix\\
$\vec{x}^T$ & Transpose of a vector\\
$\mathrm{diag}(x)$ & Diagonal matrix made from vector $\vec{x}$\\
$\mathrm{diag}(X)$ & Diagonal vector extracted from matrix $\vec{X}$\\
$\vec{I}$ or $\vec{I}_d$ & Identity matrix of size $d \times d$ (ones on diagonal, zeros of)\\
$\vec{1}$ or $\vec{1}_d$ & Vector of ones (of length $d$)\\
$\vec{0}$ or $\vec{0}_d$ & Vector of zeros (of length $d$)\\
$\abs{\abs{\vec{x}}}=\abs{\abs{\vec{x}}}_2$ & Euclidean or $\ell_2$ norm $\sqrt{\sum\limits_{j=1}^{d} x_j^2}$\\
$\abs{\abs{\vec{x}}}_1$ & $\ell_1$ norm $\sum\limits_{j=1}^{d} \abs{x_j}$\\
$\vec{X}_{:,j}$ & j'th column of matrix\\
$\vec{X}_{i,:}$ & transpose of $i$'th row of matrix (a column vector)\\
$\vec{X}_{i,j}$ & Element $(i,j)$ of matrix $\vec{X}$ \\
$\vec{x} \otimes \vec{y}$ & Tensor product of $\vec{x}$ and $\vec{y}$\\
\noalign{\smallskip}\hline\noalign{\smallskip}
\end{longtable}

\section*{Probability notation}
We denote random and fixed scalars by lower case, random and fixed vectors by bold lower case, and random and fixed matrices by bold upper case. Occasionally we use non-bold upper case to denote scalar random variables. Also, we use $p()$ for both discrete and continuous random variables

\begin{longtable}{ll}
\hline\noalign{\smallskip}
\textbf{Symbol} & \textbf{Meaning} \\
\noalign{\smallskip}\hline\noalign{\smallskip}
$X,Y$ & Random variable\\
$P()$ & Probability of a random event\\
$F()$ & Cumulative distribution function(CDF), also called distribution function\\
$p(x)$ & Probability mass function(PMF)\\
$f(x)$ & probability density function(PDF) \\
$F(x,y)$ & Joint CDF\\
$p(x,y)$ & Joint PMF \\
$f(x,y)$ & Joint PDF\\
$p(X|Y)$ & Conditional PMF, also called conditional probability\\
$f_{X|Y}(x|y)$ & Conditional PDF\\
$X \perp Y$ & X is independent of Y\\
$X \not\perp Y$ & X is not independent of Y\\
$X \perp Y | Z $ & X is conditionally independent of Y given Z\\
$X \not\perp Y | Z $ & X is not conditionally independent of Y given Z\\
$X \sim p$ & X is distributed according to distribution $p$\\
$\vec{\alpha}$ & Parameters of a Beta or Dirichlet distribution\\
$\mathrm{cov}[X]$ & Covariance of X\\
$\mathbb{E}[X]$ & Expected value of X\\
$\mathbb{E}_q[X]$ & Expected value of X wrt distribution $q$\\
$\mathbb{H}(X)$ or $\mathbb{H}(p)$ & Entropy of distribution $p(X)$\\
$\mathbb{I}(X;Y)$ & Mutual information between X and Y\\
$\mathbb{KL}(p||q)$ & KL divergence from distribution $p$ to $q$\\
$\ell(\vec{\theta})$ & Log-likelihood function\\
$L(\theta,a)$ & Loss function for taking action $a$ when true state of nature is $\theta$\\
$\lambda$ & Precision (inverse variance) $\lambda=1/\sigma^2$\\
$\Lambda$ & Precision matrix $\Lambda=\Sigma^{-1}$\\
mode$[\vec X]$ & Most probable value of $\vec X$\\
$\mu$ & Mean of a scalar distribution\\
$\vec{\mu}$ & Mean of a multivariate distribution\\
$\Phi$ & cdf of standard normal\\
$\phi$ & pdf of standard normal\\
$\vec{\pi}$ & multinomial parameter vector, Stationary distribution of Markov chain\\
$\rho$ & Correlation coefficient \\
sigm($x$) & Sigmoid (logistic) function,$\dfrac{1}{1+e^{-x}}$\\
$\sigma^2$ & Variance\\
$\Sigma$ & Covariance matrix\\
var[$x$] & Variance of $x$\\
$\nu$ & Degrees of freedom parameter\\
Z & Normalization constant of a probability distribution\\
\noalign{\smallskip}\hline\noalign{\smallskip}
\end{longtable}

\section*{Machine learning/statistics notation}
In general, we use upper case letters to denote constants, such as $C, K, M, N, T$, etc. We use lower case letters as dummy indexes of the appropriate range, such as $c=1:C$ to index classes, $i=1:M$ to index data cases, $j=1:N$ to index input features, $k=1:K$ to index states or clusters, $t=1:T$ to index time, etc.

We use $x$ to represent an observed data vector. In a supervised problem, we use $y$ or $\vec{y}$ to represent the desired output label. We use $\vec{z}$ to represent a hidden variable. Sometimes we also use $q$ to represent a hidden discrete variable.

\begin{longtable}{ll}
\hline\noalign{\smallskip}
\textbf{Symbol} & \textbf{Meaning} \\
\noalign{\smallskip}\hline\noalign{\smallskip}
$C$ & Number of classes\\
$D$ & Dimensionality of data vector (number of features)\\
$N$ & Number of data cases\\
$N_c$ & Number of examples of class $c$,$N_c=\sum_{i=1}^{N}\mathbb{I}(y_i=c)$\\
$R$ & Number of outputs (response variables)\\
$\mathcal{D}$ & Training data $\mathcal{D}=\left\{(\vec{x}_i,y_i) | i=1:N\right\}$\\
$\mathcal{D}_{test}$ & Test data\\
$\mathcal{X}$ & Input space\\
$\mathcal{Y}$ & Output space\\
$K$ & Number of states or dimensions of a variable (often latent)\\
$k(x,y)$ & Kernel function\\
$\vec{K}$ & Kernel matrix\\
$\mathcal{H}$ & Hypothesis space\\
$L$ & Loss function \\
$J(\vec{\theta})$ & Cost function\\
$f(\vec{x})$ & Decision function\\
$P(y|\vec{x})$ & TODO\\
$\lambda$ & Strength of $\ell_2$ or $\ell_1 regularizer$\\
$\phi(x)$ & Basis function expansion of feature vector $\vec{x}$\\
$\Phi$ & Basis function expansion of design matrix $\vec{X}$\\
$q()$ & Approximate or proposal distribution\\
$Q(\vec{\theta},\vec{\theta}_{old})$ & Auxiliary function in EM\\
$T$ & Length of a sequence\\
$T(\mathcal{D})$ & Test statistic for data\\
$\vec{T}$ & Transition matrix of Markov chain\\
$\vec{\theta}$ & Parameter vector\\
$\vec{\theta}^{(s)}$ & $s$'th sample of parameter vector\\
$\hat{\vec{\theta}}$ & Estimate (usually MLE or MAP) of $\vec{\theta}$\\
$\hat{\vec{\theta}}_{MLE}$ & Maximum likelihood estimate of $\vec{\theta}$\\
$\hat{\vec{\theta}}_{MAP}$ & MAP estimate of $\vec{\theta}$\\
$\bar{\vec{\theta}}$ & Estimate (usually posterior mean) of  $\vec{\theta}$\\
$\vec{w}$ & Vector of regression weights (called $\vec{\beta}$ in statistics)\\
b & intercept (called $\varepsilon$ in statistics)\\
$\vec{W}$ & Matrix of regression weights\\
$x_{ij}$ & Component (i.e., feature) $j$ of data case $i$ ,for $i=1:N ,j=1:D$\\
$\vec{x}_i$ & Training case, $i=1:N$\\
$\vec{X}$ & Design matrix of size $N \times D$\\
$\bar{\vec{x}}$ & Empirical mean $\bar{\vec{x}}=\dfrac{1}{N}\sum_{i=1}^{N} \vec{x}_i$\\
$\tilde{\vec{x}}$ & Future test case\\
$\vec{x}_*$ & Feature test case\\
$\vec{y}$ & Vector of all training labels $\vec{y} =(y_1,...,y_N)$\\
$z_{ij}$ & Latent component $j$ for case $i$\\
\noalign{\smallskip}\hline\noalign{\smallskip}
\end{longtable}

%\twocolumn


\section*{Exercises}

\begin{ex}
  Let $V$ be a vector space and suppose $\set{\vect{u}_1,\ldots,
    \vect{u}_k}$ is a set of vectors in $V$. Show that $\vect{0}$
  is in $\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}$.
  \begin{sol}
    $\vect{0} = 0\vect{u}_1 + \ldots + 0\vect{u}_k$.
  \end{sol}
\end{ex}

\begin{ex}
  Determine whether $p(x) = 4x^2-x$ is in $\sspan \set{x^2+x,~ x^2-1,~ -x+2}$.
\end{ex}

\begin{ex}
  Determine whether $p(x) = -x^2+x+2$ is in
  $\sspan \set{x^2+x+1,~ 2x^2+x}$.
\end{ex}

\begin{ex}
  \begin{enumerate}
  \item Write $A = \begin{mymatrix}{rr}
      1 & 3 \\
      0 & 0
    \end{mymatrix}$ as a linear combination of
    \begin{equation*}
      \set{
        \begin{mymatrix}{rr}
          1 & 0 \\
          0 & 1
        \end{mymatrix},\quad
        \begin{mymatrix}{rr}
          0 & 1 \\
          1 & 0
        \end{mymatrix},\quad
        \begin{mymatrix}{rr}
          1 & 0 \\
          1 & 1
        \end{mymatrix},\quad
        \begin{mymatrix}{rr}
          0 & 1 \\
          1 & 1
        \end{mymatrix}
      }.
    \end{equation*}
  \item Show that the above set of four matrices is a spanning set for
    $\Mat_{2,2}$, the vector space of all $2\times 2$-matrices.
  \end{enumerate}
\end{ex}

\begin{ex}
  Let $K$ be a field, and consider the vector space $\Seq_K$ of
  infinite sequences of scalars.  A sequence $a=(a_i)_{i\in\N}$ is
  called \textbf{finitely supported}%
  \index{finitely supported sequence}%
  \index{sequence!finitely supported} if all but finitely many
  elements of the sequence are zero. In other words, $a$ is finitely
  supported if there exists some $N\in\N$ such that $a_k=0$ for all
  $k\geq N$.  Let $e^0,e^1,e^2,\ldots$ be the sequences from
  Example~\ref{exa:spans-sequences}.  Show that
  $a\in\sspan\set{e^0,e^1,e^2,\ldots}$ if and only if $a$ is finitely
  supported.
  \begin{sol}
    Let $S=\set{e^0,e^1,e^2,\ldots}$. To prove the left-to-right
    implication, assume $a\in\sspan S$. Then there exists some finite
    subset $\set{e^{k_1},e^{k_2},\ldots,e^{k_n}}$ of $S$ and scalars
    $b_1,\ldots,b_n$ such that $a = b_1e^{k_1} + \ldots + b_ne^{k_n}$.
    Let $N\in\N$ be a number that is greater than $k_1,\ldots,k_n$.
    Since each of the sequences $e^{k_1},\ldots,e^{k_n}$ is zero after
    the first $N$ elements, the same is true for their linear
    combination $a$. Thus, $a$ is finitely supported.

    To prove the right-to-left direction, assume $a$ is finitely
    supported. Let $N\in\N$ such that $a_k=0$ for all $k\geq N$.  Then
    $a = (a_0,a_1,\ldots,a_{N-1},0,0,0,\ldots)$, with infinitely many
    zeros following. Therefore,
    $a = a_0e^0 + a_1e^1 + \ldots + a_{N-1}e^{N-1}$, and it follows
    that $a\in\sspan S$.
  \end{sol}
\end{ex}


\section{Intermediate Value Theorem for Derivatives}
\begin{theorem}[IVT for Derivatives]
	If $f$ is differentiable on the closed interval $[a,b]$, then $f^\prime$ takes on every value between $f^\prime(a)$ and $f^\prime(b)$.
\end{theorem}

\begin{example}
	Show that any function that has the following function as it derivative cannot be differentiable on the interval $-1 \leq x \leq 1$.
	\begin{equation*}
		f(x) = \begin{cases}
			0 & -1 \leq x < 0 \\
			1 & 0 \leq x \leq 1
		\end{cases}.
	\end{equation*}
\end{example}
\begin{answer}
	$f$ does not take all values between $f(-1)=0$ and $f(1)=1$.
	So, by the contrapositive of the IVT for Derivatives, any function that has $f$ as its derivative will not be differentiable on $[-1,1]$.
\end{answer}


\subsection{System to Higher Order}
Let's say we have the linear system
\begin{equation*}
	\vec{x}' = A\vec{x} + \vec{f}.
\end{equation*}
Writing the system using $x_1, \ldots x_n$ as the components of $\vec{x}$, $f_1, \ldots f_n$ as the components of $\vec{f}$, and $a_{ij}$ as the entry in $A$ on the $i^{th}$ row and $j^{th}$ column,
\begin{equation*}
	\begin{cases}
		x_1' = a_{11}x_1 + \ldots a_{1n}x_n + f_1 \\
		\vdots \\
		x_n' = a_{n1}x_1 + \ldots a_{nn}x_n + f_n
	\end{cases}.
\end{equation*}
Let's arbitrarily assign $x_1 = y$.
This will allow us to find expressions for $x_2, \ldots x_n$ in terms of $y$ and its derivatives. When we find $x_n$ in these terms and equate $x_n'$ with what's given in the system, we'll have an linear $n^{th}$ order equation.

\begin{example}
	Convert the following system of equations to a single equation.
	\begin{equation*}
		\vec{x}' = \begin{bmatrix}
			1 & 1 & 1 \\
			1 & 0 & 1 \\
			0 & 1 & 1
		\end{bmatrix}\vec{x} + \begin{bmatrix}
			1 \\
			t \\
			t^2
		\end{bmatrix}
	\end{equation*}
\end{example}
\noindent
Expanding the system out of matrix form,
\begin{equation*}
	\begin{cases}
		x_1' = x_1 + x_2 + x_3 + 1 \\
		x_2' = x_1 + x_3 + t \\
		x_3' = x_2 + x_3 + t^2
	\end{cases}.
\end{equation*}
Assuming $x_1 = y$,
\begin{align*}
	y' &= y + x_2 + x_3 + 1 \\
	&\implies x_2 = y' - y - x_3 - 1.
\end{align*}
Taking the derivative of $x_2$ and equating it with what's given in the system,
\begin{equation*}
	x_2' = y'' - y' - x_3' = y + x_3 + t.
\end{equation*}
Solving for $x_3'$ and equating it with what's given in the system,
\begin{equation*}
	x_3' = y'' - y' - t - x_3 = y' - y + t^2 - 1.
\end{equation*}
Putting this expression for $x_3'$ back into our expression for $x_2'$,
\begin{align*}
	x_2' &= y'' - 2y' + y - t^2 + 1 = y + x_3 + t \\
	&\implies x_3 = y'' - 2y' - t^2 - t + 1.
\end{align*}
Taking the derivative of $x_3$ and equating it to what's given in the system,
\begin{equation*}
	x_3' = y''' - 2y'' - 2t - 1 = y' - y - 1 + t^2.
\end{equation*}
So, we have our order 3 equation,
\begin{equation*}
	y''' - 2y'' - y' + y = t^2 + 2t.
\end{equation*}

\noindent
The auxiliary polynomial of this higher order equation $p(r)$ and the characteristic polynomial $p(\lambda)$ of the linear system will have exactly the same roots.


\section{RL Framework}

\paragraph{Markov Decision Process (MDP)}
A Markov Decision Process is a tuple $(S, A, R, P, \rho_0)$ where
\begin{itemize}
    \item $S$ is the set of all valid states
    \item $A$ is the set of all valid actions
    \item $R: S \times A \times S \rightarrow \mathbb{R}$ is the \emph{reward function}, such that $r_t = R(s_t, a_t, s_{t+1})$. In the case the reward is stochastic and $r_t$ is a random variable, we have $R(s, a, s') = \mathbb{E}[r_t \mid s_t=s, a_t=a, s_{t+1}=s']$
    \item $P: S \times A \times S \rightarrow [0, 1]$ the \emph{transition probability function}, such that $P(s' \mid s, a)$ is the probability of transitioning into state $s'$ if you are in state $s$ and take action $a$
    \item $\rho_0: S \rightarrow [0, 1]$ is the starting state distribution
\end{itemize}

\paragraph{Markov property} Transitions only depend on the most recent state and action, and no prior history : $P(s_{t+1}\mid s_t,a_t,\dots,s_1,a_0,s_0) = P(s_{t+1}\mid s_t,a_t)$. This assumption does not always hold, for instance when the observed state does not contain all necessary information (\emph{Partially Observable Markov Decision Process}), or when $P$ and $R$ actually depend on $t$ (\emph{Non-Stationary Markov Decision Process}).

When the MDP is known (e.g. small tabular environments), optimal policies can be found offline without interacting with the environment, using \nameref{section:dp} (DP) algorithms. But this is generally not the case and RL algorithms have to do trial-and-error search (like bandits problems), and have to deal with \emph{delayed rewards}. Actions may affect not only the immediate reward, but also the next state and therefore all subsequent rewards.

\paragraph{Definitions}
\begin{itemize}
    \item The \emph{policy} $\pi$ determines the behavior of our agent, who will take actions $a_t \sim \pi(\cdot \mid s_t)$. Policies can be derived from an action-value function or can be explicitly parameterized and denoted by $\pi_{\theta}$. They can also be deterministic, in which case they are sometimes denoted by $\mu_{\theta}$, with $a_t = \mu_{\theta}(s_t)$.
    \item A \emph{trajectory} $\tau = (s_0, a_0, s_1, \dots)$ is a sequence of states and actions in the world, with $s_0 \sim \rho_0$ and $s_{t+1} \sim P(\cdot \mid s_t, a_t)$. It is sampled from $\pi$ if $a_t \sim \pi(\cdot \mid s_t)$ for each $t$. Trajectories are also called \emph{episodes}.
    \item The return $R(\tau)$ is the cumulative reward over a trajectory and is the quantity to be maximized by our agent. It can refer to the \emph{finite-horizon undiscounted return} $R(\tau) = \sum_{t=0}^T r_t$ or the \emph{infinite-horizon discounted return} $R(\tau) = \sum_{t=0}^{\infty} \gamma^t r_t$ for instance. Parameter $\gamma$ is called the \emph{discount factor}.
    \item The \emph{on-policy value function}: $V^{\pi}(s) = \E_{\tau \sim \pi}[R(\tau) \mid s_0=s]$
    \item The \emph{on-policy action-value function}: $Q^{\pi}(s,a) = \E_{\tau \sim \pi}[R(\tau) \mid s_0=s, a_0=a]$. We have $V^{\pi}(s) = \E_{a \sim \pi(\cdot \mid s)}[Q^\pi(s,a)]$.
    \item The \emph{advantage function}: $A^\pi(s,a) = Q^\pi(s,a) - V^\pi(s)$
    \item The optimal value and action-value functions are obtained by acting according to an optimal policy $\pi^*$: $V^*(s) = \max_{\pi} V^{\pi}(s)$, $Q^*(s,a) = \max_{\pi} Q^{\pi}(s,a)$. We have $V^*(s) = \max_a Q^*(s,a)$, and a deterministic optimal policy can be obtained with $\pi^*(s) = \argmax_a Q^*(s,a)$.
\end{itemize}

\paragraph{Bellman Equations}
\begin{equation}
    % V^{\pi}(s) = \E_{a \sim \pi(\cdot \mid s), s' \sim P}[R(s, a, s') + \gamma V^{\pi}(s')]
    V^{\pi}(s) = \E_{\substack{a \sim \pi(\cdot \mid s) \\ s' \sim P}}[R(s, a, s') + \gamma V^{\pi}(s')]
\label{eq:bellman-v}
\end{equation}

\begin{equation}
    Q^{\pi}(s, a) = \E_{s' \sim P}\left[R(s, a, s') + \gamma \E_{a'\sim \pi(\cdot \mid s')}[Q^{\pi}(s', a')] \right]
\label{eq:bellman-q}
\end{equation}

\begin{equation}
    V^*(s) = \max_a \E_{s' \sim P}\left[R(s, a, s') + \gamma V^*(s')\right]
\label{eq:bellman-v*}
\end{equation}

\begin{equation}
    Q^*(s, a) = \E_{s' \sim P}\left[R(s, a, s') + \gamma \max_{a'} Q^*(s', a')\right]
\label{eq:bellman-q*}
\end{equation}

Where $s' \sim P$ is a shorthand for $s' \sim P(\cdot\mid s,a)$


\textbf{Consider the differential equation}
\begin{align*}
y' = \sin(t)(y^2 - \cos^2(t) -1),~~y(0)=1,~~t>0.
\end{align*}
\textbf{The exact solution to this nonlinear ODE is $y(t) = \cos(t)$.}

\textbf{Use \texttt{rk4} to solve this equation for $t \in [0,50]$. Plot the error vs. values of $\Delta t$ (use \texttt{loglog}). Does the error decay as $\mathcal{O}(\Delta t^4)$? }

\begin{figure}[H]
\center{\includegraphics[scale=.75]{problem1_a.eps}}
\caption{Error Vs. $\Delta t$}
\end{figure}

As we can see in the figure above, we have done a linear fit with the \textsl{Matlab} function \texttt{polyfit} obtaining a slope of $4$. Hence, the figure above shows that the error decays as $\mathcal{O}(\Delta t^4)$.

\textbf{You will now compare \texttt{rk4}, forward-Euler, and modified Euler methods. Solve the problem with these three methods for several $\Delta t$. on a \texttt{loglog} plot, show the error vs. the number of function evaluations used by eahc method. Using the number of function evaluations as a measure of cost, is it worth using several stages in RK methods? }

\begin{figure}[H]
\center{\includegraphics[scale=.75]{problem1_b.eps}}
\caption{Error Vs. cost}
\end{figure}

As we can see in the previous figure, it is worth using several stages in RK methods despite their cost. With the same number of funciton evaluations, RK4 reaches much higher level of accuracy.

\subsection*{Matlab code for this part}
\begin{verbatim}
clear variables; close all; format long; clc
path='Latex/FIGURES/';

linewidth=1.2;
markersize=7.5;
legendfontsize=12;
axisfontsize=16;

%% Problem 1
% Part a)
dt = 2e-1;
dydt = @(t,y) sin(t)*(y.^2-(cos(t))^2-1);
err=1;
tol=1e-12;
tspan=[0 50];
y0=1;
i=1;
while err>tol
    N=(tspan(2)-tspan(1))/dt(i);
    [t,y] = rk4(dydt,tspan,y0,N);
    err(i)=norm(y-cos(t),inf);
    dt(i+1)=dt(i)/2;
    i=i+1;
end
dt(end)=[];

% Linear fit
P=polyfit(log(dt),log(err),1);
dtfit=2*dt(1):-0.001:dt(end)/2;
errfit=exp(P(2))*dtfit.^(P(1));

figure(1)
loglog(dt,err,'r*','markersize',markersize)
grid on
hold on
loglog(dtfit,errfit,'b-.','linewidth',linewidth)
grid on
xlabel('$\log(\Delta t)$','Interpreter','latex')
ylabel('$\log(\epsilon)$','Interpreter','latex')
set(gca,'fontsize',14)
txt=[path,'problem1_a'];
saveas(gcf,txt,'epsc')

% Part b)
N=2.^(4:12);
for i=1:length(N);
    [t1,y1] = euler(dydt,tspan,y0,N(i));
    [t2,y2] = modEuler(dydt,tspan,y0,N(i));
	[t4,y4] = rk4(dydt,tspan,y0,N(i));
    err1(i)=norm(y1-cos(t1),inf);
    err2(i)=norm(y2-cos(t2),inf);
    err4(i)=norm(y4-cos(t4),inf);
end
figure
loglog(N,err1,2*N,err2,4*N,err4)
grid on
ylabel('$\log(\epsilon)$','Interpreter','latex')
set(gca,'fontsize',12)
xlabel('Function Evaluations','Interpreter','latex')
legend({'Euler','Modified Euler', 'RK4'},'fontsize',14)
txt=[path,'problem1_b'];
saveas(gcf,txt,'epsc')

function [t,w] = rk4(dydt,tspan,y0,N)

h = diff(tspan)/N;
t = tspan(1) + h*(0:N)';
w = zeros(length(y0),N+1);
w(:,1) = y0(:).';

for i = 1:N
    k1 = h*dydt(t(i)    ,w(:,i)      );
    k2 = h*dydt(t(i)+h/2, w(:,i)+k1/2);
    k3 = h*dydt(t(i)+h/2, w(:,i)+k2/2);
    k4 = h*dydt(t(i)+h  , w(:,i)+k3  );
    w(:,i+1) = w(:,i) + (k1 + 2*k2 + 2*k3 + k4)/6;
end
w = w.';
end

function [t,w] = modEuler(dydt,tspan,y0,N)

h = diff(tspan)/N;
t = tspan(1) + h*(0:N)';
w = zeros(length(y0),N+1);
w(:,1) = y0(:).';

for i = 1:N
    k1 = h*dydt(t(i)    ,w(:,i)      );
    k2 = h*dydt(t(i)+h, w(:,i)+k1);
    w(:,i+1) = w(:,i) + (k1 + k2)/2;
end
w = w.';
end

function [t,w] = euler(dydt,tspan,y0,N)

h = diff(tspan)/N;
t = tspan(1) + h*(0:N)';
w = zeros(length(y0),N+1);
w(:,1) = y0(:).';

for i = 1:N
    k1 = h*dydt(t(i)    ,w(:,i)      );
	w(:,i+1) = w(:,i) + k1;
end
w = w.';
end
\end{verbatim}