\section{Addition}

\begin{outcome}
  \begin{enumerate}
  \item Compute sums and differences of vectors algebraically and
    geometrically.
  \item Use the laws of vector addition to prove equalities between
    vector expressions.
  \end{enumerate}
\end{outcome}

Addition of vectors in $\R^n$ is defined as follows.

\begin{definition}{Addition of vectors in $\R^n$}{vector-addition}
  For vectors $\vect{u}=\begin{mymatrix}{c}
    u_1 \\
    \vdots \\
    u_n
  \end{mymatrix},\; \vect{v}= \begin{mymatrix}{c}
    v_1 \\
    \vdots \\
    v_n
  \end{mymatrix} \in \R^n$, the sum%
  \index{vector!addition}%
  \index{addition!of vectors} $\vect{u}+\vect{v}\in \R^n$ is defined
  by
  \begin{equation*}
    \vect{u}+\vect{v} = \begin{mymatrix}{c}
      u_1 \\
      \vdots \\
      u_n
    \end{mymatrix} +  \begin{mymatrix}{c}
      v_1 \\
      \vdots \\
      v_n
    \end{mymatrix}
    = \begin{mymatrix}{c}
      u_1+v_1 \\
      \vdots \\
      u_n+v_n
    \end{mymatrix}.
  \end{equation*}
\end{definition}

To add vectors, we simply add corresponding components. Therefore, in
order to add vectors, they must be the same size.  For example,
$\mat{1, 2, 3}^T + \mat{4, 5, 6}^T = \mat{1+4, 2+5, 3+6}^T =
\mat{5, 7, 9}^T$.

The geometric significance of vector addition in $\R^n$ is given in
the following proposition.

\begin{proposition}{Geometry of vector addition}{geometry-of-vector-addition}
  Let $\vect{u}$ and $\vect{v}$ be two vectors in $\R^n$. Slide
  $\vect{v}$ so that the tail of $\vect{v}$ is on the tip of
  $\vect{u}$. Then draw the arrow which goes from the tail of
  $\vect{u}$ to the tip of $\vect{v}$.  This arrow represents the
  vector $\vect{u}+\vect{v}$.

  \begin{center}
    \begin{tikzpicture}[scale=2]
      \draw[->, thick, purple](0,0) -- node[above left]{$\vect{u}+\vect{v}$}(1.5,1);
      \draw[->, thick, blue](0,0) -- node[below]{$\vect{u}$} (0.75,0);
      \draw[->, thick, red](0.75,0) -- node[below right]{$\vect{v}$} (1.5,1);
    \end{tikzpicture}
  \end{center}
\end{proposition}

\begin{example}{Commutative law}{commutative-law-geometric}
  Let $\vect{u}$ and $\vect{v}$ be two vectors. Using the geometry of
  vector addition, explain why $\vect{u}+\vect{v} = \vect{v} + \vect{u}$.
\end{example}

\begin{solution}
  In the following diagram, the vectors $\vect{u}$ and $\vect{v}$ form
  a parallelogram. Therefore, whether we line up the tail of
  $\vect{u}$ with the tip of $\vect{v}$ or vice versa, we obtain the
  same vector, which is both $\vect{u}+\vect{v}$ and
  $\vect{v}+\vect{u}$.
  \begin{center}
    \begin{tikzpicture}[scale=2.5,rotate=-20]
      \draw[->, thick, purple](0,0) -- node[above]{$\vect{v}+\vect{u}$}
      node[below]{$\vect{u}+\vect{v}$} (1.5,1);
      \draw[->, thick, blue](0,0) -- node[below]{$\vect{u}$} (0.75,0);
      \draw[->, thick, red](0.75,0) -- node[below]{$\vect{v}$} (1.5,1);
      \draw[->, thick, blue](0.75,1) -- node[above]{$\vect{u}$} (1.5,1);
      \draw[->, thick, red](0,0) -- node[above]{$\vect{v}$} (0.75,1);
    \end{tikzpicture}
  \end{center}
\end{solution}

\begin{definition}{Negative}{vector-negative}
  The \textbf{negative}%
  \index{vector!negative}%
  \index{negative!of a vector} of a vector
  $\vect{u}=\begin{mymatrix}{c}
    u_1 \\
    \vdots \\
    u_n
  \end{mymatrix}\in\R^n$ is defined by
  $-\vect{u} = \begin{mymatrix}{c}
    -u_1 \\
    \vdots \\
    -u_n
  \end{mymatrix}$.
\end{definition}

Geometrically, the vector $-\vect{u}$ has the same magnitude as
$\vect{u}$, but the opposite direction.
\begin{center}
  \begin{tikzpicture}
    \draw[->, thick, blue](0,0) -- node[above]{$\vect{u}$} +(2,1);
    \draw[->, thick, blue](5,1) -- node[above]{$-\vect{u}$} +(-2,-1);
  \end{tikzpicture}
\end{center}

To define the \textbf{subtraction}%
\index{vector!subtraction}%
\index{subtraction!of vectors} of two vectors, we simply regard
$\vect{u}-\vect{v}$ as an abbreviation for $\vect{u} + (-\vect{v})$,
exactly as we do with real numbers. Algebraically, this just amounts
to componentwise subtraction:
\begin{equation*}
  \begin{mymatrix}{c}
    u_1 \\
    \vdots \\
    u_n
  \end{mymatrix} - \begin{mymatrix}{c}
    v_1 \\
    \vdots \\
    v_n
  \end{mymatrix}
  = \begin{mymatrix}{c}
    u_1-v_1 \\
    \vdots \\
    u_n-v_n
  \end{mymatrix}.
\end{equation*}

The following example illustrates how to subtract vectors
geometrically.

\begin{example}{Graphing vector addition}{graphing-vector-addition}
  Consider the following picture of vectors $\vect{u}$ and $\vect{v}$.

  \begin{center}
    \begin{tikzpicture}[scale=2]
      \draw[->, thick, blue] (0,0) -- node[above]{$\vect{u}$} (2,1);
      \draw[->, thick, red] (4,1) -- node[above]{$\vect{v}$} (5,0.5);
    \end{tikzpicture}
  \end{center}

  Sketch a picture of $\vect{u}+\vect{v}$ and $\vect{u}-\vect{v}$.
\end{example}

\begin{solution}
  We will first sketch $\vect{u}+\vect{v}$. Begin by drawing $\vect{u}$
  and then at the point of $\vect{u}$, place the tail of $\vect{v}$ as
  shown. Then $\vect{u}+\vect{v}$ is the vector which results from
  drawing a vector from the tail of $\vect{u}$ to the tip of $\vect{v}$.

  \begin{center}
    \begin{tikzpicture}[scale=2]
      \draw[->, thick, blue] (0,0) -- node[above]{$\vect{u}$} (2,1);
      \draw[->, thick, red] (2,1) -- node[above]{$\vect{v}$} (3,0.5);
      \draw[->, thick, purple](0,0) -- node[below]{$\vect{u}+\vect{v}$} (3,0.5);
    \end{tikzpicture}
  \end{center}

  Next consider $\vect{u}-\vect{v}$. This means $\vect{u}+(-\vect{v}
  )$. From the above geometric description of vector addition, $-\vect{v}$
  is the vector which has the same length but which points in the
  opposite direction to $\vect{v}$. Here is a picture of $-\vect{v}$

  \begin{center}
    \begin{tikzpicture}[scale=2]
      \draw[<-, thick, red] (4,1) -- node[above]{$-\vect{v}$} (5,0.5);
    \end{tikzpicture}
  \end{center}

  The following picture fully represents $\vect{u}-\vect{v}$:

  \begin{center}
    \begin{tikzpicture}[scale=2]
      \draw[->, thick, blue](0,0) -- node[above]{$\vect{u}$} (2,1);
      \draw[<-, thick, red] (1,1.5) -- node[above]{$-\vect{v}$} (2,1);
      \draw[->, thick, purple](0,0) -- node[left]{$\vect{u}-\vect{v}$}(1,1.5);
    \end{tikzpicture}
  \end{center}

  Given any two vectors $\vect{u}$ and $\vect{v}$ one can create a
  parallelogram with sides these vectors and diagonals
  $\vect{u}+\vect{v}$ and $\vect{v}-\vect{u}$:

  \begin{center}
    \begin{tikzpicture}[scale=2]
      \draw[->, thick, blue](-1,0.5) -- node[above]{$\vect{u}$} (1,1.5);
      \draw[->, thick, red] (1,1.5) -- node[above]{$\vect{v}$} (2,1);
      \draw[->, thick, purple](0,0) -- node[left, near start]{$\vect{u}-\vect{v}$}(1,1.5);
      \draw[->, thick, blue](0,0) -- node[below]{$\vect{u}$} (2,1);
      \draw[->, thick, red](-1,0.5) -- node[below]{$\vect{v}$} (0,0);
      \draw[->, thick, purple](-1,0.5) -- node[above, near end]{$\vect{u}+\vect{v}$} (2,1);
    \end{tikzpicture}
  \end{center}
\end{solution}

Addition of vectors satisfies some important properties which are
outlined in the following proposition.  Recall that $\vect{0}$ is the
\textbf{zero vector}%
\index{zero vector}%
\index{vector!zero vector}, the vector from $\R^n$ in which all
components are equal to $0$.

\begin{proposition}{Properties of vector addition}{properties-vector-addition}
  The following properties hold for vectors
  $\vect{u},\vect{v},\vect{w}\in\R^n$.%
  \index{vector!properties of addition}%
  \index{vector!addition!properties}%
  \index{properties of addition!vectors}
  \begin{itemize}
  \item The commutative law of addition%
    \index{commutative law!of addition}%
    \index{vector!commutative law of addition}%
    \begin{equation*}
      \vect{u}+\vect{v}=\vect{v}+\vect{u}.
    \end{equation*}
  \item The associative law of addition
    \index{associative law!of addition}%
    \index{vector!associative law of addition}%
    \begin{equation*}
      (\vect{u}+\vect{v})+\vect{w} = \vect{u}+(\vect{v}+\vect{w}).
    \end{equation*}
  \item The existence of an additive unit%
    \index{additive unit}%
    \index{unit!of addition|see{additive unit}}%
    \index{vector!additive unit}%
    \begin{equation*}
      \vect{u}+\vect{0}=\vect{u}.
    \end{equation*}
  \item The existence of an additive inverse
    \index{additive inverse}%
    \index{vector!additive inverse}%
    \index{inverse!additive}%
    \begin{equation*}
      \vect{u}+(-\vect{u}) =\vect{0}.
    \end{equation*}
  \end{itemize}
\end{proposition}


\section{Exercises in Multivariate Random Variables}\label{S:xsMultivariateRVs}
\begin{ExerciseList}
%question 9 (cts )
\Exercise
Find the probability that none of the three bulbs in a traffic signal, that are assumed to have independent life-times (i.e., the time during which they are operational), need to be replaced during the first 1200 hours of operation if the length of time before a single bulb needs to  be replaced is a continuous random variable $X$ with density
$$f(x)\;=\;\begin{cases}6\left(0.25-(x-1.5)^2\right)&1<x<2\\0&\textrm{otherwise}\end{cases}\enspace.$$
Note: $X$ is measured in multiples of 1000 hours.
\Answer
The probability that  \emph{one}  light bulb doesn't need to be replaced in  1200 hours is:
\ba{\P(X>1.2)&\;=\;1-\P(X<1.2)\\[3pt]
&=\;1\;-\;\int^{1.2}_1 6 (0.25-(x-1.5)^2)\,dx\\[3pt]
&=\;1\;-\;\int^{1.2}_1 6(0.25-x^2+3x-2.25)\,dx\\[3pt]
&=\;1\;-\;\int^{1.2}_1 (-6x^2+18x-12)\,dx\\[3pt]
&=\;1\;-\;\left[-2x^3+9x^2-12x\right]^{1.2}_1\\[3pt]
&=\;1-0.1040\\[3pt]
&\;=\;0.8960}
Assuming  that   the three light bulbs function  independently of  each
other, the probability that none of them need to be replaced in the
first 1200 hours is
$$\P(\{X_1>1.2\}\cap\{X_2>1.2\}\cap\{X_3>1.2\}\;=\;0.8960^3\;=\;
0.7193$$
where $X_i$ is the length of time that bulb $i$ lasts.

\Exercise
Let $(X,Y)$ be a continuous \rv~with joint probability density function (JPDF)
\[
f_{X,Y}(x,y)
=
\begin{cases}
a (x^2+y) & \text{ if } 0 < x < 1 \text{ and } 0 < y < 1\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
Find the following:
\be
\item~the normalizing constant $a$ which will ensure $\P(\Omega) = \int_{-\infty}^{\infty}\int_{-\infty}^{\infty} f_{X,Y}(x,y) dx dy = 1$
\item~$f_X(x) = \int_{-\infty}^{\infty} f_{X,Y}(x,y) dy$ called the marginal probability density function (MPDF) of $X$
\item~$f_Y(y) = \int_{-\infty}^{\infty} f_{X,Y}(x,y) dx$ called the marginal probability density function (MPDF) of $Y$
\item Check if $f_X(x)f_Y(y)=f_{X,Y}(x,y)$ for every $(x,y)$ and decide whether $X$ and $Y$ are independent random variables.  {Hint: $X$ and $Y$ are said to be independent if $f_X(x)f_Y(y)=f_{X,Y}(x,y)$ for every $(x,y)$.}
\item~$F_{X,Y}(x,y)$, the joint cumulative distribution function (JCDF) of $(X,Y)$ for any $(x,y) \in (0,1) \times (0,1)$
\item~the probability that $X > 0.5$ and $Y<0.6$, i.e., $\P(X>0.5,Y<0.6)$
\item~$E(X)$, the expectation of $X$ or the first moment of $X$
\item~$E(Y)$, the expectation of $Y$ or the first moment of $Y$
\item~$E(XY)$, the expectation of $XY$
\item~$\cv(X,Y)=E(XY)-E(X)E(Y)$, the covariance of $X$ and $Y$.
\ee

\Answer
~\\
\be
\item~To find $a$ we simply set $1=\int_{-\infty}^{\infty}\int_{-\infty}^{\infty} f_{X,Y}(x,y) dx dy$ and solve for $a$ as follows:
\begin{align*}
1
&=
\int_{-\infty}^{\infty}\int_{-\infty}^{\infty} f_{X,Y}(x,y) dx dy = \int_{0}^{1}\int_{0}^{1} a(x^2+y) dx dy\\
&=
a\int_{0}^{1}\int_{0}^{1} (x^2+y) dx dy = a\int_{0}^{1} \left[ \frac{1}{3}x^3+yx \right]_{x=0^1} dy\\
&=
a\int_{0}^{1} \left( \frac{1}{3}+y - 0 \right) dy = a \left[ \frac{y}{3}+\frac{1}{2}y^2 \right]_{y=0}^{1}\\
&=
a \left( 0-\left(\frac{1}{3}+\frac{1}{2}1^2\right) \right) = a \left( \frac{1}{3}+\frac{1}{2} \right)\\
&=
a \left( \frac{5}{6} \right)
\end{align*}
Therefore $a=6/5$ and the joint PDF is
\[
f_{X,Y}(x,y)
=
\begin{cases}
\frac{6}{5} (x^2+y) & \text{ if } 0 < x < 1 \text{ and } 0 < y < 1\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
\item~First compute the marginal PDF $f_X(x)$ for any $x \in (0,1)$ by integrating over $y$
\begin{align*}
f_X(x)
&= \int_{-\infty}^{\infty} f_{X,Y}(x,y) dy
= \int_0^1 \frac{6}{5} (x^2+y) dy
= \left[ \frac{6}{5} (yx^2+y^2/2) \right]_{y=0}^1\\
&= \frac{6}{5} \left((1 \times x^2 + 1^2/2) - 0 \right)
= \frac{6}{5} \left( x^2 + \frac{1}{2} \right)
\end{align*}
Finally, the marginal PDF of the RV $X$ in the first component of the \rv~$(X,Y)$ is
\[
f_{X}(x)
=
\begin{cases}
\frac{6}{5} \left( x^2 + \frac{1}{2} \right) & \text{ if } 0 < x < 1\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
\item~Similarly, the marginal PDF $f_Y(y)$ for any $y \in (0,1)$ by integrating over $x$ is
\[
f_Y(y)=\int_{-\infty}^{\infty} f_{X,Y}(x,y) dx = \int_0^1 \frac{6}{5}(x^2+y) dx = \frac{6}{5} \left[ x^3/3+yx\right]_{x=0}^1 = \frac{6}{5}(y+1/3).
\]
Finally, the marginal PDF of the RV $Y$ in the second component of the \rv~$(X,Y)$ is
\[
f_{Y}(y)
=
\begin{cases}
\frac{6}{5} \left( y + \frac{1}{3} \right) & \text{ if } 0 < y < 1\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
\item The product of marginal PDFs of $X$ and $Y$ does not equal the joint PDF of $(X,Y)$ for values of $(x,y) \in (0,1)^2$
\[
f_X(x)f_Y(y) = \frac{6}{5} \frac{6}{5} \left( y + \frac{1}{3} \right) \left( x^2 + \frac{1}{2} \right) = \frac{6}{25} \left( 6 x^2y+2x^2+3y+1\right) \neq \frac{6}{5}(x^2+y) = f_{X,Y}(x,y)
\]
Therefore $X$ and $Y$ are not independent random variables (they are dependent!).
\item~The joint distribution function $F_{X,Y}(x,y)$ for any $(x,y) \in (0,1)^2$ is
\begin{align*}
F_{X,Y}(x,y)
&= \int_{-\infty}^{y} \int_{-\infty}^{x} f_{X,Y}(u,v) du dv
= \int_{0}^{y} \int_{0}^{x} \frac{6}{5} (u^2+v) du dv
= \frac{6}{5} \int_{0}^{y} \left[ \frac{u^3}{3}+vu) \right]_{u=0}^{x} dv\\
&= \frac{6}{5} \int_{0}^{y} \left( \frac{x^3}{3}+vx - 0 \right) dv
= \frac{6}{5} \left[ \frac{x^3v}{3}+\frac{v^2x}{2} \right]_{v=0}^{y}
= \frac{6}{5} \left( \frac{x^3y}{3}+\frac{y^2x}{2} - 0 \right)\\
&= \frac{6}{5} \left( \frac{x^3y}{3}+\frac{y^2x}{2} - 0 \right)
\end{align*}
\item~
\begin{align*}
\P(X>0.5,Y<0.6)
&= \int_{-\infty}^{0.6} \int_{0.5}^{\infty} f_{X,Y}(x,y) dx dy
= \int_{0}^{0.6} \int_{0.5}^{1} \frac{6}{5} (x^2+y) dx dy\\
&= \frac{6}{5} \int_{0}^{0.6} \left[ \frac{x^3}{3}+yx \right]_{x=0.5}^{1} dy
= \frac{6}{5} \int_{0}^{0.6} \left(\frac{7}{24} + \frac{y}{2} \right) dy\\
&= \frac{6}{5} \left[\frac{7}{24}y + \frac{y^2}{2} \right]_{y=0}^{0.6}
= \frac{6}{5} \left(\frac{7}{24}\times \frac{6}{10} + \frac{36}{400} \right)
=0.318
\end{align*}
\item~
\begin{align*}
E(X)
&= \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} x f_{X,Y}(x,y) dx dy\\
&=\frac{6}{5} \int_{0}^{1} \int_{0}^{1}  x \left( x^2 + y \right) dx dy
=\frac{6}{5} \int_{0}^{1} \int_{0}^{1}  \left( x^3 + xy \right) dx dy
= \frac{6}{5} \int_{0}^{1} \left[ \frac{x^4}{4} + \frac{1}{2}x^2y\right]_{x=0}^{1} dy\\
&=\frac{6}{5} \int_{0}^{1} \left( \frac{1}{4} + \frac{y}{2} - 0 - 0\right) dy
= \frac{6}{5} \left[ \frac{y}{4} + \frac{y^2}{4} \right]_{y=0}^{1}
= \frac{6}{5} \left(\frac{1}{4} + \frac{1}{4} - 0 - 0 \right) = \frac{6}{5}\times \frac{1}{2}=\frac{3}{5}
\end{align*}
\item~
\begin{align*}
E(Y)
&= \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} y f_{X,Y}(x,y) dx dy\\
&= \int_{0}^{1} \int_{0}^{1} y \frac{6}{5} \left( x^2 + y \right) dx dy
= \frac{6}{5} \int_{0}^{1} \int_{0}^{1}  \left( x^2y + y^2 \right) dx dy
= \frac{6}{5} \int_{0}^{1} \left[ \frac{x^3y}{3} + y^2x \right]_{x=0}^{1} dy\\
&= \frac{6}{5} \int_{0}^{1} \left( \frac{y}{3} + y^2 -0-0\right) dy
= \frac{6}{5} \left[ \frac{y^2}{6} + \frac{y^3}{3} \right]_{y=0}^{1}
= \frac{6}{5} \left( \frac{1}{6} + \frac{1}{3} + - 0 -0 \right) = \frac{6}{5} \times \frac{3}{6}=\frac{3}{5}
\end{align*}
\item~
\begin{align*}
E(XY)
&= \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} xy f_{X,Y}(x,y) dx dy\\
&=\frac{6}{5} \int_{0}^{1} \int_{0}^{1}  xy \left( x^2 + y \right) dx dy
=\frac{6}{5} \int_{0}^{1} \int_{0}^{1} x^3y+xy^2 dx dy
=\frac{6}{5} \int_{0}^{1} \left[ \frac{x^4}{4}y+\frac{x^2 y^2}{2} \right]_{x=0}^1 dy\\
&=\frac{6}{5} \int_{0}^{1} \left( \frac{y}{4} + \frac{y^2}{2} -0-0\right)dy
=\frac{6}{5} \left[ \frac{y^2}{8} + \frac{y^3}{6}\right]_{y=0}^1
=\frac{6}{5} \left(\frac{1}{8} + \frac{1}{6}-0-0\right)\\
&=\frac{6}{5} \left(\frac{3}{24}+\frac{4}{24}\right)=\frac{6}{5}\times\frac{7}{24}=\frac{7}{20}
\end{align*}
\item~
\[
\cv(X,Y) = E(XY)-E(X)E(Y) = \frac{7}{20}-\left(\frac{3}{5} \times \frac{3}{5} \right) = \frac{7}{20}-\frac{9}{25} = \frac{35}{100}-\frac{36}{100} = -\frac{1}{100}
\]
\ee

\Exercise
Logs are milled to have a width of $\mu$.
The actual width of a randomly selected item is $X$.
If $X$ is a $\normal(\mu,\sigma^2)$ random variable then find the probability density function of the {\em squared-error} of the milling process,
\[ Y\,=\, (X-\mu)^2\,.\]
\Answer
Note that $Y= (X-\mu)^2$ is not on-to-one so it is better to use the direct method by differentiating the distribution function  of $Y$, $F_Y(y)$,  to obtain $f_Y(y)$.

If $y\geqslant 0$,
\begin{align*}
 F_Y(y)\;=&\; \P(Y \leqslant y) \\[3pt]
 \;=&\;  \P( (X-\mu)^2 \leqslant y) \\[3pt]
 \;=&\; \P(-\sqrt y \leqslant X-\mu \leqslant \sqrt y)\\[3pt]
 \;=&\; \P(\mu -\sqrt y \leqslant X  \leqslant \mu + \sqrt y)\\[3pt]
\;=&\; F_X( \mu + \sqrt y) \,-\, F_X( \mu + \sqrt y)
\end{align*}

Differentiating this expression gives
\begin{align*}
f_Y(y) \;=&\; \frac{d}{dy} \left(F_X( \mu + \sqrt y) \,-\, F_X( \mu - \sqrt y)  \right) \\[3pt]
\;=&\;\frac{1}{2} y^{-\frac{1}{2}} \, f_X (\mu + \sqrt y ) \,- \, \left( -\frac{1}{2} y^{-\frac{1}{2}} \right)\, f_X (\mu - \sqrt y ) \\[3pt]
\;=&\;\frac{1}{2 \sqrt y } \left(  f_X (\mu + \sqrt y )  + f_X (\mu - \sqrt y )  \right)
\end{align*}

Note: If $y<0$ then $f_Y(y) = 0$  since  $F_Y(y)=0$ in this case.

\Exercise
Let $(X,Y)$ be a discrete random vector (\rv) with support:
\[
\mathcal{S}_{X,Y} = \{(0,0),(0,1),(1,0),(1,1)\} \enspace .
\]
Let its joint probability mass function (JPMF) be:
\[
f_{X,Y}(x,y) =
\begin{cases}
\frac{1}{4} & \text{ if } (x,y)=(0,0)\\
\frac{1}{4} & \text{ if } (x,y)=(0,1)\\
\frac{1}{4} & \text{ if } (x,y)=(1,0)\\
\frac{1}{4} & \text{ if } (x,y)=(1,1)\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
Are $X$ and $Y$ independent?
\Answer
~\\
First derive the marginal PMF of $X$ and $Y$ and then check if the JPMF is the product of the marginal PMFs.
\[
f_X(0) = \sum_{y \in \mathcal{S}_{X,Y}} f_{X,Y}(0,y) = f_{X,Y}(0,0) + f_{X,Y}(0,1) = \frac{1}{4}+\frac{1}{4}=\frac{1}{2}
\]
and
\[
f_X(1) = \sum_{y \in \mathcal{S}_{X,Y}} f_{X,Y}(1,y) = f_{X,Y}(1,0) + f_{X,Y}(1,1) = \frac{1}{4}+\frac{1}{4}=\frac{1}{2}
\]
Thus,
\[
f_{X}(x) =
\begin{cases}
\frac{1}{2} & \text{ if } x = 0\\
\frac{1}{2} & \text{ if } x = 1\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
Similarly,
\[
f_{Y}(y) =
\begin{cases}
\sum_{x \in \mathcal{S}_{X,Y}} f_{X,Y}(x,0)=f_{X,Y}(0,0) + f_{X,Y}(1,0) = \frac{1}{4}+\frac{1}{4}=\frac{1}{2} & \text{ if } y = 0\\
\sum_{x \in \mathcal{S}_{X,Y}} f_{X,Y}(x,1)=f_{X,Y}(0,1) + f_{X,Y}(1,1) = \frac{1}{4}+\frac{1}{4}=\frac{1}{2} & \text{ if } y = 1\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
Finally, the product of $f_X(x)$ and $f_Y(y)$ is
\[
f_{X}(x) \times f_{Y}(y) =
\begin{cases}
\frac{1}{2} \times \frac{1}{2}=\frac{1}{4} & \text{ if } (x,y)=(0,0)\\
\frac{1}{2} \times \frac{1}{2}=\frac{1}{4} & \text{ if } (x,y)=(0,1)\\
\frac{1}{2} \times \frac{1}{2}=\frac{1}{4} & \text{ if } (x,y)=(1,0)\\
\frac{1}{2} \times \frac{1}{2}=\frac{1}{4} & \text{ if } (x,y)=(1,1)\\
0 & \text{ otherwise} \enspace .
\end{cases}
\]
which in turn is equal to the JPMF $f_{X,Y}(x,y)$ in the question.  Therefore we have shown that the component RVs $X$ and $Y$ in the \rv~$(X,Y)$ are indeed indepedent.

\Exercise
A semiconductor product consists of three layers that are fabricated independently.
If the variances in thickness of the first, second and third third layers are $25$, $40$ and $30$ nanometers squared, what is the variance of the thickness of the final product?
\Answer
~\\
Let $X_1$, $X_2$, $X_3$ be independent RVs that denote the thickness of the first, second and third layer, respectively.  Let $X$ denote the thickness of the final product.  Then
\[
X= X_1+X_2+X_3
\]
By the property that $V\left(\sum_{i=1}^n a_i X_i\right) = \sum_{i=1}^n a_i^2 V(X_i)$, Variance of $X$ is
\[
V(X) = 1^2 V(X_1) + 1^2 V(X_2) + 1^2 V(X_3) = 25+40+30 = 95 nm^2 \enspace .
\]
This shows how the variance in each layer is propagated to the variance of the final product.

\Exercise
Find the covariance for the discrete \rv~$(X,Y)$ with joint probability mass function
\[
f_{X,Y}(x,y) =
\begin{cases}
0.2 & \text{ if } (x,y)=(0,0)\\
0.1 & \text{ if } (x,y)=(1,1)\\
0.1 & \text{ if } (x,y)=(1,2)\\
0.1 & \text{ if } (x,y)=(2,1)\\
0.1 & \text{ if } (x,y)=(2,2)\\
0.4 & \text{ if } (x,y)=(3,3)\\
0 & \text{ otherwise } \enspace .
\end{cases}
\]
[Hint: Recall that $\cv(X,Y)=E(XY)-E(X)E(Y)$]
\Answer
~\\
Find $E(XY)$, $E(X)$ and $E(Y)$ to get $\cv(X,Y)=E(XY)-E(X)E(Y)$ as follows:
\begin{multline*}
E(XY) = \sum_{(x,y) \in \mathcal{S}_{X,Y}} x \times y \times f_{X,Y}(x,y) \\
= 0 \times 0 \times 0.2 + 1 \times 1 \times 0.1+ 1 \times 2 \times 0.1 + 2 \times 1 \times 0.1 + 2 \times 2 \times 0.1 + 3 \times 3 \times 0.4 = 4.5
\end{multline*}
\begin{multline*}
E((X,Y)) = \sum_{(x,y) \in \mathcal{S}_{X,Y}} (x , y) \times f_{X,Y}(x,y) \\
= (0 , 0) \times 0.2 + (1 , 1) \times 0.1 + (1 , 2) \times 0.1 + (2 , 1) \times 0.1 + (2 , 2) \times 0.1 + (3 , 3) \times 0.4\\= (0,0)+(0.1,0.1)+(0.1,0.2)+(0.2,0.1)+(0.2,0.2)+(1.2,1.2) \\
= (1.8,1.8)
\end{multline*}
Since addition is component-wise $E((X,Y))=(E(X),E(Y))$ and therefore $E(X)=E(Y)=1.8$.

Alternatively, you can first find the marginal PMFs $f_X$ and $f_Y$ for $X$ and $Y$ and then take the expectations $E(X)=\sum_x x \times f_X(x)$ and $E(Y)=\sum_y y \times f_Y(y)$.

Finally,
\[
\cv(X,Y) = E(XY)-E(X)E(Y) = 4.5 - 1.8^2 = 1.26 \enspace .
\]

\Exercise
Consider two random variables (RVs) $X$ and $Y$ having marginal distribution functions
\[
F_X(x) =
\begin{cases}
0 & \text{ if } x < 1\\
\frac{1}{2} & \text{ if } 1 \leq x < 2\\
1 & \text{ if } x \geq 2\\
\end{cases}
\]
\[
F_Y(y) =
\begin{cases}
0 & \text{ if } y < 0\\
1-\frac{1}{2}e^{-y}-\frac{1}{2}e^{-2y} & \text{ if } y \geq 0\\
\end{cases}
\]
If $X$ and $Y$ are independent, what is their joint distribution function $F_{X,Y}(x,y)$? [Hint: you need to express $F_{X,Y}(x,y)$ for any $(x,y) \in \Rz^2$.]
\Answer
~\\
Since $X$ and $Y$ are independent, $F_{X,Y}(x,y) = F_X(x) F_Y(y)$ for all $(x,y) \in \Rz^2$, and we get:
\[
F_{X,Y}(x,y) =
\begin{cases}
0 & \text{ if } x < 1 \text{ or } y < 0\\
\frac{1}{2}\left(1-\frac{1}{2}e^{-y}-\frac{1}{2}e^{-2y}\right) & \text{ if } 1 \leq x < 2 \text{ and } y \geq 0\\
1-\frac{1}{2}e^{-y}-\frac{1}{2}e^{-2y} & \text{ if } x \geq 2 \text{ and } y \geq 0
\end{cases}
\]
{\small
You can arrive at the answer by partitioning $x$-axis into $(-\infty,1)$, $[1,2)$ and $[2,\infty)$ where $F_X(x)$ takes distinct values.  Similarly, partition the $y$-axis into  $(-\infty,0)$ and $[0,\infty)$ where $F_Y(y)$ takes distinct values.
Now $(x,y)$ can take values in one of these $3 \times 2=6$ partitions of the $x \times y$ plane as follows (make a picture!):
\[
(-\infty,1) \times (-\infty,0), \,
[1,2) \times (-\infty,0), \,
[2,\infty) \times (-\infty,0), \,
(-\infty,1) \times [0,\infty), \,
[1,2) \times [0,\infty), \,
[2,\infty) \times [0,\infty) \enspace .
\]
Now work out what $F_{X,Y}(x,y) = F_X(x) F_Y(y)$ is for $(x,y)$ in each of the above six partitions of the plane and you will get the the expression for $F_{X,Y}(x,y)$ given above.
}

\Exercise
Let $(X,Y)$ be a continuous \rv~with joint probability density function (JPDF):
\[
f_{X,Y}(x,y) =
\begin{cases}
e^{-x} & \text{ if } x \in [0,\infty) \text{ and }  y \in [2,3]\\
0 & \text{otherwise} \enspace .
\end{cases}
\]
Are $X$ and $Y$ independent?
\Answer
~\\
First obtain marginal PDF of $Y$.  If $y \in [2,3]$ then
\[
f_Y(y) = \int_{-\infty}^{\infty} f_{X,Y}(x,y) dx = \int_0^{\infty} e^{-x} dx = \left[ -e^{-x} \right]_0^{\infty} = 0 - (-1)=1 \enspace .
\]
Therefore,
\[
f_Y(y) =
\begin{cases}
1 & \text{ if } y \in [2,3]\\
0 & \text{otherwise} \enspace .
\end{cases}
\]
Now, obtain the marginal PDF of $X$.  If $x \in [0,\infty)$ then
\[
f_X(x) = \int_{-\infty}^{\infty} f_{X,Y}(x,y) dy = \int_2^3 e^{-x} dy  = e^{-x} \int_2^3 1 dy = e^{-x} \left[ y \right]_2^3 = e^{-x} (3-2) = e^{-x} \enspace .
\]
Therefore,
\[
f_X(x) =
\begin{cases}
e^{-x} & \text{ if } x \in [0,\infty)\\
0 & \text{otherwise} \enspace .
\end{cases}
\]
Finally, verifying that $f_{X,Y}(x,y) = f_X(x) f_Y(y)$ for any $(x,y) \in \Rz^2$ is done case by case.
Draw a picture on the plane to work out the cases from the distinct expressions taken by $f_{X,Y}(x,y)$.
There are only two cases to consider (when $f_{X,Y}(x,y)$ takes zero values and when $f_{X,Y}(x,y)$ takes non-zero values):
\be
\item~If $x \notin [0,\infty)$ or $y \notin [2,3]$ then $f_X(x)f_Y(y)=0=f_{X,Y}(x,y)$
\item~If $x \in [0,\infty)$ and $y \in [2,3]$ then $f_X(x)f_Y(y)=e^{-x} \times 1=e^{-x}=f_{X,Y}(x,y)$.
\ee
Thus, $X$ and $Y$ are independent.

\Exercise
In an electronic assembly, let the RVs $X_1,X_2,X_3,X_4$ denote the lifetimes of four components in hours.
Suppose that the JPDF of these variables is
\begin{align*}
~& f_{X_1,X_2,X_3,X_4}(x_1,x_2,x_3,x_4) \\
&\qquad=
\begin{cases}
9 \times 10^{-12} e^{-0.001 x_1 - 0.002 x_2 - 0.0015 x_3 - 0.003 x_4} & \text{ if } x_1 \geq 0, x_2 \geq 0, x_3 \geq 0, x_4 \geq 0\\
0 & \text{ otherwise} \enspace.
\end{cases}
\end{align*}
What is the probability that the device operates for more than 1000 hours without any failures? [Hint: The requested probability is $\P(X_1>1000,X_2>1000,X_3>1000,X_4>1000)$ since each one of the four components of the device must not fail before 1000 hours.]
\Answer
~\\
\begin{multline*}
\P(X_1>1000,X_2>1000,X_3>1000,X_4>1000) \\
= \int_{1000}^{\infty} \int_{1000}^{\infty} \int_{1000}^{\infty} \int_{1000}^{\infty} 9 \times 10^{-12} e^{-0.001 x_1 - 0.002 x_2 - 0.0015 x_3 - 0.003 x_4} dx_1 dx_2 dx_3 dx_4\\
= 9 \times 10^{-12} \int_{1000}^{\infty} \int_{1000}^{\infty} \int_{1000}^{\infty} \int_{1000}^{\infty}  e^{-0.001 x_1} e^{- 0.002 x_2} e^{- 0.0015 x_3} e^{- 0.003 x_4} dx_1 dx_2 dx_3 dx_4\\
=  9 \times 10^{-12} \int_{1000}^{\infty} e^{-0.001 x_1} \int_{1000}^{\infty} e^{- 0.002 x_2} \int_{1000}^{\infty} e^{- 0.0015 x_3} \int_{1000}^{\infty} e^{- 0.003 x_4} dx_4 dx_3 dx_2 dx_1\\
\end{multline*}
Since
\[
\int_{1000}^{\infty} e^{- a x_i} dx_i = \left[ \frac{e^{-a x_i}}{-a} \right]_{1000}^{\infty} = 0 + \frac{e^{-1000\times a}}{a} \enspace ,
\]
the above quadruply iterated integral becomes
\begin{align*}
&~ 9 \times 10^{-12} \times \frac{e^{-1000 \times 0.001}}{0.001} \times \frac{e^{-1000 \times 0.002}}{0.002}
\times \frac{e^{-1000 \times 0.0015}}{0.0015} \times \frac{e^{-1000 \times 0.003}}{0.003}\\
&= 9 \times 10^{-12} \times \frac{1000}{1} \times \frac{1000}{2} \times \frac{1000}{1.5} \times \frac{1000}{3} \times
e^{-1} \times e^{-2} \times e^{-1.5} \times e^{-3} \\
&= 9 \times 10^{-12} \times \frac{1}{9} \times 10^{12} \times e^{-7.5} = e^{-7.5} \approxeq 0.00055 \enspace .
\end{align*}

\Exercise
Suppose the RVs $Y_1$, $Y_2$ and $Y_3$ represent the thickness in micrometers of a substrate, an active layer, and a coating layer of a chemical product.
Assume $Y_1$, $Y_2$ and $Y_3$ are $\normal(10000,250^2)$, $\normal(1000,20^2)$ and $\normal(80,4^2)$ RVs, respectively.
Further suppose that they are independent.
The required specifications for the thickness of the substrate, active layer and coating layer are $[9500,10500]$, $[950,1050]$ and $[75,85]$, respectively.
What proportion of chemical products meets all thickness specifications? [Hint: this is just $\P(9500<Y_1<10500,950<Y_2<1050,75<Y_3<85)$]  Which one of the three thicknesses has the least probability of meeting specifications?
\Answer
~\\
Due to independence of $Y_1$, $Y_2$ and $Y_3$
\begin{multline*}
\P(9500<Y_1<10500,950<Y_2<1050,75<Y_3<85)\\ = \P(9500<Y_1<10500) \P(950<Y_2<1050) \P(75<Y_3<85)
\end{multline*}
After standardizing each Normal RV (subtracting its mean and dividing by its standard deviation) we get
\begin{align*}
&~ \P(9500<Y_1<10500) \P(950<Y_2<1050) \P(75<Y_3<85)\\
&=  P\left(\frac{9500-10000}{250}< Z < \frac{10500-10000}{250} \right) P\left( \frac{950-1000}{20}<Z<\frac{1050-1000}{20}\right)\\
&\qquad \qquad \qquad \qquad \qquad \qquad P\left(\frac{75-80}{4}<Z<\frac{85-80}{4}\right)\\
&= \P(-2.0<Z<2.0) \P(-2.5<Z<2.5) \P(-1.25<Z<1.25) \\
&= (\Phi(2.0)-(1-\Phi(2.0)) \times (\Phi(2.5)-(1-\Phi(2.5)) \times (\Phi(1.25)-(1-\Phi(1.25))\\
&= (2\Phi(2.0)-1) \times (2\Phi(2.5)-1) \times (2\Phi(1.25)-1)\\
&= ((2 \times 0.9772) -1) \times ((2 \times 0.9938) - 1) \times ((2 \times 0.8944)-1) \qquad \text{ using Table for $\Phi(z)$} \\
&= 0.9544 \times 0.9876 \times 0.7888 = 0.7435
\end{align*}
%To follow the fourth-last equality above see Example 8.14(d) from EMTH119. % TODO?
The values for the distribution function $\Phi(z)$ of the $\normal(0,1)$ RV $Z$ are in the table on page 67.

The thickness of the coating layer represented by $Y_3$ has the least probability ($0.7888$) of meeting specifications.  Consequently, a priority should be to reduce variability in this part of the process.

\Exercise
Soft drink cans are filled by an automated filling machine.
Assume the fill volumes of the cans are independent $\normal(12.1,0.01)$ RVs.
What is the probability that the average volume of ten cans selected from this process is less than $12.01$ fluid ounces?
\Answer
~\\
Let $X_1,X_2,\ldots,X_{10}$ denote the fill volumes of $10$ cans.  The average fill volume is the sample mean
\[
\ol{X}_{10} = \frac{1}{10} \sum_{i=1}^{10} X_i
\]
By property of Expectations and Variances for linear combinations
\[
E(\ol{X}_{10}) = E \left(\frac{1}{10} \sum_{i=1}^{10} X_i \right) = \frac{1}{10}\sum_{i=1}^nE(X_i) = \frac{1}{10}\sum_{i=1}^nE(X_1) = \frac{1}{10} \times 10 \times E(X_1) = E(X_1) = 12.1
\]
Or by directly using the ``formula'' $E(\ol{X}_{10}) = E(X_1)=12.1$ for these $10$ identically distributed RVs.
Similarly,
\[
V\left(\ol{X}_{10}\right) = V \left(\frac{1}{10} \sum_{i=1}^{10} X_i \right) = 10 \times \frac{1}{10^2} V(X_1) = \frac{1}{10} \times 0.01 = 0.001
\]
Or by directly using the ``formula'' $V(\ol{X}_{10}) = V(X_1)/10$ for these $10$ independently and identically distributed RVs.

By the special property of Normal RVs -- a linear combination of independent normal RVs is also normal -- we know that
$\ol{X}_{10}$ is a $\normal(12.1,0.001)$ RV.
Consequetly, the probability of interest is
\begin{align*}
\P(\ol{X}_{10} < 12.01)
&= P \left( \frac{\ol{X}_{10}-E(\ol{X}_{10})}{\sqrt{0.001}} < \frac{12.01 - E(\ol{X}_{10})}{\sqrt{0.001}} \right) = P\left(Z < \frac{12.01-12.1}{0.0316}\right)\\
&\approxeq \P(Z<-2.85) = 1-\P(Z<2.85) = 1-\Phi(2.85) = 1-0.9978=0.0022
\end{align*}

\Exercise
Let $X_1,X_2,X_3,X_4$ be RVs that denote the number of bits received in a digital channel that are classified as {\em excellent}, {\em good}, {\em fair} and {\em poor}, respectively.
In a transmission of $10$ bits, what is the probability that $6$ of the bits received are {\em excellent}, $2$ are {\em good}, $2$ are {\em fair} and none are {\em poor} under the assumption that the classification of bits are independent events and that the probabilities of each bit being {\em excellent}, {\em good}, {\em fair} and {\em poor} are $0.6$, $0.3$, $0.08$ and $0.02$, respectively.
[Hint: Think of $\multinomial(n=10,\theta_1=0.6,\theta_2=0.3,\theta_3=0.08,\theta_4=0.02)$ as a model for bit classification in this digital channel.]
\Answer
~\\
Using the $\multinomial(n=10,\theta_1=0.6,\theta_2=0.3,\theta_3=0.08,\theta_4=0.02)$ \rv~as our model
\begin{align*}
&~ P\left( (X_1,X_2,X_3,X_4)=(6,2,2,0); n=10,\theta_1=0.6,\theta_2=0.3,\theta_3=0.08,\theta_4=0.02 \right)\\
&= \frac{10!}{6! \times 2! \times 2! \times 0!} \times 0.6^6 \times 0.3^2 \times 0.08^2 \times 0.02^0\\
&= \frac{10 \times 9 \times 8 \times 7 \times 6 \times 5 \times 4 \times 3 \times 2 \times 1}{(6 \times 5 \times 4 \times 3 \times 2 \times 1) \times (2 \times 1) \times (2 \times 1) \times 1} \times  0.6^6 \times 0.3^2 \times 0.08^2 \times 1 \approxeq 0.03386\\
\end{align*}

\end{ExerciseList}


\subsubsection{Repeated Quadratic Factors}
\noindent
If a quadratic factor that can't be broken into linear factors is repeated, then we can write
$Q(x) = R(x)(ax^2+bx+c)^k$, $k \geq 0$, and $R(x)$ is not divisible by $(ax^2+bx+c)^k$.
Now we have to do a combination of what we did for repeated linear factors and quadratic factors.
We say
\begin{equation*}
	\frac{P(x)}{R(x)(ax^2+bx+c)^k}=\left(\text{Decomposition of }R(x)\right)+\frac{A_1x+B_1}{ax^2+bx+c}+\ldots+\frac{A_kx+B_k}{(ax^2+bx+c)^k}.
\end{equation*}
We then solve for the coefficients in the numerator.

\begin{example}
	Find the partial fraction decomposition of $\frac{3x^4-2x^3+6x^2-3x+3}{x^5+3x^4+4x^3+12x^2+4x+12}$.
\end{example}
\noindent
Factoring,
\begin{equation*}
	x^5+3x^4+4x^3+12x^2+4x+12 = (x+3)(x^2+2)^2.
\end{equation*}
So,
\begin{equation*}
	\frac{3x^4-2x^3+6x^2-3x+3}{x^5+3x^4+4x^3+12x^2+4x+12} = \frac{A_1}{x+3}+\frac{A_2x+B_2}{x^2+2}+\frac{A_3x+B_3}{(x^2+2)^2}.
\end{equation*}
Multiplying each side by the denominator,
\begin{equation*}
	3x^4-2x^3+6x^2-3x+3 = A_1(x^2+2)^2+(A_2x+B_2)(x^2+2)(x+3)+(A_3x+B_3)(x+3).
\end{equation*}
At $x=-3$,
\begin{equation*}
	363 = 121A_1 \implies A_1 = 3.
\end{equation*}
Now, we'll use our result for $A_1$ and pick a value for $x$ that minimizes the number of things we need to solve for. We'll have to solve a linear system with 4 unknowns, so we'll need up to 4 values.
At $x=0$,
\begin{equation*}
	3 = 3(2)^2 + B_2(2)(3)+B_3(3) \implies 2B_2 + B_3 = -3.
\end{equation*}
At $x=1$,
\begin{equation*}
	7 = 3(3)^2 + (A_2+B_2)(3)(4) + (A_3+B_3)(4) \implies 3A_2 + A_3 + 3B_2 + B_3 = -5.
\end{equation*}
At $x=-1$,
\begin{equation*}
	17 = 3(3)^2 + (-A_2+B_2)(3)(2) + (-A_3+B_3)(2) \implies -3A_2 - A_3 + 3B_2 + B_3 = -5.
\end{equation*}
At $x=2$,
\begin{equation*}
	53 = 3(6)^2 + (2A_2+B_2)(6)(5) + (2A_3+B_3)(5) \implies 12A_2 + 2A_3 + 6B_2 + B_3 = -11.
\end{equation*}
Now we have the following system of equations:
\begin{equation*}
	\begin{cases}
		0A_2  + 0A_3 + 2B_2 + B_3 &= -3 \\
		3A_2  + A_3  + 3B_2 + B_3 &= -5 \\
		-3A_2 - A_3  + 3B_2 + B_3 &= -5 \\
		12A_2 + 2A_3 + 6B_2 + B_3 &= -11
	\end{cases}.
\end{equation*}
Solving,
\begin{equation*}
	A_2 = 0, A_3 = 0, B_2 = -2, \text{ and } B_3 = 1.
\end{equation*}
So,
\begin{equation*}
	\frac{3x^4-2x^3+6x^2-3x+3}{x^5+3x^4+4x^3+12x^2+4x+12} = \frac{3}{x+3}-\frac{2}{x^2+2}+\frac{1}{(x^2+2)^2}.
\end{equation*}


\section{Solving PDEs with Green's Functions}
\subsection{Diffusion Equation and Fourier Transform}
Recall that the heat equation for a conducting wire is
$$\frac{\partial\Theta}{\partial t}-D\frac{\partial^2\Theta}{\partial x^2}$$
with initial condition $\Theta(x,0)=h(x)$ and boundary condition $\Theta\to 0$ as $x\to\pm\infty$.
Taking Fourier transform wrt $x$ gives
$$\frac{\partial}{\partial t}\tilde{\Theta}(k,t)=-Dk^2\tilde\Theta(k,t)\implies\tilde\Theta(k,t)=Ce^{-Dk^2t}$$
for some $C$ that is constant in $t$.
Initial condition $\tilde{\Theta}(k,0)=\tilde{h}(k)$ gives $\tilde\Theta(k,t)=\tilde{h}(k)e^{-Dk^2t}$.
Inverting it gives
\begin{align*}
    \Theta(x,t)&=\frac{1}{2\pi}\int_{-\infty}^\infty \tilde{h}(k)e^{-Dk^2t}e^{ikx}\,\mathrm dk\\
    &=\frac{1}{\sqrt{4\pi Dt}}\int_{-\infty}^\infty h(u)\exp\left( -\frac{(x-u)^2}{4Dt} \right)\,\mathrm du\\
    &=\int_{-\infty}^\infty h(u)S_d(x-u,t)\,\mathrm du
\end{align*}
where $S_d(x,t)=(4\pi Dt)^{-1/2}e^{-x^2/(4Dt)}$ is called the fundamental solution (or diffusion kernel/sourse function), which is just the FT of $e^{-Dk^2t}$.
Note that the case $\Theta(x,0)=\theta_0\delta(x)$ gives our old friend $\Theta=\theta_0(4\pi Dt)^{-1/2}e^{-\eta^2}$ where $\eta=x/(2\sqrt{Dt})$ is the similarity parameter.
\begin{example}
    Suppose initially $f(x)=\theta_0\sqrt{a/\pi}e^{-ax^2}$.
    Then
    \begin{align*}
        \Theta&=\frac{\theta_0\sqrt{a}}{\sqrt{4\pi^2Dt}}\int_{-\infty}^\infty\exp\left( -au^2-\frac{(x-u)^2}{4Dt} \right)\,\mathrm du\\
        &=\frac{\theta_0\sqrt{a}}{\sqrt{4\pi^2Dt}}\int_{-\infty}^\infty\exp\left( -\frac{1+4aDt}{4Dt}\left( u-\frac{x}{1+4aDt} \right)^2 \right)\exp\left( \frac{-ax^2}{1+4aDt} \right)\,\mathrm du\\
        &=\theta_0\sqrt{\frac{a}{\pi(1+4aDt)}}\exp\left( \frac{-ax^2}{1+4aDt} \right)
    \end{align*}
    Asymptotically the width of the Gaussian spreads as $\sqrt{t}$ and the area under the curve being constant (which can be interpreted as the conservation of heat energy).
\end{example}
\subsection{Forced Diffusion Equation}
Consider
$$\frac{\partial\Theta}{\partial t}(x,t)-D\frac{\partial^2\Theta}{\partial x^2}(x,t)=f(x,t)$$
with homogeneous initial condition $\Theta(x,0)=0$.
The Green's function $G$ of this problem would satisfy
$$\frac{\partial G}{\partial t}-D\frac{\partial^2G}{\partial x^2}=\delta(x-\xi)\delta(t-\tau)$$
with $G(x,0;\xi,\tau)=0$.
Take FT wrt $x$ gives
$$\frac{\partial \tilde{G}}{\partial t}(k,t;\xi,\tau)+Dk^2\tilde{G}(k,t;\xi,\tau)=e^{-ik\xi}\delta(t-\xi)$$
Multiplying both sides using the integration factor $e^{Dk^2t}$ allows us to integrate the equation and obtain
$$e^{Dk^2t}\tilde{G}=e^{-ik\xi}\int_0^te^{Dk^2t'}\delta(t'-\tau)\,\mathrm dt'$$
The appearance of Heaviside function is because we need to ensure that $[0,t]$ contains $\tau$ in order to make use of the property of $\delta$ function.
Inverting the whole thing gives
\begin{align*}
    G(x,t;\xi,\tau)&=\frac{H(t-\tau)}{2\pi}\int_{-\infty}^\infty e^{ik(x-\xi)}e^{-Dk^2(t-\tau)}\,\mathrm dk\\
    &=\frac{H(t')}{2\pi}\int_{-\infty}^\infty e^{ikx'}e^{-Dk^2t'}\,\mathrm dk,x'=x-\xi,t'=t-\tau\\
    &=\frac{H(t')}{\sqrt{4\pi Dt'}}e^{-x'^2/(4Dt')}\\
    &=H(t-\tau)S_d(x-\xi,t-\tau)
\end{align*}
where $S_d$ is again our good ol' fundamental solution.
Therefore the general solution is
\begin{align*}
    \Theta(x,t)&=\int_0^\infty\int_{-\infty}^\infty G(x,t;\xi,\tau)f(\xi,\tau)\,\mathrm d\xi\mathrm d\tau\\
    &=\int_0^t\int_{-\infty}^\infty f(u,\tau)S_d(x-u,t-\tau)\,\mathrm du\mathrm d\tau
\end{align*}
which looks very familiar to our previous solution for the homogeneous case with initial conditions at $t=\tau$.
But this time, the initial condition of $\Theta(u,t)=f(u)$ at $t=\tau$ is replaced by the forcing term $f(u,\tau)$ in the equation, and the effect of this, as we see now, is just integrating the solution over $\tau$.
This phenomenon is called Duhamel's Principle which relates the solution of forced PDE with homogeneous boundary conditions to solutions of homogeneous PDEs with inhomogenoeus boundary conditions.
So in this philosophy, the forcing term is acting like an initial conditions for subsequent evolution, and the integral represents a superposition of all these initial-condition-like effects for $0<\tau<t$.
\subsection{Forced Wave Equation}
Consider the forced wave equation
$$\frac{\partial^2\phi}{\partial t^2}-c^2\frac{\partial^2\phi}{\partial x}=f(x,t)$$
subject to $\phi(x,0)=\phi_t(x,0)=0$.
The Green's function for this would satisfy
$$\frac{\partial^2\phi}{\partial t^2}-c^2\frac{\partial^2\phi}{\partial x}=\delta(x-\xi)\delta(t-\tau)$$
with $G=G_t=0$ at $t=0$.
Take FT wrt $x$ again,
$$\frac{\partial^2\tilde{G}}{\partial t^2}+c^2k^2\tilde{G}=e^{-ik\xi}\delta(t-\tau)$$
Solving it yields
$$\tilde{G}=\begin{cases}
    0\text{, if $t<\tau$}\\
    e^{-ik\xi}\sin(kc(t-\tau))/(kc)\text{, if $t>\tau$}
\end{cases}=e^{-ik\xi}\frac{\sin(kc(t-\tau))}{kc}$$
Inverting this gives
\begin{align*}
    G(x,t;\xi,\tau)&=\frac{H(t-\tau)}{2\pi c}\int_{-\infty}^\infty e^{ik(x-\xi)}\frac{\sin(kc(t-\tau))}{k}\,\mathrm dk\\
    &=\frac{H(t-\tau)}{2\pi c}\int_{-\infty}^\infty \frac{\cos(kA)\sin(kB)}{k}\,\mathrm dk,A=x-\xi,B=c(t-\tau)\\
    &=\frac{H(t-\tau)}{2\pi c}\int_{-\infty}^\infty \frac{\sin(k(A+B))-\sin(k(A-B))}{k}\,\mathrm dk\\
    &=\frac{H(t-\tau)}{2\pi c}(\operatorname{sgn}(A+B)-\operatorname{sgn}(A-B))\\
    &=\frac{1}{2c}H(c(t-\tau)-|x-\xi|)
\end{align*}
which is also called the causal fundamental solution since it hints the causal structure in $t$.
The general solution is then
\begin{align*}
    \phi(x,t)&=\int_0^\infty\int_{-\infty}^\infty f(\xi,t)G(x,t;\xi,\tau)\,\mathrm d\xi\mathrm d\tau\\
    &=\frac{1}{2c}\int_0^t\int_{x-c(t-\tau)}^{x+c(t-\tau)}f(\xi,\tau)\,\mathrm d\xi\mathrm d\tau
\end{align*}
which, by relating to our previous solution, can also be seen as an example of Dudamel's principle.
\subsection{Poisson's Equation}
We want to solve Poisson's equation, which is just a forced Laplace equation $\nabla^2\phi=-\rho$ on a domain $D$ subject to Dirichlet boundary conditions $\phi|_{\partial D}=0$.
To get its fundamental solution, note that we can get the notion of $\delta$ function on $\mathbb R^3$ analogously.
So the free space Green's function $G=G_{\rm FS}(\underline{r};\underline{r}')$ is naturally the solution to $\nabla^2G_{\rm FS}(\underline{r};\underline{r}')=\delta(\underline{r}-\underline{r}')$.
We can take this to be spherically symmetric $G(\underline{r};\underline{r}')=G(|\underline{r}-\underline{r}'|)=G(r)$.
Let $B$ be the ball with centre $\underline{r}'$ and radius $r$, then we have
$$1=\int_B\delta(\underline{r}-\underline{r}')\,\mathrm d^3\underline{r}=\int_B\nabla^2G_{\rm FS}\,\mathrm d^3r=\int_{\partial B}\nabla G_{\rm FS}\cdot\underline{n}\,\mathrm dS=4\pi r^2\frac{\partial G_{\rm FS}}{\partial r}$$
So $G_{\rm FS}=-1/(4\pi r)=-1/(4\pi|\underline{r}-\underline{r}'|)$ because of the boundary condition $G\to 0$ as $r\to\infty$.
Therefore we get the general solution
$$\Phi(\underline{r})=\frac{1}{4\pi}\int_D\frac{\rho(\underline{r}')}{|\underline{r}-\underline{r}'|}\,\mathrm d^3\underline{r}'$$
By the way, we can derive the Green's function in 2D in the similar way and get $G_2(\underline{r};\underline{r}')=-(2\pi)^{-1}\log(|\underline{r}-\underline{r}'|)+C_2$ where $C_2$ is a constant (which we almost always set to $0$).\\
We now turn to Green's Identities.
Consider two scalar functions $\phi,\psi$ twice differentiable on $D$.
Then
$$\int_D(\phi\nabla^2\psi+\nabla\phi\cdot\nabla\psi)\,\mathrm d^3\underline{r}=\int_D\nabla\cdot(\phi\nabla\psi)\,\mathrm d^3\underline{r}=\int_{\partial D}\phi\nabla\psi\cdot\underline{\hat{n}}\,\mathrm dS$$
by divergence theorem.
This is called Green's first identity.
Now switch $\psi$ and $\phi$ and substract from the first identity to get Green's second identity.
$$\int_{\partial D}\left( \phi\frac{\partial\psi}{\partial n}-\psi\frac{\partial\phi}{\partial n} \right)\,\mathrm dS=\int_D(\phi\nabla^2\psi-\psi\nabla^2\phi)\,\mathrm d^3\underline{r}$$
Now excise a small spherical ball $B_\epsilon$ around $\underline{r}'$ with radius $\epsilon$.
Now if $\phi$ is a solution to $\nabla^2\phi=-\rho$ and $\psi=G_{\rm FS}(\underline{r};\underline{r}')$, then the right hand side is
$$\int_{D-B_\epsilon}(\phi\nabla^2G_{\rm FS}-G_{\rm FS}\nabla^2\phi)\,\mathrm d^3r=\int_{D-B_\epsilon}G_{\rm FS}\rho\,\mathrm d^3\underline{r}$$
and the left hand side becomes
$$\int_{\partial D}\left( \phi\frac{\partial G_{\rm FS}}{\partial n}-G_{\rm FS}\frac{\partial\phi}{\partial n} \right)\,\mathrm dS+\int_{\partial B_\epsilon}\left( \phi\frac{\partial G_{\rm FS}}{\partial n}-G_{\rm FS}\frac{\partial\phi}{\partial n} \right)\,\mathrm dS$$
Taking $\epsilon\to 0$ gives
$$\int_{\partial B_\epsilon}\left( \phi\frac{\partial G_{\rm FS}}{\partial n}-G_{\rm FS}\frac{\partial\phi}{\partial n} \right)\,\mathrm dS\to-\phi(\underline{r}')$$
Therefore we conclude Green's third identity
$$\phi(\underline{r}')=\int_DG_{\rm FS}(\underline{r};\underline{r}')(-\rho(\underline{r}))\,\mathrm d^3\underline{r}+\int_{\partial D}\left( \phi(\underline{r})\frac{\partial G}{\partial n}(\underline{r};\underline{r}')-G_{\rm FS}(\underline{r};\underline{r}')\frac{\partial\phi}{\partial n}(\underline{r}) \right)\,\mathrm dS$$
Now we want to solve $\nabla^2\phi=-\rho$ on $D$ subject to inhomogeneous Dirichlet boundary conditions $\phi|_{\partial D}=h$.
In this case, we want the Green's function $G=G(\underline{r};\underline{r}')$ to satisfy:\\
(i) $\nabla^2G(\underline{r};\underline{r}')=0$ for any $\underline{r}\neq\underline{r}'$.\\
(ii) $G(\underline{r};\underline{r}')=0$ on $\partial D$.\\
(iii) $G(\underline{r},\underline{r}')=G_{\rm FS}(\underline{r};\underline{r}')+H(\underline{r},\underline{r}')$ for some $H$ such that $\nabla^2H=0$ in $D$.\\
Green's second identity with $\nabla^2\phi=-\rho$ and $\nabla^2H=0$ gives
$$\int_{\partial D}\left( \phi\frac{\partial H}{\partial n}-H\frac{\partial\phi}{\partial n} \right)\,\mathrm dS=\int_DH\rho\,\mathrm d^3\underline{r}$$
The Green's third identity simplifies to
\begin{align*}
    \phi(\underline{r}')&=\int_D(G-H)(-\rho)\,\mathrm d^3\underline{r}+\int_{\partial D}\left( \phi\frac{\partial (G-H)}{\partial n}-(G-H)\frac{\partial\phi}{\partial n} \right)\,\mathrm dS\\
    &=\int_DG(\underline{r};\underline{r}')(-\rho(\underline{r}))\,\mathrm d^3\underline{r}+\int_{\partial D}h(\underline{r})\frac{\partial G(\underline{r};\underline{r}')}{\partial n}\,\mathrm dS
\end{align*}
So this Green's function does give a way to obtain a particular solution.
Also $G(\underline{r};\underline{r}')=G(\underline{r};\underline{r}')$ by a simple use of Green's third identity again.\\
For Neumann boundary conditions with $\partial\phi/\partial n=k(\underline{r})$ on $\partial D$, we can do a similar thing and obtain
$$\phi(\underline{r}')=\int_DG(\underline{r};\underline{r}')(-\rho(\underline{r}))\,\mathrm d^3\underline{r}+\int_{\partial D}G(\underline{r};\underline{r}')(-k(\underline{r}))\,\mathrm dS$$
\subsection{Method of Images}
For symmetric domain $D$ we can construct Green's function with $G=0$ on $\partial D$ by cancelling the boundary potential with an opposite mirror image Green's function placed outside $D$.\\
Our first target is Laplace's equation on the half-plane $D=\{(x,y,z):z>0\}$ subject to $\phi(x,y,0)=h(x,y)$ and $\phi\to 0$ as $|\underline{r}|\to\infty$.
Now $G_{\rm FS}(\underline{r};\underline{r}')\to 0$ as $|\underline{r}|\to\infty$, but it is nonzero at $z=0$.
However, we can take $G(\underline{r};\underline{r}')=G(\underline{r};\underline{r}')-G(\underline{r},\underline{r}'')$ where $\underline{r}''=(x',y',-z')$ which totally works.
Also at $z=0$, $\partial G/\partial n=\partial G/\partial z=(2\pi)^{-1}z'((x-x')^2+(y-y')^2+z'^2)^{-3/2}$.
This means that the solution is
$$\Phi(x',y',z')=\frac{z'}{2\pi}\int_{-\infty}^\infty\int_{-\infty}^\infty((x-x')^2+(y-y')^2+z'^2)^{-3/2}h(x,t)\,\mathrm dx\mathrm dy$$
We now turn to the wave equation for $x>0$.
Consider $\ddot{\phi}-c^2\phi^{\prime\prime}=f$ with Dirichlet boundary conditions $\phi(0,t)=0$.
The same philosophy gives the Green's function
$$G(x,t;\xi,\tau)=\frac{1}{2c}H(c(t-\tau)-|x-\xi|)-\frac{1}{2c}H(c(t-\tau)-|x+\xi|)$$
So if $f=0$ and the initial condition is a Gaussian pause, then this solves to
$$\phi(x,t)=\exp((x-\xi+ct)^2)-\exp((-x-\xi+ct)^2)$$


\lab{Conservation laws and heat flow}{Conservation laws and heat flow}
\label{lab:finitedifference1}

A conservation law is a balance law, and corresponds to an equation that describes how a quantity is balanced in some system throughout a given process.
(Consider how this is related to conservation laws in physics.)
For example, suppose we are keeping track of some measurable quantity in a physical system (e.g. heat, water, etc).
The fundamental conservation law then states that the rate of change of the total quantity in the system is equal to the rate of the quantity flowing into the system plus the rate at which the quantity is produced by sources inside the system.

\section*{Derivation of the Conservation equation in multiple dimensions}
Suppose $\Omega$ is a region in $\mathbb{R}^n$, and $V \subset \Omega$ is bounded with a reasonably well-behaved boundary $\partial V$.
Let $u(\vec{x},t)$ represent the density (concentration) of some quantity throughout $\Omega$.
Let $\vec{n}(x)$ represent the normal direction to $V$ at $x \in \partial V$, and let $\vec{J}(\vec{x},t)$ be the flux vector for the quantity, so that $\vec{J}(\vec{x},t) \cdot \vec{n}(x) \, dA$ represents the rate at which the quantity leaves $V$ by crossing a boundary element with area $dA$.
Note that the total amount of the quantity in $V$ is
\[ \int_V u(\vec{x},t)\, dt,\]
and the rate at which the quantity enters $V$ is
\[-\int_{\partial V} \vec{J}(\vec{x},t) \cdot \vec{n}(x) \, dA.\]

We let the source term be given by $f(\vec{x},t,u)$; we may interpret this to mean that the rate at which the quantity is produced in $V$ is
\[\int_V f(\vec{x},t,u)\, dt.\]
Then the integral form of the conservation law for $u$ is expressed as
\[\frac{d}{dt} \int_V u(\vec{x},t) \, d\vec{x} = -\int_V \vec{J}\cdot \vec{n}\, dA + \int_V f(\vec{x},t,u)\, d\vec{x}.\]

If $u$ and $J$ are sufficiently smooth functions, then we have
\[ \frac{d}{dt} \int_V u\, d\vec{x} = \int_V u_t \, d\vec{x},\]
and
\[ \int_V \vec{J}\cdot \vec{n}\, dA = \int_V \nabla \cdot \vec{J}\, d\vec{x} .\]
Since this holds for all nice subsets $V \subset \Omega$ with $V$ arbitrarily small, we obtain the differential form of the conservation law for $u$:
\[ u_t + \nabla \cdot \vec{J} = f(\vec{x},t,u) .\]

\section*{Constitutive Relations}
Currently our conservation law appears in the form
\[u_t + \nabla \vec{J} = f(\vec{x},t,u).\]
Thus the conservation law consists of one equation and 2 unknowns ($u$ and $J$).
To this equation we add other equations, called constituve relations, which are used to fully determine the system.

For example, suppose we wish to describe the flow of heat.
Since heat flows from warmer regions to colder regions, and the rate of heat flow depends on the difference in temperature between regions, we usually assume that the flux vector $\vec{J}$ is given by
\[\vec{J}(x,t) = -\nu \nabla u(x,t),\]
where $\nu$ is a diffusion constant.
This constitutive relation is called Fick's law, and is the basic model for any diffusive process.
Substituting into the conservation law we obtain
\[u_t -\nu \triangle u(x,t) = f(\vec{x},t,u).\]
The function $f$ represents heat sources/sinks within the region.

\section*{Numerically modelling heat flow}
Consider the heat flow equation in one dimension together with appropriate initial conditions and homogeneous Dirichlet boundary conditions:
\begin{align*}
	&{ } u_t = \nu u_{xx}, \quad x \in [a,b],\quad t \in [0,T], \\
	&{ } u(a,t) = 0,\quad u(b,t) = 0,\\
	&{ } u(x,0) = f(x).
\end{align*}
We will look for an approximation $U^j_i$ to $u(x_i,t_j)$ on the grid $x_i = a +  hi$, $t_j = kj$, where $h$ and $k$ are small changes in $x$ and $t$ respectively and $i$ and $j$ are indices.
Note that the index $i$ ranges over different spacial grid points and the index $j$ ranges over different time steps.
We will denote the approximate value of $u$ at the $i$'th grid point and the $j$'th time step as $U_i^j$.

A common method for modelling ordinary and partial differential equations is the finite difference method, so-named because equations containing derivatives are replaced with equations containing difference schemes.
These difference schemes can often be found using Taylor's theorem.
For example, the equation
\begin{align*}
	u(x,t_j + k) = u(x,t_j) + u_t(x,t_j)k + \mathcal{O}(k^2)
\end{align*}
yields a first-order forward difference approximation to $u_t(x,t_j)$, namely,
\begin{align*}
	u_t(x,t_j ) = \frac{u(x,t_j+k) - u(x,t_j)}{k} + \mathcal{O}(k).
\end{align*}
Similarly, by adding the equations
\begin{align*}
	u(x_i+h,t) &= u(x_i,t) + u_x(x_i,t)h + u_{xx}(x_i,t)\frac{h^2}{2} + u_{xxx}(x_i,t)h^3 + \mathcal{O}(h^4),\\
	u(x_i-h,t) &= u(x_i,t) + u_x(x_i,t)(-h) + u_{xx}(x_i,t)\frac{(-h)^2}{2} + u_{xxx}(x_i,t)(-h)^3 + \mathcal{O}(h^4),
\end{align*}
we obtain a second-order centered difference approximation to $u_{xx}(x_i,t)$:
\begin{align*}
	u_{xx}(x_i,t_j) &= \frac{u(x_i + h,t_j )-2 u(x_i,t_j)- u(x_i - h,t_j)}{h^2} + \mathcal{O}(h^2).
\end{align*}

\begin{figure}
\centering
\includegraphics[width=\textwidth]{heatexercise1a.pdf}
\caption{The graph of $U^{0}$, the approximation to the solution $u(x,t=0)$ for Problem \ref{prob:heat_exercise1}.}
\label{fig:heatexercise1a}
\end{figure}

% A simple explicit method for numerically approximating the solution of this problem is to use a forward difference approximation in time and a centered difference approximation in space.
% Recall that
% \[u_t(x_i,t_j) = \frac{u(x_i,t_j + k) - u(x_i,t_j)}{k} + \mathcal{O}(k)
% \]
%  and
% \[u_{xx}(x_i,t_j) = \frac{u(x_i + h,t_j )-2 u(x_i,t_j)- u(x_i - h,t_j)}{h^2} + \mathcal{O}(h^2).
% \]
% Let $U_{i,j}$ represent our approximation to $u(x_i,t_j)$.
These difference approximations give us the $\mathcal{O}(h^2 + k)$ explicit method
\begin{align}
	\begin{split}
	\frac{U_{i}^{j+1} - U_{i}^{j}}{k} &= \nu \frac{U_{i+1}^{j}- 2U_{i}^{j} + U_{i-1}^{j} }{h^2} ,\\
	U_{i}^{j+1} &= U_{i}^{j} + \frac{\nu k}{h^2} (U_{i+1}^{j}- 2U_{i}^{j} + U_{i-1}^{j} ).
	\end{split}\label{eqn:firstorder_explicit}
\end{align}
This method can be written in matrix form as
\[U^{j+1} = A U^j,\]
where $A$ is the tridiagonal matrix given by
\[A = \left[\begin{array}{cccccc}1-2\lambda & \lambda & & & \\ \lambda & 1-2\lambda & \lambda & & \\ & \ddots & \ddots & \ddots & \\ & & \lambda & 1-2\lambda & \lambda \\  &  &  & \lambda & 1-2\lambda\end{array}\right],\]
$\lambda = \nu k/h^2$, and $U^j$ represents the approximation at time $t_j$.
We can get this method started by using the initial condition given in our problem, so that $U_{i}^{0} = f(x_i)$.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{heatexercise1b.pdf}
\caption{The graph of $U^{10}$, the approximation to the solution $u(x,t=.4)$ for Problem \ref{prob:heat_exercise1}.}
\label{fig:heatexercise1b}
\end{figure}

\begin{info}
Finite difference schemes, though they can be \emph{represented} using matrix multiplication, should not be \emph{implemented} using raw matrix multiplication.
Using NumPy, it is best to vectorize the difference scheme so that you do not have to loop over the spatial indices.
If you are using a language with faster loops (like C, C++, Fortran, or Cython), it could work well to loop directly through the indices in both time and space.
\end{info}

To account for boundary conditions using this differencing scheme, simply set the boundary points to the appropriate values in the initial conditions, then avoid modifying them as you update for each time step.
This would be the equivalent of replacing the first and last rows of the matrix representation of the differencing scheme with the first and last rows of the identity matrix.

\begin{problem}
\label{prob:heat_exercise1}
Consider the specific initial boundary value problem
\begin{align}
	\begin{split}
	&{ } u_t = .05 u_{xx}, \quad x \in [0,1], \\
	&{ } u(0,t) = 0,\quad u(1,t) = 0,\\
	&{ } u(x,0) = 2\max\{.2 - |x-.5|,0\}.
	\end{split}
\end{align}
Approximate the solution $u(x,t)$ at time $t = .4$ be taking 6 subintervals in the $x$ dimension and 10 subintervals in time.
The graphs for $U^0$ and $U^{10}$ are given in Figures \ref{fig:heatexercise1a} and \ref{fig:heatexercise1b}.
\end{problem}

For the next problem, we need to show how Matplotlib can be used to create a 2D animation.
The following is a simple working example that animates a sine wave.

% Note: If an animation is running slowly, consider using the \li{blit=True} option when creating the animation.
% If you do this, also define an initialization function that sets the line's data to empty lists and returns a 1-tuple containing the line object.
% Pass the initialization function to the \li{FuncAnimation} function using the \li{init_func} keyword argument.

\begin{lstlisting}
import numpy as np
from matplotlib import animation, pyplot as plt

def sine_animation(res=100):
    # Make the x and y data.
    x = np.linspace(-1, 1, res+1)[:-1]
    y = np.sin(np.pi * x)
    # Initialize a matplotlib figure.
    f = plt.figure()
    # Set the x and y axes by constructing an axes object.
    plt.axes(xlim=(-1,1), ylim=(-1,1))
    # Plot an empty line to use in the animation.
    # Notice that we are unpacking a tuple of length 1.
    line, = plt.plot([], [])
    # Define an animation function that will update the line to
    # reflect the desired data for the i'th frame.
    def animate(i):
        # Set the data for updated version of the line.
        line.set_data(x, np.roll(y, i))
        # Notice that this returns a tuple of length 1.
        return line,
    # Create the animation object.
    # 'frames' is the number of frames before the animation should repeat.
    # 'interval' is the amount of time to wait before updating the plot.
    # Be sure to assign the animation a name so that Python does not
    # immediately garbage collect (delete) the object.
    a = animation.FuncAnimation(f, animate, frames=y.size, interval=20)
    # Show the animation.
    plt.show()

# Run the animation function we just defined.
sine_animation()
\end{lstlisting}

\begin{problem}
\label{prob:heat_exercise2}
Solve the specific initial boundary value problem
\begin{align}
	\begin{split}
	&{ } u_t = u_{xx}, \quad x \in [-12,12],\quad t \in [0,1], \\
	&{ } u(-12,t) = 0,\quad u(12,t) = 0,\\
	&{ } u(x,0) = \max\{1 - x^2,0\}
	\end{split}
\end{align}
using the first order explicit method \ref{eqn:firstorder_explicit}.
Use 140 subintervals in the $x$ dimension and 70 subintervals in time.
The initial and final states are shown in Figure \ref{fig:heatexercise2}.
Animate your results.

Explicit methods usually have a stability condition, called a CFL condition (for Courant-Friedrichs-Lewy).
For method \ref{eqn:firstorder_explicit} the CFL condition that must be satisfied is that
\[\lambda \leq \frac{1}{2}.\]
Repeat your computations using 140 subintervals in the $x$ dimension and 66 subintervals in time.
For these values the CFL condition is broken; you should easily see the result of this instability in the approximation $U^{66}$.
% Then the graphs for $U^0$ and $U^{70}$ are given in \eqref{heatexercise2a} and \ref{heatexercise2b}.
\end{problem}

\begin{figure}
\centering
\includegraphics[width=\textwidth]{heatexercise2.pdf}
\caption{The initial and final states for equation Problem \ref{prob:heat_exercise2}.}
\label{fig:heatexercise2}
\end{figure}

Implicit methods often have better stability properties than explicit methods.
The Crank-Nicolson method, for example, is unconditionally stable and has order $\mathcal{O}(h^2 + k^2)$.
To derive the Crank-Nicolson method, we use the following approximations:
\begin{align*}
	u_t(x_i,t_{j+1/2}) &= \frac{u_t(x_i,t_{j+1}) - u_t(x_i,t_j)}{k} + \mathcal{O}(k^2), \\
	u_{xx}(x_i,t_{j+1/2}) &= \frac{u_{xx}(x_i,t_{j+1}) + u_{xx}(x_i,t_j)}{2} + \mathcal{O}(k^2).
\end{align*}
These approximations give the method
\begin{align}
	\begin{split}
	\frac{U^{j+1}_i - U^j_i}{k} &= \frac{1}{2}\left( \frac{U^j_{i+1} - 2U^j_{i} + U^j_{i-1}}{h^2} + \frac{U^{j+1}_{i+1} - 2U^{j+1}_{i} + U^{j+1}_{i-1}}{h^2}  \right) ,\\
	U^{j+1}_i  &= U^j_i + \frac{k}{2h^2} \left( U^j_{i+1} - 2U^j_{i} + U^j_{i-1} + U^{j+1}_{i+1} - 2U^{j+1}_{i} + U^{j+1}_{i-1}   \right).
\end{split}
\end{align}
This method can be written in matrix form as
\[BU^{j+1} = A U^j,\]
where $A$ and $B$ are tridiagonal matrices given by
\begin{align*}
B &= \left[\begin{array}{cccccc}1+2\lambda & -\lambda &  &  &  \\ -\lambda & 1+2\lambda &  -\lambda & &  \\ &  \ddots &   \ddots & \ddots \\ & &  -\lambda &  1+2\lambda & -\lambda \\ &  &  & -\lambda & 1+2\lambda\end{array}\right], \\
A &= \left[\begin{array}{cccccc}1-2\lambda & \lambda &  &  &  \\ \lambda & 1-2\lambda &  \lambda & &  \\ &  \ddots &   \ddots & \ddots \\ & &  \lambda &  1-2\lambda & \lambda \\ &  &  & \lambda & 1-2\lambda\end{array}\right],
\end{align*}
where $\lambda = \nu k/(2h^2)$, and $U^j$ represents the approximation at time $t_j$.
Note that here we have defined $\lambda$ differently than we did before!

How do we know if a numerical approximation is reasonable?
One way to determine this is to compute solutions for various step sizes $h$ and see if the solutions are converging to something.
To be more specific, suppose our finite difference method is $\mathcal{O}(h^p)$ accurate.
This means that the error $E(h) \approx Ch^p$ for some constant $C$ as $h \to 0$ (i.e., for $h>0$ small enough).

So compute the approximation $y_k$ for each stepsize $h_k$, $h_1 > h_2> \ldots>h_m.$
We will think of $y_m$ as the true solution.
Then the error of the approximation for stepsize $h_k, k < m,$ is
\begin{align*}
	E(h_k) &= \max( \abs{ y_k - y_m}) \approx C h_k^p ,\\
	\log(E(h_k)) &= \log(C) + p \log(h_k).
\end{align*}
Thus on a log-log plot of $E(h)$ vs. $h,$ these values should be on a straight line with slope $p$ when $h$ is small enough to start getting convergence.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{MaximumError.pdf}
\caption{$E(h)$ represents the (approximate) maximum error in the numerical solution $U$ to Problem \ref{prob:heat_exercise3} at time $t=1$, using a stepsize of $h$.}
\label{fig:heatexercise3}
\end{figure}

% If dgtsv is ever included in SciPy, it would probably be better to use that here instead of a custom routine.
When implementing the Crank-Nicolson method, you will need some way to solve a tridiagonal system.
The following is a simple function that performs a tridiagonal solve.
\li{a}, \li{b}, and \li{c} are assumed to contain the first subdiagonal, main diagonal, and first superdiagonal of the tridiagonal matrix.
\li{x} is assumed to be the right hand side of the equation.
This function overwrites \li{x} with the solution to the system.
As has been shown in earlier labs, this function is better implemented in Cython, C, Fortran, or some other language with less overhead for array accesses and loops.
\begin{lstlisting}
def tridiag(a, b, c, x):
    # Overrides c and x.
    # The contents of x after computation will be the solution to the system.
    size = x.size
    temp = 0.
    c[0] = c[0] / b[0]
    x[0] = x[0] / b[0]
    for n in range(size-2):
        temp = 1. / (b[n+1] - a[n]*c[n])
        c[n+1] *= temp
        x[n+1] = (x[n+1] - a[n]*x[n]) * temp
    x[size-1] = (x[size-1] - a[size-2]*x[size-2]) / (b[size-1] - a[size-2]*c[size-2])
    for n in range(b.size-2, -1, -1):
        x[n] = x[n] - c[n] * x[n+1]
\end{lstlisting}

To account for boundary conditions when using the Crank-Nicolson method, set the boundary points at the beginning of the iteration and leave them constant when applying $A$ like you did before.
In the Tridiagonal solve, replace the first row of the matrix $B$ with the first row of the identity and the last row of the matrix $B$ with the last row of the identity.

\begin{problem}
\label{prob:heat_exercise3}
Using the Crank Nicolson method, numerically approximate the solution $u(x,t)$ of the problem
\begin{align}
	\begin{split}
	&{ } u_t = u_{xx}, \quad x \in [-12,12],\quad t \in [0,1],\\
	&{ } u(0,t) = 0,\quad u(1,t) = 0,\\
	&{ } u(x,0) = \max\{1 - x^2,0\}.
	\end{split}
\end{align}
Demonstrate that the numerical approximation at $t = 1$ converges to  $u(x,t=1)$.
Do this by computing $U$ at $t=1$ using $20,40,80,160,320$, and $640$ steps.
Use the same number of steps in both time and space.
Reproduce the loglog plot shown in Figure \ref{fig:heatexercise3}.
The slope of the line there shows the proper rate of convergence.

To measure the error, use the solution with the smallest $h$ (largest number of intervals) as if it were the exact solution, then sample each solution only at the x-values that are represented in the solution with the largest $h$ (smallest number of intervals).
Use the $\infty$-norm on the arrays of values at those points to measure the error.

Notice that, since the Crank-Nicolson method is unconditionally stable, there is no CFL condition and we can use the same number of spaces in time and space.
\end{problem}

% The matrix $A$ is sparse, and so we can use several functions from the package \texttt{scipy.sparse.linalg}.
% In particular, we use the functions \texttt{spdiags} and \texttt{spsolve}.
%
% \begin{verbatim}
% D1,D2,D3 = -4*np.ones((1,m**2)), np.ones((1,m**2)), np.ones((1,m**2))
% Dm = np.ones((1,m**2))
% for j in range(0,D2.shape[1]):
%     if (j%m)==m-1:
%         D2[0,j]=0
%         if (j%m)==0:
%             D3[0,j]=a0
% diags = np.array([0,-1,1,-m,m])
% data = np.concatenate((D1,D2,D3,Dm,Dm),axis=0)
%
% A = 1./h**2.*spdiags(data, diags, m**2,m**2).asformat('csr')
% \end{verbatim}
%
% \subsection{2D Heat Equation}
% Recall that the collection of finite difference equations
% \[
% \nabla^2_h U_{ij} = 0, \quad 1 \leq i,j\leq m,
% \]
% can be written in matrix form as
% $$AU + q  = 0$$
%
% The Crank-Nicolson method for the 2D heat equation is given by
% \[U_{i,\,j}^{n+1}- U_{i,\,j}^{n} = \frac{\Delta t}{2}(\nabla_h^2 U_{i,\,j}^{n} + \nabla_h^2 U_{i,\,j}^{n+1}) \text{ for each } 1 \leq i,j \leq m, \]
% is a second order accurate in both space and time. Basically we're using a midpoint scheme in time,
% and a trapezoidal scheme in space. The resulting method is implicit, and can be written in matrix form as
% \begin{align*}
% 	IU^{n+1} &= IU^n + \frac{\Delta t}{2}(AU^n + q + AU^{n+1} + q),\\
% 	(I - \frac{\Delta t}{2}A)U^{n+1}&= (I + \frac{\Delta t}{2}A)U^n + \Delta t q.
% \end{align*}
%
% TODO: What size must the time step be to ensure stability?
%
% We will need to take many time steps, where many equations must be solved with the matrix $(I - \frac{\Delta t}{2}A)$.
% The function \texttt{factorized} from \texttt{scipy.sparse.linalg} computes the LU decomposition of the matrix.
% This decomposition reduces the time required for solving consecutive time steps.
%


\subsection*{How many runs of randomized contraction do we need to be 99\% sure that a minimum cut is found?}

In RA section 1.1, the probability of finding a minimum cut is found to be larger than
$$
\frac{2}{n(n-1)}
$$

We wish to find the number, $x$, of runs of the minimum cut algorithm, such that we are 99\% sure that a minimum cut is found.

We can express, that the probability of finding a minimum cut, after $x$ repetitions of the algorithm, is at least 99\%, by
$$
\left(\frac{2}{n(n-1)}\right)^x \geq 0.99
$$

So we just need to isolate $x$.
$$
\begin{aligned}
\left(\frac{2}{n(n-1)}\right) ^x & \geq 0.99
\\
x\ln\left(\frac{2}{n(n-1)}\right) & \geq \ln(0.99)
\\
x & \geq \frac{\ln(0.99)}{\ln\left(\frac{2}{n(n-1)}\right)}
\end{aligned}
$$


%!TEX root = main.tex
\section{Modelling}
This report will analyze how a virus outbreak evolves over time on a global scale. To do so, firstly it will be described how one might model a virus outbreak in a small uniform population. Secondly, this model will be expanded to include multiple population groups and their interactions.

\subsection{The SIR Model}
\subsubsection{Assumption}
The Susceptible-Infected-Removed model is a compartmental model assuming that within some subdivision of the population containing $N$ individuals, infection rate, cure rate and spread rates are i.i.d. within each of the 3 ``compartments'' of individuals:
\begin{itemize}
	\item Susceptible to the virus
	\item Infected by the virus
	\item Recovered and immune to the virus (or alternatively dead)
\end{itemize}
In reality economic status, job type, air-conditioning etc. will affect how a virus spread \cite{zika-modelling}, however, depending on the heterogeneity of the population these assumptions constitute a good approximation.

\subsubsection{Governing equations}
The SIR model is governed by 3 differential equations \cite{sir-basics}:
\begin{align}
\frac{d S(t)}{dt} &= - \beta \frac{I(t)}{N} S(t)   \label{eq-S}\\
\frac{d I(t)}{dt} &= \beta \frac{I(t)}{N} S(t) - \gamma I(t)  \label{eq-I}\\
\frac{d R(t)}{dt} &= \gamma I(t) \label{eq-R}
\end{align}
$S(t), I(t)$ and $R(t)$ are functions describing the number of susceptible, infected and removed (recovered and immune or dead) individuals at time t, $\beta$ is the rate of infection, $\gamma$ the rate of removal (dead or cured) and $N$ the total number of individuals.

From equation \eqref{eq-S} and \eqref{eq-I} one can see that susceptible individuals become infected by some ratio $\beta$ and the ratio of already infected individuals. From \eqref{eq-I} and \eqref{eq-R} one can see that infected individuals recover with a constant factor $\gamma$.

\begin{figure}[H]
	\centering
	\includegraphics[width= 1.0 \linewidth]{plots/sir_one_region.pdf}
	\caption{Numerical solution to the nonlinear ODE system defining the SIR model.}
	\label{fig:sir_one_region}
\end{figure}

From figure \ref{fig:sir_one_region} it seen how the number of infected individuals increases quickly in the beginning, until a point where there are not enough susceptible to sustain the infection rate. This is also seen from the differential equations:
\begin{equation}
\frac{d I(t)}{dt} = \frac{\beta}{N} I(t) S(t) - \gamma I(t) = 0 \Leftrightarrow \beta\frac{S(t)}{N} = \gamma
\end{equation}

In this hypothetical case almost everybody got infected, but that is not necessarily the case and depends entirely on $\gamma$ and $\beta$.

\subsection{Multi-region SIR}
Let it now be given that instead of having a single population, one has $K$ populations each with initial $N_k, S_k(0), I_k(0)$ and $R_k(0)$. Each of these populations have a probability of transferring individuals to other populations regardless of whether the individual is susceptible, infected or recovered. The governing equations are modified to add the transfers and become
\begin{equation}
\begin{aligned}
\frac{d S_k(t)}{dt} &= - \frac{\beta}{N_k} I(t) S(t) + \sum_{i=1}^K \left( S_i(t)\tau_{i,k} - S_k(t)\tau_{k, i}\right)   &\forall k \in [1, K]\\
\frac{d I_k(t)}{dt} &= \frac{\beta}{N_k} I(t) S(t) - \gamma I(t) + \sum_{i=1}^K \left( I_i(t)\tau_{i,k} - I_k(t)\tau_{k, i}\right)  &\forall k \in [1, K]\\
\frac{d R_k(t)}{dt} &= \gamma I(t) + \sum_{i = 1}^K \left( R_i(t)\tau_{i,k} - R_k(t)\tau_{k, i}\right) &\forall k \in [1, K]
\end{aligned}
\end{equation}
, where $\tau_{i,j}$ is the probability density of transferring from population $i$ to $j$ per individual.

As an example of the above system let $K=3$ and the non zero transfer probabilities be defined by the graph:
\begin{figure}[H]
	\centering
\begin{tikzpicture}

\def \n {3}
\def \radius {1.5cm}
\def \margin {12} % margin in angles, depends on the radius

\foreach \s in {1,...,\n}
{
	\node[draw, circle] at ({360/\n * (\s - 1)}:\radius) {$\s$};
	\draw[->, >=latex] ({360/\n * (\s - 1)+\margin}:\radius)
	arc ({360/\n * (\s - 1)+\margin}:{360/\n * (\s)-\margin}:\radius);
}
\end{tikzpicture}
\end{figure}
Solving this system numerically with the outbreak starting in region 1 one yields the following curves:
\begin{figure}[H]
	\centering
	\includegraphics[width= 1.0 \linewidth]{plots/sir_three_region_theory.pdf}
	\caption{Numerical solution of the nonlinear ODE system defining a 3-region SIR model.}
\end{figure}

From the above plot it's seen that the virus first spreads in region 1. After a certain time enough infected individuals have been transferred to region 2 where the infection then accelerates. This continues to region 3.

\subsection{Stocastic Model}

To simulate the multi-region SIR model stochastically, a binomial distribution is used to sample how many people get infected, removed and transferred. The binomial probability is chosen such that the distribution's expectation is the same as the terms in the differential equations,
\begin{equation*}
\begin{aligned}
\Delta I_{k,t} &= \textsc{Binom}\left(S_{k,t}, \beta \frac{I_{k, t}}{N_k}\right) \\
\Delta R_{k,t} &= \textsc{Binom}\left(I_{k,t}, \gamma\right) \\
S_{k,t} &= S_{k,t-1} - \Delta I_{k,t} + \sum_{i = 1}^K \left(\textsc{Binom}(S_{i,t-1}, \tau_{i,k}) - \textsc{Binom}(S_{k,t-1}, \tau_{k,i})\right) \\
I_{k,t} &= I_{k,t-1} + \Delta I_{k,t} - \Delta R_{k, t} + \sum_{i = 1}^K \left(\textsc{Binom}(I_{i,t-1}, \tau_{i,k}) - \textsc{Binom}(I_{k,t-1}, \tau_{k,i})\right) \\
R_{k,t} &= R_{k,t-1} + \Delta R_{k, t} + \sum_{i = 1}^K \left(\textsc{Binom}(R_{i,t-1}, \tau_{i,k}) - \textsc{Binom}(R_{k,t-1}, \tau_{k,i})\right)
\end{aligned}
\end{equation*}

, where $\tau_{i,j}$ now is the discretized transfer probability.

\subsubsection{Transfer using Dirichlet distribution}
A problem with this approach, is the small risk that $\textsc{Binom}(S_{i, t-1}, \tau_{i,k}) = 0$ and $\textsc{Binom}(S_{k, t-1}, \tau_{k,i}) = S_k$ or similar unfortunate combination which would cause $S_{k,t}$ to become negative (i.e. there is a small non zero probability that if transfers are drawn independently that an unfeasible state would occur). In practice, this is a big problem if the number of regions is large. The only way to avoid this problem is to sample all the transfers for a region simultaneously such that $\sum_{i = 1}^K \textsc{Binom}(S_{k,t-1}, \tau_{k,i}) \le S_{k, t}$.

As there does not exist a multivariate version of the binomial distribution, the Dirichlet distribution is used as an approximation. This is convenient because $\sum_{i} X_{k, i} = 1$ if $\mathbf{X}_k \sim \textsc{Dir}(\boldsymbol{\alpha}_k)$. In addition the Dirichlet distribution has support $X_{k, i} \in [0, 1]$, thus by using $\iround*{N_k X_{k, i}}$ as the transfer amount we are guaranteed that each transfer is between $0$ and $N_k$ and that the sum of transfers is $N_k$.

Furthermore to reduced the amount of sampling, the total transfer $T$ is sampled instead of sampling $S$, $I$ and $R$ independently.

To approximate the binomial distribution using a Dirichlet distribution the expectation is set to be equal the expectation of the corresponding binomial distribution. To simplify notation region $k$ is omitted from the subscript and the summation of $\tau_k$ and $\alpha_i$ is introduced:
\begin{equation}
\tau_s = \sum_{i = 1}^{K} \tau_i, \quad \alpha_s = \sum_{i = 1}^{K+1} \tau_i
\end{equation}

\begin{equation}
\mathbb{E}[T_{i}] = \mathbb{E}[N X_i] \Leftrightarrow N \tau_i = N \frac{\alpha_i}{\alpha_s} \quad \forall i \in K
\label{dir-expected-trans}
\end{equation}

To allow $\sum_{i} X_{i,k} \le 1$ the Dirichlet distribution is extended with another variable $X_{k, K+1}$, which is how many individuals region $k$ should not transfer.
\begin{equation}
\mathbb{E}[T_{K+1}] = \mathbb{E}[N X_{K+1}] \Leftrightarrow N \left(1 - \tau_s\right) = N \frac{\alpha_{K+1}}{\alpha_s}
\label{dir-expected-stay}
\end{equation}

This is a linear system with n variables and n equations, but it turns out that it doesn't have full rank. To add another equation the variance of $T_{K+1}$ is also set to be equal.
\begin{equation}
\textsc{Var}[T_{K+1}] = \textsc{Var}[N X_{K+1}] \Leftrightarrow N\tau_s (1 - \tau_s) = N^2 \frac{\alpha_{K+1}(\alpha_s - \alpha_{K+1})}{\alpha_s^2 (\alpha_s + 1)}
\label{dir-var}
\end{equation}

The trick to solving this set of equations, is to isolate the sum $\alpha_s$ from \eqref{dir-expected-stay} and insert it into \eqref{dir-expected-trans} and \eqref{dir-var}.
\begin{align}
\alpha_s &= \frac{\alpha_{K+1}}{1 - \tau_s} \label{dir-expected-intermediate} \\
N \tau_i &= N \frac{\alpha_i}{\frac{\alpha_{K+1}}{1 - \tau_s}} \quad \forall i \in K \\
N\tau_s (1 - \tau_s) &= N^2 \frac{\alpha_{K+1}\left(\frac{\alpha_{K+1}}{1 - \tau_s} - \alpha_{K+1}\right)}{\left(\frac{\alpha_{K+1}}{1 - \tau_s}\right)^2 \left(\frac{\alpha_{K+1}}{1 - \tau_s} + 1\right)} \label{dir-var-intermediate}
\end{align}

$\alpha_{K+1}$ can now be isolated from \eqref{dir-var-intermediate} yielding
\begin{equation}
\alpha_{K+1} = (N - 1) - \tau_s (N - 1)
\end{equation}
, inserting this into \eqref{dir-expected-intermediate} gives:
\begin{equation}
\alpha_{i} = \tau_i (N - 1)
\end{equation}
Using the $\alpha_{i}$ solution $\alpha_{K+1}$ can now be reformulated as
\begin{equation}
\alpha_{K+1} = (N - 1) - \sum_{i=1}^K \alpha_k
\end{equation}
, which is computationally slightly more convenient.

Using this method for choosing $\alpha$, gives the distribution as seen in figure \ref{fig:dirichlet-validation-marginal}. As seen it has only minor errors when comparing to the binomial distribution.

\begin{figure}[H]
	\centering
	\includegraphics[width= 0.8 \linewidth]{plots/dirichlet-validation-marginal}
	\caption{Comparison of the marginal probability $P(\iround*{N X_1})$ with the binomial distribution. The Dirichlet distribution was scaled up by $N$, $\tau = (0.1, 0.1)$ and $N = 100$.}
	\label{fig:dirichlet-validation-marginal}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width= 0.8 \linewidth]{plots/dirichlet-validation-pdf}
	\caption{Shows the Dirichlet pdf with $X_3 = 1- X_1 - X_2$. The Dirichlet distribution is scaled with $N$, $\tau = (0.1, 0.1)$ and $N = 100$.}
\end{figure}

\subsection{Transfer probabilities}

The GLEaM model \cite{GLEaM} uses a gravity law for transferring people between neighboring regions within the same country and airline data for transferring people between airports. Unfortunately the GLEaM paper does not specify all the parameters for the gravity law equation and our airline dataset does not contain the traveling frequency. Instead a much simpler model, where the proportional transfer expectation is fixed to some constant, is used:
\begin{equation}
\sum_{i = 1}^{K_k} \tau_{k, i} = p_{transfer}
\end{equation}

Using an equal transfer probability one gets $\tau_{k, i} = \frac{p_{transfer}}{K_k}$. To ensure that the population doesn't change over time, this value is used to transfer both from $k$ to $i$ and from $i$ to $k$, the equivalent transfer probability of this is:
\begin{equation}
\tau_{k, i} = \frac{p_{transfer}}{K_k} + \frac{p_{transfer}}{K_i} = p_{transfer} \left(\frac{1}{K_k} + \frac{1}{K_i}\right)
\end{equation}

Finally one should be wary of transferring more people to the destination than what is already in the destination. To prevent this the transfer probability is scaled with the relative population.
\begin{equation}
\tau_{k, i} = p_{transfer} \frac{N_i}{N_k + N_i} \left(\frac{1}{K_k} + \frac{1}{K_i}\right)
\end{equation}

Using these transfer probabilities the total population for each region will be unchanged since the transfer expectation is symmetric:
\begin{equation}
\mathbb{E}[N_k \tau_{k, i}] = \mathbb{E}[N_i \tau_{i, k}] \Leftrightarrow N_k \tau_{k, i} = N_i \tau_{i, k}
\end{equation}


\subsection{Conservative Vector Fields}
\begin{definition}
	A vector field $\vec{F}$ is conservative if $\int\limits_{C}{\vec{F} \cdot \mathrm{d}\vec{r}}$ is the same for all $C$ connecting the same endpoints.
\end{definition}

\noindent
It's easy to see from this definition that vector fields of constant direction and magnitude, like $\vec{F}=\langle c, c, c \rangle$ is conservative, as its line integral only depends on the curve.\\

\begin{theorem}
	If $\vec{F}$ is conservative, then $\oint\limits_{C}{\vec{F} \cdot \mathrm{d}\vec{r}} = 0$.
\end{theorem}
\begin{proof}
	We can break the simple, closed curve, $C$ into two simple curves $C_1$ and $C_2$ that have the same endpoints and direction such that $C = C_1-C_2$.

	[INSERT IMAGE]

	\noindent
	So,
	\begin{equation*}
		\oint\limits_{C}{\vec{F} \cdot \mathrm{d}\vec{r}} = \int\limits_{C_1}{\vec{f} \cdot \mathrm{d}\vec{r}} - \int\limits_{C_2}{\vec{F} \cdot \mathrm{d}\vec{r}}.
	\end{equation*}
	Since $C_1$ and $C_2$ have the same direction and endpoints, and $\vec{F}$ is conservative, the line integrals have the same value, $L$.
	\begin{equation*}
		\oint\limits_{C}{\vec{F} \cdot \mathrm{d}\vec{r}} = L - L = 0
	\end{equation*}
\end{proof}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 6}

Show that there is no comparison sort whose running time is linear for at least half of the $n!$ inputs of length $n$.
What about a fraction of $1/n$ of the inputs of length $n$? What about a fraction $1/2^n$?

\subsection*{Solution}

\begin{enumerate}[label=(\alph*)]
\item As comparison sorts apply to decision trees, let's assume that the decision tree constructed for $n!$ inputs has height H.
Thus for half of $n!$ inputs,

\begin{equation}\label{eq61}
\frac{n!}{2} \leq n! \leq 2^H
\end{equation}

Taking logarithms with base 2 from both sides leads to Equation \ref{eq62}

\begin{equation}\label{eq62}
h \geq \log_2 (\frac{n!}{2})
\end{equation}

We know that any comparison sort algorithm requires $\Omega(n \log n)$ comparisons in the worst case.
Thus we can find a lower bound can be found for $\log_2(\frac{n!}{2})$.

\begin{equation}\label{eq63}
h \geq \log_2 (n!) - 1 = \mathcal{O}(n \log n)
\end{equation}

Therefore, Equation \ref{eq63} proves no comparison sort algorithm would do better than $\mathcal{O}(n\log n)$.

\item With the same reasoning, even for a fraction of $\frac{1}{n}$ elements of the $n!$ inputs, we will have

\begin{equation}\label{eq64}
h \geq \log_2 \frac{n!}{n} = \log_2 {(n-1)!}
\end{equation}

Therefore the lower bound would change to the following.

\begin{equation}\label{eq65}
h \geq \log_2 n! - \log_2 n = \mathcal{O}(n \log n) - \mathcal{O}(\log n) = \mathcal{O}(n \log n)
\end{equation}

This shows that achieving $\mathcal{O}(n)$ for comparison sort of  even for a fraction of $1/n$ of $n!$ inputs is not possible.

\item Similarly, for fraction of $\frac{1}{2^{n}}$ elements of the $n!$ inputs, we will have

\begin{equation}\label{eq66}
h \geq \log_2 \frac{n!}{2^{n}} = \log_2 {n!} - \log_2 {2^{n}}
\end{equation}

Therefore the lower bound would change to the following.

\begin{equation}\label{eq67}
h \geq \log_2 {n!} - n \geq \mathcal{O}(n \log n) - \mathcal{O}(n) = \mathcal{O}(n \log n)
\end{equation}

\end{enumerate}


\section{Needles in a Haystack: the Class NP}

\subsection{Exercises}

\subsubsection{Exercise 1}
If a 2-coloring of a graph exists, then we can flip all of the colors in the graph to produce another valid
two-coloring. Thus, we can proceed to construct a two-coloring by picking an uncolored vertex and then coloring
it an arbitrary color, which then determines the colors of all of the vertices it is connected to. We do
this until either the graph is completely colored, or until we need to recolor an already colored vertex
(in which case the graph is not 2-colorable). This algorithm is linear in the number of edges and vertices of the graph, so 2-coloring is in $P$.

\subsubsection{Exercise 2}

\subsubsection{Exercise 3}
Suppose we wish to determine whether a graph $G$ with  $n$ vertices is  $k$-colorable. We can extend $G$ to
be a graph $G'$ with $n + 1$ vertices by adding an additional vertex that we then connect to all of the original
$n$ vertices of  $G$ (this can be done in  $O(n)$ time). By construction, $G'$ is  $(k + 1)$-colorable if and
only if its subgraph $G$ is $k$-colorable (since node $n + 1$ is connected to all of $G$, the subgraph $G$
must be colored with only $k$ colors).

\subsubsection{Exercise 4}
Since boolean operations are commutative, we can write $\phi'$ as:
\begin{align*}
        \phi'(p, q, r) &= (p \vee q) \wedge (p \vee \bar{q}) \wedge (r \vee q) \wedge (r \vee \bar{q}) \wedge (\bar{p} \vee \bar{r}) \\
                       &= p \wedge r \wedge (\bar{p} \vee \bar{r})
\end{align*}
which is clearly not satisfiable, since $p = 1, r = 1 \implies (\bar{p} \vee \bar{r}) = 0$.

\subsubsection{Exercise 5}
For a graph to be 3-colorable (let our 3 colors be R, G, and B), each vertex in the graph must be colored
either R, G, or B, and no adjacent vertices may have the same color. If we let $v_r^{(i)}, v_g^{(i)}, v_b^{(i)}$
denote boolean variables that are true when vertex $i$ is colored R, G, and B respectively and $N(i)$ denote
all of the neighbors of vertex $i$, we can translate 3-colorability into the following boolean expression
for each vertex $i$:
\begin{align*}
\bigwedge_i (v_r^{(i)} \vee v_g^{(i)} \vee v_b^{(i)}) \wedge \big((v_r^{(i)} \wedge_{j \in N(i)} \bar{v}_r^{(j)}) \vee (v_b^{(i)} \wedge_{j \in N(i)} \bar{v}_b^{(j)}) \vee (v_g^{(i)} \wedge_{j \in N(i)} \bar{v}_g^{(j)})\big)
\end{align*}
Repeatedly applying the distributive law to the part of the above expression that consists of ors of ands
produces a CNF expression that must be true for every vertex $i$. Since satisfiability of the resulting
expression corresponds directly to whether the original graph is 3-colorable or not, we have that
3-colorability can be reduced to SAT.

\subsubsection{Exercise 6}
Factoring out $x$ in the first expression leads to $x \vee (z_1 \wedge \bar{z}_1) = x$. The second expression
is even more straightforward; we just factor out $(x \vee y)$.

\subsubsection{Exercise 7}
If any of the $x_i$ are true, then it is clear that the right-hand side can be made true by picking $(z_1, z_2)$
such that the (at most) two clauses not containing a true $x_i$ value can be made true.
For the other direction, iterating through all possible $(z_1, z_2)$ values shows that the right-hand side is
only true if at least one of the $x_i$ (where $i$ depends on the values of $z_1, z_2$) is true.

There is probably a better approach here than the above ``brute force'' reasoning; I should come back to this.

\subsubsection{Exercise 8}
The approach from Exercise 5 still works.

\subsubsection{Exercise 9}
There are only $(2n)^k$ possible unique $k$-clauses (each of the $k$ literals can either be one of the
variables or its complement), so the number of bits needed to specify a satisfiability problem is polynomial
in the number of variables (each of which requires only a single bit).


\subsection{Manifolds, charts and atlases}

A manifold is a set of points and associated charts.

A chart is a mapping from each point in a subset of the manifold to a point in a vector space.

These charts are invertible. If we are given coordinates, we can identify the point in the manifold it comes from.

For each point we have a topological neighbourhood. For each point in the neighbourhood, we can map to an element in the tangent space.

\subsubsection{Example: The sphere}

We can map a hemisphere to a subset of \(R^2\). Given a point in \(R^2\) we can identify a specific point on the hemisphere, and given a s specific point on the hemisphere we can identify a point in \(R^2\).

\subsubsection{Universal charts}

If the vector space is flat and non-repeating, then a single chart can be used to map the whole manifold.

\subsubsection{Atlases}

If we have a collection of charts which covers each point needs to be covered at least once, we have an atlas. Each chart needs to be to the same dimensional vector space.


\subsection*{Ex.29.2-6}
\subsubsection{Solve the maximum--bipartite--matching problem}

Let $G, V, E$ and $G', V', E'$ be given as in the section--26.3.

From Corollary~26.11 we know that finding a max flow in $G'$ also tells us what the maximum matching in $G$ is. So we want to define a max flow problem.
\\
We follow the approach as they do in the book, as section 29.2 \emph{Maximum flow}.

Define the capacity of the edges as
$$
c(u,v) =
\begin{cases}
1, & (u,v)\in E'\\
0, & \text{else}
\end{cases}
$$

The the problem can be stated as

\begin{align*}
&\max      \quad  &\sum_{v\in V'}f_{sv}  &- \sum_{v\in V'}f_{vs}   &\\
&\text{s.t.}      & f_{uv}               &\leq c(u,v)              &\text{for all } u,v \in V'\\
&                 & \sum_{v\in V'}f_{uv} &= \sum_{v\in V'}f_{vu}   &\text{for all } u \in V\\
&                 & f_{uv}               &\geq 0                   &\text{for all } u,v \in V'
\end{align*}

This maximization problem will solve the max flow problem, which by Corollary~26.11 solves the maximum--bipartite--matching problem.


\chapter{Adaptive basis function models}

\section{AdaBoost}

\subsection{Representation}
\begin{equation}
y=\text{sign}(f(\vec{x}))=\text{sign}\left(\sum\limits_{i=1}^m \alpha_mG_m(\vec{x})\right)
\end{equation}
where $G_m(\vec{x})$ are sub classifiers.

\subsection{Evaluation}
\begin{equation} \nonumber
L(y,f(\vec{x}))=\exp[-yf(\vec{x})] \text{  i.e., exponential loss function}
\end{equation}

\begin{equation}
(\alpha_m,G_m(x))= \arg\min_{\alpha,G} \sum_{i=1}^N \exp{[-y_i(f_{m-1}(\vec{x}_i)+\alpha G(\vec{x}_i))]}
\end{equation}

Define $\bar{w}_{mi}=\exp{[-y_i(f_{m-1}(\vec{x}_i)]}$, which is constant w.r.t. $\alpha, G$
\begin{equation}
(\alpha_m,G_m(x))= \arg\min_{\alpha,G} \sum_{i=1}^N {\bar{w}}_{mi} \exp{(-y_i \alpha G(x_i))}
\end{equation}

\subsection{Optimization}

\subsubsection{Input}
\begin{eqnarray*}
& \mathcal{D}=\{(\vec{x}_1,y_1),(\vec{x}_2,y2),\dots,(\vec{x}_N,y_N)\} \\
& \quad \text{where } \vec{x}_i \in \mathbb{R}^D,\ y_i \in \{-1,+1\} \\
& \text{ Weak classifiers } \{G_1,G_2,\dots,G_m\}
\end{eqnarray*}

\subsubsection{Output}
Final classifier: $G(x)$

\subsubsection{Algorithm}
\begin{enumerate}
\item Initialize the weights' distribution of training data(when $m=1$)
\begin{equation}\nonumber
\mathcal{D}_0=(w_{11},w_{12},\cdots,w_{1n})=(\frac{1}{N},\frac{1}{N},\cdots,\frac{1}{N})
\end{equation}
\item Iterate over $m=1,2,\dotsc,M$
\subitem (a) Use training data with current weights' distribution $\mathcal{D}_m$ to get a classifier $G_m(\vec{x})$
\subitem (b) Compute the error rate of $G_m(\vec{x})$ over the training data \\
\begin{equation}
e_m=P(G_m(\vec{x}_i)\neq y_i)=\sum_{i=1}^N {w_{mi}\mathbb{I}(G_m(\vec{x}_i) \neq y_i)}
\end{equation}

\subitem (c) Compute the coefficient of classifier $G_m(x)$
\begin{equation}
\alpha_m = \frac{1}{2}\log{\frac{1-e_m}{e_m}}
\end{equation}
\subitem (d) Update the weights' distribution of training data
\begin{equation}
w_{m+1,i}=\frac{w_{mi}}{Z_m}\exp(-\alpha_m y_i G_m(\vec{x}_i))
\end{equation}
where $Z_m$ is the normalizing constant
\begin{equation}
Z_m=\sum_{i=1}^N w_{mi}\exp(-\alpha_m y_i G_m(\vec{x}_i))
\end{equation}

\item Ensemble $M$ weak classifiers
\begin{equation}
G(x)=\text{sign}f(\vec{x})=\text{sign}\left[\sum_{m=1}^M \alpha_m G_m(\vec{x})\right]
\end{equation}
\end{enumerate}

\subsection{The upper bound of the training error of AdaBoost}
\begin{theorem}
The upper bound of the training error of AdaBoost is
\begin{equation}
\frac{1}{N} \sum_{i=1}^N \mathbb{I}(G(\vec{x}_i)\neq y_i) \leq \frac{1}{N} \sum_{i=1}^N \exp(-y_i f(\vec{x}_i))=\prod_{m=1}^M Z_m
\end{equation}

Note: the following equation would help proof this theorem
\begin{equation}
w_{mi}\exp(-\alpha_m y_i G_m(\vec{x}_i))=Z_m w_{m+1,i}
\end{equation}
\end{theorem}


\chapter{The Fundamental Theory For Spectral Methods}
\label{Chapter_2}

	\vspace{0.3cm}
	In this chapter, we will present the elements necessary to solve partial differential equations using spectral methods given as follows
	\begin{align}
	\label{general_problem}
	\left \lbrace \begin{array}{ll}
		&\frac{\partial u}{\partial t} = \mathcal{L} u, \hspace{3mm} x \in I, \hspace{3mm} t > 0,\\
		\\
		&u(x, 0) = g(x), \hspace{9mm} x \in I,
		\end{array}  \right .
	\end{align}
	where $u$ is defined in some Hilbert space $\mathcal{H}$, with initial condition $g(x) \in \mathcal{H}$ and $\mathcal{L}$ is some spatial differential operator, which allows us to represent the previous problem in another whose solution $u$ will be given by a linear combination of already known functions. \\

	To do this, suppose that $\mathcal{H}$ is a separable Hilbert space with the inner product $\langle \cdot, \cdot \rangle$. Therefore, we can represent the function $u$ in terms of a known orthonormal base of $\mathcal{H}$, which we will denote as $\{\phi_k \}_{k \in I}$, given as follows
	\begin{align*}
		\displaystyle u = \sum_{k \in I} \langle \phi_k, u \rangle \phi_k.
	\end{align*}

	There is a wide variety of families of base functions, which define different spectral methods. In this chapter, we will consider the well-known Fourier basis given by
	\begin{align}
		\label{base_phi}
		\phi_n (x) = e^{inx}.
	\end{align}
	that form an orthogonal set with the standard interior product $L^2$ in the interval $(0, 2 \pi)$, that is,
	\begin{align}
	\label{ortho_phi}
	\displaystyle \int_{0}^{2\pi} \phi_k (x) \overline{\phi_l (x)} dx = 2 \pi \delta{kl} = \left \lbrace \begin{array}{ll}
	0 \hspace{3mm} &\text{if} \hspace{3mm} k \neq l, \\
	2 \pi &\text{if} \hspace{3mm} k = l.
	\end{array}  \right.
	\end{align}

	We will denote as $B = span\{e^{inx}: |n| \leq \infty \}$ the set containing the Fourier bases. Therefore, we can define the Fourier series $F[u]$ for $u(x) \in L^2 [0, 2\pi]$ as follows
	\begin{equation}
	\label{fourier_series}
		F[u] \equiv \displaystyle \sum_{ |n| \leq \infty} \hat{u}_{n} e^{inx},
	\end{equation}
	where
	\begin{align}
	\label{coeff_fourier}
	\hat{u}_n = \frac{1}{2 \pi} \displaystyle \int_{0}^{2 \pi} u(x) e^{-inx} dx, \hspace{3mm}  k = 0, \pm 1, \pm 2, \dots.
	\end{align}
	which is known as the classical continuous series of trigonometric polynomials, where $\hat{u}_{n}$ are the Fourier coefficients. \\

    It is important noted that the integrals in (\ref{coeff_fourier}) exist if $u$ is Riemann-integrable, i.e., if $u$ is bounded and piecewise continuous in $(0, 2 \pi)$. More generally, the Fourier coefficients are defined for any function that is integrable in the Lebesgue sense. Also the relation (\ref{coeff_fourier}) associates with $u$ a sequence of complex numbers called the Fourier transform of $u$. It is possible as well to introduce a Fourier cosine transform and a Fourier sine transform of $u$, respectively, through the formulas
    \begin{align}
    \label{coeff_a_n}
    	a_n = \frac{1}{2 \pi} \displaystyle \int_{0}^{2 \pi} u(x) \cos(nx) dx, \hspace{3mm}  n = 0, \pm 1, \pm 2, \dots,
    \end{align}
    and
    \begin{align}
    \label{coeff_b_n}
    	b_n = \frac{1}{2 \pi} \displaystyle \int_{0}^{2 \pi} u(x) \sin(nx) dx, \hspace{3mm}  n = 0, \pm 1, \pm 2, \dots.
    \end{align}
    The three Fourier transforms of $u$ are related by the formula $\hat{u}_n = a_n - ib_n$ for $n = 0, \pm 1, \pm 2, \dots$. Moreover, if $u$ is a real valued function, $a_n$ and $b_n$ are real numbers, and $\hat{u}_{-n} = \hat{u}_n$. \\

	Based on the above, we will present two tools to build the methods that will be used in Chapter \ref{Chapter_3}, and that will be developed independently in the following two sections. In the first section, we will see that with the continuous Fourier expansion we can define a projection operator on a space of finite dimension that will allow us to approximate a function and its derivatives. In the second and last section, due to the complexity of the calculation of the previous integrals, we will see that it is possible to use quadrature rules to approximate them, and thus define an interpolation operator that will give us a discrete representation for a function and its derivatives.

	For these two operators, at the end of each section we will discuss the factors that determine the behavior of the series when used to approximate smooth functions, showing how fast they approach, when, and in what sense they are convergent.

	\newpage
	\input{preliminaries/Projection_Operator}

	\newpage
	\input{preliminaries/Interpolation_Operator}


\section{Sentenial Logic}
\newcommand{\definition}[1]{\textbf{#1}}
\subsection{Deductive Reasoning and Logical Connectives}
\subsection{Truth Tables}
\definition{DeMorgan's laws}
\[
\lnot (P \land Q) \equiv \lnot P \lor \lnot Q
\]
\[
\lnot (P \lor Q) \equiv \lnot Q \land \lnot Q
\]
\definition{Commutative laws}
\[
P \land Q \equiv Q \land P
\]
\[
P \lor Q \equiv Q \lor P
\]
\definition{Associative laws}
\[
P \land (Q \land R) \equiv (P \land Q) \land R
\]
\[
P \lor (Q \lor R) \equiv (P \lor Q) \lor R
\]
\definition{Idempotent laws}
\[
P \land \equiv P
\]
\[
P \lor P \equiv P
\]
\definition{Distributive laws}
\[
P \land (Q \lor R) \equiv (P \land Q) \lor (P \land R)
\]
\[
P \lor (Q \land R) \equiv (P \lor Q) \land (P \lor R)
\]
\definition{Absorption laws}
\[
P \lor (P \land Q) \equiv P
\]
\[
P \land (P \lor Q) \equiv P
\]
\definition{Double Negation law}
\[
\lnot \lnot P \equiv P
\]
\definition{Tautology laws}
\[
P \land \text{(a tautology)} \equiv P
\]
\[
P \lor \text{(a tautology)} \equiv \text{(a tautology)}
\]
\[
\lnot \text{(a tautology)} \equiv \text{(a contradiction)}
\]
\definition{Contradiction laws}
\[
P \land \text{(a contradiction)} \equiv \text{(a contradiction)}
\]
\[
P \lor \text{(a contradiction)} \equiv P
\]
\[
\lnot \text{(a contradiction)} \equiv \text{(a tautology)}
\]
\subsection{Variables and Sets}
\[
\mathbb{R}^{+,-} = \{x | x \text{ is a real number}\}
\]
(a number that can be written on the number line)
\[
\mathbb{Q}^{+,-} = \{x | x \text{ is a rational number}\}
\]
(a number that can be written as a fraction $\cfrac{p}{q}$)
\[
\mathbb{Z}^{+,-} = \{x | x \text{ is an integer}\} = \{ \ldots, -3, -2, -1, 0, 1, 2, 3, \ldots \}
\]
\[
\mathbb{N} = \{x | x \text{ is a natural number}\} = \{0, 1, 2, 3, \ldots \}
\]
\subsection{Operations on Sets}
\definition{Intersection of two sets A and B}
\[
A \cap B = \{ x | x \in A \text{ and } x \in B \}
\]
\definition{Union of two sets A and B}
\[
A \cup B = \{ x | x \in A \text{ or } x \in B \}
\]
\definition{Difference of A and B}
\[
A \setminus B = \{ x | x \in A \text{ and } x \notin B \}
\]
\definition{A is a subset of B if every element of A is also an element of B}
\[
A \subseteq B
\]
\definition{A and B are disjoint if they have no elements in common}
\[
A \land B = \emptyset
\]
\subsection{The Conditional and Biconditional Connectives}
\definition{Conditional laws}
\[
P \to Q \equiv \lnot P \lor Q
\]
\[
P \to Q \equiv \lnot (P \land \lnot Q)
\]
\definition{Contrapositive law}
\[
P \to Q \equiv \lnot Q \to \lnot P
\]