\include{config/config}

\begin{document}
% ========== Edit your name here
\title{MATH 2901 Basic Probability Lecture Notes 1}
\author{Instructor: Richard Kleeman}
\date{}
\maketitle

%\medskip

% ========== Contents begin here ==============
\section{Event as sets}
Probability studies the \emph{repeatable (or ideal) experiments}. The result of an experiment is called an \emph{outcome}.

\begin{definition}
The set of all possible outcomes of an experiment is called the \textbf{sample space} and is denoted by $\Omega$.
\end{definition}

\subsection{Cardinality of sets}
The cardinality of a set $\Omega$ refers to the number of the number of elements in this set, and is denoted by $\card(\Omega)$ or $\abs{\Omega}$.
\begin{itemize}
    \item For finite sets, $\card(\Omega)$ is a natural number.
    \item For the integer set, $\card(\Z) = \aleph_0$, which is countable.
    \item For the real number set, $\card(\R) = \aleph_1 > \aleph_0$.
\end{itemize}
The \emph{power set} of a set $\Omega$ is the set of all subsets, which is denoted by $2^\Omega$.
\begin{itemize}
    \item For $\Z$, $\card(2^\Z) = \aleph_1 = \card(\R)$.
\end{itemize}

In practical, sets in probability are: \textbf{finite, countable, reals} and \textbf{their variants}.

When we conduct an experiment, we want to know whether a subset occurs or not. For example, if we take a number from $\R$, the probability will be 0 because $\card(\R)$ is very large. Therefore,
\begin{center}
\emph{we are interested in $A \subset \Omega$ in probability, where $A$ is a collection of subsets in probability.}
\end{center}

\subsection{Events and fields}
\begin{definition}
The \textbf{events} are subsets of the sample space $\Omega$.
\end{definition}

\begin{remark}
$\varnothing$ is called the \emph{impossible event}. The set $\Omega$ is called the \emph{certain event}. Events $A$ and $B$ are called \emph{disjoint} if their intersection is the empty set.
\end{remark}

We do \textbf{NOT} need all the subsets of $\Omega$ be events. It suffices for us to think of the collection of events as a subcollection $\mathcal{F}$ of the set of all subsets of $\Omega$.
\begin{definition}
Any collection $\mathcal{F}$ of subsets of $\Omega$ which satisfies the following three conditions is called a field:
\begin{enumerate}[(a)]
    \item if $A, B \in \mathcal{F}$, then $A \cup B \in \mathcal{F}$ and $A \cap B \in \mathcal{F}$ (actually $A\cap B \in \mathcal{F}$ is redundant);
    \item if $A \in \mathcal{F}$, then $A^c \in \mathcal{F}$;
    \item the empty set $\varnothing$ belongs to $\mathcal{F}$.
\end{enumerate}
\end{definition}

\begin{remark}
Some implications from the properties of a filed $\mathcal{F}$:
\begin{itemize}
    \item if $A_1, \dots, A_n \in \mathcal{F}$, then $\bigcup_{i=1}^n A_i \in \mathcal{F}$;
    \item $\varnothing^C = \Omega \in \mathcal{F}$;
    \item $A \cap B = (A^c \cup B^c)^C \in \mathcal{F}$.
\end{itemize}
\end{remark}

\begin{question}
what if we have an $\Omega$ such that $\card(\Omega) = \aleph_0$? In this case, we may need $2^\Omega$ to study the probability.
\end{question}

\begin{definition}
A collection $\mathcal{F}$ of subsets of $\Omega$ is called a $\sigma$-field if it satisfies the following conditions:
\begin{enumerate}[(a)]
    \item $\varnothing \in \mathcal{F}$;
    \item if $A \in \mathcal{F}$, then $A^c \in \mathcal{F}$;
    \item if $A_1, A_2, \dots \in \mathcal{F}$, then $\bigcup_{i=1}^\infty A_i \in \mathcal{F}$.
\end{enumerate}
\end{definition}

\begin{remark}
$\sigma$-fields are closed under the operation of taking countable intersections.
\end{remark}

\begin{example}
The smallest $\sigma$-field associated with $\Omega$ is the collection $\mathcal{F} = \{\varnothing, \Omega \}$.
\end{example}

\begin{example}
If $A \subset \Omega$, then $\mathcal{F} = \{\varnothing, A, A^c, \Omega \}$ is a $\sigma$-field.
\end{example}

\begin{example}
The power set of $\Omega$ is a $\sigma$-field.
\end{example}

With any experiment we may associate a pair $\{\Omega, \mathcal{F}\}$, where $\Omega$ is the set of all possible outcomes or elementary events and $\mathcal{F}$ is a $\sigma$-field of subsets of $\Omega$ which contains \emph{all the events in whose occurrences we may be interested}; henceforth, to call a set $A$ an event is equivalent to asserting that $A$ belongs to the $\sigma$-field in question.

\section{Probability}
Assume we have a ``repeatable" experiment and we repeat the experiment a large number $N$ of times. Let $A \in \Omega$ and $N(A)$ be the number of $A$ occurs in the $N$ trails. Intuitively, the ratio $N(A)/ N$ appears to converge to a constant limit as $N$ increases. In practice, we have
\begin{itemize}
    \item $0 \leq N(A)/ N \leq 1$;
    \item if $A, B$ are disjoint, then $N(A \cup B) = N(A) + N(B)$. (finite additive and countably additive)
\end{itemize}

\begin{definition}
A \textbf{probability measure} $\Prob$ on $\{ \Omega, \mathcal{F} \}$ is a function $\Prob : \mathcal{F} \to [0,1]$ satisfying
\begin{enumerate}[(a)]
    \item $\Prob(\varnothing) = 0$;
    \item $\Prob(\Omega) = 1$;
    \item  if $A_1, A2, \dots$ is a collection of disjoint members of $\mathcal{F}$, in that $A_i \cap A_j = \varnothing$ for all pairs $i, j$ satisfying $i \neq j$, then
    \begin{equation*}
        \Prob\left( \bigcup_{i=1}^\infty A_i \right) = \sum_{i=1}^\infty \Prob (A_i).
    \end{equation*}
\end{enumerate}
\end{definition}
The triple $\{ \Omega, \mathcal{F}, \Prob \}$, comprising a set $\Omega$, a $\sigma$-field $\mathcal{F}$ of subsets of $\Omega$, and a probability measure $\Prob$ on $\{\Omega, \mathcal{F}\}$, is called a \textbf{probability space}. We can associate a probability space $\{ \Omega, \mathcal{F}, \Prob \}$ with any experiment, and all questions associated with the experiment can be reformulated in terms of this space.

\begin{remark}
A probability measure is a special example of what is called a \emph{measure} on the pair $\{\Omega, \mathcal{F}\}$. A measure is a function $\mu: \mathcal{F} \to [0, \infty)$ satisfying $\mu(\varnothing) = 0$ together with (c) above. A measure $\mu$ is a probability measure if $\mu(\Omega) = 1$.
\end{remark}

\begin{lemma}
We can deduce some lemmas from the definition.
\begin{enumerate}[(a)]
    \item $\Prob(A^c) = 1 - \Prob(A)$;
    \item if $A \subseteq B$, then $\Prob(B) = \Prob(A) + \Prob(B\backslash A) \geq \Prob(A)$;
    \item $\Prob(A \cup B) = \Prob(A) + \Prob(B) - \Prob(A \cap B)$;
    \item more generally, if $A_1, A_2, \dots, A_n$ are events, then
    \begin{equation*}
        \begin{split}
        \Prob\left( \bigcup_{i=1}^n A_i \right) =& \sum_{i}\Prob(A_i) - \sum_{i < j} \Prob(A_i \cap A_j) + \sum_{i<j<k} \Prob(A_i \cap A_j \cap A_k) - \cdots \\
        & +(-1)^{n+1} \Prob(A_1 \cap A_2 \cap \cdots \cap A_n)
        \end{split}
    \end{equation*}
\end{enumerate}
\end{lemma}

\begin{lemma}
Let $A_1, A_2, \dots$ be an increasing sequence of events, so that $A_1 \subset A_2 \subset A_3 \subset \cdots$, and write $A$ for their limit:
\begin{equation*}
    A = \bigcup_{i=1}^\infty A_i = \lim_{i \to \infty} A_i.
\end{equation*}
Then $\Prob(A) = \lim_{i\to\infty} \Prob(A_i)$.

Similarly, if $B_1, B_2, \dots$ is a decreasing sequence of events, so that $B_1 \supseteq  B_2 \supseteq B_3 \supseteq \cdots$, then
\begin{equation*}
    B = \bigcap_{i=1}^\infty B_i = \lim_{i \to \infty} B_i
\end{equation*}
satisfies $\Prob(B) = \lim_{i\to\infty} \Prob(B_i)$
\end{lemma}

\section{Some useful concepts}
\subsection{Conditional probability}
What if we only care how many times does $A$ occur only when $B$ occurs? $N(A\cap B) / N(B)$, the universe is changed.
\begin{definition}
If $\Prob(B) > 0$, then the \textbf{conditional probability} that $A$ occurs given that $B$ occurs is defined to be
\begin{equation*}
    \Prob(A \spacevert B) = \frac{\Prob(A \cap B)}{\Prob(B)}.
\end{equation*}
\end{definition}

\begin{definition}
Suppose $\{B_i\}$ is a finite set. If $B_i$ are all disjoint and $\bigcup_{i=1}^n B_i = \Omega$, then $\{B_i\}$ is called a \textbf{partition} of $\Omega$.
\end{definition}

\begin{lemma}
For any events $A$ and $B$ such that $0 < \Prob(B) < 1$,
\begin{equation*}
    \Prob(A) = \Prob(A \spacevert B) \Prob(B) + \Prob(A \spacevert B^c) \Prob(B^c).
\end{equation*}
More generally, let $B_1,  B_2, \dots, B_n$ be a partition of $\Omega$ such that $\Prob(B_i) > O$ for all $i$. Then
\begin{equation*}
    \Prob(A) = \sum_i \Prob(A \spacevert B_i) \Prob(B_i).
\end{equation*}
\end{lemma}

\subsection{Independence}
Intuition: an event $A$ occurs doesn't affect the probability of $A$ occurs when $B$ occurs, which means $\Prob(A) = \Prob(A\spacevert B)$.

\begin{definition}
Events $A$ and $B$ are called \textbf{independent} if
\begin{equation*}
    \Prob(A \cap B) = \Prob(A) \Prob(B).
\end{equation*}
More generally, a family $\{A_i \spacevert i \in I\}$ is called independent if
\begin{equation*}
    \Prob \left( \bigcap_{i \in J} A_i \right) = \prod_{i\in J} \Prob(A_i)
\end{equation*}
for all finite subsets $J$ of $I$.
\end{definition}

\begin{caution}
A common \emph{student error} is to make the fallacious statement that $A$ and $B$ are independent if $A \cap B = \varnothing$.
\end{caution}

\begin{remark}
If the family $\{A_i \spacevert i \in I\}$ has the property that
for all
\begin{equation*}
    \Prob(A_i \cap A_j) = \Prob(A_i) \Prob(A_j) \quad \forall i\neq j,
\end{equation*}
then it is called \emph{pairwise independent}. \textbf{Pairwise-independent families are not necessarily independent.}
\end{remark}

\section{Completeness and product space}
\begin{lemma}
If $\mathcal{F}$ and $\mathcal{G}$ are two $\sigma$-fields of subsets of $\Omega$, then their intersection $\mathcal{F} \cap \mathcal{G}$ is a $\sigma$-field also. More generally, if $\{ \mathcal{F}_i \spacevert  i \in I \}$ is a family of $\sigma$-fields of subsets of $\Omega$, then $\mathcal{G} = \bigcap_{i\in I} \mathcal{F}_i $ is a $\sigma$-field also.
\end{lemma}

\begin{newnotion}{Completeness}
Let $\{\Omega, \mathcal{F}, \Prob\}$ be a probability space. Any event $A$ which has zero probability, that is $\Prob(A) = 0$, is called \emph{null}. It may seem reasonable to suppose that any subset $B$ of a null set $A$ will itself be null, but this may be without meaning since $B$ may not be an event, and thus $\Prob(B)$ may not be defined.
\end{newnotion}

\begin{definition}
A probability space $\{\Omega, \mathcal{F}, \Prob\}$ is called \textbf{complete} if all subsets of null sets are events.
\end{definition}

Any incomplete space can be completed thus. Let $\mathcal{N}$ be the collection of all subsets of null sets in $\mathcal{F}$ and let $\mathcal{G} = \sigma (\mathcal{F} \cup \mathcal{N})$ be the smallest $\sigma$-field which contains all sets in $\mathcal{F}$
and $\mathcal{N}$. It can be shown that the domain of $\Prob$ may be extended in an obvious way from $\mathcal{F}$ to $\mathcal{G}$; $\{\Omega, \mathcal{G}, \Prob\}$ is called the completion of $\{\Omega, \mathcal{F}, \Prob\}$.

\begin{newnotion}{Product space}
Suppose two experiments have associated probability spaces $\{ \Omega_1, \mathcal{F}_1, \Prob_1 \}$ and  $\{ \Omega_2, \mathcal{F}_2, \Prob_2 \}$ respectively. The sample space of the pair of experiments, considered jointly, is the collection $\Omega_1 \times \Omega_2 = \{ (\omega_1 , \omega_2) \spacevert w_1 \in \Omega_1, \omega_2 \in \Omega_2 \}$ of ordered pairs. The appropriate $\sigma$-field of events is \textbf{more complicated} to construct. \textbf{The family of all such sets, $\mathcal{F}_1 \times \mathcal{F}_2 = \{A_1 \times A_2 \spacevert A_1 \in \mathcal{F}_1 , A_1 \in \mathcal{F}_2 \}$, is NOT in general a $\sigma$-field.}
\end{newnotion}

\begin{remark}
There exists a unique smallest $\sigma$-field $\mathcal{G} = \sigma(\mathcal{F}_1 \times \mathcal{F}_2)$ of subsets of $\Omega_1 \times \Omega_2$ which contains $\mathcal{F}_1 \times \mathcal{F}_2$. All we require now is a suitable probability function on $( \Omega1 \times \Omega_2, \mathcal{G} )$. Let $\Prob_{12}: \mathcal{F}_1 \times \mathcal{F}_2 \to [0, 1]$ be given by:
\begin{equation*}
    \Prob(A_1 \times A_2) = \Prob_1(A_1) \times \Prob_2(A_2) \quad \text{for $A_1 \in \mathcal{F}_1, A_2 \in \mathcal{F}_2$}.
\end{equation*}
It can be shown that the domain of $\Prob_{12}$ can be extended from $\mathcal{F}_1 \times \mathcal{F}_2$ to the whole of $\mathcal{G} = \sigma(\mathcal{F}_1 \times \mathcal{F}_2)$.
\end{remark}

\begin{definition}
The probability space $(\mathcal{F}_1 \times \mathcal{F}_2, \mathcal{G}, \Prob_{12})$ is called the \textbf{product space} of $(\Omega_1, \mathcal{F}_1, \Prob_1)$ and  $(\Omega_2, \mathcal{F}_2, \Prob_2)$. The measure $\Prob_{12}$ is sometimes called the `product measure'.
\end{definition}

\end{document}


\chapter[Finite differences for parabolic problems]{Finite differences for \\
parabolic problems in 1D}
Consider the following \emph{initial-boundary value problem} for~$u=u(x,t)$,
\begin{equation}\label{eq: heat ivp 1d}
\begin{aligned}
u_t-au_{xx}&=f(x,t)&&\text{for $0<x<L$ and $0<t<T$,}\\
u&=\gamma_0(t)&&\text{at $x=0$, for $0<t<T$,}\\
u&=\gamma_L(t)&&\text{at $x=L$, for $0<t<T$,}\\
u&=u_0(x)&&\text{for $0<x<L$ when $t=0$,}
\end{aligned}
\end{equation}
where $u_t=\partial u/\partial t$ and $u_{xx}=\partial^2u/\partial x^2$.  For
simplicity, we assume that the coefficient~$a$ is a positive constant.  The
problem~\eqref{eq: heat ivp 1d} provides a model of heat conduction in 1D,
where $u(x,t)$ is the temperature at position~$x$ and time~$t$.  The
coefficient~$a>0$ is the \emph{thermal conductivity}: the value of~$a$ will be
large for a material that conducts heat well, but small for a
material that conducts heat poorly (that is, for a thermal insulator).  The
\emph{source term}~$f(x,t)$ gives the density of any heat sources in the
material, the \emph{boundary conditions} specify the temperatures $\gamma_0$~and
$\gamma_L$ at the two edges of the spatial domain~$[0,L]$, and the
\emph{initial condition} gives the temperature field~$u_0(x)$ when~$t=0$.  In
this 1D model, the temperature does not vary in the $y$~and $z$ directions.

\section{Separation of variables}\label{sec: separation}

Let
\[
\gamma(x,t)=\frac{1}{L}\bigl((L-x)\gamma_0(t)+x\gamma_L(t)\bigr),
\]
so that
\[
\gamma(0,t)=\gamma_0(t)\quad\text{and}\quad\gamma(L,t)=\gamma_L(t),
\]
with
\[
\gamma_t-a\gamma_{xx}=\frac{1}{L}\bigl((L-x)\gamma_0'(t)+x\gamma_L'(t)\bigr).
\]
Thus, the difference $v=u-\gamma$ satisfies
\begin{equation}\label{eq: heat ivp homog 1d}
\begin{aligned}
v_t-av_{xx}&=g(x,t)&&\text{for $0<x<L$ and $0<t<T$,}\\
v&=0&&\text{at $x=0$, for $0<t<T$,}\\
v&=0&&\text{at $x=L$, for $0<t<T$,}\\
v&=v_0(x)&&\text{for $0<x<L$ when $t=0$,}
\end{aligned}
\end{equation}
where
\[
g(x,t)=f(x,t)-\gamma_t(x,t)
\quad\text{and}\quad
v_0(x)=u_0(x)-\gamma(x,0).
\]
In this way, it suffices to solve a problem with \emph{homogeneous} boundary
conditions.

Associated with the time-dependent problem~\eqref{eq: heat ivp homog 1d} is the
Sturm--Liouville \emph{eigenproblem},
\begin{equation}\label{eq: simple eigenproblem}
-a\phi''=\lambda\phi\quad\text{for $0<x<L$,}
	\quad\text{with $\phi(0)=0=\phi(L)$,}
\end{equation}
which has only the trivial solution~$\phi(x)\equiv0$ unless $\lambda$ is one of
the \emph{eigenvalues}
\[
\lambda_n=a\biggl(\frac{n\pi}{L}\biggr)^2\quad\text{for $n\in\{1,2,3,\ldots\}$.}
\]
When $\lambda=\lambda_n$, the solution~$\phi$ is a constant multiple of the
corresponding \emph{eigenfunction}
\[
\phi_n(x)=\sin\frac{n\pi}{L}\,x.
\]
The eigenfunctions are \emph{orthogonal} with respect to the
\emph{inner product}
\[
\langle f,g\rangle=\int_0^Lf(x)g(x)\,dx,
\]
that is,
\[
\langle\phi_n,\phi_k\rangle=0\quad\text{if $n\ne k$,}
\]
and their 2-norms are given by
\[
\|\phi_n\|^2=\langle\phi_n,\phi_n\rangle=\frac{L}{2}.
\]

Given a square-integrable function~$f(x)$, we define the
\emph{Fourier sine coefficients}
\[
\hat f_n=\frac{\langle f,\phi_n\rangle}{\|\phi_n\|^2}
    =\frac{2}{L}\int_0^Lf(x)\sin\biggl(\frac{n\pi}{L}\,x\biggr)\,dx
    \quad\text{for $n\in\{1,2,3,\ldots\}$.}
\]
The \emph{completeness} of the eigenfunctions means that the Fourier sine series
\[
Sf(x)=\sum_{n=1}^\infty\hat f_n\sin\frac{n\pi}{L}\,x
\]
converges to~$f(x)$ in the \emph{mean-square sense}.  The Fourier coefficients
of~$v(x,t)$ are functions of~$t$,
\[
\hat v_n(t)=\langle v(\cdot,t),\phi_n\rangle
    =\frac{2}{L}\int_0^Lv(x,t)\sin\biggl(\frac{n\pi}{L}\,x\biggr)\,dx,
\]
and formal term-by-term differentiation of the Fourier expansion of~$v$ gives
\[
v_t(x,t)=\frac{\partial}{\partial t}\sum_{n=1}^\infty\hat v_n(t)\phi_n(x)
    =\sum_{n=1}^\infty\frac{d\hat v_n}{dt}\,\phi_n(x).
\]
Since $\phi=\phi_n$ satisfies \eqref{eq: simple eigenproblem}
with~$\lambda=\lambda_n$,
\[
-av_{xx}=-a\,\frac{\partial^2}{\partial x^2}
    \sum_{n=1}^\infty\hat v_n(t)\phi_n(x)
    =\sum_{n=1}^\infty\hat v_n(t)\bigl(-a\phi_n''(x)\bigr)
    =\sum_{n=1}^\infty\hat v_n(t)\bigl(\lambda_n\phi_n(x)\bigr),
\]
and so
\[
v_t-av_{xx}=\sum_{n=1}^\infty
	\biggl(\frac{d\hat v_n}{dt}+\lambda_n\hat v_n\biggr)\phi_n(x),
\]
which equals $g(x,t)$ if and only if all of the Fourier coefficients match,
that is,
\[
\frac{d\hat v_n}{dt}+\lambda_n\hat v_n=\hat g_n(t)
    \quad\text{for $0<t<T$ and $n\in\{1,2,3,\ldots\}$.}
\]
Multiplying both sides of this linear, first-order ODE by the
\emph{integrating factor}~$e^{\lambda_n t}$ we obtain
\[
e^{\lambda_nt}\,\frac{d\hat v_n}{dt}+\lambda_ne^{\lambda_nt}\hat v_n
	=e^{\lambda_nt}\hat g_n(t),
\]
and since the left-hand side equals $(d/dt)(e^{\lambda_nt}\hat v_n)$ it
follows that
\[
e^{\lambda_nt}\hat v_n(t)-\hat v_n(0)=\int_0^te^{\lambda_ns}\hat g_n(s)\,ds.
\]
Thus,
\begin{equation}\label{eq: Duhamel n}
\hat v_n(t)=\hat v_n(0)e^{-\lambda_nt}
	+\int_0^te^{-\lambda_n(t-s)}\hat g_n(s)\,ds\quad\text{for $t\ge0$,}
\end{equation}
and multiplying both sides by~$\phi_n(x)$ and summing over~$n$, we obtain the
series representation
\[
v(x,t)=\sum_{n=1}^\infty\hat v_n(t)\phi_n(x)
	=\sum_{n=1}^\infty e^{-\lambda_nt}\hat v_n(0)\phi_n(x)
	+\int_0^t\sum_{n=1}^\infty e^{-\lambda_n(t-s)}\hat g_n(s)\phi_n(x)\,ds.
\]

Define the differential operator~$\mathcal{A}v=-v''$ so that
$\mathcal{A}\phi_n=\lambda_n\phi_n$, and then define the linear
operator~$e^{-t\mathcal{A}}$ by
\[
(e^{-t\mathcal{A}}v)(x)=\sum_{n=1}^\infty e^{-\lambda_nt}\hat v_n\phi_n(x).
\]
In particular, this definition means that
\[
e^{-t\mathcal{A}}\phi_n=e^{-\lambda_nt}\phi_n
	\quad\text{for $n\in\{1,2,3,\ldots\}$,}
\]
and
\[
\sum_{n=1}^\infty e^{-\lambda_nt}\hat v_n(0)\phi_n(x)=e^{-t\mathcal{A}}v_0(x)
\]
with
\[
\sum_{n=1}^\infty e^{-\lambda_n(t-s)}\hat g_n(s)\phi_n(x)
	=e^{-(t-s)\mathcal{A}}g(x,s),
\]
so \eqref{eq: Duhamel n} can be written more succinctly as
\[
v(x,t)=e^{-t\mathcal{A}}v_0(x)+\int_0^te^{-(t-s)\mathcal{A}}g(x,s)\,ds.
\]
This representation of~$v$ is known as the \emph{Duhamel formula}, and the
solution of our original problem~\eqref{eq: heat ivp 1d} is then given by
\[
u(x,t)=\gamma(x,t)+v(x,t)\quad\text{for $0\le x\le L$ and $0\le t\le T$.}
\]

\section{Semidiscrete method}

We define a uniform grid on the spatial interval~$[0,L]$,
\[
x_p=p\,\Delta x\quad\text{for $0\le p\le P$,}
	\quad\text{with $\Delta x=\frac{L}{P}$,}
\]
and let
\[
U_p(t)\approx u(x_p,t),\qquad f_p(t)=f(x_p,t),\qquad u_{0p}=u_0(x_p).
\]
Since $u_t(x_p,t)\approx dU_p/dt$ and
\[
u_{xx}(x_p,t)\approx\frac{u(x_{p+1},t)-2u(x_p,t)+u(x_{p-1},t)}{\Delta x^2}
	\approx\frac{U_{p+1}(t)-2U_p(t)+U_{p-1}(t)}{\Delta x^2},
\]
we formulate the following semidiscrete approximation to the initial-boundary
value problem~\eqref{eq: heat ivp 1d},
\begin{equation}\label{eq: semidiscrete heat 1d}
\begin{aligned}
\frac{dU_p}{dt}-a\,\frac{U_{p+1}-2U_p+U_{p-1}}{\Delta x^2}&=f_p(t)&
	&\text{for $1\le p\le P-1$ and $0<t<T$,}\\
U_0&=\gamma_0(t)&&\text{for $0<t<T$,}\\
U_P&=\gamma_P(t)&&\text{for $0<t<T$,}\\
U_p&=u_{0p}&&\text{for $1\le p\le P-1$ when $t=0$.}
\end{aligned}
\end{equation}
This system can be written in matrix-vector notation as
\begin{equation}\label{eq: ODE system}
\frac{d\boldsymbol{U}}{dt}+\boldsymbol{A}\boldsymbol{U}=\boldsymbol{f}(t)
	+\boldsymbol{g}(t),
\end{equation}
where
\[
\boldsymbol{A}=\frac{a}{\Delta x^2}\begin{bmatrix}
 2&    -1&      &      &\\
-1&     2&    -1&      &\\
  &\ddots&\ddots&\ddots&\\
  &      &    -1&     2&-1\\
  &      &      &    -1& 2\end{bmatrix},\qquad
\]
and
\begin{equation}\label{eq: U f g semidiscrete 1d}
\boldsymbol{U}(t)=\begin{bmatrix}
U_1(t)\\ U_2(t)\\ \vdots\\ U_{P-2}(t)\\ U_{P-1}(t)\end{bmatrix},\qquad
\boldsymbol{f}(t)=\begin{bmatrix}
f_1(t)\\ f_2(t)\\ \vdots\\ f_{P-2}(t)\\ f_{P-1}(t)\end{bmatrix},\qquad
\boldsymbol{g}(t)=\frac{a}{\Delta x^2}\begin{bmatrix}
\gamma_0(t)\\ 0\\ \vdots\\ 0\\ \gamma_L(t)\end{bmatrix}.
\end{equation}
Recall that for any square matrix~$\boldsymbol{A}$, the matrix exponential
is defined by the convergent infinite series
\[
e^{\boldsymbol{A}}=\sum_{k=0}^\infty\frac{\boldsymbol{A}^n}{n!}
	=\boldsymbol{I}+\boldsymbol{A}+\frac{\boldsymbol{A}^2}{2!}
	+\frac{\boldsymbol{A}^3}{3!}+\cdots.
\]
Furthermore,
\[
\frac{d}{dt}\,e^{t\boldsymbol{A}}=\boldsymbol{A}e^{t\boldsymbol{A}},
\]
so $e^{t\boldsymbol{A}}$ provides an integrating factor
for~\eqref{eq: ODE system}:
\[
\frac{d}{dt}\bigl(e^{t\boldsymbol{A}}\boldsymbol{U}\bigr)
	=\boldsymbol{A}e^{t\boldsymbol{A}}\boldsymbol{U}
	+e^{t\boldsymbol{A}}\,\frac{d\boldsymbol{U}}{dt}
	=e^{t\boldsymbol{A}}\biggl(\frac{d\boldsymbol{U}}{dt}
	+\boldsymbol{A}\boldsymbol{U}\biggr)
	=e^{t\boldsymbol{A}}\bigl(\boldsymbol{f}(t)+\boldsymbol{g}(t)\bigr),
\]
where we used the fact that $\boldsymbol{A}$ commutes
with~$e^{t\boldsymbol{A}}$.  If $t=0$ then $e^{t\boldsymbol{A}}=\boldsymbol{I}$
is the identity matrix, so
\[
e^{t\boldsymbol{A}}\boldsymbol{U}(t)-\boldsymbol{U}(0)
	=\int_0^te^{s\boldsymbol{A}}\bigl(
		\boldsymbol{f}(s)+\boldsymbol{g}(s)\bigr)\,ds.
\]
Using $e^{-t\boldsymbol{A}}=(e^{t\boldsymbol{A}})^{-1}$~and
$e^{-t\boldsymbol{A}}e^{s\boldsymbol{A}}=e^{-(t-s)\boldsymbol{A}}$, we obtain
a semidiscrete Duhamel formula,
\[
\boldsymbol{U}(t)=e^{-t\boldsymbol{A}}\boldsymbol{u}_0
	+\int_0^te^{-(t-s)\boldsymbol{A}}\bigl(
		\boldsymbol{f}(s)+\boldsymbol{g}(s)\bigr)\,ds,
\]
where $\boldsymbol{u}_0=[u_0(x_p)]_{p=1}^{P-1}$ is the vector of initial data.

\section{Explicit Euler method}

To obtain a fully-discrete scheme, we introduce a uniform grid on the time axis,
\[
t_n=n\,\Delta t\quad\text{for $0\le n\le N$,}
	\quad\text{where $\Delta t=\frac{T}{N}$,}
\]
and let
\[
U^n_p\approx u(x_p,t_n),\quad
f^n_p=f(x_p,t_n),\quad
\gamma_0^n=\gamma_0(t_n),\quad
\gamma_L^n=\gamma_L(t_n),\quad
u_{0p}=u_0(x_p).
\]
The \emph{forward difference} approximation in time,
\[
u_t(x_p,t_n)\approx\frac{u(x_p,t_n+\Delta t)-u(x_p,t_n)}{\Delta t}
	=\frac{u(x_p,t_{n+1})-u(x_p,t_n)}{\Delta t},
\]
and the second-order central difference approximation in space,
\begin{align*}
u_{xx}(x_p,t_n)
&\approx\frac{u(x_p+\Delta x,t_n)-2u(x_p,t_n)+u(x_p-\Delta x,t_n)}{\Delta x^2}\\
&=\frac{u(x_{p+1},t_n)-2u(x_p,t_n)+u(x_{p-1},t_n)}{\Delta x^2},
\end{align*}
lead to the \emph{explicit Euler method} for~\eqref{eq: heat ivp 1d}:
\begin{equation}\label{eq: explicit Euler 1d}
\begin{aligned}
\frac{U^{n+1}_p-U^n_p}{\Delta t}
	-a\,\frac{U^n_{p+1}-2U^n_p+U^n_{p-1}}{\Delta x^2}&=f^n_p&
&\text{for $1\le p\le P-1$ and $0\le n\le N-1$,}\\
U^n_0&=\gamma_0^n&&\text{for $0\le n\le N$,}\\
U^n_P&=\gamma_L^n&&\text{for $0\le n\le N$,}\\
U^0_p&=u_{0p}&&\text{for $1\le p\le P-1$.}
\end{aligned}
\end{equation}
We multiply both sides of the finite difference equation by~$\Delta t$ and put
\begin{equation}\label{eq: rho explicit Euler}
\rho=\frac{a\,\Delta t}{\Delta x^2}
\end{equation}
to obtain
\[
U^{n+1}_p-U^n_p-\rho(U^n_{p+1}-2U^n_p+U^n_{p-1})=f^n_p\,\Delta t
\]
and thus
\begin{equation}\label{eq: explicit Euler stencil}
U^{n+1}_p=f^n_p\,\Delta t+\rho U^n_{p-1}+(1-2\rho)U^n_p+\rho U^n_{p+1}.
\end{equation}
In this way, the scheme provides an \emph{explicit} formula for the solution
at~$(x_p,t_{n+1})$ given the values at the three nearest grid
points $(x_{p-1},t_n)$, $(x_p,t_n)$~and $(x_{p+1},t_n)$ from the previous time
level.  We say the these four points constitute the \emph{stencil} for the
method; see \cref{fig: explicit Euler stencil}.
\cref{alg: explicit Euler} shows how the
formula~\eqref{eq: explicit Euler stencil} is used to compute the finite
difference solution~$U^n_p$ by advancing from one time level to the next,
starting at~$t_0=0$.

\begin{figure}
\caption{Computational stencil for the explicit Euler
method~\eqref{eq: explicit Euler stencil}.}
\label{fig: explicit Euler stencil}
\begin{center}
\begin{tikzpicture}[scale=0.5]
\draw[->] (-1,0) -- (17,0);
\node[right] at (17,0) {$x$};
\node[below] at (8,0) {$x_p$};
\draw[->] (0,-1) -- (0,6);
\node[above left] at (0,6) {$t$};
\node[left] at (0,3) {$t_n$};
\foreach \x in {2, 4, ..., 16}
    \draw[thin] (\x,0) -- (\x,6);
\foreach \y in {1, ..., 5}
    \draw[thin] (0,\y) -- (16,\y);
\draw[ultra thick] (6,3) -- (8,3) -- (10,3);
\draw[fill] (6,3)  circle (0.15cm);
\draw[fill] (8,3)  circle (0.15cm);
\draw[fill] (10,3) circle (0.15cm);
\draw[ultra thick] (8,3) -- (8,4);
\draw[fill=red] (8,4)  circle (0.15cm);
\end{tikzpicture}
\end{center}
\end{figure}

\begin{algorithm}
\caption{Explicit Euler method.}
\label{alg: explicit Euler}
\begin{algorithmic}
\State Allocate storage for $x_p$, $t_n$~and $U^n_p$, where $0\le p\le P$ and
$0\le n\le N$.
\State $\Delta x=L/P$
\State$\Delta t=T/N$
\For{$p=0:P$}
    \State $x_p=p\,\Delta x$
\EndFor
\For{$n=0:N$}
    \State $t_n=n\,\Delta t$
    \State $U^n_0=\gamma_0^n$
    \State $U^n_P=\gamma_L^n$
\EndFor
\State $\rho=a\,\Delta t/\Delta x^2$
\For{$p=1:P-1$}
    \State $U^0_p=u_0(x_p)$
\EndFor
\For{$n=0:N-1$}
    \For{$p=1:P-1$}
        \State $U^{n+1}_p=f^n_p\,\Delta t
                +\rho U^n_{p-1}+(1-2\rho)U^n_p+\rho U^n_{p+1}$
    \EndFor
\EndFor
\end{algorithmic}
\end{algorithm}

We now show that the explicit Euler method is stable provided the time
step~$\Delta t$ is sufficiently small compared to the spatial grid
size~$\Delta x$.

\begin{theorem}\label{thm: explicit Euler stability}
If $\Delta x$ and $\Delta t$ satisfy
\begin{equation}\label{eq: explicit Euler timestep}
\Delta t\le\frac{\Delta x^2}{2a},
\end{equation}
or equivalently if the ratio~\eqref{eq: rho explicit Euler} satisfies
$\rho\le1/2$, then the explicit Euler method~\eqref{eq: explicit Euler 1d} is
stable:
\[
\|\boldsymbol{U}^n_{0:P}\|_\infty
	\le\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
		\|\gamma_0^{0:n}\|_\infty,\|\gamma_L^{0:n}\|_\infty\bigr\}
	+\sum_{j=0}^{n-1}\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t
\quad\text{for $0\le n\le N$.}
\]
\end{theorem}
\begin{proof}
We use finite induction on~$n$.  When~$n=0$, we have
$U^0_p=u_{0p}$ for~$1\le p\le P-1$, with $U^0_0=\gamma_0^0$~and
$U^0_P=\gamma_L^0$, so
\[
\|\boldsymbol{U}^0_{0:P}\|_\infty=
\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,|\gamma_0|,|\gamma_L|\bigr\},
\]
which agrees with the formula since the empty sum vanishes. Now let
$0\le n\le N-1$ and make the induction hypothesis.  We see
from~\eqref{eq: explicit Euler stencil} that
\[
|U^{n+1}_p|\le|f^n_p|\,\Delta t+\rho|U^n_{p-1}|+|1-2\rho|\,|U^n_p|
    +\rho|U^n_{p+1}|
\]
and thus
\[
|U^{n+1}_p|\le\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t
		+(2\rho+|1-2\rho|)\|\boldsymbol{U}^n_{0:P}\|_\infty
\quad\text{for $1\le p\le P-1$.}
\]
If $\rho\le1/2$, then $2\rho+|1-2\rho|=2\rho+1-2\rho=1$ so
\begin{align*}
\|\boldsymbol{U}^{n+1}_{1:P-1}\|_\infty
	&\le\|\boldsymbol{U}^n_{0:P}\|_\infty
		+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t\\
	&\le\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
		\|\gamma_0^{0:n}\|_\infty,\|\gamma_L^{0:n}\|_\infty\bigr\}
	+\sum_{j=0}^{n-1}\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t
		+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t\\
	&=\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
		\|\gamma_0^{0:n}\|_\infty,\|\gamma_L^{0:n}\|_\infty\bigr\}
	+\sum_{j=0}^n\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t,
\end{align*}
and since $U^{n+1}_0=\gamma_0^{n+1}$~and $U^{n+1}_P=\gamma_L^{n+1}$
\[
\|\boldsymbol{U}^{n+1}_{0:P}\|_\infty
	\le\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
		\|\gamma_0^{0:n+1}\|_\infty,\|\gamma_L^{0:n+1}\|_\infty\bigr\}
	+\sum_{j=0}^n\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t,
\]
as required.
\end{proof}

The \emph{local trunction error} for the explicit Euler method is defined by
\begin{equation}\label{eq: tau explicit Euler}
\tau(x,t)=f(x,t)-\frac{u(x,t+\Delta t)-u(x,t)}{\Delta t}
	+a\,\frac{u(x+\Delta x,t)-2u(x,t)+u(x-\Delta x,t)}{\Delta x^2},
\end{equation}
and measures the extent to which the solution~$u(x,t)$ of the continuous
problem~\eqref{eq: heat ivp 1d} fails to satisfy the finite difference equation.
We can estimate $\tau^n_p=\tau(x_p,t_n)$ by Taylor expansion.

\begin{lemma}\label{lem: tau explicit Euler}
If $u_{tt}$~and $u_{xxxx}$ are continuous for~$(x,t)\in[0,L]\times[0,T]$, then
\[
|\tau^n_p|\le\frac{\Delta t}{2}\max_{[0,L]\times[0,T]}|u_{tt}|
	+a\,\frac{\Delta x^2}{12}\max_{[0,L]\times[0,T]}|u_{xxxx}|
\]
for $1\le p\le P-1$ and $0\le n\le N-1$.
\end{lemma}
\begin{proof}
Since $u_t-au_{xx}=f$,
\[
|\tau(x,t)|=\biggl|u_t-\frac{u(x,t+\Delta t)-u(x,t)}{\Delta t}
	-a\biggl(u_{xx}-\frac{u(x+\Delta x,t)-2u(x,t)+u(x-\Delta x,t)}{\Delta x^2}
\biggr)\biggr|,
\]
and, so by Exercise~\ref{ex: forward diff}~and \cref{thm: 2nd central diff},
\[
|\tau(x,t)|\le\frac{\Delta t}{2}\max_{t\le s\le t+\Delta t}|u_{tt}(x,s)|
	+a\,\frac{\Delta x^2}{12}\,
	\max_{x-\Delta x\le y\le x+\Delta x}|u_{xxxx}(y,t)|,
\]
which implies the stated estimate for~$\tau^n_p$.
\end{proof}

The next theorem shows that, provided the
restriction~\eqref{eq: explicit Euler timestep} on the step sizes is satisfied,
the explicit Euler method is first-order accurate in time and second-order
accurate in space.

\begin{theorem}\label{thm: explicit Euler error}
Assume that $\Delta x$~and $\Delta t$ satisfy
\eqref{eq: explicit Euler timestep}, and that $u_{tt}$~and $u_{xxxx}$ are
continuous on~$[0,L]\times[0,T]$. Then the error for the explicit Euler method
satisfies
\[
|U^n_p-u(x_p,t_n)|\le Ct_n(\Delta t+\Delta x^2)
\quad\text{for $0\le p\le P-1$ and $0\le n\le N$,}
\]
where
\begin{equation}\label{eq: error const explicit Euler}
C=\max\biggl(\frac{1}{2}\max_{[0,L]\times[0,T]}|u_{tt}|,
	\frac{1}{12}\max_{[0,L]\times[0,T]}|u_{xxxx}|\biggr).
\end{equation}
\end{theorem}
\begin{proof}
By the definition~\eqref{eq: tau explicit Euler} of the local truncation error,
\[
\frac{u(x_p,t_{n+1})-u(x_p,t_n)}{\Delta t}
	-a\,\frac{u(x_{p+1},t)-2u(x_p,t_n)+u(x_{p-1},t_n)}{\Delta x^2}
	=f^n_p-\tau^n_p.
\]
Subtracting this equation from the finite difference equation
in~\eqref{eq: explicit Euler 1d}, we obtain a finite difference equation for
the error ~$E^n_p=U^n_p-u(x_p,t_n)$ in the finite difference solution, namely
\[
\frac{E^{n+1}_p-E^n_p}{\Delta t}
	-a\,\frac{E^n_{p+1}-2E^n_p+E^n_{p-1}}{\Delta x^2}=\tau^n_p
\quad\text{for $1\le p\le P-1$ and $0\le n\le N-1$.}
\]
In addition, the boundary and initial conditions satisfied by $U^n_p$~and $u$
imply that
\[
\begin{aligned}
E^n_0&=U^n_0-u(x_0,t_n)=\gamma_0^n-\gamma_0^n=0&&\text{for $0\le n\le N$,}\\
E^n_P&=U^n_P-u(x_P,t_n)=\gamma_L^n-\gamma_L^n=0&&\text{for $0\le n\le N$,}\\
E^0_p&=U^0_p-u(x_p,t_0)=u_0(x_p)-u_0(x_p)=0&&\text{for $1\le p\le P-1$.}
\end{aligned}
\]
Thus, $E^n_p$ is the explicit Euler solution obtained when $f^n_p$,
$\gamma_0^n$, $\gamma_L^n$~and $u_{0p}$ are replaced by $\tau^n_p$, $0$,
$0$~and $0$, respectively.  The restriction on the step sizes allows us to apply
the stability estimate of \cref{thm: explicit Euler stability} to deduce
that
\[
\|\boldsymbol{E}^n_{0:P}\|_\infty
	\le\sum_{j=0}^{n-1}\|\boldsymbol{\tau}^n_{1:P-1}\|_\infty\,\Delta t
	\quad\text{for $0\le n\le N$.}
\]
By Lemma~\ref{lem: tau explicit Euler},
$\|\boldsymbol{\tau}^n_{1:P-1}\|_\infty\le C(\Delta t+\Delta x^2)$, so
\[
\|\boldsymbol{E}^n_{0:P}\|_\infty\le Cn(\Delta t+\Delta x^2)\,\Delta t
	=Ct_n(\Delta t+\Delta x^2),
\]
as claimed.
\end{proof}

\section{Implicit Euler method}

Instead of using a forward difference, we can approximate $u_t$ by a
\emph{backward difference},
\[
u_t(x_p,t_n)\approx\frac{u(x_p,t_n)-u(x_p,t_n-\Delta t)}{\Delta t}
	=\frac{u(x_p,t_n)-u(x_p,t_{n-1})}{\Delta t},
\]
which leads to the \emph{im}plicit Euler method,
\begin{equation}\label{eq: implicit Euler 1d}
\begin{aligned}
\frac{U^n_p-U^{n-1}_p}{\Delta t}
	-a\,\frac{U^n_{p+1}-2U^n_p+U^n_{p-1}}{\Delta x^2}&=f^n_p&
&\text{for $1\le p\le P-1$ and $1\le n\le N$,}\\
U^n_0&=\gamma_0^n&&\text{for $0\le n\le N$,}\\
U^n_P&=\gamma_L^n&&\text{for $0\le n\le N$,}\\
U^0_p&=u_{0p}&&\text{for $1\le p\le P-1$.}
\end{aligned}
\end{equation}
Multiplying both sides of the finite difference equation by~$\Delta t$, and
defining $\rho$ as before in~\eqref{eq: rho explicit Euler}, we obtain
\[
U^n_p-U^{n-1}_p-\rho(U^n_{p+1}-2U^n_p+U^n_{p-1})=f^n_p\,\Delta t
\]
and thus
\begin{equation}\label{eq: implicit Euler stencil}
-\rho U^n_{p-1}+(1+2\rho)U^n_p-\rho U^n_{p+1}=U^{n-1}_p+f^n_p\,\Delta t
\end{equation}
for $1\le p\le P-1$ and $1\le n\le N$.  Thus, at the $n$th time level we have a
system of linear equations for the unknown $U^n_1$, $U^n_2$, \dots, $U^n_{P-1}$,
with the right-hand sides involving the solution at the previous time level.
The stencil for this scheme is shown in \cref{fig: implicit Euler stencil}.

\begin{figure}
\caption{Computational stencil for the implicit Euler
method~\eqref{eq: implicit Euler stencil}.}
\label{fig: implicit Euler stencil}
\begin{center}
\begin{tikzpicture}[scale=0.5]
\draw[->] (-1,0) -- (17,0);
\node[right] at (17,0) {$x$};
\node[below] at (8,0) {$x_p$};
\draw[->] (0,-1) -- (0,6);
\node[above left] at (0,6) {$t$};
\node[left] at (0,3) {$t_n$};
\foreach \x in {2, 4, ..., 16}
    \draw[thin] (\x,0) -- (\x,6);
\foreach \y in {1, ..., 5}
    \draw[thin] (0,\y) -- (16,\y);
\draw[ultra thick] (8,2) -- (8,3);
\draw[fill] (8,2)  circle (0.15cm);
\draw[ultra thick] (6,3) -- (8,3) -- (10,3);
\draw[fill=red] (6,3)  circle (0.15cm);
\draw[fill=red] (8,3)  circle (0.15cm);
\draw[fill=red] (10,3) circle (0.15cm);
\end{tikzpicture}
\end{center}
\end{figure}

Alternatively, we can start from the matrix form of the semidiscrete
method~\eqref{eq: ODE system}, and approximate $d\boldsymbol{U}/dt$ with a
backward difference to obtain
\begin{equation}\label{eq: implicit Euler 1d vector}
\frac{\boldsymbol{U}^n-\boldsymbol{U}^{n-1}}{\Delta t}
	+\boldsymbol{A}\boldsymbol{U}^n=\boldsymbol{f}^n+\boldsymbol{g}^n
\quad\text{for $1\le n\le N$,}
\end{equation}
where
\[
\boldsymbol{A}=\frac{a}{\Delta x^2}\begin{bmatrix}
 2&    -1&      &      &\\
-1&     2&    -1&      &\\
  &\ddots&\ddots&\ddots&\\
  &      &    -1&     2&-1\\
  &      &      &    -1& 2\end{bmatrix},\qquad
\]
and
\[
\boldsymbol{U}^n=\begin{bmatrix}
U^n_1\\ U^n_2\\ \vdots\\ U^n_{P-2}\\ U^n_{P-1}\end{bmatrix},\qquad
\boldsymbol{f}^n=\begin{bmatrix}
f^n_1\\ f^n_2\\ \vdots\\ f^n_{P-2}\\ f^n_{P-1}\end{bmatrix},\qquad
\boldsymbol{g}^n=\frac{a}{\Delta x^2}\begin{bmatrix}
\gamma_0^n\\ 0\\ \vdots\\ 0\\ \gamma_L^n\end{bmatrix}.
\]
Rearranging \eqref{eq: implicit Euler 1d vector} we find that
\[
(\boldsymbol{I}+\Delta t\,\boldsymbol{A})\boldsymbol{U}^n
	=\boldsymbol{U}^{n-1}+\Delta t\,(\boldsymbol{f}^n+\boldsymbol{g}^n)
\quad\text{for $1\le n\le N$.}
\]
Here, the coefficient matrix is symmetric, tridiagonal and positive-definite,
so we can compute~$\boldsymbol{U}^n$ via a factorization
\[
\boldsymbol{I}+\Delta t\,\boldsymbol{A}
	=\boldsymbol{L}^T\boldsymbol{D}\boldsymbol{L},
\]
as shown in \cref{alg: implicit Euler}.

\begin{algorithm}
\caption{Implicit Euler method.}
\label{alg: implicit Euler}
\begin{algorithmic}
\State Allocate storage for $x_p$, $t_n$~and $U^n_p$, where $0\le p\le P$ and
$0\le n\le N$.
\State Allocate storage for vectors $[d_1,d_2,\ldots,d_{P-1}]^T$~and
$[\ell_1,\ell_2,\ldots,\ell_{P-2}]^T$.
\State $\Delta x=L/P$
\State$\Delta t=T/N$
\For{$p=0:P$}
    \State $x_p=p\,\Delta x$
\EndFor
\For{$n=0:N$}
    \State $t_n=n\,\Delta t$
    \State $U^n_0=\gamma_0^n$
    \State $U^n_P=\gamma_L^n$
\EndFor
\State $\rho=a\,\Delta t/\Delta x^2$
\For{$p=1:P-1$}
    \State $d_p=1+2\rho$
\EndFor
\For{$p=1:P-2$}
    \State $\ell_p=-\rho$
\EndFor
\State \textsc{Factorize!}($\boldsymbol{d}$, $\boldsymbol{\ell}$)
\Comment Use \cref{alg: LDLT in place} to compute
$\boldsymbol{I}+\Delta t\,\boldsymbol{A}=\boldsymbol{L}\boldsymbol{D}
\boldsymbol{L}^T$ in place.
\For{$p=1:P-1$}
    \State $U^0_p=u_0(x_p)$
\EndFor
\For{$n=1:N$}
    \For{$p=1:P-1$}
        \State $U^n_p\gets U^{n-1}_p+\Delta t\,f(x_p,t_n)$
    \EndFor
    \State $U^n_1\gets U^n_1+a\gamma_0/\Delta x^2$
    \State $U^n_{P-1}\gets U^n_{P-1}+a\gamma_L/\Delta x^2$
    \State \textsc{Solve}!($\boldsymbol{U}^n_{1:P-1}$,
$\boldsymbol{d}$, $\boldsymbol{\ell}$)
\Comment Use \cref{alg: solve symmetric tridiagonal in place} to solve
for $\boldsymbol{U}^n_{1:P-1}$ in place.
\EndFor
\end{algorithmic}
\end{algorithm}

\begin{theorem}
The implicit Euler method is unconditionally stable: for any choice of
$\Delta x$~and $\Delta t$,
\[
\|\boldsymbol{U}^n_{0:P}\|_\infty
	\le\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
		\|\gamma_0^{0:n}\|_\infty,\|\gamma_L^{0:n}\|_\infty\bigr\}
	+\sum_{j=1}^n\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t
\quad\text{for $0\le n\le N$.}
\]
\end{theorem}
\begin{proof}
We use finite induction on~$n$.  When~$n=0$, we have
$U^0_p=u_{0p}$ for~$1\le p\le P-1$, with $U^0_0=\gamma_0^0$~and
$U^0_P=\gamma_L^0$,
so
\[
\|\boldsymbol{U}^0_{0:P}\|_\infty=
\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
	|\gamma_0^0|,|\gamma_L^0|\bigr\},
\]
which agrees with the formula since the empty sum vanishes. Now let
$1\le n\le N$ and make the induction hypothesis that
\[
\|\boldsymbol{U}^{n-1}_{0:P}\|_\infty
	\le\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
		\|\gamma_0^{0:n-1}\|_\infty,\|\gamma_L^{0:n-1}\|_\infty\bigr\}
	+\sum_{j=1}^{n-1}\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t.
\]
We see from~\eqref{eq: implicit Euler stencil} that, for $1\le p\le P-1$,
\[
(1+2\rho)U^n_p=\rho U^n_{p-1}+\rho U^n_{p+1}+U^{n-1}_p+f^n_p\,\Delta t
\]
and so
\[
(1+2\rho)|U^n_p|\le2\rho\|\boldsymbol{U}^n_{0:P}\|_\infty
	+\|\boldsymbol{U}^{n-1}_{1:P-1}\|_\infty
	+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t.
\]
Thus,
\[
(1+2\rho)\|\boldsymbol{U}^n_{1:P-1}\|_\infty
	\le2\rho\|\boldsymbol{U}^n_{0:P}\|_\infty
	+\|\boldsymbol{U}^{n-1}_{1:P-1}\|_\infty
	+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t,
\]
and since $U^n_0=\gamma_0=U^{n-1}_0$~and $U^n_P=\gamma_L=U^{n-1}_P$,
\[
(1+2\rho)\|\boldsymbol{U}^n_{0:P}\|_\infty
	\le2\rho\|\boldsymbol{U}^n_{0:P}\|_\infty
	+\|\boldsymbol{U}^{n-1}_{0:P}\|_\infty
	+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t.
\]
After canceling $2\rho\|\boldsymbol{U}^n_{0:P}\|_\infty$, it follows that
\begin{align*}
\|\boldsymbol{U}^n_{1:P-1}\|_\infty&\le\|\boldsymbol{U}^{n-1}_{0:P}\|_\infty
	+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t\\
	&\le\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
	\|\gamma_0^{0:n-1}\|_\infty,\|\gamma_L^{0:n-1}\|_\infty\bigr\}
	+\sum_{j=1}^{n-1}\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t
	+\|\boldsymbol{f}^n_{1:P-1}\|_\infty\,\Delta t\\
	&=\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
	\|\gamma_0^{0:n-1}\|_\infty,\|\gamma_L^{0:n-1}\|_\infty\bigr\}
	+\sum_{j=1}^n\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t,
\end{align*}
and so, since $U^n_0=\gamma_0^n$~and $U^n_P=\gamma_L^n$,
\[
\|\boldsymbol{U}^n_{0:P}\|_\infty
	=\max\bigl\{\|(\boldsymbol{u}_0)_{1:P-1}\|_\infty,
	\|\gamma_0^{0:n}\|_\infty,\|\gamma_L^{0:n}\|_\infty\bigr\}
	+\sum_{j=1}^n\|\boldsymbol{f}^j_{1:P-1}\|_\infty\,\Delta t,
\]
as required.
\end{proof}

The local truncation error for the \emph{im}plicit Euler method is defined by
\[
\tau(x,t)=f(x,t)-\frac{u(x,t)-u(x,t-\Delta t)}{\Delta t}
	+a\,\frac{u(x+\Delta x,t)-2u(x,t)+u(x-\Delta x,t)}{\Delta x^2},
\]
and we can again estimate $\tau^n_p=\tau(x_p,t_n)$ by Taylor expansion
(Exercise~\ref{ex: forward diff} and \cref{thm: 2nd central diff}):
\[
|\tau^n_p|\le\frac{\Delta t}{2}\max_{[0,L]\times[0,T]}|u_{tt}|
	+a\,\frac{\Delta x^2}{12}\max_{[0,L]\times[0,T]}|u_{xxxx}|
\]
for $1\le p\le P-1$~and $1\le n\le N$, which leads to the following error bound.

\begin{theorem}\label{thm: implicit Euler error}
Assume that $u_{tt}$~and $u_{xxxx}$ are continuous on~$[0,L]\times[0,T]$. Then
the error for the implicit Euler method satisfies
\[
|U^n_p-u(x_p,t_n)|\le Ct_n(\Delta t+\Delta x^2)
\quad\text{for $0\le p\le P-1$ and $0\le n\le N$,}
\]
where $C$ is defined by~\eqref{eq: error const explicit Euler}.
\end{theorem}
\begin{proof}
See Exercise~\ref{ex: implicit Euler error}.
\end{proof}

\section{Crank--Nicolson method}
Let us introduce the half-time levels
\[
t_{n-1/2}=t_n-\tfrac12\Delta t=\frac{t_n+t_{n-1}}{2}
    \quad\text{for $1\le n\le N$.}
\]
The implicit Euler method is only first-order accurate in time because the
first-order backward difference approximates $u_t(x_p,t_n)$ only to first order
in~$\Delta t$, that is,
\[
\frac{u(x,t_n)-u(x,t_{n-1})}{\Delta t}=u_t(x_p,t_n)+O(\Delta t).
\]
However, viewed as an approximation to~$u_t(x_p,t_{n-1/2})$, the backward
difference becomes a central difference (with step-size $\tfrac12\Delta t$),
and is therefore second-order accurate by \cref{thm: first central diff}:
\[
\frac{u(x,t_n)-u(x,t_{n-1})}{\Delta t}=u_t(x_p,t_{n-1/2})+O(\Delta t^2).
\]
Similarly, the theorem below shows that
\[
\frac{u(x,t_n)+u(x,t_{n-1})}{2}=u(x_p,t_{n-1/2})+O(\Delta t^2).
\]

\begin{theorem}\label{thm: average}
If $f$ is $C^2$ on the closed interval~$[x-h,x+h]$, then
\[
\biggl|f(x)-\frac{f(x+h)+f(x-h)}{2}\biggr|\le\frac{h^2}{2}
    \max_{x-h\le y\le x+h}|f''(y)|.
\]
\end{theorem}
\begin{proof}
Since
\[
f(x+h)=f(x)+f'(x)h+(R_1f)(x,h)
\quad\text{and}\quad
f(x-h)=f(x)-f'(x)h+(R_1f)(x,-h),
\]
we have
\[
f(x+h)+f(x-h)=2f(x)+(R_1f)(x,h)+(R_1f)(x,-h).
\]
Thus,
\[
\frac{f(x+h)+f(x-h)}{2}-f(x)=\frac{(R_1f)(x,h)+(R_1f)(x,-h)}{2},
\]
so the result follows from \cref{thm: Taylor remainder}.
\end{proof}

\begin{figure}
\caption{Computational stencil for the Crank--Nicolson
method~\eqref{eq: Crank-Nicolson 1d}.}\label{fig: Crank-Nicolson 1d}
\begin{center}
\begin{tikzpicture}[scale=0.5]
\draw[->] (-1,0) -- (17,0);
\node[right] at (17,0) {$x$};
\node[below] at (8,0) {$x_p$};
\draw[->] (0,-1) -- (0,6);
\node[above left] at (0,6) {$t$};
\node[left] at (0,3) {$t_n$};
\foreach \x in {2, 4, ..., 16}
    \draw[thin] (\x,0) -- (\x,6);
\foreach \y in {1, ..., 5}
    \draw[thin] (0,\y) -- (16,\y);
\draw[ultra thick] (6,2) -- (8,2) -- (10,2);
\draw[fill] (6,2)  circle (0.15cm);
\draw[fill] (8,2)  circle (0.15cm);
\draw[fill] (10,2) circle (0.15cm);
\draw[ultra thick] (8,2) -- (8,3);
\draw[ultra thick] (6,3) -- (8,3) -- (10,3);
\draw[fill=red] (6,3)  circle (0.15cm);
\draw[fill=red] (8,3)  circle (0.15cm);
\draw[fill=red] (10,3) circle (0.15cm);
\end{tikzpicture}
\end{center}
\end{figure}

This observation leads us to consider the \emph{Crank--Nicolson method}
for~\eqref{eq: heat ivp 1d}:
\begin{equation}\label{eq: Crank-Nicolson 1d}
\frac{U^n_p-U^{n-1}_p}{\Delta t}
    -a\,\frac{U^{n-1/2}_{p+1}-2U^{n-1/2}_p+U^{n-1/2}_{p-1}}{\Delta x^2}
    =f^{n-1/2}_p
\end{equation}
for $1\le p\le P-1$ and $1\le n\le N$, where we have used the notation
\[
U^{n-1/2}_p=\frac{U^n_p+U^{n-1}_p}{2}\approx u(x_p,t_{n-1/2}),
\]
and, on the right-hand side,
\[
f^{n-1/2}_p=\frac{f^n_p+f^{n-1}_p}{2}\approx f(x_p,t_{n-1/2})
\quad\text{or}\quad
f^{n-1/2}_p=f(x_p,t_{n-1/2}).
\]
The usual discrete boundary conditions are imposed,
\[
U^n_0=\gamma_0^n\quad\text{and}\quad
U^n_P=\gamma_L^n\quad\text{for $0\le n\le N$,}
\]
along with the usual discrete initial condition
\[
U^0_p=u_{0p}\quad\text{for $1\le p\le P-1$.}
\]
\cref{fig: Crank-Nicolson 1d} shows the stencil for the scheme.
In matrix notation, the Crank--Nicolson method takes the form
\begin{equation}\label{eq: Crank-Nicolson matrix}
\frac{\boldsymbol{U}^n-\boldsymbol{U}^{n-1}}{\Delta t}
    +\boldsymbol{A}\boldsymbol{U}^{n-1/2}=\boldsymbol{f}^{n-1/2}
    +\boldsymbol{g}^{n-1/2},
\end{equation}
where the $(P-1)\times(P-1)$ matrix~$\boldsymbol{A}$ and
is same as in~\eqref{eq: implicit Euler 1d vector}, with
\[
\boldsymbol{g}^{n-1/2}=\frac{\boldsymbol{g}(t_n)+\boldsymbol{g}(t_{n-1})}{2}
\quad\text{or}\quad
\boldsymbol{g}^{n-1/2}=\boldsymbol{g}(t_{n-1/2}),
\]
for $\boldsymbol{g}(t)$ as in~\eqref{eq: U f g semidiscrete 1d}.

Rearranging \eqref{eq: Crank-Nicolson matrix}, we find that
\[
\boldsymbol{U}^n-\boldsymbol{U}^{n-1}
    +\tfrac12\Delta t\,\boldsymbol{A}(\boldsymbol{U}^n+\boldsymbol{U}^{n-1})
    =\Delta t\,\bigl(\boldsymbol{f}^{n-1/2}+\boldsymbol{g}^{n-1/2}\bigr)
\]
which yields linear system that must be solved at the $n$th time step,
\[
\bigl(\boldsymbol{I}+\tfrac12\Delta t\,\boldsymbol{A}\bigr)\boldsymbol{U}^n=
\bigl(\boldsymbol{I}-\tfrac12\Delta t\,\boldsymbol{A}\bigr)\boldsymbol{U}^{n-1}
    +\Delta t\,\bigl(\boldsymbol{f}^{n-1/2}+\boldsymbol{g}^{n-1/2}\bigr).
\]
The local truncation error for the Crank--Nicolson method is
\begin{multline*}
\tau^{n-1/2}_p=f^{n-1/2}_p-\frac{u(x_p,t_n)-u(x_p,t_{n-1})}{\Delta t}
    +\frac{a}{\Delta x^2}\biggl(
    \frac{u(x_{p+1},t_n)+u(x_{p+1},t_{n-1})}{2}\\
    -2\,\frac{u(x_p,t_n)+u(x_p,t_{n-1})}{2}
    +\frac{u(x_{p-1},t_n)+u(x_{p-1},t_{n-1})}{2}\biggr)r.
\end{multline*}
If we choose $f^{n-1/2}_p=f(x_p,t_{n-1/2})$ then
$f^{n-1/2}_p=u_t(x_p,t_{n-1/2})-au_{xx}(x_p,t_{n-1/2)}$ so
\begin{align*}
\tau^{n-1/2}_p
&=\biggl(u_t(x_p,t_{n-1/2})-\frac{u(x_p,t_n)-u(x_p,t_{n-1})}{\Delta t}\biggr)\\
    &\qquad{}+a\biggl(\frac{u_{xx}(x_p,t_n)+u_{xx}(x_p,t_{n-1})}{2}
        -u_{xx}(x_p,t_{n-1/2})\biggr)\\
    &\qquad{}+\frac{a}{2}\biggl(
    \frac{u(x_{p+1},t_n)-2u(x_p,t_n)+u(x_{p-1},t_n)}{\Delta x^2}-u_{xx}(x_p,t_n)
    \biggr)\\
    &\qquad{}+\frac{a}{2}\biggl(
    \frac{u(x_{p+1},t_{n-1})-2u(x_p,t_{n-1})+u(x_{p-1},t_{n-1})}{\Delta x^2}
        -u_{xx}(x_p,t_n)\biggr).
\end{align*}
Applying Theorems \ref{thm: first central diff}~and \ref{thm: average} with
$h=\tfrac12\Delta t$, and \cref{thm: 2nd central diff} with~$h=\Delta
x$, we conclude that
\[
|\tau^{n-1/2}_p|
    \le\frac{(\tfrac12\Delta t)^2}{6}\max_{[0,L]\times[0,T]}|u_{ttt}|
    +a\,\frac{(\tfrac12\Delta t)^2}{2}\max_{[0,L]\times[0,T]}|u_{xxxx}|
    +a\,\frac{\Delta x^2}{12}\max_{[0,L]\times[0,T]}|u_{xxxx}|
\]
and hence $\tau^{n-1/2}_p=O(\Delta t^2+\Delta x^2)$.  It turns out that the
Crank--Nicolson method is unconditionally stable, and so
$U^n_p=u(x_p,t_n)+O(\Delta t^2+\Delta x^2)$.

\begin{Exercises}

\exercise
If the solution~$u(x,t)$ of~\eqref{eq: heat ivp 1d} is continuous
at $(x,t)=(0,0)$ and $(L,0)$, then we must have
\[
u_0(0)=u(0,0)=\gamma_0(0)\quad\text{and}\quad
u_0(L)=u(L,t)=\gamma_L(0).
\]
Show that if $u_t$ and $u_{xx}$ are continuous at~$(x,t)=(L,0)$ and $(L,0)$,
then
\[
\gamma_0'(0)-au_0''(0)=f(0,0)
\quad\text{and}\quad
\gamma_L'(0)-au_0''(L)=f(L,0).
\]

\exercise\label{ex: forward diff}
Use \cref{thm: Taylor remainder} to show that that if $f$ is $C^2$ on
the closed interval~$[x,x+h]$, then
\[
\biggl|f'(x)-\frac{f(x+h)-f(x)}{h}\biggr|
	\le\frac{h}{2}\max_{x\le y\le x+h}|f''(y)|,
\]
and, similarly, if $f$ is $C^2$ on~$[x-h,x]$, then
\[
\biggl|f'(x)-\frac{f(x)-f(x-h)}{h}\biggr|
	\le\frac{h}{2}\max_{x-h\le y\le x}|f''(y)|,
\]

\exercise
Count the numbers of arithmetic operations used in \cref{alg: explicit Euler}
to compute the explicit Euler solution.
\begin{ans}
About $4NP$~multiplications and $3NP$ additions, not counting the cost of
evaluating $f$ at $(N-1)(P-1)$~grid points, and $\gamma_0$~and $\gamma_L$
at $N$ grid points.
\end{ans}

\exercise
Let $\phi_k(x)=\sin(k\pi x/L)$.
\begin{description}
\item{(i)} Show that
\[
\phi_k(x_{p+1})+\phi_k(x_{p-1})=\biggl(2\cos\frac{k\pi\,\Delta x}{2}\biggr)\,
	\phi_k(x_p).
\]
\item{(ii)} Deduce that
\[
-\frac{\phi_k(x_{p+1}-2\phi_k(x_p)+\phi_k(x_{p-1})}{\Delta x^2}
	=\Lambda_k\phi_k(x_p)\quad\text{where}\quad
\Lambda_k=\frac{2}{\Delta x^2}\biggl(1-\cos\frac{k\pi\,\Delta x}{L}\biggr).
\]
\item{(iii)} Show
\[
\Lambda_k=\biggl(\frac{2}{\Delta x}\,\sin\frac{k\pi\,\Delta x}{2L}\biggr)^2.
\]
\item{(iv)}
Find $\lambda_k=\lim_{\Delta x\to0}\Lambda_k$ and verify that
$-\phi_k''=\lambda_k\phi_k$.
\item{(v)}
Suppose that $u_0(x)=\sin(k\pi x/L)$, $f(x,t)\equiv0$ and
$\gamma_0(t)\equiv 0\equiv\gamma_L(t)$.  Find $R$ such that the explicit
Euler solution is~$U^n_p=R^n\phi_k(x_p)$.
\item{(vi)} Show that if $a\,\Delta t/\Delta x^2\le1/2$ then $-1\le R<1$
and so $|U^n_p|\le1$ for all $n$~and $p$.
\end{description}
\begin{ans}
(iii) $R=1-a\lambda_k\,\Delta t$\quad(iv) $\lambda_k=(k\pi/L)^2$
\end{ans}

\exercise\label{ex: implicit Euler error}
Prove \cref{thm: implicit Euler error} by adapting the proof of
\cref{thm: explicit Euler error}.

\exercise
Consider the BDF2 (backward differentiation formula, second-order)
scheme for the heat equation,
\[
\frac{3U^n_p-4U^{n-1}_p+U^{n-2}_p}{2\Delta t}
        -\kappa\,\frac{U^n_{p+1}-2U^n_p+U^n_{p-1}}{\Delta x^2}=f^n_p,
\]
where $1\le p\le P-1$ and $2\le n\le N$.  Assume homogenous Dirichlet
boundary conditions,
\[
U^n_0=0=U^n_P\quad\text{for $1\le n\le N$,}
\]
and initial conditions $U^0_p=v(x_p)$ for~$0\le p\le P$.
\begin{description}
\item{(i)} Draw the stencil for the BDF2 scheme.
\item{(ii)} Define the local truncation error~$\tau(x,t)$ for this
scheme and show that $\tau=O(\Delta t^2+\Delta x^2)$.
\item{(iii)} Write the BDF2 scheme in matrix--vector form, and hence
describe the linear system that must be solved at the $n$th time step
for~$2\le n\le N$.
\item{(iv)} To compute $U^n_p$ when~$n=1$, we use the implicit Euler
scheme,
\[
\frac{U^1_p-U^0_p}{\Delta t}
        -\kappa\,\frac{U^1_{p+1}-2U^1_p+U^1_{p-1}}{\Delta x^2}=f^1_p.
\]
Why does the use of this method ensure that
\[
U^1_p=u(x_p,t_1)+O(\Delta t^2+\Delta x^2),
\]
when we saw in lectures that the implicit Euler method is only
first-order accurate in time?
\end{description}

\end{Exercises}


% !TEX root = main.tex

%-------------------------------------------------
\section{Non-negative variables}\label{sec:expe}

Next we define the expectation for random variables that take only non-negative values.

\begin{definition}[Expectation of non-negative random variables]\label{def:expe_non-negative}
\ben
\it
The expectation of a non-negative \emph{discrete} random variable $X$ is
\[
\expe(X) = \sum_{i=1}^{\infty} x_i f(x_i),
\]
where $f$ is the PMF of $X$ and $\{x_1,x_2,\ldots\}$ is the range of $X$.
\it
The expectation of a non-negative \emph{continuous} random variable $X$ is
\[
\expe(X) = \int_{-\infty}^{\infty} xf(x)\,dx,
\]
where $f$ is the PDF of $X$.
\een
\end{definition}

Non-negative random variables can have \emph{infinite expectation}.

\begin{example} % discrete, infinite
A long line of athletes $k=0,1,2,\ldots$ make throws of a javelin to distances $D_0,D_1,D_2,\ldots$ respectively. Assume that the distances are independent and have the same distribution, and that the probability of any two throws being exactly the same distance is equal to zero. Let $X$ be the number of throws until the initial distance $D_0$ is surpassed for the first time. Find the PMF of $X$, and show that $\expe(X)$ is infinite.
\begin{solution}
First we need to find the PMF of $X$:
\bit
\it $X$ is a discrete random variable, taking values in the set $\{1,2,\ldots\}$.
\it The event $\{X>k\}$ means that out of the first $k+1$ throws, the initial throw was the furthest.
\it Because the distances $D_0,D_1,\ldots,D_k$ have the same distribution, each has the same probability of being the largest, so $\displaystyle\prob(X>k)=\frac{1}{k+1}$.
\eit
Hence, the PMF of $X$ is
\[
\prob(X=k) = \prob(X>k-1) - \prob(X>k) = \frac{1}{k} - \frac{1}{k+1} = \frac{1}{k(k+1)},
\]
and its expected value is therefore
\[
\expe(X)
	= \sum_{k=1}^\infty k\,\prob(X=k)
	= \sum_{k=1}^\infty \frac{1}{k+1}
	= \frac{1}{2} + \frac{1}{3} + \frac{1}{4} + \ldots
	= \infty.
\]
\end{solution}
\end{example}

\begin{example}[Pareto distrubtion] % continuous, infinite (pareto)
Let $X$ be a continuous random variable with the following PDF:
\[
f(x) = \begin{cases}
	1/x^2		& \text{for}\quad x > 1, \\
	0			& \text{otherwise.}
\end{cases}
\]
Show that $\expe(X)$ is infinite.
\begin{solution}
$X$ is non-negative, so
\[
\expe(X)
	= \int_{0}^{\infty} x f(x)\,dx
	= \int_{1}^{\infty} \frac{1}{x}\,dx
	= \infty.
\]
To be explicit:
\[
\int_{1}^{\infty} \frac{1}{x}\,dx
	= \lim_{n\to\infty} \int_{1}^{n} \frac{1}{x}\,dx
	= \lim_{n\to\infty} \big[\log(x)\big]_{1}^{n}
	= \lim_{n\to\infty} \big[\log(n) - \log(1)\big]
	= \lim_{n\to\infty} \log(n)
	= \infty.
\]
\end{solution}
\end{example}


%!TEX root =  ../main.tex

\subsection{At Infinities}

\objective{Determine when a limit does and does not exist, or is infinite.}

\personfeature[0in]{\chapdir/pics/Augustin_Cauchy.jpg}{Augustin-Louis Cauchy}{1789-1857}{was a French mathematician and physicist who made pioneering contributions to analysis. His book \textit{Cours d'Analyse} is frequently noted as being the first place that inequalities, and $ \delta -\epsilon$ arguments were introduced into Calculus. \href{https://en.wikipedia.org/wiki/Augustin-Louis_Cauchy}{Wikipedia}}

Because limits are asking questions that need not have simple, numeric inputs, nor
simple, numeric outputs, we can evaluate limits involving infinities.
``Infinity'' simply means ``without end''.  Asking what a function approaches as
$x$ approaches infinity, graphically means ``what value is the output tending
towards as the input grows without bound?''.  Negative infinity is a term describing
the leftward trend of a function.

\begin{derivation}{Limit at Infinity}
For $f(x)$ which is a real function, the limit of $f$ as $x$ approaches infinity is L,
means that for all  $\varepsilon >0$, there exists $c$ such
that $|f(x) - L| < \varepsilon \, \text{ whenever } \,  x > c$. This is written in logic symbols are
$\forall \varepsilon >  0 \; \exists c \; \forall x > c :\; |f(x) - L| < \varepsilon$.

$$\lim _{x\to \infty }f(x)=L$$

\end{derivation}

The same applies at negative infinity:

\begin{derivation}{Limit at Negative Infinity}\index{limit!at infinities}
For $f(x)$ a real function, the limit of $f$ as $x$ approaches negative infinity is L,
means that for all $\varepsilon >0$ there exists $c$ such that
$ |f(x) - L| < \varepsilon \, \text{ whenever } \, x < c$.

$$ \lim_{x \to -\infty}f(x) = L$$

\end{derivation}

\begin{example}{All the way to the left}
	\exProblem
Evaluate $\displaystyle \lim_{x \to -\infty}2^x$
	\exSolution
We can observe the graph or compute numerically that $2^x$ is getting closer
and closer to 0 as we move leftward.  We can get arbitrarily close to 0 by picking
whatever large, negative exponent we wish.  Hence, the answer is 0.
\end{example}

\subsection{Infinite Forms}
Not only can we ``plug in'' infinities in limits problems, but we can also get
$\pm\infty$ as an answer.  Recall, however, that the left and right sided limits
must agree for the limit to exist.  Hence, we can say that the limit as $x$
approaches 0 of $x^{-2}$ is infinity, while the same limit taken on $\frac{1}{x}$
does not exist.

The limit of $f$ as $x$ approaches $a$ is infinity, denoted:

$$\lim_{x \to a} f(x) = \infty$$
means that for all $\varepsilon >0 \, \text{there exists} \, \delta >0 \, \text{such that} \, f(x) > \varepsilon \, \text{whenever}  \, |x - a| < \delta$.

For example:

$$ \lim_{x \to 0^+} \ln x = -\infty$$


\section{Trigonometric Substitutions}{}{}\label{sec:Trig Sub}
So far we have seen that it sometimes helps to replace a subexpression
of a function by a single variable. Occasionally it can help to
replace the original variable by something more complicated. This
seems like a ``reverse'' substitution, but it is really no different
in principle than ordinary substitution.

\begin{example}{Sine Subsitution}{Sine Subsitution}
Evaluate $\ds\int \sqrt{1-x^2}\,dx$.
\end{example}

\begin{solution}
Let $x=\sin u$ so
$dx=\cos u\,du$. Then
$$
  \int \sqrt{1-x^2}\,dx=\int\sqrt{1-\sin^2 u}\cos u\,du=
  \int\sqrt{\cos^2 u}\cos u\,du.
$$
We would like to replace $\ds \sqrt{\cos^2 u}$ by $\cos u$, but this is
valid only if $\cos u $ is positive, since $\ds \sqrt{\cos^2 u}$ is
positive. Consider again the substitution $x=\sin u$. We could just as
well think of this as $u=\arcsin x$. If we do, then by the definition
of the arcsine, $-\pi/2\le u\le\pi/2$, so $\cos u\ge0$. Then we
continue:
\begin{eqnarray*}
  \int\sqrt{\cos^2 u}\cos u\,du&=&\int\cos^2u\,du\\
	&=&\int {1+\cos 2u\over2}\,du = {u\over 2}+{\sin 2u\over4}+C\cr
  &=&{\arcsin x\over2}+{\sin(2\arcsin x)\over4}+C.
\end{eqnarray*}
This is a perfectly good answer, though the term
$\sin(2\arcsin x)$ is a bit unpleasant. It is possible to simplify
this. Using the identity $\sin 2x=2\sin x\cos x$, we can write
$\ds \sin 2u=2\sin u\cos u=2\sin(\arcsin x)\sqrt{1-\sin^2 u}=
2x\sqrt{1-\sin^2(\arcsin x)}=2x\sqrt{1-x^2}.$ Then the full
antiderivative is
$$
  {\arcsin x\over2}+{2x\sqrt{1-x^2}\over4}=
  {\arcsin x\over2}+{x\sqrt{1-x^2}\over2}+C.
$$
\end{solution}

This type of substitution is usually indicated when the function you
wish to integrate contains a polynomial expression that might allow
you to use the fundamental identity $\ds \sin^2x+\cos^2x=1$ in
one of three forms:
$$
  \cos^2 x=1-\sin^2x
  \qquad
  \sec^2x=1+\tan^2x
  \qquad
  \tan^2x=\sec^2x-1.
$$
If your function contains $\ds 1-x^2$, as in the example above, try
$x=\sin u$; if it contains $\ds 1+x^2$ try $x=\tan u$; and if it contains
$\ds x^2-1$, try $x=\sec u$. Sometimes you will need to try something a
bit different to handle constants other than one which we will describe below.
First we discuss inverse substitutions.

In a {\bf traditional} substitution we let $u=u(x)$, i.e., our new variable is defined in terms of $x$.
In an {\bf inverse} substitution we let $x=g(u)$, i.e., we assume $x$ can be written in terms of $u$.
We cannot do this arbitrarily since we do \thmfont{NOT} get to ``choose'' $x$.
For example, an inverse substitution of $x=1$ will give an obviously wrong answer.
However, when $x=g(u)$ is an invertible function, then we are really doing a $u$-substitution with $u=g^{-1}(x)$.
Now the substitution rule applies.

Sometimes with inverse substitutions involving trig functions we use $\theta$ instead of $u$.
Thus, we would take $x=\sin\theta$ instead of $x=\sin u$.
However, as we discussed above, we would like our inverse substitution $x=g(u)$ to be a one-to-one function, and $x=\sin u$ is not one-to-one.
We can overcome this issue by using the restricted trigonometric functions.
The three common trigonometric substitutions are the restricted sine, restricted tangent and restricted secant.
Thus, for sine we use the domain $[-\pi/2,~\pi/2]$ and for tangent we use $(-\pi/2,~\pi/2)$.
Depending on the convention chosen, the restricted secant function is usually defined in one of two ways.
$$\includegraphics[width=6in]{images2/inv-trig-sec}$$
One convention is to restrict secant to the region $[0,~\pi/2)\cup(\pi/2,\pi]$ as shown in the middle graph.
The other convention is to use $[0,~\pi/2)\cup[\pi,~3\pi/2)$ as shown in the right graph.
Both choices give a one-to-one restricted secant function and no universal convention has been adopted.
To make the analysis in this section less cumbersome, we will use the domain $[0,~\pi/2)\cup[\pi,~3\pi/2)$ for the restricted secant function.
Then $\sec^{-1}x$ is defined to be the inverse of this restricted secant function.

Typically trigonometric substitions are used for problems that involve radical expressions.
The table below outlines when each substitution is typically used along with their intervals of validity.

%$$\begin{array}{|c|c|c|}
%\hline
%\mbox{\thmfont{Expression}} & \mbox{\exfont{Substitution}} & \mbox{\deffont{Validity}}\\
%\hline ~ & ~ & ~ \\
%\sqrt{a^2-x^2} & x=a\sin\theta & \theta\in[-\pi/2,~\pi/2]\\
%~ & ~ & ~ \\\hline~ & ~ & ~ \\
%\sqrt{a^2+x^2}~~\mbox{or}~~a^2+x^2 & x=a\tan\theta & \theta\in(-\pi/2,~\pi/2)\\
%~ & ~ & ~ \\\hline~ & ~ & ~ \\
%\sqrt{x^2-a^2} & x=a\sec\theta & \theta\in[0,~\pi/2)\cup[\pi,~3\pi/2)\\
%~ & ~ & ~ \\\hline
%\end{array}$$

\begin{formulabox}[Trigonometric Substitution]
\begin{enumerate}
	\item[(a)] \noindent%
		\begin{minipage}[t]{.6\linewidth}%
		For integrands containing $\sqrt{a^2-x^2}$:\index{integration!trig. subst.}\\[5pt]
		Let $x=a\sin\theta$, \qquad $dx = a\cos\theta\ d\theta$\\[5pt]
	Thus $\theta = \sin^{-1}(x/a)$, for $-\pi/2\leq \theta\leq \pi/2$. \\[5pt]
	On this interval, $\cos\theta\geq 0$, so\\[5pt]
	$\sqrt{a^2-x^2} = a\cos\theta$
		\end{minipage}\qquad
	\begin{minipage}[t]{.4\linewidth}\vskip 0pt
		\begin{tikzpicture}
				\draw [very thick] (0,0) -- node[below,pos=.5] { $\sqrt{a^2-x^2}$} (3,0) -- node [right,pos=.5] { $x$} (3,2) -- node [pos=.5,above] { $a$} (0,0);
				\draw [thick] (2.7,0) -- (2.7,.3) -- (3,.3);
				\draw (.75,.25) node {$\theta$};
		\end{tikzpicture}
		\end{minipage}

	\item[(b)] \noindent
	\begin{minipage}[t]{.6\linewidth}
		For integrands containing $\sqrt{x^2+a^2}$:\\[5pt]
		Let $x=a\tan\theta$, \qquad $dx = a\sec^2\theta\ d\theta$\\[5pt]
	Thus $\theta = \tan^{-1}(x/a)$, for $-\pi/2 < \theta < \pi/2$. \\[5pt]
	On this interval, $\sec\theta> 0$, so\\[5pt]
	$\sqrt{x^2+a^2} = a\sec\theta$
		\end{minipage}\qquad
	\begin{minipage}[t]{.4\linewidth}\vskip 0pt
	\begin{tikzpicture}
			\draw [very thick] (0,0) -- node[below,pos=.5] { $a$} (3,0) -- node [right,pos=.5] { $x$} (3,2) -- node [pos=.5,above,sloped] { $\sqrt{x^2+a^2}$} (0,0);
			\draw [thick] (2.7,0) -- (2.7,.3) -- (3,.3);
			\draw (.75,.25) node {$\theta$};
	\end{tikzpicture}
		\end{minipage}

	\item[(c)] \noindent
	\begin{minipage}[t]{.6\linewidth}
		For integrands containing $\sqrt{x^2-a^2}$:\\[5pt]
		Let $x=a\sec\theta$, \qquad $dx = a\sec\theta\tan\theta\ d\theta$\\[5pt]
	Thus $\theta = \sec^{-1}(x/a)$. If $x/a\geq 1$, then $0\leq\theta<\pi/2$; if $x/a \leq -1$, then $\pi\le\theta< 3\pi/2$.\\[5pt]
	We restrict our work to where $x\geq a$, so $x/a\geq 1$, and $0\leq\theta<\pi/2$.
	On this interval, $\tan\theta\geq 0$, so\\[5pt]
	$\sqrt{x^2-a^2} = a\tan\theta$
		\end{minipage}\qquad
	\begin{minipage}[t]{.4\linewidth}\vskip 0pt
		\begin{tikzpicture}
				\draw [very thick] (0,0) -- node[below,pos=.5] { $a$} (3,0) -- node [right,pos=.5] { $\sqrt{x^2-a^2}$} (3,2) -- node [pos=.5,above] { $x$} (0,0);
				\draw [thick] (2.7,0) -- (2.7,.3) -- (3,.3);
				\draw (.75,.25) node {$\theta$};
		\end{tikzpicture}
		\end{minipage}
\end{enumerate}

\end{formulabox}

All three substitutions are one-to-one on the listed intervals.
When dealing with radicals we often end up with absolute values since
$$\sqrt{z^2}=|z|.$$
For each of the three trigonometric substitions above we will verify that we can ignore the absolute value in each case when encountering a radical.

For $x=a\sin\theta$, the expression $\sqrt{a^2-x^2}$ becomes
$$\sqrt{a^2-x^2}
= \sqrt{a^2-a^2\sin^2\theta}
= \sqrt{a^2(1-\sin^2\theta)}
= a\sqrt{\cos^2\theta}
= a|\cos\theta|
= a\cos\theta$$
This is because $\cos\theta\geq 0$ when $\theta\in[-\pi/2,~\pi/2]$.
For $x=a\tan\theta$, the expression $\sqrt{a^2+x^2}$ becomes
$$\sqrt{a^2+x^2}
= \sqrt{a^2+a^2\tan^2\theta}
= \sqrt{a^2(1+\tan^2\theta)}
= a\sqrt{\sec^2\theta}
= a|\sec\theta|
= a\sec\theta$$
This is because $\sec\theta>0$ when $\theta\in(-\pi/2,~\pi/2)$.

Finally, for $x=a\sec\theta$, the expression $\sqrt{x^2-a^2}$ becomes
$$\sqrt{x^2-a^2}
= \sqrt{a^2\sec^2\theta-a^2}
= \sqrt{a^2(\sec^2\theta-1)}
= a\sqrt{\tan^2\theta}
= a|\tan\theta|
= a\tan\theta$$
This is because $\tan\theta\geq 0$ when $\theta\in[0,~\pi/2)\cup[\pi,~3\pi/2)$.

Thus, when using an appropriate trigonometric substitution we can usually ignore the absolute value.
After integrating, we typically get an answer in terms of $\theta$ (or $u$) and need to convert back to $x$'s.
To do so, we use the two guidelines below:
\begin{itemize}
	\item For trig functions containing $\theta$, use a triangle to convert to $x$'s.
	\item For $\theta$ by itself, use the inverse trig function.
\end{itemize}

All pieces needed for such a trigonometric substitution can be summarized as follows:
\[\begin{array}{|c|c|c|c|c|}\hline	%5 columns
		&	&	&	&	\text{\textbf{Inverse of}}	\\
\text{\textbf{Expression}}	&	\text{\textbf{Substitution}}	&	\text{\textbf{Differential}}	&	\text{\textbf{Identity}}	&	\text{\textbf{Substitution}}	\\	\hline
~	&	~	&	~	&	~	&	~	\\
\sqrt{a^2-x^2}	&	x=a\sin\theta	&	dx=a\cos\theta d\theta	&	\sqrt{a^2-x^2}=a\cos\theta	&	\theta=\sin^{-1}\left(\frac{x}{a}\right)	\\
~	&	~	&	~	&	~	&	~	\\	\hline
~	&	~	&	~	&	~	&	~	\\
\sqrt{a^2+x^2}	&	&	&	&	\\
 \text{or} 	&	x=a\tan\theta	&	dx=a\sec^2\theta d\theta	&	\sqrt{a^2+x^2}=a\sec\theta	&	\theta=\tan^{-1}\left(\frac{x}{a}\right)	\\
a^2+x^2 	&	&	&	&	\\
~	&	~	&	~	&	~	&	~	\\	\hline
~	&	~	&	~	&	~	&	~	\\
\sqrt{x^2-a^2}	&	x=a\sec\theta	&	dx=a\sec\theta\tan\theta d\theta	&	\sqrt{x^2-a^2}=a\tan\theta	&	\theta=\sec^{-1}\left(\frac{x}{a}\right)	\\
~	&	~	&	~	&	~	&	~	\\	\hline
\end{array}\]

To emphasize the technique, we redo the computation for $\ds\int \sqrt{1-x^2}\,dx$.

\begin{example}{Sine Subsitution}{Sine Subsitution}
Evaluate $\ds\int \sqrt{1-x^2}\,dx$.
\end{example}

\begin{solution}
Since $\sqrt{1-x^2}$ appears in the integrand we try the trigonometric substitution $x=\sin\theta$.
(Here we are using the restricted sine function with $\theta\in[-\pi/2,~\pi/2]$ but typically omit this detail when writing out the solution.)
Then $\fbox{$dx$}=\fbox{$\cos \theta\,d\theta$}$.
$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
	\int \sqrt{1-x^2}\,\fbox{$dx$} & = & \int\sqrt{1-\sin^2\theta}\,\fbox{$\cos \theta\,d\theta$} &\mbox{Using our (inverse) substitution}\\
	& = & \int \sqrt{\cos^2\theta}\cos \theta\,d\theta &\mbox{Since $\sin^2\theta+\cos^2\theta=1$}\\
	& = & \int |\cos\theta|\cdot\cos\theta\,d\theta & \mbox{Since $\sqrt{\cos^2\theta}=|\cos\theta|$}\\
	& = & \int \cos^2\theta\,d\theta & \mbox{Since for $\theta\in[-\frac{\pi}{2},\frac{\pi}{2}]$ we have $\cos\theta\geq0$.}\\
\end{array}$$
Often we omit the step containing the absolute value by our discussion above.
Now, to integrate a power of cosine we use the guidelines for products of sine and cosine and make use of the identity
 $$\cos^2\theta=\frac{1}{2}(1+\cos(2\theta)).$$
Our integral then becomes
$$\int \sqrt{1-x^2}\,dx=\frac{1}{2}\int (1+\cos(2\theta))\,d\theta=\frac{\theta}{2}+\frac{\sin(2\theta)}{4}+C$$
To write the answer back in terms of $x$ we first use the identity $ \sin(2\theta) = 2\sin\theta \cos\theta $. So
$$\int \sqrt{1-x^2}\,dx= \frac{\theta}{2}+\frac{\sin(\theta)\cos(\theta)}{2}+C$$

%$$\includegraphics[width=1.65in]{images2/inv-trig-triangle-1}$$
\begin{minipage}{.7\textwidth}
Since	 $x=\sin\theta$, we can build the corresponding triangle and use SOH CAH TOA  we get $ \cos\theta = \frac{\sqrt{1-x^2}}{1} = \sqrt{1-x^2}.$
	This gives
	$$\int \sqrt{1-x^2}\,dx =\frac{\sin^{-1}x}{2}+\frac{x\sqrt{1-x^2}}{2}+C$$
\end{minipage}
\begin{minipage}{.3\textwidth}
	\begin{tikzpicture}
	\draw [very thick, fill=green!20] (0,0) -- node[below,pos=.5] { $\sqrt{1-x^2}$} (3,0) -- node [right,pos=.5] { $x$} (3,2) -- node [pos=.5,above,sloped] { $ 1 $} (0,0);
	\draw [thick] (2.7,0) -- (2.7,.3) -- (3,.3);
	\draw (.75,.25) node {$\theta$};
	\end{tikzpicture}
\end{minipage}

\end{solution}

\begin{example}{Secant Subsitution}{Secant Subsitution}
Evaluate $\ds\int\frac{\sqrt{25x^2-4}}{x}\,dx$.
\end{example}

\begin{solution}
We do not have $\sqrt{x^2-a^2}$ because of the $25$, but if we factor $25$ out we get:
$$\ds\int\frac{\sqrt{25(x^2-(4/25))}}{x}\,dx=\ds\int5\frac{\sqrt{x^2-(4/25)}}{x}\,dx.$$
Now, $a=2/5$, so let $x=\frac{2}{5}\sec\theta$.
Alternatively, we can think of the integral as being:
$$\ds\int\frac{\sqrt{(5x)^2-4}}{x}\,dx$$
Then we could let $u=5x$ followed by $u=2\sec\theta$, etc.
Or equivalently, we can avoid a $u$-substitution by letting $5x=2\sec\theta$.
In either case we are using the trigonometric substitution $x=\frac{2}{5}\sec\theta$, but do use the method that makes the most sense to you!
As $x=\frac{2}{5}\sec\theta$ we have $\fbox{$dx$}=\fbox{$\frac{2}{5}\sec\theta\tan\theta\,d\theta$}$.
$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
	\int\frac{\sqrt{25x^2-4}}{x}\,\fbox{$dx$} & = & \int \frac{\sqrt{25\frac{4\sec^2\theta}{25}-4}}{\frac{2}{5}\sec\theta}\,\fbox{$\frac{2}{5}\sec\theta\tan\theta\,d\theta$} &\mbox{Using the substitution}\\
	& = & \int\sqrt{4(\sec^2\theta-1)}\cdot \tan\theta\,d\theta & \mbox{Cancelling}\\
	& = & 2\int\sqrt{\tan^2\theta}\cdot \tan\theta\,d\theta & \mbox{Using $\tan^2\theta+1=\sec^2\theta$}\\
	%& = & 2\int\tan\theta\cdot \tan\theta\,d\theta & \mbox{Since $\sqrt{\tan^2\theta}=|\tan\theta|=\tan\theta$}\\
	& = & 2\int\tan^2\theta\,d\theta & \mbox{Simplifying}\\
	& = & 2\int(\sec^2\theta-1)\,d\theta & \mbox{Using $\tan^2\theta+1=\sec^2\theta$}\\
	& = & 2(\tan\theta-\theta)+C & \mbox{Since $\ds\int\sec^2\theta\,d\theta=\tan\theta+C$}\\
\end{array}$$

\begin{minipage}{.6\textwidth}
To solve for $\tan\theta$, we build a triangle where $ \sec\theta = \frac{5x}{2} $, or equivalently, $\cos\theta=\frac{2}{5x}$.
Using SOH CAH TOA, the triangle is as shown at the right:
\end{minipage}
\begin{minipage}{.4\textwidth}
\begin{center}
	\begin{tikzpicture}
\draw [very thick, fill=green!20] (0,0) -- node[below,pos=.5] { $2$} (3,0) -- node [right,pos=.5] { $\sqrt{25x^2-4}$} (3,2) -- node [pos=.5,above] { $5x$} (0,0);
\draw [thick] (2.7,0) -- (2.7,.3) -- (3,.3);
\draw (.75,.25) node {$\theta$};
\end{tikzpicture}
\end{center}
\end{minipage}
From the triangle, we get $ \ds \tan\theta = \frac{\sqrt{25x^2-4}}{2}  $.
%$$\includegraphics[height=1.5in]{images2/inv-trig-triangle-6}$$
As $\theta=\sec^{-1}(5x/2)$, we get
$$\ds\int\frac{\sqrt{25x^2-4}}{x}\,dx=2\left(\frac{\sqrt{25x^2-4}}{2}-\sec^{-1}\left(\frac{5x}{2}\right)\right)+C$$
\end{solution}

In the context of the previous example, some resources give alternate guidelines when choosing a trigonometric substitution.
\begin{center}\begin{tabular}{@{}ll@{}ll@{}}
	$\sqrt{a^2-b^2x^2}\quad\to\quad x=\ds\frac{a}{b}\sin\theta$\\
	\vspace{-0.1cm}\\
	$\sqrt{b^2x^2+a^2}~~\mbox{or}~~(b^2x^2+a^2)\quad\to\quad x=\ds\frac{a}{b}\tan\theta$\\
	\vspace{-0.1cm}\\
	$\sqrt{b^2x^2-a^2}\quad\to\quad x=\ds\frac{a}{b}\sec\theta$\\
\end{tabular}\end{center}

We next look at a tangent substitution.

%\begin{example}{Tangent Substitution}{Tangent Substitution}
%Evaluate $\ds\int\frac{1}{\sqrt{25+x^2}}\,dx$.
%\end{example}

%\begin{solution}
%Let $x=5\tan\theta$ so that $\fbox{$dx$}=\fbox{$5\sec^2\theta\,d\theta$}$.
%$${\def\arraystretch{2.2}
%\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
%	\int\frac{1}{\sqrt{25+x^2}}\,\fbox{$dx$} & = & \int \frac{1}{\sqrt{25+25\tan^2\theta}}\,\fbox{$5\sec^2\theta\,d\theta$} &\mbox{Using our substitution}\\
%	& = & \int\frac{1}{\sqrt{25(1+\tan^2\theta)}}\cdot5\sec^2\theta\,d\theta & \mbox{Factor out $25$}\\
%	& = & \int\frac{1}{5\sqrt{sec^2\theta}}\cdot5\sec^2\theta\,d\theta & \mbox{Using $\tan^2\theta+1=\sec^2\theta$}\\
%%	& = & \int\frac{1}{5\sec\theta}\cdot5\sec^2\theta\,d\theta & \mbox{Can ignore absolute value (why?)}\\
%	& = & \int\sec\theta\,d\theta & \mbox{Simplifying}\\
%	& = & \ln|\sec\theta+\tan\theta|+C & \mbox{By $\ds\int\sec \theta\,dx=\ln|\sec \theta+\tan \theta|+C$}\\
%\end{array}
%}$$
%Since $\tan\theta=x/5$, we draw a triangle:
%$$\includegraphics[height=1.5in]{images2/inv-trig-triangle-7}$$
%Then $$\ds\sec\theta=\frac{1}{\cos\theta}=\frac{\sqrt{25+x^2}}{5}.$$
%Therefore, the integral is
%$$\int\frac{1}{\sqrt{25+x^2}}\,dx=\ln\left|\frac{\sqrt{25+x^2}}{5}+\frac{x}{5}\right|+C$$
%\end{solution}

\begin{example}{Tangent Substitution}{Tangent Substitution}
Evaluate $\ds \int \frac{1}{\sqrt{5+x^2}}\ dx.$
\end{example}

\begin{solution}
Using part (b), we recognize $a=\sqrt{5}$ and  set $x= \sqrt{5}\tan \theta$. This makes $dx = \sqrt{5}\sec^2\theta\ d\theta$. We will use the fact that $\sqrt{5+x^2} = \sqrt{5+5\tan^2\theta} = \sqrt{5\sec^2\theta} = \sqrt{5}\sec\theta.$ Substituting, we have:
\begin{align*}
\int \frac{1}{\sqrt{5+x^2}}\ dx &= \int \frac{1}{\sqrt{5+5\tan^2\theta}}\sqrt{5}\sec^2\theta\ d\theta \\
			&= \int \frac{\sqrt{5}\sec^2\theta}{\sqrt{5}\sec\theta} \ d\theta\\
			&= \int \sec\theta\ d\theta\\
			&= \ln\big|\sec\theta+\tan\theta\big|+C.
\end{align*}
While the integration steps are over, we are not yet done. The original problem was stated in terms of $x$, whereas our answer is given in terms of $\theta$. We must convert back to $x$.

\begin{minipage}{.7\textwidth}
With $x=\sqrt{5}\tan\theta$, we have
$$\tan \theta = \frac x{\sqrt{5}}\quad \text{and}\quad \sec\theta = \frac{\sqrt{x^2+5}}{\sqrt{5}}.$$
This gives
\end{minipage}
\begin{minipage}{.3\textwidth}
\begin{tikzpicture}
			\draw [very thick, fill=green!20] (0,0) -- node[below,pos=.5] { $\sqrt{5}$} (3,0) -- node [right,pos=.5] { $x$} (3,2) -- node [pos=.5,above,sloped] { $\sqrt{x^2+5}$} (0,0);
			\draw [thick] (2.7,0) -- (2.7,.3) -- (3,.3);
			\draw (.75,.25) node {$\theta$};
	\end{tikzpicture}
\end{minipage}

\begin{align*}
\int \frac{1}{\sqrt{5+x^2}}\ dx &= \ln\big|\sec\theta+\tan\theta\big|+C \\
     &= \ln\left|\frac{\sqrt{x^2+5}}{\sqrt{5}}+ \frac x{\sqrt{5}}\right|+C.
\end{align*}
We can leave this answer as is, or we can use a logarithmic identity to simplify it. Note:
\begin{align*}
\ln\left|\frac{\sqrt{x^2+5}}{\sqrt{5}}+ \frac x{\sqrt{5}}\right|+C &= \ln\left|\frac{1}{\sqrt{5}}\big(\sqrt{x^2+5}+ x\big)\right|+C \\
   &= \ln\left|\frac{1}{\sqrt{5}}\right| + \ln\big|\sqrt{x^2+5}+ x\big|+C\\
	&=	\ln\big|\sqrt{x^2+5}+ x\big|+C,
\end{align*}
where the $\ln\big(1/\sqrt{5}\big)$ term is absorbed into the constant $C$. %(In Section \ref{sec:hyperbolic} we will learn another way of approaching this problem.)
\end{solution}

In the next example, we will use the technique of completing the square in order to rewrite the integrand.

\begin{example}{Completing the Square}{Completing the Square}
Evaluate $\ds\int\frac{x}{\sqrt{3-2x-x^2}}\,dx$.
\end{example}

\begin{solution}
First, complete the square to write
$$3-2x-x^2=4-(x+1)^2$$
Now, we may let $u=x+1$ so that $du=dx$ (note that $x=u-1$) to get:
$$\int\frac{x}{\sqrt{4-(x+1)^2}}\,dx=\int\frac{u-1}{\sqrt{4-u^2}}\,du$$
Let $u=2\sin\theta$ giving $du=2\cos\theta\,d\theta$:
$$\int\frac{u-1}{\sqrt{4-u^2}}\,du = \int\frac{2\sin\theta-1}{2\cos\theta}\cdot 2\cos\theta\,d\theta=\int (2\sin\theta-1)\,d\theta$$
Integrating and using a triangle we get:
\begin{eqnarray*}
\int\frac{x}{\sqrt{3-2x-x^2}}	& = &-2\cos\theta-\theta+C  \\
	& = &-\sqrt{4-u^2}-\sin^{-1}\left(\frac{u}{2}\right)+C  \\
	& = &-\sqrt{3-2x-x^2}-\sin^{-1}\left(\frac{x+1}{2}\right)+C
\end{eqnarray*}
Note that in this problem we could have skipped the $u$-substitution if instead we let $x+1=2\sin\theta$. (For the  triangle we would then use $\ds\sin\theta=\frac{x+1}{2}$.)
\end{solution}

%\bigskip
%\begin{example}{Another Inverse Sine Subsitution}{Another Inverse Sine Subsitution}\label{Another Inverse Sine Subsitution}
%Evaluate $\ds\int\sqrt{4-9x^2}\,dx$.
%\end{example}
%
%\begin{solution}
%We start by rewriting this so
%that it looks more like the previous example:
%$$
  %\int\sqrt{4-9x^2}\,dx=\int\sqrt{4(1-(3x/2)^2)}\,dx
  %=\int 2\sqrt{1-(3x/2)^2}\,dx.
%$$
%Now let $3x/2=\sin u$ so $(3/2)\,dx=\cos u \,du$ or
%$dx=(2/3)\cos u\,du$. Then
%\begin{eqnarray*}
%\int 2\sqrt{1-(3x/2)^2}\,dx&=&\int 2\sqrt{1-\sin^2u}\,(2/3)\cos u\,du\cr
%&=&{4\over3}\int \cos^2u\,du\cr
%&=&{4u\over 6}+{4\sin 2u\over12}+C\cr
%&=&{2\arcsin(3x/2)\over3}+{2\sin u \cos u\over3}+C\cr
%&=&{2\arcsin(3x/2)\over3}+{2\sin(\arcsin(3x/2))\cos(\arcsin(3x/2))\over3}+C\cr
%&=&{2\arcsin(3x/2)\over3}+{2(3x/2)\sqrt{1-(3x/2)^2}\over3}+C\cr
%&=&{2\arcsin(3x/2)\over3}+{x\sqrt{4-9x^2}\over2}+C,\cr
%\end{eqnarray*}
%using some of the work from example~\ref{Inverse Sine Subsitution}.
%\end{solution}
%
%
%
%\bigskip
%\begin{example}{Inverse Tangent Subsitution}{Inverse Tangent Subsitution}\label{Inverse Tangent Subsitution}
%Evaluate $\ds\int\sqrt{1+x^2}\,dx$.
%\end{example}
%
%\begin{solution}
%Let $x=\tan u$,
%$\ds dx=\sec^2 u\,du$, so
%$$
  %\int\sqrt{1+x^2}\,dx=\int \sqrt{1+\tan^2 u}\sec^2u\,du=
  %\int\sqrt{\sec^2u}\sec^2u\,du.
%$$
%Since $u=\arctan(x)$, $-\pi/2\le u\le\pi/2$ and $\sec u\ge0$, so
%$\ds \sqrt{\sec^2u}=\sec u$. Then
%$$\int\sqrt{\sec^2u}\sec^2u\,du=\int \sec^3 u \,du.$$
%Using the formula for the integral of $\sec^3 u$ and reverting to the original variable $x$:
%\begin{eqnarray*}
  %\int\sqrt{1+x^2}\,dx&=&{\sec u \tan u\over2}+{\ln|\sec u +\tan
    %u|\over2}+C\cr
  %&=&{\sec(\arctan x) \tan(\arctan x)\over2}
    %+{\ln|\sec(\arctan x) +\tan(\arctan x)|\over2}+C\cr
  %&=&{ x\sqrt{1+x^2}\over2}
    %+{\ln|\sqrt{1+x^2} +x|\over2}+C,\cr
%\end{eqnarray*}
%using $\tan(\arctan x)=x$ and
%$\ds \sec(\arctan x)=\sqrt{1+\tan^2(\arctan x)}=\sqrt{1+x^2}$.
%\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:Trig Sub}}

\begin{enumialphparenastyle}

%%%%%%%%%%%
%\begin{ex}
 %$\ds\int\csc x\,dx$
%\begin{sol}
 %$-\ln|\csc x+\cot x|+C$
%\end{sol}
%\end{ex}
%%%%%%%%%%%
%
%%%%%%%%%%%
%\begin{ex}
 %$\ds\int\csc^3 x\,dx$
%\begin{sol}
 %$-\csc x\cot x/2-(1/2)\ln|\csc x+\cot x|+C$
%\end{sol}
%\end{ex}
%%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int\sqrt{x^2-1}\,dx$
\begin{sol}
 $\ds x\sqrt{x^2-1}/2-\ln|x+\sqrt{x^2-1}|/2+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int\sqrt{9+4x^2}\,dx$
\begin{sol}
 $\ds x\sqrt{9+4x^2}/2+\hbox{$\ds(9/4)\ln|2x+\sqrt{9+4x^2}|+C$}$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x\sqrt{1-x^2}\,dx$
\begin{sol}
 $\ds -(1-x^2)^{3/2}/3+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x^2\sqrt{1-x^2}\,dx$
\begin{sol}
 $\arcsin(x)/8-\sin(4\arcsin x)/32+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int{1\over\sqrt{1+x^2}}\,dx$
\begin{sol}
 $\ds \ln|x+\sqrt{1+x^2}|+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int\sqrt{x^2+2x}\,dx$
\begin{sol}
 $\ds (x+1)\sqrt{x^2+2x}/2-\hbox{$\ds\ln|x+1+\sqrt{x^2+2x}|/2+C$}$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int{1\over x^2(1+x^2)}\,dx$
\begin{sol}
 $-\arctan x - 1/x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int{x^2\over\sqrt{4-x^2}}\,dx$
\begin{sol}
 $\ds 2\arcsin(x/2)-x\sqrt{4-x^2}/2+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int{\sqrt{x}\over\sqrt{1-x}}\,dx$
\begin{sol}
 $\ds \arcsin(\sqrt{x})-\sqrt{x}\sqrt{1-x}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int{x^3\over\sqrt{4x^2-1}}\,dx$
\begin{sol}
 $\ds (2x^2+1)\sqrt{4x^2-1}/24+C$
\end{sol}
\end{ex}
%%%%%%%%%%

\end{enumialphparenastyle}