\documentclass[12pt]{cdblatex}
\usepackage{examples}

\begin{document}

\section*{Elementary maths}

\vspace{-5pt}

This example is based on a similar example in the \href{../../python/examples/example-01.pdf}{Python collection}. Its purpose is to show that Cadabra is fluent in Python -- which is not surprisng since the Cadabra language is based on Python (and a subset of LaTeX).

\vspace{-8pt}

\begin{minipage}[t]{0.65\textwidth}
\begin{cadabra}
   from sympy import *
   x, y, z = symbols('x y z')
   a, b, c = symbols('a b c')
   ans = expand((a+b)**3)                                  # cdb (ans.101,ans)
   ans = factor(-2*x+2*x+a*x-x**2+a*x**2-x**3)             # cdb (ans.102,ans)
   ans = solve(x**2-4, x)                                  # cdb (ans.103,ans)
   ans = solve([2*a-b - 3, a+b+c - 1,-b+c - 6],[a,b,c])    # cdb (ans.104,ans)
   ans = N(pi,50)                                          # cdb (ans.105,ans)
   ans = apart(1/((1 + x)*(5 + x)))                        # cdb (ans.106,ans)
   ans = together((1/(1 + x) - 1/(5 + x))/4)               # cdb (ans.107,ans)
   ans = simplify(tanh(log(x)))                            # cdb (rhs.108,ans)
   ans = simplify(tanh(I*x))                               # cdb (rhs.109,ans)
   ans = simplify(sinh(3*x) - 3*sinh(x) - 4*(sinh(x))**3)  # cdb (rhs.110,ans)
   ans = tanh(log(x))                                      # cdb (lhs.108,ans)
   ans = tanh(UnevaluatedExpr(I*x))                        # cdb (lhs.109,ans)
   ans = sinh(3*x) - 3*sinh(x) - 4*(sinh(x))**3            # cdb (lhs.110,ans)
\end{cadabra}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.35\textwidth}
\begin{latex}
   \begin{align*}
      &\cdb*{ans.101}\\
      &\cdb*{ans.102}\\
      &\cdb*{ans.103}\\
      &\cdb*{ans.104}\\
      &\cdb*{ans.105}\\
      &\cdb*{ans.106}\\
      &\cdb*{ans.107}\\
      \cdb{lhs.108} &= \Cdb{rhs.108}\\
      \cdb{lhs.109} &= \Cdb{rhs.109}\\
      \cdb{lhs.110} &= \Cdb{rhs.110}
   \end{align*}
\end{latex}
\end{minipage}

\vspace{-8pt}

\begin{align*}
   &\cdb*{ans.101}\\
   &\cdb*{ans.102}\\
   &\cdb*{ans.103}\\
   &\cdb*{ans.104}\\
   &\cdb*{ans.105}\\
   &\cdb*{ans.106}\\
   &\cdb*{ans.107}\\
   \cdb{lhs.108} &= \Cdb{rhs.108}\\
   \cdb{lhs.109} &= \Cdb{rhs.109}\\
   \cdb{lhs.110} &= \Cdb{rhs.110}
\end{align*}

\clearpage

\section*{Linear Algebra}

\vspace{-10pt}

\begin{minipage}[t]{0.65\textwidth}
\begin{cadabra}
   from sympy import linsolve
   lamda = Symbol('lamda')
   mat  = Matrix([[2,3], [5,4]])                   # cdb (ans.201,mat)
   eig1 = mat.eigenvects()[0][0]                   # 1st eigenvalue
   eig2 = mat.eigenvects()[1][0]                   # 2nd eigenvalue
   v1   = mat.eigenvects()[0][2][0]                # 1st eigenvector
   v2   = mat.eigenvects()[1][2][0]                # 2nd eigenvector
   eig  = simplify(Matrix([eig1,eig2]))            # cdb (ans.202,eig)
   vec  = simplify(5*Matrix([]).col_insert(0,v1)
                               .col_insert(1,v2))  # cdb (ans.203,vec)
   det  = expand((mat - lamda * eye(2)).det())     # cdb (ans.204,det)
   rhs  = Matrix([[3],[7]])                        # cdb (ans.205,rhs)
   ans  = list(linsolve((mat,rhs),x,y))[0]         # cdb (ans.206,ans)
\end{cadabra}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.35\textwidth}
\begin{latex}
   \begin{align*}
      &\cdb*{ans.201}\\
      &\cdb*{ans.202}\\
      &\cdb*{ans.203}\\
      &\cdb*{ans.204}\\
      &\cdb*{ans.205}\\
      &\cdb*{ans.206}
   \end{align*}
\end{latex}
\end{minipage}

\begin{align*}
   &\cdb*{ans.201}\\
   &\cdb*{ans.202}\\
   &\cdb*{ans.203}\\
   &\cdb*{ans.204}\\
   &\cdb*{ans.205}\\
   &\cdb*{ans.206}
\end{align*}

\clearpage

\section*{Limits}

\vspace{-10pt}

\begin{minipage}[t]{0.65\textwidth}
\begin{cadabra}
   n, dx = symbols('n dx')
   ans = limit(sin(4*x)/x,x,0)                  # cdb (ans.301,ans)
   ans = limit(2**x/x,x,oo)                     # cdb (ans.302,ans)
   ans = limit(((x+dx)**2 - x**2)/dx, dx,0)     # cdb (ans.303,ans)
   ans = limit((4*n + 1)/(3*n - 1),n,oo)        # cdb (ans.304,ans)
   ans = limit((1+(a/n))**n,n,oo)               # cdb (ans.305,ans)
\end{cadabra}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.35\textwidth}
\begin{latex}
   \begin{align*}
      &\cdb*{ans.301}\\
      &\cdb*{ans.302}\\
      &\cdb*{ans.303}\\
      &\cdb*{ans.304}\\
      &\cdb*{ans.305}
   \end{align*}
\end{latex}
\end{minipage}

\begin{align*}
   &\cdb*{ans.301}\\
   &\cdb*{ans.302}\\
   &\cdb*{ans.303}\\
   &\cdb*{ans.304}\\
   &\cdb*{ans.305}
\end{align*}

\section*{Series}

\vspace{-10pt}

\begin{minipage}[t]{0.65\textwidth}
\begin{cadabra}
   ans = series((1 + x)**(-2), x, 1, 6)         # cdb (ans.401,ans)
   ans = series(exp(x), x, 0, 6)                # cdb (ans.402,ans)
   ans = Sum(1/n**2, (n,1,50)).doit()           # cdb (ans.403,ans)
   ans = Sum(1/n**4, (n,1,oo)).doit()           # cdb (ans.404,ans)
\end{cadabra}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.35\textwidth}
\begin{latex}
   \begin{align*}
      &\cdb*{ans.401}\\
      &\cdb*{ans.402}\\
      &\cdb*{ans.403}\\
      &\cdb*{ans.404}
   \end{align*}
\end{latex}
\end{minipage}

\begin{align*}
   &\cdb*{ans.401}\\
   &\cdb*{ans.402}\\
   &\cdb*{ans.403}\\
   &\cdb*{ans.404}
\end{align*}

\clearpage

\section*{Calculus}

This example shows how {\tt\small\verb|\Cdb|} can be used to set the equation tag on the far right hand side.

\begin{minipage}[t]{0.65\textwidth}
\begin{cadabra}
   ans = diff(x*sin(x),x)                                    # cdb (ans.501,ans)
   ans = diff(x*sin(x),x).subs(x,pi/4)                       # cdb (ans.502,ans)
   ans = integrate(2*sin(x)**2, (x,a,b))                     # cdb (ans.503,ans)
   ans = Integral(2*exp(-x**2), (x,0,oo))                    # cdb (lhs.504,ans)
   ans = ans.doit()                                          # cdb (ans.504,ans)
   ans = Integral(Integral(x**2 + y**2, (y,0,x)), (x,0,1))   # cdb (lhs.505,ans)
   ans = ans.doit()                                          # cdb (ans.505,ans)
\end{cadabra}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.35\textwidth}
\begin{latex}
   \begin{align*}
      &\cdb*{ans.501}\\
      &\cdb*{ans.502}\\
      &\cdb*{ans.503}\\
       \cdb{lhs.504}&=\Cdb{ans.504}\\
       \cdb{lhs.505}&=\Cdb{ans.505}
   \end{align*}
\end{latex}
\end{minipage}

\begin{align*}
   &\cdb*{ans.501}\\
   &\cdb*{ans.502}\\
   &\cdb*{ans.503}\\
    \cdb{lhs.504}&=\Cdb{ans.504}\\
    \cdb{lhs.505}&=\Cdb{ans.505}
\end{align*}

\clearpage

\section*{Differential equations}

\vspace{-10pt}

\begin{minipage}[t]{0.70\textwidth}
\begin{cadabra}
   y = Function('y')
   C1, C2 = symbols('C1 C2')

   ode = Eq(y(x).diff(x) + y(x), 2*a*sin(x))
   sol = expand(dsolve(ode,y(x)).rhs)                              # cdb (ans.601,sol)
   cst = solve([sol.subs(x,0)],dict=True)
   sol = sol.subs(cst[0])                                          # cdb (ans.602,sol)

   ode = Eq(y(x).diff(x,2) + y(x), 0)
   sol = expand(dsolve(ode,y(x)).rhs)                              # cdb (ans.603,sol)
   cst = solve([sol.subs(x,0),sol.diff(x).subs(x,0)-1],dict=True)
   sol = sol.subs(cst[0])                                          # cdb (ans.604,sol)

   ode = Eq(y(x).diff(x,2) + 5*y(x).diff(x) - 6*y(x), 0)
   sol = expand(dsolve(ode,y(x)).rhs)                              # cdb (ans.605,sol)
   sol = sol.subs({C1:2,C2:3})                                     # cdb (ans.606,sol)
\end{cadabra}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.30\textwidth}
\begin{latex}
   \begin{align*}
      &\cdb*{ans.601}\\
      &\cdb*{ans.602}\\
      &\cdb*{ans.603}\\
      &\cdb*{ans.604}\\
      &\cdb*{ans.605}\\
      &\cdb*{ans.606}
   \end{align*}
\end{latex}
\end{minipage}

\begin{align*}
   &\cdb*{ans.601}\\
   &\cdb*{ans.602}\\
   &\cdb*{ans.603}\\
   &\cdb*{ans.604}\\
   &\cdb*{ans.605}\\
   &\cdb*{ans.606}
\end{align*}

\end{document}


\gotosection{0}{4}
\subsection{Functions}

\begin{exercise}{9}
  \begin{enumerate}
    \item $\begin{aligned}[t]
            (f \circ g \circ h)(a) &= f(g(h(a))) = f(g(h(3))) \\
                                   &= f(g(-3+2)) = f(g(-1)) \\
                                   &= f(3 \times (-1)) = f(-3) \\
                                   &= (-3)^2-1 = 8
          \end{aligned}$
    \item $\begin{aligned}
            (f \circ g \circ h)(a) &= f(g(h(a))) = f(g(h(1))) \\
                                   &= f(g(1-3)) = f(g(-2)) \\
                                   &= f(-2-3) = f(-5) \\
                                   &= (-5)^2 = 25
          \end{aligned}$
  \end{enumerate}
\end{exercise}

\begin{exercise}{10}
  \begin{enumerate}
    \item Let the function \FunSS{f}{B}{C} and \FunSS{g}{A}{B} be onto.
          Then the composition $f \circ g$ is onto.

    \Proof{} For every $c \in C$: since \FunSS{f}{B}{C} is surjective, there
    exists $b \in B$ such that $f(b) = c$; since \FunSS{g}{A}{B} is surjective,
    there exists $a \in A$ such that $g(a) = b$. Therefore, for every $c \in C$,
    there exists $a \in A$ such that $(f \circ g)(a) = f(g(a)) = c$.
    Thus, $f \circ g$ is surjective, or onto. \QED

    \item Let the function \FunSS{f}{B}{C} and \FunSS{g}{A}{B} be one to one.
          Then the composition $f \circ g$ is one to one.

    \Proof{} For every $c \in C$ such that there exists $a \in A$ such that
    $f(g(a)) = c$, suppose there do exist $x, y \in A$ such that both $f(g(x)) = c$
    and $f(g(y)) = c$. Since $f$ is injective, $g(x) = g(y)$. Since $g$ is injective
    , $x = y$. Therefore, there is at most one $x$ such that $(f \circ g)(x) =
    f(g(x)) = c$. Thus, $f \circ g$ is injective, or one to one. \QED
  \end{enumerate}
\end{exercise}


\documentclass[paper.tex]{subfiles}

\usepackage{tikz}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{tabularx}
\usepackage{multicol}
\usepackage{algpseudocode}
\usepackage{algorithm}

% Add vertical spacing to tables
\renewcommand{\arraystretch}{1.4}

% Begin Document
\begin{document}

\section{A System of Linear Equations}

This problem can be solved by utilizing a system of linear equations.
Each equation represents the input and output of a specific router node.
First we must construct a table of the input and output flow of each node.
The input and output must be equivalent for each node.
Solving for variables for each node gives us a system of linear equations.

\begin{center}
\begin{tabular}{c | l | l}

    \textbf{Node} & \textbf{Input} & \textbf{Output} \\ \hline

    A & 100                         & $2x_1 + x_2$ \\
    B & $x_1 + x_2$                 & $x_3 + x_5$  \\
    C & $50 + x_1$                  & $x_3 + x_5$  \\
    D & $x_4 + x_5$                 & $x_2 + 120$  \\
    E & $x_2 + x_3 + x_5$ \hspace{8mm} & $x_4$

\end{tabular}

\begin{tabular}{c | l}
    \textbf{Node} & \textbf{Equations} \\ \hline
    A & $2x_1 + x_2 = 100$ \\
    B & $x_1 + x_2 - x_3 - x_5 = 0$    \\
    C & $x_1 - x_3 - x_5 = -50$        \\
    D & $-x_2 + x_4 + x_5 = 120$       \\
    E & $x_2 + x_3 - x_4 + x_5 = 0$
\end{tabular}
\end{center}

Utilizing this system of equations, we can construct a matrix equation of the form $Ax = b$ representing it.
This becomes equation (1) below:

\begin{center}
\begin{equation}
    \begin{bmatrix}
        2 & 1 & 0 & 0 & 0   \\
        1 & 1 & -1 & 0 & -1 \\
        1 & 0 & -1 & 0 & -1 \\
        0 & -1 & 0 & 1 & 1  \\
        0 & 1 & 1 & -1 & 1
    \end{bmatrix}
    \begin{bmatrix}
        x_1 \\
        x_2 \\
        x_3 \\
        x_4 \\
        x_5
    \end{bmatrix}
    =
    \begin{bmatrix}
        100 \\
        0 \\
        -50 \\
        120 \\
        0
    \end{bmatrix}
\end{equation}
\end{center}

\end{document}


%!TEX root = paper.tex
\subsection{Linear standing wave}
The linear standing wave test illustrates the different linear dispersion relations and the ability of the numerical method to represent them in an accurate manner.
Both, the linearized systems of the hydrostatic and dispersive have an analytic standing wave solution
\begin{align}
\xi(\bx,t)&=-a\sin\left(\kappa x \right)\cos\left(\kappa ct\right), \\
 u(\bx,t) &=a \frac{c}{d}\cos\left(\kappa x\right)\sin\left(\kappa ct\right), \\
 v(\bx,t) &=0, \qquad \forall \ \bx=(x, y)^T \in \Omega, \forall t \in \mathbb{R},
\end{align}
where the phase velocity $c=\frac{\omega_{\text{nh,}\fnh}}{\kappa}$ is chosen for the dispersive equation set and $c=\csw$ for the hydrostatic equation set, respectively. For the \nh\ model, the analytic vertical velocity has to be defined as
\[
w(\bx,t)=-\frac{d}{2}\partial_x u=\frac{1}{2}ac\kappa\sin\left(\kappa x \right)\sin\left(\kappa ct\right),
\]
which is derived directly from \eqref{eq:nh_closure}.
In different model runs, we vary the depth $d$ while keeping the wave length $\lambda=\frac{2\pi}{\kappa}=20$ m and the amplitude $a=0.01$m constant to get ratios for $\frac{d}{\lambda}$ between $0.05$ and $1.0$, and ratios for $\frac{a}{d}$ between $0.01$ and $0.005$, respectively.
We impose double periodic boundary conditions on a grid length of one wave length. The simulation time is chosen long enough to measure one wave period.

\subsubsection{Results of \nh\ model}
The resulting normalized phase velocities for the shallow water model and the \nh\ equation set with either the linear or the quadratic vertical pressure are displayed in figure \ref{fig:nh_standingwave}. Furthermore, they are compared to their analytical reference phase velocities and the full reference phase velocity as derived in section \ref{sec:dispersion}.
In each case, the numerical dispersion relation matches the corresponding analytical one precisely.
% The coincidence between numerical and analytical phase velocities is very good.
In a close neighborhood of the shallow water assumption (i.e., in the limit $\frac{d}{\lambda} \rightarrow 0$), the quadratic vertical pressure profile gives a better phase velocity compared to the full reference solution than the linear profile, as expected from series expansions around this state used in \Bt\ models.
% \svnote{Can we have an additional detail plot of this neighborhood?} \ajnote{Then only thinner lines would help. I could also provide the whole image with different plotting style (thinner lines and ) One could also use absolute errors between numerical values and reference phase velocities. }\jbnote{What are you expecting to see?}\svnote{I expect to see what we describe in the text, i.e., that the quadratic vertical pressure profile gives a better phase velocity than the linear one near $d/\lambda=0$, see added figure.}
However, for ratios $\frac{d}{\lambda} > 0.25$ approximately, the linear profile matches better.

\begin{figure}[htbp]
        \includegraphics*[width=0.95\textwidth]{standingwave_nh.eps}
        \caption{Standing wave: Comparison of simulated hydrostatic and \nh\ phase velocities with analytic reference values for all simulations (left) and a zoom onto the close neighborhood of the long wave limit (right)}
        \label{fig:nh_standingwave}
\end{figure}


\section{Juncai}
\subsection{Classification Problems for DNN}
Now we want to make some rigorous mathematical statement about classification problem for DNN. What we want to state and prove is about, what's the necessary and sufficient conditions for $\{x_i, y_i\}_{i \in I}$ such that, DNN model(with one hidden layer, when we talks about DNN model in this subsection, we always mean one hidden layer model.) can ``separate" them.

Let us note:
\begin{align}
A_k = \{x_i ~ | ~  y_i = e_k, i \in I\} \quad k = 1:c.
\end{align}
We need the next two assumptions for data first:
\begin{assumption}[Boundedness]We say those data $\{x_i, y_i\}$ for $i \in I$ are bounded if there exist a $M < \infty$ such that
\begin{align}
\|x_i\| < M \quad \forall i \in I.
\end{align}
\end{assumption}
\begin{assumption}[Consistence]We say those data $\{x_i, y_i\}$ for $i \in I$ are consistent for classification if
\begin{align}
A_k \cap A_k =  \emptyset \quad \forall k\neq l.
\end{align}
\end{assumption}
Then we can have the next definition for what means ``DNN can separate  $\{A_k\}_{k=1}^c$":
\begin{definition}
We say that DNN model can separate $\{A_k\}_{k=1}^c$ or DNN model can fix the classification problem for $\{x_i ,y_i\}_{i \in I}$ means that there is a function $f: \mathbb{R}^n \to \mathbb{R}^c$ such that:
\begin{align}
f(x) = e_k \quad \forall x \in A_k, ~~ k = 1:c,
\end{align}
with
\begin{align}
f(x) = a(W_2 \sigma(W_1 x + b_1) + b_2),
\end{align}
for some
\begin{align}
W_1 \in \mathbb{R}^{m \times n}, \quad b_1 \in \mathbb{R}^m \quad W_2 \in \mathbb{R}^{c\times m}, \quad b_2 \in \mathbb{R}^c, \quad \epsilon > 0,
\end{align}
and $\sigma, a: \mathbb{R} \to \mathbb{R}$.
Here $\sigma$ can be ``Sigmoid", ``ReLU", ``Heaviside" or $a$ function, with
\begin{equation}
\sigma(x) = \frac{1}{1 + e^{-x}}, \quad r(x) = \max\{0,x\}, \quad H(x) = \begin{cases}
1 ~~ &x > 0 \\
0 ~~ &x \le 0.
\end{cases}
\end{equation}
and $a$ is a variant form of Heaviside function or combine of two ReLU functions like:
%\begin{equation}
%\hat{H}_{\epsilon}(x) = \begin{cases}
%0 \quad & x < \epsilon, \\
%\frac{x}{1-2\epsilon} - \frac{\epsilon}{1-2\epsilon} \quad & \epsilon \le x < 1-\epsilon, \\
%1 \quad & x \ge 1-\epsilon.
%\end{cases}
%\end{equation}
%By the way, $H_{\epsilon}(x) = \frac{1}{1-2\epsilon}r(x - \epsilon) +  \frac{1}{1-2\epsilon}r(x - 1 + \epsilon).$
\begin{equation}
a(x) = \begin{cases}
0 \quad & x < 0, \\
x \quad & 0 \le x < 1, \\
1 \quad & x \ge 1.
\end{cases}
\end{equation}
By the way, $a(x) = r(x ) - r(x - 1).$

\end{definition}

Here we define what means ``$\{A_k\}_{k=1}^c$ are separable with positive distance":
\begin{definition}
We say that $\{A_k\}_{k=1}^c$ are separable with positive distance when there exists a small positive number $\epsilon$, such that
\begin{align}
d(\bar{A}_k, \bar{A}_l) > \epsilon, \quad \forall k \neq l,
\end{align}
where $\bar{A}_k$ means the closure of $A_k$ and $d(\bar{A}_k, \bar{A}_l)$ define the distance of two sets as:
\begin{align}
d(\bar{A}_k, \bar{A}_l) = \inf_{x \in A_k, y \in A_l} d(x,y).
\end{align}
\end{definition}

As for the ``separable with positive distance" property, we have the next lemma:
\begin{lemma}If $\{A_k\}_{k=1}^c$ are separable with positive distance $\epsilon$, then there exist a smooth feature map $\phi: \mathbb{R}^n \to \mathbb{R}^c$ such that:
\begin{align}
\phi(x) = e_k \quad \forall x \in \bar{A}_k,\quad k = 1:c.
\end{align}
\end{lemma}
\begin{proof}This proof is based on the partition of unity theorem. Because of the positive distance with $\epsilon$ property, we define
\begin{equation}
\mathcal{M}_k = \{x ~|~ d(x, \bar{A}_k) < \frac{\epsilon}{2}\}.
\end{equation}
So, $\mathcal{M}_k$ is an open set as the continuity for $d(\cdot, \bar{A}_k)$. Let
\begin{equation}
\mathcal{M}_0 = (\bigcup_{k=1:c} \bar{A}_k)^c,
\end{equation}
so $\mathcal{M}_0$ is an open set too. And
\begin{equation}
\mathbb{R}^n = (\bigcup_{k=0:c} \mathcal{M}_k).
\end{equation}
So we can apply the partition of unity for the above decomposition, getting $\{\phi_k\} \in C^{\infty}(\mathbb{R}^n)$ for $k = 0:c$ and
\begin{equation}
1 = \sum_{k=0}^c \phi_k \quad \text{with} \quad supp(\phi_k) \subset \mathcal{M}_k.
\end{equation}
Then we will see that:
\begin{equation}
\phi_k(x_l) = \delta_{kl} \quad \forall x_l \in \bar{A}_l, \quad k, l = 1:c,
\end{equation}
because of the definition of $\mathcal{M}_k$ and the results of partition of unity.

At last, we can get $\phi$ as:
\begin{align}
\phi: \mathbb{R}^n &\to \mathbb{R}^c \\
 x &\mapsto (\phi_1(x), \cdots, \phi_c(x)),
\end{align}
and it satisfy the condition.
\end{proof}

Now we can have those next theorem:
\begin{theorem}
If $I$ is a finite index set, then DNN can always separate $\{A_k\}_{k=1}^c$.
\end{theorem}

\begin{theorem}
If $\{A_k\}_{k=1}^c$ are separable with positive distance, then it can be separated by DNN.
\end{theorem}
\begin{proof}
We know that
\begin{equation}
a(e_k) = e_k \quad k = 1:c.
\end{equation}

Because of the universal approximation property for ``Sigmoid", ``ReLu", ``Heaviside" or $a(x)$ type Neural Network, we can take
\begin{align}
\hat{W}_2\sigma(W_1 x + b_1) + \hat{b}_2,
\end{align}
to approximate $\phi$ in $B_M$ such that,
\begin{align}
\|\hat{W}_2\sigma(W_1 x + b_1) + \hat{b}_2 - \phi(x) \|_{L^{\infty}} < \delta < \frac{1}{4}, \quad \forall \|x\| \le M.
\end{align}
{\bf Here $\delta$ is independent with the distant of training data, we can choose any one in $(0, \frac{1}{2})$ in fact.
This is because of the linearly separable of $\{e_k\}_{k=1}^c$ points.
}

Now we can take the DNN model as:
\begin{align}
f(x) = a(\frac{1}{1-2\delta}\hat{W}_2\sigma(W_1 x + b_1) + (\hat{b}_2 -\frac{\delta}{1-2\delta}\bm{1})).
\end{align}
So we only need to check that $f(x) = e_k$ if $x\in A_k$ with $k=1:c$.

If $x \in A_k$ for any $k=1:c$, we know that
\begin{align}
|(\hat{W}_2\sigma(W_1 x + b_1) + \hat{b}_2 - e_k)_i| < \delta, \quad  i = 1:c,
\end{align}
which means that
\begin{equation}
\begin{cases}
(\hat{W}_2\sigma(W_1 x + b_1) + \hat{b}_2 )_i < \delta \quad &i\neq k \\
(\hat{W}_2\sigma(W_1 x + b_1) + \hat{b}_2 )_i  > 1- \delta \quad &i = k.
\end{cases}
\end{equation}
Thus by the definition of $a(x)$, we know:
\begin{equation}
a(\frac{x}{1-2\delta} - \frac{\delta}{1-2\delta}) = \begin{cases}
0 \quad & x < \delta, \\
\frac{x}{1-2\delta} - \frac{\delta}{1-2\delta} \quad & \delta \le x < 1 - \delta, \\
1 \quad & x \ge 1 - \delta.
\end{cases}
\end{equation}
So we have
$f(x) = e_k$ if $x\in A_k$ with $k=1:c$, for
\begin{align}
f(x) = a(W_2\sigma(W_1 x + b_1) + b_2),
\end{align}
with
\begin{equation}
W_2 = \frac{1}{1-2\delta}\hat{W}_2, ~ \text{and} ~ b_2 = \hat{b}_2 -\frac{\delta}{1-2\delta}\bm{1}.
\end{equation}
This finishes this proof.
\end{proof}

\begin{remark}
Here $W_2$ have a special structure like as
\begin{align}
W_2 = diag\{\omega_1, \cdots, \omega_c\},
\end{align}
where $\omega_i \in \mathbb{R}^{1\times n_i}$ with $\sum_{i=1}^c n_i = m$.
\end{remark}

\begin{theorem}
If $\{A_k\}_{k=1}^c$ can be separated by DNN, then $\{A_k\}_{k=1}^c$ are separable with positive distance.
\end{theorem}
\begin{proof}First we know that,
\begin{align}
A_k \subset L_k := \{x ~|~ f(x) = e_k\}.
\end{align}
Here $L_k$ is a closed set and $d(L_k, L_l) \ge \delta > 0$ because of the continuity of $f$. So,
$\{A_k\}_{k=1}^c$ are separable with positive distance.
\end{proof}

\subsection{Linearly Separable and Global Minima}

Here we note those data sets $\{x_i, y_i\}_{i=1}^N$ as $\{A_k\}_{k=1}^c$ with $y = e_k$ for any $x \in A_k$. The simple NN model or perceptron with Heaviside function or $H_{\epsilon}$ like:
\begin{equation}
H_{\epsilon}(x) = \begin{cases}
0 \quad & x < -\epsilon, \\
\frac{x}{2\epsilon} - \frac{1}{2} \quad & -\epsilon \le x < \epsilon, \\
1 \quad & x \ge \epsilon.
\end{cases}
\end{equation}
can classify those data means there exist $W \in \mathbb{R}^{c\times n}$ and $b \in \mathbb{c}$ such that
\begin{align}
H(Wx + b) = e_k \quad x \in A_k, ~ k =1:c.
\end{align}

Note the loss function as:
\begin{align}
L(\theta) = \sum_{i=1}^N\|H(Wx_i + b) - y_i\|^2.
\end{align}

\begin{properties}Those next two statements are equivalent:
\begin{enumerate}
\item The perceptron can classify those data.
\item $\min L(\theta) = 0$.
\end{enumerate}
\end{properties}

\begin{theorem}\label{theo:linearconvex}
Let
\begin{align}
\Theta := \mathop{\arg\min}_{\theta}L(\theta).
\end{align}
Then $\Theta$ is a convex set if the perceptron can classify those data.
\end{theorem}

\begin{proof}
If the perceptron can classify those data, then $\Theta \neq \emptyset$. So, let $\theta_1, \theta_2 \in \Theta$, let us consider $\theta = t\theta_1 + (1-t)\theta_2$ with $t \in [0,1]$.
Then for any data $(x_i, y_i)$ and $\theta = (W, b)$ we have
\begin{align}
H(Wx_i + b) = H(t(W_1x_i + b_1) + (1-t)(W_2x_i + b_2)),
\end{align}
and
\begin{align}
H(W_1x_i + b_1) = H(W_2x_i + b_2) = y_i = e_k,
\end{align}
for some $1 \le k \le c$.

Because of the properties of Heaviside function(its level sets are convex sets), we know that:
\begin{align}
(W_1 x_i + b_1)_j, (W_2 x_i + b_2)_j  \le 0 \quad &\text{if}~ j \neq k \\
(W_1 x_i + b_1)_j, (W_2 x_i + b_2)_j    > 0 \quad &\text{if}~ j = k.
\end{align}
This means that $H(Wx_i + b) = e_k = y_i$, so $\theta \in \Theta$.
\end{proof}

%\begin{remark}
%This result also holds for $H_\epsilon$ and $\hat{H}_{\epsilon}$, and in fact
%\begin{align}
%\hat{H}_{\epsilon}(x) = H_{\frac{1-2\epsilon}{2}}(x -1).
%\end{align}
%\end{remark}

\begin{theorem}\label{theo:linearinfinity}
If $A_i$ are compact sets, then
there are infinity global minima for $L(\theta)$ if $\{A_i\}$ are also linearly separable.
\end{theorem}
\begin{proof}If $\{A_i\}$ are also linearly separable, then there exist $\theta=(W,b)$ such that
\begin{align}
H(Wx + b) = e_k \quad x \in A_k, ~ k =1:c.
\end{align}
Let us note that:
\begin{align}
(Wx + b)_k = \omega_k^T x + b_k,
\end{align}
so we have
\begin{align}
\omega_k^T x + b_k > 0 \quad &\text{if}~ x \in A_k \\
\omega_k^T x + b_k \le 0 \quad &\text{if}~ x \in A \setminus A_k.
\end{align}
By using the compactness for $A_k$, we have
\begin{align}
\min_{x\in A_k} \omega_k^T x + b_k = \epsilon > 0.
\end{align}
So, there at leat have those different $\theta_{k,\delta}$ with
\begin{align}
\theta_{k,\delta} = (W, \hat{b}),
\end{align}
and
\begin{align}
\begin{cases}
\hat{b}_i = b_i \quad &\text{if}~ i \neq k \\
\hat{b}_i = b_i - \delta \quad &\text{if}~ i = k,
\end{cases}
\end{align}
for any $0 \le \delta < \epsilon$.

Now, we next prove that, we can not only change $b$ but also $W$. First, we can assume that
\begin{align}
\omega_k^T x + b_k > 0 \quad &\text{if}~ x \in A_k \\
\omega_k^T x + b_k >0 \quad &\text{if}~ x \in A \setminus A_k.
\end{align}
If this is not true, we can change $b_k$ such that this $\theta$ can satisfy the above conditions with the above trick. Using the compactness again,
\begin{align}
\min_{x\in A_k} \omega_k^T x + b_k &\ge \epsilon > 0 \\
\max_{x\in A\setminus A_k} \omega_k^T x + b_k &\le -\epsilon < 0,
\end{align}
and $\|x\| < M $ for all $x \in A$.
So we may construct $\theta_{k,\delta}$ like:
\begin{align}
\theta_{k,\delta} = (\hat{W}, b),
\end{align}
and
\begin{align}
\begin{cases}
\hat{\omega}_i = \omega_i \quad &\text{if}~ i \neq k \\
\hat{\omega}_i = \omega_i + \omega \quad &\text{if}~ i = k,
\end{cases}
\end{align}
with $\omega \in B(0, \frac{\delta}{M}) \subset \mathbb{R}^n$ for any $0 < \delta < \epsilon$. Then we can check that
\begin{align}
\hat{\omega}_k^T x + b_k > 0 \quad &\text{if}~ x \in A_k \\
\hat{\omega}_k^T x + b_k >0 \quad &\text{if}~ x \in A \setminus A_k.
\end{align}
This means $\theta_{k,\delta}$ with above form also belongs to $\Theta$.
This shows that, $\Theta$ is infinity set and have both degree of freedom on $W$ and $b$.
\end{proof}

\begin{theorem}
If the perceptron can classify those data, then the SGD method for $\min L(\theta)$ will convergence with small positive learning rate.
\end{theorem}

\subsection{Separable and Global Minima}
In simple terms, we assume that those $\{A_k\}_{k=1}^c$ are compact sets. Then the positive distance can be got from the consistency.  Here we note that, we want to use general DNN model with just one hidden layer to separate those data, so the loss function will be:
\begin{align}
L(\theta) = \sum_{i=1}^N\|H(W_2\sigma(W_1x_i + b_1) + b_2) - y_i\|^2,
\end{align}
with $H$ could be $\hat{H}_{\epsilon}$ or $H_\epsilon$.

\begin{theorem}Let
\begin{align}
\Theta := \mathop{\arg\min}_{\theta}L(\theta).
\end{align}
For $(\theta_1, \theta_2) \in \Theta$, we note that
\begin{equation}
\Theta_2(\theta_1) = \{\theta ~|~ (\theta_1, \theta) \in \Theta \}.
\end{equation}
Then $\Theta_2(\theta_1)$ is a convex set if the DNN model can classify those data with compactness assumption.
\end{theorem}

\begin{proof}
Because of the continuity of $g(x) = \sigma(W_1x + b_1)$, so we have that $\{g(A_k)\}_{i=k}^c$ are also compact sets, and the consistency of $A_k$ also shows the consistency of $g(A_k)$.

Then because DNN model can classify those data, so $g(A_k)$ are also linear separable w.r.t to $\hat{H}_\epsilon$.
So, for fixed $\theta_1$, we can finish this proof by using Theorem \ref{theo:linearconvex}.
\end{proof}

\begin{theorem}If $A_i$ are compact sets, then
$\Theta_2(\theta_1)$ is a infinity set if $\{A_i\}$ can be separated by DNN model.
\end{theorem}
\begin{proof}Use the analysis in the theorem above and we can get a similar poof with Theorem \ref{theo:linearinfinity}.
\end{proof}

Next, we want to talk about some properties about $\theta_1$ part. We can also get some infinity property for
\begin{equation}
\Theta_1(\theta_2) = \{\theta ~|~ (\theta, \theta_2) \in \Theta \}
\end{equation} with some special $\theta_2$.

\subsection{Discussion with Prof. Xu on 3/10/2017}
Prof. Xu proposed those next points about vanishing gradient and random strategies in training algorithms:
\begin{enumerate}
\item ``Vanishing Gradient' is a mathematical problem.

\item For deep neural network models, ReLU activation function is better than Sigmoid in many cases just likes that singular system can be solved more easily than nearly singular system.

\item For some components in $\frac{\partial f^J_i}{\partial \theta^k_{st}} \approx 0$, we may just take those as 0.

\item For classification problems, if the class is not big, like $c = 10$, and we may have $100000$ data, and mini-batch size is $100$, which means for every mini-batch, we have $10$ samples in every class. One way for good "mini-batch" is to choose those $10$ samples in the same class as separable as possible. {\bf And in some degree, SGD is doing this. }

If the class is too big, like $c = 1000$ but mini-bath size is still $100$, we can change our mind to choose those mini-batch just comes from one class and also as separable as possible.

Here is a suggested strategy for how to choose those mini-batch  as separable as possible from Prof. Xu. For example, we have $N$ data for one class, and we want to choose $m$ data from this data set  as separable as possible. We may use next steps:
\begin{itemize}
\item Use KNN method for those data with $k = m$.
\item Use the uniform sampling strategy to choose one data in every cluster, then we get $m$ samples.
\end{itemize}

\end{enumerate}

\subsection{Train good smoother for multigrid methods}
We are going to using the idea of deconvolution to get some good smoother in multigrid method with fixing the restriction and interpolation method.

Data $\{u_k, f_k\}_{k=1}^N$ such that
\begin{equation}
A u_k = f_k,
\end{equation}
with $A$ is the linear FEM matrix with uniform grids.

Then we have stander nested multi-grids like $\{\mathcal{G}_j \}_{j=1}^J$, and $P_{j}^{j+1}$ is the interpolation operator form $\mathcal{G}_j$ to $\mathcal{G}_{j+1}$. We may have $S_j$ as the smoother in grid $\mathcal{G}$, there are also what we will trained. Under those set up, we may note $B(S_j)$ as the multigrid solve for $B \approx A^{-1}$.

So we optimization problem is:
\begin{equation}
\min_{S_j} \sum_{k=1}^N\| u_k - B(S_j)f_k\|^2.
\end{equation}

There are two points we need to take care of:
\begin{itemize}
\item How to choose those data $\{u_k, f_k\}_{k=1}^N$?

For this problem, Prof. Xu, suggested that we need to choose good data by choosing fine $f_k$ not $u_k$.

\item How to solve those above optimization problem?

One way to train this model is to compute
\begin{equation}
\sum_{k=1}^N\| u_k - B(S_j)f_k\|^2 = \sum_{k=1}^N \| (I - B(S_j)A)u_k\|^2.
\end{equation}
And then, we may use ``BP" for $I - B(S_j)A$ for $S_j$ from $j = J: -1 :1$? But, how to implement this, we may need to check more details.
\end{itemize}

\subsubsection{Another viewpoint for this problem}
We can use the idea for best convergence rate with
\begin{equation}
\min_{S_j} \|I - B(S_j)A\|^2,
\end{equation}
then the above question is the next $min-max$ problem:
\begin{equation}
\min_{S_j} \max_{\|u\| = 1}\|(I - B(S_j)A)u\|^2.
\end{equation}
And then we may take this method as:
\begin{equation}
 \sum_{k=1}^N \| (I - B(S_j)A)u_k\|^2 \approx \max_{\|u\| = 1}\|(I - B(S_j)A)u\|^2.
\end{equation}

\subsection{Seminar notes on 11/31/2017}
\begin{itemize}
\item Runge phenomenon and overfitting.
\item How about sample points and interpolation and Runge phenomenon.
\end{itemize}

\subsection{Discussion on 12/13/2017}
\subsubsection{Statistical learning based regression(classification)
  problems}

Here we suppose that for all image with label and task we want to solve, we have the next joint distribution in real world as:
\begin{equation}\label{eq:jointdit}
(X,Y) \sim p_{\rm data},
\end{equation}
as
\begin{equation}\label{eq:defPr}
\rm{Pr}(X \in I_x, Y \in I_y) = \int_{I_x \times I_y}p_{\rm data}(x,y)dxdy,
\end{equation}
with $X \in \mathbb{R}^{n\times n}$ and $Y \in \mathbb{R}^c$.

So for regression problem, the function that we want to interpolate is:
\begin{equation}\label{eq:deff}
f^*(x) = \mathbb{E}_Y (Y | X = x).
\end{equation}
This is in fact that
\begin{equation}\label{eq:f*}
f^* = \mathop{\arg\min}_{f ~\text{is a distribution}}\mathbb{E}_{X\times Y} \|y - f(x)\|^2.
\end{equation}

For classification problem, the values of $Y$ should be some discrete values such as $\{e_1, \cdots, e_c\}$. So the reals problem that we may try to solve is to use some function family to approximate the $f^*$ by:
\begin{equation}\label{eq:stat_opt}
\mathbb{E}_{X} (\|f^*(x) - f(x;\Theta)\|^2) = \mathbb{E}_{X\times Y}(\|y - f(x;\Theta)\|^2).
\end{equation}

\hrule
\paragraph{Jinchao:}  In general, we assume that the label $y$  of a given
data $x$ is also a random variable depending on $x$.  For example, for
$c$-classification problem,
$$
\sum_{k=1}^cP\{Y=e_k | X=x\}=1.
$$

$$
Y: \{e_1,\ldots e_k\}\mapsto \mathbb R^c.
$$

There are at least five different concepts:
\begin{enumerate}
\item The original expected loss
\item The empirical loss with $N$-sampled data
\item SGD directly from the original data [training]
\item SGD from the $N$-sampled data [training]
\item SGD with epoch from $N$-sampled data [training]
\end{enumerate}

Jinchao's questions:
\begin{enumerate}
\item From pure math point of view, generalization error is a concept
  for model, not for training algorithms
\item In the literature, people often relate the generalization error
  with a training algorithm.   This would only make sense if we treat
  a training algorithm as a modeling procedure.
\item In other words, the training algorithm, when finished in
  practice, has not really solved the original model problem
  accurately, namely a global minimizer or a ``good local minimizer''
\end{enumerate}

Traditionally, generalization error = test error.

For expected loss, the concept of ``training data'' and ``test data''
should not apply.

``test data'' are those data from the original data space (for
expected loss) that are not in the ``training data''

In practice, we should never aim to accurately find a global minimizer
of the empirical loss function, which is why we need ``validation
data''.  The role of the validation data

\begin{quote}
{\bf Jinchao's conjecture:}  Under reasonable assumptions, the exact
global minimizer of the empirical loss function would for sure lead to
over-fitting.
\end{quote}

In this case, we can find a DNN so that the emirical lost function is
zero.

We also note that a global minimizer of the expected loss function is
also a global minimizer of the empirical loss function.  But in
practice, it is hard to find a global minimizer of the expected loss
function.  The practical approximation of the global minimizer would
lead to over-fitting.

The above conclusions are not valid for cross-entry with soft-max.

There are many kinds of data
\begin{enumerate}
\item All feasible data:
\item training data
\item validation data
\item test data
\end{enumerate}

The labelled data can be divided to, subjectively or statistically,
into
\begin{enumerate}
\item training data
\item validation data
\item test data
\end{enumerate}
\bigskip
\hrule
\bigskip

\subsubsection{Manifold distance}
For classification problem, we may state that
\begin{equation}\label{def:manifoldAk}
A_k = \{x: f(x) = e_k\},\quad 1 \le k \le c,
\end{equation}
with $f$ is defined above \eqref{eq:deff}. Then the manifold distance means that
\begin{equation}\label{key}
\rm{dist}(A_k, A_l) \ge \delta > 0, \quad k \neq l.
\end{equation}


\section{Substitution Rule}\label{sec:SubRule}

Antiderivatives play a key role in the the process of evaluating definite integrals exactly.  In particular, the Fundamental Theorem of Calculus tells us that if $F$ is any antiderivative of $f$, then
$$\int_a^b f(x) \, dx = F(b) - F(a).$$
Furthermore, we realized that each elementary derivative rule leads to a corresponding elementary antiderivative (or indefinite integral), as summarized in Table~\ref{T:4.4.Act2}.  Thus, if we wish to evaluate an integral such as
$$\int_0^1 \left(x^3 - \sqrt{x} + 5^x \right) \,dx,$$
it is straightforward to do so, since we can easily integrate $f(x) = x^3 - \sqrt{x} + 5^x.$ In particular, since a function $F$ whose derivative is $f$ is given by $F(x) = \frac{1}{4}x^4 - \frac{2}{3}x^{3/2} + \frac{1}{\ln(5)}5^x$, the Fundamental Theorem of Calculus tells us that
\begin{eqnarray*}
\int_0^1 \left(x^3 - \sqrt{x} + 5^x\right) \,dx & = & \left. \frac{1}{4}x^4 - \frac{2}{3}x^{3/2} + \frac{1}{\ln(5)}5^x\right|_0^1 \\
								& = & \left( \frac{1}{4}(1)^4 - \frac{2}{3}(1)^{3/2} + \frac{1}{\ln(5)}5^1 \right) - \left( \frac{1}{4}(0)^4 - \frac{2}{3}(0)^{3/2} + \frac{1}{\ln(5)}5^0 \right) \\
								& = & \frac{1}{4} - \frac{2}{3} + \frac{5}{\ln(5)} - \frac{1}{\ln(5)} \\
								& = & -\frac{5}{12} + \frac{4}{\ln(5)}.
\end{eqnarray*}
Because an algebraic formula for an antiderivative of $f$ enables us to evaluate the definite integral $\int_a^b f(x) \, dx$ exactly, we see that we have a natural interest in being able to find such algebraic antiderivatives.  Note that we emphasize \emph{algebraic} antiderivatives, as opposed to any antiderivative, since we know by the Second Fundamental Theorem of Calculus that $G(x) = \int_a^x f(t) \, dt$ is indeed an antiderivative of the given function $f$, but one that still involves a definite integral.  One of our main goals in this section is to develop understanding, in select circumstances, of how to ``undo'' the process of differentiation in order to find an algebraic antiderivative for a given function.

\subsection*{Reversing the Chain Rule: Substitution}

It is usually straightforward to integrate a function of the form
$$h(x) = f(u(x)),$$
whenever $f$ is a familiar function whose antiderivative is known and $u(x)$ is a linear function.  For example, if we consider
$$h(x) = (5x-3)^6,$$
in this context the \textit{outer function} $f$ is $f(u) = u^6$, while the \textit{inner function} is $u(x) = 5x - 3$.  Since the antiderivative of $f$ is $F(u) = \frac{1}{7}u^7+C$, we
see that the antiderivative of $h$ is
$$H(x) = \frac{1}{7} (5x-3)^7 \cdot \frac{1}{5} + C = \frac{1}{35} (5x-3)^7 + C.$$
The inclusion of the constant $\frac{1}{5}$ is essential precisely because the derivative of the inner function is $u'(x) = 5$.  Indeed, if we now compute $H'(x)$, we find by the Chain Rule (and Constant Multiple Rule) that
$$H'(x) = \frac{1}{35} \cdot 7(5x-3)^6 \cdot 5 = (5x-3)^6 = h(x),$$
and thus $H$ is indeed the general antiderivative of $h$.

Hence, in the special case where the outer function is familiar and the inner function is linear, we can antidifferentiate composite functions according to the following rule.

\begin{formulabox}[]{
If $h(x) = f(ax + b)$ and $F$ is a known algebraic antiderivative of $f$, then the general antiderivative of $h$ is given by
$$H(x) = \frac{1}{a} F(ax+b) + C.$$
}
\end{formulabox}

Of course, a natural question  arises: what happens when the inner function is not a linear function?  For example, can we find antiderivatives of such functions as
$$g(x) = x e^{x^2} \ \mbox{and} \ h(x) = e^{x^2}?$$

It is important to explicitly remember that differentiation and antidifferentiation are essentially inverse processes; that they are not quite inverse processes is due to the $+C$ that arises when antidifferentiating.  This close relationship enables us to take any known derivative rule and translate it to a corresponding rule for an indefinite integral.  For example, since
$$\frac{d}{dx} \left[x^5\right] = 5x^4,$$
we can equivalently write
$$\int 5x^4 \, dx = x^5 + C.$$

Recall that the Chain Rule states that
$$\frac{d}{dx} \left[ f(g(x)) \right] = f'(g(x)) \cdot g'(x).$$
Restating this relationship in terms of an indefinite integral,
\begin{equation} \label{E:usubst}
\int f'(g(x)) g'(x) \, dx = f(g(x))+C.
\end{equation}
Hence, Equation~(\ref{E:usubst}) tells us that if we can take a given function and view its algebraic structure as $f'(g(x)) g'(x)$ for some appropriate choices of $f$ and $g$, then we can antidifferentiate the function by reversing the Chain Rule.  It is especially notable that both $g(x)$ and $g'(x)$ appear in the form of $f'(g(x)) g'(x)$; we will sometimes say that we seek to \emph{identify a function-derivative pair}\index{function-derivative pair} when trying to apply the rule in Equation~(\ref{E:usubst}).

For example: Find
$$\int 2x\cos(x^2)\,dx.$$
This is not a ``simple'' derivative, but a little thought reveals that
it must have come from an application of the chain rule. Multiplied
on the ``outside'' is $2x$, which is the derivative of the ``inside''
function $\ds x^2$. Checking:
$${d\over dx}\sin(x^2)  = \cos(x^2){d\over dx}x^2 = 2x\cos(x^2),$$
so
$$\int 2x\cos(x^2)\,dx=\sin(x^2)+C .$$

In the situation where we can identify a function-derivative pair, we will introduce a new variable $u$ to represent the function $g(x)$.  Observing that with $u = g(x)$, it follows in Leibniz notation that $\frac{du}{dx} = g'(x)$, so that in terms of differentials\footnote{If we recall from the definition of the derivative that $\frac{du}{dx} \approx \frac{\triangle{u}}{\triangle{x}}$ and use the fact that $\frac{du}{dx} = g'(x)$, then we see that $g'(x) \approx \frac{\triangle{u}}{\triangle{x}}$.  Solving for $\triangle u$, $\triangle u \approx g'(x) \triangle x$.  It is this last relationship that, when expressed in ``differential'' notation enables us to write $du = g'(x) \, dx$ in the change of variable formula.}, $du = g'(x)\, dx$.  Now converting the indefinite integral of interest to a new one in terms of $u$, we have
$$\int f'(g(x)) g'(x) \, dx = \int f'(u) \,du.$$
Provided that $f'$ is an elementary function whose antiderivative is known, we can now easily evaluate the indefinite integral in $u$, and then go on to determine the desired overall antiderivative of $f'(g(x)) g'(x)$.  We call this process \emph{$u$-substitution}.

To summarize: If we suspect that a given function is the derivative of
another via the chain rule, we let $u$ denote a likely candidate for
the inner function, then translate the given function so that it is
written entirely in terms of $u$, with no $x$ remaining in the
expression. If we can integrate this new function of $u$, then the
antiderivative of the original function is obtained by replacing $u$
by the equivalent expression in $x$.

\begin{theorem}{$ u $-Substitution Rule for Indefinite Integrals}{SubstitutionRule}
If $u=g(x)$ is a differentiable function whose range is an interval $I$ and $f$ is continuous on $I$, then
$$\int f(g(x))g'(x)\,dx=\int f(u)\,du.$$
\end{theorem}

Even in simple cases you may prefer to use this mechanical procedure,
since it often helps to avoid silly mistakes. For example, consider
again this simple problem:
$$\int 2x\cos(x^2)\,dx.$$
Let $\ds u=x^2$, then $du/dx = 2x$ or $du = 2x\,dx$. Since we have exactly
$2x\,dx$ in the original integral, we can replace it by $du$:
$$\int 2x\cos(x^2)\,dx=\int \cos u\,du=\sin u +C = \sin(x^2)+C.$$
This is not the only way to do the algebra, and typically there are
many paths to the correct answer. Another possibility, for example,
is: Since $du/dx = 2x$, $dx=du/2x$, and then the integral becomes
$$\int 2x\cos(x^2)\,dx=\int 2x\cos u\,{du\over 2x}=\int \cos u\,du.$$
The important thing to remember is that you must eliminate all
instances of the original variable $x$.

\begin{example}{Substitution Rule}{usub}\label{usub}
Evaluate the indefinite integral
$$\int x^3 \cdot \sin (7x^4 + 3) \, dx$$
and check the result by differentiating.
\end{example}

\begin{solution}
We can make two key algebraic observations stand regarding the integrand, $x^3 \cdot \sin (7x^4 + 3)$.  First, $\sin (7x^4 + 3)$ is a composite function; as such, we know we'll need a more sophisticated approach to antidifferentiating.  Second, $x^3$ is almost the derivative of $(7x^4 + 3)$; the only issue is a missing constant.  Thus, $x^3$ and $(7x^4 + 3)$ are nearly a function-derivative pair.  Furthermore, we know the antiderivative of $f(u) = \sin(u)$.  The combination of these observations suggests that we can evaluate the given indefinite integral by reversing the chain rule through $u$-substitution.

Letting $u$ represent the inner function of the composite function $\sin (7x^4 + 3)$, we have
$$u = 7x^4 + 3,$$
and thus $\frac{du}{dx} = 28x^3.$  In differential notation, it follows that $du = 28x^3 \, dx$, and thus $x^3 \, dx = \frac{1}{28} \, du$.  We make this last observation because the original indefinite integral may now be written
$$\int \sin (7x^4 + 3) \cdot x^3 \, dx,$$
and so by substituting the expressions in $u$ for $x$ (specifically $u$ for $7x^4 + 3$ and $\frac{1}{28} \, du$ for $x^3 \, dx$), it follows that
$$\int \sin (7x^4 + 3) \cdot x^3 \, dx = \int \sin(u) \cdot \frac{1}{28} \, du.$$
Now we may evaluate the original integral by first evaluating the easier integral in $u$, followed by replacing $u$ by the expression $7x^4 + 3$.  Doing so, we find
\begin{eqnarray*}
\int \sin (7x^4 + 3) \cdot x^3 \, dx & = & \int \sin(u) \cdot \frac{1}{28} \, du \\
						& = & \frac{1}{28} \int \sin(u) \, du \\
						& = & \frac{1}{28} (-\cos(u)) + C \\
						& = & -\frac{1}{28} \cos(7x^4 + 3) + C.
\end{eqnarray*}
To check our work, we observe by the Chain Rule that
$$\frac{d}{dx} \left[ -\frac{1}{28}\cos(7x^4 + 3) + C \right] = -\frac{1}{28} \cdot (-1)\sin(7x^4 + 3) \cdot 28x^3 = \sin(7x^4 + 3) \cdot x^3,$$
which is indeed the original integrand.
\end{solution}

An essential observation about our work in Example~\ref{usub}  is that the $u$-substitution only worked because the function multiplying $\sin (7x^4 + 3)$ was $x^3$.  If instead that function was $x^2$ or $x^4$, the substitution process may not (and likely would not) have worked.  This is one of the primary challenges of antidifferentiation: slight changes in the integrand make tremendous differences.  For instance, we can use $u$-substitution with $u = x^2$ and $du = 2xdx$ to find that
\begin{eqnarray*}
\int xe^{x^2} \, dx & = & \int e^u \cdot \frac{1}{2} \, du \\
			& = & \frac{1}{2} \int e^u \, du \\
			& = & \frac{1}{2} e^u + C \\
			& = & \frac{1}{2} e^{x^2} + C.
\end{eqnarray*}
If, however, we consider the similar indefinite integral
$$\int e^{x^2} \, dx,$$
the missing $x$ to multiply $e^{x^2}$ makes the $u$-substitution $u = x^2$ no longer possible.  Hence, part of the lesson of $u$-substitution is just how specialized the process is: it only applies to situations where, up to a missing constant, the integrand that is present is the result of applying the Chain Rule to a different, related function.

\begin{example}{Substitution Rule}{SubstitutionRuleex}
Evaluate $\ds\int(ax+b)^n\,dx$, assuming $a,b$ are
constants, $a\not=0$, and $n$ is a positive integer.
\end{example}

\begin{solution}
We let $u=ax+b$ so $du=a\,dx$ or $dx=du/a$. Then
$$
  \int(ax+b)^n\,dx=\int {1\over a} u^n\,du={1\over a(n+1)}u^{n+1}+C=
  {1\over a(n+1)}(ax+b)^{n+1}+C.
$$
\end{solution}

\begin{example}{Substitution Rule}{SubstitutionRulex}
Evaluate $\ds\int \sin(ax+b)\,dx$, assuming that $a$ and $b$ are
constants and $a\not=0$.
\end{example}

\begin{solution}
Again we let $u=ax+b$ so $du=a\,dx$ or $dx=du/a$. Then
$$
  \int\sin(ax+b)\,dx=\int {1\over a} \sin u\,du={1\over a}(-\cos u)+C=
-{1\over a}\cos(ax+b)+C.
$$
\end{solution}

\begin{formulabox}[Strategy for Substitution Rule]
A general strategy to follow is:
\begin{enumerate}\setlength{\itemsep}{0 in}
\item Choose a possible $u=u(x)$. \dfont{Tip:} Choose a substitution $u$ so that its derivate also appears in the integral (up to a constant).
\item Calculate $du=u'(x)~dx$.
\item Either replace $u'(x)~dx$ by $du$, or replace $dx$ by $\ds{\frac{du}{u'(x)}}$, and cancel.
\item Write the rest of the integrand in terms of $u$. If this is not possible, the substitution will not work: You must go back to step 1.
\item Find the indefinite integral. (Again, if this is not possible, try a different substitution, or a different method).
\item Rewrite the result in terms of $x$.
\end{enumerate}
\end{formulabox}

\begin{example}{Substitution}{Substitution}
Evaluate the following integral: $\ds\int \frac{2x}{\sqrt{1-4x^2}}\,dx.$
\end{example}

\begin{solution}
We try the substitution:
$$u=1-4x^2.$$
Then,
$$du=-8x~dx$$
In the numerator we have $2x~dx$, so rewriting the differential gives:
$$-\frac{1}{4}du=2x~dx.$$
Then the integral is:
\begin{eqnarray*}
\int \frac{2x}{\sqrt{1-4x^2}}\,dx&=&\int \left(1-4x^2\right)^{-1/2}(2x~dx)\\
\\
&=&\int u^{-1/2}\left(-\frac{1}{4}du\right)\\
\\
&=&\left(\frac{-1}{4}\right)\frac{u^{1/2}}{1/2}+C\\
\\
&=&-\frac{\sqrt{1-4x^2}}{2}+C
\end{eqnarray*}
\end{solution}

\begin{example}{Substitution}{Substitution}
Evaluate the following integral: $\ds\int \sech^2(7t-3)\ dt$
\end{example}

\begin{solution}
 We employ substitution, with $u = 7t-3$ and $du = 7dt$. We have:
$$ \int \sech^2 (7t-3)\ dt=  \frac17 \int \sech^2 (u)\ du= \frac17\tanh (u) + C = \frac17\tanh (7t-3) + C.$$
\end{solution}

\begin{example}{Substitution}{Substitution2}
Evaluate the following integral: $\ds\int \cos x(\sin x)^5\,dx.$
\end{example}

\begin{solution}
In this question we will let $u=\sin x$.
Then,
$$du=\cos x~dx.$$
Thus, the integral becomes:
\begin{eqnarray*}
\int \cos x(\sin x)^5\,dx&=&\int u^5\,du\\
\\
&=&\frac{u^6}{6}+C\\
\\
&=&\frac{(\sin x)^6}{6}+C
\end{eqnarray*}
\end{solution}

\begin{example}{Substitution}{Substitution3}
Evaluate the following integral:
$\ds\int \frac{\cos(\sqrt x)}{\sqrt x}\,dx.$
%\vspace{-0.5cm}
\end{example}

\begin{solution}
We use the substitution:
$$u=x^{1/2}.$$
Then,
$$du=\frac{1}{2}x^{-1/2}dx.$$
Rewriting the differential we get:
$$2~du=\frac{1}{\sqrt x}~dx.$$
The integral becomes:
\begin{eqnarray*}
\int \frac{\cos(\sqrt x)}{\sqrt x}\,dx&=&2\int \cos u\,du\\
\\
&=&2\sin u+C\\
\\
&=&2\sin(\sqrt x)+C
\end{eqnarray*}
\end{solution}

%\begin{example}{Substitution with Inverse Trigonometric Functions}{Substitution with Inverse Trigonometric Functions}
%From items \ref{invsin} and \ref{invtan} in  Theorem \ref{thm:indef_alg} and the method of substitution with $ u=\frac{x}{a} $ where $ a>0 $ (try it!), we get the following:
%\[
%\int \frac{1}{x^2+a^2}=\frac1a\arctan\left(\frac{x}{a}\right)+C \text{ and } \int \frac{1}{\sqrt{a^2-x^2}}\; dx = \arcsin\left(\frac{x}{a}\right)+C
%\]

%\end{example}

\begin{example}{Integrating by substitution}{ex_sub10}
{
Evaluate $\ds \int \sin x\cos x\ dx$.
}
\end{example}

\begin{solution}
{There is not a composition of function here to exploit; rather, just a product of functions. Do not be afraid to experiment; when given an integral to evaluate, it is often beneficial to think ``If I let $u$ be \textit{this}, then $du$ must be \textit{that} \ldots'' and see if this helps simplify the integral at all.

In this example, let's set $u = \sin x$. Then $du = \cos x\ dx$, which we have as part of the integrand! The substitution becomes very straightforward:
		\begin{align*}
		\int \sin x\cos x\ dx &=	\int u\ du \\
											&= \frac12u^2+ C \\
											&= \frac12\sin^2 x + C.
		\end{align*}
One would do well to ask ``What would happen if we let $u = \cos x$?'' The result is just as easy to find, yet looks very different. The challenge to the reader is to evaluate the integral letting $u = \cos x$ and discover why the answer is the same, yet looks different.
}
\end{solution}

Our examples so far have required ``basic substitution.'' The next example demonstrates how substitutions can be made that often strike the new learner as being ``nonstandard.''\\

%\enlargethispage{\baselineskip}

\begin{example}{Substitution}{Substitution4}
Evaluate the following integral:
$\ds\int 2x^3\sqrt{x^2+1}\,dx.$
%\vspace{-0.5cm}
\end{example}

\begin{solution}
This problem is a little bit different than the previous ones.
It makes sense to let:
$$u=x^2+1,$$
then
$$du=2x~dx.$$
Making this substitution gives:
\begin{eqnarray*}
\int 2x^3\sqrt{x^2+1}\,dx&=&\int x^2\sqrt{x^2+1}(2x)\,dx\\
\\
&=&\int x^2u^{1/2}\,du\\
\end{eqnarray*}
This is a problem because our integrals can't have a mixture of two variables in them.
Usually this means we chose our $u$ incorrectly.
However, in this case we can eliminate the remaining $x$'s from our integral by using:
$$u=x^2+1\quad\to\quad x^2=u-1.$$
We get:
\begin{eqnarray*}
\int x^2u^{1/2}\,du&=&\int (u-1)u^{1/2}\,du\\
\\
&=&\int u^{3/2}-u^{1/2}\,du\\
\\
&=&\frac{2}{5}u^{5/2}-\frac{2}{3}u^{3/2}+C\\
\\
&=&\frac{2}{5}(x^2+1)^{5/2}-\frac{2}{3}(x^2+1)^{3/2}+C
\end{eqnarray*}
\end{solution}

\begin{example}{Integrating by substitution}{ex_sub4}
Evaluate $\ds\int x\sqrt{x+3}\ dx$.
\end{example}
\begin{solution}
{Recognizing the composition of functions, set $u = x+3$. Then $du = dx$, giving what seems initially to be a simple substitution. But at this stage, we have:
	$$\int x\sqrt{x+3}\ dx = \int x\sqrt{u}\ du.$$
We cannot evaluate an integral that has both an $x$ and an $u$ in it. We need to convert the $x$ to an expression involving just $u$.

Since we set $u = x+3$, we can also state that $u-3 = x$. Thus we can replace $x$ in the integrand with $u-3$. It will also be helpful to rewrite $\sqrt{u}$ as $u^\frac12$.
\begin{align*}
		\int x\sqrt{x+3} \ dx &= \int (u-3)u^\frac12\ du \\
											&= \int \big(u^\frac32 - 3u^\frac12\big) \ du \\
											&= \frac25u^\frac52 - 2u^\frac32 + C \\
											&= \frac25(x+3)^\frac52 - 2(x+3)^\frac32 + C.
\end{align*}
Checking your work is always a good idea. In this particular case, some algebra will be needed to make one's answer match the integrand in the original problem.
}
\end{solution}

\begin{example}{Integrating by substitution}{ex_sub5}
Evaluate $\ds \int \frac{1}{x\ln x}\ dx$.
\end{example}

\begin{solution}
{This is another example where there does not seem to be an obvious composition of functions. The line of thinking used in Example \ref{exa:ex_sub4} is useful here: choose something for $u$ and consider what this implies $du$ must be. If $u$ can be chosen such that $du$ also appears in the integrand, then we have chosen well.

Choosing $u = 1/x$ makes $du = -1/x^2\ dx$; that does not seem helpful. However, setting $u = \ln x$ makes $du = 1/x\ dx$, which is part of the integrand. Thus:
\begin{align*}
	\int \frac1{x\ln x}\ dx 	&=	\int \frac{1}{\underbrace{\ln x}_{1/u}}\underbrace{\frac1x\ dx}_{du} \\
												&= \int \frac1u\ du \\
												&= \ln |u| + C \\
												&= \ln | \ln x| + C.
\end{align*}
The final answer is interesting; the natural log of the natural log. Take the derivative to confirm this answer is indeed correct.
}
\end{solution}

\subsection{Integrals Involving Trigonometric Functions}

Section \ref{sec:Powers of trigonometric functions} delves deeper into integrals of a variety of trigonometric functions; here we use substitution to establish a foundation that we will build upon.

The next three examples will help fill in some missing pieces of our antiderivative knowledge. We know the antiderivatives of the sine and cosine functions; what about the other standard functions tangent, cotangent, secant and cosecant? We discover these next.\\

\begin{example}{Integration by substitution: antiderivatives of $\tan x$}{ex_sub6}
Evaluate $\ds \int \tan x\ dx.$
\end{example}

\begin{solution}
{The previous paragraph established that we did not know the antiderivatives of tangent, hence we must assume that we have learned something in this section that  can help us evaluate this indefinite integral.

%\enlargethispage{\baselineskip}
Rewrite $\tan x$ as $\sin x/\cos x$. While the presence of a composition of functions may not be immediately obvious, recognize that $\cos x$ is ``inside'' the $1/x$ function. Therefore, we see if setting $u = \cos x$ returns usable results. We have that $du = -\sin x\ dx$, hence $-du = \sin x\ dx$. We can integrate:
\begin{align*}
		\int \tan x \ dx &= \int \frac{\sin x}{\cos x}\ dx \\
							&= \int \frac1{\underbrace{\cos x}_u}\underbrace{\sin x\ dx}_{-du} \\
							&= \int \frac {-1}u \ du\\
							&= -\ln |u| + C \\
							&= -\ln |\cos x| + C.
\end{align*}
Some texts prefer to bring the $-1$ inside the logarithm as a power of $\cos x$, as in:
\begin{align*}
-\ln |\cos x| + C &= \ln |(\cos x)^{-1}| + C\\
			&= \ln \left| \frac{1}{\cos x}\right| + C\\
			&= \ln |\sec x| + C.
\end{align*}
Thus the result they give is $\int \tan x \ dx = \ln|\sec x| + C$. These two answers are equivalent.
}\\

\end{solution}

\begin{example}{Integrating by substitution: antiderivatives of $\sec x$}{ex_sub7}
Evaluate $\ds\int \sec x\ dx$.
\end{example}

%\enlargethispage{2\baselineskip}

\begin{solution}
{This example employs a wonderful trick: multiply the integrand by ``1'' so that we see how to integrate more clearly. In this case, we write ``1'' as
$$1 = \frac{\sec x + \tan x}{\sec x + \tan x}.$$
This may seem like it came out of left field, but it works beautifully. Consider:
\begin{align*}
			\int \sec x\ dx	&=	\int \sec x\cdot \frac{\sec x + \tan x}{\sec x + \tan x}\ dx \\
							&= \int \frac{\sec^2 x + \sec x\tan x}{\sec x + \tan x}\ dx.\\
\intertext{Now let $u = \sec x+\tan x$; this means $du = (\sec x\tan x+ \sec^2 x)\ dx$, which is our numerator. Thus:}
						&= \int \frac{du}{u} \\
						&= \ln |u| + C \\
						&= \ln |\sec x+\tan x| + C.
\end{align*}
\vskip -\baselineskip
}
\end{solution}

%\clearpage

We can use similar techniques to those used in Examples \ref{ex_sub6} and \ref{ex_sub7} to find antiderivatives of $\cot x$ and $\csc x$ (which the reader can explore in the exercises.) We summarize our results here.

\begin{theorem}{Antiderivatives of Trigonometric Functions}{triganti}
{\begin{minipage}{.45\textwidth}\small\index{integration!of trig. functions}
	\begin{enumerate}
	\item		$\ds \int \sin x \ dx = -\cos x +C$
	\item		$\ds\int \cos x\ dx = \sin x + C$
	\item		$\ds \int \tan x\ dx = -\ln|\cos x|+C$
\end{enumerate}
\end{minipage}
\begin{minipage}{.55\textwidth}\small
	\begin{enumerate}\addtocounter{enumi}{3}
	\item		$\ds \int \csc x \ dx = -\ln|\csc x+\cot x| +C$
	\item		$\ds\int \sec x\ dx = \ln|\sec x+\tan x| + C$
	\item		$\ds \int \cot x\ dx = \ln|\sin x|+C$
\end{enumerate}
\end{minipage}
}
\end{theorem}

We explore one more common trigonometric integral.\\

\begin{example}{Integration by substitution: powers of $\cos x$ and $\sin x$}{ex_sub8}
{
Evaluate $\ds \int \cos^2x\ dx$.}
\end{example}

\begin{solution}
{We have a composition of functions as $\cos^2x = \big(\cos x\big)^2$.
%with $\cos x$ inside the $x^2$ function.
However, setting $u = \cos x$ means $du = -\sin x\ dx$, which we do not have in the integral. Another technique is needed.

The process we'll employ is to use a Power Reducing formula for $\cos^2x$ (perhaps consult the back of this text for this formula), which states
	$$\cos ^2x = \frac{1+\cos(2x)}{2}.$$
	The right hand side of this equation is not difficult to integrate. We have:
\begin{align*}
	\int \cos^2x\ dx &= \int \frac{1+\cos(2x)}2\ dx \\
									&=	\int \left( \frac12 + \frac12\cos(2x)\right)\ dx. \\
\intertext{Now use Key Idea \ref{idea:linearsub}:}
									&= \frac12x + \frac12\frac{\sin(2x)}{2} + C \rule[-10pt]{0pt}{5pt}\\
									&= \frac12x + \frac{\sin(2x)}4 + C.
\end{align*}
We'll make significant use of this power--reducing technique in future sections.
}\\
\end{solution}

\subsection{ Simplifying the Integrand}

It is common to be reluctant to manipulate the integrand of an integral; at first, our grasp of integration is tenuous and one may think that working with the integrand will improperly change the results. Integration by substitution works using a different logic: as long as \textit{equality} is maintained, the integrand can be manipulated so that its \textit{form} is easier to deal with. The next two examples demonstrate common ways in which using algebra first makes the integration easier to perform.\\

%\enlargethispage{2\baselineskip}

\begin{example}{Integration by substitution: simplifying first}{ex_sub9}
{
Evaluate $\ds\int \frac{x^3+4x^2+8x+5}{x^2+2x+1}\ dx$.}
\end{example}

\begin{solution}
{One may try to start by setting $u$ equal to either the numerator or denominator; in each instance, the result is not workable.

When dealing with rational functions (i.e., quotients made up of polynomial functions), it is an almost universal rule that everything works better when the degree of the numerator is less than the degree of the denominator. Hence we use polynomial division.

We skip the specifics of the steps, but note that when $x^2+2x+1$ is divided into $x^3+4x^2+8x+5$, it goes in $x+2$ times with a remainder of $3x+3$. Thus
	$$\frac{x^3+4x^2+8x+5}{x^2+2x+1} = x+2 + \frac{3x+3}{x^2+2x+1}.$$
Integrating $x+2$ is simple. The fraction can be integrated by setting $u = x^2+2x+1$, giving $du = (2x+2)\ dx$. This is very similar to the numerator. Note that $du/2 = (x+1)\ dx$ and then consider the following:
\begin{align*}
\int \frac{x^3+4x^2+8x+5}{x^2+2x+1}\ dx & = \int \left(x+2 + \frac{3x+3}{x^2+2x+1}\right)\ dx  \rule[-13pt]{0pt}{5pt} \\
					&= \int (x+2)\ dx + \int \frac{3(x+1)}{x^2+2x+1}\ dx  \rule[-13pt]{0pt}{5pt}\\
					& = \frac12x^2+2x+C_1 + \int \frac{3}{u}\frac{du}{2}  \rule[-13pt]{0pt}{5pt}\\
					&= \frac12x^2+2x+C_1 + \frac32\ln|u| + C_2 \rule[-13pt]{0pt}{5pt}\\
					&= \frac12x^2+2x+\frac32\ln|x^2+2x+1| + C.
\end{align*}
In some ways, we ``lucked out'' in that after dividing, substitution was able to be done. In later sections we'll develop techniques for handling rational functions where substitution is not directly feasible.
}\\
\end{solution}

\begin{example}{Integration by alternate methods}{ex_sub11}
{
Evaluate $\ds\int \frac{x^2+2x+3}{\sqrt{x}}\ dx$ with, and without, substitution.}
\end{example}

\begin{solution}
{We already know how to integrate this particular example. Rewrite $\sqrt{x}$ as $x^\frac12$ and simplify the fraction:
	$$ \frac{x^2+2x+3}{x^{1/2}} = x^\frac32 + 2x^\frac12 + 3x^{-\frac12}.$$
We can now integrate using the Power Rule:
\begin{align*}
	\int \frac{x^2+2x+3}{x^{1/2}}\ dx &= \int\left(x^\frac32 + 2x^\frac12 + 3x^{-\frac12}\right)\ dx\\
						&=	\frac25x^\frac52 + \frac43x^\frac32 + 6x^\frac12 + C
\end{align*}
This is a perfectly fine approach. We demonstrate how this can also be solved using substitution as its implementation is rather clever.

Let $u = \sqrt{x} = x^\frac12$; therefore
		$$du = \frac12x^{-\frac12}dx = \frac{1}{2\sqrt{x}}\ dx \quad \Rightarrow \quad 2du = \frac{1}{\sqrt{x}}\ dx.$$

This gives us $\ds \int \frac{x^2+2x+3}{\sqrt{x}}\ dx = \int (x^2+2x+3)\cdot2\ du$. What are we to do with the other $x$ terms? Since $u = x^\frac12$, $u^2 = x$, etc. We can then replace $x^2$ and $x$ with appropriate powers of $u$. We thus have
\begin{align*}
\int \frac{x^2+2x+3}{\sqrt{x}}\ dx &= \int (x^2+2x+3)\cdot2\ du\\
											&= \int 2(u^4 + 2u^2 + 3)\ du \\
											&= \frac25u^5 + \frac43u^3 + 6u + C \\
											&= \frac25x^\frac52 + \frac43x^\frac32 + 6x^\frac12+C,
\end{align*}
which is obviously the same answer we obtained before. In this situation, substitution is arguably more work than our other method. The fantastic thing is that it works. It demonstrates how flexible integration is.
}
\end{solution}

\subsection{Substitution and Inverse Trigonometric Functions}

%In Section \ref{sec:deriv_inverse_function}
When studying derivatives of inverse functions, we learned that $$\frac{d}{dx}\big(\tan^{-1}x\big) = \frac{1}{1+x^2}.$$ Applying the Chain Rule to this is not difficult; for instance, $$\frac{d}{dx}\big(\tan^{-1}5x\big) = \frac{5}{1+25x^2}.$$ We now explore how Substitution can be used to ``undo'' certain derivatives that are the result of the Chain Rule applied to Inverse Trigonometric functions. We begin with an example.\\

\begin{example}{Integrating by substitution: inverse trigonometric functions}{ex_subst14}
{
Evaluate $\ds \int \frac{1}{25+x^2}\ dx$.}
\end{example}

\begin{solution}
{The integrand looks similar to the derivative of the arctangent function. Note:
\begin{align*}
\frac{1}{25+x^2} &= \frac{1}{25(1+\frac{x^2}{25})}\\
							&= \frac{1}{25(1+\left(\frac{x}{5}\right)^2)} \\
							&= \frac{1}{25}\frac{1}{1+\left(\frac{x}{5}\right)^2}\ .
\end{align*}
Thus $$\int\frac{1}{25+x^2}\ dx = \frac{1}{25}\int \frac{1}{1+\left(\frac{x}{5}\right)^2}\ dx.$$ This can be integrated using Substitution. Set $u = x/5$, hence $du = dx/5$ or $dx=5du$. Thus
\begin{align*}
\int\frac{1}{25+x^2}\ dx &= \frac{1}{25}\int \frac{1}{1+\left(\frac{x}{5}\right)^2}\ dx \\
										&= \frac15\int \frac{1}{1+u^2}\ du \\
										&= \frac15\tan^{-1}u + C \\
										&= \frac15\tan^{-1}\left(\frac x5\right)+C
\end{align*}
\vskip -\baselineskip
}
\end{solution}

Example \ref{exa:ex_subst14} demonstrates a general technique that can be applied to other integrands that result in inverse trigonometric functions. The results are summarized here.

\begin{theorem}{Integrals Involving Inverse Trigonomentric Functions}{int_inverse_trig}
{Let $a>0$.
%\noindent\begin{minipage}[t]{.5\linewidth}
\begin{enumerate}
\item		$\ds \int \frac{1}{a^2+x^2}\ dx = \frac1a\tan^{-1}\left(\frac{x}{a}\right) + C$
%\addtocounter{enumi}{1}
\item		$\ds \int \frac{1}{\sqrt{a^2-x^2}}\ dx = \sin^{-1}\left(\frac{x}{a}\right)+C$
%\end{enumerate}
%\end{minipage}
%\begin{minipage}[t]{.5\linewidth}
%\begin{enumerate}\addtocounter{enumi}{1}
\item		$\ds \int \frac{1}{x\sqrt{x^2-a^2}}\ dx = \frac1a\sec^{-1}\left(\frac{|x|}{a}\right)+C$
\end{enumerate}
%\end{minipage}
}
\end{theorem}

Let's practice using Theorem \ref{thm:int_inverse_trig}.\\

\begin{example}{Integrating by substitution: inverse trigonometric functions}{ex_subst15}
{Evaluate the given indefinite integrals.
$$\int \frac{1}{9+x^2}\ dx,\quad \int \frac{1}{x\sqrt{x^2-\frac{1}{100}}}\ dx\quad \text{ and }\quad  \int \frac{1}{\sqrt{5-x^2}}\ dx.$$
}
\end{example}

\begin{solution}
{Each can be answered using a straightforward application of Theorem \ref{thm:int_inverse_trig}.\\

$\ds \int \frac{1}{9+x^2}\ dx = \frac13\tan^{-1} \frac x3 + C$, as $a = 3$.\vskip 10pt

$\ds \int  \frac{1}{x\sqrt{x^2-\frac{1}{100}}}\ dx = 10\sec^{-1}10x + C$, as $a = \frac1{10}$.\vskip 10pt

$\ds \int \frac{1}{\sqrt{5-x^2}} = \sin^{-1}\frac{x}{\sqrt{5}}+C$, as $a = \sqrt{5}$.\\
}
\end{solution}

\enlargethispage{2\baselineskip}
Most applications of Theorem \ref{thm:int_inverse_trig} are not as straightforward. The next examples show some common integrals that can still be approached with this theorem.\\

\begin{example}{Integrating by substitution: completing the square}{ex_subst16}
{Evaluate $\ds \int\frac{1}{x^2-4x+13}\ dx$.}
\end{example}

\begin{solution}
{Initially, this integral seems to have nothing in common with the integrals in Theorem \ref{thm:int_inverse_trig}. As it lacks a square root, it almost certainly is not related to arcsine or arcsecant. It is, however, related to the arctangent function.

We see this by \textit{completing the square} in the denominator. We give a brief reminder of the process here.

Start with a quadratic with a leading coefficient of 1. It will have the form of $x^2 + bx + c$. Take 1/2 of $b$, square it, and add/subtract it back into the expression. I.e.,
\begin{align*}
x^2+bx+ c &= \underbrace{x^2 + bx + \frac{b^2}4}_{(x+b/2)^2} - \frac{b^2}4 + c\\
   &= \left(x+\frac b2\right)^2 + c-\frac{b^2}4
\end{align*}
In our example, we take half of $-4$ and square it, getting $4$. We add/subtract it into the denominator as follows:

\begin{align*}
\frac{1}{x^2-4x+13} &= \frac{1}{\underbrace{x^2-4x+4}_{(x-2)^2}-4+13}\\
				&=\frac{1}{(x-2)^2 + 9}
\end{align*}
We can now integrate this using the arctangent rule. Technically, we need to substitute first with $u=x-2$, but we omit this step here. Thus we have
$$ \int \frac{1}{x^2-4x+13}\ dx = \int \frac{1}{(x-2)^2+9}\ dx = \frac13\tan^{-1}\frac{x-2}{3}+C.$$
\vskip -\baselineskip
}
\end{solution}

\begin{example}{Integrals requiring multiple methods}{ex_subst17}
{
Evaluate $\ds \int \frac{4-x}{\sqrt{16-x^2}}\ dx$.}
\end{example}

\begin{solution}
{This integral requires two different methods to evaluate it. We get to those methods by splitting up the integral:
$$ \int \frac{4-x}{\sqrt{16-x^2}}\ dx = \int \frac{4}{\sqrt{16-x^2}}\ dx - \int \frac{x}{\sqrt{16-x^2}}\ dx.$$
The first integral is handled using a straightforward application of Theorem \ref{thm:int_inverse_trig}; the second integral is handled by substitution, with $u = 16-x^2$. We handle each separately.

$\ds \int \frac{4}{\sqrt{16-x^2}}\ dx = 4\sin^{-1}\frac{x}{4} + C$.
\vskip 10pt

$\ds \int\frac{x}{\sqrt{16-x^2}}\ dx$: Set $u = 16-x^2$, so $du = -2xdx$ and $xdx = -du/2$. We have
\begin{align*}
\int\frac{x}{\sqrt{16-x^2}}\ dx &= \int\frac{-du/2}{\sqrt{u}}\\
				&= -\frac12\int \frac{1}{\sqrt{u}}\ du \\
				&= - \sqrt{u} + C\\
				&= -\sqrt{16-x^2} + C.
\end{align*}
Combining these together, we have
$$ \int \frac{4-x}{\sqrt{16-x^2}}\ dx = 4\sin^{-1}\frac x4 + \sqrt{16-x^2}+C.$$
}
\end{solution}

% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %

This section has focused on evaluating indefinite integrals as we are learning a new technique for finding antiderivatives. However, much of the time integration is used in the context of a definite integral. Definite integrals that require substitution can be calculated using the following workflow:

\begin{enumerate}
\item		Start with a definite integral $\ds \int_a^b f(x)\ dx$ that requires substitution.
\item		Ignore the bounds; use substitution to evaluate $\ds \int f(x)\ dx$ and find an antiderivative $F(x)$.
\item		Evaluate $F(x)$ at the bounds; that is, evaluate $F(x)\Big|_a^b = F(b) - F(a)$.
\end{enumerate}
This workflow works fine, but substitution offers an alternative that is powerful and amazing (and a little time saving).  The next example shows how to use the Substitution Rule when dealing with definite integrals.

\begin{example}{Substitution Rule}{SubstitutionRuledefex}
Evaluate $\ds\int_2^4 x\sin(x^2)\,dx$.
\end{example}

\begin{solution}
First we compute the
antiderivative, then evaluate the integral.
Let $\ds u=x^2$, so $du=2x\,dx$ or $x\,dx=du/2$. Then
$$
  \int x\sin(x^2)\,dx=\int {1\over 2} \sin u\,du={1\over 2}(-\cos u)+C=
  -{1\over 2}\cos(x^2)+C.
$$
Now
$$
  \int_2^4 x\sin(x^2)\,dx=\left.-{1\over 2}\cos(x^2)\right|_2^4
  =-{1\over 2}\cos(16)+{1\over 2}\cos(4).
$$
A somewhat neater alternative to this method is to change the original
limits to match the variable $u$. Since $\ds u=x^2$, when $x=2$, $u=4$,
and when $x=4$, $u=16$. So we can do this:
$$
  \int_2^4 x\sin(x^2)\,dx=
  \int_4^{16} {1\over 2} \sin u\,du=\left.-{1\over 2}(\cos u)\right|_4^{16}
  =-{1\over 2}\cos(16)+{1\over 2}\cos(4).
$$
An incorrect, and dangerous, alternative is something like this:
$$
  \int_2^4 x\sin(x^2)\,dx=\int_2^4 {1\over 2} \sin u\,du=
  \left.-{1\over 2}\cos (u)\right|_2^4=
  \left.-{1\over 2}\cos(x^2)\right|_2^4=-{1\over 2}\cos(16)+{1\over
  2}\cos(4).
$$
This is incorrect because $\ds\int_2^4 {1\over 2} \sin u\,du$
means that $u$ takes on values between 2 and 4, which is wrong. It
is dangerous, because it is very easy to get to
the point $\ds\left.-{1\over 2}\cos (u)\right|_2^4$ and forget to substitute
$\ds x^2$ back in for $u$, thus getting the incorrect answer
$\ds -{1\over 2}\cos(4)+{1\over 2}\cos(2)$. An acceptable alternative is something like:
$$
  \int_2^4 x\sin(x^2)\,dx=\int_{x=2}^{x=4} {1\over 2} \sin u\,du=
  \left.-{1\over 2}\cos (u)\right|_{x=2}^{x=4}=
  \left.-{1\over 2}\cos(x^2)\right|_2^4=-{\cos(16)\over 2}+{\cos(4)\over2}.
$$
\end{solution}

\subsection{Substitution and Definite Integrals}

The following theorem states how the bounds of a definite integral can be changed as the substitution is performed.

\begin{theorem}{Substitution Rule for Definite Integrals}{SubstitutionRuledef}
If $g'$ is continuous on $[a,b]$ and $f$ is continuous on the range of $u=g(x)$, then
$$\int_a^b f(g(x))g'(x)\,dx=\int_{g(a)}^{g(b)}f(u)\,du.$$
\end{theorem}

In effect, Theorem \ref{thm:SubstitutionRuledef} states that once you convert to integrating with respect to $u$, you do not need to switch back to evaluating with respect to $x$. A few examples will help one understand.\\

\begin{example}{Definite integrals and substitution: changing the bounds}{ex_sub12}
{
Evaluate $\ds\int_0^2 \cos(3x-1)\ dx$ using Theorem \ref{thm:SubstitutionRuledef}.}
\end{example}

\begin{solution}
{Observing the composition of functions, let $u=3x-1$, hence $du = 3dx$. As $3dx$ does not appear in the integrand, divide the latter equation by 3 to get $du/3 = dx$.

By setting $u = 3x-1$, we are implicitly stating that $g(x) = 3x-1$. Theorem \ref{thm:SubstitutionRuledef} states that the new lower bound is $g(0) = -1$; the new upper bound is $g(2) = 5$. We now evaluate the definite integral:
\begin{align*}
\int_1^2 \cos(3x-1) \ dx &=	\int_{-1}^5 \cos u \frac{du}{3} \\
								&= \frac{1}{3} \sin u\Big|_{-1}^5 \\
								&= \frac{1}{3}\big(\sin 5- \sin (-1)\big)\approx -0.039.
								%&\approx -0.039.
\end{align*}
Notice how once we converted the integral to be in terms of $u$, we never went back to using $x$.

\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=.8\textwidth]{figures/figsubst12a}
  \caption{}
  \label{fig:sub1}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=.8\textwidth]{figures/figsubst12b}
  \caption{}
  \label{fig:sub2}
\end{subfigure}
\caption{Graphing the areas defined by the definite integrals of Example \ref{exa:ex_sub12}. \label{fig:subst12}}
\label{fig:test}
\end{figure}

%\begin{figure}
%\centering
%
%\includegraphics[width=0.7\linewidth]{figures/figsubst12a}
%
%\caption{Graphing the areas defined by the definite integrals in Example \ref{exa:ex_sub12}}
%\label{fig:figsubst12a}
%\end{figure}
%
%
%\mtable{.7}{Graphing the areas defined by the definite integrals of Example \ref{exa:ex_sub12}.}{fig:subst12}{\begin{tabular}{ccc}
%\includegraphics{figures/figsubst12a} & &\includegraphics{figures/figsubst12b}\\
%(a) & & (b)
%\end{tabular}

The graphs in Figure \ref{fig:subst12} tell more of the story. In (a) the area defined by the original integrand is shaded, whereas in (b) the area defined by the new integrand is shaded. In this particular situation, the areas look very similar; the new region is ``shorter'' but ``wider,'' giving the same area.
}\\
\end{solution}

\begin{example}{Definite integrals and substitution: changing the bounds}{ex_subst13}
{
Evaluate $\ds \int_0^{\pi/2} \sin x \cos x\ dx$ using Theorem \ref{thm:SubstitutionRuledef}.
}
\end{example}

\begin{solution}
{We saw the corresponding indefinite integral in Example \ref{exa:ex_sub10}. In that example we set $u = \sin x$ but stated that we could have let $u = \cos x$. For variety, we do the latter here.

Let $u = g(x) = \cos x$, giving $du = -\sin x\ dx$ and hence $\sin x\ dx = -du$. The new upper bound is $g(\pi/2) = 0$; the new lower bound is $g(0) = 1$. Note how the lower bound is actually larger than the upper bound now. We have
\begin{align*}
	\int_0^{\pi/2} \sin x\cos x\ dx &= \int_1^0 -u\ du \quad \text{\scriptsize (switch bounds \& change sign)}\\%&= \int_1^0u\ (-1)du\\
											%&= \int_1^0 -u\ du \quad \text{\scriptsize (switch bounds \& change sign)}\\
											&=	\int_0^1 u\ du\\
											&= \frac12u^2\Big|_0^1= 1/2.%\\
											%&= 1/2.
\end{align*}
In Figure \ref{fig:subst13} we have again graphed the two regions defined by our definite integrals. Unlike the previous example, they bear no resemblance to each other. However, Theorem \ref{thm:SubstitutionRuledef} guarantees that they have the same area.

\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=.8\linewidth]{figures/figsubst13a}
  \caption{}
  \label{fig:sub1}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=.8\linewidth]{figures/figsubst13b}
  \caption{}
  \label{fig:sub2}
\end{subfigure}
\caption{Graphing the areas defined by the definite integrals of Example \ref{exa:ex_subst13}. \label{fig:subst13}}
\label{fig:test}
\end{figure}

%%\ifthenelse{\boolean{longpage}}
%%{% if longpage
%\mtable{.3}{Graphing the areas defined by the definite integrals of Example \ref{ex_subst13}.}{fig:subst13}{\begin{tabular}{ccc}
%\includegraphics{figures/figsubst13a} & &\includegraphics{figures/figsubst13b}\\
%(a) & & (b)
%\end{tabular}
%
%} %ends \mtable
%}% ends if longpage
%{% not longpage
%\mtable{.32}{Graphing the areas defined by the definite integrals of Example \ref{ex_subst13}.}{fig:subst13}{\begin{tabular}{c}
%\myincludegraphics{figures/figsubst13a} \\ (a) \\ \myincludegraphics{figures/figsubst13b}\\
%(b)
%\end{tabular}
%}% ends figure
%}% ends if not longpage
%\vskip-\baselineskip
}
\end{solution}

\begin{example}{Substitution Rule}{SubstitutionRuledef2}
Evaluate $\ds\int_{1/4}^{1/2}{\cos(\pi t)\over\sin^2(\pi t)}\,dt$.
\end{example}

\begin{solution}
Let $u=\sin(\pi t)$ so $du=\pi\cos(\pi t)\,dt$ or $du/\pi=\cos(\pi
t)\,dt$.
We change the limits to $\ds \sin(\pi/4)=\sqrt2/2$ and
$\sin(\pi/2)=1$.
Then
$$
  \int_{1/4}^{1/2}{\cos(\pi t)\over\sin^2(\pi t)}\,dt=
  \int_{\sqrt2/2}^{1}{1\over \pi}{1\over u^2}\,du=
  \int_{\sqrt2/2}^{1} {1\over \pi}u^{-2}\,du=
  \left.{1\over \pi}{u^{-1}\over -1}\right|_{\sqrt2/2}^{1}=
  -{1\over\pi}+{\sqrt2\over\pi}.
$$
\end{solution}

The following theorem sometimes allows us to greatly simplify the calculations of integrals, by exploiting their symmetry.

\begin{theorem}{Integrals of Symmetric Functions}{symm_integrals}
Suppose $f$ is continuous on $[-a, a]$.
\begin{enumerate}
\item  If $f$ is  even  (that is,  {$f(-x) = f(x)$}), then $\displaystyle  { \int\limits_{-a}^a f(x) \ dx} = 2  {\int\limits_0^a f(x) \ dx}$.
\item  If $f$ is  {odd} (that is,  {$f(-x) = -f(x)$}), then $\displaystyle   {\int\limits_{-a}^a f(x) \  dx }=  {\int\limits_{0}^a f(x) \ dx}+  {\int\limits_{-a}^0 f(x) \ dx}= 0$.
\end{enumerate}
\end{theorem}

\begin{proof}
Since the definite integral is additive with respect to the interval of integration, we have
\[
\int_{-a}^{a} f(x) \; dx = \int_{-a}^{0} f(x) \; dx + \int_{0}^{a} f(x) \; dx =\star
\]
Substitute $ u=-x $ in the first integral to get
\[
\star = \int_{ a}^{0} f(-u) \; -du + \int_{0}^{a} f(x) \; dx = \int_{ 0}^{a} f(-u) \; du + \int_{0}^{a} f(x) \; dx
\]
Since $ u $ is simply a "dummy variable" we can replace it with $ x $ to get
\[
\star = \int_{ 0}^{a} f(-x) \; dx + \int_{0}^{a} f(x) \; dx = \int_{0}^{a} f(x) + f(-x) \; dx
\]
If $ f $ is even, then $ f(-x)=f(x) $, and   $ f $ is odd then $ f(x)+f(-x)=0 $ giving the equations in the theorem.
\end{proof}

\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \begin{tikzpicture}[/pgf/declare function={f= x^2+1;}]
    \begin{axis}[
      axis lines=middle,
       xtick       = {-2,0,2},
      xticklabels = {$-a$,$0$,$a$},
       ytick       = {},
       yticklabels = {},
      samples     = 160,
      domain      = -2:2,
      xmin = -2, xmax = 2,
      ymin = 0, ymax = 5,
    ]
    \addplot[name path=poly, black, thick, mark=none, ] {f};
     \addplot [draw=none,name path=B] {0};     % “fictional” curve
  %  \addplot[name path=line, gray, no markers, line width=1pt] {3};
   \addplot [green!60] fill between[of = poly and B,soft clip={domain=-2:2}]; % filling
    %\addplot [red!60] fill between[of = poly and B,soft clip={domain=2:4.2}]; % filling
    ];
    %% Choosing the coordinates manually is annoying:
    \node at (axis cs:-1,1) {$A_1$};
    \node at (axis cs:1,1) {$A_2$};
  \end{axis}
  \end{tikzpicture}
  \caption{Even symmetry, $ A_1=A_2$}
  \label{fig:symm1}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
\begin{tikzpicture}[/pgf/declare function={f=x^3;}]
  \begin{axis}[
      axis lines=middle,
      xtick       = {-2,0,2},
          xticklabels = {$-a$,$0$,$a$},
           ytick       = {},
          yticklabels = {},
      samples     = 160,
      domain      = -2:2,
      xmin = -2, xmax = 2,
      ymin = -8, ymax = 8,
    ]
  \addplot[name path=poly, black, thick, mark=none, ] {f};
   \addplot [draw=none,name path=B] {0};     % “fictional” curve
%  \addplot[name path=line, gray, no markers, line width=1pt] {3};
 \addplot [green!60] fill between[of = poly and B,soft clip={domain=0:2}]; % filling
  \addplot [red!60] fill between[of = poly and B,soft clip={domain=-2:0}]; % filling
  ];
  %% Choosing the coordinates manually is annoying:
  \node at (axis cs:-1.5,-1) {$A_1$};
  \node at (axis cs:1.5,1) {$A_2$};
\end{axis}
\end{tikzpicture}
\caption{Odd symmetry, $ A_1=-A_2$.}
  \label{fig:symm2}
\end{subfigure}
\label{fig:symm_integral}
\end{figure}

\begin{example}{Odd Symmetry and Integrals}{symm-integral1}
Use properties of integrals to evaluate $\displaystyle \int_{-3}^{3}1-x^4 {d}{x}$
\end{example}

\begin{solution}
Since $ 1-x^4 $ is an odd function, that is, if $ f(x)=1-x^4 $, then $ f(-x) = 1-(-x)^4=1-x^4=f(x)$. So by symmetry (Theorem \ref{thm:symm_integrals}) we have
\[
\int_{-3}^{3}1-x^4 {d}{x}=2\int_{0}^{3}1-x^4 {d}{x}= 2\left(x-\frac{x^5}{5}\right|+0^3=2\left(3-\frac{243}{5} \right)
= -\frac{456}{5}\]
\end{solution}

\begin{example}{Even Symmetry and Integrals}{symm-integral2}
Use properties of integrals to evaluate $\displaystyle{{{\int_{{-{5}}}^{{5}}}{\left(\frac{\sin(x)}{x^2+1} \right)}{d}{x}}}
$
\end{example}

\begin{solution}
Since $ sin(x) $ is an odd function, and $ x^2+1 $ is an even function, the quotient of the two is odd.  That is, if $ f(x)= \frac{\sin(x)}{x^2+1}   $, then $ f(-x) =  \frac{\sin(-x)}{(-x)^2+1}  = \frac{-\sin(x)}{x^2+1} = -f(x) $. So by symmetry (Theorem \ref{thm:symm_integrals}) we have
\[
\int_{-5}^{5}{\left(\frac{\sin(x)}{x^2+1} \right)}{d}{x}=0
\]
\end{solution}

Integration by substitution is a powerful and useful integration technique. Section \ref{sec:Parts} introduces another technique, called Integration by Parts. As substitution ``undoes'' the Chain Rule, integration by parts ``undoes'' the Product Rule. Together, these two techniques provide a strong foundation on which most other integration techniques are based.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for Section \ref{sec:SubRule}}

\begin{enumialphparenastyle}

Find the following indefinite and definite integrals.

%%%%%%%%%%
\begin{ex}
 $\ds\int (1-t)^9\,dt$
\begin{sol}
 $\ds -(1-t)^{10}/10+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int (x^2+1)^2\,dx$
\begin{sol}
 $\ds x^5/5+2x^3/3+x+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int x(x^2+1)^{100}\,dx$
\begin{sol}
 $\ds (x^2+1)^{101}/202+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int {1\over\root 3 \of {1-5t}}\,dt$
\begin{sol}
 $\ds -3(1-5t)^{2/3}/10+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin^3x\cos x\,dx$
\begin{sol}
 $\ds (\sin^4x)/4+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int x\sqrt{100-x^2}\,dx$
\begin{sol}
 $\ds -(100-x^2)^{3/2}/3+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int {x^2\over\sqrt{1-x^3}}\,dx$
\begin{sol}
 $\ds \ds -2\sqrt{1-x^3}/3+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int \cos(\pi t)\cos\bigl(\sin(\pi t)\bigr)\,dt$
\begin{sol}
 $\ds \sin(\sin\pi t)/\pi+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int {\sin x\over\cos^3 x}\,dx$
\begin{sol}
 $\ds \ds 1/(2\cos^2 x)=(1/2)\sec^2x+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int\tan x\,dx$
\begin{sol}
 $-\ln|\cos x|+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
  $\ds\int_0^\pi\sin^5(3x)\cos(3x)\,dx$
\begin{sol}
 $0$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int\sec^2x\tan x\,dx$
\begin{sol}
 $\ds \tan^2(x)/2+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int_0^{\sqrt{\pi}/2} x\sec^2(x^2)\tan(x^2)\,dx$
\begin{sol}
 $1/4$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int {\sin(\tan x)\over\cos^2x}\,dx$
\begin{sol}
 $-\cos(\tan x)+C$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int_3^4 {1\over(3x-7)^2}\,dx$
\begin{sol}
 $1/10$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int_0^{\pi/6}(\cos^2x - \sin^2x)\,dx$
\begin{sol}
 $\ds \sqrt3/4$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int {6x\over(x^2 - 7)^{1/9}}\,dx$
\begin{sol}
 $\ds (27/8)(x^2-7)^{8/9}$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int_{-1}^1 (2x^3-1)(x^4-2x)^6\,dx$
\begin{sol}
 $\ds -(3^7+1)/14$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int_{-1}^1 \sin^7 x\,dx$
\begin{sol}
 $0$
\end{sol}
\end{ex}

%%%%%%%%%%
\begin{ex}
 $\ds\int f(x) f'(x)\,dx$
\begin{sol}
 $\ds f(x)^2/2$
\end{sol}
\end{ex}

\end{enumialphparenastyle}