\documentclass[12pt, letterpaper, preprint]{aastex}

\usepackage{graphicx}
\usepackage{amsmath}

\newcommand{\tmin}[1][]{t_{\mathrm{min} #1}}
\newcommand{\tmax}[1][]{t_{\mathrm{max} #1}}
\newcommand{\amin}{a_{\mathrm{min}}}
\newcommand{\amax} {a_{\mathrm{max}}}

\newcommand{\dlt}{\Delta\log t_i}
\newcommand{\dt}{\Delta t_i}

\newcommand{\tintegral}{\int_{\tmin[,i]}^{\tmax[,i]} dt}
\newcommand{\tinterval}{\right|_{\tmin[,i]}^{\tmax[,i]}}
\newcommand{\clip}[3][]{{#1}_{\top {#2}}^{\bot {#3}}}

\newcommand{\sftrunc}{T_{\mathrm{trunc}}}
\newcommand{\tage}{t_{\mathrm{age}}}
\newcommand{\sfzero}{T_{\mathrm{zero}}}
\newcommand{\tzero}{t_{\circ}}
\newcommand{\sfslope}{m_{\mathrm{SF}}}

\begin{document}
\author{B. Johnson}

%\begin{center}
%\today
%\end{center}

\section{Introduction}
Throughout this document we use lowercase $t$ to refer to lookback times (or SSP ages), and capital $T$ to refer to time since the start of star formation.

We wish to find the proper weighting of discrete SSPs to reproduce the spectrum $F$ from SFR occurring within some bin of  lookback times $(\amin, \amax)$.
We approximate the spectrum $f(t)$ at any lookback time $t$ as a linear interpolation of the (total stellar mass normalized) spectra $f_i$ at the discrete SSP lookback times $t_i$, for $i=1,...,N$
\begin{eqnarray}
F & = & \int_{\amin}^{\amax} dt \, f(t) \, SFR(t) \nonumber \\
f(t) & = & w_{i}(t) \, f_i + w_{i+1}(t) \, f_{i+1} \, , \quad t_i < t < t_{i+1}  \nonumber \\
w_{i}(t) & = & \frac{t_{i+1} - t}{t_{i+1}  - t_i} \nonumber \\
w_{i+1}(t) & = & 1 - w_i(t) \quad . \nonumber
\end{eqnarray}
Then we can write the \emph{total} contribution of SSP $i$ to the total spectrum as a sum of the integrated weight due to SF in the sub-bin to the ``right'' (higher lookback times) and the  sub-bin to the ``left'' (lower lookback times).
The sub-bins are defined by the spacing of the SSP $t_i$ values.
The limits of each integral are determined either by the spacing of the SSP points $t_i$ or, when the bin does not completely span the sub-bin, by the $\amin$ and/or $\amax$ values.
\begin{eqnarray}
W_i & = &  \int_{\amin}^{\amax} dt \, \mathrm{SFR}(t) \, w_i(t) \nonumber \\
 & = & W_{i, \mathrm{right}} + W_{i, \mathrm{left}} \nonumber \\
W_{i, \mathrm{right}} & = & \tintegral \, \mathrm{SFR}(t) \, \frac{t_{i+1} - t}{t_{i+1}  - t_i} \nonumber \\
W_{i, \mathrm{left}} & = & \int_{\tmin[,i-1]}^{\tmax[,i-1]} dt \, \mathrm{SFR}(t) \, \frac{t - t_{i-1}}{t_{i}  - t_{i-1}} \nonumber \\
\tmin[,i] & = & \clip[\amin]{t_i}{t_{i+1}} \nonumber \\
\tmax[,i] & = & \clip[\amax]{t_i}{t_{i+1}} \nonumber
\end{eqnarray}
where $\clip[x]{a}{b}$ denotes $x$ \emph{clipped} to the interval $(a,b)$. If we take $\mathrm{SFR}(t) = 1$ then we can define
\begin{eqnarray}
g_i(x) & \equiv & \tintegral \, \mathrm{SFR}(t) \, \frac{x - t}{t_{i+1}  - t_i} \nonumber  \\
 & = & \left. \frac{1}{t_{i+1} - t_i} \, \left[ x\, t - \frac{t^2}{2} \right] \tinterval
\end{eqnarray}
which leads to
\begin{eqnarray}
W_i & = & g_{i}(t_{i+1}) - g_{i-1}(t_{i-1})
\end{eqnarray}
and one only has to calculate the $\{g_i(t_i)\}, \{g_i(t_{i+1})\}$ once and then take differences between shifted versions to obtain the total weights.
Note that for the smallest $i$ we define $g_{i-1}(t_{i-1})\equiv 0$ (there is no younger or ``left'' sub-bin)
and for the largest $i$ we define $g_{i}(t_{i+1}) \equiv 0$ (there is no older or ``right'' sub-bin) .

This formalism works even if $\amin$ and $\amax$ both fall between the same $t_i$ and $t_{i+1}$; that is, if the desired bin is narrower than the SSP defined sub-bins.
It also works if a sub-bin falls completely outside the $(\amin, \amax)$ interval.
This is because the clipping will cause $\tmin$ and $\tmax$ to be the same for bins not at least partially within the $(\amin, \amax)$ interval, and thus all integrals will evaluate to zero.

\section{Interpolation in $\log t$}

For the case of a linear interpolation in $\log t$, which is probably more appropriate, we can write
\begin{eqnarray}
w_{i}(t) & = & \frac{ \log t_{i+1} - \log t}{ \log t_{i+1}  - \log t_i} \nonumber
\end{eqnarray}
in the first set of equations above. The we simply replace $g_i$ by $s_i$ defined as
\begin{eqnarray}
s_i(x) & \equiv & \tintegral \, \mathrm{SFR}(t) \, \frac{x - \log t}{\log t_{i+1}  - \log t_i} \nonumber \\
 & = & \left. \frac{1}{\log t_{i+1} - \log t_i} t \, \left( x - \log t + \log e \right) \tinterval
\end{eqnarray}
to obtain a total weight for SSP $i$ of
\begin{eqnarray}
W_i & = & s_{i}(\log t_{i+1}) - s_{i-1}(\log t_{i-1}) \nonumber
\end{eqnarray}

The key in both cases is to properly calculate $\tmax$ and keep track of the indices
(especially for the first and last SSP, where $s_{i-1}$ and $s_i$ are not defined, respectively.)

\section{Lookback Time Zero}
The youngest age for which an SSP spectrum is availables is generally of order 1 Myr.
However, we need to account for SF occuring from 0-1 Myr lookback time.
There's a couple ways to deal with this necessary extrapolation.
Probably the most straightforward is nearest-neighbor extrapolation.
That is, simply approximate the $t=0$ spectrum by the youngest available SSP.

In the case of logarithmic interpolation one has to define some minimum age, say 1 year, to act as zero.
In practice the exact choice of minimum age does not matter,
as the total weights for the zeroth and first SSP converge such that fractional error is below $\sim 10^{-4}$ for a minimum age less than 10 years or so...
The totally correct way to do this is to work out the definite integrals properly with $\tmin[,i]=0$.

\section{More complex SFHs}
For more complicated SFH only the integrals in the basic definition of $s_i$ and $g_i$ must be calculated again by hand,
(remembering here that $t$ is \emph{lookback time}.)
We define $s_i(x)$ for interpolation in $\log t$ and $g_i(x)$ for interpolation in $t$.
For simplicity and clarity we take the cases $x = \log t_{i+1}$ for $s_i$ and $x = t_{i+1}$ for $g_i$,
but substitution for $x$ is trivial (as long as it is not $t$ dependent):
\begin{eqnarray}
s_i & \equiv & \tintegral \, \mathrm{SFR}(t) \, \frac{\log t_{i+1} - \log t}{\dlt} \\
g_i & \equiv & \tintegral \, \mathrm{SFR}(t) \, \frac{t_{i+1} - t}{\dt}
\end{eqnarray}
where for brevity we've defined in this section
\begin{eqnarray}
\dt & \equiv & t_{i+1} -  t_i  \nonumber \\
\dlt & \equiv & \log t_{i+1} - \log t_i \nonumber
\end{eqnarray}
In general the limits of the integral ($\tmin, \tmax$) will be defined by the SSP temporal grid $\{t_i\}$ as well as any discontinuities in the derivative of the SFR (e.g. the edges of step function already enocountered).

For convenience we will also note that the integral
\begin{eqnarray}
H_i & \equiv & \tintegral \, e^{t/\tau} \, \log t \nonumber \\
  & = & \left. \tau \left[ \log t \, e^{t/\tau} - \log(e) \, \mathrm{Ei}(t/\tau) \right]\tinterval
\end{eqnarray}
where Ei is the exponential integral.

\subsection{$\tau$-model}
Here we have a SFH defined by three parameters
\[
\mathrm{SFR}(t) =
\begin{cases}
K \, e^{-T/\tau}, &  \text{if } 0 < T \leq \sftrunc \\
0, & \text{otherwise}
\end{cases}
\]
\begin{eqnarray}
T & = & \tage - t \nonumber \\
t_q & = & \tage - \sftrunc \nonumber
\end{eqnarray}
where $T$ is defined as the time since the start of star formation, \emph{not} lookback time.
The lookback time  (in the reference frame of the galaxy) is given by $t$, which goes from 0 to $\tage$.
The parameters are:
$\tage$, the age of the galaxy since the start of SF (\texttt{tage} in FSPS),
$\tau$, the $e$-folding time (\texttt{tau} in FSPS), and
$\sftrunc$, the time since the start of SF that truncation occurs (\texttt{sf\_trunc} in FSPS).
Note that in the current FSPS definition, $\sftrunc = 0$ corresponds to no truncation, so it is actually $\sftrunc= \tage$ and $t_q=0$.
The FSPS \texttt{sf\_start} is basically redundant with \texttt{tage} and confusing so we ignore it.

In this case we have, for the logarithmic time interpolation,
\begin{eqnarray}
s_i  & = & \tintegral \, K \, e^{(t-\tage) /\tau}\, \frac{\log t_{i+1} - \log t}{\dlt} \nonumber \\
%      & = & \frac{K \, e^{-\tage/\tau}}{\dlt} \left[ \left. \tau \, \log t_{i+1} \, e^{t/\tau} \tinterval -  H_i\right] \nonumber \\
\tmin[,i] & = & \clip[t_q]{t_i}{t_{i+1}} \nonumber \\
\tmax[,i] & = & \clip[\tage]{t_i}{t_{i+1}} \quad . \nonumber
\end{eqnarray}
This yields,
\begin{eqnarray}
s_i & = & \left. \frac{K \, \tau \, e^{-\tage/\tau}}{\dlt} \left [(\log t_{i+1} - \log t)\, e^{t/\tau} +
          \log(e) \, \mathrm{Ei}(t/\tau)\right]  \tinterval
\end{eqnarray}
For interpolation in linear time the integral is a bit more straightforward
\begin{eqnarray}
g_i & = & \left. \frac{K \, \tau \, e^{-\tage/\tau}}{\dt} (t_{i+1} - t + \tau)\, e^{t/\tau} \tinterval
\end{eqnarray}
where the limits $\tmin$ and $\tmax$ are defined as above.
One can either calculate the normalization constant $K$ analytically, or simply renormalize the final weights $W_i$ by their sum.

\subsection{Delayed-$\tau$}
Here the SFH is also defined by three parameters
\[
\mathrm{SFR}(t) =
\begin{cases}
K \, T \, e^{-T/\tau}, &  \text{if } 0 < T \leq \sftrunc\\
0, & \text{otherwise}
\end{cases}
\]
\begin{eqnarray}
T & = & \tage - t \nonumber \\
t_q & = & \tage - \sftrunc\nonumber
\end{eqnarray}
So now we have
\begin{eqnarray}
s_i  & = & \tintegral \, K \, (\tage-t) \, e^{(t-\tage) /\tau}\, \frac{\log t_{i+1} - \log t}{\dlt}  \nonumber \\
%      & = & \frac{K \, e^{-\tage/\tau}}{\dlt} \tintegral \, \left(\tage \, \log t_{i+1} \, e^{t/\tau}  - \tage \, \log t \, e^{t/\tau} - t \, \log t_{i+1} \, e^{t/\tau} + t \, \log t \, e^{t/\tau}\right) \quad .\nonumber
%\end{eqnarray}
%The first and third terms are fairly straightforward.  The second term is basically the integral we did for the $\tau$-model.  The last term can be solved by integrating by parts (with $u=t\,\log t$, $dv=e^{t/\tau}dt$ and hence $du=(\log e + \log t)\, dt$, $v=\tau e^{t/\tau}$.  Substituting and collecting terms gives, finally,
%\begin{eqnarray}
%s_i  & = & \frac{K \, \tau \,  e^{-\tage/\tau}}{\dlt} \left[\left.\left(t\,\log \frac{t}{t_{i+1}} + \tage\,\log t_{i+1}  + \tau\,\log \frac{t_{i+1}}{e}\right) e^{t/\tau}\tinterval - (\tage/\tau+1)\, H_i\right] \\
 & = & \frac{K \, \tau \,  e^{-\tage/\tau}}{\dlt} \left. \left[\left((t - \tage - \tau) \, \log \frac{t}{t_{i+1}} - \tau \, \log (e) \right) \, e^{t/\tau}    + (\tage + \tau) \, \log (e) \, \mathrm{Ei}(t/\tau) \right]\tinterval %\quad .
\end{eqnarray}
The limits $\tmin$ and $\tmax$ are as for the $\tau$-model above.

For interpolation in linear time, we have
\begin{eqnarray}
g_i  & = & \tintegral \, K \, (\tage-t) \, e^{(t-\tage) /\tau}\, \frac{t_{i+1} - t}{\dt}  \nonumber \\
%      & = & \frac{K \, e^{-\tage/\tau}}{\dt} \tintegral \, \left(\tage \, t_{i+1} \, e^{t/\tau}  - \tage \, t \, e^{t/\tau} - t \,  t_{i+1} \, e^{t/\tau} + t^2 \, e^{t/\tau}\right) \nonumber \\
     & = & \frac{K \, \tau \, e^{-\tage/\tau}}{\dt} \left. e^{t/\tau} \, \left[ \tage \, t_{i+1} - (\tage + t_{i+1})\, (t-\tau) + t^2 - 2\, t\, \tau + 2\, \tau^2\right] \tinterval \quad .
\end{eqnarray}

One could, in principle, redefine the SSP $t_i$ to be in forward time instead of lookback time.  While this would simplify the integrals, keeping them in lookback time is conceptually simpler.

\subsection{Simha}
This is the SFH from \citet{simha14}.
It is defined by a delayed-$\tau$ SFH until some time $\sftrunc$, after which the SFH is linearly increasing or decreasing with some slope $m_{\mathrm{SF}}$, which has units of inverse time.
The SFR is continuous at $\sftrunc$, but the derivative is not.
For this reason, we will treat the SFH in two separate pieces.
The trick is to get the proper normalization at $\sftrunc$ and, for linearly decreasing SF, find the time where the SF reaches zero.
\[
\mathrm{SFR}(t) =
\begin{cases}
K \, T \, e^{-T/\tau}, &  \text{if } 0 < T \leq \sftrunc \\
L\, \left[1 - \sfslope \, (T - \sftrunc)\right] & \text{if } \sftrunc < T  < \sfzero \\
0 & \text{otherwise}
\end{cases}
\]
\begin{eqnarray}
T & = & \tage - t \nonumber \\
t_q & = & \tage - \sftrunc \nonumber \\
L & = & K \, \sftrunc \, e^{-\sftrunc/\tau} \nonumber \\
\sfzero & = & \clip[\left(\sftrunc + \sfslope^{-1}\right)]{\sftrunc}{\tage}, \nonumber \\
\tzero & = & \clip[\left( t_q - \sfslope^{-1}\right)]{0}{t_q} \nonumber
\end{eqnarray}
where $\tzero$ is the \emph{lookback} time at which the SFR becomes zero.  The clipping here is the wrong terminology.  Basically we want $\tzero$ to be 0 unless it falls within the interval $(t_q -\sfslope^{-1}, t_q)$, or if $\sfslope=0$.

The portion of the SFH that is a delayed-$\tau$ can be dealt with using the formulae in the last section.  The linear portion is relatively straightforward:
\begin{eqnarray}
s_i  & = & \tintegral \,  L\, \left[1 - \sfslope \, (t_q - t)\right] \, \frac{\log t_{i+1} - \log t}{\dlt}  \nonumber \\
      & = & \frac{L}{\dlt} \left.\left[(1-\sfslope\, t_q) \, t \, \left(\log t_{i+1} - \log t +\log e \right) +
                                                     \frac{\sfslope \, t^2}{2} \, \left(\log t_{i+1} - \log t +\frac{\log e}{2}\right) \right] \tinterval \\
\tmin[,i] & = & \clip[\tzero]{t_i}{t_{i+1}} \nonumber \\
\tmax[,i] & = & \clip[t_q]{t_i}{t_{i+1}} \quad . \nonumber
\end{eqnarray}

The linear interpolation version of the linear SFH is given by
\begin{eqnarray}
g_i  & = & \tintegral \,  L\, \left[1 - \sfslope \, (t_q - t)\right] \, \frac{t_{i+1} - t}{\dt}  \nonumber \\
      & = & \frac{L}{\dt} \left.\left[(1-\sfslope\, t_q) \, t_{i+1} \, t + \left(\sfslope\, t_{i+1} + \sfslope \, t_q -1\right)\, \frac{t^2}{2} - \frac{\sfslope\, t^3}{3}\right] \tinterval
\end{eqnarray}
We verify that these expressions reduce to the constant SFR case when $\sfslope=0$.

Generally we will want to express the normalization as a ratio of total masses formed in the two segments (the delayed-$\tau$ and linear portions).

\subsection{Bursts}

Here we have
\[
\mathrm{SFR}(t) = \delta(T - T_{burst})
\]
\begin{eqnarray}
T & = & \tage - t \nonumber \\
T_{burst} & = & \tage - t_b
\end{eqnarray}
and the integrals are easy as pie
\[
s_i =
\begin{cases}
\frac{\log t_{i+1} - \log t_b}{\dlt}&  \text{if } t_i < t_b \leq t_{i+1} \\
0 & \text{otherwise}
\end{cases}
\]
and
\[
g_i =
\begin{cases}
\frac{t_{i+1} - t_b}{\dt}&  \text{if } t_i < t_b \leq t_{i+1} \\
0 & \text{otherwise}
\end{cases}
\]

\section{Old FSPS Integrals}

Previously, FSPS was doing the something like following integral (ignoring \texttt{dust\_tesc}, \texttt{sf\_start} and \texttt{sf\_trunc})
\begin{eqnarray}
f & = & f_{i_{max}+1} \, \int _{t_{i_{max}}}^{\tage} dt\, SFR(t) + \sum_{i=1}^{i_{max}} \frac{f_i + f_{i+1}}{2}\int _{t_{i}-t_1}^{t_{i+1}-t_1} dt\, SFR(t) \\
%i_{max} & = &
\end{eqnarray}
This is basically trapezoidal integration, a numerical approximation to the fully analytic integrals we are doing.
It works great as long as the SFH in the bin is approximately linear across the entire bin.

\end{document}


\section{Powers of Trigonometric Functions}{}{}\label{sec:Powers of trigonometric functions}
Functions involving trigonometric functions are useful as they are good at describing periodic behavior. % Learning techniques to integrate such functions helps solve problems involving periodic behavior and also helps build general problem solving skills.
 This section describes several techniques for finding antiderivatives of certain combinations of trigonometric functions.\\

Functions consisting of powers of the sine and cosine can be
integrated by using substitution and trigonometric identities. These
can sometimes be tedious, but the technique is straightforward. A similar
technique is applicable to powers of secant and tangent (and also cosecant
and cotangent, not discussed here).

%The trigonometric substitutions we will focus on in this section are summarized in the table below:
%\begin{center}
%	\begin{tabular}{|c|c|c|c|c|}
%	\hline
%	\thmfont{Substitution} & $u=\sin x$ & $u=\cos x$ & $u=\tan x$ & $u=\sec x$\\[0.5em]
%	\thmfont{Derivative} & $du=\cos x\, dx$ & $du=-\sin x\,dx$ & $du=\sec^2x\,dx$ & $du=\sec x\tan x\,dx$\\
%	\hline
%	\end{tabular}
%\end{center}

\subsection*{Integrals of the form $\ds \int \sin^m x\cos^n x\ dx$}

Using the technique of Substitution, we see the integral $\int \sin x\cos x\ dx$  could easily be evaluated  by letting $u=\sin x$ or by letting $u = \cos x$. This integral is easy since the power of both sine and cosine is $ 1 $.

We generalize this integral and consider integrals of the form $\int \sin^mx\cos^nx\ dx$, where $m,n$ are nonnegative integers. Our strategy for evaluating these integrals is to use the Pythagorean identity $\cos^2x+\sin^2x=1$ to convert high powers of one trigonometric function into the other, leaving a single sine or cosine term in the integrand. We summarize the general technique in the following

\begin{formulabox}[Products of Sine and Cosine]
	When evaluating $\ds\int\sin^mx \cos^nx\,dx$:
	\begin{enumerate}
	\item \ffont{The power of sine is odd ($m$ odd):}\\
		(a) Use $u=\cos x$ and $du=-\sin x\,dx$.\\
		(b) Replace $dx$ using (a), thus cancelling one power of $\sin x$ by the substitution of $du$, and be left with an even number of sine powers.\\
		(c) Use $\sin^2x=1-\cos^2x~(=1-u^2)$ to replace the leftover sines.
	\item \ffont{The power of cosine is odd ($n$ odd):}\\
		(a) Use $u=\sin x$ and $du=\cos x\,dx$.\\
		(b) Replace $dx$ using (a), thus cancelling one power of $\cos x$ by the substitution of $du$, and be left with an even number of cosine powers.\\
		(c) Use $\cos^2x=1-\sin^2x~(=1-u^2)$ to replace the leftover cosines.
	\item \ffont{Both $m$ and $n$ are odd}:\\ Use either $1$ or $2$ (both will work).
	\item \ffont{Both $m$ and $n$ are even}:\\
	 Use $\cos^2x=\frac{1}{2}\left(1+\cos(2x)\right)$ and/or $\sin^2x=\frac{1}{2}\left(1-\cos(2x)\right)$ to reduce to a form that can be integrated.
	\end{enumerate}
	NOTE: As $m$ and $n$ get large, multiple steps will be needed.
\end{formulabox}

%\begin{formulabox}[Integrals Involving Powers of Sine and Cosine]
%	Consider $\ds \int \sin^mx\cos^nx\ dx$, where $m,n$ are nonnegative integers.\index{integration!of trig. powers}
%		\begin{enumerate}
%		\item		If $m$ is odd, then $m=2k+1$ for some integer $k$. Rewrite \small
%				$$ \sin^mx = \sin^{2k+1}x = \sin^{2k}x\sin x = (\sin^2x)^k\sin x = (1-\cos^2x)^k\sin x.$$\normalsize
%				Then \small
%				$$\int \sin^mx\cos^nx\ dx = \int (1-\cos^2x)^k\sin x\cos^nx\ dx = -\int (1-u^2)^ku^n\ du,$$\normalsize
%				where $u = \cos x$ and $du = -\sin x\ dx$.
%		\item		If $n$ is odd, then using substitutions similar to that outlined above we have
%				\small
%				$$ \int \sin^mx\cos^nx\ dx = \int u^m(1-u^2)^k\ du,$$ \normalsize
%				where $u = \sin x$ and $du = \cos x\ dx$.
%		\item		If both $m$ and $n$ are even, use the power--reducing identities
%			\small$$  \cos^2x = \frac{1+\cos (2x)}{2} \quad \text{and}\quad \sin^2x = \frac{1-\cos(2x)}2$$\normalsize
%		to reduce the degree of the integrand. Expand the result and try again.
%		\end{enumerate}
%%	NOTE: The integral will become more tedious as $m$ and $n$ get large, since multiple steps will be needed.
%\end{formulabox}

Now a few examples to practice the approach.

\begin{example}{Integrating powers of sine and cosine}{}
Evaluate $\ds\int \sin^6 x\cos^5 x\,dx$.
\end{example}

\begin{solution}
Since the power of cosine is odd, we use the substitution $\red{u=\sin x}$ and $du=\cos x\,dx$, that is, $\fbox{$dx$}=\fbox{$\ds\frac{du}{\cos x}$}$.
Then $\ds\int \sin^6 x\cos^5 x\,dx$ is equal to:
$${\def\arraystretch{2.2}
\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
 & = & \int u^6\cos^5x \,\fbox{$\ds\frac{du}{\cos x}$}	 & \mbox{Using the substitution} \\
	&=&\int u^6\left(\cos^2x\right)^2 \,du  & \mbox{Canceling a $\cos x$ and rewriting $\cos^4x$}\\
	&=&\int u^6(1-\sin^2x)^2\,du  & \mbox{Using trig identity $\cos^2x=1-\sin^2x$}\\
	&=&\int u^6(1-u^2)^2\,du  & \mbox{Writing integral in terms of $u$'s}\\
	&=&\int u^6-2u^8+u^{10}\,du  & \mbox{Expand and collect like terms}\\
	&=&\frac{u^7}{7}-\frac{2u^9}{9}+\frac{u^{11}}{11}+C & \mbox{Integrating}\\
	&=&\frac{\sin^7x}{7}-\frac{2\sin^9x}{9}+\frac{\sin^{11}x}{11}+C  & \mbox{Replacing $u$ back in terms of $x$}
\end{array}
}$$
\end{solution}

\begin{example}{Odd Power of Cosine}{}
Evaluate $\ds\int \cos^3 x\,dx$.
\end{example}

\begin{solution}
Since the power of cosine is odd, we use the substitution $\red{u=\sin x}$ and $du=\cos x\,dx$.
This may seem strange at first since we don't have $\sin x$ in the question, but it does work!
$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
\int \cos^3 x\,\fbox{$dx$} & = & \int \cos^3x \,\fbox{$\ds\frac{du}{\cos x}$}	 & \mbox{Using the substitution} \\
	&=&\int \cos^2x \,du  & \mbox{Canceling a $\cos x$}\\
	&=&\int (1-\sin^2x)\,du  & \mbox{Using trig identity $\cos^2x=1-\sin^2x$}\\
	&=&\int (1-u^2)\,du  & \mbox{Writing integral in terms of $u$'s}\\
	&=&u-\frac{u^3}{3}+C & \mbox{Integrating}\\
	&=&\sin x-\frac{\sin^3x}{3}+C  & \mbox{Replacing $u$ back in terms of $x$}
\end{array}$$
\end{solution}

\begin{example}{Integrating powers of sine and cosine}{Integrating powers of sine and cosine}\label{Odd Power of Sine}
Evaluate $\ds\int \sin^5 x\,dx$.
\end{example}

\begin{solution}
Since the power of sine is odd, we factor out one $ \sin(x) $, and exploit the Pythagorean identity: $\sin^2x+\cos^2x=1$.
\begin{eqnarray*}
  \int \sin^5 x\,dx&=&\int \sin x \sin^4 x\,dx\\
	&=&
  \int \sin x (\sin^2 x)^2\,dx\\
	&=&
  \int \sin x (1-\cos^2 x)^2\,dx.
\end{eqnarray*}%\vskip-20pt
Now use $u=\cos x$, $du=-\sin x\,dx$:
\begin{eqnarray*}
  \int \sin x (1-\cos^2 x)^2\,dx&=&\int -(1-u^2)^2\,du\\
  &=&\int -(1-2u^2+u^4)\,du\\
  &=&\int 1+2u^2-u^4\,du\\
  &=&-u+{\frac{2}{3}}u^3-{\frac{1}{5}}u^5+C\\
  &=&-\cos x+\frac{2}{3}\cos^3 x-\frac{1}{5}\cos^5x+C.
\end{eqnarray*}\vskip-20pt
\end{solution}

%Observe that by taking the substitution $u=\cos x$ in the last example, we ended up with an even power of sine from which we can use the Pythagorean identity $\sin^2x+\cos^2x=1$ to replace any remaining sine terms.
%We then ended up with a polynomial in $u$ in which we could expand and integrate quite easily.

%This technique works for products of powers of sine and cosine.
%We summarize it below.
%\begin{formulabox}[Products of Sine and Cosine]
%	When evaluating $\ds\int\sin^mx \cos^nx\,dx$:
%	\begin{enumerate}
%	\item \ffont{The power of sine is odd ($m$ odd):}\\
%		(a) Use $u=\cos x$ and $du=-\sin x\,dx$.\\
%		(b) Replace $dx$ using (a), thus cancelling one power of $\sin x$ by the substitution of $du$, and be left with an even number of sine powers.\\
%		(c) Use $\sin^2x=1-\cos^2x~(=1-u^2)$ to replace the leftover sines.
%	\item \ffont{The power of cosine is odd ($n$ odd):}\\
%		(a) Use $u=\sin x$ and $du=\cos x\,dx$.\\
%		(b) Replace $dx$ using (a), thus cancelling one power of $\cos x$ by the substitution of $du$, and be left with an even number of cosine powers.\\
%		(c) Use $\cos^2x=1-\sin^2x~(=1-u^2)$ to replace the leftover cosines.
%	\item \ffont{Both $m$ and $n$ are odd}:\\ Use either $1$ or $2$ (both will work).
%	\item \ffont{Both $m$ and $n$ are even}:\\
%	 Use $\cos^2x=\frac{1}{2}\left(1+\cos(2x)\right)$ and/or $\sin^2x=\frac{1}{2}\left(1-\cos(2x)\right)$ to reduce to a form that can be integrated.
%	\end{enumerate}
%	NOTE: The integral will become more tedious as $m$ and $n$ get large, since multiple steps will be needed.
%\end{formulabox}

\begin{example}{Integrating powers of sine and cosine}{Integrating powers of sine and cosine}\label{Odd Power of Sine}
Evaluate $\ds\int\sin^5x\cos^8x\ dx$.
\end{example}

\begin{solution}
The power of the sine term is odd, so we rewrite $\sin^5x$ as $$\sin^5x = \sin^4x\sin x = (\sin^2x)^2\sin x = (1-\cos^2x)^2\sin x.$$

Our integral is now $\ds \int (1-\cos^2x)^2\cos^8x\sin x\ dx$. Let $u = \cos x$, hence $du = -\sin x\ dx$. Making the substitution and expanding the integrand gives
$$\int (1-\cos^2)^2\cos^8x\sin x\ dx = -\int (1-u^2)^2u^8\ du = -\int \big(1-2u^2+u^4\big)u^8\ du = -\int \big(u^8-2u^{10}+u^{12}\big)\ du.$$
This final integral is not difficult to evaluate, giving
\begin{align*} -\int \big(u^8-2u^{10}+u^{12}\big)\ du &= -\frac19u^9 + \frac2{11}u^{11} - \frac1{13}u^{13} + C \\
										&=-\frac19\cos^9 x + \frac2{11}\cos^{11} x - \frac1{13}\cos^{13} x + C.
\end{align*}
\end{solution}

\begin{example}{Product of Even Powers of Sine and Cosine}{Product of Even Powers of Sine and Cosine}\label{Product of Even Powers of Sine and Cosine}
Evaluate $\ds\int \sin^2x\cos^2x\,dx$.
\end{example}

\begin{solution}
Use the formulas
$\ds \sin^2x =(1-\cos(2x))/2$ and $\ds \cos^2x =(1+\cos(2x))/2$ to get:
$$
  \int \sin^2x\cos^2x\,dx=\int {1-\cos(2x)\over2}\cdot
  {1+\cos(2x)\over2}\,dx.
$$
We then have
\begin{eqnarray*}
\int \sin^2x\cos^2x\,dx&=&\int {1-\cos(2x)\over2}\cdot{1+\cos(2x)\over2}\,dx\\
&=&{1\over4}\int 1-\cos^2 2x\,dx\\
&=&{1\over4}\left(x-\int\cos^2 2x\,dx\right)\\
&=&{1\over4}\left(x-{1\over2}\int 1+\cos4x \,dx\right)\\
&=&{1\over4}\left(x-{1\over2}\left(x+{\sin 4x\over 4}\right)\right)\\
&=&{1\over4}\left(x-{x\over2}-{\sin 4x\over 8}\right)+C
\end{eqnarray*}
\end{solution}

\begin{example}{Even Power of Sine}{Even Power of Sine}\label{Even Power of Sine}
Evaluate $\ds\int \sin^6 x\,dx$.
\end{example}

\begin{solution}
Use $\ds \sin^2x =(1-\cos(2x))/2$ to
rewrite the function:
\begin{eqnarray*}
  \int \sin^6 x\,dx&=&\int (\sin^2 x)^3\,dx\\
	&=&\int {(1-\cos 2x)^3\over 8}\,dx\\
  &=&{1\over 8}\int 1-3\cos 2x+3\cos^2 2x-\cos^3 2x\,dx.
\end{eqnarray*}
Now we have four integrals to evaluate:
$$\int 1\,dx=x+C$$
and
$$\int -3\cos 2x\,dx = -{3\over 2}\sin 2x+C$$
The $\ds \cos^3 2x$ integral is like the previous example:
\begin{eqnarray*}
  \int -\cos^3 2x\,dx&=&\int -\cos 2x\cos^2 2x\,dx\\
  &=&\int -\cos 2x(1-\sin^2 2x)\,dx\\
  &=&\int -{1\over 2}(1-u^2)\,du\\
  &=&-{1\over 2}\left(u-{u^3\over 3}\right)+C\\
  &=&-{1\over 2}\left(\sin 2x-{\sin^3 2x\over 3}\right)+C.
\end{eqnarray*}
And finally we use another trigonometric identity,
$\ds \cos^2x=(1+\cos(2x))/2$:
$$
  \int 3\cos^2 2x\,dx=3\int {1+\cos 4x\over 2}\,dx=
  {3\over 2}\left(x+{\sin 4x\over 4}\right)+C.
$$
So at long last, gathering and combining the arbitrary constants we get
$$
  \int \sin^6 x\,dx = {x\over8} -{3\over 16}\sin 2x
  -{1\over 16}\left(\sin 2x-{\sin^3 2x\over 3}\right)
  +{3\over 16}\left(x+{\sin 4x\over 4}\right)+C.
$$\vskip-10pt
\end{solution}

\begin{example}{Integrating powers of sine and cosine}{ex_trigint2}
Evaluate $\ds \int\sin^5x\cos^9x\ dx$.
\end{example}

\begin{solution}
The powers of both the sine and cosine terms are odd, therefore we can apply our techniques to either power. We choose to work with the power of the cosine term since the previous example used the sine term's power.

We rewrite $\cos^9x$ as
\begin{align*} \cos^9 x &= \cos^8x\cos x \\
				&= (\cos^2x)^4\cos x \\
				&= (1-\sin^2x)^4\cos x \\
				&= (1-4\sin^2x+6\sin^4x-4\sin^6x+\sin^8x)\cos x.
\end{align*}

We rewrite the integral as
$$\int\sin^5x\cos^9x\ dx = \int\sin^5x\big(1-4\sin^2x+6\sin^4x-4\sin^6x+\sin^8x\big)\cos x\ dx.$$

Now substitute and integrate, using $u = \sin x $ and $du = \cos x\ dx$.

\small\noindent
%\begin{gather}
%\int\sin^5x\big(1-4\sin^2x+6\sin^4x-4\sin^6x+\sin^8x\big)\cos x\ dx =\notag\\
%\int u^5(1-4u^2+6u^4-4u^6+u^8)\ du = \int\big(u^5-4u^7+6u^9-4u^{11}+u^{13}\big)\ du  \notag
%\end{gather}
$\ds \int\sin^5x\big(1-4\sin^2x+6\sin^4x-4\sin^6x+\sin^8x\big)\cos x\ dx =$% \\
\vskip-.8\baselineskip
\begin{align*}
 \int u^5(1-4u^2+6u^4-4u^6+u^8)\ du &= \int\big(u^5-4u^7+6u^9-4u^{11}+u^{13}\big)\ du \\
				&= \frac16u^6-\frac12u^8+\frac35u^{10}-\frac13u^{12}+\frac{1}{14}u^{14}+C\\
				&= \frac16\sin^6 x-\frac12\sin^8 x+\frac35\sin^{10} x+\ldots\\
				&\phantom{=}-\frac13\sin^{12} x+\frac{1}{14}\sin^{14} x+C.
\end{align*}
\end{solution}

\noindent\textbf{Technology Note:} The work we are doing here can be a bit tedious, but the skills developed (problem solving, algebraic manipulation, etc.) are important. Nowadays problems of this sort are often solved using a computer algebra system. The powerful program \textit{Mathematica}\textsuperscript{\textregistered}, or \textit{Wolfram Alpha} integrates $\int \sin^5x\cos^9x\ dx$ as \small$$f(x)=-\frac{45 \cos (2 x)}{16384}-\frac{5 \cos (4 x)}{8192}+\frac{19 \cos (6
   x)}{49152}+\frac{\cos (8 x)}{4096}-\frac{\cos (10 x)}{81920}-\frac{\cos (12
   x)}{24576}-\frac{\cos (14 x)}{114688},$$\normalsize
which clearly has a different form than our answer in Example \ref{exa:ex_trigint2}, which is
$$g(x)=\frac16\sin^6 x-\frac12\sin^8 x+\frac35\sin^{10} x-\frac13\sin^{12} x+\frac{1}{14}\sin^{14} x.$$ Figure \ref{fig:trigint2} shows a graph of $f$ and $g$; they are clearly not equal, but they differ \emph{only by a constant}. That is $g(x) = f(x) + C$ for some constant $C$. So we have two different antiderivatives of the same function, meaning both answers are correct. \\%We leave it to the reader to recognize why both answers are correct.\\

\begin{figure}
\begin{center}
\begin{tikzpicture}
\begin{axis}[width=\marginparwidth+25pt,%
tick label style={font=\scriptsize},axis y line=middle,axis x line=middle,name=myplot,axis on top,%
			%x=.37\marginparwidth,
			%y=.37\marginparwidth,
%			xtick=\empty,%
%			extra x ticks={.5,3},
%			extra x tick labels={$a$,$b$},
			ytick={-.002,.002,.004},
			yticklabels={$-0.002$,$0.002$,$0.004$},
			%minor y tick num=1,
%			extra y ticks={0.001},%
%			minor x tick num=4,
			ymin=-.003,ymax=0.005,%
			xmin=-.1,xmax=3.15,%
			scaled ticks=false
]

\addplot [{\colortwo},thick,smooth] coordinates {(0,-0.0027879) (0.15708,-0.0027856) (0.31416,-0.0026798) (0.47124,-0.0020322) (0.62832,-0.00060997) (0.7854,0.00089518)(0.94248,0.0017233) (1.0996,0.0019485) (1.2566,0.0019734) (1.4137,0.0019741) (1.5708,0.0019741) (1.7279,0.0019741) (1.885,0.0019734) (2.042,0.0019485) (2.1991,0.0017233) (2.3562,0.00089518) (2.5133,-0.00060997) (2.6704,-0.0020322) (2.8274,-0.0026798) (2.9845,-0.0027856) (3.1416,-0.0027879)};

\draw (axis cs:2.6,0.004) node {\scriptsize $g(x)$};

\addplot [{\colorone},thick,smooth] coordinates {(0,0) (0.15708,0) (0.31416,0.00010807) (0.47124,0.0007557) (0.62832,0.0021779) (0.7854,0.003683) (0.94248,0.0045111)
(1.0996,0.0047364) (1.2566,0.0047612) (1.4137,0.0047619)
(1.5708,0.0047619) (1.7279,0.0047619) (1.885,0.0047612) (2.042,0.0047364) (2.1991,0.0045111) (2.3562,0.003683) (2.5133,0.0021779) (2.6704,0.0007557) (2.8274,0.00010807) (2.9845,0) (3.1416,0)};

\draw (axis cs:2.4,-0.002) node {\scriptsize $f(x)$};
\end{axis}

\node [right] at (myplot.right of origin) {\scriptsize $x$};
\node [above] at (myplot.above origin) {\scriptsize $y$};
\end{tikzpicture}
\caption{A plot of $f(x)$ and $g(x)$ from Example \ref{exa:ex_trigint2} and the Technology Note. \label{fig:trigint2}}
\end{center}
\end{figure}

\begin{example}{Integrating powers of sine and cosine}{Integrating powers of sine and cosine}
Evaluate $\ds\int\cos^4x\sin^2x\ dx$.
\end{example}

\begin{solution}
The powers of sine and cosine are both even, so we employ the power--reducing formulas and algebra as follows.
\begin{align*}
\int \cos^4x\sin^2x\ dx &= \int\left(\frac{1+\cos(2x)}{2}\right)^2\left(\frac{1-\cos(2x)}2\right)\ dx \\
				&= \int\frac{1+2\cos(2x)+\cos^2(2x)}4\cdot\frac{1-\cos(2x)}2\ dx\\
				&=	\int \frac18\big(1+\cos(2x)-\cos^2(2x)-\cos^3(2x)\big)\ dx
\end{align*}
The $\cos(2x)$ term is easy to integrate, especially with Key Idea \ref{idea:linearsub}. The $\cos^2(2x)$ term is another trigonometric integral with an even power, requiring the power--reducing formula again. The $\cos^3(2x)$ term is a cosine function with an odd power, requiring a substitution as done before. We integrate each in turn below.

$$\int\cos(2x)\ dx = \frac12\sin(2x)+C.$$

$$\int\cos^2(2x)\ dx = \int \frac{1+\cos(4x)}2\ dx = \frac12\big(x+\frac14\sin(4x)\big)+C.$$

Finally, we rewrite $\cos^3(2x)$ as $$\cos^3(2x) = \cos^2(2x)\cos(2x) = \big(1-\sin^2(2x)\big)\cos(2x).$$
Letting $u=\sin(2x)$, we have $du = 2\cos(2x)\ dx$, hence
\begin{align*}
\int \cos^3(2x)\ dx &= \int\big(1-\sin^2(2x)\big)\cos(2x)\ dx\\
							&= \int \frac12(1-u^2)\ du\\
							&= \frac12\Big(u-\frac13u^3\Big)+C\\
							&=	\frac12\Big(\sin(2x)-\frac13\sin^3(2x)\Big)+C
\end{align*}

Putting all the pieces together, we have
\begin{align*}
\int \cos^4x\sin^2x\ dx &=\int \frac18\big(1+\cos(2x)-\cos^2(2x)-\cos^3(2x)\big)\ dx \\
					&= \frac18\Big[x+\frac12\sin(2x)-\frac12\big(x+\frac14\sin(4x)\big)-\frac12\Big(\sin(2x)-\frac13\sin^3(2x)\Big)\Big]+C \\
					&=\frac18\Big[\frac12x-\frac18\sin(4x)+\frac16\sin^3(2x)\Big]+C
\end{align*}
\end{solution}

\subsection*{Integrals of the form {\small{$\ds \int\sin(mx)\sin(nx)\ dx,$ $\ds\int \cos(mx)\cos(nx)\ dx$}}, and {\small{$\ds\int \sin(mx)\cos(nx)\ dx$.}}}

Functions that contain products of sines and cosines of differing periods are important in many applications including the analysis of sound waves. Integrals of the form
$$\int\sin(mx)\sin(nx)\ dx,\quad \int \cos(mx)\cos(nx)\ dx \quad \text{and}\quad\int \sin(mx)\cos(nx)\ dx$$
are best approached by first applying the Product to Sum Formulas found in the back cover of this text, namely
\begin{align*}
\sin(mx)\sin(nx) &= \frac12\Big[\cos\big((m-n)x\big)-\cos\big((m+n)x\big)\Big] \\
\cos(mx)\cos(nx) &= \frac12\Big[\cos\big((m-n)x\big)+\cos\big((m+n)x\big)\Big] \\
\sin(mx)\cos(nx) &=	\frac12\Big[\sin\big((m-n)x\big)+\sin\big((m+n)x\big)\Big]
\end{align*}

\begin{example}{Integrating products of $\sin(mx)$ and $\cos(nx)$}{ex_trigint4}
Evaluate $\ds\int\sin(5x)\cos(2x)\ dx$.
\end{example}

\begin{solution}
The application of the formula and subsequent integration are straightforward:
\begin{align*}
\int\sin(5x)\cos(2x)\ dx &= \int \frac12\Big[\sin(3x)+\sin(7x)\Big]\ dx \\
												&= -\frac16\cos(3x) - \frac1{14}\cos(7x) + C
\end{align*}
\end{solution}

\section*{Integrals of the form $\ds\int\tan^mx\sec^nx\ dx$.}

When evaluating integrals of the form $\int \sin^mx\cos^nx\ dx$, the Pythagorean identity allowed us to convert even powers of sine into even powers of cosine, and vise--versa. If, for instance, the power of sine was odd, we pulled out one $\sin x$ and converted the remaining even power of $\sin x$ into a function using powers of $\cos x$, leading to an easy substitution.

The same basic strategy applies to integrals of the form $\int \tan^mx\sec^n x\ dx$, albeit a bit more nuanced. The following three facts will prove useful:
\begin{itemize}
\item $\frac{d}{dx}(\tan x) = \sec^2x$,
\item $\frac{d}{dx}(\sec x) = \sec x\tan x$ , and
\item	$1+\tan^2x = \sec^2x$ (the Pythagorean Theorem).
\end{itemize}

If the integrand can be manipulated to separate a $\sec^2x$ term with the remaining secant power even, or if a $\sec x\tan x$ term can be separated with the remaining $\tan x$ power even, the Pythagorean Theorem can be employed, leading to a simple substitution. This strategy is outlined in the following.

%Next, we turn our attention to products of secant and tangent.
%Some we already know how to do.
%$$\int \sec^2x\,dx = \tan x+C\qquad\qquad\int\sec x\tan x\,dx=\sec x+C$$

\index{integration!products of secant and tangent}
\begin{formulabox}[Products of Secant and Tangent]
	When evaluating $\ds\int\sec^mx \tan^nx\,dx$:
	\begin{enumerate}
	\item \ffont{The power of secant is even ($m$ even):}\\
		(a) Use $u=\tan x$ and $du=\sec^2 x\,dx$.\\
		(b) Cancel $\sec^2 x$ by the substitution of $dx$, and be left with an even number of secants.\\
		(c) Use $\sec^2x=1+\tan^2x~(=1+u^2)$ to replace the leftover secants.
	\item \ffont{The power of tangent is odd ($n$ odd):}\\
		(a) Use $u=\sec x$ and $du=\sec x\tan x\,dx$.\\
		(b) Cancel one $\sec x$ and one $\tan x$ by the substitution of $dx$.\\
		\quad~~The number of remaining tangents is even.\\
		(c) Use $\tan^2x=\sec^2x-1~(=u^2-1)$ to replace the leftover tangents.
	\item \ffont{$m$ is even or $n$ is odd}:\\ Use either $1$ or $2$ (both will work).
	\item \ffont{The power of secant is odd and the power of tangent is even}:\\
	 No guidelines. Remember that $\ds\int\sec x\,dx$ and $\ds\int\sec^3 x\,dx$ can usually be looked up.
	\end{enumerate}
\end{formulabox}

%\begin{formulabox}[Integrals Involving Powers of Tangent and Secant]
%Consider $\ds\int\tan^mx\sec^nx\ dx$, where $m,n$ are nonnegative integers.\index{integration!of trig. powers}
%\begin{enumerate}
%\item		If $n$ is even, then $n=2k$ for some integer $k$. Rewrite $\sec^nx$ as
%$$\sec^nx = \sec^{2k}x = \sec^{2k-2}x\sec^2x = (1+\tan^2x)^{k-1}\sec^2x.$$
%Then
%$$\int\tan^mx\sec^nx\ dx=\int\tan^mx(1+\tan^2x)^{k-1}\sec^2x\ dx = \int u^m(1+u^2)^{k-1}\ du,$$
%where $u = \tan x$ and $du = \sec^2x\ dx$.
%
%\item		If $m$ is odd, then $m=2k+1$ for some integer $k$. Rewrite $\tan^mx\sec^nx$ as
%$$\tan^mx\sec^nx = \tan^{2k+1}x\sec^nx = \tan^{2k}x\sec^{n-1}x\sec x\tan x = (\sec^2x-1)^k\sec^{n-1}x\sec x\tan x.$$
%Then
%$$\int\tan^mx\sec^nx\ dx=\int(\sec^2x-1)^k\sec^{n-1}x\sec x\tan x\ dx = \int(u^2-1)^ku^{n-1}\ du,$$
%where $u = \sec x$ and $du = \sec x\tan x\ dx$.
%
%\item If $n$ is odd and $m$ is even, then $m=2k$ for some integer $k$. Convert $\tan^mx $ to $(\sec^2x-1)^k$. Expand the new integrand and use Integration By Parts, with $dv = \sec^2x\ dx$.
%
%\item		If $m$ is even and $n=0$, rewrite $\tan^mx$ as
%$$\tan^mx = \tan^{m-2}x\tan^2x = \tan^{m-2}x(\sec^2x-1) = \tan^{m-2}\sec^2x-\tan^{m-2}x.$$
%So
%$$\int\tan^mx\ dx = \underbrace{\int\tan^{m-2}\sec^2x\ dx}_{\text{\small apply rule \#1}}\quad - \underbrace{\int\tan^{m-2}x\ dx}_{\text{\small apply rule \#4 again}}.$$
%
%\end{enumerate}
%\end{formulabox}

The techniques described in items 1 and 2 are relatively straightforward, but the techniques in items 3 and 4 can be rather tedious. A few examples will help with these methods.\\

We can integrate $\tan x$ quite easily using substitution.\\

\begin{example}{Integrating Tangent}{int_tan}
Evaluate $\ds\int\tan x\,dx$.
\end{example}

\begin{solution}
Note that $\ds\tan x = \frac{\sin x}{\cos x}$ and let $u=\cos x$, so that $du=-\sin x\,dx$. %, i.e., $\fbox{$dx$}=\fbox{$\ds\frac{du}{-\sin x}$}$:
$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
\int \tan x\,dx & = & \int \frac{\sin x}{\cos x}\,\fbox{$dx$} & \mbox{Rewriting $\tan x$} \\
	&=&\int \frac{\sin x}{u}\,\fbox{$\ds\frac{du}{-\sin x}$}  & \mbox{Using the substitution}\\
	&=&-\int \frac{1}{u}\,du  & \mbox{Cancelling and pulling the $-1$ out}\\
	&=&-\ln|u|+C & \mbox{Using formula $\ds\int\frac{1}{u}\,dx=\ln|u|+C$}\\
	& = & -\ln|\cos x|+C & \mbox{Replacing $u$ back in terms of $x$}\\
	& = & \ln|\sec x|+C & \mbox{Using log properties and $\sec x=1/\cos x$}\\
\end{array}$$
\end{solution}

%Let's take a moment to realize this result! A common mistake is to believe that $\int\tan x\,dx$ is $\sec^2(x)+C$ -- this is \emph{not} true.

Higher poers of $ \tan(x) $ require the methods outlined above.

\begin{example}{Integrating Tangent Squared}{}
Evaluate $\ds\int\tan^2 x\,dx$.
\end{example}

\begin{solution}
The power on tangent is even, so we are in the fourth case outlined in the method. Ao we exploit the fact that   $\ds\tan^2 x = \sec^2x-1$.
$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
\int \tan^2 x\,dx & = & \int \sec^2x-1\,dx & \mbox{Rewriting $\tan x$} \\
	&=&\tan x-x+C & \mbox{Since $\ds\int\sec^2x\,dx=\tan x+C$}\\
\end{array}$$
\end{solution}

In problems with tangent and secant, two integrals come up frequently:
$$\ds \int\sec^3x\,dx\qquad\mbox{and}\qquad\int\sec x\,dx.$$
Both have relatively nice expressions but they are a bit tricky to discover.

First we do $\ds\int\sec x\,dx$, which we
will need to compute $\ds \int\sec^3x\,dx$.

\begin{example}{Integral of Secant}{Integral of Secant}
Evaluate $\ds\int\sec x\,dx$.
\end{example}

\begin{solution}
\begin{eqnarray*}
  \int\sec x\,dx&=&\int\sec x\,{\sec x +\tan x\over \sec x +\tan x}\,dx\cr
  &=&\int{\sec^2 x +\sec x\tan x\over \sec x +\tan x}\,dx.\cr
\end{eqnarray*}
Now let $u=\sec x +\tan x$, $\ds du=\sec x \tan x + \sec^2x\,dx$, exactly
the numerator of the function we are integrating. Thus
\begin{eqnarray*}
  \int\sec x\,dx&=&\int{\sec^2 x +\sec x\tan x\over \sec x +\tan x}\,dx\cr
	&=&\int{1\over u}\,du=\ln |u|+C\cr
  &=&\ln|\sec x +\tan x|+C.
\end{eqnarray*}
\end{solution}

Now we compute the integral $\ds \int\sec^3 x\,dx$.

\begin{example}{Integral of Secant Cubed}{Integral of Secant Cubed}
Evaluate $\ds\int\sec^3 x\,dx$.
\end{example}

\begin{solution}
\begin{eqnarray*}
  \sec^3x&=&{\sec^3x\over2}+{\sec^3x\over2}={\sec^3x\over2}+{(\tan^2x+1)\sec
    x\over 2}\cr
		\\
  &=&{\sec^3x\over2}+{\sec x \tan^2 x\over2}+{\sec x\over 2}\\
	\\
	&=&
  {\sec^3x+\sec x \tan^2x\over 2}+{\sec x\over 2}.
\end{eqnarray*}

We already know how to integrate $\sec x$, so we just need the first
quotient. This is ``simply'' a matter of recognizing the product rule
in action:
$$\int \sec^3x+\sec x \tan^2x\,dx=\sec x \tan x.$$
So putting these together we get
$$
  \int\sec^3x\,dx={\sec x \tan x\over2}+{\ln|\sec x +\tan x|\over2}+C,
$$
Note: Once we learn a technique called Integration by Parts, we will see another way to solve this integral.
\end{solution}

%For products of secant and tangent it is best to use the following guidelines.
%
%\begin{formulabox}[Products of Secant and Tangent]
%	When evaluating $\ds\int\sec^mx \tan^nx\,dx$:
%	\begin{enumerate}
%	\item \ffont{The power of secant is even ($m$ even):}\\
%		(a) Use $u=\tan x$ and $du=\sec^2 x\,dx$.\\
%		(b) Cancel $\sec^2 x$ by the substitution of $dx$, and be left with an even number of secants.\\
%		(c) Use $\sec^2x=1+\tan^2x~(=1+u^2)$ to replace the leftover secants.
%	\item \ffont{The power of tangent is odd ($n$ odd):}\\
%		(a) Use $u=\sec x$ and $du=\sec x\tan x\,dx$.\\
%		(b) Cancel one $\sec x$ and one $\tan x$ by the substitution of $dx$.\\
%		\quad~~The number of remaining tangents is even.\\
%		(c) Use $\tan^2x=\sec^2x-1~(=u^2-1)$ to replace the leftover tangents.
%	\item \ffont{$m$ is even or $n$ is odd}:\\ Use either $1$ or $2$ (both will work).
%	\item \ffont{The power of secant is odd and the power of tangent is even}:\\
%	 No guidelines. Remember that $\ds\int\sec x\,dx$ and $\ds\int\sec^3 x\,dx$ can usually be looked up.
%	\end{enumerate}
%\end{formulabox}

%\begin{example}{Even Power of Secant}{}
%Evaluate $\ds\int\sec^6x\tan^6x\,dx$.
%\end{example}
%
%\begin{solution}
%Since the power of secant is even, we ue $u=\tan x$, so that $du=\sec^2x\,dx$.
%$${\def\arraystretch{2.2}
%\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
%\int \sec^6x\tan^6 x\,dx &=&\int \sec^6x\,(u^6)\,\fbox{$\ds\frac{du}{\sec^2x}$}  & \mbox{Using the substitution}\\
%	&=&\int \sec^4x(u^6)\,du  & \mbox{Cancelling a $\sec^2x$}\\
%	&=&\int (\sec^2x)^2(u^6)\,du  & \mbox{Rewriting $\sec^4x$}\\
%	&=&\int (1+\tan^2x)^2(u^6)\,du  & \mbox{Using $\sec^2x=1+\tan^2x$}\\
%	&=&\int (1+u^2)^2(u^6)\,du  & \mbox{Using the substitution}\\
%	\end{array}
%}$$
%To integrate this product the easiest method is expand it into a polynomial and integrate term-by-term.
%$${\def\arraystretch{2.2}
%\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
%\int \sec^6x\tan^6 x\,dx	&=&\int ( u^6+2u^8+u^{10})\,du  & \mbox{Expanding}\\
%	&=&\frac{u^7}{7}+\frac{2u^9}{9}+\frac{u^{11}}{11}+C & \mbox{Integrating}\\
%	& = & \frac{\tan^7x}{7}+\frac{2\tan^9x}{9}+\frac{\tan^{11}x}{11}+C & \mbox{Rewriting in terms of $x$}\\
%\end{array}
%}$$
%\end{solution}

\begin{example}{Even Power of Secant}{intevensec}
Evaluate $\ds\int \tan^2x\sec^6x\ dx$.
\end{example}

\begin{solution}
Since the power of secant is even, we use rule \#1 and pull out a $\sec^2x$ in the integrand. We convert the remaining powers of secant into powers of tangent.
\begin{align*}
\int \tan^2x\sec^6x\ dx &= \int\tan^2x\sec^4x\sec^2x\ dx \\
		&= \int \tan^2x\big(1+\tan^2x\big)^2\sec^2x\ dx \\
\intertext{Now substitute, with $u=\tan x$, with $du = \sec^2x\ dx$.}
		&=\int u^2\big(1+u^2\big)^2\ du\\
\intertext{We leave the integration and subsequent substitution to the reader. The final answer is}
		&=\frac13\tan^3x+\frac25\tan^5x+\frac17\tan^7x+C.
\end{align*}
\end{solution}

%\begin{example}{Odd Power of Tangent}{}
%Evaluate $\ds\int\sec^5x\tan x\,dx$.
%\end{example}
%
%\begin{solution}
%Since the power of tangent is odd, we use $u=\sec x$, so that $du=  \sec x\tan x\,dx$.
%Then we have:
%$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
%\int \sec^5x\tan x\,dx &=&\int \sec^5x\tan x\,\fbox{$\ds\frac{du}{\sec x\tan x}$}  & \mbox{Substituting $dx$ first}\\
%	&=&\int \sec^4x\,du  & \mbox{Cancelling}\\
%	&=&\int u^4\,du  & \mbox{Using the substitution}\\
%	&=&\frac{u^5}{5}+C & \mbox{Integrating}\\
%	& = & \frac{\sec^5x}{5}+C & \mbox{Rewriting in terms of $x$}\\
%\end{array}$$
%\end{solution}

\begin{example}{Odd Power of Tangent}{intoddtan}
Evaluate $\ds\int\sec^5x\tan x\,dx$.
\end{example}

\begin{solution}
Since the power of tangent is odd, we use rule \# 2, and factor out $\sec x\tan x$ and substitute $ u=\sec(x)$.
Then we have:
$$\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
\int \sec^5x\tan x\,dx &=&\int \sec^4x\sec x\tan x\; dx  & \mbox{Substituting $dx$ first}\\
	%&=&\int \sec^4x\,du  & \mbox{Cancelling}\\
	&=&\int u^4\,du  & \mbox{Using the substitution}\\
	&=&\frac{u^5}{5}+C & \mbox{Integrating}\\
	& = & \frac{\sec^5x}{5}+C & \mbox{Rewriting in terms of $x$}\\
\end{array}$$
\end{solution}

\begin{example}{Odd Power of Secant and Even Power of Tangent}{}
Evaluate $\ds\int\sec x\tan^2x\,dx$.
\end{example}

\begin{solution}
The guidelines don't help us in this scenario. However, since $\ds\tan^2x=\sec^2x-1$, we have
$${\def\arraystretch{2.2}
\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
\int \sec x\tan^2x\,dx & = & \int \sec x(\sec^2x-1)  \,dx &  \\
	& = & \int (\sec^3x-\sec x)\,dx  &  \\
	& = & \frac{1}{2}\left(\sec x\tan x+\ln|\sec x+\tan x|\right) - \ln|\sec x+\tan x|+C &  \\
	& = & \frac{1}{2}\sec x\tan x+\frac{1}{2}\ln|\sec x+\tan x| - \ln|\sec x+\tan x|+C &  \\
	& = & \frac{1}{2}\sec x\tan x-\frac{1}{2}\ln|\sec x+\tan x|+C & \\
\end{array}
}$$
\end{solution}

\begin{example}{Integrating powers of tangent and secant}{ex_trigint7}
{
Evaluate $\ds\int\tan^6x\ dx$.
}
\end{example}

\begin{solution}
{We employ rule \#4.
\begin{align*}
\int \tan^6x\ dx &= \int \tan^4x\tan^2x\ dx \\
			&= \int\tan^4x\big(\sec^2x-1\big)\ dx\\
			&= \int\tan^4x\sec^2x\ dx - \int\tan^4x\ dx \\
\intertext{Integrate the first integral with substitution, $u=\tan x$; integrate the second by employing rule \#4 again.}
			&=	\frac15\tan^5x-\int\tan^2x\tan^2x\ dx\\
			&=	\frac15\tan^5x-\int\tan^2x\big(\sec^2x-1\big)\ dx \\
			&= \frac15\tan^5x -\int\tan^2x\sec^2x\ dx + \int\tan^2x\ dx\\
\intertext{Again, use substitution for the first integral and rule \#4 for the second.}
			&= \frac15\tan^5x-\frac13\tan^3x+\int\big(\sec^2x-1\big)\ dx \\
			&=	 \frac15\tan^5x-\frac13\tan^3x+\tan x - x+C.
\end{align*}
\vskip-\baselineskip
}
\end{solution}

Some of these examples were admittedly long, with repeated applications of the same rule. Try to not be overwhelmed by the length of the problem, but rather admire how robust this solution method is. A trigonometric function of a high power can be systematically reduced to trigonometric functions of lower powers until all antiderivatives can be computed.

The next section introduces an integration technique known as Trigonometric Substitution, a clever combination of Substitution and the Pythagorean Theorem.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:Powers of trigonometric functions}}

\begin{enumialphparenastyle}

Find the antiderivatives.

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin^2 x\,dx$
\begin{sol}
 $x/2-\sin(2x)/4+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin^3 x\,dx$
\begin{sol}
 $\ds -\cos x+(\cos^3x)/3+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin^4 x\,dx$
\begin{sol}
 $3x/8-(\sin 2x)/4+(\sin 4x)/32+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \cos^2 x\sin^3 x\,dx$
\begin{sol}
 $\ds (\cos^5 x)/5-(\cos^3x)/3+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \cos^3 x\,dx$
\begin{sol}
 $\ds \sin x-(\sin^3x)/3+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \cos^3 x \sin^2 x\,dx$
\begin{sol}
 $\ds (\sin^3x)/3-(\sin^5x)/5+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin x (\cos x)^{3/2}\,dx$
\begin{sol}
 $\ds -2(\cos x)^{5/2}/5+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \sec^2 x\csc^2 x\,dx$
\begin{sol}
 $\tan x-\cot x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \tan^3x \sec x\,dx$
\begin{sol}
 $\ds (\sec^3x)/3-\sec x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \left(\frac{1}{\csc x}+\frac{1}{\sec x}\right)\,dx$
\begin{sol}
 $\ds -\cos x+\sin x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int\frac{\cos^2x+\cos x+1}{\cos^3x}\,dx$
\begin{sol}
 $\ds \frac{3}{2}\ln|\sec x+\tan x|+\tan x+\frac{1}{2}\sec x\tan x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x\sec^2(x^2)\tan^4(x^2)\,dx$
\begin{sol}
 $\ds \frac{\tan^5(x^2)}{10}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

\end{enumialphparenastyle}


\subsection{Principle of proof by contradiction}

Let us suppose, that we want to (have to) prove the implication
\begin{equation}
 \label{eq:contradiction_implication}
 v_1 \Rightarrow v_2,
\end{equation}
where $v_1, v_2$ are statements\footnote{statement is a meaningful declarative sentence that is either true or false}. Suppose that "standart" direct proof in form
\begin{displaymath}
 v_1 \Rightarrow ~~~
 \hat{v}_1 \Rightarrow
 \dots
 \Rightarrow \hat{v}_m
 ~~~ \Rightarrow v_2
\end{displaymath}
is not suitable or too complicated. Here $\hat{v}_1, \dots, \hat{v}_m$ denote auxiliary statements presenting the small partial steps during the proof.

At first, let us recall the \emph{Truth table} of the implication and other interesting relationship between statements (i.e. composed statements).
In Table \ref{table:statements}, $1$ denotes that the statement is \emph{true}, $0$ denotes \emph{false}, and $v'$ is the negation of statement $v$.

\begin{table}[h]
%\def\arraystretch{1.3}
\centering
\begin{tabular}{C{0.05\linewidth} | C{0.05\linewidth} || C{0.1\linewidth} | C{0.1\linewidth} | C{0.1\linewidth} }
\hline
 $v_1$ & $v_2$ & $v_1 \Rightarrow v_2$ & $v_1' \vee v_2$ & $v_1 \wedge v_2'$ \\
\hline
 1 & 1 & 1 & 1 & 0 \\
 1 & 0 & 0 & 0 & 1 \\
 0 & 1 & 1 & 1 & 0 \\
 0 & 0 & 1 & 1 & 0 \\
\hline
\end{tabular}
\caption{Truth of selected composed statements; notice that 3th and 4th columns are equivalent, 5th is the negation of 3th and 4th column}
\label{table:statements}
\end{table}

Please, notice that from the table it is clear that statement $v_1 \Rightarrow v_2$ is equivalent to $v_1' \vee v_2$. However, instead of proving this statement (the first one
or the second one, it does not matter which one, since they are equivalent), we can prove that the negation  of this statement is not true
\footnote{because if the negation is not true, than the original statement is true}. The negation of implication is given by $v_1 \wedge v_2'$, see Table \ref{table:statements}.
To see the real relationship between $v_1' \vee v_2$ and $v_1 \wedge v_2'$, please, see also so-called \emph{De-Morgan's laws}.

Therefore, if we want to prove the implication \eqref{eq:contradiction_implication}, then we rather examine $v_1 \wedge v_2'$, i.e. we suppose, that the assumptions of the implication are true and the result in not true in the same time.
To prove that this statement is not true, it is enought to show that it does not hold in any case, i.e. we state the \emph{contradiction}.

Moreover, if we consider additional quantifiers (like \emph{for all} or \emph{there exists}) and our statements depend on the parameters,
then during the negation of original implication statement, we have to perform negation also to all quantifiers.
For example the negation of
\begin{displaymath}
 \forall \alpha: v_1(\alpha) \Rightarrow v_2(\alpha)
\end{displaymath}
is given by
\begin{displaymath}
 \exists \alpha: v_1(\alpha) \wedge v_2'(\alpha).
\end{displaymath}
So, if we prove that this $\alpha$ does not exist, then we are done.

\subsection{Absolute values in constraints}

\begin{theorem}
Let
\begin{itemize}
 \item $f: \mathbb{R}^n \rightarrow \mathbb{R}, f \in C^1(\mathbb{R}^n)$ be a convex function,
 \item $c \in \mathbb{R}^{+}$ is arbitrary constant,
 \item $S \in \mathbb{R}^{n,2n}$ is a matrix defined by
 \begin{displaymath}
  S :=
  \left[
   \begin{array}{ccccccc}
     1 & -1 & & & & & \\
      &  & 1 & -1 & & & \\
      &  & &  & \ddots & & \\
      &  &  &  & & 1 & -1
   \end{array}
  \right],
 \end{displaymath}
 \item $h(y) := f(Sy), h: \mathbb{R}^{2n} \rightarrow \mathbb{R}$.
\end{itemize}
Then if there exists a solution of optimization problem
\begin{equation}
 \label{eq:absvalconstr1}
 \bar{y} := \arg \min h(y) ~~~ \textrm{subject to} ~~~ \sum\limits_{i=1}^{2n} y_i \leq c ~~ \textrm{and} ~~ y \geq 0,
\end{equation}
then $\bar{x} := S \bar{y}$ is a solution of optimization problem
\begin{equation}
 \label{eq:absvalconstr2}
 \bar{x} = \arg \min f(x) ~~~ \textrm{subject to} ~~~ \Vert x \Vert_1 \leq c.
\end{equation}
Moreover
\begin{equation}
 \label{eq:absvalconstr3}
\forall i = 1,\dots,n: \bar{y}_{2i-1} \cdot \bar{y}_{2i} = 0.
\end{equation}
\end{theorem}

\begin{proof}
At first, notice that $h \in C^1(\mathbb{R}^{2n})$ is also a convex function and (here $\langle g(x),x=\hat{x} \rangle$ denotes function value of $g(x)$ in point $\hat{x}$)
\begin{equation}
  \label{eq:absvalconstr4}
  \langle \nabla h(y), y = \hat{y} \rangle =
  \langle \nabla f(Sy), y = \hat{y} \rangle =
  \langle S^T \nabla f(x), x = S \hat{y} \rangle.
\end{equation}
Since we suppose existence of $\bar{y}$ as a solution of \eqref{eq:absvalconstr1}, then from necessary optimality condition (see \todo{give cite Boyd, Vandenberghe: Convex Optimization, page 139, equation 4.21})
we get (here $\langle v,w \rangle$ denotes scalar product of vectors $v,w \in \mathbb{R}^m$)
\begin{equation}
 \label{eq:absvalconstr5}
  \langle \nabla h(\bar{y}), y - \bar{y} \rangle \geq 0, ~~~ \forall y \in \Omega_y,
\end{equation}
where $\Omega_y \subset \mathbb{R}^{2n}$ is a feasible set of \eqref{eq:absvalconstr1}. Using \eqref{eq:absvalconstr4}, we can write condition \eqref{eq:absvalconstr5}
in form
\begin{equation}
 \label{eq:absvalconstr6}
  \langle S^T \nabla f(S \bar{y}), y - \bar{y} \rangle \geq 0, ~~~ \forall y \in \Omega_y.
\end{equation}
Let us denote $\bar{x} := S \bar{y}$. Using the definition of scalar product
\begin{displaymath}
 \forall v,w \in \mathbb{R}^m: \langle v,w \rangle = v^T w = \sum\limits_{i = 1}^{m} v_i w_i,
\end{displaymath}
we can write the left part of \eqref{eq:absvalconstr6} in form
\begin{equation}
 \label{eq:absvalconstr7}
 \langle S^T \nabla f(\bar{x}), y - \bar{y} \rangle
 = \left( S^T \nabla f(\bar{x}) \right)^T \left( y - \bar{y} \right)
 = \left( \nabla f(\bar{x}) \right)^T S \left( y - \bar{y} \right)
 = \langle \nabla f(\bar{x}), Sy - \bar{x} \rangle.
\end{equation}
It remains to prove the relationship between $\Omega_x$ (feasible set of \eqref{eq:absvalconstr2}) and $\Omega_y$ (feasible set of \eqref{eq:absvalconstr1}) in form
\begin{equation}
 \label{eq:absvalconstr8}
 \begin{array}{ll}
  \forall y \in \mathbb{R}^{2n}: & y \in \Omega_y ~~ \Rightarrow ~~ Sy \in \Omega_x, \\
  \forall x \in \mathbb{R}^{n}: & x \in \Omega_x ~~ \Rightarrow ~~ \exists ! y \in \Omega_y: Sy = x,
 \end{array}
\end{equation}
because if \eqref{eq:absvalconstr8} holds, then we are able to write \eqref{eq:absvalconstr6} using \eqref{eq:absvalconstr7} in form
\begin{displaymath}
 \forall x \in \Omega_x: \langle \nabla f(\bar{x}), x - \bar{x} \rangle \geq 0,
\end{displaymath}
that concludes that $\bar{x} \in \Omega_x$ is solution of \eqref{eq:absvalconstr5}.\newline

\vspace{0.5cm}

\noindent To prove the first statement of \eqref{eq:absvalconstr8}, let us consider $y \in \Omega_y$, i.e. $y \in \mathbb{R}^{2n}$ such that
\begin{equation}
 \label{eq:absvalconstr9}
 \sum\limits_{i = 1}^{2n} y_i \leq c, y \geq 0.
\end{equation}
In following, we examine the L1-norm of $x := Sy$
\begin{equation}
 \label{eq:absvalconstrst}
 \Vert x \Vert_1 = \Vert Sy \Vert_1 = \Vert [y_2 - y_1, y_4 - y_3, \dots, y_{2n} - y_{2n-1}]^T \Vert_1 = \sum\limits_{i =1}^n \vert y_{2i} - y_{2i-1} \vert
\end{equation}
using auxiliary inequality; please, notice that
\begin{displaymath}
 \forall \alpha, \beta \in \mathbb{R}: \vert \alpha - \beta \vert \leq \vert \alpha \vert + \vert \beta \vert,
\end{displaymath}
therefore we can estimate \eqref{eq:absvalconstrst}
\begin{displaymath}
 \sum\limits_{i = 1}^{n} \vert y_{2i} - y_{2i-1} \vert \leq \sum\limits_{i=1}^n \left( \vert y_{2i} \vert + \vert y_{2i-1} \vert \right) = \sum\limits_{i=1}^{2n} \vert y_i \vert .
\end{displaymath}
Since we assume that $y \in \Omega_y$, i.e. \eqref{eq:absvalconstr9}, we can continue with estimation
\begin{displaymath}
 \sum\limits_{i=1}^{2n} \vert y_i \vert = \sum\limits_{i=1}^{2n} y_i  \leq c,
\end{displaymath}
i.e. $\Vert x \Vert_1 \leq c$ and therefore $x \in \Omega_x$. \newline

\vspace{0.5cm}

\noindent Now we prove the second statement of \eqref{eq:absvalconstr8}. Suppose $x \in \Omega_x$, i.e.
\begin{equation}
 \label{eq:absvalconstr12}
 \Vert x \Vert_1 = \sum\limits_{i=1}^n \vert x_i \vert \leq c.
\end{equation}
Please, notice that for any $\alpha \in \mathbb{R}$, there exist $\alpha^{+}, \alpha^{-} \geq 0$ such that
\begin{equation}
 \label{eq:absvalconstr10}
 \begin{array}{rcl}
  \alpha & = & \alpha^{+} - \alpha^{-}, \\
  \vert \alpha \vert & = & \alpha^{+} + \alpha^{-},
 \end{array}
\end{equation}
(for instance
if $\alpha \geq 0$, then we can choose $\alpha^{+}:=\alpha, \alpha^{-}:=0$;
if $\alpha < 0$, then we can choose $\alpha^{+}:=0, \alpha^{-}:= -\alpha$ )
\footnote{in fact, in the end of the proof it will be clear that this decomposition is only one possible choice}.
Using \eqref{eq:absvalconstr10}, we can decompose each component $x_i, i=1,\dots,n$ into
\begin{equation}
 \label{eq:absvalconstr11}
 \begin{array}{rcl}
  x_i & = & y_{2i-1} - y_{2i}, \\
  \vert x_i \vert & = & y_{2i-1} + y_{2i},
 \end{array}
\end{equation}
where $y \in \mathbb{R}^{2n}, y \geq 0$ is a new vector.
We examine the first condition of \eqref{eq:absvalconstr9} using the second equality of \eqref{eq:absvalconstr11}
and assumption \eqref{eq:absvalconstr12}
\begin{displaymath}
 \sum\limits_{i=1}^{2n} y_i = \sum\limits_{i =1}^{n} y_{2i-1} + y_{2i} = \sum\limits_{i=1}^{n} \vert x_i \vert \leq c.
\end{displaymath}
Therefore, we can state that $y \in \Omega_y$. Moreover, notice that from the first equality in \eqref{eq:absvalconstr11}, it holds $x = Sy$.

\vspace{0.5cm}

\noindent To prove \eqref{eq:absvalconstr3}, it is sufficient to show that \eqref{eq:absvalconstr10} gives us $\alpha^{+} \cdot \alpha^{-} = 0$
\footnote{notice that we do not need assumption $\alpha^{+},\alpha^{-} \geq 0$},
i.e.
\begin{equation}
 \label{eq:absvalconstr17}
 \left.
 \begin{array}{rcl}
  \alpha & = & \alpha^{+} - \alpha^{-} \\
  \vert \alpha \vert & = & \alpha^{+} + \alpha^{-}
 \end{array}
 \right\rbrace ~~~ \Rightarrow
 ~~~ \alpha^{+} \cdot \alpha^{-} = 0.
\end{equation}
Suppose by contradiction that $\alpha^{+} \cdot \alpha^{-} \neq 0$, i.e.
\begin{equation}
 \label{eq:absvalconstr13}
 \alpha^{+} \neq 0 ~~~~ \textrm{and} ~~~~ \alpha^{-} \neq 0.
\end{equation}
Now we get rid of absolute value in system in assumption of \eqref{eq:absvalconstr17}.
We examine both of possible situations:
\begin{itemize}
 \item if $\alpha \geq 0$, then the solution of system
  \begin{displaymath}
   \begin{array}{rcl}
    \alpha & = & \alpha^{+} - \alpha^{-}, \\
    \alpha& = & \alpha^{+} + \alpha^{-},
   \end{array}
  \end{displaymath}
  is given by $\alpha^{+} = \alpha, \alpha^{-} = 0$, which is a contradition with \eqref{eq:absvalconstr13}.
 \item if $\alpha < 0$, then the solution of system
  \begin{displaymath}
   \begin{array}{rcl}
    \alpha & = & \alpha^{+} - \alpha^{-}, \\
    -\alpha& = & \alpha^{+} + \alpha^{-},
   \end{array}
  \end{displaymath}
  is given by $\alpha^{+} = 0, \alpha^{-} = -\alpha$, which is a contradition with \eqref{eq:absvalconstr13}.
\end{itemize}

We proved that the solution of the system \eqref{eq:absvalconstr11} is unique, therefore for each $x \in \Omega_x$ there exists unique $y \in \Omega_y$ and $x = Sy$.

\end{proof}

\begin{example}
Let us consider the basic quadratic programming problem - the projection (Euclidean) onto feasible set. In this case, we will consider a feasible set described by the L1-norm.
On this simple example, we demonstrate the computation of gradient in equation \eqref{eq:absvalconstr4}.
For any $p \in \mathbb{R}^2$, we define projection onto set $\Omega$ as a solution of optimization problem
\begin{displaymath}
 P_{\Omega} (p) = \arg \min\limits_{x \in \Omega} \Vert x - p \Vert, ~~~ \Omega := \lbrace x \in \mathbb{R}^2: \Vert x \Vert_1 \leq 1 \rbrace
\end{displaymath}
(find the point from $\Omega$ which is the nearest to given arbitrary $p$).
Notice that the problem is equivalent to
\begin{displaymath}
 P_{\Omega} (y) = \arg \min\limits_{x \in \Omega} \frac{1}{2}\Vert x - p \Vert^2 = \arg \min\limits_{x \in \Omega} \underbrace{\frac{1}{2} x^T I x - p^T x}_{=: f(x)}
\end{displaymath}
with gradient
\begin{displaymath}
 \nabla f(x) = x - p.
\end{displaymath}
Let us define the function $h: \mathbb{R}^{4n} \rightarrow \mathbb{R}$ by
\begin{displaymath}
 h(y) := f(Sy) = \frac{1}{2} (Sy)^T I Sy - p^T Sy = \frac{1}{2} y^T S^T S y - (S^Tp)^T y.
\end{displaymath}
The gradient of this function is given by
\begin{displaymath}
 \nabla h(y) = S^T S y - S^Tp = S^T ( Sy-p) = S^T \nabla f(Sy).
\end{displaymath}
Instead of solving the original problem, we will rather solve
\begin{displaymath}
 \hat{y} = \arg \min\limits_{y \in \Omega_y} \frac{1}{2} y^T S^T S y - (S^Tp)^T y, ~~~ \Omega_y := \lbrace y \in \mathbb{R}^4: y_1 + y_2 + y_3 + y_4 \leq c ~ \vee ~ y \geq 0 \rbrace
\end{displaymath}
and the solution of original problem will be given by $P(p) = S\hat{y}$. Notice that $S^TS$ is SPS Hessian matrix of $h$, therefore $h$ is convex function (but not strictly convex).
\end{example}

\subsection{QP solvability}


\section{Ordinary differential equations}

For the clearest description of the model, we refer the reader to our code repository, because our object-oriented approach to software development is intended to be highly transparent and readable. For those who prefer dynamical systems such as those presented in the form of ordinary differential equations, we present the following.

    \[\frac{dS_{a}}{dt}=-\lambda_{a}(t)\times\sigma_{a}\times S_{a}\]
    \[\frac{dE_{a}}{dt}=\lambda_{a}(t)\times\sigma_{a}\times S_{a}-\alpha E_{a}\]
    \[\frac{dP_{a,c}}{dt}=p_{a,c}(t)\times \alpha E_{a}-\nu P_{a,c}\]
    \[\frac{dI_{a,c}}{dt}=\nu P_{a,c}-\gamma_{c}I_{a,c}\]
    \[\frac{dL_{a,c}}{dt}=\gamma_{c}I_{a,c}-\delta_{a,c}L_{a,c}-\mu_{a,c}L_{a,c}\]
    \[\frac{dR_{a}}{dt}=\sum_{c}{}\delta_{a,c}L_{a,c}\]
    where
    \[\lambda_{a}=\beta\left[\sum_{j,c}\frac{\epsilon\times P_{j}}{N_{j}}\times C_{a,j}(t)+\sum_{j,c}\frac{I_{j,c}\times\iota_{c}+L_{j,c}\times\kappa_{c}}{N_{j}}\times C_{a,j}(t)\right]\]
    \[\sum_{c}p_{a,c}(t)=1,\forall t\in\mathbb{R}\]
    \[\textbf{C}_{0}=\textbf{C}_{H}+\textbf{C}_{S}+\textbf{C}_{W}+\textbf{C}_{L}\]
    \[\textbf{C}(t)=h(t)\times\textbf{C}_{H}+s(t)\times\textbf{C}_{S}+w(t)\times\textbf{C}_{W}+l(t)\times\textbf{C}_{L}\]
    \[l(t)=\frac{re(t)+gr(t)+pa(t)+tr(t)}{4}\]

\begin{table}[t]

    \begin{tabular}{| p{3.4cm} | p{10.4cm} |}
        \hline
        \textbf{Symbol} & \textbf{Explanation} \\
        \hline
        \textit{S} & Persons susceptible to infection \\
   		\textit{E} & Persons in the non-infectious incubation period \\
    	\textit{P} & Persons in the incubation period \\
   		\textit{I} & Persons in the early active disease period, before isolation or 			   			hospitalisation may occur \\
    	\textit{L} & Persons in the late active disease period, after isolation or hospitalisation 			may have occurred \\
    	\textit{R} & Persons in the recovered period, from which re-infection cannot occur\\
     	\hline
    \end{tabular}

\end{table}

\clearpage
\begin{table}[t]

    \begin{tabular}{| p{3.4cm} | p{10.4cm} |}
	\hline
	\textbf{Symbol} & \textbf{Explanation} \\
	\hline
    \textit{t} & Time  \\
    {\textit{a}} & Compartment of age group a \\
    {\textit{c}} & Compartment of clinical stratification c \\
    $\sigma$ & Relative susceptibility to infection \\
    $\alpha$ & Rate of progression from non-infectious to infectious incubation period \\
    $\nu$ & Rate of progression from infectious incubation to early active disease \\
    $\gamma$ & Rate of progression from early active disease to late active disease \\
    $\mu$ & Rate of disease-related death \\
    $\epsilon$ & Relative infectiousness of pre-symptomatic compartment \\
    $\iota$ & Clinical stratification infectiousness vector for early active compartment \\
    $\kappa$ & Clinical stratification infectiousness vector for late active compartments \\
    $\beta$ & Probability of infection per contact between an infectious and susceptible individual \\
    \textit{j} & Infectious populations \\
    \textit{p} & Proportion progressing to each clinical stratification \\
    \textbf{C} & Mixing matrix \\
    \textbf{H} & Household contribution to mixing matrix \\
    \textbf{W} & Workplace contribution to mixing matrix \\
    \textbf{O} & Other locations contribution to mixing matrix \\
    \textbf{S} & Schools contribution to mixing matrix \\
    \textit{l} & Other locations macrodistancing function of time \\
    \textit{w} & Function fit to Google mobility data for workplaces \\
    \textit{s} & Function fit to Google mobility data for schools \\
    \textit{re} & Function fit to Google mobility data for retail and recreation \\
    \textit{gr} & Function fit to Google mobility data for grocery and pharmacy \\
    \textit{pa} & Function fit to Google mobility data for parks \\
    \textit{tr} & Function fit to Google mobility data for transit stations \\
    \hline
    \end{tabular}

\end{table}

\clearpage


\section{Scheduling for M/G/1}
\label{sec:Scheduling-M-G-1}

The scheduling policy can hugely influence the mean response time and other metrics.
Scheduling policies can be categorized based on whether the policy is preemptive or not, and whether it assumes knowledge of job size or not.

A \textit{work-conserving} scheduling policy is one that always performs work on some job when there is a job in the system, and does not create new work.
Given an arrival sequence and a time, all work-conserving policies have the same work left in system and server utilization.This does not imply that all work-conserving policies have the same mean response time.

When evaluating a scheduling policy, we take into account the following metrics:

\begin{description}

	\item [Response Time ($T$)]
	We denote with $T(x)$ the response time for job size $x$.

	\item [Queueing Time ($T_{Q}$)]

	\item [System Jobs ($N$)]

	\item [Queueing Jobs ($N_{Q}$)]

	\item [Slowdown ($Slowdown$)]
	\begin{equation}
	\label{eqn:Slowdown}
	Slowdown=\frac{T}{S}
	\end{equation}
	We denote with $Slowdown(x)$ the slowdown for job size $x$.

	\item [Tail Behavior ($Tail-Behavior$)]
	\begin{equation}
	\label{eqn:Tail-Behavior}
	Tail-Behavior=\probability{T>x}
	\end{equation}

	\item [Starvation]
	We say that policy $\mathit{P}$ produces \textit{starvation} if for some $x$
	\begin{equation}
	\label{eqn:Starvation}
	\expected{Slowdown^{\mathit{P}}(x)}>\expected{Slowdown^{\mathit{PS}}(x)}
	\end{equation}
	where $\mathit{PS}$ is the Processor-Sharing scheduling policy.

	\item [Fairness]
	We say that policy $\mathit{P}$ is fair if for all $x$
	\begin{equation}
	\label{eqn:Fairness}
	\expected{Slowdown^{\mathit{P}}(x)}<\expected{Slowdown^{\mathit{PS}}(x)}
	\end{equation}
	where $\mathit{PS}$ is the Processor-Sharing scheduling policy.
	Alternatively, we say that policy $\mathit{P}$ is fair if $Slowdown^{\mathit{P}}(x)$ is equal for all $x$.

\end{description}

More often we consider absolute mean metrics and mean metrics with respect to job sizes.
Sometimes however, variance in response time and variance in slowdown are more important than the respective means.

Few books analyze scheduling policies in stochastic environment: the best are \cite{conway2012theory,kleinrock1976queueing}.
The fairness metric we used is defined in \cite{bansal2001analysis}. The slowdown metric has received attention only recently in \cite{hyytia2012minimizing}.

In the following sections, we evaluate various scheduling policies for $M/G/1$. We base our evaluation on $\expected{T},\expected{T(x)},\expected{Slowdown(x)}$.

Notice that $\expected{Slowdown}\neq\frac{\expected{T}}{\expected{S}}$. We derive the Slowdown as follows:

\begin{equation*}
\expected{Slowdown(x)}=\expected{\frac{T}{S}|S=x}=\expected{\frac{T(x)}{x}}=\frac{\expected{T(x)}}{x}
\end{equation*}

and

\begin{equation*}
\expected{Slowdown}=\int_{x}\expected{Slowdown(x)}f_{S}(x)\partial x=\int_{x}\frac{\expected{T(x)}}{x}f_{S}(x)\partial x
\end{equation*}

The \textit{job age} is the total service it has received so far. If job size distribution has decreasing failure rate, then the greater the job age, the greater its expected remaining service time.


\documentclass{beamer}

\input{../../shared_slides.tex}

\title{Matrix games}
\date{\today}

\begin{document}
\maketitle
\frame{\tableofcontents}

\section{Introduction}%

\begin{frame}
  \frametitle{Introduction}
  Given
  \begin{itemize}
    \item Player I (rows, Alice)
          \item Player II (columns, Bob)
          \item a \emph{payoff} matrix $A \in \R^{m \times n}$
  \end{itemize}
  Every round
  \begin{enumerate}
    \item Alice picks (row) strategy $i\in [m]:= \{1,\dots, m\}$
          Bob picks (col) strategy $j\in [n]$
    \item Bob pays Alice the amount $a_{i,j}$
  \end{enumerate}
  \begin{center}
    \textbf{zero-sum game}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Example: penalty game}
  \begin{figure}[ht]
    \centering
    \includegraphics[scale=0.4]{penalty-game}
    \caption{penalty game}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Example: prisoners dilemma}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=0.5\textwidth]{prison}
    \caption{prisoners dilemma}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Worst case}

  \begin{itemize}
    \item if Alice chooses strategy $i$ she gets (at least): $\min_{j\in [n]} a_{i,j}$
    \onslide<2->{\item Alice can ensure payoff $\max_{i\in [m]} \min_{j\in [n]} a_{i,j}$}
    \onslide<3->{\item Bob pays (at most) $\min_{j\in [n]} \max_{i\in [m]} a_{i,j}$}
  \end{itemize}
  \onslide<4->{%
    We claim:
    \begin{equation}
      \max_i \min_j a_{i,j} \le \min_{j} \max_i a_{i,j}
    \end{equation}
    \begin{center}
      \textit{``Tallest dwarf is not as tall as the smallest giant.''}
    \end{center}
    But: \textbf{No equality in general!}
  }
\end{frame}

\begin{frame}
  \frametitle{Proof of the min-max theorem}
  \begin{equation}
    \begin{aligned}
      a_{ij} &\le a_{ij} & \forall i,j \\
      \onslide<2->{a_{ij} &\le \max_i a_{ij} & \forall i,j }\\
      \onslide<3->{\min_j a_{ij} &\le \min_j \max_i a_{ij} & \forall i}\\
    \end{aligned}
  \end{equation}
  \onslide<4->{%
  \begin{definition}
    We call $(i^*, j^*)$ a saddle point (or \emph{Nash equilibrium}) if
    \begin{equation}
      \max_i a_{ij^*} = a_{i^*j^*} = \min_j a_{i^*j}.
    \end{equation}
    These are called \emph{pure strategies}.
  \end{definition}
  }
\end{frame}

% Talk about stackelberg?

\begin{frame}
  \frametitle{Rock paper scissors}
  \begin{equation}
    \begin{array}{l|ccc}
        & R & P & S \\
      \hline
      R & 0 & -1 & 1 \\
      P & 1 & 0 & -1 \\
      S & -1 & 1 & 0
    \end{array}
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Mixed Strategies}
  \textit{With pure strategies we do not always have a saddle point.}

  \begin{block}{von Neumann (1928) --- Mixed strategies}
    \begin{itemize}
      \item Alice picks strategies $1, \dots, m$ \emph{with probabilities} $x\in \Delta_m$
      \item Bob picks strategies $1, \dots, n$ \emph{with probabilities} $x\in \Delta_n$
    \end{itemize}
    Expected gain of Alice is
    \begin{equation}
      \langle x, Ay \rangle = \sum_{i,j} a_{ij} x_i y_j
    \end{equation}
  \end{block}
  \onslide<2->{%
    \begin{theorem}{Saddle point exists}
      Expected gain of Alice $=$ expected loss of Bob
      \begin{equation}
        \max_{x \in \Delta} \min_{y \in \Delta} \langle x, Ay \rangle = \min_{y \in \Delta} \max_{x \in \Delta} \langle x, Ay \rangle.
      \end{equation}

    \end{theorem}
  }
\end{frame}

\section{Algorithms}%
\label{sec:}

\begin{frame}
  \frametitle{Stopping criteria}
  \begin{equation}
    \begin{aligned}
      \min_{x \in \Delta} \max_{y \in \Delta} \langle x, Ay \rangle =: v \\
      \max_{y \in \Delta} \min_{x \in \Delta} \langle x, Ay \rangle = v
    \end{aligned}
  \end{equation}
  \begin{block}{stopping criterion}
    \begin{equation}
      \begin{aligned}
        f_p(x) - f_p(x^*) &= f_p(x) - v \onslide<2->{\textcolor{red}{\, \le \epsilon/2}}\\
        f_d(y^*) - f_d(y) &= v - f_d(y) \onslide<2->{\textcolor{red}{\, \le \epsilon/2}} \\
        \onslide<3->{\Rightarrow f_p(x) - f_d(y) \le \epsilon}
      \end{aligned}
    \end{equation}
  \end{block}
  \begin{center}
    Before we never had the optimal value!
  \end{center}
  % how does this fit in?
  Solution will always be on the boundary:
  \begin{equation}
    f_p(x) = \max_{y \in \Delta} \langle A^T x, y \rangle = \max_{j} \langle A^T x, e_j \rangle
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{}
  Consider
  \begin{equation}
      \min_{x \in \Delta} \max_{y \in \Delta} \langle x, Ay \rangle
  \end{equation}
  as a minimization problem
  \begin{equation}
      \min_{x \in \Delta}  f_p(x) = \langle x, A y^* \rangle .
  \end{equation}
  Then, by the \textbf{first-order optimality condition}
  \begin{equation}
    x^* \in \argmin_{x \in \Delta} f_p(x)  \Leftrightarrow \langle \nabla f_p(x^*), x-x^* \rangle \ge 0 \quad \forall x \in \Delta
  \end{equation}
  Thus
  \begin{equation}
    \begin{aligned}
      \langle A^T y^*, x-x^* \rangle &\ge 0 \quad \forall  x \in \Delta \\
      \langle -A x^*, y-y^* \rangle &\ge 0 \quad \forall  y \in \Delta \\
    \end{aligned}
  \end{equation}
  Concatenate the two conditions to get
  \begin{equation}
  \left\langle \begin{bmatrix}
      0 & A^T \\
      -A & 0
    \end{bmatrix}
    \left(\begin{array}{c}
      x^*\\ y^*
    \end{array}  \right),
    \left(\begin{array}{c}
      x \\ y
    \end{array}  \right)
    -
    \left(\begin{array}{c}
      x^* \\
      y^*
    \end{array} \right)
 \right\rangle \ge 0.
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Games as Variational Inequalities}
  We had:
  \begin{equation}
  \left\langle \begin{bmatrix}
      0 & A^T \\
      -A & 0
    \end{bmatrix}
    \left(\begin{array}{c}
      x^*\\ y^*
    \end{array}  \right),
    \left(\begin{array}{c}
      x \\ y
    \end{array}  \right)
    -
    \left(\begin{array}{c}
      x^* \\
      y^*
    \end{array} \right)
 \right\rangle \ge 0.
  \end{equation}
  By rewriting $z=(x,y)$ and $F(z) = [A^T y; Ax]$, then
  \begin{equation}\tag{VI}
    \label{eq:VI}
    \langle F(z^*), z-z^* \rangle \ge 0 \quad \forall z \in \Delta_n \times \Delta_m =: C
  \end{equation}
  \begin{center}
    \textbf{Variational inequality}
  \end{center}

  \begin{block}{}
  If $F = \nabla \phi$ then~\eqref{eq:VI} would be equivalent to
  \begin{equation}
    \min_{z\in C} \, \phi(z)
  \end{equation}
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Potential - integrability}

  \textbf{Question:} Does there exist a potential $\phi$ for $F$, such that $F = \nabla \phi$

  \begin{block}{Integrability condition (from calculus)}
    Is the case if
    \begin{equation}
      \frac{\partial \phi}{\partial x \partial y} = \frac{\partial \phi}{\partial y \partial x}
    \end{equation}
  \end{block}
  But
  \begin{equation}
    \frac{\partial F_1}{\partial y} = A^T \neq - A = \frac{\partial F_2}{\partial x}
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{VI as Fixed point equation}
  \begin{block}{}
    \vspace{-0.5cm}
  \begin{align}
    \langle F(z^*), z-z^* \rangle \ge 0 \quad \forall z \in C \\
    \Leftrightarrow z^* = P_C (z^* - F(z^*)) \label{eq:FP}\tag{FP}
  \end{align}
  \end{block}
  \begin{proof}
    Applying the property of the projection
    \begin{equation}
      \langle P_C(x)-x, x' - P_C(x) \rangle \ge 0 \quad \forall x' \in C
    \end{equation}
    with~\eqref{eq:FP}, gives
    \begin{equation}
    \langle z^* - (z^* - F(z^*)), z-z^* \rangle \ge 0 \quad \forall z \in C.\hfill \qedhere\qed
    \end{equation}
  \end{proof}
  \begin{itemize}
    \item should remind us of (projected) gradient descent
          \item when you see a fixed point equation: iterate!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{But is it any good?}
  \begin{equation}
    z_{k+1} = z_k - \alpha F(z_k)
  \end{equation}
  Then
  \begin{equation}
    \begin{aligned}
      \Vert z_{k+1} \Vert^2 &= \Vert z_k \Vert^2 - \underbrace{2 \alpha \langle F(z_k), z_k \rangle}_{=0} + \alpha^2 \Vert F(z_k) \Vert^2 \\
      &=\Vert z_k \Vert^2 + \alpha^2 \Vert F(z_k) \Vert^2 \\
    \end{aligned}
  \end{equation}
  Resulting in $\Vert z_{k+1} \Vert \ge \Vert z_k \Vert$.
  \begin{center}
    $\Rightarrow$ \textbf{No bueno!}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{}

We can still show
  \begin{theorem}
    complexity of $\mathcal{O}(1/\sqrt{k})$ for \textbf{averaged iterates} in terms of $f_p(x) - f_d(y)$.
  \end{theorem}

  \begin{itemize}
    \item with the same analysis as for subgradient descent
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Sketch of the proof}
  With the notation $g_k = F(z_k)$ we get
  \begin{equation}
    \begin{aligned}
      \Vert x_{k+1} - x^* \Vert^2 &\le \Vert x_k - \alpha_k g_k - x^* \Vert^2 \\
      &= \Vert x_k-x^* \Vert^2 + 2 \alpha_k \langle g_k, x^*-x_k \rangle + \alpha^2 \Vert g_k \Vert^2.
    \end{aligned}
  \end{equation}
  But now $\langle g_k, x^* - x_k \rangle = [f_d(y_k) - f_p(x_k)]$.
  Rest of the proof is left as an exercise.
\end{frame}

\end{document}


\chapter{Formal Power Series}
\label{chapter:formal-power-series}
Formal power series is an algebraic analogy of power series from analysis.
A formal power series is something like
$a_0 + x a_1 + x^2 a_2 + \dots a_n x^n + \dots$; to describe such an object it
is enough to define the sequence $\set{a_n}_{n \ge 0}$ since $x$ is a variable.
\begin{definition}
  We say that $F(x)$ is a \emph{formal power series} in the variable $x$, if
  $F(x) = \set{f_n}_{n \ge 0}$. To distinguish between formal power series and
  sequences, we write formal power series as $\sum_{n \ge 0} f_n x^n$.
  We say that $f_n$ is the coefficient of $x^n$ in $F(x)$.

  We say that two formal power series $F(x)$ and $G(x)$ are equal iff for all
  $n \ge  0$, the coefficients of $x^n$ in $F(x)$ and $G(x)$ are the same.

  The set of all the power series in the variable $x$ is denoted as
  $\R[[x]]$.
\end{definition}
\nomenclature[S]{$\R[[x]]$}{denotes the set all the power series in the
variable $x$}

\section{Arithmetic Operations}

We can perform all the standard operations with the formal power series:
\begin{gather*}
  \sum_{n \ge 0} a_n x^n \pm \sum_{n \ge 0} b_n x^n =
  \sum_{n \ge 0} (a_n \pm b_n) x^n, \\
  c \sum_{n \ge 0} a_n x^n =
  \sum_{n \ge 0} (c a_n) x^n,
  \\
  \text{and} \\
  \sum_{n \ge 0} a_n x^n \sum_{n \ge 0} b_n x^n =
  \sum_{n \ge 0} (\sum_{k = 0}^n a_{k} b_{n - k}) x^n.
\end{gather*}

These operations satisfy all the properties we may expect from them.
\begin{theorem}
  Let $F(x)$, $G(x)$, and $H(x)$ be some formal power series. Then the following
  equalities hold:
  \begin{itemize}
    \item $(F(x) + G(x)) + H(x) = F(x) + (G(x) + H(x))$,
    \item $F(x) + G(x) = G(x) + F(x)$,
    \item $(F(x) G(x)) H(x) = F(x) (G(x) H(x))$,
    \item $F(x) G(x) = G(x) F(x)$, and
    \item $(F(x) + G(x)) H(x) = F(x)H(x) + G(x)H(x)$.
  \end{itemize}
\end{theorem}

For example, $(1 - x) (1 + x + x^2 + \dots) = 1$. Thus we can say that the
series $(1 - x)$ has an inverse, and that inverse is equal to $1 + x + x^2 +
\dots$.
\begin{theorem}
  A formal power series $\sum_{n \ge 0} f_n x^n$ has an inverse iff
  $f_0 \neq 0$ and moreover this inverse is unique.
\end{theorem}
\begin{proof}
  Assume that a power series $F(x) = \sum_{n \ge 0} f_n x^n$ has an inverse
  $G(x) = \sum_{n \ge 0} g_n x^n$. In this case $F \cdot G = 1$ i.e.
  $f_0 g_0 = 1$ and $f_0 \neq 0$. Moreover,
  $\sum_{k = 0}^n f_k g_{n - k} = 0$; from which we can conclude that
  \begin{equation}
    \label{equation:inverse-of-formal-power-series}
    g_n = -\frac{1}{f_0} \sum_{k > 0} f_k g_{n - k}.
  \end{equation}
  This determine $g_n$ uniquely, as stated.

  Conversely, if $f_0 \neq 0$, (\ref{equation:inverse-of-formal-power-series})
  determines the sequence $\set{g_n}_{n \ge 0}$.
\end{proof}

\section{Composition}
Another operation we may need to perform is composition; a composition of
the power series $F(x)$ and $G(x)$ is a power series $F(G(x))$; i.e.
$F(G(x)) = \sum_{n \ge 0} a_n G^n(x)$, where $F(x) = \sum_{n \ge 0} a_n x^n$
Note that the composition is well-defined iff the coefficient of $x^0$ in $G(x)$
is $0$ or if $F(x)$ is a polynomial.

\section{Derivative}
Let $F(x) = \sum_{n \ge 0} f_n x^n$ be a formal power series. Then the
derivative $F'(x)$ (we also denote it as $\frac{d}{dx} F(x)$) of $F(x)$ is
equal to
$\sum_{n \ge 1} n f_n x^{n - 1} = \sum_{n \ge 0} (n + 1) f_{n + 1} x^n$.

The derivatives of formal power series satisfy the same properties as
derivatives of functions.
\begin{theorem}
  Let $F(x)$, $G(x)$, and $H(x)$ be some formal power series. Then the following
  equalities hold:
  \begin{itemize}
    \item $\frac{d}{dx}(F(x) + G(x)) = F'(x) + G'(x)$, and
    \item $\frac{d}{dx}(F(x) G(x)) = F'(x)G(x) + F(x)G'(x)$.
  \end{itemize}
\end{theorem}

As a corollary of these statements we can derive a formula for the derivative of
$1 / F(x)$.
\begin{corollary}
  Let $F(x)$ be a formal power series such that $1 / F(x)$ exists. In this
  case $\frac{d}{dx} \frac{1}{F(x)} = -\frac{F'(x)}{F^2(x)}$.
\end{corollary}
\begin{proof}
  Note that $F(x) \frac{1}{F(x)} = 1$. Hence,
  $\frac{d}{dx}(F(x) \frac{1}{F(x)}) = 0$. Using the formula for the derivative
  of a product we may conclude that $F'(x)\frac{1}{F(x)} +
  F(x)\frac{d}{dx}\frac{1}{F(x)} = 0$. As a result,
  $-\frac{F'(x)}{F^2(x)} = \frac{d}{dx} \frac{1}{F(x)}$.
\end{proof}

\begin{remark}
  If $F'(x) = 0$, then $F(x) = a_0$.
\end{remark}

We denote the formal power series $\sum_{n \ge 0} \frac{1}{n!} x^n$ by $e^x$
(since the Taylor series of $e^x$ is equal to
$\sum_{n \ge 0} \frac{1}{n!} x^n$).
\begin{remark}
  If $F'(x) = F(x)$, then $F(x) = c e^x$ for some $c \in \R$.
\end{remark}