\section{Derivation of the two stage promoter equation}

Shahreaei and Swain derive the full analytical protein distribution for a two
stage (i.e. an unregulated promoter) and a three stage (i.e. a promoter that
transitions between active and inactive) promoter \cite{Shahrezaei2008}. In
this section wi will follow the derivation augmenting the details at each step
for clarity.

\subsection{From the master equation}

First let us write the chemical master equation for this system. Let $p$ and $m$
be the protein and mRNA copy numbers, respectively, and $P_{m,p}(t)$ be the
probability of having $m$ mRNA and $p$ proteins at time t. Then we can use the
``spread the butter'' approach to write the discrete difference equation
\begin{equation}
\begin{aligned}
P_{m,p}(t + \Delta t) =
P_{m,p}(t) +
\overbrace{r_m \Delta t
\left[ P_{m-1,p}(t) - P_{m,p}(t) \right]}^\text{mRNA production}
+ \overbrace{r_p m \Delta t
\left[ P_{m, p-1}(t) - P_{m, p}(t) \right]}^\text{protein production}\\
+ \underbrace{\gamma_m \Delta t
\left[ (m + 1) P_{m+1,p}(t) - m P_{m, p}(t) \right]}_\text{mRNA degradation}
+ \underbrace{\gamma_p \Delta t
\left[ (p + 1) P_{m, p+1}(t) - p P_{m, p}(t) \right]}_\text{protein
degradation},
\end{aligned}
\end{equation}
where $r_m$ and $r_p$ are the mRNA and protein production rates respectively,
$\gamma_m$ and $\gamma_p$ are the mRNA and protein degradation rates
respectively, and $\Delta t$ is a time interval small enough so that only one
event can take place.

We rearrange the terms, divide both sides by $\Delta t$ , obtaining
\begin{equation}
  \begin{aligned}
\frac{P_{m,p}(t + \Delta t) - P_{m,p}(t)}{\Delta t} =
r_m \left[ P_{m-1,p}(t) - P_{m,p}(t) \right]
+ r_p m \left[ P_{m, p-1}(t) - P_{m, p}(t) \right]\\
+ \gamma_m \left[ (m + 1) P_{m+1,p}(t) - m P_{m, p}(t) \right]
+ \gamma_p \left[ (p + 1) P_{m, p+1}(t) - p P_{m, p}(t) \right].
  \end{aligned}
\end{equation}
Note that the production of protein is taken per mRNA.
We now take the limit $\Delta t \rightarrow 0$ to obtain the final form of the
chemical master equation
\begin{equation}
  \begin{aligned}
\frac{\partial P_{m,p}}{\partial t} =
r_m \left[ P_{m-1,p}(t) - P_{m,p}(t) \right] +
r_p m \left[ P_{m, p-1}(t) - P_{m, p}(t) \right]\\
+ \gamma_m \left[ (m + 1) P_{m+1,p}(t)
- m P_{m, p}(t) \right]
+ \gamma_p \left[ (p + 1) P_{m, p+1}(t) - p P_{m, p}(t) \right].
  \end{aligned}
\end{equation}

\mrm{Eq. 1 of the paper.}

Let us now divide by the slowest rate, i.e. $\gamma_p$
\begin{equation}
\begin{aligned}
\frac{1}{\gamma_p} \frac{\partial P_{m,p}}{\partial t} =
\frac{r_m}{\gamma_p} \left[ P_{m-1,p}(t) - P_{m,p}(t) \right]
+ \frac{r_p}{\gamma_p} m  \left[ P_{m, p-1}(t) - P_{m, p}(t) \right]\\
+ \frac{\gamma_m}{\gamma_p} \left[ (m + 1) P_{m+1,p}(t) - m P_{m, p}(t) \right]
+ \left[ (p + 1) P_{m, p+1}(t) - p P_{m, p}(t) \right].
\end{aligned}
\label{eq_cme_over_gammap}
\end{equation}
We now introduce the following variables:
\begin{align}
  a \equiv \frac{r_m}{\gamma_p}\\
  b \equiv \frac{r_p}{\gamma_m}\\
  \gamma \equiv \frac{\gamma_m}{\gamma_p}\\
  \tau \equiv \gamma_p \cdot t
\end{align}
Substituting these variables into \eref{eq_cme_over_gammap} we obtain
\begin{equation}
\begin{aligned}
\frac{\partial P_{m,p}}{\partial \tau} =
a \left[ P_{m-1,p}(t) - P_{m,p}(t) \right]
+ b \gamma m  \left[ P_{m, p-1}(t) - P_{m, p}(t) \right]\\
+ \gamma \left[ (m + 1) P_{m+1,p}(t) - m P_{m, p}(t) \right]
+ \left[ (p + 1) P_{m, p+1}(t) - p P_{m, p}(t) \right].
\end{aligned}
\label{eq_cme_tau}
\end{equation}
Note that we used $\frac{1}{\gamma_p}\frac{\partial}{\partial t} =
\frac{\partial}{\partial \tau}$.

We now define the generating function
\begin{equation}
F\left[ s, z \right] = \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p P_{m, p},
\label{eq_generating_function}
\end{equation}
where from now on we abbreviate $P_{m, p}(t)$ as $P_{m, p}$. The generating
function will allow us to write a single PDE rather than an infinite system of
PDEs for each mRNA and protein copy number. The generating function time
derivative is given by
\begin{equation}
\frac{\partial F}{\partial \tau} =
\frac{\partial}{\partial \tau} \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p
P_{m, p} =
\sum_{m=0}^{\infty}
\sum_{p=0}^{\infty} s^m z^p \frac{\partial}{\partial \tau} P_{m, p}.
\label{eq_dF_dtau}
\end{equation}
We now substitute \eref{eq_cme_tau} into \eref{eq_dF_dtau}
\begin{equation}
\begin{aligned}
\frac{\partial F}{\partial \tau} =
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p
\left\{ a \left[ P_{m-1,p} - P_{m,p} \right]
+ b \gamma m  \left[ P_{m, p-1} - P_{m, p} \right] \right. \\
\left. + \gamma \left[ (m + 1) P_{m+1,p} - m P_{m, p} \right]
+ \left[ (p + 1) P_{m, p+1} - p P_{m, p} \right] \right\}.
\end{aligned}
\label{eq_dF_dtau_complete}
\end{equation}

In order to make progress with this equation we note that we can distribute
the terms in parenthesis as
\begin{equation}
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p \left( P_{m+1, p} - P_{m, p}
\right) =
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p P_{m+1, p}
- \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p P_{m, p}.
\label{eq_split_sum}
\end{equation}
We can now factorize $s^{-1}$ from the first term on the left hand side of
\eref{eq_split_sum} and redefine the variable to sum over.
\begin{equation}
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p \left( P_{m+1, p} - P_{m, p}
\right) =
s^{-1} \sum_{(m+1)=0}^{\infty} \sum_{p=0}^{\infty} s^{m+1} z^p P_{m+1, p}
- \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p P_{m, p}.
\end{equation}
But since both sums on the left hand side are taken over the same ranges we can
write it as
\begin{equation}
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p \left( P_{m+1, p} - P_{m, p}
\right) =
\left( s^{-1} - 1 \right) \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p P_{m,
p}
\end{equation}

With this identity we can rewrite \eref{eq_dF_dtau_complete} as
\begin{equation}
\begin{aligned}
\frac{\partial F}{\partial \tau} =
(s - 1) \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p \left( a P_{m,p} \right)
+ (z - 1) \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p b \gamma m P_{m,p}
\\
+ \left( s^{-1} - 1 \right) \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p
\gamma m P_{m,p}
+ \left( z^{-1} - 1 \right) \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p
\gamma p P_{m,p}.
\end{aligned}
\label{eq_dF_dtau_identity}
\end{equation}

Another useful identity can be derived if we note that
\begin{equation}
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p m P_{m, p} =
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} z^p s \frac{\partial s^m}{\partial s}
P_{m, p}.
\end{equation}
But since $z^p$ and $P_{m, p}$ do not depend on $s$ we can include these terms
inside the derivative, obtaining
\begin{equation}
\sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p m P_{m, p} =
s \frac{\partial}{\partial s}\left( \sum_{m=0}^{\infty} \sum_{p=0}^{\infty} s^m z^p P_{m, p} \right) =
s \frac{\partial}{\partial s}F.
\end{equation}

Using this identity in \eref{eq_dF_dtau_identity} allow us to remove all the
sums, obtaining a single PDE of the form
\begin{equation}
\frac{\partial F}{\partial \tau} =
(s - 1) a F + (z - 1) b \gamma s \frac{\partial F}{\partial s}
+ \left( s^{-1} - 1 \right) \gamma s \frac{\partial F}{\partial s}
+ \left( z^{-1} - 1 \right) z \frac{\partial F}{\partial Z}.
\end{equation}
Rearranging terms we obtain
\begin{equation}
\frac{\partial F}{\partial \tau} =
a (sF - s)
+ b \gamma \left( z s \frac{\partial F}{\partial s} - s \frac{\partial
F}{\partial s} \right)
+ \gamma \left( \frac{\partial F}{\partial s} - s \frac{\partial F}{\partial s}
\right)
+ \left( \frac{\partial F}{\partial z} - z \frac{\partial F}{\partial z}
\right).
\label{eq_df_dtau_PDE}
\end{equation}

With the generating function we were able to pass from an infinite system of PDE
for each mRNA and protein copy number to a single PDE. We now have to find a
solution for this equation and then infer back the probability distribution
given the definition of the generating function.

We now note a pattern in \eref{eq_df_dtau_PDE}. If we define $u \equiv s - 1$ and $v \equiv z - 1$, which satisfy
\begin{align}
  \frac{\partial}{\partial s} = \frac{\partial}{\partial u},\\
  \frac{\partial}{\partial z} = \frac{\partial}{\partial v},
\end{align}
we can rewrite \eref{eq_df_dtau_PDE} as
\begin{equation}
  \frac{\partial F}{\partial \tau} =
  a u F
  + b \gamma v \left( u + 1  \right) \frac{\partial F}{\partial u}
  - \gamma u \frac{\partial F}{\partial u}
  - v \frac{\partial F}{\partial v}.
\end{equation}

We now divide by $v$ and rearrange terms obtaining
\begin{equation}
  \frac{\partial F}{\partial v}
  - \gamma \left[ b (u + 1) - \frac{u}{v} \right] \frac{\partial F}{\partial u}
  + \frac{1}{v} \frac{\partial F}{\partial \tau}
  = a \frac{u}{v} F,
  \label{eq_swain_2}
\end{equation}
\mrm{which is Eq. (2) in \cite{Shahrezaei2008}.}

\eref{eq_swain_2} is a semi-linear PDE that can be solved using the method of
the characteristics (this method is described in the Appendix
\ref{seq_method_characteristics}). In order to apply this method we write
\eref{eq_swain_2} as
\begin{equation}
  A(v, u, \tau) \frac{\partial F}{\partial v}
  + B(v, u, \tau) \frac{\partial F}{\partial u}
  + C(v, u, \tau) \frac{\partial F}{\partial \tau}
  = f(u, v, \tau, F),
\end{equation}
where
\begin{equation}
  \begin{aligned}
  A &= 1,\\
  B &= - \gamma \left[ b (1 + u) - \frac{u}{v} \right],\\
  C &= \frac{1}{v},\\
  f &= a \frac{u}{v} F.
  \label{eq_coef_definition}
  \end{aligned}
\end{equation}

The method of characteristics requires us to solve the PDE along the
characteristic lines. To do so we can parametrize the characteristics with
parameter $r$. This allow us to write the equality
\begin{equation}
  {{dv \over dr} \over A(v, u, \tau)} =
  {{du \over dr} \over B(v, u, \tau)} =
  {{d\tau \over dr} \over C(v, u, \tau)} =
  {{dF \over dr} \over f(v, u, \tau, F)}.
  \label{eq_lagrange_charpit}
\end{equation}
Substituting \eref{eq_coef_definition} into \eref{eq_lagrange_charpit} gives
\begin{equation}
  {{dv \over dr} \over 1} =
  {{du \over dr} \over - \gamma \left[ b (1 + u) - \frac{u}{v} \right]} =
  {{d\tau \over dr} \over {1 \over v}} =
  {{dF \over dr} \over a \frac{u}{v} F}.
\end{equation}
Or, if a particular parametrization $r$ of the curves is fixed, then these
equations can be written as a system of ODEs as
\begin{align}
  \frac{d v}{d r} &= 1,
  \label{eq_ODE_1}\\
  \frac{d u}{d r} &= - \gamma \left[ b (1 + u) - \frac{u}{v}
  \right],
  \label{eq_ODE_2}\\
  \frac{d \tau}{d r} &= \frac{1}{v},
  \label{eq_ODE_3}\\
  \frac{d F}{d r} &= \frac{a u}{v} F.
  \label{eq_ODE_4}
\end{align}

\subsubsection{Solving system of ODEs}

Once we find the system of ODEs that integrate the variables along the
characteristic line we can try to solve this system in order to find the general
integral surface that solves the PDE. The initial conditions for the system are
given by $\tau = 0$, $u = u_0$, $v = v_0$. Let's start with \eref{eq_ODE_1}.
For this equation we use separation of variables which simply gives
\begin{equation}
  {\partial v \over \partial r} = 1 \Rightarrow
  r = v + C_1,
  \label{eq_ODE_sol_1}
\end{equation}
where $C_1$ is an integration constant.

For \eref{eq_ODE_3} we can write
\begin{equation}
  {\partial \tau \over \partial r} = {1 \over v} \Rightarrow
  {\partial r \over \partial \tau} = v.
\end{equation}
If we now substitute \eref{eq_ODE_sol_1} we find
\begin{equation}
  {\partial r \over \partial \tau} = r - C_1.
\end{equation}
Using separation of variables on this equation we get
\begin{equation}
  \int {dr \over r - C_1} = \int d\tau,
\end{equation}
which when integrated gives
\begin{equation}
  \ln (r - C_1) = \tau + C_2'.
\end{equation}
where $C_2'$ is an integration constant. If we now solve for $r$ we find
\begin{equation}
  r = C_2e^{\tau} + C_1,
  \label{eq_ODE_sol_3}
\end{equation}
where $C_2 \equiv e^{C_2'}$.

If we substitute \eref{eq_ODE_sol_3} back on \eref{eq_ODE_sol_1} we obtain a
functional relationship between $v$ and $\tau$ of the form
\begin{equation}
  C_2 e^{\tau} + C_1 = v + C_1 \Rightarrow
  v = C_2 e^{\tau}.
\end{equation}

Using the initial conditions $\tau = 0$, $v=v_0$ we find that $C_2 = v_0$,
therefore the functional relationship between $\tau$ and $v$ is of the form
\begin{equation}
  v = v_0 e^{\tau}.
  \label{eq_v_tau_relation}
\end{equation}

Using \eref{eq_ODE_1} and \eref{eq_ODE_2} we can compute
$\partial u / \partial v$ as
\begin{equation}
  {\partial u \over \partial v} =
  - \gamma \left[ b (1 + u) - {u \over v} \right].
\end{equation}
This is a first order linear ODE which we can rewrite as
\begin{equation}
  {du \over dv} = -\gamma b
  - \gamma b u
  + {\gamma u \over v} =
  -\gamma b
  + u \left( {\gamma \over v} - \gamma b \right).
\end{equation}
Grouping terms with respect to $u$ gives
\begin{equation}
  {du \over dv} + \left( \gamma b - {\gamma \over v} \right) u = - \gamma b.
  \label{eq_dudv}
\end{equation}
\eref{eq_dudv} can be solved by the integrating factor method. In this case
the integrating factor is given by
\begin{equation}
  e^{\int \gamma \left( b - {1 \over v} \right) dv} =
  e^{\gamma b v - \gamma \ln v} = v^{-\gamma} e^{\gamma b v}
  \label{eq_dudv_integ_factor}
\end{equation}

Multiplying both sides of \eref{eq_dudv} by \eref{eq_dudv_integ_factor} we find
that the left hand side is transformed into the derivative of the integrating
factor times $u$, i.e.
\begin{equation}
  {d \over dv}\left[ v^{-\gamma} e^{\gamma b v} \cdot u \right] =
  -\gamma b v^{-\gamma} e^{-\gamma b v}.
\end{equation}

We can now integrate both sides, obtaining
\begin{equation}
u v^{-\gamma} e^{\gamma b v} =
-\gamma b \int dv \; v^{-\gamma} e^{\gamma b v} + C,
\end{equation}
where $C$ is an integrating constant. Solving for $u$ we find
\begin{equation}
  u(v) = v^{\gamma} e^{-\gamma b v} \left[ C -
  \gamma b \int dv {e^{\gamma b v} \over v^{\gamma}} \right].
  \label{eq_implicit_sol_u}
\end{equation}
\mrm{Eq. (26)}

To solve the integral we expand the exponential as a Taylor series, i.e.
\begin{equation}
  e^{-\gamma b v} = \sum_{n=0}^{\infty} {\left( \gamma b v \right)^n \over n!}.
\end{equation}

Using this on the integral in \eref{eq_implicit_sol_u} gives
\begin{equation}
  \int dv {e^{\gamma b v} \over v^{\gamma}} =
  \int dv {1 \over v^{\gamma}}
  \sum_{n=0}^{\infty} {\left( \gamma b v \right)^n \over n!}
\end{equation}
We can take out the terms that do not depend on $v$ out of the integral to
obtain
\begin{equation}
\int dv {1 \over v^{\gamma}}
  \sum_{n=0}^{\infty} {\left( \gamma b v \right)^n \over n!} =
  \sum_{n=0}^{\infty} {\left( \gamma b \right)^n \over n!}
  \int dv \; v^{n - \gamma} =
  \sum_{n=0}^{\infty} {\left( \gamma b \right)^n \over n!}
  {v^{n - \gamma + 1} \over n - \gamma + 1}.
  \label{eq_integral_implicit_sol_u}
\end{equation}

Substituting \eref{eq_integral_implicit_sol_u} into \eref{eq_implicit_sol_u}
gives
\begin{equation}
  u(v) = v^\gamma e^{-\gamma b v}
  \left[ C - \gamma b \sum_{n=0}^\infty {\left( \gamma b \right)^n \over n!}
  {v^{n - \gamma + 1} \over n - \gamma + 1} \right].
\end{equation}
This can be simplified to obtain
\begin{equation}
  u(v) = e^{-\gamma b v} \left[ C v^{\gamma}
  - \sum_{n=0}^\infty {\left( \gamma b v \right)^{n+1} \over
  n! \left( n - \gamma + 1 \right)} \right].
  \label{eq_SI_27_uofv}
\end{equation}
\mrm{Eq (27) in the SI.}

The question now becomes how can we evaluate the sum
\begin{equation}
  S(\gamma b v) = \sum_{n=0}^\infty {\left( \gamma b v \right)^{n+1} \over
  n! \left( n - \gamma + 1 \right)}.
\end{equation}
Just as Shahrenzaei \& Swain we will follow Bender \& Orszag and use the
so-called Laplace's method for sums. The idea behind this method is to find the
leading behavior of the sum. What this means is that if there are some very
dominant terms in the sum we can just add those in the vincinity to get a decent
approximation of the sum.

To find the leading behavior of the sum we first need to identify the largest
terms in the series. Let us start by looking at the ratio of the
$(n+1)^{\text{th}}$ term to the $n^{\text{th}}$ term of the sum. This is given
by
\begin{equation}
{{(\gamma b v)^{n+2} \over (n+1)! (n - \gamma + 2)}
\over
{(\gamma b v)^{n+1} \over n! (n - \gamma + 1)}}
=
{\gamma b v \over n + 1} \cdot
{(n - \gamma + 1) \over (n - \gamma + 2)}.
\label{eq_ratio_series}
\end{equation}
From this ratio we can see that since we assumed that $\gamma \gg 1$ we can
discard the second term on the right hand side of \eref{eq_ratio_series} since
\begin{equation}
{(n - \gamma + 1) \over (n - \gamma + 2)} \approx 1.
\end{equation}

Taking this approximation we can see that the ratio is greater than 1 if
$n + 1 < \gamma b v$ and it is less than one if the opposite is true, i.e.
$n + 1 > \gamma b v$. That implies that terms in the series increase until they
reach $n \sim \gamma b v$, to then decrease with increasing $n$.

As $\gamma b v \rightarrow \infty$ the series become sharply peaked around
$n = \gamma b v$. Therefore we could approximate this sum by just summing in the
vicinity of $n = \gamma b v$. This would give us
\begin{align}
  S(\gamma b v) =
\sum_{n=0}^\infty {\left( \gamma b v \right)^{n+1} \over
  n! \left( n - \gamma + 1 \right)} \approx
\sum_{n=\gamma b v (1 - \epsilon)}
    ^{\gamma b v (1 + \epsilon)}
    {\left( \gamma b v \right)^{n+1} \over
  n! \left( n - \gamma + 1 \right)},
\end{align}
with errors increasingly smaller as $\gamma b v \rightarrow \infty$.

If this approximation is valid, that means we can use Stirling's formula to
approximate the factorial term remaining in the sum. If we let
$n \approx \gamma b v + s$, where $s$ is small compared to $\gamma b v$, then by
Stirling's approximation we have
\begin{equation}
  n! \approx \left( {n \over e} \right)^n \sqrt{2 \pi n}.
\end{equation}
Substituting our approximation for $n$ gives
\begin{equation}
  n! \approx (\gamma b v + s)^n e^{-n} \sqrt{2 \pi n}.
\end{equation}
For this equation we left the exponent with $n$ untouched. That violates our
approximation of $n \approx \gamma b v + s$, but we will come back to that
later. For now it is convenient to carry it as $n$ and substitute at the end.

We now factorize out the term $\gamma b v$ obtaining
\begin{equation}
  n! \approx \left[ \gamma b v \left( 1 + {s \over \gamma b v} \right) \right]^n
  e^{-n} \sqrt{2 \pi n}.
\end{equation}
We can rewrite the term in parenthesis in a convenient way as
\begin{equation}
  n! \approx (\gamma b v)^n
  \exp \left\{ n \ln \left( 1 + {s \over \gamma b v} \right) \right\}
  e^{-n} \sqrt{2 \pi n}.
\end{equation}
The term in the exponential can be Taylor expanded up to second order as
\begin{equation}
n \ln \left( 1 + {s \over \gamma b v} \right) \approx
n \left[ {s \over \gamma b v}
- {1 \over 2} \left( {s \over \gamma b v}  \right)^2 \right].
\end{equation}

Putting all these together we obtain an approximation of the form
\begin{equation}
  n! \approx (\gamma b v)^n
             e^{-n}
             e^{n {s \over \gamma b v}}
             e^{-{n \over 2} {s^2 \over (\gamma b v)^2}}
             \sqrt{2 \pi n}.
\end{equation}
Let's now substitute what we didn't do at the beginning for the exponents.
Let's replace $n = \gamma b v + s$ in the exponents obtaining
\begin{equation}
  e^{-n}
  e^{n {s \over \gamma b v}}
  e^{-{n \over 2} {s^2 \over (\gamma b v)^2}} =
  \exp \left\{ -\gamma b v - s
               + (\gamma b v + s){s \over \gamma b v}
               - {(\gamma b v + s) \over 2} {s^2 \over (\gamma b v)^2} \right\}.
\end{equation}
Expanding the terms in the exponent gives
\begin{equation}
\exp \left\{ -\gamma b v - s
             + (\gamma b v + s){s \over \gamma b v}
             - {(\gamma b v + s) \over 2} {s^2 \over (\gamma b v)^2} \right\} =
\exp \left\{ -\gamma b v - s
             + s
             + {s^2 \over \gamma b v}
             - {1 \over 2} {s^2 \over \gamma b v}
             - {1 \over s} {s^3 \over (\gamma b v)^2} \right\}
\end{equation}
Simplifying terms gives
\begin{equation}
  \exp \left\{ -\gamma b v
             - s
             + s
             + {s^2 \over \gamma b v}
             - {1 \over 2} {s^2 \over \gamma b v}
             - {1 \over s} {s^3 \over (\gamma b v)^2} \right\} =
\exp \left\{ -\gamma b v
             + {1 \over 2} {s^2 \over \gamma b v}
             - {1 \over s} {s^3 \over (\gamma b v)^2} \right\}
\end{equation}
Note that by assumption $s^3 \ll (\gamma b v)^2$, therefore we can remove the
third term in the exponent. All these steps finally give an approximation for
$n!$ of the form
\begin{equation}
  n! \approx (\gamma b v)^n e^{-\gamma b v} e^{s^2 \over 2 \gamma b v}
             \sqrt{2 \pi \gamma b v}, \;\;
             \text{for} \; (\gamma b v) \rightarrow \infty.
\end{equation}
\mrm{Eq (28) in the SI}

So far the Laplace approximation for sums allowed us to find the leading terms
of the sum to then have an approximation for the $n!$ term. We can now
substitute back this approximation into the sum
\begin{align}
  S(\gamma b v) &= \sum_{n = 0}^{\infty} {(\gamma b v)^{n + 1} \over
                                        n! (n - \gamma + 1)},\\
  &\approx \sum_{n = 0}^{\infty}
  {(\gamma b v)^{n + 1} \over (n - \gamma + 1)} \cdot
  \overbrace{
  { (\gamma b v)^{-n} e^{\gamma b v} e^{-s^2 \over 2 \gamma b v}
  \over
  \sqrt{2 \pi \gamma b v}
  }}^{1 \over n!}.
\end{align}
We will now substitute for all $n \approx \gamma b v + s$ and approximate the
sum as an integral on $s$ with an extended range from $-\infty$ to $\infty$.
This is valid since the terms around $n = \gamma b v$ dominate the sum, so the
extra terms would have a marginally small impact in the result. With this then
we have
\begin{equation}
  S(\gamma b v) \approx \int_{-\infty}^{\infty} ds {
  \gamma b v e^{\gamma b v} e^{-s^2 \over 2 \gamma b v}
  \over
  \left[ \gamma (b v - 1) + s + 1 \right] \sqrt{2 \pi \gamma b v}
  }.
\end{equation}

Now we factorize a term $\gamma (b v - 1)$ from the denominator, obtaining
\begin{equation}
  S(\gamma b v) \approx \int_{-\infty}^{\infty} ds {
  \gamma b v e^{\gamma b v} e^{-s^2 \over 2 \gamma b v}
  \over
  \gamma (b v - 1) \left[ 1 + {s + 1 \over \gamma (b v - 1) }\right]
  \sqrt{2 \pi \gamma b v}
  }.
\end{equation}
Since we assumed $\gamma \gg 1$ and $\gamma b v \gg s$ we can safely approximate
$\left[ 1 + {s + 1 \over \gamma (b v - 1) }\right] \approx 1$, obtaining
\begin{equation}
  S(\gamma b v) \approx {b v e^{\gamma b v} \over b v - 1}
  \int_{-\infty}^{\infty} ds
  {e^{-s^2 \over 2 \gamma b v}
  \over
  \sqrt{2 \pi \gamma b v}}.
\end{equation}
The term inside the integral integrates to 1 by the Gaussian integral, so we
have
\begin{equation}
  S(\gamma b v) \approx {b v e^{\gamma b v} \over b v - 1}.
\end{equation}
\mrm{Eq (29) in the SI.}

With this rather convoluted result we can go back to \eref{eq_SI_27_uofv} to
write
\begin{align}
  u(v) &= e^{-\gamma b v} \left[ C v^{\gamma}
  - \sum_{n=0}^\infty {\left( \gamma b v \right)^{n+1} \over
  n! \left( n - \gamma + 1 \right)} \right].\\
  &\approx e^{-\gamma b v}
  \left[ C v^{\gamma} - {b v e^{\gamma b v} \over b v - 1} \right]
\end{align}
Simplifying terms we have
\begin{equation}
  u(v) \approx C v^{\gamma} e^{-\gamma b v}
  - {b v \over b v - 1}
  \label{eq_SI_30_uofv}
\end{equation}
\mrm{Eq (30) in the SI.}

To obtain the value of the integration constant $C$ we use the initial
conditions $u(0) = u_o$ and $v(0) = v_o$. This gives us
\begin{equation}
  u_o = C v_o^{\gamma} e^{-\gamma b v_o}
  - {b v_o \over b v_o - 1}.
\end{equation}
Solving for $C$ gives
\begin{equation}
  C = \left[ u_o - {b v_o \over b v_o - 1} \right] v_o^{-\gamma} e^{\gamma b v}.
\end{equation}

That means that \eref{eq_SI_30_uofv} is given by
\begin{align}
  u(v) &= \left[ u_o - {b v_o \over b v_o - 1} \right]
  v_o^{-\gamma} e^{\gamma b v_o} v^{\gamma} e^{-\gamma b v}
  - {b v \over b v - 1}\\
  &= \left( u_o - {b v_o \over b v_o - 1}  \right)
  \left( {v \over v_o} \right)^{\gamma}
  e^{\gamma b v (v_o - v)}
  + {b v \over 1 - b v}.
  \label{eq_uofv_with_C}
\end{align}

Recall that \eref{eq_v_tau_relation} tells us the relationship between $v$ and
$\tau$, i.e. $v = v_o e^{\tau}$. This means that since $\tau > 0$ then
$v = v_o e^{\tau} > v_o$, therefore since the first term of
\eref{eq_uofv_with_C} is dominated by the exponential, and
$e^{\gamma b (v_o - v)}$ quickly converges to zero for large $\gamma$ we can
approximate \eref{eq_uofv_with_C} as
\begin{equation}
  u(v) \approx {b v \over 1 - b v}.
  \label{eq_SI_31_uofv}
\end{equation}
\mrm{Eq (30) in the SI and (5) in the main text.}
Interestingly since in the generating function \eref{eq_generating_function}
$u$ was indirectly related to mRNA copy number and $v$ to protein,
\eref{eq_SI_31_uofv} implies that since $u$ quickly converges to a fixed
function of $v$, for most of the protein lifetime the mRNA is at steady state.

\subsection{Finding the generating function}

With so much mathematical details so far it is easy to lose track of what is
what we are trying to achieve. Let's recall that the ODEs that define the
characteristics of the generating function are given byj
\begin{align}
  \frac{d v}{d r} &= 1,
  \\
  \frac{d u}{d r} &= - \gamma \left[ b (1 + u) - \frac{u}{v}
  \right],
  \\
  \frac{d \tau}{d r} &= \frac{1}{v},
  \\
  \frac{d F}{d r} &= \frac{a u}{v} F.
\end{align}
So far we found functional relationships between them given by
\begin{equation}
  v = v_o e^{\tau},
\end{equation}
and
\begin{equation}
  u = {b v \over 1 - b v}.
\end{equation}

Now we can combine ${dv \over dr}$ and ${dF \over dr}$ with these relationships
to obtain
\begin{align}
  {\partial F \over \partial v} &= a {u \over v} F, \\
  &= a {b v \over v (1 - b v)} F, \\
  &= {a b \over 1 - b v} F
\end{align}
This ODE can be solved by separation of variables as
\begin{equation}
  \int {dF \over F} = \int {a b \over 1 - b v} dv.
\end{equation}
Upon defining $x = 1 - b v, \; dx = - b\; dv$ we have
\begin{equation}
  \ln F = \int {a b \over x} \cdot {1 \over -b} dx = -a \ln x + C'.
\end{equation}
Therefore
\begin{equation}
  \ln F = -a \ln (1 - b v) + C',
  \label{eq_lnF_constant}
\end{equation}
whith $C'$ being an integration constant. To find the value of this constant
we must use the initial conditions. But we originally didn't set initial
conditions for $F$. If we start our birth and death process with $k$ proteins
at time $\tau = 0$, we would have that
\begin{align}
  F(\tau = 0) &= \sum_{p=0}^{\infty}P_p z^p\\
  &= \sum_{p=0}^{\infty} \delta (p - k) z^p = z^k.
\end{align}
This initial condition doesn't sum over mRNA, because as explained before for
\eref{eq_SI_31_uofv}, for most of the protein lifetime the mRNA is in steady
state. Formally this initial condition is valid for times of order
${\gamma_p \over \gamma_m} = \gamma^{-1}$. Now $z = 1 + v$, therefore
$F(\tau = 0) = (1 - v_o)^k$. Using this initial condition we have
\begin{equation}
  \ln (1 + v_o)^k = -a \ln (1 - b v_o) + C'.
\end{equation}
Solving for $C'$ gives
\begin{equation}
  C' = \ln (1 + v_o)^k + a \ln (1 - b v_o).
\end{equation}
Therefore substituting this in \eref{eq_lnF_constant} gives
\begin{align}
  \ln F &= a \ln \left( {1 - b v_o \over 1 - b v} \right)^a
  + \ln (1 + v_o)^k,\\
  &= \ln \left[ \left( {1 - b v_o \over 1 - b v} \right)^a
  (1 + v_o)^k \right].
\end{align}
This gives us the generating function of the form
\begin{equation}
  F =   \left( {1 - b v_o \over 1 - b v} \right)^a
  (1 + v_o)^k .
  \label{eq_generating_function_onv}
\end{equation}

We would like to write the generating function as a function of the original
variable $z$ and $\tau$. This is done by recalling that we defined $z = 1 + v$,
in combination with \eref{eq_v_tau_relation} gives $v_o = (z - 1) e^{-\tau}$.
Using this on \eref{eq_generating_function_onv} gives
\begin{equation}
  F = \left( {1 - b (z - 1) e^{-\tau}
  \over
  1 - b (z - 1)
  } \right)^a
  \left[ 1 + (z - 1) e^{-\tau} \right]^k.
\end{equation}
\mrm{Eq (35) in the SI.}

If we set $k = 0$ we get
\begin{equation}
  F = \left( {1 - b (z - 1) e^{-\tau}
  \over
  1 - b (z - 1)
  } \right)^a.
  \label{eq_generating_function_onz}
\end{equation}
\mrm{Eq (7) in the main text.}

\subsection{Deriving the probability distribution for proteins}

Given that we know the generating function we can backtrack the entire
distribution. The difficulty resides in being able to write
\eref{eq_generating_function_onz} in the standard form
\begin{equation}
  F = \sum_{p} z^p P_p.
\end{equation}
By the definition of the generating function we can see that to compute any
probability we can compute
\begin{equation}
  P_p = {1 \over p!} \left. {\partial ^p F \over \partial z^p}.
  \right\rvert_{z=0}
  \label{eq_prob_from_generating}
\end{equation}
So let's rewrite \eref{eq_generating_function_onz} in a more convenient form
\begin{equation}
  F = \left( {1 + b (1 - z) e^{-\tau}
  \over
  1 + b (z - 1)
  }  \right)^a.
\end{equation}

Now we want to write $F$ as a product of two factors, one of them without any
$z$ term. This means we can write
\begin{align}
  F(z, \tau) &= \left( {1 + b e^{-\tau} \over 1 + b } \right)^a
  \left[
  \left( {1 + b} \over 1 + b (1 - z) \right)^a
  \left( {1 + b (1 - z) e^{-\tau} \over 1 + b e^{-\tau}}  \right)^a
  \right], \\
  &= \left( {1 + b e^{-\tau} \over 1 + b } \right)^a
  { \left( {1 + b (1 - z) \over 1 + b} \right)^{-a}
  \over
  \left( {1 + b (1 - z) e^{-\tau} \over 1 + b e^{-\tau}} \right)^{-a}
  }, \\
  &= \left( {1 + b e^{-\tau} \over 1 + b } \right)^a
  { \left( {1 + b - b z) \over 1 + b} \right)^{-a}
  \over
  \left( {1 + b e^{-\tau} - b z e^{-\tau} \over 1 + b e^{-\tau}} \right)^{-a}
  }, \\
  &= \left( {1 + b e^{-\tau} \over 1 + b } \right)^a
  { \left( 1 - {b z \over 1 + b} \right)^{-a}
  \over
  \left( 1 - {b z \over e^{\tau} + b} \right)^{-a}
  }. \label{eq_generating_final}
\end{align}
\mrm{Eq. (36) in the SI.}

There are two identities that Shahrezaei \& Swain quote for the following steps.
The first one is given by
\begin{equation}
    \left. {\partial ^n \over \partial z^n}
    \left[ 1 - q z \right]^{-a} \right\vert _{z=0} =
    {\Gamma(a + n) \over \Gamma(a)} q^n.
\end{equation}
To proof this relationship let's first take the first derivative of the function
\begin{align}
  {\partial \over \partial z}
  \left. \left[ 1 - q z \right]^{-a} \right\vert _{z=0}
  &= \left. -a \left[ 1 - q z \right]^{-(a + 1)} (-q) \right\vert _{z=0}\\
  &= a q.
\end{align}

Now let's take the second derivative
\begin{align}
  {\partial^2 \over \partial z^2}
  \left. \left[ 1 - q z \right]^{-a} \right\vert _{z=0}
  &= {\partial \over \partial z}
  \left. a q \left[ 1 - q z \right]^{-(a + 1)} \right\vert _{z=0}\\
  &= \left. -a (a + 1) q \left[ 1 - q z \right]^{-(a + 2)} (-q)
  \right\vert _{z=0} \\
  &= a(a + 1) q^2.
\end{align}

The pattern seems to be emerging now. For completeness let's take the 3rd
derivative
\begin{align}
  {\partial^3 \over \partial z^3}
  \left. \left[ 1 - q z \right]^{-a} \right\vert _{z=0}
  &= {\partial \over \partial z}
  \left. -a (a + 1) q \left[ 1 - q z \right]^{-(a + 2)} (-q)
  \right\vert _{z=0} \\
  &= \left. -a (a + 1) (a + 2) q^2
  \left[ 1 - q z \right]^{-(a + 3)} (-q) \right\vert _{z=0} \\
  &= a(a + 1) (a + 2) q^3.
\end{align}

We have that $a (a + 1) (a + 2) \cdots (a + n - 1) = {(a + n - 1)! \over (a -
1)!}$ for $n, a \in \mathbb{Z}$, or in general for any positive $n, a$ we have
$a (a + 1) (a + 2) \cdots (a + n - 1) = {\Gamma(a + n) \over \Gamma(a)}$.
Therefore for the $n^{\text{th}}$ derivative it must be true that
\begin{equation}
    \left. {\partial ^n \over \partial z^n}
    \left[ 1 - q z \right]^{-a} \right\vert _{z=0} =
    {\Gamma(a + n) \over \Gamma(a)} q^n.
    \label{eq_derivative_identity}
\end{equation}
\mrm{Eq (37) in the SI.}

The second identity the authors quote is more convoluted. It could be proof by
induction but it would be a really long proof. For convenience we will quote
the result. The identity tells us that the $n^{\text{th}}$ derivative of a ratio
of functions is given by
\begin{equation}
  {\partial ^n \over \partial z^n} {x(z) \over y(z)} =
  n! \sum_{k=0}^n {\partial ^{n-k} \over \partial z^{n-k}} x(z)
  \sum_{j = 0}^k (-1)^j (k + 1) y(z)^{-j-1}
  {\partial^k \over \partial z^k} y(z)^j
  \label{eq_derivative_ratio}
\end{equation}
\mrm{Eq (39) in SI.}

\eref{eq_prob_from_generating} gives us the recipe for how to compute any
probability from the generating function we derived in
\eref{eq_generating_final}. To use \eref{eq_derivative_ratio} we define from
\eref{eq_generating_final}
\begin{equation}
  x(z) = \left( 1 - {b z \over 1 + b} \right)^{-a},
\end{equation}
and
\begin{equation}
  y(z) = \left( 1 - {b z \over e^{\tau} + b} \right)^{-a}.
\end{equation}
With these definitions in hand we use \eref{eq_prob_from_generating} along with
\eref{eq_derivative_ratio} to compute
\begin{equation}
  \scriptscriptstyle
  P_p = \left. {1 \over p!} \left( {1 + b e^{-\tau} \over 1 + b} \right)^a
  \left[ p! \sum_{k=0}^p  {\partial^{p-k} \over \partial z^{p-k}}
  \left(1 - {b z \over 1 + b} \right) \cdot
  \sum_{j=0}^k {(-1)^j (k + 1) \over (j + 1)! (p - k)! (k - j)!} \cdot
  \left[ \left( 1 - {b z \over e^\tau + b} \right)^{-a} \right]^{-(j+1)} \cdot
  {\partial^k \over \partial z^k}
  \left[ \left( 1 - {b z \over e^\tau + b}  \right)^{-a} \right]^j
  \right]\right\vert _{z=0}.
\end{equation}
Using \eref{eq_derivative_identity} we can rewrite the derivatives as
\begin{equation}
  \scriptstyle
  P_p = \left( {1 + b e^{-\tau} \over 1 + b} \right)^a
  \sum_{k=0}^p {\Gamma(a + p - k) \over \Gamma(a)}
  \left( {b \over 1 + b} \right)^{p - k} \cdot
  \sum_{j=0}^k {(-1)^j (k + 1) \over (j + 1)! (p - k)! (k - j)!} \cdot
  {\Gamma(aj + k) \over \Gamma(aj)} \left( {b \over e^\tau + b} \right)^k
\end{equation}
\mrm{Eq (39) in SI.}

For the second sum the authors quote the following identity
\begin{equation}
  \sum_{j=0}^k {(-1)^j \Gamma(aj + k) \over (j + 1)! \Gamma(aj) (k - j)!} =
  {(-1)^k \Gamma(a + 1) \over \Gamma(a - k + 1) (k + 1)!}.
\end{equation}
\mrm{Eq (40) in SI. I tried really hard deriving this result, but I
couldn't figure it out. Nevertheless computationally it can be shown to be true
for several values of $k$.}

Using this we have
\begin{equation}
  \scriptstyle
  P_p = \left( {1 + b e^{-\tau} \over 1 + b} \right)^a
  \sum_{k=0}^p {\Gamma (a + p - k) \over \Gamma(a)}
  \left( {b \over 1 + b} \right)^{p - k}
  {(k + 1) \over (p - k)!} \left( b \over e^\tau + b \right)^k
  {(-1)^k \Gamma(a + 1) \over \Gamma(a - k + 1) (k + 1)!}.
\end{equation}
This can be further simplified to
\begin{equation}
  P_p = \left( {1 + b e^{-\tau} \over 1 + b} \right)^a
  \left( {b \over 1 + b} \right)^p
  \sum_{k=0}^p {(-1)^k \over k!}
  {a \Gamma(a + p - k) \over (p - k)! \Gamma(a - k + 1)}
  \left( {1 + b \over e^\tau + b} \right)^k.
  \label{eq_dist_nohyper}
\end{equation}
\mrm{Eq (41) in the SI. Shahrezaei and Swain missed one factor of
$a$ since $\Gamma(a + 1) / \Gamma(a) = a$, but the error doesn't propagate to
the other equations.}

\subsection{Gauss Hypergeometric Function}

An important function for the solution is the Hypergeometric function
$_2F_1(a; b; c; z)$. This funciton is defined as
\begin{equation}
  _2F_1(a; b; c; z) \equiv \sum_{k=0}^\infty
  {(a)_k (b)_k \over (c)_k} {z^k \over k!}.
\end{equation}

The series is indetermined if $c$ is a non-positive integer. The $(q)_k$ terms
are the so-called rising factorials or rising Pochhammer symbols defined as
\begin{equation}
  (q)_k = \begin{cases}
  1, & \text{for }k = 0.\\
  q (q + 1)\cdots (q + k - 1) & \text{for k > 0},
  \end{cases}
\end{equation}
or using gamma functions
\begin{equation}
  (q)_k = {\Gamma (q + k) \over \Gamma (q)}.
\end{equation}

If $a$ or $b$ are non-positive integers the series reduces to the polynomial
\begin{equation}
_2F_1(-n; b; c; z) = \sum_{k=0}^\infty (-1)^k {n \choose k}
                    {(b)_k \over (c)_k} z^k.
\end{equation}
Or for non-integer values we have
\begin{equation}
_2F_1(-n; b; c; z) = \sum_{k=0}^\infty (-1)^k
                    {\Gamma (n + 1) \over \Gamma (n - k + 1)}
                    {(b)_k \over (c)_k} {z^k \over k!}.
\end{equation}

From the definition of the Pochhammer symbols they must satisfy
\begin{equation}
  (-a)_k = {\Gamma (a + 1) \over \Gamma (a - k + 1)}.
\end{equation}
This relationship can be shown using the Euler reflection formula
\begin{equation}
  \Gamma (z) \Gamma (1 - z) = {\pi \over \sin (\pi z)}.
\end{equation}

If $z \equiv -a + k$ we have
\begin{equation}
  \Gamma(-a + k) \Gamma(1 - (-a + k)) = {\pi \over \sin(\pi (-a + k))}.
\end{equation}
On the other hand if $z \equiv -a$ we have
\begin{equation}
  \Gamma(-a) \Gamma(1 - (-a)) = {\pi \over \sin(\pi (-a))}.
\end{equation}

From the definition of $(-a)_k$ we have
\begin{equation}
  (-a)_k = {\Gamma(-a + k) \over \Gamma(-a)}
         = {{\pi \over \sin(\pi (-a + k)) \Gamma(1 - (-a + k))}
         \over
           {\pi \over \sin(\pi(-a)) \Gamma(1 - (-a))}}.
\end{equation}
Simplifying this we have
\begin{equation}
  (-a)_k = {\Gamma(1 - (-a)) \over \Gamma(1 - (-a + k))}\cdot
           {\sin(\pi (-a + k)) \over \sin(\pi (-a))}.
\end{equation}

For $\sin(\pi (-a + k))$ we can use the identity
\begin{equation}
  \sin(\pi (-a + k)) = \sin(\pi (-a)) \cos(\pi k) -
                       \cos(\pi (-a)) \sin(\pi k).
\end{equation}
Since for $k \in \mathbb{Z}$ we have that $\sin(\pi k) = 0$, and
$\cos(\pi k) = (-1)^k$, it must be true that for $a \notin \mathbb{N}$
\begin{align}
  (-a)_k &= {\Gamma(a + 1) \over \Gamma(a - k + 1)}
           {(-1)^k \sin(\pi (-a)) \over \sin(\pi (-a))},\\
         &= (-1)^k {\Gamma(a + 1) \over \Gamma(a - k + 1)}.
\end{align}
\mrm{Eq (43) in SI}.

From \eref{eq_dist_nohyper} we can rewrite the term inside the sum as a
hypergeometric function by rewriting $\Gamma(a - k + n) =
\Gamma(a + n - 1 - k + 1)$. This means that if we carefully look at the elements
of the sum we can rewrite it as a hypergeometric function like
\begin{align}
  \sum_{k=0}^p {(-1)^k \over k!}
  {a \Gamma(a + p - k) \over \Gamma(p - k + 1) \Gamma(a - k + 1)}
  \left( {1 + b \over e^\tau + b} \right)^k =
  a
  \overbrace{
  {\Gamma(p + a - 1 + 1) \over \Gamma(a + 1) \Gamma(p + 1)}
  }^{\text{terms to complete } {}_2F_1}
  \times \\
  \sum_{k=0}^p
  \overbrace{
  (-1)^k {\Gamma(p + 1) \over \Gamma(p - k + 1)}
  }^{(a)_k = (-p)_k}
  \overbrace{
  (-1)^k {\Gamma(a + 1) \over \Gamma(a - k + 1)}
  }^{(b)_k = (-a)_k}
  \underbrace{
  {1 \over (-1)^k }{\Gamma(p + a - 1 + k + 1) \over \Gamma(p + a - 1 + 1)}
  }_{{1 \over (c)_k} = {1 \over (-p - a + 1)_k}}
  \underbrace{
  {1 \over k!} \left( {1 + b \over e^\tau + b} \right)^k
  }_{z^k \over k!}.
\end{align}
Therefore the sum can be simply written as
\begin{equation}
\sum_{k=0}^p {(-1)^k \over k!}
  {a \Gamma(a + p - k) \over \Gamma(p - k + 1) \Gamma(a - k + 1)}
  \left( {1 + b \over e^\tau + b} \right)^k =
  {a \Gamma(p + a - 1 + 1) \over \Gamma(a + 1) \Gamma(p + 1)}
  {}_2F_1 \left( -p, -a, 1 - a - n, {1 + b \over e^\tau + b} \right)
\end{equation}

This relationship means that we can rewrite \eref{eq_dist_nohyper} as
\begin{equation}
  P_p = \left( {1 + b e^{-\tau} \over 1 + b} \right)^a
  \left( {b \over 1 + b} \right)^p
  {\Gamma(p + a) \over \Gamma(a) p!}
  {}_2F_1 \left( -p, -a, 1 - a - n, {1 + b \over e^\tau + b} \right),
\end{equation}
This is the final expression for the protein distribution valid for
$\Gamma \gg 1$ and $\tau \gg \Gamma^{-1}$.
\mrm{Eq (44) in SI and (8) in main text.}

\subsection{Steady State Distribution}
To compute the steady state distribution we take the limit when
$\tau \rightarrow \infty$. That gives ${}_2F_1(a, b, c, 0)$ which cancels all
terms in the sum except for the first one. Therefore ${}_2F_1(a, b, c, 0) = 1$.
This means that we have
\begin{equation}
  P_p^{ss} = {\Gamma(p + a) \over \Gamma(p + 1) \Gamma(a)}
  \left( 1 - {b \over 1 + b}  \right)^a
  \left( {b \over 1 + b} \right)^p.
\end{equation}
\mrm{Eq (9) in main text.}

This is a negative binomial distribution with mean
\begin{equation}
  \left\langle p \right\rangle = a \cdot b =
  {r_m \over \gamma_p}{r_p \over \gamma_m},
\end{equation}
and variance
\begin{equation}
  \sigma_p^2 = ab(1 + b) =
  {r_m \over \gamma_p}{r_p \over \gamma_m} \left(1 + {r_p \over \gamma_m}\right)
\end{equation}


\documentclass{report}
\usepackage{amsfonts, amsmath, amssymb, hyperref}
\renewcommand{\chaptername}{}
\begin{document}

\tableofcontents

\chapter{Brusselator}

The Brusselator is characterized by the reactions

\begin{align}
  &A \rightarrow X \\
  &2X + Y \rightarrow 3X \\
  &B + X \rightarrow Y + D \\
  &X \rightarrow E
\end{align}

and the rate equations are

\begin{align}
  \frac{\mathrm{d}}{\mathrm{d}t} \{X\} &= \{A\} + \{X\}^2\{Y\} - \{B\}\{X\} - \{X\} \\
  \frac{\mathrm{d}}{\mathrm{d}t} \{Y\} &= \{B\}\{X\} - \{X\}^2\{Y\}
\end{align}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Damped spring}

The damped spring is described by the simple equation

\begin{equation}
  m\frac{\mathrm{d}^2x}{\mathrm{d}t^2} = -kx + -c\dot{x}
\end{equation}

where $k$ is the spring constant and $c$ is the damping coefficient. We can trivially write this as the following 1st-order ODEs:

\begin{align}
  \frac{\mathrm{d}x}{\mathrm{d}t} &= v \\
  \frac{\mathrm{d}v}{\mathrm{d}t} &= -kx + -cv
\end{align}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Double pendulum}

\section{Lagrangian}

\begin{equation}
  L = \frac{1}{6} ml^2 \left[ \dot{\theta}^2_2 + 4\dot{\theta}^2_1 + 3 \dot{\theta}_1 \dot{\theta}_2 \cos(\theta_1 - \theta_2) \right] + \frac{1}{2} mgl(3 \cos\theta_1 + \cos\theta_2)
\end{equation}

\section{Equations of motion}

The momenta are
\begin{align}
  p_{\theta_1} &= \frac{\partial L}{\partial \dot{\theta}_1} = \frac{1}{6} ml^2 \left[8 \dot{\theta}_1 + 3 \dot{\theta}_2 \cos(\theta_1 - \theta_2) \right] \\
  p_{\theta_2} &= \frac{\partial L}{\partial \dot{\theta}_2} = \frac{1}{6} ml^2 \left[2 \dot{\theta}_2 + 3 \dot{\theta}_1 \cos(\theta_1 - \theta_2) \right]
\end{align}
which can be inverted to
\begin{align}
  \dot{\theta}_1 &= \frac{6}{ml^2} \frac{2 p_{\theta_1} - 3 \cos(\theta_1 - \theta_2) p_{\theta_2}}{16 - 9 \cos^2(\theta_1 - \theta_2)} \\
  \dot{\theta}_2 &= \frac{6}{ml^2} \frac{8 p_{\theta_2} - 3 \cos(\theta_1 - \theta_2) p_{\theta_1}}{16 - 9 \cos^2(\theta_1 - \theta_2)}
\end{align}
Finally,
\begin{align}
  \dot{p}_{\theta_1} &= \frac{\partial L}{\partial\theta_1} = - \frac{1}{2} ml^2 \left[\dot{\theta}_1 \dot{\theta}_2 \sin(\theta_1 - \theta_2) + 3 \frac{g}{l} \sin\theta_1 \right] \\
  \dot{p}_{\theta_2} &= \frac{\partial L}{\partial\theta_2} = - \frac{1}{2} ml^2 \left[- \dot{\theta}_1 \dot{\theta}_2 \sin(\theta_1 - \theta_2) + \frac{g}{l} \sin\theta_2 \right]
\end{align}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Duffing equation}

The Duffing equation is given by the second-order ODE
\begin{equation}
  \frac{\mathrm{d}^2x}{\mathrm{d}t^2} + \delta \frac{\mathrm{d}x}{\mathrm{d}t} + \alpha x + \beta x^3 = \gamma \cos(\omega t)
\end{equation}
which we can cast into the first-order ODEs:
\begin{align}
  \frac{\mathrm{d}x}{\mathrm{d}t} &= v \\
  \frac{\mathrm{d}v}{\mathrm{d}t} &= \gamma \cos(\omega t) - \delta v - \alpha x - \beta x^3
\end{align}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Lorenz system}

\begin{align}
  \frac{\mathrm{d}x}{\mathrm{d}t} &= \sigma(y - x) \\
  \frac{\mathrm{d}y}{\mathrm{d}t} &= x(\rho - z) - y \\
  \frac{\mathrm{d}z}{\mathrm{d}t} &= xy - \beta x
\end{align}

\begin{itemize}
  \item $\sigma = 10, \beta = 8/3, \rho = 28$
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Lotka-Volterra equations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Van der Pol oscillator}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{2-D Van der Pol oscillator}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Symmetric top}

\section{Moments of inertia}

\begin{align}
  I_1 = I_2 &= \frac{3}{20} m \left( r^2 + \frac{l^2}{4} \right) \\
  I_3 &= \frac{3}{m r^2}
\end{align}

\section{Lagrangian}

\begin{equation}
  L = \frac{1}{2} I_1 \left( \dot{\theta}^2 + \dot{\varphi}^2 \sin^2\theta \right) + \frac{1}{2} I_3 \left( \dot{\psi} + \dot{\varphi}\ \cos\theta \right)^2 - mgl \cos\theta
\end{equation}

\section{Equations of motion}

The Euler-Lagrange equation gives
\begin{equation}
  \frac{d}{dt} \frac{\partial L}{\partial \dot{\theta}} - \frac{\partial L}{\partial \theta} = 0
\end{equation}

\textit{Fill in derivation later\dots} \\

Solving for $\ddot{\theta}$,
\begin{equation}
  \ddot{\theta} = \frac{\dot{\varphi}^2 \sin\theta \cos\theta(I_1 - I_3) - I_3 \dot{\varphi} \dot{\psi} \sin\theta + mgl \sin\theta}{I_1}
\end{equation}

\textit{Fill in derivation later\dots} \\

Solving for $\ddot{\varphi}$,
\begin{equation}
  \ddot{\varphi} = \frac{2(I_3 - I_1) \dot{\varphi} \dot\theta \sin\theta \cos\theta - I_3 \dot{\varphi} \dot{\theta} \sin\theta \cos\theta + I_3 \dot{\psi} \dot\theta \sin\theta}{I_1 \sin^2\theta}
\end{equation}

\textit{Fill in derivation later\dots} \\

Solving for $\ddot{\psi}$,
\begin{equation}
  \ddot{\psi} = \frac{\cos\theta \left[ \frac{(I_1 \sin^2\theta + I_3 \cos^2\theta) \dot{\varphi} \dot{\theta} \sin\theta}{\cos\theta} - 2(I_3 - I_1) \dot{\varphi} \dot{\theta} \sin\theta \cos\theta - I_3 \dot{\psi} \dot{\theta} \sin\theta \right]}{I_1 \sin^2\theta}
\end{equation}

\end{document}


\documentclass[12pt]{article}
    \usepackage{mathtools}
    \usepackage[hidelinks]{hyperref}
    \usepackage{color}
    \usepackage{fancyref}
    \usepackage{lastpage}
    \usepackage{fancyhdr}

    \pagestyle{fancy}
    \fancyhf{}
    \fancyhead[LO]{TMA4140: Homework Set 3}
    \fancyhead[RO]{Henry S. Sjøen}
    \fancyfoot[CO]{\thepage\ of \pageref{LastPage}}

    \definecolor{darkred}{RGB}{200, 0, 0}

  \author{Henry S. Sjøen}
  \title{
  \textbf{TMA4140 - Homework Set 3}\\
  Basic structures: Sets, Functions, Sequences and Sums\\
    \color{darkred}{\textbf{RETTES}}
  }
\begin{document}
    \maketitle
    \thispagestyle{empty}
    \tableofcontents

    \pagebreak
    \section{Chapter 2.6 Matrices}
    \subsection{Exercise 27c}
    Let
    $ A =
        \begin{bmatrix}
          1 & 0 & 1 \\
          1 & 1 & 0 \\
          0 & 0 & 1
        \end{bmatrix}
    $
    and
    $
    B =
        \begin{bmatrix}
            0&1&1\\
            1&0&1\\
            1&0&1
        \end{bmatrix}
    $. Find $A \odot B$.

    \begin{equation}
        \begin{split}
            A \odot B =&
            \begin{bmatrix}
                1 & 0 & 1 \\
                1 & 1 & 0 \\
                0 & 0 & 1
            \end{bmatrix}
                \odot
            \begin{bmatrix}
                0 & 1 & 1\\
                1 & 0 & 1\\
                1 & 0 & 1
            \end{bmatrix}\\
            =&
            \begin{bmatrix}
                (1\wedge 0) \vee (0\wedge1) \vee (1\wedge1) & (1\wedge1) \vee (0\wedge0) \vee (1\wedge0) & (1\wedge1) \vee (0\wedge1) \vee (1\wedge1) \\
                (1\wedge0) \vee (1\wedge1) \vee (0\wedge1) & (1\wedge1) \vee (1\wedge0) \vee (0\wedge0) & (1\wedge1) \vee (1\wedge1) \vee (0\wedge1) \\
                (0\wedge0) \vee (0\wedge1) \vee (1\wedge1) & (0\wedge1) \vee (0\wedge0) \vee (1\wedge0) & (0\wedge1) \vee (0\wedge1) \vee (1\wedge1)
            \end{bmatrix}\\
            =&
            \begin{bmatrix}
                0\vee0\vee1&1\vee0\vee0&1\vee0\vee1\\
                0\vee1\vee0&1\vee0\vee0&1\vee1\vee0\\
                0\vee0\vee1&0\vee0\vee0&0\vee0\vee1
            \end{bmatrix}\\
            =&
            \begin{bmatrix}
                1&1&1\\
                1&1&1\\
                1&0&1
            \end{bmatrix}
        \end{split}
    \end{equation}

    \pagebreak
    \section{Chapter 3.1 Algorithms}
    \subsection{Exercise 53}
    Use the greedy algorithm to make change using quarters (25), dimes (10), nickels (5), and pennies (1) for\label{e:53}:\\
    \textbf{a)} 51 cents = 2 quarters (25) + 1 penny (1) $\Rightarrow$ 3 coins\\
    \textbf{b)} 69 cents = 2 quarters (25) + 1 dime (10) + 1 nickel (5) + 4 pennies (1) $\Rightarrow$ 8 coins\\
    \textbf{c)} 76 cents = 3 quarters (25) + 1 penny (1) $\Rightarrow$ 4 coins\\
    \textbf{d)} 60 cents = 2 quarters (25) + 1 dime (10) $\Rightarrow$ 3 coins

    \subsection{Exercise 55 }
    Use the greedy algorithm to make change using quarters, dimes, and pennies (but no nickels) for each of the amounts given in Exercise 53. For which of these amounts does the greedy algoritm use the fewest coins of these denominations possible?\label{e:55}\\
    \textbf{a)} 51 cents $\Rightarrow$ 2 quarters (25) + 1 penny (1) $\Rightarrow$ 3 coins\\
     Exercise 53 and 55 provides the same answer.\\
    \textbf{b)} 69 cents = 2 quarters (25) + 1 dimes (10) + 9 pennies (1) $\Rightarrow$ 12 coins\\
    Exercise 53 uses the fewest amount of coins of the two.\\
    \textbf{c)} 76 cents = 3 quarters (25) + 1 penny (1) $\Rightarrow$ 4 coins\\
    Exercise 53 and 55 provides the same answer.\\
    \textbf{d)} 60 cents = 2 quarters (25) + 1 dime (10) $\Rightarrow$ 3 coins\\
    Exercise 53 and 55 provides the same answer.

    \subsection{Exercise 56}
    Show that if there were a coin worth 12 cents, the greedy algoritm using quarters, 12-cent coins, dimes, nickels, and pennies would not always produce change using the fewest coins possible.

    If we wanted change for 15 cents then the greedy algorithm would give us
    1 (12) + 3 pennies (1) = 15 cents $\Rightarrow$ 4 coins.
    But a more fitting change would be 1 dime + 1 nickel $\Rightarrow$ 2 coins.

    % Another example
    % 21=> 12 + 5 + 1 + 1 + 1 + 1
    % 21=> 10 + 10 + 1

    \pagebreak
    \section{Chapter 3.2 The Growth of Functions}
    \subsection{Exercise 27a, 27b}
    Give a big-$O$ estimate for each of these functions. For the function $g$ in your estimate that $f(x)$ is $O(g(x))$, use a simple function $g$ of the smallest order.\\
    a)$n log(n^2+1)+n^2 log n = O(n^2logn)$\\
    b)$(n log n + 1)^2+(log n+1)(n^2+1)=O(n^2(logn)^2)$

    \subsection{Exercise 30c, 30e}
    Show that each of these pairs of functions are of the same order.\\
    c)$\lfloor x + 1/2 \rfloor,x$\\
    Let $f(x) = \lfloor x + 1/2 \rfloor$ and $g(x)=x$.
    $x<2 \lfloor x+1/2\rfloor$ and for $x>2$: $| \lfloor x +1/2 \rfloor | > \frac{1}{2}\cdot|x|$\\
    That means $\lfloor x+1/2 \rfloor$ is $\Omega(x)$.\\
    $\lfloor x + 1/2 \rfloor <2x for x>2$ it follows that $|\lfloor x+1/2 \rfloor| <2 cdot |x|$\\
    Thus, $\lfloor x+1/2 \rfloor$ is $O(x)$.\\
    And we conclude that $\lfloor x+1/2 \rfloor$ and $x$ are of the same order.\\
    e)$log_{10}x,log_{2}x$\\
    Let $f(x)=log_{10}x$ and $g(x)=log_2x$.
    \begin{equation}
        \begin{split}
            f(x)&=log_{10}x\\
                &=\frac{log x}{log 10}\\
                &=\frac{log 2}{log 2}\frac{log x }{log 10}\\
                &=\frac{log 2}{log 10}\frac{log x}{log 2}\\
                &=\frac{1}{log_{2}10}log_{2}x
        \end{split}
    \end{equation}
    Thus $|log_{10}x| \leq \frac{1}{log_{2}10}x|$ and it follows that $log_{10}x$ is $O(log_2 x)$

    \subsection{Exercise 34}
    Show that $3x^2+x+1$ is $\theta (3x^2)$ by directly finding the constants $k, C_1$ and $C_2$ in Exercise 33.

    \textbf{Exercise 33:}
    Show that if $f(x)$ and $g(x)$ are functions from the set of real numbers to the set of real numbers, then $f(x)$ is $\theta (g(x))$ if and only if there are positive constants $k, C_1$ and $C_2$ such that $C_1|g(x) \leq |f(x)| \leq C_2|g(x)|$ whenever $x > k$.

    \begin{equation}
        3x^2+x+1 = \theta (3x^2)
    \end{equation}

    \subsection{Exercise 42}
    Suppose that $f(x)$ is $O(g(x))$. Does it follow that $2^{f(x)}$ is $O(2^{g(x)})$?\\
    No it does not. For example $f(x)=2x$ and $g(x)=x$.
    Then $f(x)$ is $O(g(x))$. But $2f(x)$ is not $O(2^{g(x)})$.

    \pagebreak
    \section{Chapter 4.1 Divisibility and Modular Arithmetic}
    \subsection{Exercise 11}
    \textbf{What time does a 12-hour clock read:}\\
    \textbf{a)} 80 hours after it reads 11:00?\\
    \emph{Answer:} 7:00\\
    \textbf{b)} 40 hours before it reads 12:00?\\
    \emph{Answer:} 8:00\\
    \textbf{c)} 100 hours after it reads 6:00?\\
    \emph{Answer:} 10:00
\end{document}


\documentclass[12pt]{article}
\usepackage{pylatex}
\usepackage{mmalatex}
\usepackage{geometry}
\usepackage{amsmath}
\usepackage{pgf}
\usepackage{caption}
\usepackage{hyperref}
\usepackage{examples}

% portrait
\geometry{papersize={210mm,297mm},hmargin=2cm,tmargin=1.0cm,bmargin=1.5cm}
\parskip=8pt plus 4pt minus 2pt

% landscape
% \geometry{papersize={297mm,210mm},hmargin=2cm,tmargin=1.0cm,bmargin=1.5cm}
% \parskip=6pt plus 3pt minus 2pt

\begin{document}

\section*{A mixed Mathematica-Python example}

This example demonstrates a cooperative effort where Mathematica is used to do the analytic computations while Python is used to plot the data.

The example chosen here is to find and plot the solution to the boundary value problem defined by
\begin{align*}
   \frac{d^2y}{dx^2} + 2 \frac{dy}{dx} + 10 y = 0\quad\quad\text{with }y(0)=3,\> y'(0)=0
\end{align*}

This example requires two passes, once for Mathematica and once for Python (and in that order). This example can be run using

\vspace{5pt}

\begin{lstlisting}
   mmalatex.sh -x -i mixed
   pylatex.sh  -x -i mixed
   pdflatex          mixed
\end{lstlisting}

\vspace{5pt}

Note that the last pair of commands could also be combined as {\small\tt pylatex.sh -i mixed}.

\subsection*{The Mathematica code}

Here Mathematica is used to first find the general solution of th differential equation.
The boundary conitions are then imposed and finally a uniform sampling of the solution
is written to a file for later use by Python and Matplotlib.

\vspace{10pt}

\begin{mathematica}
   sol = DSolve[y''[x] + 2 y'[x] + 10 y[x] == 0, y, x]
   ans = y[x]/.sol[[1]]                  (* mma(ans.101,ans)*)

   sol = DSolve[{y''[x] + 2 y'[x] + 10 y[x] == 0, y[0]==3, y'[0]==0}, y, x]
   foo = y[x]/.sol[[1]]                  (* mma(ans.102,foo)*)
   bah = Simplify[y'[x]/.sol[[1]]]       (* mma(ans.103,bah)*)

   (* now sample y and dy at selected points *)
   myData = Table[{x, foo, bah}, {x, 0, 2 Pi, 0.02}];
   Export["mixed.txt", myData, "Table",  "FieldSeparators" -> " "];
\end{mathematica}

\vspace{10pt}

The general solution of the differential equation is
\begin{equation*}
  y(x) = \mma{ans.101}
\end{equation*}
while the particular solution satifying the boundary conditions is given by
\vspace{5pt}
\begin{align*}
    y(x) &= \mma{ans.102}
\end{align*}

\clearpage

\subsection*{The Python code}

This is a straighforward use of Matplotlib to plot two functions. The code reads the datafile created previously by Mathematica and then calls Matplotlib to plot that data.

\begin{python}
   import numpy as np
   import matplotlib.pyplot as plt

   plt.matplotlib.rc('text', usetex = True)
   plt.matplotlib.rc('grid', linestyle = 'dotted')
   plt.matplotlib.rc('figure', figsize = (5.5,4.1)) # (width,height) inches

   x, y, dy = np.loadtxt ('mixed.txt', unpack=True)

   plt.plot (x,y)
   plt.plot (x,dy)

   plt.xlim (0.0,4.0)

   plt.legend(('$y(x)$', '$dy(x)/dx$'), loc = 0)
   plt.xlabel('$x$')
   plt.ylabel('$y(x),\> dy/dx$')
   plt.grid(True)
   plt.tight_layout(0.5)

   plt.savefig('mixed-fig.pdf')
\end{python}

\vspace{10pt}

\begin{minipage}{\textwidth}
   \centering
   \IfFileExists{mixed-fig.pdf}%
   {\includegraphics[width=0.75\textwidth]{mixed-fig.pdf}}{Failed to create pdf plot.}
   \captionof{figure}{The function and its derivative.}
\end{minipage}

\end{document}


% $Id$
%
% Author: David Fournier
% Copyright (c) 2008 Regents of the University of California
%

Let $y_i$ be an $N$-dimensional multivariate time series for
$i=1,\ldots,n$, where $y_i$ is a random vector with probability
density function $p(y_i | \alpha_i)$.  For each $i$,
the $\alpha_i$ are random vectors that satisfy the condition
\begin{equation}
  \alpha_i=T_i(\alpha_{i-1},y_{i-1})+\eta_i
\end{equation}
where $\mu_{\eta_i}=0$ and $\sigma^2_{\eta_i}=\sigma^2_\eta$.

Let $p(\alpha_1)$ be the probability density function for
$\alpha_1$ before $y_1$ is observed. After observing
$y_1$, we want to calculate the probability distribution of $\alpha_1$
given $y_1$. This is given by
\begin{equation}
  p(\alpha_1 | y_1)=p(y_1|\alpha_1)\,p(\alpha_1)\big/p(y_1)
\end{equation}
where
\begin{equation}\label{eq:xx6}
  p(y_1)=\int_{-\infty}^\infty p(y_1|\alpha_1)\, p(\alpha_1) \,\,\textrm{d}\alpha_1
\end{equation}
Let $\phi(y_1,\alpha_1)=\log(p(y_1|\alpha_1)p(\alpha_1))$ and
$\hat\alpha_1(y_1)=\max_{\alpha_1} \{\phi(y_1,\alpha_1)\}$.
Approximate $\phi$ by its second-order Taylor expansion in
$\alpha_1$ at $\hat\alpha_1$.
\begin{equation}
  \phi(y_1,\alpha_1)\approx \phi(y_1,\hat\alpha_1)
  + D^2_{\alpha_1\alpha_1}\phi \big(y_1,\hat\alpha_1(y_1)\big) \big(\alpha_1-\hat\alpha_1(y),
  \alpha_1-\hat\alpha_1(y) \big)
\end{equation}
so that
\begin{equation}\label{eq:xx5}
  p(y)\approx e^{\phi(y_1,\hat\alpha_1(y_1))} \kern-.5em
  \int_{-\infty}^\infty \kern-.25em \exp\bigg\{-
  \Big(-D^2_{\alpha_1\alpha_1}\phi
      \big(y_1,\hat\alpha_1(y_1) \big)\,
      \big (\alpha_1-\hat\alpha_1(y),  \alpha_1-\hat\alpha_1(y) \big) \,
   \Big) \bigg\}\,\textrm{d}\alpha_1
\end{equation}
Making a change of variables and integrating, we obtain
\begin{equation}\label{eq:xx4}
  p(y_1)\approx e^{\phi (y_1,\hat\alpha_1(y_1) )}  (2\pi)^{n/2}
  \Big | -D^2_{\alpha_1\alpha_1} \phi \big(y_1,\hat\alpha_1(y_1) \big) \Big |^{-1/2}
\end{equation}
\X{Laplace approximation}
\XX{Laplace approximation}{in Kalman filter}
\X{robust Kalman filter}
This is the Laplace approximation to the integral in equation~(\ref{eq:xx6}).

If the distribution of $\alpha_1$ is (multivariate) normal and
the distribution of $y_1|\alpha_1$ is multivariate normal,
then $\phi(y_1,\alpha_1)$ is a quadratic function of $\alpha_1$, so
the Laplace approximation is exact. The advantage of the
Laplace approximation is that it can be employed for non-normal distributions.

To illustrate this advantage, consider the simple 1-dimensional case where
$\alpha_1$ has a (univariate) normal distribution with mean~0 and
variance~$\sigma_\alpha^2$.
Assume that the distribution of $y_1|\alpha_1$ is a fat-tailed
distribution, which is a mixture of $0.95$  normal distribution and $0.05$ Cauchy
distribution. Then,
\begin{align}
\nonumber \phi(y_1,\alpha_1)=& \log \bigg[ 0.95\exp \Big(\kern-.1em -0.5 (y_1-\alpha_1)^2  \Big/  \sigma_y^2 \Big)
     + 0.05\sqrt{2/\pi} \Big/ \Big(1+ (y-\alpha_1)^2 \big/ \sigma_y^2 \Big) \bigg] \\
   & -0.5\alpha_1^2 \big/ \sigma_\alpha^2 + \textrm{const}
   \label{eq:xx9.7}
\end{align}
whereas if $y_1$ is assumed to have a normal distribution,
\begin{equation}
  \phi(y_1,\alpha_1)=-0.5(y_1-\alpha_1)^2 \big/  \sigma_y^2
    -0.5\alpha_1^2 \big/ \sigma_\alpha^2\ + \textrm{const}
    \label{eq:xx9.8}
\end{equation}
where ``$\textrm{const}$'' denotes some constant independent of~$\alpha_1$.
There are two drawbacks to the use of equation~(\ref{eq:xx9.8}). %fixed $5.b$
If the value of $y_1$ is an outlier from the point of the normal
model, then it will have too much influence on the mode of the estimate of
$p(\alpha_1 | y_1)$.  Also, since the variance %xx added a { to middle term:
\begin{equation}
  \sigma_{\alpha_1|y_1}^2=\big\{D^2_{\alpha_1\alpha_1}\phi(y_1,\beta_i)\big\}^{-1}=\Big[1/\sigma_y^2 + 1/\sigma_\alpha^2\Big]^{-1}
\end{equation}
is independent of the value of $y_1$ observed,
$\sigma_{\alpha_1|y_1}^2$ will be underestimated.  This is incorrect behavior,
since if $y_1$ is an outlier, it contains (almost) no information
about the value of $p(\alpha_1|y_1)$.  So,
$p(\alpha_1|y_1)$ should be almost equal to~$p(\alpha_1)$.
The likelihood function based on equation~(\ref{eq:xx9.7}) %fixed $5a$
has the desired behavior.

To calculate expression~(\ref{eq:xx4}), it is necessary to maximize $\phi(y_1,\alpha_1)$ with respect to $\alpha_1$,
and to calculate its Hessian matrix with respect to $\alpha_1$.

For the maximization, we employ the Newton-Raphson algorithm.
Let $\beta_0=\mu_{\alpha_1}$
\begin{equation}
 \beta_{i+1}=\beta_i-\big\{D^2_{\alpha_1\alpha_1}\phi(y_1,\beta_i)\big\}^{-1}
  \big(D_{\alpha_1}\phi(y_1,\beta_i) \big)
\end{equation}
This operation is carried out a fixed number, $r$, times and then
$\hat\alpha_1(y_1)\approx\beta_r$.
For ``well behaved'' problems, the sequence $\beta_i$ converges
quadratically to $\hat\alpha_1(y_1)$.
We approximate $p(\alpha_1|y_1)$ by a multivariate normal with
\begin{align*}
  \mu_{\alpha_1|y_1}&=\beta_r\\[6pt]
  \sigma^2_{\alpha_1|y_1}&=
  \big\{-D^2_{\alpha_1\alpha_1}\phi(y_1,\beta_r)\big\}^{-1}\\
\end{align*}
and approximate $p(\alpha_2|y_1)$ by a multivariate normal with
\begin{align*}
  \mu_{\alpha_2|y_1}&=T(\beta_r,y_1)\cr
  \sigma^2_{\alpha_2|y_1}&=
  D_{\alpha_1}\, T_1(\beta_r,y_1)\sigma^2_{\alpha_1|y_1}
  D_{\alpha_1}\, T_1(\beta_r,y_1) ^{\prime}+
  \sigma^2_\eta\cr
\end{align*}
Now,
\begin{equation}\label{eq:xx3}
  p(y_2|y_1)=\int_{-\infty}^\infty p(y_2|\alpha_2)\, p(\alpha_2|y_1)
  \,\textrm{d}\alpha_2
\end{equation}
As above, we maximize the integrand of equation~(\ref{eq:xx3}) with respect to
$\alpha_2$ and use the Laplace approximation to the integral.
This produces the sequence of conditional probabilities $p(y_i|y_{i-1})$.
The log-likelihood function for the observed sequence ${y_i}$
is given by
\begin{equation}
  \sum_{i=1}^n \log\Big(p(y_i|y_{i-1})\Big)
\end{equation}

\section{Parameter estimation}

Although we have not explicitly shown them, the conditional likelihood
functions $p(y_i|y_{i-1})$  depend on a number of
parameters. These parameters include the specification of $T$, other
parameters in the probability density $p(y_i|\alpha_i)$, and parameters
that determine $\sigma^2_\eta$. If we denote these parameters by
$\theta$ and write $\big(p(y_i|y_{i-1},\theta)\big)$ to indicate this
dependence, the log-likelihood function becomes
\begin{equation}\label{eq:xx1}
  \sum_{i=1}^n \log\Big(p(y_i|y_{i-1},\theta)\Big)
\end{equation}
The maximum likelihood estimates for the parameter vector $\theta$ are
found by maximizing expression~(\ref{eq:xx1}) with respect to~$\theta$.

\X{stochastic volatility model}
\section{The stochastic volatility model}

The version of the stochastic volatility model presented here is from
\cite{RePEc:eee:empfin:v:5:y:1998:i:2:p:155-173}.

It is assumed that $y_i$ has a multivariate normal distribution with
$\mu_{y_i}=0$ and covariance matrix
$\Omega_i(\alpha_i)=H_i(\alpha_i)RH_i(\alpha_i)$.
$H_i(\alpha_i)$ is an $m\times m$ diagonal matrix whose $j^\textrm{th}$ element
on the diagonal is given by $\exp(\alpha_{ij})/2$, where the
$\alpha_{ij}$ satisfy the relationship
\begin{equation}
  \alpha_i=w+\ep(\delta,\alpha_{i-1}) +\ep(\lambda_1,y_{i-1})
   + \ep(\lambda_2,|y_{i-1}|) + \eta_i
\end{equation}
where, in turn, $\eta_i$ is a multivariate normal random variable with
$\mu_{\eta_i}=0$ and $\sigma^2_{\eta_i}=\sigma^2_\eta$.
If $u$ and $v$ are two vectors with $j^\textrm{th}$ component $u_j$ and $v_j$,
$\ep(i,v)$ is the vector with $j^\textrm{th}$ component $u_jv_j$.
$R$ is an  $m\times m$ positive definite matrix satisfying $r_{jj}=1$,
that is, a correlation matrix.
Then,
\begin{equation}
  \log\big(p(y_i | \alpha_i)\big)=
     -0.5\log|\Omega_i(\alpha_i)|-0.5y^\prime_i\,\,\Omega_i(\alpha_i)^{-1}y_i
\end{equation}
and the distribution of $\alpha_i|y_{i-1}$ is multivariate normal, with
mean vector and covariance matrix given by
\begin{align}
  \mu_{\alpha_i|y_{i-1}}&=w+\ep(\delta,\mu_{\alpha_{i-1}|y_{i-1}})
     +\ep(\lambda,y_{i-1})\\[4pt]
  \sigma^2_{\alpha_i|y_{i-1}}&=i\,\diag(\delta)
   \sigma^2_{\alpha_{i-1}|y_{i-1}}\diag(\delta)
  +\sigma^2_\eta
\end{align}
$\diag(\delta)$ is the diagonal matrix whose diagonal is equal to the vector
$\delta$.
\begin{align} \label{eq:xx2}
  \log\big(p(y_i|\alpha_i)\,p(\alpha_i|y_{i-1})\big)&=
     -0.5\log|\Omega_i(\alpha_i)|-0.5y_i^\prime\Omega_i(\alpha_i)^{-1}y_i
     -0.5\log|\sigma^2_{\alpha_i|y_{i-1}}|\\[6pt]
     &\quad   -0.5(\alpha_i-\mu_{\alpha_i|y_{i-1}})^\prime
          (\sigma^2_{\alpha_i|y_{i-1}})^{-1}
             (\alpha_i-\mu_{\alpha_i|y_{i-1}})
\end{align}
To perform the Newton-Raphson calculations, it is necessary to calculate
the first and second derivatives of expression~(\ref{eq:xx2}) with respect
to the parameter vector~$\alpha$. This is the most involved part of the
calculations and will depend on the particular form of the model. In the present
case, the calculations are simplified by the fact that $\Omega_i$ only depends on
$\alpha$ through the diagonal matrix $H(\alpha_i)$.

The probability density function $p(\alpha_1)$ is
assumed to be multivariate normal with $\mu_{\alpha_1}=\theta_0$
and $\sigma^2_{\alpha_1}=0$.

\section{The data}

The data consist of the daily Mark/Dollar and Yen/dollar exchange rates
and the U.S. and Japanese stock index data. There are 1301
time periods, with some missing data. The missing data, which are denoted by the
impossibly large value of 10,000, were replaced with
the average from the period before and after. They can, however, easily
be estimated in the model, if desired.

\section{The results}

The model was fit with various combinations of the parameters,
and the log-likelihood was examined to investigate the improvement in
fit due to the addition of the parameters.  See Table~\ref{tab:parameters}.
\begin{table}[htbp]
\begin{center}
\begin{tabular}{@{\vrule height 16pt depth 6pt width0pt}@{\extracolsep{1em}}c c c}
\\
\hline
\bf Parameters in model & \bf Number of parameters & \bf Log-likelihood \\[3pt]
\hline
%\noalign{\medskip}
 $w,\delta,R,\sigma^2_\eta$ & 24 & 3774.7 \\
 $w,\delta,R,\sigma^2_\eta,\lambda_1$ & 28 & 3806.6 \\
 $w,\delta,R,\sigma^2_\eta,\lambda_1,\theta_0$ & 32 & 3808.6 \\
 $w,\delta,R,\sigma^2_\eta,\lambda_1,\theta_0,\lambda_2$ & 36 & 3811.2 \\[6pt]
\hline
\\
\end{tabular}
\end{center}
\emptycaption
\label{tab:parameters}
\end{table}

The parameters $\theta_0$ and $\lambda_2$ did not produce a significant
improvement to the fit.  $\lambda_2$ measures the asymmetry
in the response of the variance to positive and negative shocks.

Here are the parameter estimates and their standard deviations for
the model with $w,\delta,R,\sigma^2_\eta$, and $\lambda_1$:
\begin{lstlisting}
 index   name    value      std.dev
    1   w(1)       -1.3749e-001 4.9434e-002
    2   w(2)       -6.5649e-001 1.6161e-001
    3   w(3)        3.1693e-002 1.0574e-002
    4   w(4)       -1.2973e-002 1.5375e-002
    5   lambda1(1)  1.5564e-001 4.9688e-002
    6   lambda1(2)  1.8647e-001 6.9525e-002
    7   lambda1(3)  -6.9265e-002 1.4158e-002
    8   lambda1(4)  -1.6689e-001 3.1626e-002
    9   delta(1)    8.2229e-001 4.6074e-002
   10   delta(1)    5.0848e-001 1.0785e-001
   11   delta(1)    9.5763e-001 1.4602e-002
   12   delta(1)    9.3610e-001 1.8812e-002
   29   R(1,1)      1.0000e+000 0.0000e+000
   30   R(1,2)      5.3821e-001 2.2883e-002
   31   R(1,3)      -7.1704e-002 2.9477e-002
   32   R(1,4)      -3.8796e-002 2.9278e-002
   33   R(2,1)      5.3821e-001 2.2883e-002
   34   R(2,2)      1.0000e+000  0.0000e+000
   35   R(2,3)      -1.2932e-001 2.9111e-002
   36   R(2,3)      -4.1466e-002 2.9468e-002
   37   R(3,1)      -7.1704e-002 2.9477e-002
   38   R(3,2)      -1.2932e-001 2.9111e-002
   39   R(3,3)      1.0000e+000  0.0000e+000
   40   R(1,4)      8.8811e-002 2.9085e-002
   41   R(4,1)      -3.8796e-002 2.9278e-002
   42   R(4,2)      -4.1466e-002 2.9468e-002
   43   R(4,3)      8.8811e-002 2.9085e-002
   44   R(4,4)      1.0000e+000  0.0000e+000
   45   Omega(1,1)  6.5973e-001 6.3099e-002
   46   Omega(1,2)  1.9827e-001 1.6129e-002
   47   Omega(1,3)  -1.3395e-001 5.4982e-002
   48   Omega(1,4)  -3.5161e-002 2.6676e-002
   49   Omega(2,1)  1.9827e-001 1.6129e-002
   50   Omega(2,2)  2.0570e-001 2.3994e-002
   51   Omega(2,3)  -1.3489e-001 3.2608e-002
   52   Omega(2,4)  -2.0985e-002 1.5016e-002
   53   Omega(3,1)  -1.3395e-001 5.4982e-002
   54   Omega(3,2)  -1.3489e-001 3.2608e-002
   55   Omega(3,3)  5.2895e+000 5.7872e-001
   56   Omega(3,4)  2.2791e-001 7.9318e-002
   57   Omega(4,1)  -3.5161e-002 2.6676e-002
   58   Omega(4,2)  -2.0985e-002 1.5016e-002
   59   Omega(4,3)  2.2791e-001 7.9318e-002
   60   Omega(4,4)  1.2451e+000 1.7043e-001
   61   Z(1,1)      2.3967e-001 7.4268e-002
   62   Z(1,2)      2.0711e-001 5.5599e-002
   63   Z(1,3)      3.8832e-002 1.8505e-002
   64   Z(1,4)      2.4097e-002 2.0344e-002
   65   Z(2,1)      2.0711e-001 5.5599e-002
   66   Z(2,2)      4.6309e-001 1.1143e-001
   67   Z(2,3)      3.4298e-002 2.3017e-002
   68   Z(2,4)      9.6831e-003 2.9999e-002
   69   Z(3,1)      3.8832e-002 1.8505e-002
   70   Z(3,2)      3.4298e-002 2.3017e-002
   71   Z(3,3)      3.9101e-002 1.6885e-002
   72   Z(3,4)      2.4602e-002 1.1053e-002
   73   Z(4,1)      2.4097e-002 2.0344e-002
   74   Z(4,2)      9.6831e-003 2.9999e-002
   75   Z(4,3)      2.4602e-002 1.1053e-002
   76   Z(4,4)      9.6109e-002 3.4268e-002
\end{lstlisting}

The AD Model Builder \textsc{tpl} file for the model is given below:
\begin{lstlisting}
DATA_SECTION
  init_int ndim
  init_int nobs
  int ndim1
  int ndim2
 !! ndim1=ndim*(ndim+1)/2;
 !! ndim2=ndim*(ndim-1)/2;
  init_matrix Y(1,nobs,1,ndim)
 LOC_CALCS
  // replace missing values (10000) with the average of before and after.
  for (int i=2;i<nobs;i++)
    for (int j=1;j<=ndim;j++)
      if (Y(i,j)==10000)
      {
        int i2=i+1;
        do
        {
          if (Y(i2,j)==10000)
            i2++;
          else
            break;
        }
        while(1);
        Y(i,j)=(Y(i-1,j)+Y(i2,j))/2.;
        if (Y(i,j)>100.0)     // did this work
          cerr << " Y(i,j) too big " << Y(i,j) << endl;
      }
 END_CALCS

PARAMETER_SECTION
  matrix h_mean(1,nobs,1,ndim)
  3darray h_var(1,nobs,1,ndim,1,ndim)
  number ldR;
  init_vector theta0(1,ndim,3);
  vector lmin(1,nobs)
  init_bounded_vector w(1,ndim,-10,10)
  vector w1(1,ndim)
  init_vector lambda(1,ndim,2)
  init_vector lambda2(1,ndim,-1)
  init_bounded_vector delta(1,ndim,0,.98)
  sdreport_matrix R(1,ndim,1,ndim)
  sdreport_matrix Omega(1,ndim,1,ndim)
  matrix ch_R(1,ndim,1,ndim)
  matrix Rinv(1,ndim,1,ndim)
  init_bounded_vector v_R(1,ndim2,-1.0,1.0)
  sdreport_matrix Z(1,ndim,1,ndim)
  matrix ch_Z(1,ndim,1,ndim)
  init_bounded_vector v_Z(1,ndim1,-1.0,1.0)
  matrix S(1,ndim,1,ndim);
  objective_function_value f
INITIALIZATION_SECTION
  delta 0.9
PROCEDURE_SECTION

  fill_the_matrices();
  int sgn;
  ldR=ln_det(R,sgn);
  Rinv=inv(R);
  dvar_vector tmp(1,ndim);
  dvar_matrix sh(1,ndim,1,ndim);
  h_mean(1)=theta0;
  h_var(1)=0;
  for (int i=2;i<=nobs;i++)
  {
    dvar_vector tmean=update_the_means(w,h_mean(i-1),Y(i-1));
    dvar_matrix v=update_the_variances(h_var(i-1));
    tmp=tmean;
    dvar_vector h(1,ndim);
    dvar_vector gr(1,ndim);
    for (int ii=1;ii<=4;ii++)  // do the Newton-Raphson 4 times
    {
      xfp12(tmp, Y(i),tmean,v,gr,sh); // get 1st and 2nd derivatives
      h=-solve(sh,gr);  //sh is hessian and gr is the gradient
      tmp+=h;  // add new step h
    }
    double nh=norm2(value(h)); // check size of h for convergence
    if (nh>1.e-1)
      cout << "No convergence in NR " << nh << endl;
    if (nh>1.e+02)
    {
      f+=1.e+7;   // this ensures that the function minimizer will take a
      return;    // smaller step
    }
    h_mean(i)=tmp;
    h_var(i)=inv(sh);
    lmin(i)=fp(tmp,Y(i),tmean,v);
    int sgn;
    f+=lmin(i)+0.5*ln_det(sh,sgn);  // Laplace approximation
  }
  f-=0.5*nobs*ndim*log(2.*PI);
  Omega=S;

FUNCTION  dvar_vector update_the_means(dvar_vector& w,dvar_vector& m,dvector& e)
  dvar_vector tmp= w+elem_prod(delta,m)+elem_prod(lambda,e);
  if (active(lambda2))
    tmp+=elem_prod(lambda2,fabs(e));
  return tmp;

FUNCTION  dvar_matrix update_the_variances(dvar_matrix& v)
  dvar_matrix tmp(1,ndim,1,ndim);
  for (int i=1;i<=ndim;i++)
  {
    for (int j=1;j<=i;j++)
    {
      tmp(i,j)=delta(i)*delta(j)*v(i,j);
      if (i!=j) tmp(j,i)=tmp(i,j);
    }
  }
  tmp+=Z;
  return tmp;

FUNCTION dvariable fp(dvar_vector& h, dvector& y, dvar_vector& m,dvar_matrix& v)
  dvar_vector eh=exp(.5*h);
  for (int i=1;i<=ndim;i++)
  {
    for (int j=1;j<=i;j++)
    {
      S(i,j)= eh(i)*eh(j)*R(i,j);
      if (i!=j) S(j,i)=S(i,j);
    }
  }

  dvariable lndet;
  dvariable sgn;
  dvar_vector u=solve(S,y,lndet,sgn);
  dvariable l;
  l=.5*lndet+.5*(y*u);
  dvar_vector hm=h-m;
  w1=solve(v,hm,lndet,sgn);
  l+=.5*lndet+.5*(w1*hm);
  return l;

FUNCTION void xfp12(dvar_vector& h, dvector& y,dvar_vector& m,dvar_matrix& v,
 dvar_vector gr,dvar_matrix& hess)
  dvar_vector ehinv=exp(-.5*h);
  dvariable lndet;
  dvariable sgn;
  dvar_vector ys=elem_prod(ehinv,y);
  dvar_vector u=Rinv*ys;
  gr=0.5;
  dvar_vector vv=elem_prod(ys,u);
  gr-=.5*vv;
  dvar_vector hm=h-m;
  dvar_vector w=solve(v,hm,lndet,sgn);
  gr+=w;
  for (int i=1;i<=ndim;i++)
  {
    for (int j=1;j<=i;j++)
    {
      hess(i,j)=0.25*ys(i)*ys(j)*Rinv(i,j);
      if (i!=j) hess(j,i)=hess(i,j);
    }
  }
  for (i=1;i<=ndim;i++)
  {
    hess(i,i)+=.25*vv(i);
  }
  hess+=inv(v);

FUNCTION  fill_the_matrices
  int ii=1;
  ch_Z.initialize();
  for (int i=1;i<=ndim;i++)
  {
    for (int j=1;j<=i;j++)
      ch_Z(i,j)=v_Z(ii++);
    ch_Z(i,i)+=0.5;
  }
  Z=ch_Z*trans(ch_Z);
  ch_R.initialize();
  ii=1;
  for (i=1;i<=ndim;i++)
  {
    for (int j=1;j<i;j++)
      ch_R(i,j)=v_R(ii++);
    ch_R(i,i)+=0.1;
    ch_R(i)/=norm(ch_R(i));
  }
  R=ch_R*trans(ch_R);

REPORT_SECTION
  report<<"observed"<<Y<<endl;
  for (int i=1;i<=nobs;i++)
  {
    report<< "mean" <<endl;
    report<< h_mean(i) <<endl;
    report<< "covariance" <<endl;
    report<<h_var(i)<<endl;
    report<<endl;
  }
  report<< "S(nobs) " << endl;
  report<< Omega << endl;
  report<< "Z " << endl;
  report<< Z << endl;
  report<< "R " << endl;
  report<< R << endl;

TOP_OF_MAIN_SECTION
  arrmblsize=20000000;
  gradient_structure::set_CMPDIF_BUFFER_SIZE(25000000);
  gradient_structure::set_GRADSTACK_BUFFER_SIZE(1000000);
\end{lstlisting}


%================================
\section{Subspaces}
%================================

% Let B be a base for X and let Y be a subspace of X. Then if we intersect each element of B with Y, the resulting collection of sets is a base for the subspace Y.

%--------------------------------
\begin{definition}
	[subspace topology]
	\label{def: subspace topology}
	Let $(X, \mathcal T)$ be a topological space and let $A \subseteq X$. The \textit{subspace topology} $\mathcal T_A$ on $A$ is defined to be the family of the intersections of open sets in $(X, \mathcal T)$ and $A$. That is,
	$$
	\mathcal T_A = \left\{ U \cap A : \ U \in \mathcal T \right\}.
	$$
\end{definition}
%--------------------------------