\chapter{}

\rmk{1} We follow the convention that the derivative with respect to a vector is the transpose of the gradient, the latter being a row vector. Wherever present, the acronym CRLB stands for “Cramer-Rao lower bound”.

\begin{ex} \label{ex:5.1}
    For an underlying parametric family $\{f(x|\theta)\}_{\theta\in\Theta}$, let $\theta=H(\xi)$ be a reparametrization, where $H(\cdot)$ is differentiable and injective in a vicinity of $\xi$. Show that
    \[
        I(\xi)=I(H(\xi))[H'(\xi)]^2.
    \]
    \emph{Hint:} since\[
        I(\xi)=\mathbb{E}\left(\frac{\pder}{\pder \xi}\log f(x|H(\xi))\right)^2,
    \]
    you can use the chain rule.
\end{ex}

\begin{solution}
    \[
        \begin{aligned}
            I(\xi)&=\mathbb{E}\left(\frac{\pder \log f(x|H(\xi))}{\pder \xi}\right)^2\\
            &=\mathbb{E}\left(\frac{\pder \log f(x|H(\xi))}{\pder H(\xi)}\cdot\frac{\pder H(\xi)}{\pder \xi}\right)^2\\
            &=\mathbb{E}\left[\left(\frac{\pder \log f(x|H(\xi))}{\pder H(\xi)}\right)^2\cdot\left(\frac{\pder H(\xi)}{\pder \xi}\right)^2\right]\\
            &=\mathbb{E}\left(\frac{\pder \log f(x|H(\xi))}{\pder H(\xi)}\right)^2\cdot\left(\frac{\pder H(\xi)}{\pder \xi}\right)^2\\
            &=I(H(\xi))[H'(\xi)]^2.
        \end{aligned}
    \]
\end{solution}

\begin{ex} \label{ex:5.2}
    In this problem, we finish a proof we started in class. Consider the following setting.
    \begin{enumerate}[(i)]
        \item $\mathcal{P}=\{\mathbb{P}_\theta\}_{\theta\in\Theta}$, 1-dimensional exponential family generated by $(T,h)$, $\mathbf{X}\sim \mathcal{P}$, i.e.,
        \[
            f(\mathbf{x|\theta})=h(\mathbf{x})\exp\left(H(\theta)T(\mathbf{x})-B(\theta)\right);
        \]
        \item $\tau(\theta):=\mathbb{E}_\theta T(\mathbf{X})$, $\eta=H(\theta)\in\mathcal{E}$;
        \item the parameter functions $\theta\mapsto B(\theta), \tau(\theta), H(\theta)$ are, respectively, $C^1, C^1$ and $C^2$ in a
        vicinity of the true parameter value $\theta$ and satisfy $|\tau'(\theta)H'(\theta)H''(\theta)|>0$;
        \item $\Theta$ is open in $\mathbb{R}$.
    \end{enumerate}
    \begin{enumerate}[(a)]
        \item Prove that
        \[
        \tau(\theta):=\mathbb{E}_{\theta} T(\mathbf{X})=\frac{B^{\prime}(\theta)}{H^{\prime}(\theta)}, \quad  Var_{\theta} T(\mathbf{X})=\frac{B^{\prime \prime}(\theta)}{\left(H^{\prime}(\theta)\right)^{2}}-\frac{\eta^{\prime \prime}(\theta) B^{\prime}(\theta)}{\left(H^{\prime}(\theta)\right)^{3}}=\frac{B^{\prime \prime}(\theta)-\eta^{\prime \prime}(\theta) \tau(\theta)}{\left(\eta^{\prime}(\theta)\right)^{2}}
        \]
        You may assume that
        \[
            \frac{\partial}{\partial \theta} \int_{\mathbb{R}} f(\mathbf{x} \mid \theta) G(d \mathbf{x})=\int_{\mathbb{R}} \frac{\partial}{\partial \theta} f(\mathbf{x} \mid \theta) G(d \mathbf{x}),
        \]
        \[
            \frac{\partial^{2}}{\partial \theta^{2}} \int_{\mathbb{R}} f(\mathbf{x} \mid \theta) G(d \mathbf{x})=\int_{\mathbb{R}} \frac{\partial^{2}}{\partial \theta^{2}} f(\mathbf{x} \mid \theta) G(d \mathbf{x}).
        \]
        \item Conclude that \(Var_{\theta} T(\mathbf{X})\) attains the CRLB, i.e.,
        \[
         Var_{\theta} T(\mathbf{X})=\frac{1}{I(\tau(\theta))}
        \]
        Also conclude that, almost surely, it is the only unbiased estimator of \(\tau(\theta)\) to do so.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item \label{ex:5.2.a} Let $\theta=H^{-1}(\eta)$, then
        \[
            \mathbb{E}_\theta T(X) = A'(\eta) =\frac{\pder B(H^{-1}(\eta))}{\pder \eta}=B'(H^{-1}(\eta))\frac{1}{H'(\theta)}=\frac{B'(\theta)}{H'(\theta)}.
        \]
        \[
            \begin{aligned}
                Var_\theta T(X)&=A''(\eta)\\
                &=\left(\frac{B'(H^{-1}(\eta))}{H'(H^{-1}(\eta))}\right)'\\
                &=\frac{B''(H^{-1}(\eta))(H^{-1}(\eta))'}{H'(H^{-1}(\eta))}-\frac{B'(H^{-1}(\eta))H''(H^{-1}(\eta))(H^{-1}(\eta))'}{\left(H'(H^{-1}(\eta))\right)^2}\\
                &=\frac{B''(\theta)}{(H'(\theta))^2}-\frac{B'(\theta)H''(\theta)}{(H'(\theta))^3}\\
                &=\frac{B''(\theta)-\eta''(\theta)\tau(\theta)}{(\eta'(\theta))^2}.
            \end{aligned}
        \]
        \item \[
            I(\theta)=Var_\theta\left(\frac{f'(\mathbf{x|\theta})}{f(\mathbf{x|\theta})}\right)=Var_{\theta}(\eta'(\theta)T(\mathbf{x})-B'(\theta))=Var_\theta(T(\mathbf{x}))(\eta'(\theta))^2.
        \]
        Then from Exercise \ref{ex:5.1},
        \begin{equation}
            I(\tau(\theta))=\frac{I(\theta)}{(\tau'(\theta))^2}=\left[\frac{\eta'(\theta)}{\tau'(\theta)}\right]^2Var_\theta T(\mathbf{x}).
            \label{eq:5.2.1}
        \end{equation}
        And from Exercise \ref{ex:5.2} \ref{ex:5.2.a},
        \[
            Var_\theta T(X)=\frac{B''(\theta)-\eta''(\theta)\tau(\theta)}{(\eta'(\theta))^2}, \qquad \tau(\theta)=\frac{B'(\theta)}{\eta'(\theta)}.
        \]
        \[
            \tau'(\theta)=\frac{B''(\theta)-\tau(\theta)\eta''(\theta)}{\eta'(\theta)},
        \]
        and \[
            \frac{\tau'(\theta)}{\eta'(\theta)}=Var_\theta T(X).
        \]
        So,
        \[
            I(\tau(\theta))=\left[\frac{\eta'(\theta)}{\tau'(\theta)}\right]^2Var_\theta T(\mathbf{x})=\frac{\eta'(\theta)}{\tau'(\theta)}=\frac{1}{Var_\theta T(\mathbf{x})}.
        \]
        If a statistic $\delta$ can attain the lower bond, then it can be written as following form:
        \[
           \delta=a\left(\eta'(\theta)T(X)-B'(\theta)\right)+b.
        \]
        And if $\mathbb{E}(\delta)=\tau(\theta)$, $Var_\theta(\delta)=\frac{(\tau'(\theta))^2}{I(\theta)}$. Solve them, we can get that $a=1/H'(\theta)$, $b=\tau(\theta)$. So, the statistic can only be $T(X)$.
    \end{enumerate}
\end{solution}

\begin{ex}
    Complete the following table:
    \begin{center}
        \begin{tabular}{ccc}
            \hline
            distribution & parameter $\tau(\theta)$ & $I(\tau(\theta))$ \\ \hline
            $\mathcal{N}(\mu, \sigma^2)$ & $\mu$ &  \\
            $\mathcal{N}(\mu, \sigma^2)$ & $\sigma^2$ &  \\
            $B(n,p)$ & $p$ &  \\
            $Poi(\lambda)$ & $\lambda$ &  \\
            $\Gamma(\alpha, \lambda)$ & $\beta=\lambda^{-1}$ &  \\
            \hline
            \end{tabular}
    \end{center}
\end{ex}

\begin{solution}
    In order to simplify the problem, we only calculate the 1-dimension case:
    \begin{itemize}
        \item \[
            p_\mu(x)=\frac{1}{\sqrt{2\pi}\sigma}\exp\left(-1/2\sigma^2(x^2-2\mu x+\mu^2)\right),
        \]
        Then
        \[
            h(x)=\frac{1}{\sqrt{2\pi}\sigma}\exp\left(-x^2/2\sigma^2\right), \eta(\mu) T(x)-B(\mu)=\frac{\mu}{\sigma^2}x-\frac{\mu^2}{2\sigma^2}.
        \]
        So, if $h(\mu)=\mu$, from \ref{eq:5.2.1},
        \[
            I(\mu)=\left(\frac{\eta'(\mu)}{h'(\mu)}\right)^2Var(T(X))=\frac{1}{\sigma^4}\sigma^2=1/\sigma^2.
        \]
        \item \[
        \begin{aligned}
            \mathcal{L}_\sigma&=\log\left(\frac{1}{\sqrt{2\pi}\sigma}\exp\left(-1/2\sigma^2(x-\mu)^2\right)\right)\\
            &=-\frac{1}{2}\left(x-\mu\right)^2\sigma^{-2}-\log(\sqrt{2\pi}\sigma)
        \end{aligned}
        \]
        \[
            \mathcal{L}'_\sigma = \left(x-\mu\right)^2\sigma^{-3}-\sigma^{-1},
        \]
        \[
            \mathcal{L}''_\sigma = -3\left(x-\mu\right)^2\sigma^{-4}+\sigma^{-2},
        \]
        Then $E(X)=\mu$, $E(X^2)=Var(X)+(EX)^2=\sigma^2+\mu^2$,
        \[
            I(\sigma)=-E(\mathcal{L''_\sigma})=2\sigma^{-2}.
        \]
        \[
            I(\sigma^2)=\frac{I(\sigma)}{(2\sigma)^2}=\frac{1}{2\sigma^4}.
        \]
        \item \[
            f_p(x)=\binom{n}{x}p^x(1-p)^{n-x}=\binom{n}{x}\exp\left(x\log(p)-x\log(1-p)+n\log(1-p)\right),
        \]
        $\eta(p)T(x)-B(p)=\log\frac{p}{1-p}x-n\log\frac{1}{1-p}$,
        \[
            I(np)=\left(\frac{\eta'(p)}{n}\right)^2Var(T(X))=\left(\frac{1}{np(1-p)}\right)^2np(1-p)=\frac{1}{np(1-p)}.
        \]
        \[
            I(p)=\frac{I(np)}{1/n^2}=\frac{n}{p(1-p)}.
        \]
        \item For Poisson distribution, $\eta(\lambda)=\log \lambda$, $T(x)=x$, $E(x)=Var(x)=\lambda$. So,
        \[
            I(\lambda)=\frac{1}{\lambda^2}\lambda=1/\lambda.
        \]
        \item For Gamma distribution, $\eta(\lambda)=-\lambda$, $T(x)=x$, $E(x)=\alpha/\lambda$, $Var(x)=\alpha/\lambda^2$. So,
        \[
            I(\alpha\beta)=\left(\frac{1}{-\alpha/\lambda^2}\right)^2\frac{\alpha}{\lambda^2}=\lambda^2/\alpha=\frac{1}{\alpha\beta^2},
        \]
        So, \[
            I(\beta)=\frac{I(\alpha\beta)}{(1/\alpha)^2}=\frac{\alpha}{\beta^2}.
        \]
    \end{itemize}
\end{solution}

\begin{ex}
    Let $X\sim \Gamma(\alpha_0, \lambda)$, $\lambda>0$, where $\alpha_0$ is known. We see in the class that $I(\alpha_0/\lambda)=\lambda^2/\alpha_0$. Use this result to find $I(1/\lambda)$.
\end{ex}

\begin{solution}
    Let $\xi=\alpha_0/\lambda$, $\theta=H(\xi)=\frac{1}{\alpha_0}\xi=1/\lambda$. Then
    \[
        I(1/\lambda)=I(\theta)=\frac{I(\xi)}{(H'(\xi))^2}=\frac{\lambda^2}{\alpha_0}\alpha_0^2=\alpha_0\lambda^2.
    \]
\end{solution}

\begin{ex}
    Let \(X \sim \mathcal{N}\left(\mu, \sigma_{0}^{2}\right), \mu>0\), where \(\sigma_{0}^{2}>0\) is known.
    \begin{enumerate}[(a)]
        \item Find \(I\left(\mu / \sigma_{0}^{2}\right)\) (hint: no calculations needed).
        \item Use the result in (a) to compute \(I\left(\mu^{2}\right)\).
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item From previous problem, $\eta(\mu)=\mu$, $T(X)=x/\sigma^2$. And $E(T(X))=\mu/\sigma^2$, $Var(T(X))=1/\sigma^2$. So,
        \[
            I(\mu/\sigma^2)=\left(\frac{1}{1/\sigma^4}\right)1/\sigma^2=\sigma^2.
        \]
        \item Let $\theta=\mu/\sigma^2$, $h(\theta)=\sigma^4\theta^2=\mu^2$. So,
        \[
            I(h(\theta))=\frac{I(\theta)}{(h'(\theta))^2}=\frac{\sigma^2}{4\sigma^4\mu^2}=\frac{1}{4\sigma^2\mu^2}.
        \]
    \end{enumerate}
\end{solution}

\begin{ex}
    Prove the lemma stated in class.
    \begin{enumerate}[(i)]
        \item Suppose that the conditions A-(a),(b),(c) hold, and that
        \[
        \frac{\partial}{\partial \theta} \int_{\mathbb{R}^{n}} f(\boldsymbol{x} \mid \theta) G(d \boldsymbol{x})=\int_{\mathbb{R}^{n}} \frac{\partial}{\partial \theta} f(\boldsymbol{x} \mid \theta) G(d \boldsymbol{x}), \quad \theta \in \Theta .
        \]
        Then,
        \[
        \mathbb{E}_{\theta}\left(\frac{\partial}{\partial \theta} \log f(\boldsymbol{X} \mid \theta)\right)=0, \quad I(\theta)= Var_{\theta}\left(\frac{\partial}{\partial \theta} \log f(\boldsymbol{X} \mid \theta)\right).
        \]
        \item If, in addition,
        \[
        \begin{gathered}
        \exists \frac{\partial^{2}}{\partial \theta^{2}} \log f(\boldsymbol{x} \mid \theta), \quad x \in \operatorname{supp} f(\boldsymbol{x} \mid \theta), \\
        \frac{\partial^{2}}{\partial \theta^{2}} \int_{\mathbb{R}^{n}} f(\boldsymbol{x} \mid \theta) G(d \boldsymbol{x})=\int_{\mathbb{R}^{n}} \frac{\partial^{2}}{\partial \theta^{2}} f(\boldsymbol{x} \mid \theta) G(d \boldsymbol{x}),
        \end{gathered}
        \]
        for \(\theta \in \Theta\), then,
        \[
        I(\theta)=-\mathbb{E}_{\theta}\left(\frac{\partial^{2}}{\partial \theta^{2}} \log f(\boldsymbol{X} \mid \theta)\right).
        \]
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(i)]
        \item \[
            \begin{aligned}
                E  \left[\left.{\frac {\partial }{\partial \theta }}\log f(X|\theta )\right|\theta \right]={}&\int _{\mathbb {R} }{\frac {{\frac {\partial }{\partial \theta }}f(X|\theta )}{f(X|\theta )}}f(X|\theta )\der x\\[3pt]={}&{\frac {\partial }{\partial \theta }}\int _{\mathbb {R} }f(X|\theta )\der x\\[3pt]={}&{\frac {\partial }{\partial \theta }}1=0.
            \end{aligned}
        \]
        \[
            \begin{aligned}
                I(\theta)&=E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2\\
                &=E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)- E  \left[{\frac {\partial }{\partial \theta }}\log f(X|\theta )\right]\right)^2\\
                &=Var\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)
            \end{aligned}
        \]
        \item \[
            \begin{aligned}
                \frac {\partial ^{2}}{\partial \theta ^{2}}\log f(X|\theta )&=\frac{\pder}{\pder \theta}\left(\frac{f'(X|\theta)}{f(X|\theta)}\right)\\
                &=\frac{\frac{\pder^2}{\pder\theta^2}f(X|\theta)}{f(X|\theta)}-\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2
            \end{aligned}
        \]
        and
        \[
            E \left[{\frac {{\frac {\partial ^{2}}{\partial \theta ^{2}}}f(X|\theta )}{f(X|\theta )}}\right]=\int_{\mathbb{R}}{\frac {{\frac {\partial ^{2}}{\partial \theta ^{2}}}f(X|\theta )}{f(X|\theta )}}f(X|\theta )\der x={\frac {\partial ^{2}}{\partial \theta ^{2}}}\int _{\mathbb {R} }f(X|\theta )\der x=0.
        \]
        So, \[
            \frac {\partial ^{2}}{\partial \theta ^{2}}\log f(X|\theta )=-\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2,
        \]
        \[
            E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2=-E\left(\frac {\partial ^{2}}{\partial \theta ^{2}}\log f(X|\theta )\right).
        \]
    \end{enumerate}
\end{solution}

\begin{ex}
    Consider the following lemma. Assume that
    \begin{enumerate}[(a)]
        \item \(\Theta\) is an open interval;
        \item  \(A:=\{x ; f(x \mid \theta)>0\}\) is independent of \(\theta\);
        \item \(\delta(X) \in \Delta\).
    \end{enumerate}
    Let \(\psi(x \mid \theta)=\frac{\partial}{\partial \theta} \log f(x \mid \theta)\), assumed to exist at every pair \(x, \theta\). For some \(\varepsilon>0\), let \(b_{\theta}\) be a function that satisfies
    \[
    \begin{aligned}
    \mathbb{E}_{\theta} b_{\theta}^{2}(X) &<\infty \\
    \left|\frac{1}{\Delta} \frac{f(x \mid \theta+\Delta)-f(x \mid \theta)}{f(x \mid \theta)}\right| & \leq b_{\theta}(x) \quad \text { if }|\Delta|<\varepsilon .
    \end{aligned}
    \]
    Then,
    \[
    \mathbb{E}_{\theta} \psi(X, \theta)=0
    \]
    and
    \[
    \frac{\partial}{\partial \theta} \mathbb{E}_{\theta} \delta(X)=\mathbb{E}_{\theta}(\delta(X) \psi(X, \theta))= Cov_{\theta}(\delta, \psi)=\int \delta(x) \frac{\partial}{\partial \theta} \log f(x \mid \theta) G(d x)
    \]
    (and thus the \(C R L B\) is \(\left(\frac{\partial}{\partial \theta} \mathbb{E}_{\theta}(\delta)\right)^{2} / I(\theta)\)).

    Assume \(f\) is the density of a Cauchy \((0,1)\) random variable, and consider the location family associated with
    \[
    f(x-\theta)=\frac{1}{\pi} \frac{1}{1+(x-\theta)^{2}}, \quad \theta \in \mathbb{R} .
    \]
    Use the above lemma to establish the information inequality for the family \(\{f(x-\theta)\}_{\theta \in \mathbb{R}}\).

    (suggestion: however clumsy the technical condition looks, this is an easy problem. For this location family, just show that
    \[
    \left|\frac{1}{\Delta} \frac{f(x \mid \theta+\Delta)-f(x \mid \theta)}{f(x \mid \theta)}\right| \leq 2+\varepsilon
    \]
    This can be done by expanding the left-hand side of the inequality. Now define
    \[
    b_{\theta}(X):=2+\varepsilon,
    \]
    and note that
    \[
    \mathbb{E}_{\theta} b_{\theta}^{2}(X)<\infty
    \]
    By using the lemma above, conclude that the information inequality holds for any estimator \(\delta(X)\) such that \(\left.\mathbb{E}_{\theta} \delta^{2}(X)<\infty\right)\)
\end{ex}

\begin{solution}
    For Cauchy$(0,1)$,
    \begin{align*}
        &\quad\,\left|\frac{1}{\Delta} \frac{f(x \mid \theta+\Delta)-f(x \mid \theta)}{f(x \mid \theta)}\right|  \\
        &= \left|\frac{1}{\Delta} \frac{1+(x-\theta)^2-1-(x-\Delta-\theta)^2}{1+(x-\Delta-\theta)^2}\right|\\
        &= \left|\frac{1}{\Delta} \frac{2\Delta(x-\theta)-\Delta^2}{1+(x-\Delta-\theta)^2}\right|\\
        &= \left|2\frac{x-\theta}{1+(x-\Delta-\theta)^2}-\frac{\Delta}{1+(x-\Delta-\theta)^2}\right|\\
        &= \left|2\frac{x-\Delta-\theta}{1+(x-\Delta-\theta)^2}+\frac{\Delta}{1+(x-\Delta-\theta)^2}\right|\\
        &\leqslant2\frac{|x-\Delta-\theta|}{1+(x-\Delta-\theta)^2}+\frac{|\Delta|}{1+(x-\Delta-\theta)^2}\\
        &\leqslant2+\varepsilon.
    \end{align*}
    Because $|x|<1+x^2$. Let $b(X)=2+\varepsilon$, then $\mathbb{E}b^2<\infty$. So, the information inequality holds for any estimator $\delta(X)$ with $\mathbb{E}_{\theta} \delta^{2}(X)<\infty$.
    \[
        \mathcal{L}'=\frac{\pder}{\pder \theta}\log f(x)=-\frac{2(x-\theta)}{1+(x-\theta)^2},
    \]
    \[
        \begin{aligned}
            E\left(\mathcal{L}'\right)^2&=4E\left(\frac{x-\theta}{1+(x-\theta)^2}\right)^2,
        \end{aligned}
    \]
    \[
        \begin{aligned}
            E_{\theta}\left[\frac{X-\theta}{1+(X-\theta)^2}\right]^2
            &=\frac{1}{\pi}\int_{\mathbb R}\left[\frac{x-\theta}{1+(x-\theta)^2}\right]^2\frac{1}{1+(x-\theta)^2}\der x
            \\&=\frac{1}{\pi}\int_{\mathbb R}\frac{(x-\theta)^2}{(1+(x-\theta)^2)^3}\der x\\&=\frac{2}{\pi}\int_0^\infty\frac{t^2}{(1+t^2)^3}\der t
            \\&=\frac{1}{\pi}\int_0^\infty\frac{\sqrt u}{(1+u)^3}\der u
            \\&=\frac{1}{\pi}B\left(\frac{3}{2},\frac{3}{2}\right)
            \\&=\frac{1}{8}
        \end{aligned}
    \]
    So, $I(\theta)=1/2$,
    \[
        Var(\delta)\geqslant2
    \]
\end{solution}

\begin{ex}
    The following theorem establishes a bound for the variance of an unbiased estimator in a multiparameter setting.

    Suppose the following conditions hold.
    \begin{enumerate}[(i)]
        \item \(\delta(\boldsymbol{X}) \in \Delta\) : an \(\mathbb{R}\)-valued unbiased estimator;
        \item \(\boldsymbol{\Psi}(\boldsymbol{X} \mid \theta)\) : an \(\mathbb{R}^{r}\)-valued function with finite second moments;
        \item \[
        \begin{aligned}
            \mathbb{R}^{r} \ni \gamma(\theta) &= Cov_{\theta}(\delta(\boldsymbol{X}), \boldsymbol{\Psi}(\boldsymbol{X} \mid \theta))\\
            &:=\mathbb{E}_{\theta}\left\{\left(\delta(\boldsymbol{X})-\mathbb{E}_{\theta} \delta(\boldsymbol{X})\right)\left(\boldsymbol{\Psi}(\boldsymbol{X} \mid \theta)-\mathbb{E}_{\theta}(\boldsymbol{\Psi}(\boldsymbol{X} \mid \theta))\right)\right\};
        \end{aligned}
        \]
        \item \[
            \begin{aligned}
                C(\theta) &= Cov_{\theta}(\boldsymbol{\Psi}(\boldsymbol{X} \mid \theta), \boldsymbol{\Psi}(\boldsymbol{X} \mid \theta))\\
                &:=\mathbb{E}_{\theta}\left\{\boldsymbol{\Psi}(\boldsymbol{X} \mid \theta)-\mathbb{E}_{\theta}(\boldsymbol{X} \mid \theta)\right\}\left\{\boldsymbol{\Psi}(\boldsymbol{X} \mid \theta)-\mathbb{E}_{\theta} \boldsymbol{\Psi}(\boldsymbol{X} \mid \theta)\right\}^T;
            \end{aligned}
        \]
    \end{enumerate}
    Then,
        \[
            Var_{\theta} \delta(\boldsymbol{X}) \geq \gamma^T(\theta) C^{-1}(\theta) \gamma(\theta)
        \]
    We will break up its proof into simple steps.
        \begin{enumerate}[(a)]
            \item Let \(\mathbf{a} \in \mathbb{R}^{r}\). Show that
            \[
                Var_{\theta} \delta(\mathbf{X}) \geq \frac{\left[ Cov_{\theta}\left(\delta(\mathbf{X}), \mathbf{a}^T \boldsymbol{\Psi}(\mathbf{X} \mid \theta)\right)\right]^{2}}{ Var_{\theta}\left(\mathbf{a}^T \boldsymbol{\Psi}(\mathbf{X})\right)};
            \]
            \item Conclude that
            \[
                Var_{\theta} \delta(\mathbf{X}) \geq \sup _{\mathbf{a} \in \mathbb{R}^{n}} \frac{\left(\mathbf{a}^T \gamma(\theta)\right)^{2}}{\mathbf{a}^T C(\theta) \mathbf{a}}.
            \]
            \item Consider the following result.
            Let \(p\) be a \(r \times 1\) column vector, let \(P=p p^T\), and let \(Q\) be an \(r \times r\) real matrix. Then
            \[
            \sup _{\mathbf{a} \in \mathbb{R}^{r}} \frac{\mathbf{a}^T P \mathbf{a}}{\mathbf{a}^T Q \mathbf{a}}=\text { largest eigenvalue of } Q^{-1} P=p^T Q^{-1} p.
            \]
            Use the lemma to conclude that
            \[
             Var_{\theta} \delta(\mathbf{X}) \geq \gamma^T(\theta) C^{-1}(\theta) \gamma(\theta).
            \]
        \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item From Cauchy-Schwarz inequality,
        \[
            [Cov(X,Y)]^2\leqslant Var(X) Var(Y).
        \]
        So, \[
            Var(\delta(X))\geqslant \frac{[Cov(\delta(X), a^T\psi(X))]^2}{Var(a^T\psi(X))}.
        \]
        \item $Cov(\delta(X), a^T\psi(X))=a^TCov(\delta(X), \psi(X))$, $Var(a^T\psi(X))=a^T\psi(T(X))a$. So,
        \[
            Var(\delta(X))\geqslant \max_a\frac{[a^T\gamma(\theta)]^2}{a^TC(\theta)a}.
        \]
        \item Using the lemma, $P=\gamma\gamma^T$
        \[
            Var(\delta(X))\geqslant \gamma^T(\theta)C^{-1}(\theta) \gamma(\theta).
        \]
    \end{enumerate}
\end{solution}

\begin{ex}
    Consider the following set of assumptions.
    \begin{enumerate}[(a)]
        \item \(\Theta \subseteq \mathbb{R}^{r}\) is an open rectangle;
        \item the set
        \(
            A=\{\mathbf{x}: f(\mathbf{x} \mid \theta)>0\}
        \)
        is independent of \(\theta\);
        \item \(\frac{\partial}{\partial \theta} f(\mathbf{x} \mid \theta)\) exists for all \(\theta \in \Theta\).
    \end{enumerate}
    For \(\theta \in \mathbb{R}^{r}\), define the information matrix at \(\theta\) by
    \[
        I(\theta)=\mathbb{E}_{\theta}\left(\frac{\partial}{\partial \theta} \log f(\mathbf{X} \mid \theta)\right)\left(\frac{\partial}{\partial \theta} \log f(\mathbf{X} \mid \theta)\right)^T.
    \]
    Prove the following theorem (a particular case of Theorem 2.6.6 in Lehmann \& Casella \((1998)\), p. 127).

    \thm{1} (multiparameter information inequality)

    Suppose the following assumptions hold.
    \begin{enumerate}[(i)]
        \item \(\{f(x \mid \theta)\}_{\theta}\) is a family of generalized densities with respect to the \(\sigma\)-finite measure \(G(d x)\);
        \item \(\mathrm{A}-(\mathrm{a}),(\mathrm{b}),(\mathrm{c})\);
        \item \(\mathbb{E}_{\theta} \nabla_{\theta} \log f(\boldsymbol{X} \mid \theta)=0, \theta \in \Theta\);
        \item \(I(\theta)\) is nonsingular, \(\theta \in \Theta\);
        \item \(\mathbb{E}_{\theta} \delta^{2}(\boldsymbol{X})<\infty, \theta \in \Theta\);
        \item \(\nabla_{\theta} \mathbb{E}_{\theta} \delta(\boldsymbol{X})=\int \delta(\boldsymbol{x}) \frac{\partial}{\partial \theta} f(\boldsymbol{x} \mid \theta) G(d \boldsymbol{x})\).
    \end{enumerate}
    Then,
    \[
         Var_{\theta} \delta(\boldsymbol{X}) \geq \nabla_{\theta} \mathbb{E}_{\theta} \delta(\boldsymbol{X}) I(\theta)^{-1} \nabla_{\theta} \mathbb{E}_{\theta} \delta(\boldsymbol{X})^T, \quad \theta \in \Theta
    \]
    (hint: use the previous problem).
\end{ex}

\begin{solution}
    In previous problem, let $\psi(X)=\frac{\pder }{\pder \theta}\log f(x)$, then
    \[
        C=I(\theta);
    \]
    \[
        \begin{aligned}
            \nabla_\theta E(\delta)^T&=\frac{\partial}{\partial \theta}\int \delta(X)f(X)\der x\\
            &=\int\delta(X)f'(X)\der x\\
            &=\int\delta(X)\frac{\pder}{\pder \theta}\log(f(X))f(X)\der x\\
            &=\int\delta(X)\psi(X)f(X)\der x\\
            &=E\left(\delta(X)-E(\delta(X))\right)\left(\psi(X)-E(\psi(X))\right)\\
            &=Cov(\delta, \psi)\\
            &=\gamma.
        \end{aligned}
    \]
    Hence, we finish the proof.
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu, \sigma^{2}\right), \mu \in \mathbb{R}, \sigma^{2}>0\).
    \begin{enumerate}[(a)]
        \item Prove that
        \[
        I\left(\mu, \sigma^{2}\right)=\left(\begin{array}{cc}
        \frac{n}{\sigma^{2}} & 0 \\
        0 & \frac{n}{2 \sigma^{4}}
        \end{array}\right)
        \]
        Here you can use the equality
        \[
        \mathbb{E}_{\theta} \frac{\partial}{\partial \theta} \log f(\mathbf{X} \mid \theta) \frac{\partial}{\partial \theta} \log f(\mathbf{X} \mid \theta)^T=-\mathbb{E}_{\theta} \frac{\partial^{2}}{\partial \theta \partial \theta^T} \log f(\mathbf{X} \mid \theta).
        \]
        \item Conclude that the variance of an UMVU estimator does not necessarily attain the CRLB.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item Knowing that
        \[
            I(\theta;\mathbf{X})=nI(\theta;X),
        \]
        \[
            I(\mu)=\frac{n}{\sigma^2}.
        \]
        \[
            I(\sigma^2)=\frac{n}{2\sigma^4}.
        \]
        \[
            \frac{\pder}{\pder \mu \,\pder \sigma}\log(f(x))=\frac{\pder}{\pder \sigma \,\pder \mu}\log(f(x))=-2(x-\mu)\sigma^{-3},
        \]
        \[
            E\left(-2(x-\mu)\sigma^{-3}\right)=0.
        \]
        So,
        \[
            E\left(\frac{\pder}{\pder \mu \,\pder \sigma^2}\log(f(x))\right)=E\left(\frac{\pder}{\pder \sigma^2 \,\pder \mu}\log(f(x))\right)=0.
        \]
        \item Consider $\frac{(n-1)S^2}{\sigma^2}\sim \chi^2
        (n-1)$,
        \[
            Var\left(\frac{\sigma^2}{n-1}\frac{(n-1)S^2}{\sigma^2}\right)=\frac{\sigma^4}{(n-1)^2}2(n-1)=\frac{2\sigma^4}{n-1}>\frac{2\sigma^4}{n}.
        \]
    \end{enumerate}
\end{solution}

\begin{ex}
    (n.b.: the purpose of this exercise is to provide a simple 1-dimensional example where the variance of the UMVU estimator does not attain the CRLB)

    Let \(X \sim \operatorname{Poi}(\lambda), \lambda>0\), and let
    \[
        \delta(X):= \begin{cases}1, & \text { if } X=0 \\ 0, & \text { otherwise }\end{cases}.
    \]
    \begin{enumerate}[(a)]
        \item Conclude that \(\delta(X)\) is UMVU for \(\mathbb{E}_{\lambda} \delta(X)\).
        \item Find \(\mathbb{E}_{\lambda} \delta(X)\) and \(\operatorname{Var}_{\lambda} \delta(X)\).
        \item Find the CRLB for \(\operatorname{Var}_{\lambda} \delta(X)\) (hint: use the formula for the change of variables in the information function).
        \item Conclude that the CRLB cannot be met.
        \item Why doesn't this contradict the result we showed in class on the attainment of the CRLB in exponential families? (see problem \#2 in this Problem set.)
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item $\delta(X)$ is unbiased, and $X$ is a sufficient statistic. So, $\delta(X)$ is UMVU.
        \item \[
            E(\delta(X))=1\cdot e^{-\lambda}=e^{-\lambda}, \ Var(\delta(X))=E(\delta(X))^2-(E\delta(X))^2=\frac{e^\lambda-1}{e^{2\lambda}}.
        \]
        \item \[
            Var(\delta)\geqslant\frac{\left(-e^{-\lambda}\right)^2}{I(\lambda)}=\frac{\lambda}{e^{2\lambda}}.
        \]
        \item When $\lambda>0$, $e^{\lambda}-1>\lambda$. Hence, CRLB cannot be met.
        \item Because $\delta(X)$ is not $C^1$.
    \end{enumerate}
\end{solution}

\begin{ex}
    Let \(\mathbf{X}\) and \(\mathbf{Y}\) be independent random vectors, where \(\mathbf{X} \sim f(\mathbf{x} \mid \theta), \mathbf{Y} \sim g(\mathbf{y} \mid \theta), \theta \in \Theta\), where \(f\) and \(g\) are generalized densities. Assume the following conditions hold.
    \begin{enumerate}[(i)]
        \item A-(a),(b), (c);
        \item \[
            \mathbb{E}_{\theta}\left(\frac{\partial}{\partial \theta} \log f(\mathbf{X} \mid \theta)\right)=0=\mathbb{E}_{\theta}\left(\frac{\partial}{\partial \theta} \log g(\mathbf{X} \mid \theta)\right), \quad \theta \in \Theta.
        \]
    \end{enumerate}
    \begin{enumerate}[(a)]
        \item Show that \(I_{\mathbf{X}, \mathbf{Y}}(\theta)=I_{\mathbf{X}}(\theta)+I_{\mathbf{Y}}(\theta)\), namely, the information contained in each subsample adds up to the information in the whole sample.
        \item Use (a) to rewrite the CRLB for an i.i.d. sample \(X_{1}, \ldots, X_{n}\).
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item Let $\psi(X)=\frac{\pder }{\pder \theta}\log f(X; \theta)$, then
        \[
            \begin{aligned}
                I_{X,Y}(\theta)&=E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)f(Y;\theta)\right)^2\\
                &=E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)+\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2\\
                &=E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2+E\left(\frac{\pder}{\pder \theta}\log f(Y;\theta)\right)^2\\
                &=I_X(\theta)+I_Y(\theta)
            \end{aligned}
        \]
        \item Let $\delta(X)$ be an unbiased statistics for $\theta$,
        \[
            I_X(\theta)=\sum_{i=1}^nI_{X_i}(\theta)=nI_x(\theta).
        \]
        \[
            Var_\theta(\delta(X))\geqslant\frac{1}{nI_x(\theta)}.
        \]
    \end{enumerate}
\end{solution}

\begin{ex}
    Consider an estimator \(\delta(\mathbf{X}) \in \Delta\) under a parametric family \(\mathbf{X} \sim \mathcal{P}=\{f(\mathbf{x} \mid \theta)\}_{\theta \in \Theta}\), where \(f(\mathbf{x} \mid \theta)\) is a generalized density. Suppose, in addition, that the following conditions hold.
    \begin{enumerate}[(i)]
        \item \(\mathbb{E}_{\theta} \delta(\mathbf{X})=g(\theta), \theta \in \Theta\), where the function \(g(\cdot)\) is differentiable;
        \item the estimator can be explicitly written as
        \[
        \delta(\mathbf{x})=g(\theta)+\frac{g^{\prime}(\theta)}{I(\theta)} \frac{\partial}{\partial \theta} \log f(\mathbf{x} \mid \theta), \quad I(\theta)>0, \quad \theta \in \Theta.
        \]
        \item \[
            \frac{\partial}{\partial \theta} \int_{\mathbb{R}^{n}} f(\mathbf{x} \mid \theta) G(d \mathbf{x})=\int_{\mathbb{R}^{n}} \frac{\partial}{\partial \theta} f(\mathbf{x} \mid \theta) G(d \mathbf{x}), \quad \theta \in \Theta .
        \]
        Show that \(\operatorname{Var}_{\theta} \delta(\mathbf{X})\) attains the CRLB.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{align*}
        Var(\delta(X))&=E(\delta(X)-E\delta(X))^2\\
        &=E\left(\frac{g'(\theta)}{I(\theta)}\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2\\
        &=\left(\frac{g'(\theta)}{I(\theta)}\right)^2E\left(\frac{\pder}{\pder \theta}\log f(X|\theta)\right)^2\\
        &=\left(\frac{g'(\theta)}{I(\theta)}\right)^2I(\theta)\\
        &=\frac{(g'(\theta))^2}{I(\theta)}\\
        &=\frac{\left(\frac{\pder}{\pder \theta}E(\delta(X))\right)^2}{I(\theta)}\\
        &=CRLB.
    \end{align*}
\end{solution}


\section{Methods for pattern selection}

\subsection{Based on Conditional Mutual Information}
\begin{itemize}
\item When studying methylation we are faced with two main questions:
  \begin{enumerate}
  \item Which genes exhibit an L-shape, and
  \item What is the optimal threshold for binarizing
methylation data for each L-shape gene.
  \end{enumerate}
\item Following \cite{Liu} in order to determine whether methylation $X$ and expression $Y$ of a gene exhibit an L--shape, the conditional Mutual Information $cMI(t)$ for different choices of threshold $t$ is computed.
\[
\mathit{cMI}(t)=I(X,Y|X>t)P(X>t) + I(X,Y|X\le t)P(X\le t)
\]
\item If the relation between methylation and expression shows an L-shape  as $t$ moves from 0 to 1, $\mathit{cMI}(t)$ first decreases and then increases, its value approaching zero when $t$ coincides with the reflection point.
\end{itemize}


\documentclass[a4paper,10pt]{article}

\usepackage[utf8]{inputenc}

\usepackage{mathtools}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{graphicx}
\usepackage{listings}
\usepackage{hyperref}
\usepackage[show]{ed}

\usepackage[english]{babel}

\title{Elements of Stochastic Processes\\Asssignment Sheet 2}
\author{Tom Wiesing}
\date{\today}

\begin{document}
\maketitle

\section{Exercise 5}
\subsection{Problem}

Assume that $X$ is a real valued continuous random variable with density $f_X$
and cumulative distribution function $F_X$. Show that $$
  \mathbb{E}\left[X\right] = \int_{-\infty}^{0}{F_x(t)\mathrm{d}t} + \int_{0}^{\infty}{1 - F_x(t)\mathrm{d}t}
$$.

\subsection{Solution}

For now assume that $X \geq 0$, i.e. $X$ never assumes negative values. Then we have:

$$
  \mathbb{E}\left[X\right] = \int_{0}^{\infty}{t f_x(t)\mathrm{d}t} = \int_{-\infty}^{\infty}{t P[X = t]\mathrm{d}t}
$$
$$
  = \int_{0}^{\infty}{P[X > t]\mathrm{d}t} = \int_{0}^{\infty}{1 - P[X \leq t]\mathrm{d}t} = \int_{0}^{\infty}{1 - F_X(t)\mathrm{d}t}
$$ Notice that in the middle of this computation, we switch the integral around.
Intuitively, the integral measures the area under the curve $t P[X = t]$ on the right hand side of the origin by slicing it vertically.
Equivalently, we can also measure the area by slicing it horizontally, which is what we are doing in the next line.

To compute the expected value of a generalised random variable (which may now be negative), we write it as a sum of two random variables $$
X = Y - Z
$$ with both $Y, Z \geq 0$. As the expected value is linear, we can then write

$$
  \mathbb{E}\left[X\right] = \mathbb{E}\left[Y\right] - \mathbb{E}\left[Z\right] = \int_{0}^{\infty}{1 - F_Y(t)\mathrm{d}t} - \int_{0}^{\infty}{1 - F_Z(t)\mathrm{d}t}
$$
$$
  = \int_{0}^{\infty}{1 - F_X(t)\mathrm{d}t} + \int_{-\infty}^{0}{F_x(t)\mathrm{d}t}
$$.

\section{Exercise 6}
\subsection{Problem}

For $\alpha \geq 1$ suppose that the random variable $X$ has the density function
$$
f_x(t) =
\begin{cases}
  \alpha e^{-\alpha t} & \mbox{if } t \geq 0 \\
  0 & \mbox{else}
\end{cases}
$$ Compute $\mathbb{E}[e^X]$.

\subsection{Solution}

$$
\mathbb{E}[e^X] = \int_{-\infty}^{\infty}{e^t \begin{cases}
  \alpha e^{-\alpha t} & \mbox{if } t \geq 0 \\
  0 & \mbox{else}
\end{cases}\mathrm{d}t}
= \int_{0}^{\infty}{e^t e^{-\alpha t} \mathrm{d}t}
= \int_{0}^{\infty}{e^{(1 - \alpha) t} \mathrm{d}t}
= \frac{1}{\alpha - 1}
$$.

{\raggedleft{}$\square$}

\section{Exercise 7}
\subsection{Problem}

Let $X$ and $Y$ be independent random variables with uniform distribution in the interval $[0, 1]$. Let $Z$ = $XY$. Find
\begin{enumerate}[a)]
  \item the joint probability distribution function of $X$ and $Y$,
  \item the joint probability distribution function of $X$ and $Z$,
  \item the joint density function of $X$ and $Z$
\end{enumerate}.

\subsection{Solution}
\subsubsection{a)}
$$
  F(x, y) = P[X \leq x, Y \leq y] = P[X \leq x]P[Y \leq y] = x y
$$

\subsubsection{b)}
$$
  F(x, z) = P[X \leq x, Z \leq z] = P[X \leq x, X + Y \leq z]
$$
$$
  = P[X \leq x, Y \leq z - x] = P[X \leq x]P[Y \leq z - x] = x (z - x)
$$

\subsubsection{c)}
$$
  f(x, z) = P[X = x, Z = z] = P[X = x, X + Y = z]
$$
$$
   = P[X = x, Y = z - x] = P[X = x]P[Y = z - x] = 1
$$

\section{Exercise 8}
\subsection{Problem}

Let $n \geq 2$ be a natural number and let the joint probability mass function of the discrete random variables $X$ and $Y$ be given by
$$
  p_{X,Y}(x, y) = \begin{cases}
    k (x + y) & \mbox{if }1 \leq x,y \leq n \\
    0 & \mbox{else}
  \end{cases}
$$

\begin{enumerate}[a)]
  \item Determine the value of the constant $k$.
  \item Determine the marginal probability mass functions of $X$ and $Y$.
  \item Find $P(X \geq Y)$. \\
  \textit{Hint: You can simplify the calculations by observing that $P(X \geq Y) = P(Y \geq X)$. }
\end{enumerate}

\subsection{Solution}
\subsubsection{a)}
We observe that $$
  1 = \sum_{x = 1}^{n}{\sum_{y = 1}^{n}{k (x + y)}} = k n^2 (n + 1)
$$ and thus have $k = \frac{1}{n^2 (n + 1)}$.
\subsubsection{b)}
We observe that $$
  P[X = x] = \sum_{y = 1}^{n}{k (x + y)} = k (\sum_{y = 1}^{n}{(x + y)})
  = k (nx + \sum_{y = 1}^{n}{y})
  = \frac{x}{n (n + 1)} + \frac{1}{2 n}
$$ By symmetry, we have
$$
  P[Y = y] = \frac{y}{n (n + 1)} + \frac{1}{2 n}
$$
\subsubsection{c)}
$$
  P[X \leq Y] = \sum_{y=1}^{n}{P[X \leq y, Y=y]} = \sum_{y=1}^{n}{\sum_{x=1}^{y}{P[X = x, Y = y]}}
$$
$$
  = \sum_{y=1}^{n}{\sum_{x=1}^{y}{k (x + y)}} = \frac{1}{2} k n (n + 1)^2 = \frac{1}{2n}
$$

\end{document}


\lab{Image Segmentation}{Image Segmentation}

\objective{Understand some basic applications of eigenvalues to graph theory.}
\label{lab:ImgSeg_eigenvalues}

\section*{Graph Theory}
\begin{figure}

 \begin{tikzpicture}[auto,node distance=1.5cm,
 thick,main node/.style={circle,draw}]

  \node[main node] (5) [] {6};
  \node[main node] (2) [below right of=5] {3};
  \node[main node] (3) [above right of=5] {4};
  \node[main node] (4) [right of=3] {5};
  \node[main node] (1) [right of=2] {2};
  \node[main node] (0) [below right of=4] {1};

  \foreach \s/\t in {5/3, 3/4, 4/0, 0/1, 1/2, 2/3, 1/4, 5/0} {
   \path[draw] (\s) edge (\t);}
\end{tikzpicture}
\caption{An undirected graph that is connected.}
\label{fig:example_graph}
\end{figure}

\begin{figure}

 \begin{tikzpicture}[auto,node distance=2cm,
 thick,main node/.style={circle,draw}]

  \node[main node] (0) [] {1};
  \node[main node] (1) [below of=0] {2};
  \node[main node] (2) [right of=0] {3};
  \node[main node] (3) [below of=2] {4};
  \node[main node] (4) [right of=2] {5};
  \node[main node] (5) [right of=3] {6};

   \path[draw] (0) edge node [left] {3} (1);
   \path[draw] (2) edge node [left] {1} (3);
   \path[draw] (4) edge node{-1} (5);
   \path[draw] (3) edge node{2} (4);
   \path[draw] (3) edge node [below]{.5} (5);
\end{tikzpicture}
\caption{A weighted undirected graph that is not connected.}
\label{fig:example_graph2}
\end{figure}

% \begin{tikzpicture}[auto,node distance=1.5cm,
% thick,main node/.style={circle,draw}]
%
%  \node[main node] (2) [] {2};
%  \node[main node] (1) [below left of=2] {1};
%  \node[main node] (0) [below right of=2] {0};
%
%  \foreach \s/\t in {1/1, 1/2, 1/3, 2/3} {
%   \path[draw] (\s) edge (\t);}
%\end{tikzpicture}
%\caption{An undirected graph that is not simple.}
%\label{fig:example_graph}
%\end{figure}

%\begin{figure}
% \begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto,node distance=1.5cm,
% thick,main node/.style={circle,draw}]
%
%  \node[main node] (A) [] {A};
%  \node[main node] (B) [below of=A] {B};
%  \node[main node] (C) [right of=A] {C};
%  \node[main node] (D) [below of=C] {D};
%  \node[main node] (E) [right of=C] {E};
%  \node[main node] (F) [right of=D] {F};
%
%  \foreach \s/\t in {A/C, B/A, B/D, C/E, C/F, D/C} {
%   \path[draw] (\s) edge (\t);}
%\end{tikzpicture}
%\caption{A simple directed graph}
%\end{figure}

Graphs represent relationships between objects.
%For example, the transition diagram in Figure \ref{fig:markov1} in Lab \ref{lab:EigSolve} shows the relationship between states in a Markov chain.
An \emph{undirected graph} is a set of nodes (or vertices) and edges, where each edge connects exactly two nodes and (see Figure \ref{fig:example_graph}).
A \emph{directed graph} has the additional information of an arrow on each edge.
In this lab we will only consider undirected graphs, which we will simply call graphs (unless we wish to emphasize the fact that they are undirected).

%A graph is \emph{simple} if no edge connects a node to itself.
%The graph in Figure [TODO!] is simple, but the graph in Figure [TODO!] is not.

A \emph{weighted} graph is a graph with a weight attached to each edge.
For example, a weighted graph could represent a collection of cities with roads connecting them.
The vertices would be cities, the edges roads, and weight of an edge would be the length of a road.
Such a graph is depicted as in Figure \ref{fig:example_graph2}.

Any graph can be thought of as a weighted graph by assigning a weight of 1 to each edge.

\subsection*{Adjacency, degree, and Laplacian matrices}
We will now introduce three special matrices associated to a graph.
Throughout this section, assume we are working with a weighted undirected graph with $N$ nodes, and that $w_{ij}$ is the weight attached to the edge connecting node $i$ and node $j$.
We first define the adjacency matrix.

\begin{definition} The \emph{adjacency matrix} is an $N \times N$ matrix whose $(i,j)$-th entry is
\begin{center}
	$ \begin{cases}  w_{ij} & \mbox{if an edge connects node i and node j} \\ 0 & \mbox{otherwise.} \end{cases}$
\end{center}
% If the graph is not simple, there are differing conventions for how to define the diagonal of the adjacency matrix.
\end{definition}

For example, the graph in Figure \ref{fig:example_graph} has the adjacency matrix $A_1$ and the graph in Figure \ref{fig:example_graph2} has the adjacency matrix $A_2$, where
\[
A_1 = \begin{pmatrix}
0 & 1 & 0 & 0 & 1 & 1\\
1 & 0 & 1 & 0 & 1 & 0\\
0 & 1 & 0 & 1 & 0 & 0\\
0 & 0 & 1 & 0 & 1 & 1\\
1 & 1 & 0 & 1 & 0 & 0\\
1 & 0 & 0 & 1 & 0 & 0
\end{pmatrix}. \qquad A_2 =
 \begin{pmatrix}
0 & 3 & 0 & 0 & 0 & 0\\
3 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 1 & 0 & 0\\
0 & 0 & 1 & 0 & 2 & .5\\
0 & 0 & 0 & 2 & 0 & -1\\
0 & 0 & 0 & .5 & -1 & 0
\end{pmatrix}
\]
Notice that these adjacency matrices are symmetric. This will always be the case for undirected graphs.

\begin{comment}
Raising the adjacency matrix to a power yields some very interesting information.
We can discover the number of paths of length $n$ between two nodes by raising a graph's adjacency matrix to the $n$th power.
For example, by squaring $A$, we can find the number of paths of length two between every pair of nodes.
\begin{lstlisting}
>>> A = np.array([[0,1,0,0,1,0],[1,0,1,0,1,0],
                  [0,1,0,1,0,0],[0,0,1,0,1,1],
                  [1,1,0,1,0,0],[0,0,0,1,0,0]])

>>> np.linalg.matrix_power(A,2)
array([[2, 1, 1, 1, 1, 0],
       [1, 3, 0, 2, 1, 0],
       [1, 0, 2, 0, 2, 1],
       [1, 2, 0, 3, 0, 0],
       [1, 1, 2, 0, 3, 1],
       [0, 0, 1, 0, 1, 1]])
\end{lstlisting}
We can see that no paths of length two exist between node 0 and node 5 because $A^2_{0,5} = 0$.
By calculating $A^6$ we can find the number of paths of length six from node 3 to itself.
\begin{lstlisting}
>>> np.linalg.matrix_power(A, 6)
array([[45, 54, 38, 45, 54, 16],
       [54, 86, 29, 77, 51, 11],
       [38, 29, 55, 15, 70, 27],
       [45, 77, 15, 75, 31,  4],
       [54, 51, 70, 31, 93, 34],
       [16, 11, 27,  4, 34, 14]])
\end{lstlisting}
We see that there are 75 unique paths of length six from node 3 to itself.
Imagine trying to count all of those paths by hand!
It would be very easy to count incorrectly.
This method makes it very simple to count paths without mistakes.

Adjacency matrices can also be composed of \li{True} and \li{False} values.
In this case, the $n$th power of such a matrix (using boolean arithmetic)
is again a matrix of
boolean values which simply indicate whether there exists a path of length $n$ between the given pair of nodes, rather than indicating the number of such
paths.

\begin{problem}
Let the following matrix represent a directed graph
\[
\begin{pmatrix}
0 & 0 & 1 & 0 & 1 & 0 & 1 \\
1 & 0 & 0 & 0 & 0 & 1 & 0 \\
0 & 0 & 0 & 0 & 0 & 1 & 0 \\
1 & 0 & 0 & 0 & 1 & 0 & 0 \\
0 & 0 & 0 & 1 & 0 & 0 & 0 \\
0 & 0 & 1 & 0 & 0 & 0 & 1 \\
0 & 1 & 0 & 0 & 0 & 0 & 0
\end{pmatrix}
\]
Between which pair of nodes does there exist the greatest number of paths
of length five?
From which node to which node is there no path of length seven?
\end{problem}
\end{comment}

The second matrix is the degree matrix.
\begin{definition} The \emph{degree matrix} is an $N \times N$ diagonal matrix whose $(i,i)$-th entry is
\[
\sum_{j=1}^N w_{ij}.
\]
This quantity is just the sum of the weight of each edge that touches node $i$.
\end{definition}
%For a directed graph, each node has an \emph{out-degree} (the number of edges directed away from a node) and an \emph{in-degree} (the number edges directed toward a node).
We call the $(i, i)-th$ entry of the degree matrix the \emph{degree} of node $i$. As an example, the degree matrices of the graphs in Figures \ref{fig:example_graph} and \ref{fig:example_graph2} are $D_1$ and $D_2$, respectively, where

\[
D_1 = \begin{pmatrix}
3 & 0 & 0 & 0 & 0 & 0\\
0 & 3 & 0 & 0 & 0 & 0\\
0 & 0 & 2 & 0 & 0 & 0\\
0 & 0 & 0 & 3 & 0 & 0\\
0 & 0 & 0 & 0 & 3 & 0\\
0 & 0 & 0 & 0 & 0 & 2
\end{pmatrix}. \qquad D_2 =
 \begin{pmatrix}
3 & 0 & 0 & 0 & 0 & 0\\
0 & 3 & 0 & 0 & 0 & 0\\
0 & 0 & 1 & 0 & 0 & 0\\
0 & 0 & 0 & 3.5 & 0 & 0\\
0 & 0 & 0 & 0 & 1 & 0\\
0 & 0 & 0 & 0 & 0 & -.5
\end{pmatrix}
\]

Finally, we can combine the degree matrix and the adjacency matrix into the Laplacian matrix.
% Wikipedia defines the Laplacian of a simple graph only. I don't know why.
% The graph in our application is NOT simple.
% However, the non-simple parts cancel out, meaning that the Laplacian of the graph is the same as if you removed all self edges and then computed the Laplacian.
% So I just define the Laplacian this way and don't talk about simple graphs.
\begin{definition}
The \emph{Laplacian matrix} of a graph is
\[D - A \]
where $D$ is the degree matrix and $A$ is the adjacency matrix of the graph.
\end{definition}

For example, the Laplacian matrix of the graphs in Figures \ref{fig:example_graph} and \ref{fig:example_graph2} are $L_1$ and $L_2$, respectively, where

\[
L_1 = \begin{pmatrix}
3 & -1 & 0 & 0 & -1 & -1\\
-1 & 3 & -1 & 0 & -1 & 0\\
0 & -1 & 2 & -1 & 0 & 0\\
0 & 0 & -1 & 3 & -1 & -1\\
-1 & -1 & 0 & -1 & 3& 0\\
-1 & 0 & 0 & -1 & 0 & 2
\end{pmatrix}. \qquad L_2 =
 \begin{pmatrix}
3 & -3 & 0 & 0 & 0 & 0\\
-3 & 3 & 0 & 0 & 0 & 0\\
0 & 0 & 1 & -1 & 0 & 0\\
0 & 0 & -1 & 3.5 & -2 & -.5\\
0 & 0 & 0 & -2 & 1 & 1\\
0 & 0 & 0 &- .5 & 1 & -.5
\end{pmatrix}
\]

In this lab we will learn about graphs by studying their Laplacian matrices.
While the Laplacian matrix seems simple, we can learn surprising things from its eigenvalues.

\begin{problem}
Write a function that accepts the adjacency matrix of a graph as an argument.
Your function should return the Laplacian matrix.
Hint: You can compute the diagonal of the degree matrix in one line by summing over an axis (see Lab \ref{lab:NumPyArrays}).
Another hint: Test your function on the graphs in Figures \ref{fig:example_graph} and \ref{fig:example_graph2}.
\label{prob:laplacian}
\end{problem}

\subsection*{Connectivity: first application of Laplacians}

A \emph{connected graph} is a graph where every vertex is connected to every other vertex by at least one path.
The graph in Figure \ref{fig:example_graph} is connected, whereas the graph in Figure \ref{fig:example_graph2} is not.
In applications, it is often important to know if a graph is connected.
A naive approach to determine connectivity of a graph is to search every possible path from each vertex.
While this works for very small graphs, most interesting graphs will have thousands of vertices (for example, the internet), and for such graphs this approach is not feasible.

Fortunately, there is a better way.
Recall that the adjacency matrix of an undirected graph is always symmetric.
Thus, it will have real eigenvalues.
Surprisingly, a graph is connected if the second smallest eigenvalue of its Laplacian matrix is positive. [TODO: cite something!]
In many applications the Laplacian matrix is sparse, so by taking advantage of this sparsity, we can cheaply determine if a graph is connected.

\begin{problem}
\leavevmode
\begin{enumerate}
\item Write a function that accepts a symmetric adjacency matrix as an argument and returns the second smallest eigenvalue of the Laplacian matrix.
Use the \li{scipy.linalg} package to compute the eigenvalue.

\item Here is a function that creates a random symmetric matrix of Boolean values with sparsity determined by the input \li{c}.
\begin{lstlisting}
def sparse_generator(n, c):
    ''' Return a symmetric nxn matrix with sparsity determined by c.
    Inputs:
        n -- an integer
        c -- a decimal number in [0,1]. Larger values of c will produce
             matrices with more entries equal to zero.
    '''
    A = np.random.rand(n**2).reshape((n, n))
    A = ( A > c**(.5) )
    return A.T.dot(A)
\end{lstlisting}
\end{enumerate}

Run your function from part 1 of this problem on matrices created by \li{sparse_generator} with inputs $n = 10, 100$ and $c = .25, .5, .95$.
What do you notice about the likelihood that a random graph is connected?
\end{problem}

\section*{Image Segmentation: second application of Laplacians}

\begin{figure}
    \centering
    \begin{subfigure}{0.31\textwidth}
        \includegraphics[width=\textwidth]{RegMon.png}
    \end{subfigure}
   \hspace*{\fill}
    \begin{subfigure}{0.31\textwidth}
        \includegraphics[width=\textwidth]{NegMon.png}
    \end{subfigure}
    \hspace*{\fill}
    \begin{subfigure}{0.31\textwidth}
        \includegraphics[width=\textwidth]{PosMon.png}
    \end{subfigure}

\caption{An image and its segments.}
\label{fig:monument}
\end{figure}
Image segmentation is the process of finding natural boundaries in an image (see for example Figure \ref{fig:monument}).
This is an easy task for humans, who can easily pick out portions of an image that ``belong together.''
In this lab, you will learn one way to program a computer to segment images.

The algorithm we will present comes from a paper by Jianbo Shi and Jitendra Malik in 2000 (\cite{Shi2000}).
Their idea is to represent an image as a weighted graph as follows.
To a computer, an \emph{image} is a collection of \emph{pixels}.
Each pixel has a brightness and coordinates describing its location in the image.
To define a graph representing this image, we let every pixel be a vertex.
Two pixels are connected if the distance between their coordinates is small (less than $r$).
The weight of the edge connecting two pixels is related their similarity in brightness, where a low weight means they are very different.

After defining this graph, we will segment the image by ``cutting'' (or removing) edges with low weights, which represent lines of high contrast in the image.
The ``cut'' is the total weight of the edges removed. Thus, to segment an image, we wish to minimize the ``cut.''
We can ``cut'' an image multiple times to segment an image into more than two pieces.

Now let us define the adjacency matrix of the graph associated to an image.
Since an $N \times N$ image has $N^2$ pixels, the adjacency matrix will be $N^2 \times N^2$.
After choosing a radius $r$ and some constants $\sigma_I$ and $\sigma_d$, we define the adjacency matrix to be $W = (w_{ij})$, where

\begin{equation}
\label{eq:adjacency}
w_{ij} = \begin{cases} \exp(-\frac{|I(i) - I(j)|}{\sigma_I^2}-\frac{d(i,j)}{\sigma_d^2}) & \mbox{ for $d(i,j) < r$} \\ 0 & \mbox{ otherwise,} \end{cases}
\end{equation}
where
\begin{itemize}
	\item$d(i,j)$ is the Euclidean distance between pixel $i$ and pixel $j$
	\item $|I(i) - I(j)|$ is the difference in brightness of pixels $i$ and $j$.
\end{itemize}

Notice that $W$ will be sparse as long as $r$ is small. Figure \ref{fig:adjacency} shows what the adjacency matrix looks like for a $4x4$ image when $r=1.2$.

\begin{figure}
\begin{tikzpicture}[dot/.style={circle,fill=black,minimum
	size=4pt,inner sep=0pt,outer sep=-1pt}, >=stealth]
%scale=.85, transform shape,

%image
\draw[step=.75,thick](2.999,0)grid(6,3);
%numbers 1-16
\foreach \x in {1,2,3,4}
	\foreach \y in {4}
		\node[draw=none, anchor=south west]at(\x*.75+2.6, \y-1.4){\x};
\foreach \x [evaluate=\x as \r using int(\x+4)]in {1,2,3,4}
	\foreach \y in {3}
		\node[draw=none, anchor=south west]at(\x*.75+2.6, \y-1.2){\r};
\foreach \x [evaluate=\x as \r using int(\x+8)] in {1,2,3,4}
	\foreach \y in {2}
		\node[draw=none, anchor=south west]at(\x*.75+2.5, \y-.95){\r};
\foreach \x [evaluate=\x as \r using int(\x+12)] in {1,2,3,4}
	\foreach \y in {1}
		\node[draw=none, anchor=south west]at(\x*.75+2.5, \y-.7){\r};

\node[draw=none](image)at(4.5, -.5){$image$};
\node[draw=none](flattened)at(7.25,-3){\textit{flattened image}};
\node[draw=none](adjacency)at(12,-3){\textit{adjacency image}};

%dots within grid
\foreach \x in {1,2,3,4}
	\foreach \y in {1,2,3,4}
		\node[draw, dot]at(\x*.75+2.6,\y*.75-.4){};

%color fill
\foreach \x/\y in {2/3.75, 1.25/3, 2/3, 2.75/3, 2/2.25} {\node[draw, minimum
	size=.75cm, fill=green!30!black, fill opacity=.25]at(\x+2.118,\y-1.118){};}

%circle in image
\node[draw, circle, minimum size=2cm,thick](circle)at(4.13,1.86){};

%flattened image
\draw[step=.5, thick](6.999,-2.5)grid(7.5,5.5);
\foreach \x in {7}
	\foreach \y in {1,...,16}
		\node[draw=none]at(\x+.25,\y*-.5+5.75){\y};

\draw[->,thick](6.1, 1.5)--(6.9,1.5);

%adjancey matrix
\draw[step=.5](7.9999,-2.5)grid(16,5.5);
%\draw[step=2,thick](7.999,-2.5)grid(16,5.5);

%outside labels
\foreach \x in {1,5,9,13} {\node[draw=none]at(\x*.5+7.75,5.8){\x};}
\foreach \y in {1,5,9,13}{\node[draw=none]at(16.3,\y*-.5+5.8){\y};}

%shading of boxes
\foreach \x/\y in {8.5/5.5, 9/5.5, 8.5/5, 9/5, 9.5/5,9/4.5,9.5/4.5,10/4.5,
	9.5/4, 10/4, 10.5/5.5, 11/5,11.5/4.5, 12/4, 12.5/3.5,
	13.5/2.5, 14/2, 14.5/1.5, 15/1, 15.5/.5,16/0, 10.5/3.5, 11/3.5, 11/2.5,
	11.5/2.5,12/2.5, 11.5/2,12/2, 12.5/1.5, 13/1.5, 12.5/1, 13/1,
	13.5/1,13/.5,13.5/.5,14/.5, 13.5/0,14/0, 14.5/-.5,15/-.5,14.5/-1,
	15/-1,15.5/-1, 15/-1.5, 15.5/-1.5, 16/-1.5, 15.5/-2, 16/-2, 8.5/3.5,
	9.5/2.5,10/2,10.5/1.5,11/1,11.5/.5,12/0, 12.5/-.5, 13/-1, 13.5/-1.5, 14/-2}
	{\node[draw, minimum size=.5cm, fill=black, fill opacity=.25]
	at(\x-.25,\y-.25){};}

%green shaded boxes
\foreach \x/\y in {9/3,11/3, 11.5/3, 10.5/3, 13/3} {\node
	[draw, minimum size=.5cm, fill=green!30!black, fill opacity=.25]
	at(\x-.25,\y-.25){};}

\node[draw=none]at(8.75,2.75){2};
\node[draw=none]at(10.25,2.75){5};
\node[draw=none]at(10.75,2.75){6};
\node[draw=none]at(11.25,2.75){7};
\node[draw=none]at(12.75,2.75){10};

\end{tikzpicture}

\caption{The grid at left represents a $4\times4$ image, which has 16 pixels. Thus the adjacency matrix at right is $16 \times 16$. We have not calculated the weights in the adjacency matrix, but we have shaded the nonzero entries. For example, the $6^{th}$ row corresponds to the $6^{th}$ pixel. Within that row, entries are nonzero if they correspond to pixels that are at most 1.2 away from pixel 6.}
\label{fig:adjacency}
\end{figure}

\subsection*{Computing the adjacency matrix}
Let us write a function that accepts an image \li{img} and constants \li{radius}, \li{sigma_I}, and \li{sigma_d}. The function will return the adjacency matrix defined in (\ref{eq:adjacency}) and the diagonal of the corresponding degree matrix. The input \li{img} should be a 2-D array of brightness values. Here is the function definition, which includes some default values for the constants.
\begin{lstlisting}
1. def adjacency(img, radius=5.0, sigma_I = .15, sigma_d = 1.7):
\end{lstlisting}

Later, our function will iterate through the rows of the adjacency matrix, initializing one row at a time. Each row corresponds to a pixel of the original image. Thus, we begin by flattening the image (see Figure \ref{fig:adjacency}). We also store the dimensions of \li{img} for later.
\begin{lstlisting}
2.     flat_img = img.flatten()
3.     height, width = img.shape
\end{lstlisting}

Next we want to initialize the adjacency and degree matrices. As can be seen in Figure \ref{fig:adjacency}, the adjacency matrix should be sparse, so we initialize it as a sparse matrix \li{W}. The sparse matrix type \li{lil_matrix} is optimized for building a matrix one entry at a time, which is what we will do. On the other hand, we only need to compute the diagonal of the degree matrix, so we initialize this diagonal as a regular NumPy array \li{D}.
\begin{lstlisting}
4.     W = spar.lil_matrix((flat_img.size, flat_img.size), dtype=float)
5.     D = np.zeros((1, flat_img.size))
\end{lstlisting}

Now for each pixel in the image, we initialize the corresponding row of the adjacency matrix.
The sum of the entries of this row will be the corresponding entry in \li{D}.

The function \li{getNeighbors} returns two flat arrays: \li{indices} and \li{distances} (code is provided at the end of this lab). The array \li{indices} contains the indices of those pixels with \li{radius} of the input \li{pixel}. The array \li{distances} contains the corresponding distances of those pixels from the input \li{pixel} (all entries of \li{distances} will be at most \li{radius}). According to (\ref{eq:adjacency}), the array \li{indices} contains exactly the indices of the nonzero entries of the current row of \li{W}.\footnote{Note that \li{W} will be a symmetric matrix. We could possibly speed up this function a lot by taking advantage of this fact.}

\begin{lstlisting}
6.     for pixel in xrange(flat_img.size):
7.         indices, distances = getNeighbors(pixel, radius, height, width)
8.         weights = # weights[j] should be W[pixel, indices[j]]
9.         W[pixel, indices] = weights
10.        D[0,pixel] = weights.sum()
\end{lstlisting}

Finally, we convert \li{W} to the sparse matrix type \li{csc_matrix}, which is faster for computations. Then we return \li{W} and \li{D}.
\begin{lstlisting}
11.    W = W.tocsc()
12.    return W, D
\end{lstlisting}

\begin{problem}
Finish writing the function \li{adjacency} described in this section by (a) adding comments and (b) filling in the following line.
\begin{lstlisting}
8.         weights = # weights[j] should be W[pixel, indices[j]]
\end{lstlisting}
Your computation should use (\ref{eq:adjacency}) and take exactly one line.

Images are typically stored as 2-D arrays of RGB values. To convert such an array to a 2-D array of brightness values, run the following code.
\begin{lstlisting}
# read in the image
img_color = plt.imread('dream.png')
# convert to grayscale
img = (img_color[:,:,0]+img_color[:,:,1]+img_color[:,:,2])/3.0
\end{lstlisting}
Test your function on the image \li{dream.png} (which is currently a 2-D array of RGB values).

%Notice that for each pixel you can save time by only checking the pixels $r$ rows and columns away.
%For that you'll have to handle the pixels on the edges and corners of the image carefully.
%I gave them new helper code, which abstracts away the edge cases.
\label{prob:adjacency_dream}
\end{problem}

\subsection*{Minimizing the 'cut'}

As was mentioned before, we are trying to minimize the `cut', or the total weight of the edges we remove to create segments.
Let $L$ be the Laplacian of the adjacency matrix defined in \ref{eq:adjacency} and let $D$ be the degree matrix.
Shi and Malik proved that we can minimize the `cut' by finding the second smallest eigenvalue of $D^{-1/2}LD^{-1/2}$.
Because both $D$ and $L$ will be symmetric matrices, all eigenvalues of $D^{-1/2}LD^{-1/2}$ will be real, and so it makes sense to ask for the second smallest one.

The eigenvector associated to the second smallest eigenvalue will have $N^2$ entries, some positive and some negative.
We can reshape this vector to be an $N \times N$ mask defining two segments, segments that correspond to the positive and negative values of the eigenvector.
Shi and Malik proved that these are the segments we desire.

Here is the definition of a function that will segment an image.
\begin{lstlisting}
1. def segment(img):
\end{lstlisting}

We use the function \li{adjacency} from Problem \ref{prob:adjacency_dream} to compute the adjacency matrix and the diagonal of the degree matrices of the image.
\begin{lstlisting}
2.     W, D = adjacency(img)
\end{lstlisting}

Next we create sparse matrices corresponding to $D$ and $D^{-1/2}$ in Shi and Malik's algorithm. Remember that the sparse matrix type \li{csc_matrix} is best for computations.
\begin{lstlisting}
3.     Dsq = # calculate the square root of D
4.     D_matrix = spar.spdiags(D, 0, D.shape[1], D.shape[1], format = 'csc')
5.     Dsq_matrix = # create a sparse matrix with diagonal Dsq
\end{lstlisting}
Now it is simple to compute $D^{-1/2}LD^{-1/2}$. We call this matrix \li{P}.
\begin{lstlisting}
6.     L = # compute the Laplacian
7.     P = # compute D^{-1/2}*L*D^{-1/2} as in Shi and Malik's algorithm
\end{lstlisting}
According to Shi and Malik, we need the eigenvector corresponding to the second smallest eigenvalue of \li{P}. We compute this with the \li{eigs()} method of the \li{scipy.sparse.linalg} module. We set the parameter \li{which='SR'} in the function call in order to compute the eigenvalues with Smallest Real part and their corresponding eigenvectors. The parameter \li{k} in the function call can be used to specify how many eigenvalues the method computes.
\begin{lstlisting}
8.     e = # compute the two smallest eigenvalues of P and their eigenvectors
9.     eigvec = # eigenvector of the second smallest eigenvalue
\end{lstlisting}

Next we create a mask that is \li{True} wherever \li{eigvec} is positive and reshape it to be the size of \li{img}.
\begin{lstlisting}
10.    mask = # create mask
\end{lstlisting}
Once we have a mask of True-False values, we multiply it by the image entrywise. This zeros out the pixels in the matrix corresponding to the \li{False} entries in the mask and leaves the pixels corresponding to \li{True} entries unaffected. We can negate the mask using the tilde operator, which lets us compute the other segment of the image. Finally we return the two segments.
\begin{lstlisting}
11.    pos = # compute positive segment
12.    neg = # compute negative segment
13.    return pos, neg
\end{lstlisting}

\begin{problem} Finish writing the function \li{segment} described in this section by (a) adding comments and (b) filling in the missing lines.
You should be able to fill in these lines with exactly the space indicated. Test your function on the image \li{dream.png}. Your segments should look like the segments in Figure \ref{fig:dream_solution} (the original image is on the left). Here are some hints:

\begin{enumerate}
\item Note that in Line 6, you should NOT use your solution to Problem \ref{prob:laplacian}, because in this problem we are working with sparse matrices, and you have already computed the degree matrix.

\item Also, in Line 7, multiply sparse matrices \li{A} and \li{B} with \li{A.dot(B)}.
\item Here is some code that will plot the segments \li{neg} and \li{pos} as well as the original \li{img}.
\begin{lstlisting}
plt.subplot(131)
plt.imshow(neg)
plt.subplot(132)
plt.imshow(pos)
plt.subplot(133)
plt.imshow(img_color)
plt.show()
\end{lstlisting}

\end{enumerate}

\end{problem}

\begin{figure}
\centering
    \centering
    \begin{subfigure}{0.31\textwidth}
        \includegraphics[width=\textwidth]{RegDream.png}
    \end{subfigure}
    \hspace*{\fill}
    \begin{subfigure}{0.31\textwidth}
        \includegraphics[width=\textwidth]{NegDream.png}
    \end{subfigure}
    \hspace*{\fill}
    \begin{subfigure}{0.31\textwidth}
        \includegraphics[width=\textwidth]{PosDream.png}
    \end{subfigure}
\caption{Segments of \li{dream.png}}
\label{fig:dream_solution}
\end{figure}

\section*{Appendix: helper code for Problem \ref{prob:adjacency_dream}}
Here is the function \li{getNeighbors} which you can use to compute the adjacency matrix of an image, as in Problem \ref{prob:adjacency_dream}.

\lstinputlisting[style=fromfile]{getNeighbors.py}


\section{The Chain Rule}\label{sec:multivariable chain rules}

Consider the surface $z=x^2y+xy^2$, and suppose that
$x=2+t^4$ and $y=1-t^3$. We can think of the latter two equations as
describing how $x$ and $y$ change relative to, say, time. Then
$$z=x^2y+xy^2=(2+t^4)^2(1-t^3)+(2+t^4)(1-t^3)^2$$
tells us explicitly how the $z$ coordinate of the corresponding point on the
surface depends on $t$. If we want to know $dz/dt$ we can compute it
more or less directly, but it's actually a bit simpler to use product and chain
rules:
\begin{align*}
  {dz\over dt}&=x^2y'+2xx'y+x2yy'+x'y^2	\\
  &=(2xy+y^2)x'+(x^2+2xy)y'	\\
  &=(2(2+t^4)(1-t^3)+(1-t^3)^2)(4t^3)+((2+t^4)^2+2(2+t^4)(1-t^3))(-3t^2)	\\
\end{align*}
If we look carefully at the middle step,
$dz/dt=(2xy+y^2)x'+(x^2+2xy)y'$, we notice that $2xy+y^2$ is $\partial
z/\partial x$, and $x^2+2xy$ is $\partial z/\partial y$.
This turns out to be true in general, and gives us a new chain rule:

\begin{theorem}{Multivariate Chain Rule}{MultivariateChainRule}
Suppose that $z=f(x,y)$, $f$ is differentiable,
$x=g(t)$, and $y=h(t)$.
Assuming that the relevant derivatives exist,
$${dz\over dt}={\partial z\over \partial x}{dx\over dt}+
{\partial z\over \partial y}{dy\over dt}.
$$\index{chain rule!multivariate}
\end{theorem}
\begin{proof}
If $f$ is differentiable, then
$$\Delta z=f_x(x_0,y_0)\Delta x+f_y(x_0,y_0)\Delta y+\epsilon_1\Delta
x + \epsilon_2\Delta y,$$
where $\epsilon_1$ and $\epsilon_2$ approach 0 as
$(x,y)$ approaches $(x_0,y_0)$. Then
\begin{equation}\label{eq:dz over dt}
{\Delta z\over\Delta t}=
f_x{\Delta x\over\Delta t}+f_y{\Delta y\over\Delta t}+\epsilon_1{\Delta
x\over\Delta t} + \epsilon_2{\Delta y\over\Delta t}.
\end{equation}
As $\Delta t$ approaches 0, $(x,y)$ approaches $(x_0,y_0)$ and so
\begin{align*}
\lim_{\Delta t\to0}{\Delta z\over\Delta t} &=  {dz\over dt}	\\
\lim_{\Delta t\to0}\epsilon_1{\Delta x\over\Delta t} &= 0\cdot{dx\over dt}	\\
\lim_{\Delta t\to0}\epsilon_2{\Delta y\over\Delta t} &= 0\cdot{dy\over dt}	\\
\end{align*}
and so taking the limit of~(\ref{eq:dz over dt})
 as $\Delta t$ goes to 0 gives
$$
{dz\over dt}=
f_x{dx\over dt}+f_y{dy\over dt},
$$
as desired.
\end{proof}

We can write the chain rule in a way that is somewhat closer to the
single variable chain rule:
$${df\over dt}=\langle f_x,f_y\rangle\cdot\langle x',y'\rangle,$$
or (roughly) the derivatives of the outside function ``times'' the
derivatives of the inside functions.
Not surprisingly, essentially the same chain rule works for functions
of more than two variables, for example, given a function of three
variables $f(x,y,z)$, where each of $x$, $y$ and $z$ is a function of
$t$,
$${df\over dt}=\langle f_x,f_y,f_z\rangle\cdot\langle x',y',z'\rangle.$$

We can even extend the idea further. Suppose that $f(x,y)$ is a
function and $x=g(s,t)$ and $y=h(s,t)$ are functions of two variables
$s$ and $t$. Then $f$ is ``really'' a function of $s$ and $t$ as well, and
$${\partial f\over\partial s}=f_xg_s+f_yh_s\qquad
{\partial f\over\partial t}=f_xg_t+f_yh_t.$$
The natural extension of this to $f(x,y,z)$ works as well.

Recall that we used the ordinary chain rule to do implicit
differentiation\index{implicit differentiation}. We can do the same
with the new chain rule.

\begin{example}{Equation of a Sphere}{EqSphereExample}
Find the partial derivative of $x^2+y^2+z^2 = 4$.
\end{example}
\begin{solution}
The equation $x^2+y^2+z^2 = 4$ defines a sphere, which is not a function of
$x$ and $y$, though it can be thought of as two functions, the top and
bottom hemispheres. We can think of $z$ as one of these two functions,
so really $z=z(x,y)$, and we can think of $x$ and $y$ as particularly
simple functions of $x$ and $y$, and let $f(x,y,z)=x^2+y^2+z^2$.
Since $f(x,y,z)=4$, $\partial f/\partial x=0$, but
using the chain rule:
\begin{align*}
0={\partial f\over\partial x}&=f_x{\partial x\over\partial x}+
f_y{\partial y\over\partial x}+f_z{\partial z\over \partial x}	\\
&=(2x)(1)+(2y)(0)+(2z){\partial z\over\partial x},	\\
\end{align*}
noting that since $y$ is temporarily held constant its derivative
${\partial y/\partial x}=0$. Now we can solve for $\partial z/\partial
x$:
$${\partial z\over \partial x}=-{2x\over 2z}=-{x\over z}.
$$
In a similar manner we can compute $\partial z/\partial y$.
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:multivariable chain rules}}

\begin{enumialphparenastyle}

\begin{ex}
Use the chain rule to compute $dz/dt$ for
$z=\sin(x^2+y^2)$, $x=t^2+3$, $y=t^3$.
\begin{sol}
$4xt\cos(x^2+y^2)+6yt^2\cos(x^2+y^2)$
\end{sol}
\end{ex}

\begin{ex}
Use the chain rule to compute $dz/dt$ for
$z=x^2y$, $x=\sin(t)$, $y=t^2+1$.
\begin{sol}
$2xy\cos t+2x^2t$
\end{sol}
\end{ex}

\begin{ex}
Use the chain rule to compute $\partial z/\partial s$ and
$\partial z/\partial t$ for
$z=x^2y$, $x=\sin(st)$, $y=t^2+s^2$.
\begin{sol}
$2xyt\cos(st)+2x^2s$, $2xys\cos(st)+2x^2t$
\end{sol}
\end{ex}

\begin{ex}
Use the chain rule to compute $\partial z/\partial s$ and
$\partial z/\partial t$ for
$z=x^2y^2$, $x=st$, $y=t^2-s^2$.
\begin{sol}
$2xy^2t-4yx^2s$, $2xy^2s+4yx^2t$
\end{sol}
\end{ex}

\begin{ex}
Use the chain rule to compute $\partial z/\partial x$ and
$\partial z/\partial y$ for $2x^2+3y^2-2z^2=9$.
\begin{sol}
$x/z$, $3y/(2z)$
\end{sol}
\end{ex}

\begin{ex}
Use the chain rule to compute $\partial z/\partial x$ and
$\partial z/\partial y$ for $2x^2+y^2+z^2=9$.
\begin{sol}
$-2x/z$, $-y/z$
\end{sol}
\end{ex}

\begin{ex}Chemistry students will recognize the {\em ideal gas law}, given
  by $PV=nRT$ which relates the Pressure, Volume, and Temperature of
  $n$ moles of gas.  (R is the ideal gas constant).  Thus, we can view
  pressure, volume, and temperature as variables, each one dependent
  on the other two.

\begin{enumerate}
	\item If pressure of a gas is increasing at a rate of $0.2
	  Pa/\hbox{min}$ and temperature is increasing at a rate of $1
	  K/\hbox{min}$, how fast is the volume changing?
	\item If the volume of a gas is decreasing at a rate of $0.3
	  L/\hbox{min}$ and temperature is increasing at a rate of $.5
	  K/\hbox{min}$, how fast is the pressure changing?
	\item If the pressure of a gas is decreasing at a rate of $0.4
	  Pa/\hbox{min}$ and the volume is increasing at a rate of $3
	  L/\hbox{min}$, how fast is the temperature changing?
\end{enumerate}
\begin{sol}
\begin{enumerate}
	\item	$\ds V'=(nR-0.2V)/P$
	\item	$\ds P'=(nR+0.6P)/2V$
	\item	$\ds T' = (3P-0.4V)/(nR)$
\end{enumerate}
\end{sol}
\end{ex}

\begin{ex}
Verify the following identity in the case of the ideal gas law:
$${\partial P\over \partial V} {\partial V\over \partial T}
{\partial T\over \partial P}=-1$$
\end{ex}

\begin{ex}
The previous exercise was a special case of the following
fact, which you are to verify here: If $F(x,y,z)$ is a function of 3
variables, and the relation $F(x,y,z)=0$ defines each of the variables
in terms of the other two, namely $x=f(y,z)$, $y=g(x,z)$ and
$z=h(x,y)$, then
$${\partial x\over \partial y} {\partial y\over \partial z}
{\partial z\over \partial x}=-1$$
\end{ex}

\end{enumialphparenastyle}


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amsfonts}
\usepackage{graphicx}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage[margin=1in]{geometry}
\usepackage{enumitem}
\usepackage{listings}
\usepackage{soul}

\title{Depth First Learning Problem Set 1: Signal propagation}

\date{\today}

\begin{document}
\maketitle

\section*{Problem 1: NN Signal propagation framework}

In this problem, we will work with the basic framework for analyzing signal propagation in a feedforward neural networks as the width of the network's hidden layers grows towards infinity.

To set up the notation, let $x^0 \in \mathbb{R}^{di}$ be the (vector-valued) input to the network.  Each layer, $l$, of the network, is a function from $\mathbb{R}^N \rightarrow \mathbb{R}^N$ defined via the two equations:
\begin{eqnarray}
h^l &=& W^l x^{l-1} + b^l \\
x^l &=& \phi(h^l)
\end{eqnarray}
Here $x^{l-1}$ is the input to the layer, $x^l$ is the output, $W^l$ is an $N\times N$ matrix containing the layer's weights, and $b^l$ is an $N$-dimensional vector containing the layer's biases.  The function $\phi$ is the nonlinearity (e.g., sigmoid, ReLU, etc.) used by the network.  We sometimes call the inputs to the nonlinearity ($h^l$) the \emph{pre-activations}, and we call the outputs of the nonlinearity (which are also the outputs of the layer) the \emph{activations}.

Note here that we are considering a specific neural-netowrk architecture in which the number of hidden units does not vary from layer to layer, i.e. each layer is $N$-dimensional.

\begin{enumerate}[label=(\alph*)]
\item To initialize the network, we usually draw the weights and biases randomly from some distribution.  Let's ignore the bias term for now (i.e. set $b^l=0$ for all layers $l$).  Since the pre-activations $h^l$ are functions of the random variables, they themselves are (vector-valued, $N$-dimensional) random variables. We want to understand what we can say about the distribution of the $h^l$'s as we move through layers of the network, i.e. as a function of $l$.

Suppose we initialize each weight matrix $W^l$ from a zero-mean Gaussian with variance $\sigma^2$, i.e. $W^l \sim \mathcal{N}(0, \sigma^2)$. For simplicity, assume the nonlinearity is just the identity function.  We will relax this assumption in the subsequent problems.

\textbf{What are the mean and variance of the distribution of a single-component, $h^l_i$ of $h^l$, given the mean and the variance of the the previous layer's preactivations ($h^{l-1}_i$)?}

\item You should find in the previous part that the mean of the distribution of $h^l_i$ is always zero, but that each layer of the network multiplies the variance of the distribution by a factor of $N\sigma^2$.  Typically, the variance of the distribution of pre-activations is a proxy for how much of the nonlinearity we're making use of (e.g. for the sigmoid, if the variance is very small, we're basically in the linear region of the nonlinearity).  To be able to vary the number of layers in a network and not have the behavior change too much, we'd like to have an initialization strategy which keeps the variance of the $h^l$ distribution the same as we change the number of layers.  Clearly, our current strategy ($W^l \sim \mathcal{N}(0, \sigma^2)$) doesn't accomplish this.

\textbf{Suggest a simple modification of the initialization strategy which would achieve this goal.}

\item Now let's add back in the bias term.  Imagine that we initialize according to a simple zero-mean Gaussian, where the variance (as for the weights in part (a)) was independent of the number of hidden units $N$, that is $b^l \sim \mathcal{N}(0, \sigma_b^2)$.

\textbf{Does this initialization suffer from the same problem as the weight initialization described in part (b)?  If so, how can we fix it?}

\end{enumerate}

\section*{Problem 2: $N\rightarrow\infty$ and the mean-field approximation}

In this problem, we use the knowledge we gained in problem 1 to properly choose to initialize the weights and biases according to $W^l \sim \mathcal{N}(0, \sigma_w^2/N)$ and $b^l \sim \mathcal{N}(0, \sigma_b^2)$. We'll investigate some techniques that will be useful in understanding precisely how the network's random initialization influences what the net does to its inputs; specifically, we'll be able to take a look at how the \textit{depth} of the network together with the initialization governs the propagation of an input point as it flows forward through the network's layers.

\begin{enumerate}[label=(\alph*)]
\item \textbf{A natural property of input points to study as the input flows through the net layer by layer is its length. Intuitively, this is closely related to how the net transforms the input space, and to how the depth of the net relates to that transformation. Compute the length $q^l$ of the activation vector outputted by layer $l$. When considering non-rectangular nets, where layer $l$ has length $N_l$, we want to distinguish this activation norm from the width of individual layers, so what's a more appropriate quantity we can track to understand how the lengths of activation vectors change in the net?}

\item \textbf{What probabilistic quantity of the neuronal activations does $q^l$ approximate (with the approximation improving for larger $N$)?}

\textit{Hint: Recall that all neuronal activations $h^l_i$ are zero-mean, and consider the definition of $q^l$ from part (a) in terms of the empirical distribution of $h^l_i$.}

\item \textbf{Calculate the variance of an individual neuron's pre-activations, that is, the variance of $h_i^l$.  Your answer should be a recurrence relation, expressing this variance in terms of $h^{l-1}$ (and the parameters $\sigma_w$ and $\sigma_b$}).

(Note 1: You basically did this in problem 1; the differences here are just that the weights are initialized slightly differently and that the bias term exists, and now the noninearity is not just the identity)

(Note 2: This part of the problem does not yet assume that $N\rightarrow \infty$)

\item Now consider the limit that the number of hidden neurons, $N$, approaches infinity.

\textcolor{red}

\textbf{Use the central limit theorem to argue that in this limit, the pre-activations will be zero-mean Gaussian distributed. Be explicit about the conditions under which this result holds.}

\item With this zero-mean Gaussian approximation of $q^l$, we have a single parameter characterizing this aspect of signal propagation in the net: the variance, $q^l$, of individual neuronal activations (a proxy for squared activation vector lengths). Let's now look at how this variance changes from layer to layer, by deriving the relationship between $q^l$ and $q^{l - 1}$.

\textbf{In part (a), your answer should have included a term $\langle (x^{l-1})^2 \rangle$.  In terms of the activation function $\phi$ and the variance $q^{l-1}$, write this expectation value as an integral over the standard Gaussian measure.}

\textbf{Use this result to write a recursion relation for $q^l$ in  terms of $q^{l-1}$, $\sigma_w$, and $\sigma_b$}.

\end{enumerate}

\section*{Problem 3: Fixed points and stability}

In the previous problem, we found a recursion relation relating the length of a vector at layer $l$ of a network to the length of the vector at the previous layer, $l-1$ of the network.  In this problem, we are interested in studying the properties of this recursion relation.  In the \emph{Resurrecting the sigmoid} paper, the results of this problem are used to understand at which bias point to evaluate the Jacobian of the input-output map of the network.  For more information on this topic, see either of the two papers which are suggested reading for this week:
\begin{itemize}
	\item \emph{Exponential expressivity in deep neural networks through transient chaos}
	\item \emph{Deep information propagation}
\end{itemize}

Note that in this problem, we are just taking the recursion relation as a given, i.e. we do not need to worry about random variables or probabilities; all of that went into determining teh recursion relation.

\begin{enumerate}[label=(\alph*)]
\item A simple example of a dynamical system is a recurrence defined by some initial value $x_0$ and a relation $x_n = f(x_{n-1})$ for all $n>0$.  This system defines the resulting sequence $x_n$.  Sometimes, these systems have \emph{fixed points}, which are values $x^*$ such that $f(x^*) = x^*$.

\textbf{If the value of the system, $x_m$, at some time-step $m$, happens to be a fixed point $x^*$, what is the subsequent evolution of the system?}

\item \textbf{For the recurrence relation you derived in the previous problem, what is the equation which a fixed-point of the variance, $q^*$, must satisfy?}

\textbf{Under some conditions (i.e. for some values of $\sigma_w$ and $\sigma_b$), the value $q^*=0$ is a fixed point of the system.  What are these conditions?}

\item Now let us be concrete, and look at the recurrence relation in the special case of a nonlinearity $\phi(h)$ which is both monotonically increasing and satisfies $\phi(0) = 0$.  Note that both of the nonlinearities considered in the paper we are studying, the $tanh$ and ReLU nonlinearities, satisfy this property.

\textbf{Show that those two properties (monotonicity and $\phi(0)=0$) imply that the length map $q^l(q^{l-1})$ is monotonically increasing.}

\textbf{Optional: Show that these two properties imply that the length map $q^l(q^{l-1})$ is a concave function.}

(Note: We have not managed to prove this ourselves (and not for lack of trying!), so feel free to skip)

\textbf{What is the maximum number of times any concave function can intersect the line $y = x$?  What does this imply about the number of fixed points the length map $q^l(q^{l-1})$ can have?}

\item Let's be concrete now and consider the nonlinearity to be a ReLU.

\textbf{Compute (analytically) the length map $q^l = f(q^{l-1})$, which will also depend on $\sigma_w$ and $\sigma_b$.  For what values of $\sigma_w$ and $\sigma_b$ does the system have fixed point(s)? How does the value of the fixed point depend on $\sigma_w$ and $\sigma_b$? }

\item Now let's consider the sigmoid nonlinearity $\phi(h) = tanh(h)$.  In this case the length map cannot be computed analytically, but it can be done numerically.

\textbf{Numerically plot the length map, $q^l=f(q^{l-1})$, for a few values of $\sigma_w$ and $\sigma_b$ in the following regimes: (i) $\sigma_b=0$ and $\sigma_w < 1$, (ii) $\sigma_b = 0$ and $\sigma_w > 1$, and (iii) $\sigma_b > 0$.  Describe qualitatively the fixed points of the map in each regime.}

\item Let’s now talk about the stability of fixed points. In a dynamical system, once the system reaches (or starts at) a fixed point, by definition it can never leave. But what happens if the system gets or starts near a fixed point?  In real physical systems, this question is very relevant because physical systems almost always have some noise which pushes the system away from a fixed point.

In general, the fixed point can be either stable or unstable.  For a stable fixed point, initializing the system near the fixed point will result in behavior which converges to the fixed point, i.e reducing the magnitude of the perturbation away from the fixed point.  Conversely, for an unstable fixed point, the system initialized nearby will be repelled from the fixed point.

\textbf{Relate the stability of a particular fixed point to the derivative of the length map evaluated at that fixed point.}

\item With this understanding of stability, revisit your result in part (e) for the $tanh$ nonlinearity.

\textbf{Specifically, discuss the stability of the fixed points in each of the three regimes.  You can evaluate the derivative of the length map by looking at the graphs.}

\item \textbf{Do the same stability analysis for the ReLU network.}

\item \textbf{(Optional) You should have found above that the both the ReLU and $tanh$ systems never had more than one stable fixed point.  Show that this is a consequence of the concavity of the length map.}

(Hint: You can just draw a picture for this one)

\end{enumerate}

\section*{Problem 4: Correlation maps}

In the previous problem, we discovered a very interesting property of wide neural networks: the existence of fixed points of activation vector lengths.  In this problem, we will explore a similar analysis for correlations between activations due to different inputs to the network and connect this to vanishing/exploding gradients.

Define the correlation $q_{12}^l$ between two inputs $x^{0,1}$ and $x^{0,2}$ at the $l^{th}$ layer of the network via the following inner product:
\begin{equation}
    q^l_{12} = \frac{1}{N_L} \sum_{i=1}^{N_l} h_i^l(x^{0,1}) h_i^l(x^{0,2}),
\end{equation}
where the notation $h_i^l(x^{0,1})$ means the pre-activation of the $i^{th}$ neuron of the $l^{th}$ layer of the network given the input $x^{0,1}$.

\begin{enumerate}[label=(\alph*)]

\item As in the previous problem, though the quantity $q_{12}^l$ is defined for a specific realization of the network, averaged over all neurons, we can use the self-averaging assumption treat this as an estimate of a quantity which characterizes a single neuron but averaged over realizations of the network.

\textbf{What is this quantity?  (It is a quantity that comes up quite often when dealing with correlated random variables)}

\item Now we want to find a recurrence relation to describe the behavior of $q_{12}^l$ as we go through the network.

\textbf{Show that}
\begin{align*}{q_{12}^{l}=\mathcal{C}\left(c_{12}^{l-1}, q_{11}^{l-1}, q_{22}^{l-1} | \sigma_{w}, \sigma_{b}\right) \equiv \sigma_{w}^{2} \int \mathcal{D} z_{1} \mathcal{D} z_{2} \phi\left(u_{1}\right) \phi\left(u_{2}\right)+\sigma_{b}^{2}} \\ {u_{1}=\sqrt{q_{11}^{l-1}} z_{1}, \quad u_{2}=\sqrt{q_{22}^{l-1}}\left[c_{12}^{l-1} z_{1}+\sqrt{1-\left(c_{12}^{l-1}\right)^{2}} z_{2}\right]},\end{align*}
\textbf{where $\mathcal{D} z_{1}$ and $\mathcal{D} z_{2}$ indicate integration with respect to two independent standard normal (Gaussian) variables $z_1$ and $z_2$.  Here $c_{12}^l = q_{12}^l (q^l_{11} q^l_{22})^{-1/2}$ is known as the correlation coefficient.}

(Hint: don't be scared by the ugly-looking definitions of $u_1$ and $u_2$.  This is just done so that we can have the integral over independent Gaussians.  Once you have an expression in terms of dependent Gaussians, a simple change-of-variables should get the result you see above.)

\item Once you have arrived at the above recurrence relation, note that you have a dynamical system which determines the value of $q_{12}^l$ in terms of $q_{12}^{l-1}$, $q_{11}^{l-1}$, and $q_{22}^{l-1}$.  You know from the previous problem, however, that $q_{11}^{l-1}$ and $q_{22}^{l-1}$ converge to fixed points.  Though you'll have to take it on faith for now, it turns out that the convergence of these values to their fixed point happens quickly (compared to the dynamics of $q_{12}^l$), so it is a reasonable approximation to just replace $q_{11}^{l-1}$ and $q_{22}^{l-1}$ with $q^*$.

Also, in the following we will be interested not in $q_{12}^l$ itself but instead $c_{12}^l$ (defined above).

\textbf{In terms of} $\mathcal{C}\left(c_{12}^{l-1}, q_{11}^{l-1}, q_{22}^{l-1} | \sigma_{w}, \sigma_{b}\right)$\textbf{, what is the recurrence relation for the quantity $c_{12}^l$?}

\item \textbf{What are the allowed values of $c_{12}^l$?}

\item It turns out that the recurrence relation you derived above always has a fixed point at $c^* = 1$.

\textbf{Show this analytically, and then argue why you could have arrived at this result without any calculation.}

\item Now let's examine the stability of the fixed point $c^* = 1$.  To do this, as before, we look at the derivative of the recurrence relation, i.e. $dc^l_{12}/dc^{l-1}_{12}$, evaluated at the point $c^* = 1$.  We will call this quantity $\chi$.  It can be shown that
\begin{equation}
    \chi = \sigma_w^2 \int~\mathcal{D}z \left[\phi'(\sqrt{q^*} z)\right]^2
\end{equation}

\textbf{If you want, prove the above relation.  Since this is just algebra and not too instructive, feel free to skip this part. }

\end{enumerate}

\section*{Problem 5: Connection between $\chi$ and exploding/vanishing gradients}

In the previous problem, we saw that a unit correlation coefficient (the highest value which is allowed) is a fixed point under evolution through the neural network.  However, it was not always stable.  The stability depended on whether the quantity $\chi$, defined as the derivative of $dc_{12}^l/dc^{l-1}_{12}$, is greater or less than 1.  In this problem, we want to understand the connection between the quantity $\chi$ and exploding or vanishing gradients.

A starting point to understand the connection between $\chi$ and gradients is to consider what it means for the fixed point $c^*=1$ to be stable or unstable.  If $\chi > 1$, and the fixed point $c^* = 1$ is thus unstable, then two input vectors which are highly correlated will de-correlate as they are processed by the network.  Conversely, if $\chi < 1$, then two vectors will become more correlated as they are processed.  Thus it seems like if $\chi > 1$, space is stretched, while if $\chi < 1$, space is contracted.  In this problem we find that $\chi$ is precisely the factor by which space is streched or contracted by each layer of the neural network.

\begin{enumerate}[label=(\alph*)]
\item Start by considering a plain linear transformation, $\mathbf{y} = \mathbf{J}\mathbf{x}$, where $\mathbf{x}$ and $\mathbf{y}$ are vectors and $\mathbf{J}$ is a matrix.

\textbf{Averaged over all possible directions in which $\mathbf{x}$ can point, what is the ratio of the squared length of $\mathbf{y}$ to that of $\mathbf{x}$, in relation to the singular values of $\mathbf{J}$?}

\item Consider the transformation enacted by layer $l$ of a neural network,
\begin{equation}
h^l &=& W^l \phi(h^{l-1}) + b^l.
\end{equation}
Let $\mathbf{D}$ be a diagonal matrix whose entries are $D_{ii} = \phi'(h_i^{l-1})$.

\textbf{In terms of $W^l$ and $D$, what is the Jacobian of the transformation $h^{l-1}\rightarrow h^l$?}

\textbf{From the problem above, we want the mean-squared singular value of the Jacobian.  Show that this mean-squared singluar value is exactly equal to the expression for $\chi$ calculated earlier, when we take the expectation with respect to the distribution of the weight matrix and the distribution of the pre-activations}

\item How does the value of $\chi$ relate to exploding and vanishing gradients?

\item To emphasize that $\chi$ is a function of $\sigma_w$ and $\sigma_b$, we go back to the expression for $\chi$,
\begin{equation}
    \chi = \sigma_w^2 \int~\mathcal{D}z \left[\phi'(\sqrt{q^*} z)\right]^2,
\end{equation}
and explicitly write the dependence of the fixed point $q^*$ on $\sigma_w$ and $\sigma_b$.  Remember that this dependence can be written implicitly via the equation
\begin{equation}
q^{*}=\sigma_{w}^{2} \int \phi\left(\rho \sqrt{q^{*}}\right)^{2} \mathrm{d} \rho+\sigma_{b}^{2}.
\end{equation}

\textbf{Numerically calculate the value of $\chi$ for several values of $\sigma_w$ and $\sigma_b$, for the hyperbolic tangent nonlinearity.  Make a contour plot, and specifically indicate the curve corresponding to $\chi=1$}

\end{enumerate}

\end{document}


\section{Conclusions}
\label{sec:conclusions}

In the following table we collect congruences proved in previous sections,
respect to $\equiv_{2}$ over coefficients $c_{n,k}=\frac{k+1}{n+1}{{2n-k}\choose{n-k}}\in\mathcal{C}$ :
\begin{displaymath}
    \begin{split}
        & \frac{1}{n+1}{{2n}\choose{n}} \equiv_{2} 1 \leftrightarrow \exists \alpha\in\mathbb{N}: n=2^{\alpha}-1  \\
        & \forall\alpha\in\mathbb{N}:\frac{k+1}{2^{\alpha}}{{2^{\alpha+1}-2-k}\choose{2^{\alpha}-1-k}} \equiv_{2} 1  \\
        & \forall\alpha\in\mathbb{N}:\frac{k+1}{2^{\alpha}+1}{{2^{\alpha+1}-k}\choose{2^{\alpha}-k}} \equiv_{2} 0 \leftrightarrow \exists j\in\mathbb{N}: k=2j+1   \\
        & \forall\alpha\in\mathbb{N}:\frac{2^{\alpha}}{s+1}{{2s-2^{\alpha}+1}\choose{s-2^{\alpha}+1}} \equiv_{2} 0 \leftrightarrow
            s\in\lbrace 2^{\alpha}, \ldots, 2^{\alpha+1}-2\rbrace  \\
        & \forall\alpha\in\mathbb{N}:\frac{s-2^{\alpha}+2}{s+1}{{s+2^{\alpha}-1}\choose{2^{\alpha}-1}} \equiv_{2}
            \frac{2^{\alpha}}{s+1}{{2s-2^{\alpha}+1}\choose{s-2^{\alpha}+1}} \leftrightarrow
                s\in\lbrace 2^{\alpha}, \ldots, 2^{\alpha+1}-2\rbrace  \\
        & \frac{k+1}{n+1}{{2n-k}\choose{n-k}} \equiv_{2} 0 \leftrightarrow
            n \in\lbrace 2^{\alpha}, \ldots, 2^{\alpha+1} - 2\rbrace \wedge k \in \lbrace 0, \ldots, n - 2^{\alpha}\rbrace \\
    \end{split}
\end{displaymath}

The following two congruences hold if and only if $\alpha\in\mathbb{N} \wedge e\in\lbrace1,\ldots,s-2^{{\alpha}}\rbrace$:
\begin{displaymath}
    \begin{split}
        & {{2s-e-2^{{\alpha}}+1}\choose{s-e}} - {{2s-e-2^{{\alpha}}+1}\choose{s-e+1}} \equiv_{2}
            {{2s-2^{{\alpha}}+1-e}\choose{s}} - {{2s-2^{{\alpha}}+1-e}\choose{s+1}}\\
        & {{2s-2^{{\alpha}}+1-e}\choose{s}} - {{2s-2^{{\alpha}}+1-e}\choose{s+1}} \equiv_{2}
            {{2s-2^{{\alpha}+1}-e+1}\choose{s-2^{{\alpha}}}} - {{2s-2^{{\alpha}+1}-e+1}\choose{s-2^{{\alpha}}+1}}\\
    \end{split}
\end{displaymath}

To summarize our work, we provide a study of the congruence
$\equiv_{2}$ mapped to the Catalan array $\mathcal{C}$, both a formal
approach to characterize different regions and implementations are
given. Even though array $\mathcal{P}_{\equiv_{2}}$ is very
interesting and deeply studied, is our opinion that
$\mathcal{C}_{\equiv_{2}}$ should deserve interest as well because of
its recursive flavor and connection to universal counting Catalan
numbers and combinatorial objects.

We finish pointing out directions for further studies: we would like
to understand how a generalization to congruence $\equiv_{p}$, for $p$
prime, can be proved using either a closed formula for the generic
coefficient or algebraic generating function manipulations; at last,
find a modular characterization of $\mathcal{C}_{\equiv_{2}}^{-1}$,
the inverse group element of $\mathcal{C}_{\equiv_{2}}$, depicted in
\autoref{fig:catalan-traditional-inverse-ignore-negatives-centered-colouring-127-rows-mod2-partitioning-triangle}.

\input{catalan/catalan-traditional-inverse-ignore-negatives-centered-colouring-127-rows-mod2-partitioning-include-figure.tex}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%% Supplemental Material%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\clearpage
%\newpage
\section{Choosing Window Size}
In genome-wide scans for detecting selection, we apply the \comale\
statistic on sliding windows of length $L$bp. The single locus
statistic values within the window are averaged to get the composite
statistic. While the statistic is robust to variation in window-size,
choosing a very large window where LD has decayed will weaken the
composite signal, and choosing a small window will decrease the power
of composite likelihoods. Here, we use a systematic calculation to
choose $L$ as the distance where the LD between the favored mutation
and a site $L/2$bp away remains strong.
\label{sec:winSize}
\ignore{
The optimum
choice of window size depends on the linkage disequilibrium (LD)
between the favored allele and its nearby variants.  In an E\&R
selection experiment, dynamic of the exact LD between the favored
allele to other variants depends on a number of parameters including
recombination rate ($r$), genomic distance ($l$), LD at the onset of
selection ($\rho_0$), initial frequency of the favored allele
($\nu_0$), strength of selection ($s$) and span of experiment
($\tau$). Moreover, observed LD, additionally depend on the sample
size $n$ and sequencing coverage. For simplicity we exclude sampling
noise due to finite sampling for sequencing and finite sequencing
coverage in our analysis.
}

Consider a segregating site $l$ bp away from the favored allele in a
selective sweep.  Let $\rho_\tau$ be the LD between the favored allele
and the site, $\tau$ generations after the onset of selection. Then,
we have (see Eqs. 30-31 in \cite{stephan2006hitchhiking}):
\beq
\rho_\tau= \alpha_\tau\beta_\tau \rho_0=e^{-r\tau l}
\left(\frac{K^{(\tau)}}{K^{(0)}}\right)\rho_0\label{eq:ldt},
\eeq
where $K^{(\tau)}=2\nu_\tau(1-\nu_\tau)$ is the heterozygosity at the
selected site, $r$ is the recombination rate (crossovers/bp/gen). The
`decay factor', $\alpha_\tau=e^{-r\tau l}$, and `growth factor',
$\beta_\tau$, are due to recombination and selection, respectively.
Under regular parameter settings, linkage to the favored allele is
expected to increase after onset of selection and then
decreases due to crossover events
(See~\ref{fig:winSize}-A).  While $\rho_0$ is
unknown in pool-seq E\&R experiments, we compute the value of $l$ so
that
\beq
\alpha_\tau \beta_\tau=1.\label{eq:eq}
\eeq
In E\&R scenarios, we let $\tau$ be the time of the last sampling. For
given $s$, we aim to compute the smallest window size $L$ over all
possible starting frequencies. Specifically,

\beq L=2\min_{\nu_0} \left\{
  \frac{1}{r\tau}\log\left(\frac{\hat{\nu}_\tau(1-\hat{\nu}_\tau)}{\nu_0(1-\nu_0)}\right)\right\},
  \label{eq:winSize}
\eeq
where the term $\hat{\nu}_\tau$ depends on initial frequency
$\nu_0$ and selection strength $s$ (Eq.~\ref{eq:transition}).

We used \dmel dataset parameters, $N=250,r=2\times10^{-8}$ and
$\tau=59$ to compute the optimal window size for different values of
$Ns$, ranging from weak selection to strong selection:
$Ns\in\{20,100,200,500\}$, or $s\in\{0.08,0.4,0.8,2\}$.  We set
$L=30$Kbp (See~\ref{fig:winSize}-B) to provide good resolution for
detecting weak selection.