\documentclass[a4paper]{article}

\input{temp}

\setcounter{section}{-1}

\begin{document}

\title{Introduction to Discrete Analysis}

\maketitle

\newpage

\tableofcontents

\newpage

\section{Introduction}
asdasd

\newpage

\section{The discrete Fourier transform}

Let $N$ be a fixed positive integer. Write $\omega$ for $e^{2\pi i/N}$, and $\Z_N$ for $\Z/n\Z$. Let $f:\Z_N \to \C$. Given $r \in \Z_N$, define $\hat{f}(r)$ to be
\begin{equation*}
\begin{aligned}
\frac{1}{N} \sum_{x \in \Z_N} f(x) \omega^{-rx}
\end{aligned}
\end{equation*}
From now on we use the notation $\E_{x \in \Z_N}$ for $\frac{1}{N}\sum_{x \in \Z_N}$, so $\hat{f}(r) = \E_x f(x) e^{-\frac{2\pi irx}{N}}$.

If we write $\omega_r$ for the function $x \to \omega^{rx}$, and $\bra f,g\ket$ for $\E_x f(x) \overline{g(x)}$, then $\hat{f}(r) = \bra f,\omega_r \ket$. So the discrete fourier transforn is basically expanding the function $f$ in the set of orthonormal basis $\omega_r$.

Let us write $||f||_p$ for $\E_x |f(x)|^p)^{1/p}$ (the $L_p$-norm), and call the resulting space $L_p(\Z_n)$.

Important convention: we use \emph{averages} for the 'original functions' in 'physical spaces', and \emph{sums} for their Fourier transforms in 'frequency space' (referring to $\E$: $\bra,\ket$ is average in the original space but just $\sum$ in frequency space, i.e. for $\hat{f},\hat{g}$ etc.)

\begin{lemma} (1, Parseval's identity)\\
    If $f,g: \Z_n \to \C$, then $\bra \hat{f},\hat{g}\ket = \bra f,g\ket$.
    \begin{proof}
        \begin{equation*}
            \begin{aligned}
                \bra\hat{f},\hat{g}\ket &= \sum_r \hat{f}(r) \overline{\hat{g}(r)}\\
                &= \sum_r (\E_x f(x) \omega^{-rx}) (\overline{\E_y g(y) \omega^{-ry}})\\
                &= \E_x \E_y f(x) \overline{g(y)} \sum_r \omega^{-r(x-y)}\\
                &= \E_x \E_y f(x) \overline{g(y)} n\delta_{xy}\\
                &= \bra f,g\ket
            \end{aligned}
            \end{equation*}
    \end{proof}
\end{lemma}

\begin{lemma} (2, Convolution identity)\\
    \begin{equation*}
        \begin{aligned}
            \widehat{f*g}(r) = \hat{f}(r)\hat{g}(r)
        \end{aligned}
    \end{equation*}
    where
    \begin{equation*}
        \begin{aligned}
            (f*g)(x) = \E_{y+z=x}f(y)g(z) = \E_y f(y)g(x-y)
        \end{aligned}
    \end{equation*}
    \begin{proof}
        \begin{equation*}
            \begin{aligned}
                \widehat{f*g}(r) &= \E_x f*g(x)\omega^{-rx}\\
                &=\E_x \E_{y+z=x}f(y)g(z)\omega^{-rx}\\
                &=\E_x\E_{y+z=x} f(y)g(z) \omega^{-ry}\omega^{-rz}\\
                &=\E_y\E_z f(y) \omega^{-ry} g(z) \omega^{-rz}\\
                &=\hat{f}(r) \hat{g}(r)
            \end{aligned}
        \end{equation*}
    \end{proof}

\end{lemma}

\begin{lemma} (3, Inversion formula)\\
    \begin{equation*}
        \begin{aligned}
            f(x) = \sum_r \hat{f}(r) \omega^{rx}
        \end{aligned}
    \end{equation*}
    (note the sign of $\omega^{rx}$).
    \begin{proof}
        \begin{equation*}
            \begin{aligned}
                \sum_r \hat{f}(r) \omega^{rx} &= \sum_r \E_y f(y) \omega^{r(x-y)}\\
                &= \E_y f(y) \sum_r \omega^{r(x-y)}\\
                &= \E_y f(y) n\delta_{xy}\\
                &=f(x)
            \end{aligned}
        \end{equation*}
        This is really just the statement that we get the original vector back when we sum up its components.
    \end{proof}
\end{lemma}

Further observations: If $f$ is real-valued, then $\hat{f}(-r) = \E_x f(x) \omega^{rx} = \overline{\E_x f(x) \omega^{-rx}} = \overline{\hat{f}(r)}$.

If $A \subset \Z_n$, write $A$ (instead of $1_A,\chi_A$) for the characteristic function of $A$. Then $\hat{A}(0) = \E_x A(x) = \frac{|A|}{N}$, the \emph{density} of $A$.

Also, $||\hat{A}||^2_2 = \bra\hat{A},\hat{A}\ket = \bra A,A\ket = \E_x A(x)^2 = \E_x A(x) = \frac{|A|}{N}$, again the density.

Let $f:\Z_n \to \C$. Given $\mu \in \Z_n$, define $f_\mu(x)$ to be $f(\mu^{-1}x)$ (so we need $(\mu,N) = 1$). Then
\begin{equation*}
    \begin{aligned}
        \hat{f}_\mu(r) &= \E_x f_\mu (x) \omega^{-rx}\\
        &= \E_x f(x/\mu) \omega^{-rx}\\
        &=\E_x f(x) \omega^{-r\mu x}\\
        ^= \hat{f}(\mu r)
    \end{aligned}
\end{equation*}

\subsection{Roth's theorem}
\begin{thm} (4)
    For every $\delta > 0$, $\exists N$ s.t. if $A \subset \{1,...,N\}$ is a set of size at least $\delta N$, then $A$ must contain an arithmetic progression of length 3.\\
    This is also true for 4,5,..., but the proof is much harder -- Szemeredi's theorem.\\
    Basic strategy of proof: show that if $A$ has density $\delta$ and no AP of length 3 (3AP), then there's a long AP in $P \subset \{1,2,...,n\}$ s.t.
    \begin{equation*}
        \begin{aligned}
            |A \cap P| \geq (\delta+c(\delta)) |p|
        \end{aligned}
    \end{equation*}
    where $c(\delta)$ is some positive number. But then we can continue this argument to expand $A\cap P$ to infinity (note that $|A\cap P|$ is an integer, so each time increase by 1 at least).

    The best known relationship between $\delta$ and the $N$ required is around $\delta \sim \frac{c}{\log\log N}$ for some constant $c$.
\end{thm}

---Lecture 2---

\begin{lemma} (5)\\
    Let $N$ be odd, $A,B,C \subset \Z_N$ have densties $\alpha,\beta,\gamma$.\\
    If $\max_{r \neq 0} |\hat{A}(r)| \leq \frac{\alpha(\beta\gamma)^{1/2}}{2}$ and $\frac{\alpha\beta\gamma}{2} > \frac{1}{N}$, then there exists $x,d \in \Z_N$ with $d \neq 0$ s.t. $(x,x+d,x+2d) \in A \times B \times C$.
    \begin{proof}
        \begin{equation*}
            \begin{aligned}
                \E_{x,d} A(x) B(x+d) C(x+2d) &= \E_{x+z=2y} A(x)B(y)C(z)\\
                &=\E_u (\E_{x+z=u}A(x)C(z)) \E_{2y=u} B(y)\\
                &=\E_u A * C(u) B_2(u)\\
                &=\bra A*C,B_2\ket\\
                &=\bra\widehat{A*C},\hat{B}_2\ket\\
                &=\bra \hat{A}\hat{C},\hat{B}_2\ket\\
                &=\sum_r \hat{A}(r)\hat{C}(r)\hat{B}(-2r)\\
                &=\alpha\beta\gamma + \sum_{r \neq 0} \hat{A}(r)\hat{C}(r)\hat{B}(-2r)
            \end{aligned}
        \end{equation*}
        Recall here the notation is $B_2(u) = B(u/2)$. now
        \begin{equation*}
            \begin{aligned}
                |\sum_{r\neq 0} \hat{A}(r)\hat{B}(-2r)\hat{C}(r)| &\leq \frac{\alpha(\beta\gamma)^{1/2}}{2}\sum_{r \neq 0} |\hat{B}(-2r)||\hat{C}(r)|\\
                &\leq \frac{\alpha(\beta\gamma)^{1/2}}{2} \left(\sum_r |\hat{B}(-2r)^2\right)^{1/2}\left(\sum_r |\hat{C}(r)|^2\right)^{1/2} \ \text{By Cauchy-Schwarz}\\
                &=\frac{\alpha(\beta\gamma)^{1/2}}{2} ||\hat{B}||_2 ||\hat{C}||_2\\
                &= \frac{\alpha(\beta\gamma)^{1/2}}{2} ||B||_2||C||_2\\
                &= \frac{\alpha\beta\gamma}{2}
            \end{aligned}
        \end{equation*}
        The contribution to $\E_{x,d} A(x)B(x+d)C(x+2d)$ from $d=0$ is at most $\frac{1}{N}$, so if $\frac{\alpha\beta\gamma}{2} > \frac{1}{N}$, we are done.
    \end{proof}
\end{lemma}

Now let $A$ be a subset of $\{1,...,N\}$ with density $\geq \delta$ and let $B=C=A \cap [\frac{N}{3},\frac{2N}{3})$. If $B$ has density $<\frac{\delta}{5}$ (??), then either $A \cap [1,\frac{N}{3}]$ or $A \cap [\frac{2N}{3},N]$ has density at least $\frac{2\delta}{5}$. In that case we find an AP $P$ of length about $N/3$ such that $|A \cap P| / |P| \geq \frac{6\delta}{5}$.\\
Otherwise, we find that if $\max_{r \neq 0}|\hat{A}(r)| \leq \frac{\delta}{10}$ and $\frac{\delta^3}{50} > \frac{1}{N}$, then $A \times B \times C$ contains a 3AP, so $A$ contains a 3AP.\\
So if $A$ does not contain a 3AP, then either we find $P$ of length about $N/3$ with $|A\cap P| / |P| \geq \frac{6\delta}{5}$, or ther exists $r \neq 0$ s.t. $|\hat{A}(r)| \geq \frac{\delta}{10}$.

\begin{defi}
    If $X$ is a finite set and $f:X \to \C$, $Y \subset X$, write $osc(f|_Y)$ to mean $\max_{y_1,y_2 \in Y} |f(y_1)-f(y_2)|$ (I think \emph{amplitude} is a better word for this).
\end{defi}

\begin{lemma} (6)\\
    Let $r \in \Z_n$ and let $\varepsilon > 0$. Then there is a partition of $\{1,2,...,N\}$ into arithmetic progressions $P_i$ of length at least $c(\varepsilon)\sqrt{N}$ such that
    \begin{equation*}
        \begin{aligned}
            osc(\omega_r|_{P_i}) \leq \varepsilon
        \end{aligned}
    \end{equation*}
    for each $i$.
    \begin{proof}
        Let $t = \lfloor \sqrt{N}\rfloor$. Of the numbers $1,\omega^r,...,\omega^tr$, there must be two that differ by at most $\frac{2\pi}{t}$.\\
        If $|\omega^{ar}-\omega^{br}| \leq \frac{2\pi}{t}$ with $a < b$, then $|1-\omega^{dr}| \leq \frac{2\pi}{t}$ where $d=b-a$. Then $|\omega^{urd}-\omega^{vrd}| \leq |\omega^{urd}-\omega^{(u+1)rd}| + ... + |\omega^{(v-1)rd}-\omega^{vrd}| \leq \frac{2\pi}{t}(v-u)$.\\
        So if $P$ is a progression with common differnece $d$ and length $l$, then $osc(\omega_r|_P) \leq \frac{2\pi l}{t}$. So divide up $\{1,...,N\}$ into residue classes mod $d$, and partition each residue class into parts of length between $\frac{\varepsilon t}{4\pi}$ and $\frac{\varepsilon t}{2\pi}$ (possible, since $d \leq t \leq \sqrt{N}$).\\
        We are done, with $c(\varepsilon) = \frac{\varepsilon}{16}$ (a casual choice).
    \end{proof}
\end{lemma}
Now let us use the information that $r \neq 0$ and $|\hat{A}(r)| \geq \frac{\delta^2}{10}$.\\
Define the \emph{balanced function} $f$ of $A$ by $f(x) = A(x)=\frac{|A|}{N}$ for each $x$.\\
Note that $\hat{f}(0) = 0$ and $\hat{f}(r) = \hat{A}(r)$ for all $r \neq 0$.

Now let $P_1,...,P_m$ be given by Lemma 6 with $\varepsilon = \delta^2/20$. Then
\begin{equation*}
    \begin{aligned}
        \frac{\delta^2}{10} &\leq |\hat{f}(r)|\\
        &= \frac{1}{N} |\sum_x f(x) \omega^{-rx}|\\
        &\leq \frac{1}{N} \sum_{i=1}^m |\sum_{x \in P_i} f(x) \omega^{-rx}|\\
        &\leq \frac{1}{N} \sum_{i=1}^N \left[\left|\sum_{x \in P_i} f(x) \omega^{-rx_i}\right| + \left|\sum_{x \in P_i} f(x) (\omega^{-rx} - \omega^{-rx_i})\right|\right] \ x_i \in P_i \text{ arbitrary}\\
        &\leq \frac{1}{N} \sum_{i=1}^m |\sum_{x \in P_i} f(x)| + \frac{\delta^2}{20}
    \end{aligned}
\end{equation*}
Therefore $\sum_{i=1}^m \left|\sum_{x \in P_i} f(x)\right| \geq \frac{\delta^2 N}{20}$.

We also have $\sum_{i=1}^m \sum_{x\in P_i} f(x) = 0$, so
\begin{equation*}
    \begin{aligned}
        \sum_{i=1}^m \left(\left|\sum_{x \in P_i} f(x) \right| + \sum_{x \in P_i}f(x) \right) \geq \frac{\delta^2}{20} \sum_{i=1}^m |P_i|
    \end{aligned}
\end{equation*}
Therefore,
\begin{equation*}
    \begin{aligned}
        &|\sum_{x \in P_i} f(x)| + \sum_{x \in P_i} f(x) \geq \frac{\delta^2}{20} |P_i|\\
        &\implies \sum_{x \in P_i} f(x) \geq \frac{\delta^2}{40}|P_i|\\
        &\implies |A \cap P_i| \geq \left(\delta+\frac{\delta^2}{40}\right)|P_i|
    \end{aligned}
\end{equation*}

---Lecture 3---

Now let $A \subset \Z_N$, $|A| \geq \delta N$. Then:\\
$\bullet$ either $A$ contains a 3AP,\\
$\bullet$ or $N$ is even,\\
$\bullet$ or $\exists P \subset \{1,...,N\}$, $|P| \geq N/3$ s.t. $|A \cap P| \geq \frac{6\delta}{5} |P|$,\\
$\bullet$ or $\exists P \subset \{1,...,N\}$, $|P|\geq \frac{\delta^2}{640}\sqrt{N}$ (casual) s.t. $|A\cap P| \geq (\delta+\frac{\delta^2}{40})|P|$.

Note that the third case is strictly worse than the fourth.\\
Well if the first is true then we're done. Suppose now the seconds hols. Write $N=N_1+N_2$ with $N_1,N_2$ odd, $N_1,N_2 \approx\frac{N}{2}$. Then $A$ has density at least $\delta$ in one of $\{1,...,N_1\}$ or $\{N_1+1,...,N_1+N_2\}$.\\
If (4) holds (note (3) $\implies$ (4)), then we pass to $P$ and start to again. After $\frac{40}{\delta}$ iterations, the density at least doubles. Therefore the toatl number of iterations we can have is at most $\frac{40}{\delta} + \frac{40}{2\delta}+... \leq \frac{80}{\delta}$.\\
If $\frac{\delta^2}{640} \sqrt{N} \geq N^{1/3}$ (to account for the above, and also for the possible use of (2)) at each iteration, and $\frac{\delta^3}{25} \geq N^{-1}$ (which follows from the first condition), then after $\frac{80}{\delta}$ iterations we have $N \geq N^{(1/3)^{80/\delta}}$, so the argument works provided
\begin{equation*}
    \begin{aligned}
        N^{(1/3)^{80\delta}} \geq \left(\frac{640}{\delta^2}\right)^6
    \end{aligned}
\end{equation*}
taking logs and simplify a bit, we need
\begin{equation*}
    \begin{aligned}
        &-\frac{80}{\delta} \log 3 + \log \log N \geq \log 6 + \log (\log 640 + 2 \log \frac{1}{\delta}))\\
        &\Leftarrow \log\log N \geq \frac{160}{\delta}\\
        &\Leftarrow \delta \geq \frac{160}{\log\log N}
    \end{aligned}
\end{equation*}

\subsection{Bogolyubov's method}
Let $K \subset \hat{\Z}_N$ and let $\delta > 0$. The \emph{Bohr set} $B(K,\delta)$ has two definitions (not exactly equivalent, but quite equivalent):\\
$\bullet$ (1) $B(K,\delta) = \{x \in \Z_N: rx \in [-\delta N,\delta N] \forall r \in K\}$ (arc-length definition);\\
$\bullet$ (2) $B(K,\delta) = \{x \in \Z_N: |1-\omega^{rx}| \leq \delta \forall r \in K\}$ (chord-length definition).

\begin{defi}
    Let $G$ be an abelian group and let $A,B$ be subsets of $G$. Then write $A + B = \{a+b: a \in A, b \in B\}$ and the obvious definition for $A-B$. We also write $rA = \{a_1+...+a_r:a_1,...,a_r\in A\}$ (note this might be different than what you think this notation should mean).
\end{defi}

\begin{lemma} (7)\\
    Let $A \subset \Z_N$ be a set of density $\alpha$. Then $2A-2A$ contains a Bohr set $B(K,1/4)$ (arc) with $|K| \leq \alpha^{-2}$.
    \begin{proof}
        Observe that $x \in 2A-2A$ iff $A*A*(-A)*(-A)(x) \neq 0$ (this makes more sense if we write it as $\E_{a+b-c-d=x} A(a)A(b)A(c)A(d) \neq 0$, i.e. we are basically just counting the number of ways $x$ can be written as $a+b-c-d$ where $a,b,c,d \in A$.)\\
        But
        \begin{equation*}
            \begin{aligned}
                A*A*(-A)*(-A)(x) &= \sum_r \widehat{A*A*(-A)*(-A)}(r) \omega^{rx} \text{ inversion formula}\\
                &= \sum |\hat{A}(r)|^4 \omega^{rx}
            \end{aligned}
        \end{equation*}
        Let $K = \{r: |\hat{A}(r)| \geq \alpha^{3/2}\}$. Then $\alpha = ||\hat{A}||_2^2 = \sum_r |\hat{A}(r)|^2 \geq \alpha^3 |K|$.\\
        So $|K| \leq \alpha^{-2}$.\\
        Now suppose that $x \in B(K,1/4)$. Then
        \begin{equation*}
            \begin{aligned}
                \sum_r |\hat{A}(r)|^4 \omega^{rx} &= \alpha^4 + \sum_{r \in K, r \neq 0} |\hat{A}(r)|^4 \omega^{rx} + \sum_{r \not\in K} |\hat{A}(r)|^4 \omega^{rx}
            \end{aligned}
        \end{equation*}
        The real part of the second term is non-negative since $rx \in [-N/4,N/4]$ when $r \in K$.\\
        Also
        \begin{equation*}
            \begin{aligned}
                \left|\sum_{r \not\in K} |\hat{A}(r)|^4 \omega^{rx}\right| &\leq \sum_{r \not\in K} |\hat{A}(r)|^4\\
                &< \alpha^3 \sum_{r \not\in K} |\hat{A}(r)|^2\\
                &\leq \alpha^4
            \end{aligned}
        \end{equation*}
        So it follows that the real part of $\sum_r |\hat{A}(r)|^4 \omega^{rx} > 0$, i.e. it is non-zero. So $x \in 2A - 2A$.
    \end{proof}
\end{lemma}

\begin{lemma} (8)\\
    Let $K \subset \Z_N$ and let $\delta > 0$. Then:\\
    (i) $B(K,\delta)$ (arc) has density at least $\delta^{|K|}$;\\
    (ii) $B(K,\delta)$ contains a mod-$N$ artihmetic progression of length at least $\delta N^{1/|K|}$.
    \begin{proof}
        (i) Let $K = \{r_1,...,r_k\}$. Consider the $N$ $k$-tuples $(r_1x,...,r_kx) \in \Z_N^k$. If we intersect this set of $k$-tuples with a random 'box' $[t_1,t_1+\delta N] \times ... \times [t_k,t_k+\delta N]$ (here we are thinking $t_i$ as real numbers), then the expected number of the $k$-tuples in the box is $\delta^k N$ (since each one has a probability $\delta^k$).\\
        But if $(r_1x,...,r_kx)$ and $(r_1y,...,r_ky)$ belong to this box, then $x-y \in B(K,\delta)$.\\
        (ii) If we take $\eta > N^{-1/k}$, then by (i) we get that $|B(K,\eta)| > 1$, therefore at least 2. So $\exists x \in B(K,\eta)$ s.t. $x \neq 0$. But then $dx\in B(K,d\eta)$ for every $d$.\\
        So if $d\eta \leq \delta$ then $dx \in B(K,\delta)$. That gives us an AP of length at least $\frac{\delta}{\eta}$. So we get one of length at least $\delta N^{1/k}$.
    \end{proof}
\end{lemma}

\begin{defi}
    Let $A,b$ be subsets of Abelian groups and let $\phi:A \to B$. Then $\phi$ is a \emph{Freiman homomorphism of order $k$} if
    \begin{equation*}
        \begin{aligned}
            a_1+...+a_k=a_{k+1}+...+a_{2k} \implies \phi(a_1)+...+\phi(a_k) = \phi(a_{k+1})+...+\phi(a_{2k})
        \end{aligned}
    \end{equation*}
    If $k=2$, we call this just a \emph{Freiman homomorphism}. In that case, the condition is equivalent to $a-b=c-d \implies \phi(a)-\phi(b)=\phi(c)-\phi(d)$.\\
    If $\phi$ has an inverse which is also a F-homomorphism of order $k$, then $\phi$ is a F-\emph{isomorphism} of order $k$.
\end{defi}

\begin{lemma} (9)\\
    Assume $0 \not \in K$, and $N$ is prime. If $\delta < 1/4$, then $B(K,\delta)$ (arc) is Freiman isomorphic to the intersection in $\R^{|K|}$ of $[-\delta N, \delta N]^{|K|}$ with some lattice $\Lambda$.
    \begin{proof}
        Let $K = \{r_1,...,r_k\}$, and let $\Lambda = N\Z^k + \{(r_1 x,...,r_kx):x \in \Z\}$. Write $\mathbf{r}$ for $(r_1,...,r_k)$. Claim that $B(K,\delta) \cong \Lambda \cap [-\delta N, \delta N]^k$.\\
        Define a map $\phi:B(K,\delta) \to \Lambda \cap [-\delta N, \delta N]^k$ by sending $x$ to $(\bra r_1 x\ket, ...,\bra r_k x\ket)$ where $\bra u\ket$ means the least-modulus residue $u$ mod $N$.\\
        If $x+y = z+w$, then $\mathbf{r} x +\mathbf{r} y = \mathbf{r}z+\mathbf{r}w$ in $\Z_N^k$. But for each $i$, $\bra r_ix\ket+\bra r_iy\ket-\bra r_iz\ket-\bra r_iw\ket \in [-4\delta N, 4\delta N]$. Since $\delta< 1/4$, that implies that $\bra r_ix\ket +\bra r_iy\ket-\bra r_iz\ket-\bra r_iw\ket =0$. So $\bra\mathbf{r}x\ket+\bra\mathbf{r}y\ket=\bra\mathbf{r}z\ket+\bra\mathbf{r}w\ket$.\\
        That already implies that $\phi$ is an injection.\\
        If $\mathbf{r}x + \mathbf{a} N \in [-\delta N,\delta N]^k$, then $r_i x \in [-\delta N,\delta N]$ mod $N$ for each $i$. So $x \in B(K,\delta)$ and $\phi(x) = \mathbf{r}x + \mathbf{a} N$. So $\phi$ is a surjection as well.\\
        If $\mathbf{r}x+\mathbf{a}N + \mathbf{r}y+\mathbf{b}N = \mathbf{r}z+\mathbf{c}N+\mathbf{r}w+\mathbf{d}N$, then $r_1(x+y) = r_1(z_w)$ mod $N$, so $x+y=z+w$ mod $N$. So the inverse of $\phi$ is also a Freiman homomorphism.
    \end{proof}
\end{lemma}

\begin{lemma} (10)\\
    Let $\Lambda$ be a lattice and $C$ be a symmetric convex body, both in $\R^k$. Then $|\Lambda \cap C| \leq 5^k |\Lambda \cap \frac{C}{2}$|.
    \begin{proof}
        let $x_1,...,x_m$ be a maximal subset of $\Lambda \cap C$ such that for all $i \neq j$, $x_j \not\in x_i + \frac{C}{2}$. Then by maximality, the sets $x_i+\frac{C}{2}$ over(are?) all of $\Lambda \cap C$. Also, the sets $x_i+\frac{C}{4}$ are disjoint subsets of $\R^k$, and they are all contained in $C+\frac{C}{4} = \frac{5}{4} C$. So $m \leq \frac{vol(\frac{5}{4}C)}{vol(\frac{1}{4}C)} = 5^k$.
    \end{proof}
\end{lemma}

\begin{coro} (11)\\
    If $N$ is prime, $0 \not \in K$, $|K| = k$, $\delta < 1/4$, then $|B(K,\delta)| \leq 5^k |B(K,\frac{\delta}{2})|$.
\end{coro}

\newpage

\section{Sumsets and their structure}
It's to be shown that $|A+A| \leq K|A|$ $\implies$ $|rA-sA| \leq K^{r+s}|A|$ (Ruzsa).

\begin{lemma} (1, Petridis)\\
    Let $A_0,B$ be finite subsets of an Abelian group such that $|A_0+B| \leq K_0|A_0|$. Then there exist a non-empty subset $A \subset A_0$ and $K \leq K_0$ s.t. $|A+B+C| \leq K|A+C|$ for every finite subset $C$ of the group.
    \begin{proof}
        Let $A$ minimize the ratio $\frac{|A+B|}{|A|}$, and let the minimal ratio be $K$.\\
        Claim: this works. We prove this by induction on $C$.\\
        If $C =\phi$, then the result holds. Now assume it for $C$ and let $x \not\in C$. Then $A+(C \cup \{x\}) = (A+C) \cup [(A+x) \setminus (A'+x)]$ where $A' = \{a \in A:a+x \in A+C\}$.\\
        This is a disjoint union, so $|A+(C\cup \{x\})| = |A+C|+|A|-|A'|$, $A+B+(C\cup\{x\}) = (A+B+C) \cup ((A+B+x)\setminus (A'+B+x))$, since if $a+x \in A+C$ then $a+B+x \subset A+B+C$. So
        \begin{equation*}
            \begin{aligned}
                |A+B+(C\cup\{x\})| &\leq |A+B+C|+|A+B|-|A'+B|\\
                &\leq K|A+C|+K|A|-K|A'|
            \end{aligned}
        \end{equation*}
        by induction and minimality property of $A$.
    \end{proof}
\end{lemma}

---Lecture 5---

\begin{coro} (2)\\
    If $A,B$ are finite subsets of an abelian group, and $|A+B| \leq K^r|A|$, then there exists $A'\subset A$, $A' \neq \phi$ such that $|A'+rB| \leq K|A'|$ for every positive integer $r$.
    \begin{proof}
        Choose $A'$ as we choose $A$ in lemma 1. Then $|A'+rB| = |A'+B+(r-1)B| \leq K|A'+(r-1)B|$.\\
        And $|A'+B| \leq K|A'|$, so we are done by induction.
    \end{proof}
\end{coro}

\begin{coro} (3)\\
    If $|A+A| \leq K|A|$ or $|A-A| \leq K|A|$, then $|rA| \leq K^r|A|$.
    \begin{proof}
        Set $B=A$ or $-A$ in corollary 2. (think about this)
    \end{proof}
\end{coro}

\begin{lemma} (4, Ruzsa triangle inequality)\\
    Let $A,B,C$ be finite subsets of an abliean group. Then $|A||B-C| \leq |A-B||A-C|$.
    \begin{proof}
        Define a map $\phi: A\times(B-C) \to (A-B) \times (A-C)$. Given $(a,x)$ with $a\in A, x \in B-C$, choose, \emph{somehow}, $b(x) \in B$ and $c(x) \in C$ s.t. $b(x) - c(x) = x$, and set $\phi(a,x) = (a-b(x),a-c(x))$. Note that $(a-c(x)) - (a-b(x)) = b(x)-c(x) = x$ (!). And then, having worked out $x$, we know $b(x)$, and $a=a-b(x)+b(x)$, so $a$ is determined too. So $\phi$ is an injection.
    \end{proof}
\end{lemma}

The proof was easy, but why is this called the triangle inequality? We can rewrite it as
\begin{equation*}
    \begin{aligned}
        \frac{|B-C|}{|B|^{1/2}|C|^{1/2}} \leq \frac{|A-B|}{|A|^{1/2}|B|^{1/2}} \cdot \frac{|A-C|}{|A|^{1/2}|C|^{1/2}}
    \end{aligned}
\end{equation*}
So if we define the \emph{Ruzsa distance} $d(A,B) = \frac{|A-B|}{|A|^{1/2}|B|^{1/2}}$, then the inequality says $d(B,C) \leq d(A,B) d(A,C)$.

\begin{coro} (5)\\
    If $|A+B| \leq K|A|$, then $|rB-sB| \leq K^{r+s}|A|$ for all $r,s$.
    \begin{proof}
        Pick $a'$ as before. Then by corollary 2 with $B$ replaced by $-B$, $|A'-rB| \leq K^r|A'|$ and $|A'-sB| \leq K^s|A'|$.\\
        Therefore, by Rusza triangle inequality (lemma 4), $|A'||rB-sB| \leq K^{r+s}|A'|^2$, so $|rB-sB| \leq K^{r+s} |A|$.
    \end{proof}
\end{coro}

One finally corollary:
\begin{coro} (6, Plunnecke's theorem)\\
    If $|A+A| \leq K|A|$ or $|A-A| \leq K|A$, then $|rA-sA| \leq K^{r+s} |A|$.
    \begin{proof}
        Just apply corollary 5 with $B=-A$ or $B=A$.
    \end{proof}
\end{coro}

\begin{lemma} (7, Ruzsa's embedding lemma)\\
    Let $A \subset \Z$ be finite and suppose that $|kA-kA| \leq C|A|$. Then there exists a prime $p \leq 4C|A|$ and a subset $A'\subset A$ of size at least $|A|/k$ such that $A'$ is Freiman isomorphic of order $k$ to a subset of $\Z_p$.
    \begin{proof}
        Consider the following composition of maps $\Z \xrightarrow{\text{ reduce mod }q}\Z_q \xrightarrow{\times \text{ by random non-zero } r} \Z_q \xrightarrow{\text{least non-negative residue}} \Z \xrightarrow{\text{reduce mod }p} \Z_p$, where $q$ is a prime bigger than $diam A$ and $p$ is a prime $\in (2C|A|,4C|A|]$ by Bertrand's postulate.\\
        Let $|phi$ be the composition. THe first, second and fourth parts are group homomorphisms, and thus Freiman homomorphisms of all order. Also, the third map is a Freiman homomorphism of order $k$ if you restrict to a subinterval of $[0,q-1]$ of lemgth $\leq q/k$. To see this, write $\bra u \ket$ for least non-negative residue. Then if $I$ has length $\leq q/k$ (and therefore $<q/k$) and $u_{1},...,u_{2k} \in I$, then if $i_1+...+u_k-u_{k+1}-...-k_{2k} = 0$, then $\bra u_1 \ket + ... + \bra u_k \ket - \bra u_{k+1} \ket - ... - \bra u_{2k} \ket \equiv 0 \pmod q$, and also has modulus less than $q$. So it is zero.\\
        By the pigeonhole principle, for any $r$ we can find $I$ of length $\leq q/k$ such that $A' = \{a \in A: ra \in I\}$ has size at least $|A| / k$.\\
        Then $\phi|_{A'}$ is a Freiman homomorphism of order $k$. It's now remain to prove that $\phi$ is an isomorphism to its image, i.e. we must show that if $a_1+...+a_k-a_{k+1}-...-a_{2k} \neq 0 (a_i \in A)$, then
        $$ \bra ra_1\ket + ... + \bra ra_k\ket - \bra ra_{k+1} \ket - ... - \bra ra_{2k} \ket \not\equiv 0 \pmod p$$
        But if the $a_i$ are chosen sucth that the $ra_i$ all belong to the same interval of length $\leq q/k$, then
        $$|\bra ra_1\ket + ... + \bra ra_k\ket - \bra ra_{k+1} \ket - ... - \bra ra_k\ket| < q$$
        and is congruent to $r(a_1+...+a_k-a_{k+1}-...-a_{2k}) \pmod q$.\\
        So all that can go wrong is if $r(a_1+...+a_k-a_{k+1} - ... - a_{2k}$ is $xp$ for some $x \neq 0$, with $|x| < q/p$. The number of values to avoid is at most $2q/p$, so for each $a_1+...+a_k-a_{k+1} - ... - a_{2k}$, the probability of going wrong if $r$ is chosen randomly is at most $2/p$. So since $|kA-kA| \leq C|A|$, the probability of going wrong is at most $\frac{2}{p} C|A|$. Since $p > 2C|A|$, there exists $r$ s.t. we get an Freiman isomorphism of order $k$.
    \end{proof}
\end{lemma}

\end{document}


\chapter{Summary of Saleena and Paleri}
\label{chap:chapter4}

Saleena and Paleri gave an algorithm for
\textbf{global value numbering (GVN)}. GVN works by assigning a value
number to variables and expressions. The same value number is
assigned to those variables and expressions which are provably
equivalent. A notable difference between Herbrand equivalence and GVN
is that in Herbrand equivalence we talk about equivalences at a
particular program point but in GVN are concerned with equivalence
between expressions at two different program points.

The data structure used in the algorithm is called \textbf{value
expression} - an expression with value numbers as operands. Two
expressions are equivalent if they have same value expression.
So, a value expression can be used to represent a set of equivalent
program expressions.

\section{Notation}
\label{sec:Notation}
Input is a flow graph atmost one assignment statement in each node
which has one of the following forms
$$x\; ::=\; e$$
$$e\; ::=\; x\: |\: c\: |\: x\,op\,x$$
The flow graph also has two additional empty $ENTRY$ and $EXIT$ nodes.
For a node $n$, $IN_n$ and $OUT_n$ denotes the input and output
program points of the node.

Expression pool at a program point, is a partition of expressions at
that point, in which equivalent expression belongs to the same
partition. Each class will have a value number which we will consider
as its first element. For a node $n$, $EIN_n$ and $EOUT_n$ denotes
the expression pools at input and output program points of the node.

\section{Value Expression}
\label{sec:ValueExpression}
The value expression corresponding to an expression is obtained by
replacing actual operands with their corresponding value numbers.
Example - For the expression-pool $\{\{v_1, a, x\}$, $\{v_2, b, y \}\}$
and statement $z ::= x + y$ , the value-expression for $x + y$ will be
$v_1 + v_2$. Instead of $x + y$, its value-expression is included in
the expression-pool, with a new value number ie. the new
expression-pool would be
$\{\{v_1, a, x\},\{v_2, b, y\}, \{v_3, v_1 + v_2, z\}\}$.

The value expression $v_1 + v_2$ represents not just $x + y$ but the
set of equivalent expressions $\{a + b, x + b, a + y, x + y\}$. Its
presence indicates that an expression from this set is already
computed and this information is enough for detection of redundant
computations. Also, a single binary value expression can represent
equivalence among any numbre of expressions of any length. Example -
$v_1 + v_3$ represents, $a + z$, $x + z$, $a + (a + b)$,
$a + (x + b)$ and so on.

\section{Algorithm}
\label{sec:Algorithm}
Similar to Gulwani's algorithm, the algorithm consists of two main
functions - a transfer function for changes in expression pool across
assignment statements and a confluence function to find the
expression pool at points were two branches meet. The algorithm
starts with $EOUT_{ENTRY} = \phi$, and uses transfer and confluence
functions to calculate expression pools at other points. This process
is repeated till there is any change in the equivalence information.
For detailed implementation refer to \cite{Saleena}.


\chapter{Sampling with sufficient statistics}
This chapter will go in depth of generating samples from the NHPP given its sufficient statistics. The main focus will be a Gibbs sampler with a Metropolis-Hastings step, and discuss possible modifications.
\section{Gibbs sampler}
The Gibbs sampler is Markov chain Monte Carlo (MCMC) sampler. A MCMC sampler is a way to sample from a distribution by contructing a Markov chain with spesific equilibrium. From \cite{casella1992explaining} we have that the Gibbs sampler is used when direct sampling is difficult or cannot be done. This sampler samples from a conditional distribution. This can be done with sufficient statistics.
\section{Metropolis-Hastings sampler}
This is also a MCMC sampler where direct sampling is difficult. However this is used when the conditional posterior is unknown. Hence this uses a proportional density instead, and has an acceptance rejection step to determine if the new sample is from the desired distribution.
\section{Gibbs with Hastings step}
\label{sec:gibbsmet}
A Gibbs sampler with a Metropolis-Hastings step is what we used in this project \cite{gilks1995adaptive}. Since the values of the sufficient statistics cannot differ from the original data set, the sampler must generate new samples with the same values for the sufficient statistics. Given a sample with $N$ points and number of sufficient statistics, one can draw three arbitrary points and draw a new value for one of the points given the two others. By setting one of the points one can calculate the two others. This is where the Metropolis-Hastings step is used to draw a new value. How to find a new value is shown in section \ref{sec:draw}. Further when this new value is drawn the two other values must be adjusted to fulfill the sufficient statistics. This should be done serveral times for each sample. The reason for this is to make the samples independent.
\subsection{Drawing a new value}
\label{sec:draw}
Assume we have chosen three points ($X_1, X_2, X_3$) from our sample and that we have the same sufficient statistics as in section \ref{sec:sufstat}. It is these three points that are going to be updated. The idea is to draw a new value for one of them as stated in section \ref{sec:gibbsmet}. Assume $\tau = 1$. Hence all values in the samples must be between 0 and 1.  This makes it easier for the uniform distribution. To draw a new value we need to use tranformation of variables from section \ref{sec:transvar}. The tranformation is
\begin{equation*}
(Z_1, Z_2, Z_3) = (\sum_{i=1}^{3} X_i, \prod_{i=1}^{3} X_i, X_3).
\end{equation*}
This will yield the probability density $f_{Z_1, Z_2, Z_3}(z_1, z_2, z_3)$. However we are interested in $f_{Z_3 | Z_1, Z_2}(z_3 | z_1, z_2)$, where $Z_3=X_3$ is the variable drawn from this distribution. Fortunately from Bayes rule  \cite{casella2002statistical} the distribution becomes
\begin{equation*}
f_{Z_3 | Z_1, Z_2}(z_3 | z_1, z_2) \propto f_{Z_1, Z_2, Z_3}(z_1, z_2, z_3).
\end{equation*}
Further we continue with tranformation of variables. The determinant of the Jacobian matrix then becomes
\begin{equation*}
|J| = X_3 (X_2 - X_1).
\end{equation*}
We also have the following relations from the tranformation, where we have put $a = Z_1$ and $b = Z_2$
\begin{align*}
X_1 + X_2 + X_3 = a \\
X_1 + X_2 = a - X_3 \\
X_1 \cdot X_2 \cdot X_3 = b \\
X_1 \cdot X_2 = \frac{b}{X_3}.
\end{align*}
Then $X_1$ and $X_3$ are the roots of
\begin{equation*}
X^2 - (a - X_3) X^2 + \frac{b}{X_3} = 0,
\end{equation*}
which are
\begin{equation*}
X_{(1,2)} = \frac{(a - X_3) ± \sqrt{(a - X_3)^2 - \frac{4b}{X_3}}}{2}.
\end{equation*}
From this the jacobian becomes
\begin{equation*}
|J| = X_3\sqrt{(a - X_3)^2 - \frac{4b}{X_3}}.
\end{equation*}
From paper \cite{lockhart2007use} the determinant of the inverse jacobian is
\begin{equation*}
|J|^{-1} = \frac{1}{X_3\sqrt{(a - X_3)^2 - \frac{4b}{X_3}}}.
\end{equation*}
Since we have that
\begin{equation*}
f_{X_1, X_2, X_3}(x_1, x_2, x_3) = 1, \quad 0 \leq x_i \leq 1 \text{ for all }i,
\end{equation*}
the density to draw new values for $X_3$ becomes
\begin{equation*}
\Pi \propto \frac{1}{X_3\sqrt{(a - X_3)^2 - \frac{4b}{X_3}}}.
\end{equation*}
This is where the Metropolis-Hastings step is used. The requirements for this density are listed below.
\begin{align}
0 \leq X_3 \leq \min(1, a) \label{eq:req1} \\
(a - X_3)^2 - \frac{4b}{X_3} \geq 0 \label{eq:req2}\\
(a - X_3) + \sqrt{(a - X_3)^2 - \frac{4b}{X_3}} \leq 1. \label{eq:req3}
\end{align}
So when we draw from a proposal density like a uniform distribution we need to make sure that the new $X_3$ fulfills these requirements. If the requirements are not fulfilled then the probability of accepting this new value is zero. If the requirements are fulfilled then the acceptance probability is defined as,
\begin{equation}
\alpha = \min\left(1, \frac{\Pi(X_{prop})}{\Pi(X_{curr})}\right)
\label{eq:alpha}
\end{equation}
An overview of the algorithm is shown below.
\begin{algorithm}
\caption{Generate new samples with sufficient statistics}
\label{alg:simdata}
\begin{algorithmic}
\STATE Draw 3 arbitrary indices from $\{1,2,...,n\}$
\STATE Caclulate sum $a$ and product $b$ of the corresponding $X$'s.
\STATE Draw a proposal $X_3$ from $U[0,\min(a,1)]$.
\STATE Check requirements in equations \ref{eq:req1}, \ref{eq:req2} and \ref{eq:req3}.
\STATE Calculate $\alpha$ from equation \ref{eq:alpha}.
\STATE Accept with probability $\alpha$
\STATE Repeat from top desired times to make sample approximately independent.
\end{algorithmic}
\end{algorithm}
\subsection{Possible modifications}
In our current version if the sampler generates invalid values for the three new indices in a sample, the algorithm tries over again with three new indices. One possible modification is to not try over again with three new indices, but keep the three indices and draw new random value. However the area for valid values might be small. Hence this modification might not be recommended.
\\
\\
Another modification is to use the indices of the entire sample when generating a new sample. This has been done before for related distribution in \cite{lockhart2007use}. The pros of this method is that you have chance of getting new  values for more of the indices. However the cons is that the values for the sub sums and sub products are sensitive to large and small values when a larger number of indices are used.


%%!TEX TS-program = latex
\documentclass[11pt]{article} %DIF >
\usepackage{etex}
\usepackage[utf8]{inputenc}
\input ../AuxFiles/PreambleNotes.tex

\begin{document}
\onehalfspace

\vspace*{\fill}
\begingroup
\centering

\Large {\scshape Introduction to Statistics}\\

(Lectures 11-12: Confidence Sets)

\endgroup
\vspace*{\fill}

\newpage

\section{Confidence Sets}

{\scshape Overview:} \noindent Lectures 11-12 will focus on the construction of \emph{confidence sets}. The actions are subsets of the parameter space. Each action is interpreted as a region that (presumably) contains the parameter selected by nature. The loss function we work with is a convex combination of ``volume’’ and ``coverage’’, in a sense we will make precise.

We first show that Bayes rules for this problem are ``highest posterior density” sets: the collection of parameters with a sufficiently high posterior probability. We then argue that in large samples, highest posterior density sets can be approximated by ``inverting’’ a Wald test. When the parameter of the model is scalar, inverting the Wald test is tantamount to reporting the interval formed by the Maximum Likelihood estimator $\pm$ $c$ times its estimated “standard error”.

We then introduce and analyze two popular algorithms for summarizing the uncertainty in parameter estimation: (parametric) bootstrap confidence intervals and Bayesian ``credible’’ intervals. We use these algorithms to construct confidence/credible regions for one of the possibly many coefficients of the Linear Regression model.

\subsection{Description of the problem}

Let $X$ be a random variable and let $\{f(x | \theta)\}_{\theta \in \Theta}$ be a statistical model. The problem of constructing a confidence set for $\theta$ starts as follows. In the first stage nature picks an element of the parameter space, which in this section we model explicitly as a subset of $\mathbb{R}^{k}$. The econometrician cannot observe the parameter selected by nature but observes data. Based on the data, the econometrician has to suggest a ``region’’ of the parameter space that is likely to contain the true parameter selected by nature.

In principle, the action space for the econometrician based on the description above is the power set of $\Theta$. A minor technical restriction is that the econometrician can only report regions that are ``measurable’’. This happens because the ``volume’’ of whatever set is reported will be part of the loss function. Let $\mathcal{C}$ denote the class of regions that the econometrician can report and let $c$ denote an element of $\mathcal{C}$.

A \emph{decision rule/algorithm/strategy} for this problem is a mapping
\begin{equation*}
C:X \rightarrow \mathcal{C}
\end{equation*}
These decision rules will be called ``confidence’’ regions.

\subsection{Loss function}

The payoff/loss for the econometrician depends on the action taken ($c \in \mathcal{C}$) and the true parameter ($\theta$). The following loss is sometimes used to evaluate confidence sets:
\[ \mathcal{L}_{\delta}(c, \theta) = \delta \mathbf{1}\{ \theta \notin c \} + (1-\delta)\int_{c} d\theta, \]

\noindent where $\delta \in (0,1)$. The first component in the loss function above penalizes the econometrician whenever $\theta$ does not belong to $c$. The econometrician could avoid the penalty by reporting a ``large’’ region (for example, reporting $\Theta$).  The second component makes large regions unattractive by taking into account their volume.

The expected loss (risk) of a confidence region $C$ is given by

\begin{equation} \label{equation:TypeI}
\mathbb{E}_{f(x|\theta)} [  \mathcal{L}_{\delta}(C(x),\theta)   ] = \delta (1-P_{f(x|\theta)} ( \theta \in C(x) ))  + (1-\delta) \mathbb{E}_{f(x|\theta)} [ \textrm{Vol}(C(x))].
\end{equation}

\noindent The term
\[ P_{f(x|\theta)} ( \theta \in C(x) ) \]
is typically referred to as the \emph{coverage probability} of $C(\cdot)$ at $\theta$. This captures how often the confidence region contains the parameter $\theta$ when the data is indeed generated by it. The worst coverage probability over the parameter space:

\[ \inf_{\theta \in \Theta} P_{f(x|\theta)} ( \theta \in C(x) )  \]

\noindent is called the \emph{confidence level} of the confidence set $C(x)$.

\subsection{Minimizing Posterior-Loss}

We now present a rule that minimizes posterior-loss. Any such rule, by construction, cannot be uniformly improved: there is no other region that has better coverage everywhere and also smaller expected volume. Let $\pi(\cdot)$ denote a prior over $\Theta$ and let $\pi(\theta | x)$ denote the posterior.

The posterior loss of an action $c$ is given by:

\[ \delta(1- P_{\pi(\theta | x)}( \theta \in c  )) + (1-\delta) \int_{c} d \theta.   \]

\noindent The term
\[ P_{\pi(\theta | x)}( \theta \in c ) = \int_{c} \pi(\theta | x) d \theta  \]
\noindent is the posterior probability that $\theta$ falls in a set $c$. Such probability is usually referred to as the \emph{credibility} of $c$. Algebra shows that the posterior loss can be written as

\[ \delta - \int_{c}  \left(  \delta \pi(\theta | x) - (1-\delta) \right) d\theta.  \]

\noindent Thus, for each $x$, the posterior loss is minimized by choosing
\begin{equation} \label{equation:HPD}
C^{\textrm{Bayes}}_{\delta}(x) = \{ \theta \in \Theta \: | \: \pi(\theta | x) \geq (1-\delta)/\delta  \}.
\end{equation}
The region above is called a ``highest posterior density’’ region. As it is name suggests, the region collects the points in the parameter space for which the posterior density is above certain threshold.

In general, it is hard to give closed form expressions for the highest posterior density region. However, in large samples it is possible to provide (heuristic) approximations for these regions; exploiting the fact that---regardless of the prior---the posterior distribution is approximately distributed as
\[  \mathcal{N}_{k} \left(  \widehat{\theta}_{\textrm{ML}} \quad, \quad \widehat{I}^{-1} \right),   \]
\noindent where $\widehat{I}$ is the observed information. This means that highest posterior density regions will be approximately of the form
\begin{eqnarray*}
\left \{ \theta \in \Theta \: \Bigg | \:  \frac{1}{ (2 \textrm{det} \widehat{I}))^{k/2} } \exp \left( -\frac{1}{2} (\theta - \widehat{\theta}_{\textrm{ML}}) ’\widehat{I}   (\theta - \widehat{\theta}_{\textrm{ML}})\right)  \geq (1-\delta)/\delta \right \}, \\
\end{eqnarray*}
which can be written as
\[ \left \{ \theta \in \Theta \: \Bigg | \:  (\widehat{\theta}_{\textrm{ML}} - \theta) ’\widehat{I}   (\widehat{\theta}_{\textrm{ML}}-\theta)   \leq \widehat{c}_{\delta} \right \},
\]
\noindent where $\widehat{c}_{\delta}$ is some function that depends on $\delta$ and $\widehat{I}$ but not on $\theta$. Recognize the quadratic form as the Wald statistic for the problem
\[ \mathbf{H}_0: \theta = \theta_0, \textrm{ vs. } \mathbf{H}_1: \theta \neq \theta_0. \]
Thus, in large samples, highest-posterior density regions are approximately given by those points in the parameter space in which the Wald statistic is below certain threshold that might depend on the data.

When $\theta$ is scalar, the approximation takes the form
\[ \Big[ \widehat{\theta}_{\textrm{ML}} - \sqrt{\frac{\widehat{c}_{\delta}}{\widehat{I}}} \:, \: \widehat{\theta}_{\textrm{ML}} + \sqrt{\frac{\widehat{c}_{\delta}}{\widehat{I}}} \Big]  \]

\noindent {\scshape Example:} We derive the highest posterior density region for a linear regression model with known variance. We have shown that if the prior density is that of a $\mathcal{N}_{k}(0, \sigma^2 (\mathbb{I}_n/\lambda))$, the posterior of $\beta$ is given by
\[  \beta | Y, X \sim \mathcal{N}_{k} ( m , V   ),  \]
where
\begin{eqnarray*}
m  &\equiv& (X’X + \lambda \mathbb{I}_n)^{-1} X’Y, \\
V  & \equiv & \sigma^2  (X’X + \lambda \mathbb{I}_n)^{-1}.
\end{eqnarray*}
\noindent The highest-posterior density region is thus
\begin{eqnarray*}
C_{\delta}(x) &= & \left \{ \beta \in \mathbb{R}^{k} \: \Big | \:  \frac{1}{ (2 \textrm{det} V))^{k/2} } \exp \left( -\frac{1}{2 } (\beta - m) ’V^{-1}  (\beta - m) \right)  \geq (1-\delta)/\delta \right \}, \\
&=&  \left \{ \beta \in \mathbb{R}^{k} \: \Big | \:   (\beta - m) ’V^{-1}  (\beta - m)  \leq \widehat{c}_{\delta,X,\sigma^2} \right \}.
\end{eqnarray*}

\noindent This is an ellipse centered at the posterior mean. In large samples,
$$ m \approx \widehat{\beta}_{\textrm{OLS}}, \quad V \approx \sigma^2 (X’X)^{-1}.$$
Hence, the highest-posterior density region is approximately given by
\[ \left \{ \beta \in \mathbb{R}^{k} \: \Big | \:  \frac{1}{\sigma^2}(\widehat{\beta} - \beta) ’(X’X)  (\widehat{\beta} - \beta)  \leq \widehat{c}_{\delta,X,\sigma^2} \right \}. \]

\noindent This is set of all parameter values for which the Wald test for the null $\beta = \beta_0$ is low enough.

\subsection{Confidence Regions and Hypothesis Testing}
There is a connection between confidence regions and tests of hypothesis that goes beyond the approximation argument above. Any family of tests of size $\alpha$ can be ``inverted’’ to construct a confidence region with confidence level of at least $1-\alpha$. Likewise, any confidence region with confidence level $1-\alpha$ can be transformed into a statistical test with rate of Type 1 error of at most $\alpha$.

{\scshape Inverting a family of tests:} Let $\{\phi_{\theta}\}_{\theta \in \Theta}$ denote a collection of non-randomized tests. Suppose each element in the collection is a test for the problem
\[ \mathbf{H}_0: \theta =\theta_0, \quad \mathbf{H}_1: \theta \neq \theta_0. \]
Suppose further that each test has size of at most $\alpha$ in the sense that
\[ \mathbb{P}_{f(x|\theta)} [\phi_{\theta}(x)=1] \leq \alpha. \]

\noindent The confidence region obtained by ``test inversion’’ is given by

\[ C  (x) \equiv \{ \theta \in \Theta \: | \: \phi_{\theta}(x)=0\}. \]

\noindent The coverage of such confidence region at $\theta$ is given by
\begin{eqnarray*}
P_{f(x|\theta)} [\theta \in C(x)] &=& 1-P_{f(x|\theta)} [\theta \notin C(x)]  \\
&=& 1- P_{f(x|\theta)} [ \phi_{\theta}(x) = 1] \\
& \geq & 1-\alpha.
\end{eqnarray*}
\noindent Therefore,
\[\inf_{\theta \in \Theta} P_{f(x|\theta)} [\theta \in C(x)] \geq 1-\alpha,   \]
\noindent which implies that the confidence region formed by test inversion has confidence level, provided each of the tests $\phi_{\theta}$ have size of at $\alpha$. \\

{\scshape Using a confidence region to test a hypothesis:} A confidence region with $C(x)$ confidence level of at least $1-\alpha$ can be readily used to test a hypothesis of the form
\[ \mathbf{H}_0: \theta =\theta_0 \quad \textrm{vs.} \quad \mathbf{H}_1: \theta \neq \theta_0. \]

Consider the test $\phi$ that rejects the null hypothesis above whenever $\theta \notin C(x)$. The rate of Type I error of the test is
\begin{eqnarray*}
P_{f(x| \theta)} [ \phi(x)=1 ]  &=&  P_{f(x| \theta)} [\theta \notin C(x)] \\
&=&   1-  P_{f(x| \theta)} [\theta \in C(x)]  \\
&\leq& 1- \inf_{\theta \in \Theta} P_{f(x| \theta)} [\theta \in C(x)] \\
&\leq& 1-(1-\alpha) =\alpha.
\end{eqnarray*}

\subsection{Parametric Bootstrap and Bayesian Credible sets}

\subsubsection{The Parametric Bootstrap}

Suppose that we have a statistical model $\{ f(x | \theta)\}_{\theta \in \Theta}$ for data $X$.  The estimator $\widehat{\theta}_{\textrm{ML}}$ is a random variable, as it depends on the data. If $\theta$ were known, we could approximate the distribution of $\widehat{\theta}_{\textrm{ML}}$ by Monte-Carlo methods: generate $I$ independent draws from $x \sim f(x;\theta)$, and evaluate $\widehat{\theta}_{\textrm{ML}}$ over the $I$ new data sets that we have generated.  \\

The true parameter that generated the data is unknown; hence it is not possible to generate draws from the model $f(x;\theta)$. However, since there is an estimator for $\theta$, one could perform the Monte-Carlo approximation exercise described above by using draws from the model:

\[ f(x | \widehat{\theta}_{\textrm{ML}}).\]

If we do this, we will effectively end with $I$ estimators (one for each new data set). Suppose that the sample size is large enough to guarantee that $\widehat{\theta}_{\textrm{ML}}$ is close to the true $\theta$ (whatever the true value might be) with high probability. If the parametric model $f(x | \theta)$ varies smoothly with $\theta$, the c.d.f. based on the $I$ estimators can be shown to be a reasonable approximate for the distribution of $\widehat{\theta}_{\textrm{ML}}$. If the parameter of interest is some function $g(\theta)$, the procedure above also provides an approximation for $g(\widehat{\theta}_{\textrm{ML}})$, provided $g$ is smooth. \\

The exercise described above is known as the \underline{parametric bootstrap} and it is a general statistical technique used to estimate the distribution of $\widehat{\theta}_{\textrm{ML}}$ or any function smooth function $g(\widehat{\theta}_{\textrm{ML}})$. \\

\noindent {\scshape Example:} Consider the linear regression model with unknown variance
\[ Y \sim \mathcal{N}_n (X\beta, \sigma^2 \mathbb{I}_n). \]
The parameters of the model are $\beta = (\beta_1, \ldots, \beta_{k})’$ and $\sigma^2$. Suppose we want to construct a confidence region for the $\beta_1$. The ML estimators for $\beta$ and $\sigma^2$ are
\[ \widehat{\beta}_{\textrm{ML}} = (X’X)^{-1}X’Y, \quad \widehat{\sigma}^2_{\textrm{ML}} = (Y-X \widehat{\beta}_{\textrm{ML}})’ (Y-X \widehat{\beta}_{\textrm{ML}})/n.  \]

\noindent The parametric bootstrap algorithm starts by generating $I$ new data sets, independently according to the distribution
\[ Y(i)  \sim \mathcal{N}_{n}( X \widehat{\beta}_{\textrm{ML}}\:, \: \widehat{\sigma}^2_{\textrm{ML}} \mathbb{I}_n).   \]
\noindent For each of these data sets, we can compute

\[ \widehat{\beta}_{\textrm{ML}}(i) = (X’X)^{-1}X’Y(i), \quad \widehat{\sigma}^2_{\textrm{ML}} = (Y(i)-X \widehat{\beta}_{\textrm{ML}}(i))’ (Y(i)-X \widehat{\beta}_{\textrm{ML}}(i))/n,  \]

\noindent and collect the first component of $ \widehat{\beta}_{\textrm{ML}}(i)$:

\[  \widehat{\beta}_{\textrm{ML}}(i)_1 = e_1’ \widehat{\beta}_{\textrm{ML}}(i), \]

\noindent where $e_1$ is the first column of the identity matrix of dimension $k$. Let $\widehat{q}_{\eta}$ denote the $\eta$ quantile of $\{ \widehat{\beta}_{\textrm{ML}}(i)_1\}_{i=1}^{I}$. The parametric bootstrap confidence interval for $\beta_1$ (of ``nominal’’ level $1-\alpha$) for $\beta_1$ is thus given by
\[ C_{\textrm{Bootstrap}}(Y,X) \equiv [ \widehat{q}_{\alpha/2} \:, \: \widehat{q}_{(1-\alpha)/2} ]. \]

\subsection{Credible Sets based on the quantiles of the posterior}
The quantiles of the posterior provide an off-the-shelf algorithm to construct ``credible’’ regions for any real-valued function $g(\theta)$. Start with the parametric model $\{f(x|\theta)\}$ and let $\pi(\theta)$ be a prior. Suppose that we can generate $I$ draws from the posterior distribution $\pi(\theta | x)$. Denote these draws $\{ \theta(i) \})_{i=1}^{I}$. For each of these draws we can construct
\[ \Big\{ g(\theta(i)) \Big\}_{i=1}^{I}. \]
Let $\widehat{p}_{\eta}$ denote the $\eta$ quantile of $\{ g(\theta(i))\}_{i=1}^{I}$. The interval with credibility of approximately $1-\alpha$ is equal to
\[ C_{\textrm{Bayes}} \equiv [ \widehat{b}_{\alpha/2} \:, \: \widehat{b}_{(1-\alpha)/2} ]. \]

\noindent {\scshape Example:} Let’s go back to the linear regression model with unknown variance. The parameter of interest is still $\beta_1$. The prior is $\mathcal{N}_{k}(0, \sigma^2 (\mathbb{I}_n/\lambda))$. The posterior of $\beta$ (conditional on $\sigma^2$) is given by
\[  \beta | Y, X, \sigma^2 \sim \mathcal{N}_{k} ( m , V   ),  \]
where
\begin{eqnarray*}
m  &\equiv& (X’X + \lambda \mathbb{I}_k)^{-1} X’Y, \\
V  & \equiv & \sigma^2  (X’X + \lambda \mathbb{I}_k)^{-1}.
\end{eqnarray*}
The posterior distribution for distribution for $\sigma^2$ is an Inverse-Gamma($a_n$,$b_n$) where
\[ a_n = a_0 + n/2, \quad b_n = b_0 + \frac{1}{2}( (Y-Xm)’(Y-Xm) + \lambda m’m ). \]

We can generate $I$ draws for $\sigma^2$ from the Inverse-Gamma posterior. For each draw $\sigma^2(i)$ we can also take a draw from $\beta | Y,X, \sigma^2$, and collect the first element $\beta(i)_1$. The $\alpha/2$ and $(1-\alpha)/2$ quantiles, provide an approximation to the credible set for $\beta_1$.

\newpage

\newpage

\bibliographystyle{../AuxFiles/ecta}
\bibliography{../AuxFiles/BibMaster}

\end{document}


\subsection{Homogeneity}
\begin{definition}
	A differential equation is homogeneous if all terms involve the dependent variable or its derivatives.
\end{definition}

\noindent
Below is a table of homogeneity and equation numbers.
\begin{table}[H]
	\centering
	\begin{tabular}{c|c}
		Homogeneity & Equation Number \\
		\hline
		Homogeneous &  3, 7, 8 \\
		Heterogeneous & 1, 2, 4, 5, 6, 9, 10 \\
	\end{tabular}
\end{table}
\noindent
Homogeneous equations have some nice properties, like always having at least a trivial solution of 0.


\section{Introduction}\label{sec:intro}

\mypar{Motivation} The longest common subsequence (LCS) problem is the problem of finding the longest (not necessarily contiguous) subsequence that is shared by two input strings. This problem occurs very widely in practice. For example, the two sequences $ABCD$ and $ACBAD$ have the longest common subsequences $ACD$ and $ABD$. The length of the LCS is unique, although multiple subsequences with that length might exist.

LCS algorithms are at the core of comparison tools like ``diff'', which are ubiquitous when working with source code. These comparisons are also used extensively in version control systems. Outside of the software domain, the longest common subsequence problem is extremely important in biology - LCS is the exact solution to DNA nucleotide \emph{sequence alignment}. However, it is important to note that heuristic methods (BLAST \cite{altschul_basic_1990}) are often used today, since they allow fast search of very large databases of genetic sequences.

There exist various specialized LCS algorithms which achieve faster runtime or lower memory usage than general algorithms. One class of specialized algorithms provides improved runtime when both input sequences have significant similarities (edit distance not too large). This is generally the case when comparing two versions of a text document. For this reason algorithms of this class are nearly universally used by tools similar to ``diff''. However such algorithms are generally sequential.

We adapt the sequential LCS algorithm of Myers \cite{myers_anond_1986}, which is by far the most commonly used LCS algorithm for similar sequences, for parallel execution on a distributed memory system.

\mypar{Related work} Bergroth et al. prepared a survey \cite{bergroth_survey_2000} of various sequential LCS algorithms. They group the algorithms into three categories: ``row-by-row methods'', ``contour methods'' and ``diagonal methods''. The classic dynamic programming algorithm for LCS \cite{wagner_string--string_1974} is classified as a row-by-row method. Myers' algorithm \cite{myers_anond_1986} belongs to the class of diagonal methods. Diagonal methods are designed to be efficient when both input sequences are similar.

Most parallel algorithms for the LCS problem are based on row-by-row sequential algorithms. Lu and Lin \cite{mi_lu_parallel_1994} presented a parallel algorithm of this class, achieving optimal complexity with a shared-memory machine model. Yang et al. \cite{jiaoyun_efficient_2010} designed a parallel LCS algorithm for GPUs (graphics processing units) by restructuring the classic DP (dynamic programming) table to reduce data dependencies.\linebreak These algorithms are fundamentally based on row-by-row methods, which are less efficient than diagonal methods for similar input sequences.

Allison and Dix \cite{allison_bit-string_1986} present a ``bit-parallel'' LCS algorithm (row-by-row method). Assuming a sufficiently small alphabet, their algorithm allows calculating multiple elements of the DP table in parallel, for example by using vector instructions. Their bit-parallel technique can be applied to other algorithms. We do not incorporate it into our algorithm, but do use vectorization within each DP cell.

Since Myers' algorithm \cite{myers_anond_1986} is an extremely popular LCS algorithm, we attempted to parallelize it. As far as we are aware, there are no published parallel versions of an LCS algorithm from the ``diagonal methods'' class, which Myers' algorithm belongs to.


\section{Numerical Experiment}
\label{sec:4}
In this section, we apply various methods to solve the LP form of optimal transport \ref{lp} and the entropic regularization of optimal transport \ref{equ:kl}. All of the numerical experiments are conducted on an Intel Core i7-6500U Processor with 2 cores and 4 threads.

\subsection{Datasets of Optimal Transport}
We perform the numerical experiments on various datasets, including randomly generated data, DOTmark\cite{DOTmark}, ellipse example \cite{multiscale}, Caffarelli's example \cite{multiscale} and Gaussian mixture model.

\subsubsection{Randomly generated data}
In order to make the feasible set not empty, we first generate a random matrix $R\in \mbR^{m\times n}$, where $R_{i,j}\sim \mcN(0,1)\, i.i.d.$. and then generate $\mu_i, \nu_j$ by
$$
\mu_i=\frac{\sum_{j=1}^n|R_{i,j}|}{\sum_{i=1}^m\sum_{j=1}^n|R_{i,j}|}, \, \forall i=1, 2, \dots m\\
$$
$$
\nu_j=\frac{\sum_{i=1}^m|R_{i,j}|}{\sum_{i=1}^m\sum_{j=1}^n|R_{i,j}|}, \, \forall j=1, 2, \dots n\\
$$
It is easy to verify that $\sum_{i=1}^m\mu_i=\sum_{j=1}^n\nu_j=1$. We then generate $C_{i,j} \sim \mcN(0,1)\, i.i.d.$. We denote $C_{\mathrm{min}}=\min_{i,j}C_{i,j}$ and let $C_{i,j}=C_{i,j}-C_{\mathrm{min}}$ to ensure $C_{i,j}\geq 0$.

\subsubsection{DOTmark}
DOTmark \cite{DOTmark} provides $10$ classes of $10$ different images, each of which is available at the different resolutions from $32\times32$ to $512\times512$. Suppose we select two images $x$ and $y$ with same resolution $l\times l$ from one class. The corresponding parameter for the standard form of LP is given in the following way: $m=n=l^2$, $\mu_i=x_i$, $\nu_j=y_j$ and $C_{i,j}=\|x_i-y_j\|_p$, where $(x_i)_{1\leq i\leq m}$ and $(y_j)_{1\leq j\leq n}$ form a regular square grid in $\mbR^2$. In practice, we take $p=2$. In solving this LP, we actually calculate the Wasserstein distance between $x$ and $y$.

\subsubsection{Ellipse Example}
The ellipse example consists of two uniform samples, source and target data set, of size $n$ from the unit circle with normal distributed noise added with zero mean and deviation 0.1. Slightly different from that in the paper \cite{multiscale}, we construct the source data example by scaling in the x-Axis by 2.0 and in the y-Axis by 0.5. The target data sample is then scaled in the x-Axis by 0.5 and y-Axis by 2.0. The corresponding parameter for the standard form of LP is as follows. $m = n$, $\mu_i=1, \nu_i=1(i=1, ..., n)$ and $C_{i, j}=\|x_i-y_j\|_2$, which is the Euclidean distance between two points $x_i$ and $y_j$. Figure \ref{ellipse} gives an example.
\begin{figure}[h]
\centering
\includegraphics[width=12cm]{final/ellipse1.png}
\caption{An Ellipse Example with sample size $n=50$. The darker the line, the greater the $\pi_{i, j}$.}
\label{ellipse}
\end{figure}

\subsubsection{Caffarelli's Example}
This example, mentioned in \cite{multiscale}, consists of two uniform samples on $[-1, 1]^2$ of size $n$. Then any point outside the unit circle are then discarded. The source and target data sample is split along the x-Axis at 0 and shifted by -2 and +2, respectively. Note that the true number of points may be less than $n$. When $n$ is large, about $\frac{\pi}{4}n$ points are preserved. For ease of notation, we still assume there are $n$ points. The corresponding parameter for the standard form of LP is as follows. $m = n$, $\mu_i=1, \nu_i=1(i=1, ..., n)$ and $C_{i, j}=\|x_i-y_j\|_2$, which is the Euclidean distance between two points $x_i$ and $y_j$. Figure \ref{caff} gives an example.
\begin{figure}[h]
\centering
\includegraphics[width=12cm]{final/caff1.png}
\caption{An Caffarelli's Example with sample size $n=50$. The darker the line, the greater the $\pi_{i, j}$.}
\label{caff}
\end{figure}

\subsubsection{Gaussian Mixture Model}
Suppose $p_\mu$ and $p_\nu$ are the probability density function of two different Gaussian mixture models and $N$ is the number of discrete points. In this example, $\mu$ and $\nu$ are the normalized discrete distribution of Gaussian mixture models, which satisfy
$$
\mu_i=\frac{p_\mu(\frac{i-1}{N-1})}{\sum_{i=1}^Np_\mu(\frac{i-1}{N-1})}, \quad \nu_j=\frac{p_\nu(\frac{j-1}{N-1})}{\sum_{j=1}^Np_\nu(\frac{j-1}{N-1})}
$$
And $C_{i,j}=|i-j|$.

\subsection{Different methods on Mosek and Gurobi}
`prim', `dual', `int' represent primal simplex, dual simplex and interior point method respectively. `(M)' means Mosek, `(G)' means Gurobi. For each method, we record the time and the number of iteration it takes and evaluate their performance. We examine the accuracy of the solution $\pi$ by the value of the objective function ('objval')
$$
\sum_{i=1}^m\sum_{j=1}^nC_{i,j}\pi_{i,j}
$$
and the violation of the constraints ('vltcst')
$$
\sum_{i=1}^m|\mu_i - \sum_{j=1}^{n}\pi_{i, j}| + \sum_{j=1}^n|\nu_j-\sum_{i=1}^m\pi_{i, j}|$$
In the table, we only give the exact value of `objval' for prim(M). In `objval' of other methods, we give its relative difference to prim(M). Namely, if `objval' for prim(M) is $f_1$ and `objval' for prim(G) is $f_2$, in `objval' of prim(G), we give $\frac{f_1-f_2}{f_1}$.

\subsubsection{Randomly generated data}
For simplicity, we only consider the case when $m=n$. We take $m=128,256,512,1024$.

\input{final/RGD_mg}

From Table \ref{RGD_mg}, we can see on Mosek, dual simplex method and interior point method take much more time than primal simplex does, especially when $m, n$ are large. But on Gurobi, dual simplex method is the fastest method and the time primal simplex method takes is close to the time interior point takes. Interior point method takes much smaller number of iterations than  We can observe that compared to other methods, primal simplex method on Mosek gets the largest value of objective function and the largest value of the violation of constraints. On Mosek, dual simplex methods achieves the lowest value of `vltcst', which partly explains why it takes the longest time. On Gurobi, the solution from all methods satisfies constraints perfectly.

\subsubsection{DOTmark}
We then test the performance of Mosek and Gurobi on DOTmark \cite{DOTmark}. We number the classes of DOTmark with the order of alphabet in the following way.
\begin{table}[!htp]
\centering
\begin{tabular}{|c|c|c|c|c|}
\hline
CauchyDensity&ClassicImages&GRFmoderate&GRFrough&GRFsmooth\\\hline
1&2&3&4&5\\\hline
LogGRF&LogitGRF&MicroscopyImages&Shapes&WhiteNoise\\\hline
6&7&8&9&10\\\hline
\end{tabular}
\end{table}

We test on pictures with resolution $32\times32$.
\input{final/DOTmark_mg}

From Table \ref{DOTmark_mg}, we observe that dual simplex method on Mosek takes much longer time than other methods. The cpu-time of primal simplex method on Mosek and primal simplex method and dual simplex method on Gurobi are close. On Mosek and Gurobi, the iteration number vary with different classes of DOTmark. Compared to other methods, primal simplex method on Mosek gets the largest `vltcst' and the largest `vltcst'. On Mosek, dual simplex methods achieves the smallest `vltcst'. On Gurobi , the solution $\pi$ from all methods satisfies the constraints perfectly.

\subsubsection{Ellipse Example \& Caffarelli's Example}
Similar to randomly generated data, we only consider the case when $m=n$. We take $m=128,256,512,1024$.
\input{final/ellipse_mg}
\input{final/caff_mg}

Different from the results on randomly generated data, Table \ref{ellipse_mg} shows that on Ellipse Example, dual simplex method takes much longer time and more iterations than primal simplex method, no matter on Mosek or on Gurobi. Similar to Ellipse Example, Table \ref{caff_mg} shows that dual simplex method takes much longer time than the primal simplex method, no matter on Mosek or on Gurobi. The iteration number of primal simplex method and dual simplex method are close. Similarly, compared to other methods, primal simplex method on Mosek gets the largest `vltcst' and the largest `vltcst'. On Mosek, dual simplex methods achieves the smallest `vltcst'. On Gurobi , the solution $\pi$ from all methods satisfies the constraints perfectly.

\subsection{First order methods}
`prim(M)' means primal method of Mosek. `ADMM-p' means ADMM primal method \ref{alg:ADMM}. `ADMM-d' means ADMM dual method \ref{alg:ADMM-dual}. `ADMM-s' means another ADMM splitting method for primal problem \ref{alg:ADMM-split}. BADMM means Bregman ADMM \ref{alg:BADMM}.
\subsubsection{Art of Tuning Parameters}
The success of ADMM always lies in an appropriate choice of the coefficient $t$ in the quadratic term of the augmented Lagrangian function. Thanks to the normalization of constraints, i.e., letting $\sum_{i=1}^m\mu_i = \sum_{j=1}^n\nu_j = 1$, the tuning of parameters is relatively easier. Intuitively, the value of $t$ should be dependent on the problem size, $m$ and $n$, and the objective coefficients $C$. In our implementation, for primal problem, we let $t$ to be proportional to $(m+n)\bar C$, where $\bar C$ is the mean of the coefficient matrix $C$ ($\frac{1}{mn}\sum_{i, j}C_{i, j}$). For dual problem, we set $t$ to be inversely proportional to $(m+n)\bar C$. For the Bregman ADMM, we set $t$ to be proportional to $\bar C$. The detailed proportion will be clear in the following. Empirically, we found that such strategy performs well in our experiments.

\subsubsection{Randomly Generated Data}
\input{final/RGD_fo}
In Table \ref{RGD_fo}, we present the numerical results on random generated data.
Detailed tuning parameters is listed in Table \ref{param_rgd_fo}.
\begin{table}[h]
\centering
\caption{Tuned Parameters: Random Generated Data}
\label{param_rgd_fo}
\begin{tabular}{|c|c|c|}
\hline
Solver & $t$ & Stopping Rule \\
\hline
ADMM-p & $5(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 5e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
ADMM-d & $1/(8(m+n)\bar C)$ & $\text{`vltcst'}\leqslant$ 2e-04 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
ADMM-s & $2(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 1e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
BADMM & $10\bar C$ & $\|\pi-\tilde\pi\|_1\leqslant$ 1e-06 or $\text{`iter'}\geqslant$ 2e+04 \\
\hline
\end{tabular}
\end{table}

In this setting, both `ADMM-p' and `ADMM-s' achieves relatively better results. They both satisfy the constraints quite well and achieve lower objective value. `ADMM-p' spends moderate time in all problems because all computation has closed form. `ADMM-s' performs quite well in `objval' and `vltcst', especially when the problem size is larger. However, such accuracy is at the cost of much longer time. In fact, from our observation, much of the time by `ADMM-s' is spent on the projection on the simplex(constraints). There exists more efficient algorithms for projection, but since time is limited, we do not implement them. The projection is quite essential in `ADMM-s' because it imposes $\pi$ to satisfy constraints, and this may explain why the constraints are easy to satisfy within tolerance.

It's worth noting that `ADMM-d' has weakness in satisfying constraints. This is because in Section 2.1.2 we regard the ``dual of dual'' as an approximation of initial variables, which is numerically unstable. In our experiments, we found that `ADMM-d' converges faster at the beginning to the optimal value, but then continuously vibrates around optimal value for a long time. `BADMM' appears to perform worse than other three algorithms. The selection of $t$ is inconsistent with other ADMM due to the property of KL divergence. If $t$ is selected to be too big, then the exponential term in \ref{badmm_pipit_upd} may be so close to 1 that leads to bad precision. Furthermore, in our experiments, we found that when we increase $t$, `BADMM' indeed has slightly better performance in `vltcst' but deteriorates rapidly in `objval'. Our parameter is a balance between the two targets.

\subsubsection{DOTmark}
Some detailed tuning parameters is listed in Table \ref{param_DOTmark_fo}.
\begin{table}[htbp]
\centering
\caption{Tuned Parameters: DOTmark}
\label{param_DOTmark_fo}
\begin{tabular}{|c|c|c|}
\hline
Solver & $t$ & Stopping Rule \\
\hline
ADMM-p & $5(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 5e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
ADMM-d & $1/(8(m+n)\bar C)$ & $\text{`vltcst'}\leqslant$ 2e-04 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
ADMM-s & $2(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 4e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
BADMM & $10\bar C$ & $\|\pi-\tilde\pi\|_1\leqslant$ 4e-06 or $\text{`iter'}\geqslant$ 2e+04 \\
\hline
\end{tabular}
\end{table}

\input{final/DOTmark_fo}
Table \ref{DOTmark_fo} presents the numerical results on DOTmark.

As to constraints, all methods perform similar to that in the setting of random generated data, since our stopping rule is built on `vltcst' or the norm of difference between $\pi$ and $\tilde\pi$.

As to objective value, the three traditional ADMM methods are not stable on `objval' and tend to achieve higher value than Mosek. This may be because of the different structure between random generated data and DOTmark. `BADMM' tends to achieve lower objective value. However, the results are not stable and the constraints are not satisfied well.

As to iteration numbers and time spent, surprisingly, `ADMM-p' and `ADMM-s' need much less iterations than others. This phenomenon implies that we could tighten our stopping rule by decreasing the thresholds. Due to the time limit, further experiments could be conducted in the future.

\subsubsection{Ellipse Example \& Caffarelli's Example}
The detailed parameters is listed in Table \ref{param_ellcaff_fo}.
\begin{table}[htbp]
\centering
\caption{Tuning Parameters: Ellipse \& Caffarelli}
\label{param_ellcaff_fo}
\begin{tabular}{|c|c|c|}
\hline
Solver & $t$ & Stopping Rule \\
\hline
ADMM-p & $5(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 5e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
ADMM-d & $1/(8(m+n)\bar C)$ & $\text{`vltcst'}\leqslant$ 2e-04 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
ADMM-s & $2(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 2e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
BADMM & $10\bar C$ & $\|\pi-\tilde\pi\|_1\leqslant$ 4e-06 or $\text{`iter'}\geqslant$ 2e+04 \\
\hline
\end{tabular}
\end{table}

\input{final/ellipse_fo}
\input{final/caff_fo}

Table \ref{ellipse_fo} and \ref{caff_fo} presents numerical results for Ellipse Example and Caffarelli's Example, respectively.

The constraints are generally satisfied before we stop iterations. The solution from `BADMM' does not satisfy the constraints very well. However, the optimal value is not within our expectation. The vibration of objective values around optimal values are observed in the three traditional ADMM methods. Among these algorithms, `ADMM-s' performs relatively better on `iter', `objval', and `vltcst', though it takes more time in each iteration. Nevertheless, `objval' of `BADMM' is relatively low.

\subsection{Algorithms for entropic regularized OT}
`sinkhorn' represents the Sinkhorn's algorithm with numerical stability and continuation strategy \ref{alg:sansc}. `ADMM' means the algorithm described in \ref{alg:ADMM-ER}. Addition to `time', `iter', `objval' and `vltcst', we use `entval' to denote the objective value with extropy regularization term. The values in `objval' are, same as before, compared to `prim(M)'.

The parameters and stopping criterions of our algorithms are given in Table \ref{param_er}. In 'sinkhorn', we apply continuation strategy. If the coefficient of the regularized term is $\epsilon$, we select $10^i\epsilon(i=4, 3, 2, 1, 0)$ as the sequence and run 4000 iterations in optimizing every subproblem.

\begin{table}[htbp]
\centering
\caption{Tuned Parameters: algorithms for entropic regularized OT}
\label{param_er}
\begin{tabular}{|c|c|c|}
\hline
method & $t$ & Stopping Rule \\
\hline
ADMM & $5(m+n)\bar C$ & $\text{`vltcst'}\leqslant$ 1e-07 or $\text{`iter'}\geqslant$ 2e+04 \\\hline
sinkhorn & / & $\text{`iter'}\geqslant$ 2e+04 \\
\hline
\end{tabular}
\end{table}

\subsubsection{Gaussian Mixture Model: The effect of continuation strategy}

We consider to test Sinkhorn's algorithm with numerical stability on Gaussian mixture model and test the effect of continuation strategy. We first take $N=128$ as a first exmaple. Figure \ref{gmm3} gives the probability density function of two Gaussian mixture models.
\begin{figure}[htbp]
\centering
\includegraphics[width=12cm]{final/gmm3.jpg}
\caption{The probability distribution function of two Gaussian mixture models $G_1$ and $G_2$. $G_1$ (blue one) has mean $[0.2,0.5]$, variance $[0.05^2, 0.03^2]$ and component proportion $[0.6, 0.4]$; $G_2$ (red one) has mean $[0.6,0.7]$, variance $[0.03^2, 0.05^2]$ and component proportion $[0.3, 0.7]$. }
\label{gmm3}
\end{figure}

We then take $\epsilon=10^{-2},10^{-3},10^{-4},10^{-5}$. If we use the vanilla Sinkhorn's algorithm \ref{alg:sa} for $\epsilon=10^{-4},10^{-5}$, $K^{\epsilon}$ would be a zero matrix. For Sinkhorn's algorithm with numerical stability \ref{alg:sans}, we take the maximum iteration number to be $10,100,1000,4000$ respectively. For Sinkhorn's algorithm with numerical stability and continuation strategy \ref{alg:sansc}, we take $\alpha=0.1$, $\epsilon_0=\epsilon\times 10^5$ and the maximum iteration number to be $2,20,200,800$ respectively.

\begin{figure}[!htb]
\centering
\includegraphics[width=12cm]{final/gmm4.jpg}
	\caption{Impact of $\epsilon$ on the coupling between two gaussian mixture models \ref{gmm1}. From left to right, $\epsilon=10^{-2},10^{-3},10^{-4},10^{-5}$. The upper row is without continuation strategy and the lower row is with continuatation strategy. The purple one is the solution to the entropic regularized LP. The green one is the solution to the original LP.}
\label{gmm4}
\end{figure}

From Figure \ref{gmm4}, we can find that with same iteration number, Sinkhorn's algorithm with numerical stability and continuation strategy converges faster than the one only with numerical stability. This shows that with proper choice of the parameter of continuation, we can achieve acceleration in Sinkhorn's algorithm with continuation strategy.

We then perform a comprehensive numerical experiment on Gaussian mixture model. Detailed parameters and the stopping criterions are listed in Table \ref{param_er}.

\input{final/gmm_er}
Table \ref{gmm_er} gives the comprehensive numerical results for Gaussian Mixture Model. The setting is same as Figure \ref{gmm1}, except that we test examples for different problem sizes, i.e., we let $N=128, 256, 512, 1024$, respectively. We observe that, in most cases, although `sinkhorn' takes much longer time and more iterations than `ADMM', `sinkhorn' outperforms `ADMM' in both `objval' and `vltcst'.

\subsubsection{Randomly generated data}
The parameters and the stopping criterions are listed in Table \ref{param_er}. Table \ref{RGD_er} presents numerical results on random generated data.

\input{final/RGD_er}

We find that the `objval' and `vltcst' of `sinkhorn' is getting worse with the decrease of $\epsilon$. This can be explained by the computation complexity of Sinkhorn's algorithm. As shown in Section 3.2, the $\tau$-approximate solution of the unregularized OT problem takes $O(n^2\log(n)\tau^{-3})$ operations where $\epsilon=\tau/\log(n)$. This means that if we decrease the $\epsilon$ by $0.1$, theoretically we would need $10^3$ times iteration to get a precise solution. Although our improvements of Sinkhorn's algorithm makes it adapt to small $\epsilon$, but the computation is costly. If we want to use Sinkhorn's algorithm to get an approximation solution of the orignial problem \ref{lp}, we should pay attention to choose an appropriate $\epsilon$.

On the contrary, `ADMM' appears robustness in satisfying constraints within acceptable iteration number. `vltcst' of all test samples are below 1e-06, though it's not comparable to `sinkhorn'. `objval' of ADMM appears to be gradually lower than `prim(M)' when $\epsilon$ decreases. It's worth noting that in `ADMM' we do not apply continuation strategy. In fact, we observe that with the carefully selected $t$, `ADMM' converges quite fast to the optimal value at the beginning, whatever $\epsilon$ is, although vibration is unavoidable in the latter steps. Of course, for $\epsilon$ not large and small or moderate problem sizes, `sinkhorn' performs better than `ADMM'.

\subsubsection{DOTmark}
Detailed parameters are listed in \ref{param_er}. (The stopping rule for `ADMM' changes to $\text{`vltcst'}\leqslant$ 1e-07 or $\text{`iter'}\geqslant$ 2e+04.) Table \ref{DOTmark_er1} and \ref{DOTmark_er2} present numerical results on DOTmark.
\input{final/DOTmark_er1}
\input{final/DOTmark_er2}

Same as before, `objval' and `vltcst' of `sinkhorn' is getting worse with the decrease of $\epsilon$. All of the results are within 1e-09 in `vltcst', but `objval' deteriorate rapidly when $\epsilon$ decreases. This can be again explained by the computation complexity of Sinkhorn's algorithm.

'ADMM' can achieve relatively lower objective value on average, and need moderate iterations before `vltcst' arrives at a threshold. The performance is similar to random generated data setting with larger problem sizes. We should note that due to the different problem structures, it seems difficult sometimes for `ADMM' to outperform `prim(M)'.

\subsubsection{Ellipse Example \& Caffarelli's Example}
\input{final/ellipse_er}
\input{final/caff_er}
Table \ref{param_er} lists the detailed parameters.Table \ref{ellipse_er} and Table \ref{caff_er} present numerical results for Ellipse Example and Caffarelli's Example, respectively. Through our previous experiments, we observe that with the decrease of $\epsilon$, `ADMM' does not require so much increase of computational complexity as `sinkhorn'. This can be explained by the limit when $\epsilon\to0$. When $\epsilon\to0$, `ADMM' reduce to ADMM for the primal problem \ref{lp}. But with $\epsilon\to0$, we have shown in Section 3.3.1 that `sinkhorn' will diverge due to the loss of strong convexity. And with the decrease of $\epsilon$, the strong convexity of problem \ref{equ:kl} decreases. This also explains why `sinkhorn' requires such enormous iterations to find the optimal solution. In summary, we shall use `sinkhorn' to solve the entropic regularization of OT \ref{equ:kl} with moderate $\epsilon$. With large $\epsilon$, we shall use the original Sinkhorn's algorithm \ref{alg:sa} because it is faster in computation. `ADMM' seems to be suitable to arbitrary small $\epsilon$.


% This LaTeX was auto-generated from an M-file by MATLAB.
% To make changes, update the M-file and republish this document.

%%% \documentclass{article}
%%% \usepackage{graphicx}
%%% \usepackage{color}

%%% \sloppy
%%% \definecolor{lightgray}{gray}{0.5}
\setlength{\parindent}{0pt}

%%% \begin{document}

\subsection*{Four parameter sine wave fitting}

\begin{par}
Example for algorithm ThreePSF.
\end{par} \vspace{1em}
\begin{par}
ThreePSF is an algorithm for estimating the amplitude, phase and offset of the sine waveform according standard IEEE Std 1241-2000';
\end{par} \vspace{1em}

\subsubsection*{Contents}

\begin{itemize}
\setlength{\itemsep}{-1ex}
   \item Generate sample data
   \item Call algorithm
   \item Display results
\end{itemize}

\subsubsection*{Generate sample data}

\begin{par}
Two quantities are prepared: \lstinline{t} and \lstinline{y}, representing 1 second of sinus waveform of nominal frequency 1 kHz, nominal amplitude 1 V, nominal phase 1 rad and offset 1 V sampled at sampling frequency 10 kHz.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI = [];
Anom = 2; fnom = 100; phnom = 1; Onom = 0.2;
t = [0:1/1e4:1-1/1e4];
DI.y.v = Anom*sin(2*pi*fnom*t + phnom) + Onom;
DI.Ts.v = 1e-4;
DI.f.v = fnom;
\end{lstlisting}

\subsubsection*{Call algorithm}

\begin{par}
Use QWTB to apply algorithm \lstinline{ThreePSF} to data \lstinline{DI}.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
CS.verbose = 1;
DO = qwtb('ThreePSF', DI, CS);
\end{lstlisting}

        \begin{lstlisting}[style=output]
QWTB: no uncertainty calculation
\end{lstlisting} \color{black}

\subsubsection*{Display results}

\begin{par}
Results is the amplitude, phase and offset of sampled waveform.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
A = DO.A.v
ph = DO.ph.v
O = DO.O.v
\end{lstlisting}

        \begin{lstlisting}[style=output]

A =

    2.0000

ph =

    1.0000

O =

    0.2000

\end{lstlisting} \color{black}
    \begin{par}
Errors of estimation in parts per milion:
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
Aerrppm = (DO.A.v - Anom)/Anom .* 1e6
pherrppm = (DO.ph.v - phnom)/phnom .* 1e6
Oerrppm = (DO.O.v - Onom)/Onom .* 1e6
\end{lstlisting}

        \begin{lstlisting}[style=output]

Aerrppm =

  -8.4377e-09

pherrppm =

  -9.7700e-09

Oerrppm =

   8.3267e-10

\end{lstlisting} \color{black}

%%% \end{document}


% !Mode:: "TeX:UTF-8"
% !TEX program = xelatex
\title{Project 5: Solving the FDMs for Two-asset Options}
\author{Iydon Liang}
\date{\today}

\maketitle\tableofcontents\clearpage

\begin{abstract}
    It's clear that how to solve the FDM for one-asset option, either European, American or Path-dependent types. In this report, we discussed the two-asset options case. And gave three methods to determine the price for a two-asset option.
\end{abstract}

\section{Two-asset Option}
\[
    \dd S_i(t) = S_i(t)\alpha_i\dd t + S_i(t)\sigma_i\dd W_i(t)
\]
where $\dd W_i(t)$ $(i=1,\ldots,n)$ are standard Brownian motion which satisfy
\[
    \begin{aligned}
        \EE(\dd W_i) &= 0 \\
        \var(\dd W_i) &= \dd t \\
        \cov(\dd W_i, \dd W_j) &= \delta_{ij} p_{ij} \dd t.
    \end{aligned}
\]

\subsection{Pricing by Monte Carlo Method}
We know that price of the two stocks follows a bi-variate log-normal distribution. And we can generate it from a bi-variate normal distribution with $\mu=0$ and
\[
    \Sigma = \begin{pmatrix}
        1 & \rho \\
        \rho & 1
    \end{pmatrix}.
\]
Then we can sample $(S_1(T), S_2(T))$ from the distribution, and pricing by Monte Carlo method is possible by so.

\section{FDMs for Two-asset Option}
\[
    \frac{\partial V}{\partial t} + rS_1\frac{\partial V}{\partial S_1} + \frac{1}{2}\sigma_1^2S_1^2\frac{\partial^2 V}{\partial S_1^2} + rS_2\frac{\partial V}{\partial S_2} + \frac{1}{2}\sigma_2^2S_2^2\frac{\partial^2 V}{\partial S_2^2} - rV = 0
\]

\subsection{Approximate the Differential Quotients}
Suppose $f(i, j, k) = V(S_1=i\Delta S, S_2=j\Delta S, t=k\Delta k)$,
\[
    \begin{aligned}
        f_t &\approx \frac{f(i, j, k+1) - f(i, j, k)}{\Delta t} \\
        f_{S_1} &\approx \frac{f(i+1, j, k) - f(i-1, j, k)}{2\Delta t} \\
        f_{S_1S_2} &\approx \frac{f(i+1, j, k) - 2f(i, j, k) + f(i-1, j, k)}{\Delta^2S} \\
        f_{S_2} &\approx \frac{f(i, j+1, k) - f(i, j-1, k)}{2\Delta t} \\
        f_{S_1S_2} &\approx \frac{f(i, j+1, k) - 2f(i, j, k) + f(i, j-1, k)}{\Delta^2S}
    \end{aligned}
\]

\subsection{Recursion Form}
Substitute the approximations in to the BS equation, and put $f(\cdot, \cdot, k+1)$ to the left side, put $f(\cdot, \cdot, k)$ to the right hand side. Then we get
\[
    \begin{aligned}
        f(i, j, k+1) =& \quad\alpha_i f(i-1, j, k) \\
        &+ \beta_i f(i+1, j, k) \\
        &+ \gamma_j f(i, j-1, k) \\
        &+ \theta_j f(i, j+1, k) \\
        &+ \lambda_{1i} f(i, j, k) \\
        &+ \lambda_{2j} f(i, j, k)
    \end{aligned}
\]

Here we divide the coefficient of $f(i, j, k)$ into two parts for the convenience of following steps.
\[
    \begin{aligned}
        \alpha_i &= \frac{1}{2}\Delta t(ri-\sigma_1^2i^2) \\
        \beta_i &= -\frac{1}{2}\Delta t(ri+\sigma_1^2i^2) \\
        \gamma_j &= \frac{1}{2}\Delta t(rj-\sigma_2^2j^2) \\
        \theta_j &= -\frac{1}{2}\Delta t(ri-\sigma_2^2j^2) \\
        \lambda_{1i} &= \frac{1}{2} + \Delta t(\frac{1}{2}r+\sigma_1^2i^2) \\
        \lambda_{2j} &= \frac{1}{2} + \Delta t(\frac{1}{2}r+\sigma_2^2j^2) \\
    \end{aligned}
\]

\subsection{Matrix Form}
The relationship can be written as a matrix equation
\[
    AX + XB = C
\]
where matrix $C(i, j) = f(i, j, k+1)$, $X(i, j) = f(i, j, k)$

\begin{figure}[H]
    \centering
    \includegraphics[width=.7\textwidth]{figures/2019-12-25-matrix-form.png}
\end{figure}

\subsection{Solve the Matrix Equation}
Matrix equation of the form $AX + XB = C$ is called \textbf{Sylvester equation}. The equation has a unique solution when the eigenvalues of $A$ and $−B$ are distinct. In \texttt{MATLAB}, there is a function \textbf{sylvester(A,B,C)}, which can solve the Sylvester equation.