\documentclass{article}
\usepackage{geometry}
\usepackage{amsmath, amsfonts, amsthm, amssymb}

\title{Exercises and Solutions for \textbf{An Introduction to Stochastic Differential Equations}}
\author{Zehao Dou}
\date{September 2021}

\begin{document}
\allowdisplaybreaks[4]
\maketitle

\newpage
\paragraph{Problem 1} Show, using the formal manipulations for Ito's chain rule, that
\[Y(t) := e^{W(t)-\frac{t}{2}}\]
solves the stochastic differential equation
\begin{equation*}
\begin{cases}
dY &= YdW\\
Y(0) &= 1.
\end{cases}
\end{equation*}
(Hint: If $X(t):=W(t)-\frac{t}{2}$, then $dX=-\frac{dt}{2}+dW$.)

\paragraph{Solution of Problem 1}
According to the Ito's chain rule: since function $X(t):=W(t)-\frac{t}{2}$ satisfies the equation $X(t)=-\frac{dt}{2}+dW$, for $Y(t)=\exp(X(t))$, it holds that:
\[dY=\left(e^{X}\cdot\left(-\frac{1}{2}\right)+\frac12\cdot e^{X}\right)dt+e^{X}dW= 0+YdW=YdW.\]
Also, it's easy to verify that $Y(0)=1$. Therefore, the solution
\[Y(t) := e^{W(t)-\frac{t}{2}}\]
actually solves the SDE we want.

\paragraph{Problem 2} Show that,
\[S(t)=s_0e^{\sigma W(t)+\left(\mu-\frac{\sigma^2}{2}\right)t}\]
solves
\begin{equation*}
\begin{cases}
dS &= \mu Sdt + \sigma SdW\\
S(0) &= s_0.
\end{cases}
\end{equation*}

\paragraph{Solution of Problem 2}
Again we use the Ito's chain rule: for function $X(t):= W(t)+\frac{\mu-\sigma^2/2}{\sigma}t$, we have:
\[dX= dW + \frac{\mu-\sigma^2/2}{\sigma} dt.\]
Therefore, for function $S(t)=s_0 e^{\sigma\cdot X(t)}$, it holds that:
\begin{equation*}
\begin{aligned}
dS &= \left(\sigma s_0 e^{\sigma X(t)}\cdot \frac{\mu-\sigma^2/2}{\sigma} +\frac12\cdot \sigma^2 s_0 e^{\sigma X(t)}\right)dt + \sigma s_0 e^{\sigma X(t)}dW\\
&= \mu s_0 e^{\sigma X(t)}dt + \sigma s_0 e^{\sigma X(t)}dW = \mu Sdt+\sigma SdW.
\end{aligned}
\end{equation*}
Also, it's easy to verify that $S(0)=s_0$, which comes to our conclusion.
\paragraph{Problem 3} (1) Let $(\Omega, \mu, P)$ be a probability space and let $A_1\subseteq A_2\subseteq\ldots\subseteq A_n\subseteq\ldots$ be events. Show that
\[P\left(\bigcup_{n=1}^{\infty}A_n\right)=\lim_{m\rightarrow \infty} P(A_m).\]
(Hint: Look at the disjoint events $B_n :=A_{n+1}-A_n$.)\\
~\\
(2) Likewise, show that if $A_1\supseteq A_2 \supseteq\ldots\supseteq A_n\supset\ldots$, then
\[P\left(\bigcap_{n=1}^{\infty}A_n\right)=\lim_{m\rightarrow\infty} P(A_m).\]

\paragraph{Solution of Problem 3}~\\
(1) Consider the disjoint events $B_n=A_{n+1}-A_n$, then we know that these events are disjoint since for $\forall m<n$, it holds that:
\[B_m=A_{m+1}-A_m\subseteq A_{m+1}\subseteq A_n, ~A_{n+1}-A_n =B_n\cap A_n = \varnothing,\]
so we can conclude that $B_m\cap B_n=\varnothing$, which means $\{B_n\}$ are disjoint events. Also, we have $\bigcup_{n=1}^{\infty}A_n=\bigcup_{n=0}^{\infty}B_n$. It is because for $\forall x\in \bigcup_{n=1}^{\infty}A_n$, there exists a smallest $k$ such that $x\in A_k\Rightarrow x\in A_k-A_{k-1}=B_k\subseteq\bigcup_{n=0}^{\infty}B_n$, which leads to
\[\bigcup_{n=1}^{\infty}A_n\subseteq\bigcup_{n=0}^{\infty}B_n.\]
On the other hand, for $\forall x\in \bigcup_{n=0}^{\infty}B_n$, there exists $k$ such that $x\in B_k\subseteq A_{k+1}\subseteq\bigcup_{n=1}^{\infty}A_n$, which leads to
\[\bigcup_{n=1}^{\infty}A_n\supseteq\bigcup_{n=0}^{\infty}B_n.\]
Therefore, we conclude that $\bigcup_{n=1}^{\infty}A_n=\bigcup_{n=0}^{\infty}B_n$. Also, notice that $A_m=B_0\cup B_1\cup\ldots B_{m-1}$, which leads to
\[P(A_m)=\sum_{n=0}^{m-1}P(B_m).\]
Here, we use the countable additivity of $P(\cdot)$ under disjoint events $\{B_n\}$. Finally, it holds that:
\begin{equation*}
\begin{aligned}
&P\left(\bigcup_{n=1}^{\infty}A_n\right) = P\left(\bigcup_{n=0}^{\infty}B_n\right) = \sum_{n=0}^{\infty}P(B_n) =\lim_{m\rightarrow\infty}\sum_{n=0}^{m-1}P(B_n) = \lim_{m\rightarrow\infty} P(A_m)
\end{aligned}
\end{equation*}
In the final step, we use the fact that the sequence $p_m := \sum_{n=0}^{m-1}P(B_n)=P(A_m)$ is non-decreasing and upper bounded by 1, and therefore it converges. \\
~\\
(2) Similarly, we have $A_1^c\subseteq A_2^c\subseteq\ldots\subseteq A_n^c\subseteq\ldots$. By using the conclusion of (1), we have:
\[P\left(\bigcap_{n=1}^{\infty}A_n\right)=1-P\left(\bigcup_{n=1}^{\infty}A_n^c\right)=1-\lim_{m\rightarrow\infty}P(A_m^c)=\lim_{m\rightarrow\infty}P(A_m),\]
which comes to our conclusion.

\paragraph{Problem 4} Let $\Omega$ be any set and $\mathcal A$ any collections of subsets of $\Omega$. Show that there exists a unique smallest $\sigma$-algebra $\mu$ of subsets of $\Omega$ containing $\mathcal A$. We call $\mu$ the $\sigma$-algebra generated by $\mathcal A$. \\
(Hint: Take the intersection of all the $\sigma$-algebras containing $\mathcal A$. )

\paragraph{Solution of Problem 4} Obviously, there exists a $\sigma$-algebra containing $\mathcal A$ (for example, $2^{\Omega}$ itself). Now, we take the intersection of all the $\sigma$-algebras containing $\mathcal A$, denoted by $\mu$. In order to prove that there exists a unique smallest $\sigma$-algebra $\mu$ of subsets of $\Omega$ containing $\mathcal A$, we only need to prove that the intersection of all $\sigma$-algebras is also a $\sigma$-algebra of $\Omega$ containing $\mathcal A$. On one hand, since all the $\sigma$-algebras considered by us contain $\mathcal A$, so their intersection $\mu$ also contains $\mathcal A$. On the other hand,
\begin{itemize}
\item For all $\sigma$-algebra $S$ that contains $\mathcal A$, we have $\varnothing \in S$ by the definition of $\sigma$-algebra. Therefore, the intersection holds $\varnothing\in \mu$ obviously.
\item For $\forall A\in\mu$, we have $A\in S$ for any $\sigma$-algebra $S$ that contains $\mathcal A$, so it holds that $A^c\in S$. Therefore, we have $A^c\in\mu$.
\item For $A_1, A_2, \ldots, \in\mu$, we know that for any $\sigma$-algebra $S$ that contains $\mathcal A$, $\bigcup_{i=1}^{+\infty} A_i\in S$. Therefore:
\[\bigcup_{i=1}^{+\infty} A_i\in \mu.\]
\end{itemize}
After combining all the items above, we know that $\mu$ is also a $\sigma$-algebra of $\Omega$ containing $\mathcal A$, which comes to our conclusion.

\paragraph{Problem 5} Show that if $A_1, A_2,\ldots, A_n$ are events, then
\begin{equation*}
\begin{aligned}
P\left(\bigcup_{i=1}^{n}A_i\right) &= \sum_{i=1}^{n} P(A_i) -\sum_{1\leqslant i<j\leqslant n}P(A_i\cap A_j)\\
&~~~+ \sum_{1\leqslant i<j<k\leqslant n} P(A_i\cap A_j\cap A_k)\\
&~~~-\ldots+(-1)^n P(A_1\cap A_2\cap\ldots\cap A_n).
\end{aligned}
\end{equation*}
(Hint: Do the case $n=2$ first and then the general case by induction.)

\paragraph{Solution of Problem 5} We use the method of induction. When $n=2$, we know that:
\[P(A\cup B)=P(A)+P((A\cup B)-A)=P(A)+P(B-(A\cap B))=P(A)+P(B)-P(A\cap B).\]
Suppose the condition holds for $n$, then for $n+1$, we have:
\begin{align}
&~~P(A_1\cup A_2\cup\ldots\cup A_{n+1}) = P((A_1\cup A_2\cup\ldots\cup A_n)\cup A_{n+1})\notag\\
&= P(A_1\cup A_2\cup\ldots\cup A_n)+P(A_{n+1})-P((A_1\cup A_2\cup\ldots\cup A_n)\cap A_{n+1})\notag\\
&= P(A_{n+1})+\sum_{i=1}^{n} P(A_i) -\sum_{1\leqslant i<j\leqslant n}P(A_i\cap A_j)+\ldots+(-1)^n P(A_1\cap A_2\cap\ldots\cap A_n)\notag\\
&~~-P\left((A_1\cap A_{n+1})\cup (A_2\cap A_{n+1})\cup\ldots\cup (A_n\cap A_{n+1})\right)\notag\\
&= P(A_{n+1})+\sum_{i=1}^{n} P(A_i) -\sum_{1\leqslant i<j\leqslant n}P(A_i\cap A_j)+\ldots+(-1)^n P(A_1\cap A_2\cap\ldots\cap A_n)\notag\\
&~~-\sum_{i=1}^{n} P(A_i\cap A_{n+1})+\sum_{1\leqslant i<j\leqslant n}P((A_i\cap A_{n+1})\cap(A_j\cap A_{n+1}))-\ldots\notag\\
&~~-(-1)^n P((A_1\cap A_{n+1})\cap (A_2\cap A_{n+1})\cap\ldots\cap (A_n\cap A_{n+1}))\notag\\
&= \sum_{i=1}^{n+1} P(A_i) -\sum_{1\leqslant i<j\leqslant n+1}P(A_i\cap A_j)+\ldots+(-1)^{n+1} P(A_1\cap A_2\cap\ldots\cap A_{n+1})\notag,
\end{align}
which completes the induction and comes to our conclusion.

\paragraph{Problem 6} Let $X=\sum_{i=1}^{k}a_i\chi_{A_i}$ be a simple random variable, where the real numbers $a_i$ are distinct, the events $A_i$ are pairwise disjoint, and $\Omega=\bigcup_{i=1}^{k}A_i$. Let $\mu(X)$ be the $\sigma$-algebra generated by $X$.\\
(1) Describe precisely which sets are in $\mu(X)$.\\
(2) Suppose the random variable $Y$ is $\mu(X)$-measurable. Show that $Y$ is constant on each set $A_i$.\\
(3) Show that therefore $Y$ can be written as a function of $X$.

\paragraph{Solution of Problem 6}~\\
(1) Since the real numbers $a_i$ are distinct, we can assume that
\[a_1<a_2<\ldots<a_k\]
without loss of generality. We are going to prove that $\mu(X)=\sigma(\{A_i:~i\in[k]\})$ which is the $\sigma$-algebra generated by the pairwise disjoint events $A_i$. On one hand, notice that: $\forall i\in[k]$,
\[A_i=\{X=a_i\}\in\mu(X)~\Rightarrow~\sigma(\{A_i:~i\in[k]\}\subseteq \mu(X).\]
On the other hand, for any $B\in\mathcal B(\mathbb{R})$, we have:
\[\{X\in B\}=\bigcup_{a_i\in B}A_i \in\sigma(\{A_i:~i\in[k]\}),\]
which leads to:
\[\mu(X)=\sigma\left(\{X\in B\}\right)\subseteq \sigma(\{A_i:~i\in[k]\}).\]
To sum up, $\mu(X)=\sigma(\{A_i:~i\in[k]\})$, which means all the sets in $\mu(X)$ are $\bigcup_{i\in S}A_i$ for some $S\subseteq [k]$.

~\\
(2) Random variable $Y$ is $\mu(X)$-measurable. Then, for any $B\in\mathcal B(\mathbb{R})$, it holds that event $\{Y\in B\}\in\mu(X)$. From the conclusion of (1), we know that
\[\mu(X)=\left\{\bigcup_{i\in S}A_i:~S\subseteq [k]\right\}.\]
Now we suppose that $Y$ is not constant on each set $A_i$. Without loss of generality, we assume $Y$ is not constant on $A_1$, then there exists $x,y\in A_1$ such that $Y(x)<Y(y)$. Consider the event $E = \{Y=Y(x)\}\in\mu(X)$, then $x\in E$ but $y\notin E$. Since $E$ must be the union of some $A_i$-s, from $x\in E$, we know that $A_1\subseteq E$. From $y\notin E$, we know that $A_1\cap E=\varnothing$, which contradict with each other. Therefore, $Y$ must be constant on each set $A_i$.

~\\
(3) According to the conclusion of (2), we can assume that $Y$ equals to constant $b_i$ on the set $A_i$. Then:
\[Y=\sum_{i=1}^{k}b_i\chi_{A_i}.\]
Therefore, $Y$ can be written as $f(X)$ where $f(a_i)=b_i$ for $\forall i\in[k]$, which comes to our conclusion.

\paragraph{Problem 7} Verify:
\[\int_{-\infty}^{\infty}e^{-x^2}dx=\sqrt{\pi},~~~\frac{1}{\sqrt{2\pi\sigma^2}}\int_{-\infty}^{\infty}xe^{-\frac{(x-m)^2}{2\sigma^2}}dx=m,\]
\[\frac{1}{\sqrt{2\pi\sigma^2}}\int_{-\infty}^{\infty}(x-m)^2e^{-\frac{(x-m)^2}{2\sigma^2}}dx=\sigma^2.\]

\paragraph{Solution of Problem 7} For the first conclusion, we consider the following equation: denote $S=\int_{-\infty}^{\infty}e^{-x^2}dx$, then:
\begin{equation*}
\begin{aligned}
S^2&=\int_{-\infty}^{\infty}e^{-x^2}dx \cdot\int_{-\infty}^{\infty}e^{-y^2}dy = \int_{\mathbb{R}^2}e^{-(x^2+y^2)}dxdy\\
& = \int_{0}^{+\infty}\int_{-\pi}^{\pi} e^{-r^2}r\cdot drd\theta = \pi\cdot \int_{0}^{+\infty}e^{-r^2}\cdot 2rdr = \pi\cdot \int_{0}^{+\infty}e^{-r^2}dr^2 = \pi.
\end{aligned}
\end{equation*}
It is obvious that $S>0$, therefore $S=\sqrt{\pi}$. For the following two equations, let $x=m+\sqrt{2}\sigma y$, then:
\begin{equation*}
\begin{aligned}
&\frac{1}{\sqrt{2\pi\sigma^2}}\int_{-\infty}^{\infty}xe^{-\frac{(x-m)^2}{2\sigma^2}}dx = \frac{1}{\sqrt{\pi}}\int_{-\infty}^{\infty}(m+\sqrt{2}\sigma y)e^{-y^2}dy\\
=~&\frac{m}{\sqrt{\pi}}\int_{-\infty}^{\infty}e^{-y^2}dy+\frac{\sqrt{2}\sigma}{\sqrt{\pi}}\int_{-\infty}^{\infty}ye^{-y^2}dy = m+0=m.
\end{aligned}
\end{equation*}
Here, we use the conclusion of $\int_{-\infty}^{\infty}e^{-y^2}dy=\sqrt{\pi}$ and the fact that $ye^{-y^2}$ is an odd function, whose integral over $\mathbb{R}$ must be 0.
\begin{equation*}
\begin{aligned}
&\frac{1}{\sqrt{2\pi\sigma^2}}\int_{-\infty}^{\infty}(x-m)^2e^{-\frac{(x-m)^2}{2\sigma^2}}dx = \frac{1}{\sqrt{\pi}}\int_{-\infty}^{\infty}2\sigma^2 y^2 e^{-y^2}dy\\
=~&\frac{\sigma^2}{\sqrt{\pi}}\int_{-\infty}^{\infty}y\cdot 2ye^{-y^2}dy = \frac{\sigma^2}{\sqrt{\pi}}\int_{-\infty}^{\infty}-y de^{-y^2} = \frac{\sigma^2}{\sqrt{\pi}}\cdot \left(-ye^{-y^2}\Bigg|_{-\infty}^{\infty}+\int_{-\infty}^{\infty}e^{-y^2}dy\right)\\
=~&\frac{\sigma^2}{\sqrt{\pi}}\cdot\sqrt{\pi}=\sigma^2.
\end{aligned}
\end{equation*}
Here, we again use the equation $\int_{-\infty}^{\infty}e^{-y^2}dy=\sqrt{\pi}$.

\paragraph{Problem 8} Suppose $A$ and $B$ are independent events in some probability space. Show that $A^c$ and $B$ are independent. Likewise, show that $A^c$ and $B^c$ are independent.

\paragraph{Solution of Problem 8} Since $A$ and $B$ are independent events, we have:
\[P(A\cap B)=P(A)P(B).\]
Therefore, we have the following two equations:
\[P(A^c\cap B)=P(B)-P(A\cap B)=P(B)-P(A)P(B)=(1-P(A))P(B)=P(A^c)P(B).\]
\begin{equation*}
\begin{aligned}
P(A^c\cap B^c) &= 1-P(A\cup B)=1-P(A)-P(B)+P(A\cap B)=1-P(A)-P(B)+P(A)P(B)\\
&=(1-P(A))(1-P(B))=P(A^c)\cdot P(B^c).
\end{aligned}
\end{equation*}
These two equations conclude that $A^c$ and $B$ are independent. Also, $A^c$ and $B^c$ are independent.

\paragraph{Problem 9} Suppose we have three cards: one is red on both sides, one is red on one side and white on the other side, and one is white on both sides. \\
(1) Pick a card and then one of its sides at random. What is the probability it is red?\\
(2) Given that the side of the card is red, what is the probability that the other side is red?

\paragraph{Solution of Problem 9}~\\
(1) We denote R as red and W as white, then:
\[P(R)=P(RR)\cdot 1+P(RB)\cdot \frac12 + P(BB)\cdot 0 = \frac13+\frac16=\frac12.\]
The probability of red side is $\frac12$.

~\\
(2) By using Bayes' formula, we have:
\[P(RR|R)=\frac{P(RR)\cdot P(R|RR)}{P(RR)\cdot P(R|RR)+P(RB)\cdot P(R|RB)+P(BB)\cdot P(R|BB)}=\frac{1/3}{1/3+1/6+0}=\frac23.\]
The probability for the other side to be red is $\frac23$.

\paragraph{Problem 10} Suppose that $A_1, A_2,\ldots, A_m$ are disjoint events, each of positive probability, such that $\Omega=\bigcup_{j=1}^{m} A_j$. Prove Bayes' formula:
\[P(A_k|B)=\frac{P(B|A_k)P(A_k)}{\sum_{j=1}^{m}P(B|A_j)P(A_j)}\]
for $k=1,2,\ldots, m$ provided $P(B)>0$.

\paragraph{Solution of Problem 10} Since $A_1, A_2, \ldots, A_m$ all have positive probabilities, so:
\[P(B|A_j)P(A_j)=P(B\cap A_j).\]
Also, they are disjoint and $\bigcup_{j=1}^{m}A_j=\Omega$, which leads to $\{B\cap A_j\}_{j\in[m]}$ are also disjoint and $\bigcup_{j=1}^{m}(B\cap A_j)=B\cap \Omega=B$. Therefore:
\[\frac{P(B|A_k)P(A_k)}{\sum_{j=1}^{m}P(B|A_j)P(A_j)}=\frac{P(B\cap A_k)}{\sum_{j=1}^{m}P(B\cap A_j)}=\frac{P(B\cap A_k)}{P(B)}=P(A_k|B),\]
which comes to our conclusion. Here, we used the additivity of probability measures.

\paragraph{Problem 11} During one fall semester 105 women applied to Miskatonic University, of whom 76 were accepted, and 400 men applied, of whom 230 were accepted. During the subsequent spring semester, 300 women applied, of whom 100 were accepted, and 112 men applied, of whom 21 were accepted. Calculate numerically:\\
(a) the probability of a female applicant being accepted during the fall,\\
(b) the probability of a male applicant being accepted during the fall, \\
(c) the probability of a female applicant being accepted during the spring,\\
(d) the probability of a male applicant being accepted during the spring.\\
Consider now the total applicant pool for both semesters together and calculate. \\
(e) the probability of a female applicant being accepted,\\
(f) the probability of a male applicant being accepted.\\
Are the university's admission policies biased towards females or towards males?

\paragraph{Solution of Problem 11} ~\\
(a) $76/105 = 0.7238$.\\
(b) $230/400 = 0.5750$.\\
(c) $100/300 = 0.3333$.\\
(d) $21/112 = 0.1875$.\\
(e) $(100+76)/(300+105) = 0.4346$.\\
(f) $(230+21)/(400+112) = 0.4902$.\\
~\\
Although in each semester, the probability of acceptance of females is higher than males, the university's admission policies biased towards males.

\paragraph{Problem 12} Let $X$ be a real-valued, $\mathcal N(0,1)$ random variable, and set $Y:=X^2.$ Calculate the density $g$ of the distribution function for $Y$. \\
(Hint: You must find $g$ so that $P(-\infty<Y\leqslant a)=\int_{-\infty}^{a}gdy $ for all $a$.)

\paragraph{Solution of Problem 12} Notice that $Y=X^2 \geqslant 0$ always holds, which means $P(Y<0)=0$. For $\forall t\geqslant 0$, we have:
\[P(-\infty<Y\leqslant t)=P(X^2\leqslant t)=P(-\sqrt{t}\leqslant X\leqslant \sqrt{t})=\int_{-\sqrt{t}}^{\sqrt{t}}\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{x^2}{2}\right)dx = 2\phi(\sqrt{t}).\]
where $\phi(u)=\int_{0}^{u}\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{x^2}{2}\right)dx$. Then: $\phi'(u)=\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{u^2}{2}\right)$. Now, we can conclude that, the density function for distribution $Y$ is:
\[g(t)=\frac{d}{dt}2\phi(\sqrt{t})= \frac{\frac{1}{\sqrt{2\pi}}\cdot\exp(-t/2)}{\sqrt{t}}=\frac{\exp(-t/2)}{\sqrt{2\pi t}}~~~~~(t\geqslant 0).\]
To sum up, the density function is:
\[g(t)=\frac{\exp(-t/2)}{\sqrt{2\pi t}}\cdot\mathbb{I}[t>0].\]

\paragraph{Problem 13} Take $\Omega = [0,1]\times [0,1]$, with $\mathcal U$ the Borel sets and $P$ Lebesgue measure. Let $g:[0,1]\rightarrow\mathbb{R}$ be a continuous function.\\
Define the random variables
\[X_1(\omega):=g(x_1), X_2(\omega):=g(x_2)~~\text{for}~\omega=(x_1,x_2)\in\Omega.\]
Show that $X_1$ and $X_2$ are independent and identically distributed.

\paragraph{Solution of Problem 13} For $\forall A\in\mathcal B(\mathbb{R})$, we have:
\[P(X_1\in A) = P(g^{-1}(A)\times [0,1])= P([0,1]\times g^{-1}(A))=P(X_2\in A).\]
Here, since $g$ is a continuous function, we know that $g^{-1}(A)\in \mathcal B([0,1])$. Therefore, $X_1$ and $X_2$ are identically distributed. Next, we are going to prove that they are independently distributed. For $\forall A, B\in\mathcal B(\mathbb{R})$, we have:
\[P(X_1\in A, X_2\in B) = P(g^{-1}(A)\times g^{-1}(B))=P'(g^{-1}(A))\times P'(g^{-1}(B)) = P(X_1\in A)\cdot P(X_2\in B),\]
where $P'$ is the Lebesgue measure on $\mathbb{R}$. Therefore, we can conclude that $X_1$ and $X_2$ are independently distributed.

\paragraph{Problem 14} Let $f:[0,1]\rightarrow\mathbb{R}$ be continuous and define the Bernstein polynomial
\[b_n(x):=\sum_{k=0}^{n}f\left(\frac{k}{n}\right)\binom{n}{k}x^k(1-x)^{n-k}.\]
Prove that $b_n\rightarrow f$ uniformly on $[0,1]$ as $n\rightarrow\infty$, by providing the details for the following steps.\\
(1) Since $f$ is uniformly continuous, for each $\varepsilon > 0$ there exists $\delta(\varepsilon)>0$ such that $|f(x)-f(y)|\leqslant \varepsilon$ if $|x-y|\leqslant \delta(\varepsilon)$.\\
(2) Given $x\in[0,1]$, take a sequence of independent random variables $X_k$ such that $P(X_k=1)=x, P(X_k=0)=1-x$. Write $S_n=X_1+X_2+\ldots+X_n$. Then $b_n(x)=\mathbb{E}f\left(\frac{S_n}{n}\right)$.\\
(3) Therefore
\begin{equation*}
\begin{aligned}
|b_n(x)-f(x)|&\leqslant \mathbb{E}\left|f\left(\frac{S_n}{n}\right)-f(x)\right|\\
&= \int_A\left|f\left(\frac{S_n}{n}\right)-f(x)\right|dP + \int_{A^c}\left|f\left(\frac{S_n}{n}\right)-f(x)\right|dP,
\end{aligned}
\end{equation*}
for $A:=\left\{\omega\in\Omega|~|\frac{S_n}{n}-x|\leqslant \delta(\varepsilon)\right\}$. \\
(4) Then show that
\[|b_n(x)-f(x)|\leqslant \varepsilon+\frac{2M}{\delta(\varepsilon)^2}\cdot V\left(\frac{S_n}{n}\right)=\varepsilon+\frac{2M}{n\delta(\varepsilon)^2}V(X_1),\]
for $M=\max|f|$. Conclude that $b_n\rightarrow f$ uniformly.

\paragraph{Solution of Problem 14} ~\\
(1) For any $\varepsilon > 0, x\in[0,1]$, there exists $\delta_x>0$ such that $|u-x|< \delta_x\Rightarrow |f(u)-f(x)|\leqslant \varepsilon/2$. Then: $\cup_{x\in[0,1]}(x-\delta_x, x+\delta_x)\supseteq[0,1]$. According to the Heine-Borel Covering Theorem: there exists a finite sub-covering. We denote it by $(a_1, b_1), (a_2, b_2), \ldots, (a_n, b_n)$.  Let $\delta = \min_{i,j}\{|a_i-b_j|>0\}$, then for $\forall x,y\in[0,1]$ such that $|x-y|<\delta$, they belong to the same interval, which leads to $|f(x)-f(y)|\leqslant \varepsilon$, and it comes to our conclusion.

~\\
(2) Notice that:
\begin{equation*}
\begin{aligned}
\mathbb{E}f\left(\frac{S_n}{n}\right) &= f\left(\frac{k}{n}\right)\cdot P(\{\text{There are }k\text{ 1-s and }n-k\text{ 0-s in }X_1, X_2, \ldots, X_n\})\\
&= f\left(\frac{k}{n}\right)\cdot\binom{n}{k}x^k(1-x)^{n-k}=b_n(x),
\end{aligned}
\end{equation*}
which comes to our conclusion.

~\\
(3) Notice that for any random variable $X$, we have:
\[-\mathbb{E}|X|\leqslant \mathbb{E}X \leqslant \mathbb{E}|X|~\Rightarrow |\mathbb{E}X|\leqslant \mathbb{E}|X|.\]
Therefore:
\begin{equation*}
\begin{aligned}
|b_n(x)-f(x)| &= |\mathbb{E}[f(S_n/n)-f(x)]|\leqslant \mathbb{E}\left|f\left(\frac{S_n}{n}\right)-f(x)\right|\\
&= \int_A\left|f\left(\frac{S_n}{n}\right)-f(x)\right|dP + \int_{A^c}\left|f\left(\frac{S_n}{n}\right)-f(x)\right|dP,
\end{aligned}
\end{equation*}
for $A:=\{\omega\in\Omega|~|\frac{S_n}{n}-x|\leqslant \delta(\varepsilon)\}.$

~\\
(4) If $\omega\in A$, we have:$|\frac{S_n}{n}-x|\leqslant \delta(\varepsilon)\Rightarrow \left|f\left(\frac{S_n}{n}\right)-f(x)\right|\leqslant \varepsilon$. Therefore:
\[\int_A\left|f\left(\frac{S_n}{n}\right)-f(x)\right|dP\leqslant \varepsilon\cdot P(A)\leqslant \varepsilon.\]
On the other hand,
\begin{equation*}
\begin{aligned}
&\int_{A^c}\left|f\left(\frac{S_n}{n}\right)-f(x)\right|dP \leqslant 2M\cdot P(A^c)\leqslant 2M\cdot\mathbb{E}\frac{\left(\frac{S_n}{n}-x\right)^2}{\delta(\varepsilon)^2}\\
&=\frac{2M}{\delta(\varepsilon)^2}V\left(\frac{S_n}{n}\right)=\frac{2M}{n\delta(\varepsilon)^2}V(X_1).
\end{aligned}
\end{equation*}
After adding up the two inequalities above, we have:
\[\|b_n-f\|_{\infty}\leqslant \varepsilon+\frac{2M}{n\delta(\varepsilon)^2}V(X_1).\]
Then we have:
\[\lim_{n\rightarrow\infty}\|b_n-f\|_{\infty}\leqslant \varepsilon.\]
Since $\varepsilon > 0$ can be any positive real number, we have:
\[\lim_{n\rightarrow\infty}\|b_n-f\|_{\infty} = 0,\]
which comes to our conclusion.

\paragraph{Problem 15} Let $X$ and $Y$ be independent random variables, and suppose that $f_X$ and $f_Y$ are the density functions for $X, Y$. Show that the density function for $X+Y$ is
\[f_{X+Y}(z)=\int_{-\infty}^{\infty} f_X(z-y)f_Y(y)dy.\]
(Hint: If $g:\mathbb{R}\rightarrow\mathbb{R}$, we have
\[\mathbb{E}[g(X+Y)]=\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}f_{X,Y}(x,y)g(x+y)dxdy,\]
where $f_{X,Y}$ is the joint density function of $X,Y$. )

\paragraph{Solution of Problem 15} Notice that,
\[\mathbb{E}[g(X+Y)]=\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}f_{X,Y}(x,y)g(x+y)dxdy.\]
Let $Z=X+Y$, then:
\begin{equation*}
\begin{aligned}
&\int_{-\infty}^{\infty} g(z)\cdot f_Z(z)dz = \mathbb{E}[g(X+Y)] = \int_{-\infty}^{\infty}\int_{-\infty}^{\infty}f_{X,Y}(x,y)g(x+y)dxdy\\
&= \int_{-\infty}^{\infty}\int_{-\infty}^{\infty}f_X(x) f_Y(y)\cdot g(x+y)dxdy = \int_{-\infty}^{\infty}\int_{-\infty}^{\infty} f_Y(y)f_X(z-y) g(z) dzdy\\
&= \int_{-\infty}^{\infty}g(z)\left(\int_{-\infty}^{\infty}f_Y(y)f_X(z-y)dy\right)dz.
\end{aligned}
\end{equation*}
Here, we use the fact that $X,Y$ are independent random variables and we also use the coordinate change of integration. Since $g$ can be any function on $Z$, we can conclusion that:
\[f_{X+Y}(z) = \int_{-\infty}^{\infty}f_Y(y)f_X(z-y)dy,\]
which comes to our conclusion.

\paragraph{Problem 16} Let $X$ and $Y$ be two independent positive random variables, each with density
\[f(x)=\begin{cases}e^{-x}~~~~\mbox{if $x\geqslant 0$}\\ 0~~~~~~~\mbox{if $x < 0$}\end{cases}.\]
Find the density of $X+Y$.

\paragraph{Solution of Problem 16} Notice that for $\forall z\geqslant 0$, we have:
\begin{equation*}
\begin{aligned}
&P(X+Y\leqslant z) =\mathbb{E}\mathbb{I}[X+Y\leqslant z] = \mathbb{E}_Y [\mathbb{E}\mathbb{I}[X+Y\leqslant z]|Y] = \mathbb{E}_Y P(X\leqslant z-Y)\\
&= \int_{0}^{z} e^{-y}dy\cdot \int_{0}^{z-y} e^{-x} dx = \int_{0}^{z} e^{-y}(1-e^{-(z-y)})dy = 1-e^{-z}-ze^{-z} = 1-(z+1)e^{-z}.
\end{aligned}
\end{equation*}
After taking derivative over $z$, we know that the density of $X+Y$ is:
\[g(z)=ze^{-z}~~(z\geqslant 0).\]

\paragraph{Problem 17} Show that
\[\lim_{n\rightarrow\infty}\int_0^1\int_0^1\ldots\int_0^1  f\left(\frac{x_1+x_2+\ldots+x_n}{n}\right)dx_1 dx_2\ldots dx_n = f\left(\frac12\right)\]
for each continuous function $f$.

(Hint: Let $X_1, X_2,\ldots, X_n$ be independent random variables, each of which has density function $f_i(x)=1$ if $0\leqslant x\leqslant 1$ and $=0$ otherwise. Then $P\left(\left|\frac{X_1+X_2+\ldots+X_n}{n}-\frac12\right|>\varepsilon\right)\leqslant \frac{1}{\varepsilon^2}V\left(\frac{X_1+X_2+\ldots+X_n}{n}\right)=\frac{1}{12\varepsilon^2 n}$.)

\paragraph{Solution of Problem 17} Let $X_1, X_2,\ldots, X_n$ be independent random variables, each of which follows the uniform distribution over $[0,1]$. Then, we only need to prove that:
\[\lim_{n\rightarrow\infty} \mathbb{E} f\left(\frac{X_1+X_2+\ldots+X_n}{n}\right)=f(1/2).\]

As we know that:
\[P\left(\left|\frac{X_1+X_2+\ldots+X_n}{n}-\frac12\right|>\varepsilon\right)\leqslant \frac{1}{\varepsilon^2}V\left(\frac{X_1+X_2+\ldots+X_n}{n}\right)=\frac{1}{12\varepsilon^2 n}.\]
Since $f$ is continuous function, $f$ is bounded on the closed interval $[0,1]$, which means there exists $L>0$ such that $|f(x)-f(1/2)|<L$ holds for $\forall x\in [0,1]$. Also, $f$ is continuous at $1/2$. Therefore, for $\forall \varepsilon > 0$, there exists $\delta > 0$ such that $|x-1/2|<\delta\Rightarrow |f(x)-f(1/2)|<\varepsilon$. Then:
\[P\left(\left|f\left(\frac{X_1+X_2+\ldots+X_n}{n}\right)-f(1/2)\right|>\varepsilon\right)\leqslant P\left(\left|\frac{X_1+X_2+\ldots+X_n}{n}-\frac12\right|>\delta\right)\leqslant \frac{1}{12\delta^2 n},\]
which leads to:
\[\left|\mathbb{E}f\left(\frac{X_1+X_2+\ldots+X_n}{n}\right) -f(1/2)\right|\leqslant \varepsilon + \frac{L}{12\delta^2 n}.\]
Let $n\rightarrow\infty$:
\[\lim_{n\rightarrow\infty}\left|\mathbb{E}f\left(\frac{X_1+X_2+\ldots+X_n}{n}\right)-f(1/2)\right|\leqslant \varepsilon.\]
Since $\varepsilon$ can be any positive real number, we can finally conclude that:
\[\lim_{n\rightarrow\infty}f\left(\frac{X_1+X_2+\ldots+X_n}{n}\right)=f(1/2).\]

\paragraph{Problem 18} Prove that:\\
(1) $\mathbb{E}[\mathbb{E}[X|\mathcal V]]=\mathbb{E}[X]$.\\
(2) $\mathbb{E}[X] = \mathbb{E}[X|\mathcal W]$, where $\mathcal W = \{\emptyset, \Omega\}$ is the trivial $\sigma$-algebra.

\paragraph{Solution of Problem 18} According to the definition of conditional expectation, we know that $\mathbb{E}[X|\mathcal V]$ is a random variable on $\Omega$ such that $\mathbb{E}[X|\mathcal V]$ is $\mathcal V$-measurable and $\int_A XdP = \int_A \mathbb{E}[X|\mathcal V]dP$ for all $A\in\mathcal V$.\\
(1) $\mathbb{E}[\mathbb{E}[X|\mathcal V]] = \int \mathbb{E}[X|\mathcal V] d\mu = \int Xd\mu = \mathbb{E}[X]$.\\
(2) Since $\mathcal W=\{\emptyset, \Omega\}$ is the trivial $\sigma$-algebra, and random variable $\mathbb{E}[X|\mathcal W]$ is $\mathcal W$-measurable, which means $\mathbb{E}[X|\mathcal W]$ is a constant variable. By the conclusion of (1), we have:
\[\mathbb{E}[X|\mathcal W]=\mathbb{E}[X],\]
which comes to our conclusion.

\paragraph{Problem 19} Let $X,Y$ be two real-valued random variables and suppose their joint distribution function has the density $f(x,y)$. Show that
\[\mathbb{E}[X|Y]=\Phi(Y)~~a.s.\]
for
\[\Phi(y)=\frac{\int_{-\infty}^{\infty}xf(x,y)dx}{\int_{-\infty}^{\infty}f(x,y)dx}.\]

\paragraph{Solution of Problem 19} According to the definition of conditional expectation, we know that: for all $A\in\mathcal U(Y)$,
\[\int_A XdP=\int_A \mathbb{E}[X|Y] dP = \int_A \Phi(Y)dP.\]
For any Borel subset of $\mathbb{R}$, denoted by $B$, let $A=\{Y\in B\}\in \mathcal U(Y)$:
\[\int_A XdP=\int_{\omega}\chi_B(Y)XdP=\int_{-\infty}^{\infty}\int_B xf(x,y)dydx.\]
\[\int_A \Phi(Y)dP = \int_{-\infty}^{\infty} \int_B \Phi(y)f(x,y)dydx.\]
After switching the order of integrals, we know that:
\[\int_{-\infty}^{\infty}xf(x,y)dx = \int_{-\infty}^{\infty}\Phi(y)f(x,y)dx = \Phi(y)\int_{-\infty}^{\infty}f(x,y)dx.\]
It directly leads to
\[\Phi(y) = \frac{\int_{-\infty}^{\infty}xf(x,y)dx}{\int_{-\infty}^{\infty}f(x,y)dx},\]
which comes to our conclusion.

\paragraph{Problem 20} A smooth function $\Phi:\mathbb{R}\rightarrow\mathbb{R}$ is called convex if $\Phi''(x)\geqslant 0$ for all $x\in\mathbb{R}$.\\
(1) Show that if $\Phi$ is convex, then:
\[\Phi(y)\geqslant \Phi(x)+\Phi'(x)(y-x)~~~\text{for all }x,y\in\mathbb{R}.\]
(2) Show that
\[\Phi\left(\frac{x+y}{2}\right)\leqslant \frac12\Phi(x)+\frac12\Phi(y)~~~\text{for all }x,y\in\mathbb{R}.\]
(3) A smooth function $\Phi:\mathbb{R}^n\rightarrow\mathbb{R}$ is called convex if the matrix $D^2\Phi=((\Phi_{x_i x_j}))$ is nonnegative definite for all $x\in\mathbb{R}^n$. (This means that $\sum_{i,j=1}^n \Phi_{x_i x_j}\xi_i\xi_j\geqslant 0$ for all $\xi\in\mathbb{R}^n$.) Prove that
\[\Phi(y)\geqslant \Phi(x)+D\Phi(x)\cdot (y-x),\]
\[\Phi\left(\frac{x+y}{2}\right)\leqslant \frac12\Phi(x)+\frac12\Phi(y)\]
for all $x,y\in\mathbb{R}^n$.

\paragraph{Solution of Problem 20} ~\\
(1) Notice that, by using Lagrangian Theorem
\[\Phi(y)-\Phi(x) = \Phi'(x')(y-x)\]
holds for some $x'=x+t(y-x)~(t\in[0,1])$. Then:
\[\Phi(y)-\Phi(x)-\Phi'(x)(y-x)=(y-x)(\Phi'(x')-\Phi(x))=(y-x)(x'-x)\Phi''(x'')=t(y-x)^2\Phi''(x'').\]
Since $t\geqslant 0$ and $\Phi''(x'')\geqslant 0$, we can conclude that:
\[\Phi(y)\geqslant \Phi(x)+\Phi'(x)(y-x).\]
(2) By using Lagrangian Theorem, there exists $t_1, t_2 \in[0,1/2]$, such that:
\[\Phi\left(\frac{x+y}{2}\right)-\Phi(x)=\Phi'(x+t_1(y-x))\cdot\frac{y-x}{2},\]
\[\Phi(y)-\Phi\left(\frac{x+y}{2}\right) = \Phi'(y-t_2(y-x))\cdot\frac{y-x}{2}.\]
Therefore, we have:
\begin{equation*}
\begin{aligned}
&\frac12\Phi(x)+\frac12\Phi(y)-\Phi\left(\frac{x+y}{2}\right) = \frac{y-x}{4}\cdot \left[\Phi'(y-t_2(y-x))-\Phi'(x+t_1(y-x))\right]\\
&=\frac{(1-t_1-t_2)(y-x)^2}{4}\cdot\Phi''(x'')\geqslant 0,
\end{aligned}
\end{equation*}
since $1-t_1-t_2\geqslant 0$ and $\Phi''(x'')\geqslant 0$, which comes to our conclusion.

~\\
(3) Notice that:
\begin{equation*}
\begin{aligned}
&\Phi(y)-\Phi(x)-D\Phi(x)\cdot(y-x) = \int_0^1 \langle D\Phi(x+t(y-x)), y-x\rangle dt - D\Phi(x)\cdot(y-x)\\
=& \int_0^1 \langle D\Phi(x+t(y-x))- D\Phi(x), y-x\rangle dt =\int_0^1 dt\int_0^t (y-x)^{\top} D^2\Phi(x+t'(y-x))\cdot (y-x)dt'\geqslant 0,
\end{aligned}
\end{equation*}
which comes to our conclusion. Then, by using this inequality, we have:
\[\Phi(x)-\Phi\left(\frac{x+y}{2}\right)\geqslant -D\Phi\left(\frac{x+y}{2}\right)\cdot\frac{y-x}{2},\]
\[\Phi(y)-\Phi\left(\frac{x+y}{2}\right)\geqslant D\Phi\left(\frac{x+y}{2}\right)\cdot\frac{y-x}{2}.\]
After adding them up, we obtain that:
\[\Phi\left(\frac{x+y}{2}\right)\leqslant \frac12\Phi(x)+\frac12\Phi(y).\]

\paragraph{Problem 21} ~\\
(1) Prove Jensen's Inequality:
\[\Phi(\mathbb{E}X)\leqslant \mathbb{E}(\Phi(X)),\]
for a random variable $X:\Omega\rightarrow\mathbb{R}$, where $\Phi$ is convex. (Hint: Use assertion (3) from the previous exercise.)\\
(2) Prove the conditional Jensen Inequality:
\[\Phi(\mathbb{E}(X|\mathcal V))\leqslant \mathbb{E}(\Phi(X)|\mathcal V).\]

\paragraph{Solution of Problem 21}~\\
(1) According to the assertion (3) of the previous exercise, we have:
\[\Phi(X)-\Phi(\mathbb{E}(X))\geqslant D\Phi(\mathbb{E}(X))\cdot(X-\mathbb{E}X).\]
Therefore, after taking the expectation, it holds that:
\[\mathbb{E}\Phi(X)-\Phi(\mathbb{E}(X))\geqslant D\Phi(\mathbb{E}(X))\cdot \mathbb{E}\left(X-\mathbb{E}X\right) = 0,\]
which comes to our conclusion. \\
(2) Again by using assertion (3), we know that:
\[\Phi(X)-\Phi(\mathbb{E}(X|\mathcal V))\geqslant D\Phi(\mathbb{E}(X|\mathcal V))\cdot(X-\mathbb{E}(X|\mathcal V)).\]
By taking conditional expectation $p(\cdot|\mathcal V)$:
\[\mathbb{E}(\Phi(X)|\mathcal V)-\Phi(\mathbb{E}(X|\mathcal V))\geqslant D\Phi(\mathbb{E}(X|\mathcal V))\cdot\mathbb{E}\left(X-\mathbb{E}(X|\mathcal V)|\mathcal V\right)=0,\]
which comes to our conclusion.

\paragraph{Problem 22} Let $W(\cdot)$ be a one-dimensional Brownian motion. Show that
\[\mathbb{E}[W^{2k}(t)]=\frac{(2k)!t^k}{2^k k!}~~(t>0).\]

\paragraph{Solution of Problem 22} According to the property of Brownian motions, $W(t)\sim\mathcal N(0,t)$. Therefore:
\[\mathbb{E}[W^{2k}(t)]=\mathbb{E}_{z\sim\mathcal N(0,1)}(\sqrt{t}z)^{2k}= t^k\cdot\mathbb{E}[z^{2k}]=t^k(2k-1)!!=\frac{(2k)!t^k}{2^k k!},\]
which comes to our conclusion.

\paragraph{Problem 23} Show that if $W(\cdot)$ is an $n$-dimensional Brownian motion, then so are:\\
(1) $W(t+s)-W(s)$ for all $s\geqslant 0$.\\
(2) $cW(t/c^2)$ for all $c>0$. (Brownian scaling)

\paragraph{Solution of Problem 23} ~\\
(1) For any $k\in[n]$, $W^k(t)$ is a one-dimensional Brownian motion, then $W^k(t+s)-W^k(s)$ is a Gaussian process and
\[\mathbb{E}[(W^k(u+s)-W^k(s))(W^k(v+s)-W^k(s))]=\min(u+s,v+s)+s-\min(u+s,s)-\min(v+s,s)=\min(u,v),\]
which means $W^k(t+s)-W^k(s)$ is also a one-dimensional Brownian motion. Also $\{W^k(t+s)-W^k(s)\}_{k\in[n]}$ are independent. Therefore, $W(t+s)-W(s)$ is a Brownian motion. \\
(2) For any $k\in[n]$, $W^k(t)$ is a one-dimensional Brownian motion, then $cW^k(t/c^2)$ is a Gaussian process and
\[\mathbb{E}[cW(u/c^2)\cdot cW(v/c^2)]=c^2\min(u/c^2,v/c^2)=\min(u,v).\]
So $cW^k(t/c^2)$ is a one-dimensional Brownian motion. Also $\{cW^k(t/c^2)\}_{k\in[n]}$ are independent. Therefore, $cW(t/c^2)$ is a Brownian motion.

\paragraph{Problem 24} Let $W(\cdot)$ be a one-dimensional Brownian motion, and define
\[\overline{W}(t)=\begin{cases}tW\left(\frac{1}{t}\right)~~~\mbox{for $t>0$}\\ 0 ~~~~~~~~~~~~\mbox{for $t=0$}\end{cases}.\]
Show that $\overline{W}(t)-\overline{W}(s)$ is $\mathcal N(0,t-s)$ for times $0\leqslant s\leqslant t$. ($\overline{W}(\cdot)$ also has independent increments and so is a one-dimensional Brownian motion. You do not need to show this.)

\paragraph{Solution of Problem 24} For times $0\leqslant s\leqslant t$, we know that:
\[\overline{W}(t)-\overline{W}(s) = tW\left(\frac{1}{t}\right) - sW\left(\frac{1}{s}\right)= (t-s)W\left(\frac{1}{t}\right)-s\left(W\left(\frac{1}{s}\right)-W\left(\frac{1}{t}\right)\right).\]
Notice that $0<\frac1t\leqslant \frac1s$, and so $W\left(\frac{1}{t}\right)$ are independent with $W\left(\frac{1}{s}\right)-W\left(\frac{1}{t}\right)$. We know that:
\[W\left(\frac{1}{t}\right)\sim\mathcal N(0,1/t),~~W\left(\frac{1}{s}\right)-W\left(\frac{1}{t}\right)\sim\mathcal N(0, 1/s-1/t).\]
Therefore, $\overline{W}(t)-\overline{W}(s)$ follows the zero-centered Gaussian distribution with its variance
\[(t-s)^2\cdot\frac1t + s^2\cdot\left(\frac1s-\frac1t\right)=t-s.\]
To sum up, we conclude that $\overline{W}(t)-\overline{W}(s)\sim\mathcal N(0,t-s)$.

\paragraph{Problem 25} Define $X(t):=\int_0^t W(s)ds$, where $W(\cdot)$ is a one-dimension Brownian motion. Show that
\[\mathbb{E}[X^2(t)]=\frac{t^3}{3}~~~~~\text{for each }t>0.\]

\paragraph{Solution of Problem 25} The Brownian motion can be written as: $W(t)=\sum_{k=0}^{+\infty} A_k s_k(t)$. Therefore:
\[X(t)=\sum_{k=0}^{+\infty}A_k \int_0^t s_k(s)ds:=\sum_{k=0}^{+\infty}A_k U_k(t).\]
Then, since $\{A_k\}$ are independent random variables sampled from standard Gaussian. Therefore:
\[\mathbb{E}[X^2(t)]=\sum_{k=0}^{+\infty} U_k^2(t).\]
Notice that:
\begin{equation*}
\begin{aligned}
\sum_{k=0}^{+\infty} U_k^2(t) &= \sum_{k=0}^{+\infty}\int_0^t\int_0^t s_k(x)s_k(y)dxdy = \int_0^t\int_0^t \left[\sum_{k=0}^{+\infty}s_k(x)s_k(y)\right]dxdy\\
&= \int_0^t\int_0^t\min(x,y)dxdy = 2\cdot\int_0^t xdx\int_x^t dy = 2\int_0^t x(t-x)dx= 2\left(\frac{t^3}{2}-\frac{t^3}{3}\right)=\frac{t^3}{3},
\end{aligned}
\end{equation*}
which comes to our conclusion.

\paragraph{Problem 26} Define $X(t)$ as in the previous exercise. Show that
\[\mathbb{E}e^{\lambda X(t)}=e^{\frac{\lambda^2 t^3}{6}}~~~~\text{for each }t>0.\]
(Hint: $X(t)$ is a Gaussian random variable, the variance of which we know from the previous exercise.)

\paragraph{Solution of Problem 26} From the exercise above, we know that $X(t)=\sum_{k=0}^{+\infty}A_k U_k(t)$. Then:
\begin{equation*}
\begin{aligned}
\mathbb{E}e^{\lambda X(t)}&=\prod_{k=0}^{+\infty}\mathbb{E} e^{\lambda A_k U_k(t)} = \prod_{k=0}^{+\infty}\exp\left(-\frac{\lambda^2 U_k(t)^2}{2}\right)\\
&= \exp\left(-\frac{\lambda^2 \sum_{k=0}^{+\infty}U_k(t)^2}{2}\right) = e^{-\frac{\lambda^2 t^3}{6}}.
\end{aligned}
\end{equation*}
Here, we use the conclusion from the previous exercise.

\paragraph{Problem 27} Define $U(t) := e^{-t} W(e^{2t})$, where $W(\cdot)$ is a one-dimensional Brownian motion. Show that
\[\mathbb{E}[U(t)U(s)]=e^{-|t-s|}~~~\text{for all}~~-\infty<s,t<\infty.\]

\paragraph{Solution of Problem 27} Without loss of generality, we can assume that $s\leqslant t$. Then:
\begin{equation*}
\begin{aligned}
\mathbb{E}[U(t)U(s)] &= e^{-t-s}\cdot\mathbb{E}[W(e^{2t})W(e^{2s})] = e^{-t-s}\cdot\mathbb{E}\left[W(e^{2s})^2 + W(e^{2s})\cdot(W(e^{2t})-W(e^{2s}))\right]\\
&= e^{-t-s}\cdot(e^{2s} + 0)  = e^{-t+s} = e^{-|t-s|},
\end{aligned}
\end{equation*}
which comes to our conclusion. Here, we use the fact that $W(x)\sim\mathcal N(0,x)$ for all $x\geqslant 0$ and $W(x)-W(y)\sim \mathcal N(0,x-y)$ for all $0<y<x$ which is independent of $W(y)$.

\paragraph{Problem 28} Let $W(\cdot)$ be a one-dimensional Brownian motion. Show that
\[\lim_{m\rightarrow\infty}\frac{W(m)}{m}=0~~~\text{almost surely.}\]
(Hint: Fix $\varepsilon > 0$ and define the event $A_m:=\left\{\left|\frac{W(m)}{m}\right|\geqslant \varepsilon\right\}$. Then $A_m=\{|X|\geqslant \sqrt{m}\varepsilon\}$ for the $\mathcal N(0,1)$ random variable $X=\frac{W(m)}{\sqrt{m}}$. Apply the Borel-Cantelli Lemma.)

\paragraph{Solution of Problem 28} Notice that, the event $\left\{\lim_{m\rightarrow\infty}\frac{W(m)}{m}\neq 0\right\}$ is equivalent to the event that there exists $\delta>0$ such that there are infinitely many $m$-s satisfy $\left|\frac{W(m)}{m}\right|>\delta$, which means:
\[E := \left\{\lim_{m\rightarrow\infty}\frac{W(m)}{m}\neq 0\right\} = \lim_{k\rightarrow\infty}E_k\]
where
\[E_k = \left\{\text{There are infinitely many }m\text{-s satisfy }\left|\frac{W(m)}{m}\right|>\frac1k\right\}.\]
Notice that
\[E_k = \lim\sup_{m\rightarrow\infty}\left\{\left|\frac{W(m)}{m}\right|>\frac1k\right\}.\]
According to the fact that $W(m)\sim\mathcal N(0,m)$, we have:
\[P\left(\left|\frac{W(m)}{m}\right|>\frac1k\right)=P\left(\left|\frac{W(m)}{\sqrt{m}}\right|>\frac{\sqrt{m}}{k}\right) = \widetilde{\Phi}\left(\frac{\sqrt{m}}{k}\right)\leqslant \exp\left(-\frac{m}{2k^2}\right).\]
Then their sum:
\[\sum_{m=1}^{\infty}P\left(\left|\frac{W(m)}{m}\right|>\frac1k\right)\leqslant \sum_{m=1}^{\infty}\exp\left(-\frac{m}{2k^2}\right) = \frac{1}{1-\exp(-1/2k^2)},\]
which means the infinite sum actually converges. By using Borel-Cantelli Lemma, we know that:
\[P(E_k)=\left(\lim\sup_{m\rightarrow\infty}\left\{\left|\frac{W(m)}{m}\right|>\frac1k\right\}\right)=0,\]
and then:
\[P(E)=\lim_{k\rightarrow\infty}P(E_k)=0~~~\Rightarrow~~P(\overline{E})=1,\]
which comes to our conclusion.

\paragraph{Problem 29} (1) Let $0<\gamma\leqslant 1$. Show that $f:[0,T]\rightarrow \mathbb{R}^n$ is uniformly Holder continuous with exponent $\gamma$, it is also uniformly Holder continuous with each exponent $0<\delta<\gamma$. \\
(2) Show that $f(t)=t^{\gamma}$ is uniformly Holder continuous with exponent $\gamma$ on the interval $[0,1]$.

\paragraph{Solution of Problem 29} (1) Since $f$ is uniformly Holder continuous with exponent $\gamma$, we have:
\[\|f(x)-f(y)\|\leqslant K |x-y|^{\gamma}\]
holds for $\forall x,y\in[0,T]$ and a constant $K$. Then if $|x-y|\leqslant 1$, we have: $|x-y|^{\gamma}\leqslant |x-y|^{\delta}$, which directly leads to
\[\|f(x)-f(y)\|\leqslant K |x-y|^{\delta}.\]
If $|x-y|>1$, we have: $|x-y|^{\gamma}=|x-y|^{\gamma-\delta}\cdot|x-y|^{\delta}\leqslant T^{\gamma-\delta}\cdot |x-y|^{\delta}$. To sum up, for $\forall x,y\in[0,T]$, it holds that:
\[\|f(x)-f(y)\|\leqslant K\max(1,T^{\gamma-\delta}) |x-y|^{\delta},\]
which comes to our conclusion.\\

(2) We only need to prove that for $\forall x,y\in[0,1]$, it uniformly holds that:
\[|x^{\gamma}-y^{\gamma}|<K|x-y|^{\gamma}\]
for some constant $K$. Without loss of generality, we can assume that $0\leqslant y\leqslant x\leqslant 1$. Let $\delta = x-y$, then we need to prove that:
\[g(y):=(y+\delta)^{\gamma}-y^{\gamma} < K \delta^{\gamma}\]
for some constant $K$. Since $\gamma\in(0,1]$, we have:
\[g'(y)=\gamma\cdot\left((y+\delta)^{\gamma-1}-y^{\gamma-1}\right)\leqslant 0.\]
It means $g$ is a decreasing function. Therefore:
\[g(y)\leqslant g(0) = \delta^{\gamma}\]
holds for $\forall y$. We can make $K=2$ and it comes to our conclusion.

\paragraph{Problem 30} Let $0<\gamma<\frac12$. We showed in Chapter 3 that if $W(\cdot)$ is a one-dimensional Brownian motion, then for almost every $\omega$ there exists a constant $K$, depending on $\omega$, such that
\[|W(t,\omega)-W(s,\omega)|\leqslant K|t-s|^{\gamma}~~~\text{for all }0\leqslant s,t\leqslant 1.\]
Show that there does not exist a constant $K$ such that it holds for almost all $\omega$.

\paragraph{Solution of Problem 30} Assume there exists a constant $K$ such that if holds for almost all $\omega$. Then:
\begin{equation*}
\begin{aligned}
1&=\mathbb{P}_{\omega}\left[|W(t,\omega)-W(s,\omega)|\leqslant K|t-s|^{\gamma}~~\forall 0\leqslant s,t\leqslant 1\right] \leqslant \mathbb{P}_{\omega}\left[|W(1,\omega)-W(0,\omega)|\leqslant K\right]\\
&= \mathbb{P}_{\omega}\left[|W(1,\omega)|\leqslant K\right] = 1-2\widetilde{\phi}(K).
\end{aligned}
\end{equation*}
which leads to a contradiction.

\paragraph{Problem 31} Prove that if $G,H\in\mathbb{L}^2(0,T)$, then:
\[\mathbb{E}\left(\int_0^T GdW \int_0^T HdW\right) = \mathbb{E}\left(\int_0^T GHdt\right).\]
(Hint: $2ab=(a+b)^2-a^2-b^2$.)

\paragraph{Solution of Problem 31} Notice that:
\[\mathbb{E}\left(\int_0^T GdW +\int_0^T HdW\right)^2 = \|G+H\|_2^2, ~\mathbb{E}\left(\int_0^T GdW\right)^2 = \|G\|_2^2, ~\mathbb{E}\left(\int_0^T HdW\right)^2 = \|H\|_2^2.\]
Therefore: we have
\[\mathbb{E}\left(\int_0^T GdW \int_0^T HdW\right) = \frac{1}{2}\left(\|G+H\|_2^2-\|G\|_2^2-\|H\|_2^2\right)=\langle G,H\rangle=\int_0^T GHdt,\]
which comes to our conclusion.

\paragraph{Problem 32} Let $(\Omega, \mathcal U, P)$ be a probability space, and take $\mathcal F(\cdot)$ to be a filtration of $\sigma$-algebras. Assume $X$ to be an integrable random variable, and define $X(t):=\mathbb{E}[X|\mathcal F(t)]$ for times $t\geqslant 0$. Show that $X(\cdot)$ is a martingale.

\paragraph{Solution of Problem 32} According to the definition of martingale, we know that $\mathcal F(t)\supseteq \mathcal F(s)$ when $t>s$. So for $\forall t>s$, it holds that:
\[\mathbb{E}[X(t)|\mathcal F(s)]=\mathbb{E}[\mathbb{E}[X|\mathcal F(t)]|\mathcal F(s)] = \mathbb{E}[X|\mathcal F(t)\cap \mathcal F(s)]=\mathbb{E}[X|\mathcal F(s)]=X(s),\]
which shows us that $\{X(t)\}$ is a martingale.

\paragraph{Problem 33} Show directly that $I(t):=W^2(t)-t$ is a martingale. \\
(Hint: $W^2(t)=(W(t)-W(s))^2-W^2(s)+2W(t)W(s)$. Take the conditional expectation with respect to $\mathcal W(S)$, the history of $W(\cdot)$, and then condition with respect to the history of $I(\cdot)$.)

\paragraph{Solution of Problem 33} Notice that $W^2(t)=(W(t)-W(s))^2-W^2(s)+2W(t)W(s)$, then:
\[\mathbb{E}[W^2(t)-t|\mathcal W(s)] = (t-s)+W^2(s)-t = W^2(s)-s,\]
holds for $\forall t>s$, which means $\mathbb{E}[I(t)|\mathcal W(s)]=I(s)$. Then, we need to take condition with respect to the history of $I(\cdot)$. Since the conditional expectation above only depends on $W^2(s)$, so:
\[\mathbb{E}[W^2(t)-t|\mathcal W^2(s)] = W^2(s)-s,\]
which leads to:
\[\mathbb{E}[I(t)|\mathcal I(s)] = I(s).\]
It shows us that $I(t)$ is a martingale.

\paragraph{Problem 34} Suppose $X(\cdot)$ is a real-valued martingale and $\Phi:\mathbb{R}\rightarrow\mathbb{R}$ is convex. Assume also that $\mathbb{E}(|\Phi(X(t))|)<\infty$ for all $t\geqslant 0$. Show that $\Phi(X(\cdot))$ is a submartingale. (Hint: Use the conditional Jensen Inequality.)

\paragraph{Solution of Problem 34} Since $X(\cdot)$ is a real-valued martingale, we have: for $\forall t>s$
\[\mathbb{E}(X(t))|\mathcal F(s))=X(s).\]
Then, by using conditional Jensen Inequality, we have:
\[\mathbb{E}(\Phi(X(t))|\mathcal F(s))\geqslant \Phi(\mathbb{E}(X(t)|\mathcal F(s)))=\Phi(X(s)).\]
Therefore, $\Phi(X(\cdot))$ is a submartingale.

\paragraph{Problem 35} Use the Itô chain rule to show that $Y(t):=e^{\frac{t}{2}}\cos(W(t))$ is a martingale.

\paragraph{Solution of Problem 35} By using the Itô chain rule, let $X(t) = W(t)$, we know that: $dX(t)= dW(t)$. Then: $dY(t)=d u(X,t)$ where $u(X,t)=e^{t/2}\cos X$. Therefore,
\begin{equation*}
\begin{aligned}
dY(t)&=du(X,t)=(u_t+\frac12 u_{xx})dt + u_x dW = \left(\frac12 e^{t/2}\cos X-\frac12 e^{t/2}\cos X\right)dt-e^{t/2}\sin W(t)dW\\
&= -e^{t/2}\sin W(t)dW,
\end{aligned}
\end{equation*}
which shows us that $Y(t)$ is a martingale.

\paragraph{Problem 36} Let $\mathbf{W}(\cdot)=(W^1, \ldots, W^n)$ be an $n$-dimensional Brownian motion, and write $Y(t):=|\mathbf{W}(t)|^2-nt$ for times $t\geqslant 0$. Show that $Y(\cdot)$ is a martingale. (Hint: Compute $dY$.)

\paragraph{Solution of Problem 36} We are going to compute $dY(t)$. Since $dW^2 = 2WdW+ dt$, so it's easy for us to compute that:
\[dY(t) = \sum_{i=1}^{N} d(W^i(t))^2 - ndt = \sum_{i=1}^{N} \left[2W^i(t) dW^i(t) + dt\right]-ndt = \sum_{i=1}^{N} 2W^i(t) dW^i(t),\]
which shows us that $Y(t)$ is a martingale.

\paragraph{Problem 37} Show that
\[\int_0^T W^2dW = \frac13 W^3(T)-\int_0^T Wdt\]
and
\[\int_0^T W^3 dW = \frac14 W^4(T) - \frac32 \int_0^T W^2dt.\]

\paragraph{Solution of Problem 37}~\\
(1) By using the Itô's chain rule, we know that:
\[dW^3 = 3W^2 dW + 3W dt.\]
Therefore:
\[W^3(T)=\int_0^T dW^3(t) = 3\int_0^T W^2 dW + 3\int_0^T Wdt~~\Rightarrow~~\int_0^T W^2 dW=\frac13 W^3(T)-\int_0^T Wdt,\]
which comes to our conclusion.\\

(2) Again by using the Itô's chain rule, we know that:
\[dW^4 = 4W^3 dW + 6W^2 dt.\]
Therefore:
\[W^4(T)=\int_0^T dW^4(t) = 4\int_0^T W^3 dW + 6\int_0^T W^2 dt~~\Rightarrow~~\int_0^T W^3 dW=\frac14 W^4(T)-\frac32 \int_0^T W^2 dt,\]
which comes to our conclusion.

\paragraph{Problem 38} Recall from the text that
\[Y := e^{\int_0^t gdW - \frac12 \int_0^t g^2 ds}\]
satisfies
\[dY = gYdW.\]
Use this to prove
\[\mathbb{E}\left(e^{\int_0^T gdW}\right)=e^{\frac12 \int_0^T g^2 ds}.\]

\paragraph{Solution of Problem 38} Define $F = \int_0^t gdW - \frac12 \int_0^t g^2 ds$, then we know that:
\[dF = - \frac12 g^2 dt + gdW.\]
Since $Y=e^F$, we have:
\[dY = e^F dF + \frac12 e^F g^2dt = e^F gdW = gY dW.\]
Then, we can conclude that $Y(t)$ is a martingale, which leads to the fact that:
\[\mathbb{E}Y(t)=\mathbb{E}Y(0)=1~~\Rightarrow~~\mathbb{E}\left(e^{\int_0^T gdW}\right)=e^{\frac12 \int_0^T g^2 ds}.\]

\paragraph{Problem 39} Let $u=u(x,t)$ be a smooth solution of the backwards diffusion equation
\[u_t+\frac12 u_{xx}=0,\]
and suppose $W(\cdot)$ is a one-dimensional Brownian motion. Show that for each time $t>0$
\[\mathbb{E}(u(W(t),t)) = u(0,0).\]

\paragraph{Solution of Problem 39} Let $Y(t) = u(W(t),t)$. By using Itô's chain rule, we know that:
\[dY = u_x dW + u_t dt +\frac12 u_{xx} dt = u_x dW + (u_t+\frac12 u_{xx})dt = u_x dW.\]
Therefore, $Y(t)$ is a martingale, which leads to:
\[\mathbb{E} u(W(t),t)=\mathbb{E} Y(t) = Y(0)=u(0,0).\]

\paragraph{Problem 40} Calculate $\mathbb{E}(B^2(t))$ for the Brownian bridge $B(\cdot)$, and show in particular that $\mathbb{E}(B^2(t))\rightarrow 0$ as $t\rightarrow 1^{-}$.

\paragraph{Solution of Problem 40} We have already known that the Brownian bridge has the following formulation:
\[B(t)=(1-t)\int_0^t\frac{1}{1-s}dW(s).\]
Then, we can obtain that:
\[\mathbb{E}[B^2(t)]=(1-t)^2\cdot\int_0^t\frac{1}{(1-s)^2}ds = (1-t)^2\cdot\left(\frac{1}{1-t}-1\right)=t(1-t).\]
Furthermore, it's obvious to verify that when $t\rightarrow 1^-$, $\mathbb{E}[B^2(t)]=t(1-t)\rightarrow 0$, which comes to our conclusion.

\paragraph{Problem 41} Let $X$ solve the Langevin equation, and suppose that $X_0$ is an $\mathcal N(0,\frac{\sigma^2}{2b})$ random variable. Show that
\[\mathbb{E}[X(s)X(t)]=\frac{\sigma^2}{2b}e^{-b|t-s|}.\]

\paragraph{Solution of Problem 41} According to the formulation of the solution for the Langevin equation, we have:
\[X(t) = e^{-bt}X_0+\sigma\int_0^t e^{-b(t-c)}dW(c),~~X(s) = e^{-bs}X_0+\sigma\int_0^s e^{-b(s-c)}dW(c).\]
Without loss of generality, we assume that $t>s$, then:
\[\mathbb{E}[X(t)X(s)]=e^{-b(t+s)}\mathbb{E}X_0^2 + \sigma^2\cdot\mathbb{E}\left[\int_0^t e^{-b(t-c)}dW(c)\cdot \int_0^s e^{-b(s-c)}dW(c)\right]\]
since the other two terms have expectation 0. Then:
\begin{equation*}
\begin{aligned}
&\mathbb{E}\left[\int_0^t e^{-b(t-c)}dW(c)\cdot \int_0^s e^{-b(s-c)}dW(c)\right] \\
&= \mathbb{E} \left[\int_0^s e^{-b(t-c)}dW(c)\cdot \int_0^s e^{-b(s-c)}dW(c) + \int_s^t e^{-b(t-c)}dW(c)\cdot \int_0^s e^{-b(s-c)}dW(c)\right]\\
&= \int_0^s e^{-b(t+s-2c)}dc + 0 = \frac{e^{-b(t-s)}-e^{-b(t+s)}}{2b}.
\end{aligned}
\end{equation*}
Also, $\mathbb{E}X_0^2 = \frac{\sigma^2}{2b}$. To sum up, we finally obtain that:
\[\mathbb{E}[X(s)X(t)]= \frac{\sigma^2}{2b}e^{-b(t-s)}= \frac{\sigma^2}{2b}e^{-b|t-s|}.\]

\paragraph{Problem 42} (1) Consider the ODE
\begin{equation*}
\begin{cases}
\dot{x} &= x^2~~~(t>0)\\
x(0) &= x_0.
\end{cases}
\end{equation*}
Show that if $x_0 > 0$, the solution "blows up to infinity" in finite time.

~\\
(2) Next, look at the ODE
\begin{equation*}
\begin{cases}
\dot{x} &= x^{1/2}~~~(t>0)\\
x(0) &= 0.
\end{cases}
\end{equation*}
Show that this problem has infinitely many nonnegative solutions. \\
(Hint: $x\equiv 0$ is a solution. Find also a solution which is positive for times $t>0$, and then combine these solutions to find ones which are zero for some time and then become positive.)

\paragraph{Solution of Problem 42} ~\\
(1) Notice that $\dot{x}=x^2\geqslant 0$, which means $x(t)$ is a monotonic function and therefore $x(t)\geqslant x(0)=x_0 > 0$ holds for all $t\geqslant 0$. Since:
\[\frac{dx}{dt}=x^2~~\Rightarrow~~\frac{dx}{x^2}=dt~~\int_{0}^T \frac{dx(t)}{x(t)^2}=T,\]
where we used $x(t)> 0$. Then we have:
\[\frac{1}{x_0}-\frac{1}{x(T)}=T~~\Rightarrow~~x(T)=\frac{x_0}{1-Tx_0}\]
holds for all $T > 0$. Therefore, this solution blows up to infinity at $T=\frac{1}{x_0}$, which is a finite time.

~\\
(2) Consider the following functions:
\[x_{s}(t)=\begin{cases}&0~~~\mbox{when $0\leqslant t \leqslant s$}\\ &\frac{(t-s)^2}{2}~~~\mbox{when $t>s$}\end{cases}.\]
Here $s > 0$. Then, we notice that for $\forall s >0$, function $x_s(t)$ satisfies the given ODE, and it is non-negative, which comes to our conclusion.

\paragraph{Problem 43} (1) Use the substitution $X=u(W)$ to solve the SDE
\[\begin{cases}dX&=-\frac12 e^{-2X}dt+e^{-X}dW\\ X(0)&=x_0\end{cases}.\]

~\\
(2) Show that the solution blows up at a finite, random time.

\paragraph{Solution of Problem 43} ~\\
(1) We use the substitution $X=u(W)$, then:
\[dX = u'(W(t))dW(t) + \frac12 u''(W(t))dt.\]
We need:
\[u' = e^{-u}, u''=-e^{-2u}, u(0)=x_0.\]
Then $u(t)=\ln(t+e^{x_0})$. To sum up, the solution of the SDE is:
\[X(t)=\ln(W(t)+e^{x_0}).\]

(2) The blowup time is the following stopping time $\tau = \min\{t : W(t) = -e^{x_0}\}$. For any positive integer $n$, define the event $E_n := \{\tau\geqslant n\}$. Then: by using Reflection Theorem, we know that:
\[\mathbb{P}\{E_n\} = \mathbb{P}\{|W(n)| < e^{x_0}\}=2\Phi\left(\frac{e^{x_0}}{\sqrt{n}}\right).\]
Also, it is obvious that $E_1 \supseteq E_2 \supseteq \ldots$, and when $n\rightarrow \infty$, we have:
\[\lim_{n\rightarrow\infty}\mathbb{P}\{E_n\}=0,\]
which means:
\[0=\mathbb{P}\left\{\lim_{n\rightarrow\infty}E_n\right\}=\mathbb{P}\{E\}\]
where $E:=\{\tau=+\infty\}$. It means the solution blows up at time $\tau$, which is a random time. With probability 1, this time is finite.

\paragraph{Problem 44} Solve the SDE $dX=-Xdt+e^{-t}dW$.

\paragraph{Solution of Problem 44} We use the substitution $X=e^{-t}W(t)+f(t)$, then:
\[dX = -e^{-t}W(t)dt+e^{-t}dW(t) + f'(t)dt = e^{-t}dW(t) + (-e^{-t}W(t)+f'(t))dt.\]
Now we just need $f(t)=-f'(t)~\Rightarrow~ f(t)=Ae^{-t}$. To sum up, the solution of the SDE is:
\[X = e^{-t}W(t)+Ae^{-t},\]
where $A=X(0)$ can be any real constant.

\paragraph{Problem 45} Let $\mathbf{W}=(W^1,W^2,\ldots, W^n)$ be an $n$-dimensional Brownian motion and write
\[R := |\mathbf{W}|=\left(\sum_{i=1}^{n}(W^i)^2\right)^{1/2}.\]
Show that $R$ solves the stochastic Bessel equation
\[dR=\frac{n-1}{2R}dt+\sum_{i=1}^{n}\frac{W^i}{R}dW^i.\]

\paragraph{Solution of Problem 45} Denote function  $F:\mathbb{R}^n\rightarrow\mathbb{R}$ as $F(x):= \|x\|_2$. Then:
\[F_k'(x)=\frac{x_k}{\|x\|_2},~~F_{kk}''(x)=\frac{\|x\|_2^2-x_k^2}{\|x\|_2^3}.\]
According to Itô's chain rule, it holds that:
\[dR = \frac{1}{R}\sum_{i=1}^{n} W^i dW^i + \frac{1}{2R^3}\sum_{i=1}^{n}(R^2-x_k^2) dt = \sum_{i=1}^{n}\frac{W^i}{R}dW^i + \frac{nR^2-R^2}{2R^3}dt = \frac{n-1}{2R}dt+\sum_{i=1}^{n}\frac{W^i}{R}dW^i,\]
which comes to our conclusion.

\paragraph{Problem 46} (1) Show that $\mathbf{X}=(\cos(W),\sin(W))$ solves the system of SDE
\[\begin{cases}dX^1&=-\frac12 X^1dt-X^2dW\\ dX^2&=-\frac12 X^2 dt+X^1 dW\end{cases}.\]
(2) Show also that if $\mathbf{X}=(X^1,X^2)$ is any other solution, then $|\mathbf{X}|$ is constant in time.

\paragraph{Solution of Problem 46} ~\\
(1) We only need to verify the SDE solution.
\[dX^1 = d\cos(W) = -\sin(W) dW -\frac12 \cos(W) dt = -X^2 dW -\frac12 X^1 dt.\]
\[dX^2 = d\sin(W) = \cos(W) dW -\frac12 \sin(W) dt = X^1 dW - \frac12 X^2 dt.\]
(2) Denote $F = (X^1)^2 + (X^2)^2$, then:
\begin{equation*}
\begin{aligned}
dF &= 2X^1 dX^1 + (X^2)^2 dt + 2X^2 dX^2 + (X^1)^2 dt\\
&= 2X^1(-X^2 dW -\frac12 X^1 dt) + 2X^2(X^1 dW - \frac12 X^2 dt) + Fdt \\
&= -Fdt + Fdt = 0
\end{aligned}
\end{equation*}
Therefore, $F=|\mathbf{X}|^2$ is constant in time, which leads to the fact that $|\mathbf{X}|$ is also constant in time.

\paragraph{Problem 47} Solve the system
\[\begin{cases} dX^1 &= dt+dW^1\\ dX^2 &= X^1 dW^2,\end{cases}\]
where $\mathbf{W}=(W^1, W^2)$ is a Brownian motion.

\paragraph{Solution of Problem 47} From the first equation, we easily know that
\[X^1 = t+W^1(t)+c_1\]
where $c_1$ can be any real constant. Then, $X^2$ satisfies:
\[dX^2 = (t+c_1+W^1(t))dW^2.\]
We can simply write down the following formulation:
\[X^2 = c_1W^2(t) + c_2 + \int_{0}^t (s+W^1(s))dW^2(s).\]
To sum up, the solution of the SDE is:
\begin{equation*}
\begin{cases}
X^1 &= t+W^1(t)+c_1\\
X^2 &= c_1 W^2(t) + c_2 + \int_0^t (s+W^1(s))dW^2(s).
\end{cases}
\end{equation*}

\paragraph{Problem 48} Solve
\begin{equation*}
\begin{cases}
dX^1 &= X^2 dt + dW^1\\
dX^2 &= X^1 dt + dW^2.
\end{cases}
\end{equation*}

\paragraph{Solution of Problem 48} Denote $W_1 := \frac{W^1+W^2}{\sqrt{2}}, W_2 := \frac{W^1-W^2}{\sqrt{2}}$. Also, we denote $X=X_1+X_2, Y=X_1-X_2$. Then, $\{W_1(t)\}, \{W_2(t)\}$ are Brownian motions. After adding the two equations above, we obtain that:
\[dX = Xdt + \sqrt{2}dW_1,\]
which is a Langevin equation. The solution is:
\[X = e^t\left(c_1 + \sqrt{2}\int_0^t e^{-s}dW_1(s)\right)= e^t\left(c_1 + \int_0^t e^{-s}dW^1(s)+\int_0^t e^{-s}dW^2(s)\right).\]
Similarly, after subtracting the two equations above, we obtain that:
\[dY = -Ydt + \sqrt{2}dW_2,\]
which is also a Langevin equation. The solution is:
\[Y = e^{-t}\left(c_2 + \sqrt{2}\int_0^t e^{s}dW_2(s)\right)= e^{-t}\left(c_2 + \int_0^t e^{s}dW^1(s)-\int_0^t e^{s}dW^2(s)\right). \]
Therefore, the solution of this SDE is:
\[\begin{cases}
X^1 &= \frac{c_1 e^t + c_2 e^{-t}}{2} + \int_0^t \frac{e^{t-s}+e^{s-t}}{2}dW^1(s) + \int_0^t \frac{e^{t-s}-e^{s-t}}{2}dW^2(s)\\
X^2 &= \frac{c_1 e^t - c_2 e^{-t}}{2} + \int_0^t \frac{e^{t-s}-e^{s-t}}{2}dW^1(s) + \int_0^t \frac{e^{t-s}+e^{s-t}}{2}dW^2(s).
\end{cases}\]

\paragraph{Problem 49} Solve
\[\begin{cases}
dX &= \frac12 \sigma'(X)\sigma(X)dt + \sigma(X) dW\\
X(0) &= 0
\end{cases}\]
where $W$ is a one-dimensional Brownian motion and $\sigma$ is a smooth, positive function.\\
(Hint: Let $f(x):=\int_0^x \frac{dy}{\sigma(y)}$ and set $g:=f^{-1}$, the inverse function of $f$. Show that $X=g(W)$.)

\paragraph{Solution of Problem 49} Denote function $f(x):=\int_0^x \frac{dy}{\sigma(y)}$ and $Y = f(X)$. Then $Y(0)=0$ and
\begin{equation*}
\begin{aligned}
dY &= f'(X) dX + \frac12 f''(X)\cdot\sigma^2(X) dt = \frac{1}{\sigma(X)}dX -\frac{\sigma'(X)}{2\sigma^2(X)}\cdot  \sigma^2(X) dt\\
&= \frac12 \sigma'(X)dt + dW - \frac12 \sigma'(X)dt = dW.
\end{aligned}
\end{equation*}
Since $Y(0)=0$, we have: $Y(t)=W(t)$. Therefore, the solution of the SDE is:
\[X = f^{-1}(Y)=g(W).\]

\paragraph{Problem 50} Let $\tau$ be the first time a one-dimensional Brownian motion hits the half-open interval $(a,b]$. Show that $\tau$ is a stopping time.

\paragraph{Solution of Problem 50} We are going to prove $\tau$ is a stopping time by the definition. When $a<0\leqslant b$, we have $\tau=0$, which means for $\forall t \geqslant 0$, the event $\{\tau\leqslant t\}$ is always true. When $a\leqslant 0$, it holds that:
\[\{\tau> t\} = \{\forall s\leqslant t, W(s)\leqslant a\}\in \mathcal F(t).\]
When $b>0$, it holds that:
\[\{\tau> t\} = \{\forall s\leqslant t, W(s)>b\}\in \mathcal F(t).\]
To sum up, $\tau$ is a stopping time, which comes to our conclusion.

\paragraph{Problem 51} Let $\mathbf{W}$ denote an $n$-dimensional Brownian motion for $n\geqslant 3$. Write $\mathbf{X}=\mathbf{W}+x_0$, where the point $x_0$ lies in the region $U=\{0<R_1<|x|<R_2\}$. Calculate explicitly the probability that $\mathbf{X}$ will hit the outer sphere $\{|x|=R_2\}$ before hitting the inner sphere $\{|x|=R_1\}$. \\
(Hint: Check that
\[\Phi(x)=\frac{1}{|x|^{n-2}}\]
satisfies $\Delta\Phi=0$ for $x\neq 0$. Modify $\Phi$ to build a function $u$ which equals 0 on the inner sphere and 1 on the outer sphere.

\paragraph{Solution of Problem 51} Suppose the probability function we want is $u(x)$ where $R_1<|x|<R_2$. Then the function $u$ satisfies the following conditions.
\begin{equation*}
\begin{cases}
\nabla u &= 0\\
u(x) &= 0 \mbox{$|x|=R_1$}\\
u(x) &= 1 \mbox{$|x|=R_2$}
\end{cases}
\end{equation*}
As we see, for $\Phi(x)=\frac{1}{|x|^{n-2}}$, we have:
\[\frac{\partial}{\partial x_i}\Phi(x) =\sum_{i=1}^{n}-\frac{(n-2)x_i}{|x|^n},~\frac{\partial^2}{\partial x_i^2}\Phi(x) =\sum_{i=1}^{n}-\frac{(n-2)}{|x|^n}+\frac{n(n-2)x_i^2}{|x|^{n+2}}.\]
Therefore:
\[\Delta\Phi = \sum_{i=1}^{n}-\frac{(n-2)}{|x|^n}+\frac{n(n-2)x_i^2}{|x|^{n+2}} = -\frac{n(n-2)}{|x|^n}+\frac{n(n+2)}{|x|^n}=0.\]
So, for any constant $a,b$, we have $\Delta(a\Phi+b)=0$. Let
\[a = -\frac{R_1^{n-2}R_2^{n-2}}{R_2^{n-2}-R_1^{n-2}},~b=\frac{R_2^{n-2}}{R_2^{n-2}-R_1^{n-2}}.\]
Then, function $a\Phi+b$ meets our satisfaction. To sum up, the probability function we need is:
\[u(x)=\frac{R_2^{n-2}}{R_2^{n-2}-R_1^{n-2}}\cdot\left(1-\frac{R_1^{n-2}}{|x|^{n-2}}\right).\]

\end{document}


\chapter{Background}
\label{chapter:overview}

We define term rewriting systems and give some background. We briefly describe Halide and the Halide expression language we will be operating on throughout this work. We then describe the uses of term rewriting systems within the Halide compiler, including the term rewriting algorithm used by the compiler, the requirements that led to its design choices, and the two term rewriting systems we use as case studies in this work, the simplifier and the variable solver.

\section{Term Rewriting Systems}

Term rewriting systems~\citep{gorn1967} are sets of \textit{rewrite rules} used to transform expressions into a new form.  Such systems are widely
used in theorem proving~\citep{baader1999term} and abstract interpretation~\citep{cousot1977abstract, cousot1979systematic}.

Terms are defined inductively over a set of variables $V$ and a set of function symbols $\Sigma$. Every variable $v \in V$ is a term, and for any function symbol $f \in \Sigma$ with arity $n$ and any terms $t_1, ..., t_n$, the application of the symbol to the terms $f(t_1, ..., t_n)$ is also a term. (Constants are considered zero-arity functions.) We refer to the set of terms constructed from the variables $V$ and the function symbols $\Sigma$ as $T(\Sigma, V)$.

A \emph{rewrite rule} is a directed binary relation $l \rewrites r$ such that $l$ is not a variable, and all variables present in $r$ are also present in $l$ (i.e., $\mathcal{V}ar(l) \supseteq \mathcal{V}ar(r)$). A set of rewrite rules is called a \emph{term rewriting system}.

Consider a set of terms $T(\Sigma, V)$ such that $\Sigma = \{\clubsuit, \diamondsuit\}$ and $V$ is an infinite set of variables.
\newpage
Let the term rewriting system $R$ consist of a single rule:

\[ R = \{ x_1 \clubsuit x_2 \rewrites x_1 \diamondsuit x_2 \} \]
We use $R$ to rewrite the term

\[
(y_1 \diamondsuit y_1) \clubsuit (y_2 \clubsuit y_3)
\]
The first step is matching; we find a substitution that will unify the left-hand side (LHS) of the rule with the term we are rewriting. Here, one possible substitution is:

\[
\{ x_1 \mapsto (y_1 \diamondsuit y_1), x_2 \mapsto (y_2 \clubsuit y_3) \}
\]
We then apply this substitution to the right-hand side (RHS) of the rule to obtain the rewritten version of the original term:

\[
(y_1 \diamondsuit y_1) \diamondsuit (y_2 \clubsuit y_3)
\]

\section{Halide and the Halide expression language}

The Halide compiler contains term rewriting systems that manipulate terms in the Halide expression language. This language
operates over vectors and scalars of integers, booleans, and real values.  However, in
this work we concentrate on the TRS as it applies to integer and boolean values, for
both vectors and scalars, because the most important uses of the TRS within the compiler
apply to these types.
The Halide expression language contains the usual arithmetic operators and uses the Euclidean definition for division and modulo. It also contains the boolean operators and, or, and not, as well as the usual comparators. The \texttt{select} operator takes three arguments and behaves like an if-then-else statement, returning the second argument if its first evaluates to true and the third argument otherwise. There are two operators that can create vectors from scalars: $\texttt{broadcast}(v, s)$, which creates a vector of size $s$ with each value initialized to $v$, and $\texttt{ramp}(v, s, k)$, which creates a vector of size $s$ where the first value is initialized to $v$ and each subsequent value is increased by the stride $k$. Most operators that take integer or boolean arguments can also take vector arguments whose values are integer or boolean type respectively. For the full Halide expression grammar, see appendix~\ref{a:grammar}.

\section{Term rewriting systems in the Halide compiler}

\begin{figure*}
\centering
\begin{tabular}{cccc}

%\Tree [.+ [.- [.min a b ] [.max c c ] ] [.max c c ]]
\begin{tikzpicture}[level distance=12mm]
\tikzstyle{level 1}=[sibling distance=15mm]
\tikzstyle{level 2}=[sibling distance=10mm]
\tikzstyle{level 3}=[level distance=10mm,sibling distance=5mm]

%\Tree [.+ [.- [.min a b ] c ] [.max c c ]]
\node (+) {+}
  child { node (-) {-}
    child { node (min) {\hmin}
      child {node (a) {a}}
      child {node (b) {b}}
    }
    child { node (max2) {\hmax}
      child {node (c4) {c}}
      child {node (c5) {c}}
    }
  }
  child { node (max) {\hmax}
    child { node (c2) {c}}
    child { node (c3) {c}}
  };
\begin{pgfonlayer}{background}
\fill[red,opacity=0.3] \convexpath{c4,max2,c5}{10pt};
\end{pgfonlayer}
\end{tikzpicture}

&
\begin{tikzpicture}[level distance=12mm]
\tikzstyle{level 1}=[sibling distance=15mm]
\tikzstyle{level 2}=[sibling distance=7mm]
\tikzstyle{level 3}=[level distance=10mm,sibling distance=5mm]
%\Tree [.+ [.- [.min a b ] c ] [.max c c ]]
\node (+) {+}
  child {  node (-) {-}
    child { node (min) {\hmin}
      child {node (a) {a}}
      child {node (b) {b}}
    }
    child { node (c) {c}}
  }
  child { node (max) {\hmax}
    child { node (c2) {c}}
    child { node (c3) {c}}
  };
\begin{pgfonlayer}{background}

\fill[blue,opacity=0.3] \convexpath{c2,max,c3}{10pt};
%\fill[red,opacity=0.3] \convexpath{c}{10pt};
\draw[fill=red,opacity=0.3,draw=none](c) circle (10pt);

\end{pgfonlayer}
\end{tikzpicture}
&
\begin{tikzpicture}[level distance=12mm]
\tikzstyle{level 1}=[sibling distance=15mm]
\tikzstyle{level 2}=[sibling distance=10mm]
\tikzstyle{level 3}=[level distance=10mm,sibling distance=5mm]
%\Tree [.+ [.- [.min a b ] c ] c ]
\node (+) {+}
  child { node (-) {-}
    child { node (min) {\hmin}
      child {node (a) {a}}
      child {node (b) {b}}
    }
    child { node (c) {c} }
  }
  child { node (c2) {c}};

  \begin{pgfonlayer}{background}
\fill[green,opacity=0.3] \convexpath{b,a,min,-,+,c2,c}{10pt};
\draw[fill=blue,opacity=0.3,draw=none](c2) circle (10pt);
%\draw[red,fill=blue,opacity=0.3](c2.north) to[closed,curve through={($(c2.north east)!1.0!(c2.south east)$) .. ($(c2.south west)!1.0!(c2.north west)$)}] (c2.north);
\end{pgfonlayer}
\end{tikzpicture}
&
\vspace{0pt}
\begin{tikzpicture}[level distance=12mm]
\tikzstyle{level 1}=[sibling distance=15mm]
\tikzstyle{level 2}=[sibling distance=10mm]
\tikzstyle{level 3}=[level distance=10mm,sibling distance=5mm]
\node (min) {\hmin}
  child { node (a) {a}
     % [red,opacity=0.0]
    child {     [red,opacity=0.0] node (fake1) {f}
      child {    [red,opacity=0.0] node (fake2) {f}}
      child {    [red,opacity=0.0] node (fake3) {f}}
    }
    child {     [red,opacity=0.0] node (fake4) {f}
    [red,opacity=0.0]
    child { [red,opacity=0.0] node (fake) {f}}
    child { [red,opacity=0.0] node (fake2) {f}}
    }
  }
  child { node (b) {b}
     % [red,opacity=0.0]
    child {     [red,opacity=0.0] node (fake5) {f}}
    child {     [red,opacity=0.0] node (fake6) {f}}
  };
  \begin{pgfonlayer}{background}
\fill[green,opacity=0.3] \convexpath{a,min,b}{10pt};
\end{pgfonlayer}

\end{tikzpicture} \\
(i) & (ii) & (iii) & (iv)
\end{tabular}
\caption{We demonstrate the Halide rewriting algorithm using a TRS $R = \{\hmax(x, x) \rewrites x, (x - y) + y \rewrites x\}$ and an expression $\hmin(a,b) - \hmax(c,c) + \hmax(c,c)$. The algorithm attempts to simplify all subtrees bottom up; here, no rule applies to $\hmin(a,b)$ so it is not changed. Next (i), rule 1 rewrites $\hmax(c,c)$ to $c$. No rule applies to $\hmin(a,b) - c$, so we move to the rightmost subtree and rewrite again (ii) to obtain $c$ from $\hmax(c,c)$. Finally, we consider the entire tree $\hmin(a,b) - c) + c$ (iii) and apply rule 2 to produce $\hmin(a,b)$. No rules match this expression, so we are left with $\hmin(a,b)$ (iv).}
\label{fig:algoexample}
\end{figure*}

\subsection{The Halide rewriting algorithm}
\label{sec:customalgo}

A term rewriting system is simply a set of rewrite rules; when we use a TRS to transform an expression, we also need to define a procedure for
applying those rewrite rules to the expression.  In a TRS, a single rule may be able to match an input expression in
multiple ways, and there may be multiple rules in the ruleset which could be used
to rewrite the expression. A term rewriting algorithm might choose one of many alternatives
and later backtrack if it turns out not to be fruitful; it might make use of
heuristics to choose a next step; it might exercise all the alternatives and keep
the results in equivalence classes, as in an e-graph. The Halide term rewriting algorithm
keeps only one expression in state and applies rules greedily, in a fixed priority.
This is very fast and requires very little memory; the tradeoff is that the algorithm
may pick the ``wrong'' rule and have no way of undoing that decision.
Since the rewriter is invoked thousands of times with each call of the compiler,
it chooses to sacrifice some solving power in exchange for performance.

The Halide term rewriting algorithm simplifies an input expression in a
depth-first, bottom-up traversal of the expression AST. At each node, it
uses the root node to pick a list of rules, then
attempts to match the subtree expression with the rule LHSs in a fixed priority. Matching
is performed purely syntactically, using C++ template metaprogramming.
 Halide rewrite rules contain special metavariables,
called \emph{symbolic constants}, that can match only with constant values; all other
variables can match any subterm as usual.
When a match is found, the algorithm rewrites the
subtree expression using the RHS of that rule, and then recurses and starts applying rewrites on the newly rewritten expression. If no rule matches the subtree, the traversal
continues; when the entire expression cannot be simplified further, the
rewritten expression is returned. See Figure~\ref{fig:algoexample} for a worked
example.

The rewrite rules optionally contain a predicate guard.
These guards contain only special variables whose values are known at the time of rule application\footnote{Existing
rules sometimes have predicates that check if
  non-constant variables can be shown to have certain properties at compile
  time, but these are expensive and used sparingly.}; when the LHS of a rule
matches an expression, its guard is evaluated and only if it
is true will the rewrite be applied.

Halide rewrite rules are applied in a fixed priority, organized so that the TRS
first attempts very basic rules such as constant folding, then tries more specific
rules before more general ones. (We do not evaluate the current rule priority in this work.)

Associativity and commutativity laws are particularly troublesome for term rewriting systems.
For one expression $e$, the number of semantically equivalent expressions grows
exponentially in terms of the number of AC operations $e$ contains. Some term rewriting
systems perform a full \emph{AC matching} step during rewriting. Halide's TRS does not
perform this matching, but instead
includes multiple AC variations of rules.
However, a small number of Halide's rewrite rules have the effect of canonicalizing some commutative
expressions. (For example, if a commutative expression has a multiplication as its
first operand and a subtraction as its second operand, a rule will switch their positions.)
These rules are all early in the application priority, so later rules can rely on
expressions having a quasi-canonical form.

\subsection{The rewriting algorithm's design requirements}

Because a reasoning engine implemented as a term rewriting system may be invoked thousands of times in a single compilation, the rewriting algorithm outlined above was designed to be fast and use very little memory. For that reason the algorithm is non-backtracking and keeps only one expression in state, sacrificing some potential solving power for performance.
In this work, we operate within the scope of Halide's TRS algorithm
  and work to make the TRSs used by that algorithm as correct, general, and robust as possible.  Because
  the space of expressions we consider constitutes an undecidable theory, any complete
  TRS is impossible.  Instead, we strive to improve correctness by ensuring the TRSs
  will always terminate on any expression and that each individual rewrite
  preserves semantics; and we improve generality by expanding the rulesets to
  contain rewrites that apply to real-world expressions, rather than
  arbitrary new rules that may not apply to any expressions the compiler will encounter.

These improvements require overcoming challenging obstacles.  First,
  we must perform a post-hoc verification of a large body of existing rules;
  proving a subset of rules correct or that a subset of rules do not result in infinite
  rewriting loops is insufficient to guarantee robust behavior.  Secondly,
  these rewrites operate in an undecidable theory, making automated verification
  difficult.  Finally, because of this undecidability, we cannot necessarily
  rely on traditional automated techniques to discover new rules.

\begin{table}
\caption{We compare the performance of Z3 and the Halide TRS in proving a set of 4304 expressions gathered from realistic compiler output. Note that expressions in the ``not proven'' column include expressions that are true but not found to be so by the solvers as well as expressions that are not true. }
\centering
\begin{tabular}{l|r|r|r}
Tool & Runtime & Proven expressions & Not proven \\
\hline
Z3 & 7m29s & 1125 & 3179 \\
Halide TRS & 2s & 885 & 3419
\end{tabular}
\label{tab:simplifiervsz3}
\end{table}

Given that we make use of the Z3 solver~\citep{de2008z3} for both verification and synthesis, it is natural to ask why Halide could not simply call Z3 as its reasoning engine. Z3 is the product of extensive development and is a very powerful, general-purpose solver. However, the Halide term rewriting system has a few key properties that Z3 does not: deterministic output, low memory and compute requirements, and domain-specific optimizations.

As discussed above, the Halide compiler must return the same schedule every time the same pipeline is run. Z3 can fix a random seed, but long-running queries may complete on a more powerful server while timing out on a different machine.

While the Halide algorithm is less powerful than Z3, its deterministic, greedy rule application strategy
gives it a smooth performance curve, whether it succeeds or fails in simplifying an input expression.
A solver like Z3 tends to give very good performance most of the time but gets bogged down in difficult cases, requiring the use of timeouts. The Halide algorithm ``fails fast'': on an input expression which does not match any rule,  the Halide algorithm will complete in time linear to the size of the expression, taking on the order of one CPU cycle per term in the expression per rule in the TRS. To demonstrate this performance tradeoff, we gathered 4304 expressions from queries the Halide compiler made when compiling realistic pipelines, including both provably true expressions and expressions that are not provably true. Z3 could prove approximately 30\% more expressions true (within a 60 second timeout), but was starkly less performant. As shown in Table~\ref{tab:simplifiervsz3}, Z3 took over 7 minutes to check the set of expressions while the Halide TRS took just 2 seconds. This set of expressions is much smaller than the number of calls the compiler makes to the rewriter in compiling a single pipeline.

Because the Halide algorithm at every step chooses one rule to apply to the single expression it is working on, it scales well in terms of the number of rules in the TRS. See Section~\ref{ssec:compilationspeed} for an evaluation of the effects of adding newly synthesized rules on the performance of the compiler.

Finally, Z3 is a very powerful general-purpose solver, but Halide TRSs can be targeted for their specific use cases. For example, when rewriting an expression to be shorter, gathering like terms in some cases can actually prevent Halide or LLVM optimizations from applying. The Halide TRSs use domain-specific strategies to guide expressions into more optimizable forms and can be changed or tuned as needed if further optimizations are discovered.

\subsection{The Simplifier}
\label{sec:uses-of-trs}

To compile an image processing pipeline written in the
  Halide language, the compiler must perform a variety of analyses of
  the pipeline's properties. For example, if the user marks a
  loop to be fully unrolled, the compiler must infer a constant upper
  bound for the extent of the loop. If the user marks
  a loop as parallel, the compiler must prove the absence of data
  races. These analyses also affect performance more than in most
  compilers. In Halide, the compiler infers loop bounds and allocation sizes.
  If these are overestimated, the generated code may
  perform an amount of wasted work sufficient to alter the
  computational complexity of the algorithm. These analyses all depend
  critically on the quality of Halide's expression simplifier. In
  fact, Halide relies so heavily on its simplifier that restricting it
  to mere constant-folding causes a geomean 5.1$\times$ increase in
  compilation times and a 26.4$\times$ increase in runtimes across
  Halide's benchmark suite.

While the Halide compiler makes use of the TRS in numerous ways, the most important
  applications of the TRS are its uses as a fast simplifier and as a proof engine.
  In many parts of the compiler, the TRS is used to rewrite expressions into simpler forms,
  which are easier for the compiler to reason about, and result in less code being generated for
  LLVM to consume at the backend.  Most importantly, the compiler uses the TRS to simplify expressions
  into constants or expressions that are monotonic with respect to loop bounds; these simplifications are core to Halide's
  ability to generate drastically different loop nests for different schedules.

For example, consider the simple two-stage imaging pipeline $g(x) = f(x - 1) + f(x) + f(x + 1)$.
  Halide enables programmers to fuse the computation of $f$ into $g$ at an arbitrary granularity
  using the \texttt{compute\_at} scheduling directive.  This requires Halide to automatically reason
  about which region of $f$ is required for a specific sub-region (or tile) of $g$, using interval
  arithmetic over symbolic values for the size of a tile of $g$.  For a tile size of 8, a tile of $g$
  is the region \texttt{[g.tile\_min, g.tile\_min+7]};  the region of $f$ required is
  \texttt{[g.tile\_min-1, g.tile\_min+8]}; and the number of values of $f$ to compute is then
  \texttt{g.tile\_min + 8 + 1 - (g.tile\_min - 1)}.  If the TRS can determine this is a static value
  of 10, the Halide compiler can then safely perform transformations requested by the user.  In this
  case, the compiler can use stack memory instead of inserting a dynamic allocation; or the loop can be
  completely unrolled; the loop can be vectorized; or $f$ can be mapped to GPU threads (since a single
  threadblock must have a compile-time-known size).  More generally, this kind of region analysis
  operates most effectively when the expressions are monotonic in the loop bounds; otherwise, interval
  arithmetic can result in vast overestimates of required regions.  These simplifications are essential
  for the compiler to work, and are usually not as simple as this example.

The rules for simplifying to perform cancellations and
  ensure monotonicity are incredibly important for compiler
  performance. When we disabled all but the constant-folding rules to
  measure the importance of the simplifier, it was the absence of
  these specific rules that caused the (26.4$\times$) slow-down
  mentioned in Section~\ref{sec:intro}. Without these rules,
  Halide is useless for high-performance image
  processing.

%% Fodder from synthesis section:
%% For example, Halide relies on symbolic interval arithmetic to determine how much memory
%% to allocate and how many values to compute for each stage.  Symbolic interval arithmetic
%% is exact when an expression monotonically increases or decreases over a loop.}
%% For example, if $x \in [0, 100]$, then symbolic interval arithmetic states that
%% $\hmax(x, x/2 + 20) \in [20, 100]$, which is the tightest correct
%% bound; this bound is obtained by substituting in the lower and upper bounds of $x$
%% into the expression. However, in the presence of anti-correlated subexpressions
%% interval arithmetic becomes inexact, and is prone to overestimating
%% bounds. The expression $\hmin(x, 100 - x)$ when $x \in [0, 100]$ is bounded above by 50, but
%% symbolic interval arithmetic makes the weaker claim that it is bounded
%% above by 100, by setting the first instance of $x$ to 100 and the
%% second instance of $x$ to zero.

The use as a proof engine occurs when the compiler must prove properties about the code in order to guarantee the
  correctness of specific transformations or the relationships between bounds of
  different loops or producer-consumer relationships.  In such cases, the compiler constructs
  an expression that must be true (or false) in order to guarantee correctness, then applies
  the TRS to see if the expression simplifies to a single constant boolean value.

For example, Halide uses Euclidean division, which rounds according to the sign of the
  denominator.  Lowering this to code requires emitting several instructions, which can be
  slower than native division.  When the compiler can statically prove the signs of the numerator
  and denominator, in some cases the code can be replaced by native division or even a different
  instruction altogether.  For example, for an expression \texttt{x / max(y, 1)} the compiler
  will try to prove $0 < \hmax(y, 1)$.  The TRS first invokes a rule to transform this to
  $0 < y\; ||\; 0 < 1$, which then is transformed to true (since the second clause is always true).
  Thus, the compiler is able to replace Euclidean division with machine division.

Simplifier failures have adverse results on the compiler, making it unpredictable and
  thus difficult for Halide program authors to reason about the performance of their schedules.
  When the TRS fails to properly simplify an expression or
  prove a property, the consequences include:
\begin{itemize}
\item Insufficiently tight bounds on loops and allocations, which may result in
  runtime failures (e.g. due to memory overallocation) or performance issues;

\item Failure of the compiler to apply optimizations, also resulting in slow performance;

\item Dynamic checks in the generated code for properties that could have been proven
  at compile time, leading to slower code;

\item Compilation failures, when the compiler is unable to correctly produce code
  even though the properties required hold, or when the proof engine itself crashes
  or loops infinitely.
\end{itemize}

Thus, correctness and generality of the simplifier are essential qualities for making the compiler
robust and able to generate fast code.

\subsection{The Variable Solver}
\label{sec:thevariablesolver}

The Halide compiler contains a $\texttt{SolveExpression}$ class that implements functionality that we will refer to as a variable solver in this work. It takes an expression and a variable name as inputs and `solves' the expression for that variable, isolating it on the left of an expression.

$\texttt{SolveExpression}$ currently rewrites expressions using a recursive visitor that ``matches'' the expression using if statements. We have translated this into a term rewriting system that contains about a hundred rules. This system is much smaller and less mature than the simplifier, and using synthesis to author new rulesets will potentially result in much bigger performance gains than we saw with the more highly optimized simplifier.

We formally define the purpose of the variable solver as follows. Let $|t|_x$ represent the number of occurrences of the variable $x$ in the term $t$. We use the special variable $x^t$ to stand for the target variable, or the variable that the TRS is 'solving' for.

We say that a term $t$ is in \emph{solved form} if it is in the form:
\begin{enumerate}
  \item $|t|_{x^t} = 0$ (the term $t$ does not contain the target variable $x^t$). If we took the term $(x^t - x^t) + y$ and rewrote it to $y$, $y$ would be in solved form, as it does not contain any instances of $x^t$.
  \item $t = x^t$ (the term $t$ is precisely the target variable $x^t$). If we took the term $x^t + (y * (z - z))$ and rewrote it to $x^t$, $x^t$ would then be in solved form, as it consists of the target variable alone.
  \item $t = x^t \odot t'$, where $\odot$ is any binary operator in the Halide expression language, and $|t'|_{x^t} = 0$. Terms in this form are only considered `solved' if no term $u$ in either of the above two forms exist such that $t =_e u$. If we took the term $(y + x^t) + z$ and rewrote it to $x^t + (y + z)$, it would then be in solved form. (Note that $x^t + (z + y)$ would also in solved form.) However, $x^t + (y - y)$ would not be in solved form, since a semantically equivalent term $x^t$ is in the second form.
\end{enumerate}

The variable solver in the Halide compiler is used to reason about the bounds over variables and about their dependencies. As a worked example, let's consider the TrimNoOps stage of the compiler, in which the compiler tries to identify regions of loops in which no work is performed and thus those loop iterations can be skipped completely.

%\begin{itemize}
%	\item \textbf{Trimming no-ops} The variable solver is used in analyzing the conditions under which work is performed within nested for-loops. For example, imagine that the body of an inner for-loop over some variable $x$ only performs an operation when the condition of some if-statement is true. If the compiler can show that this condition will only hold for a limited range of the values for $x$, then it can skip the regions where only a no-op will be performed by truncating the loop. Isolating $x$ as much as possible is helpful in performing this analysis.
%	\item \textbf{Breaking dependencies in GPU code} By `solving' expressions within functions for relevant variables, the variable solver can remove spurious dependencies and allow the compiler to place computations over GPU code more efficiently.
%	\item \textbf{Proving associativity} If the operations within a function can be shown to be associative (for example, a sequence of adds and multiplies), the compiler can rearrange the associative components in whatever order will be most efficient. `Solving' such an expression for each variable in turn has the effect of normalizing it, making associativity easier to prove.
%\end{itemize}

Consider a simple Halide function over the variables $x$ and $y$. Whenever $2 \cdot x$ is greater than $y$, the function returns 5, otherwise it returns nothing. The function is then tiled using a 4 by 4 tile size.

\begin{verbatim}
	Func F;
	Var x, y;
	f(x, y) = select(2 * x < y, 5, undef<int>());

	Var xi, yi;
	f.tile(x, y, xi, yi, 4, 4);
\end{verbatim}

The code is lowered to a set of nested for loops. The variables $x$ and $y$ now represent the number of tiles in their respective dimensions; at each iteration, they calculate the starting point for the next tile as the variables $yi\_base$ and $xi\_base$. The variables $xi$ and $yi$ then iterate over each point in the current tile.

\begin{verbatim}
for(y=0; y < (y_extent+3)/4; y++) {
    yi_base = min(y * 4, y_extent - 4);
    for(x=0; x < (x_extent+3)/4; x++) {
        xi_base = min(x * 4, x_extent - 4);
        for(yi=0; yi<4; yi++) {
            for(xi=0; xi<4; xi++) {
                if (xi_base + xi)*2 < (yi_base + yi) {
                    f[idx] = 5;
                }
            }
        }
    }
}
\end{verbatim}

Note that the body of the innermost for loop is an if statement. If the condition of the if statement is true, the loop will set the current location of the output (here idealized as $idx$) to the value 5. If the condition of the if statement is not true, then the loop will do nothing. Thus, if we can identify regions of the innermost loop where the condition of the if statement can never be true, we can optimize by skipping those regions entirely.

To see if we can identify such a region, we take the condition of the if statement:

\begin{verbatim}
	(xi_base + xi) * 2 < (yi_base + yi)
\end{verbatim}

and use the variable solver to `solve' for the variable $xi$.

\begin{verbatim}
	xi <= ((yi_base + yi) - (xi_base* 2) - 1)/2
\end{verbatim}

Here the variable solver is successful in rewriting the condition in terms of $xi$. Furthermore, we are able to state the expression as an upper bound of $xi$. We can thus calculate the maximum value of $xi$ for which the inner loop will perform any work.

\begin{verbatim}
	xi_new_max = ((yi_base + yi) - (xi_base* 2) - 1)/2
\end{verbatim}

We can then use this new maximum as a new extent for the loop, allowing us to skip loop iterations where no work will be performed:

\begin{verbatim}
    for(xi=0; xi<(xi_new_max + 1); xi++) {
        if (xi_base + xi)*2 < (yi_base + yi) {
            f[idx] = 5;
        }
    }
\end{verbatim}

The variable solver is used in various other places in the compiler as well. For example, by `solving' expressions within functions for relevant variables, the variable solver can remove spurious dependencies and allow the compiler to place computations over GPU code more efficiently. The variable solver is also used to attempt to prove associativity of functions. If the operations within a function can be shown to be associative (for example, a sequence of adds and multiplies), the compiler can rearrange the associative components in whatever order will be most efficient. `Solving' such an expression for each variable in turn has the effect of normalizing it, making associativity easier to prove.


\section{Integral Theorems}
\subsection{Green's Theorem}
\begin{proposition}[Green's Theorem]
    For continuously differentiable functions $P=P(x,y),Q=Q(x,y)$ and a bounded region $A\subset\mathbb R^2$ with piecewise smooth boundary $\partial A$, we have
    $$\oint_{\partial A}P\,\mathrm dx+Q\,\mathrm dy=\iint_A\left( \frac{\partial Q}{\partial x}-\frac{\partial P}{\partial y} \right)\,\mathrm dx\,\mathrm dy$$
    where the direction of $\partial A$ is taken such that the region on the left of motion.
\end{proposition}
Note that the choice of direction is consistent with the convention we used for surfaces in $\mathbb R^3$ if we consider the normal to be pointing out of paper.
We shall prove the case where $A$ is rectangular, i.e. $A=\{(x,y):x\in [a,b],y\in[c,d]\}$.
\begin{proof}[Proof of Rectangular Case]
    \begin{align*}
        \iint_A\left( \frac{\partial Q}{\partial x}-\frac{\partial P}{\partial y} \right)\,\mathrm dx\,\mathrm dy
        &=\int_c^d\int_a^b\frac{\partial Q}{\partial x}\,\mathrm dx\,\mathrm dy-\int_a^b\int_c^d\frac{\partial P}{\partial y}\,\mathrm dy\,\mathrm dx\\
        &=\int_c^dQ(b,y)-Q(a,y)\,\mathrm dy-\int_a^bP(x,d)-P(x,c)\,\mathrm dx\\
        &=\oint_{\partial A}P\,\mathrm dx+Q\,\mathrm dy
    \end{align*}
    As desired.
\end{proof}
The general case can be thought of gluing many rectangles together.
\begin{example}
    Suppose $Q=x/2,P=-y/2$, then
    $$\oint_{\partial A}P\,\mathrm dx+Q\,\mathrm dy=\iint_A\,\mathrm dx\,\mathrm dy=\operatorname{Area}(A)$$
    Let $A$ be the ellipse $x^2/a^2+y^2/b^2\le1$, which by integrating the line integral on the left, we get $\operatorname{Area}(A)=\pi ab$.
\end{example}
\subsection{Stokes' Theorem}
\begin{proposition}
    For a continuously differentiable vector field $\underline{F}$ and any orientable surface $S$ with piecewise smooth boundary, then
    $$\int_S\nabla\times \underline{F}\cdot\mathrm d\underline{S}=\oint_{\partial S}\underline{F}\cdot\mathrm d\underline{x}$$
\end{proposition}
The orientability is important since we will need a consistent choice of normal on $S$ that varies smoothly from point to point.
So surfaces can be said to have two sides, the inside and outside.
An example of a non-orientable surface is the Mobius strip.
\begin{example}
    Consider a spherical cap
    $$S=\{\underline{x}=(\cos\phi\sin\theta,\sin\phi\sin\theta,\cos\theta)^\top:\phi\in[0,2\pi],\theta\in[0,\alpha]\}$$
    Let $F(\underline{x})=(-x^2y,0,0)^\top$, so $\nabla\times\underline{F}=(0,0,x^2)^\top$.
    Now $\mathrm d\underline{S}=\underline{e_r}\sin\theta\,\mathrm d\theta\mathrm d\phi$.
    So
    $$\int_S\nabla\times\underline{F}\cdot\mathrm d\underline{S}=\int_0^\alpha\int_0^{2\pi}(\cos\phi\sin\theta)^2\cos\theta\sin\theta\,\mathrm d\phi\mathrm d\theta=\frac{\pi}{4}\sin^4\alpha$$
    Now $\partial S:[0,2\pi]\ni t\mapsto (\cos t\sin\alpha,\sin t\sin\alpha,\cos\alpha)^\top$, we can calculate to find
    $$\oint_{\partial S}\underline{F}\cdot\mathrm d\underline{x}=\frac{\pi}{4}\sin^4\alpha$$
    Which is equal to the original value.
\end{example}
\begin{example}
    If $S$ is a closed surface, then its boundary is $0$, hence by Stokes' Theorem,
    $$\int_S\nabla \times\underline{F}\cdot\mathrm d\underline{S}=0$$
    which just looks like what we get when we integrate a closed loop.
\end{example}
\begin{proposition}
    If $\underline{F}$ is continuously differentiable and
    $$\oint_C\underline{F}\cdot\mathrm d\underline{x}=0$$
    for any closed loop $C$, then $\nabla F=\underline{0}$.
\end{proposition}
Hence zero circulation implies irrotaion.
\begin{proof}
    Suppose $\underline{F}$ satisfies all conditions but $\nabla\times\underline{F}\neq \underline{0}$, then there is a unit vector $\underline{k}$ such that it is nonzero in the $\underline{k}$ direction, then if there is some $\epsilon>0$ such that $\underline{k}\cdot(\nabla\times\underline{F}(\underline{x_0}))>\epsilon$, then there is some $\delta>0$ such that $|\underline{x}-\underline{x_0}|<\delta$ implies $\underline{k}\cdot(\nabla\times\underline{F})>\epsilon/2>0$.\\
    Now consider the ball $|\underline{x}-\underline{x_0}|<\delta$ and we choose a disk $D$ inside it, we have
    $$0=\left|\oint_{\partial D}\underline{F}\,\mathrm d\underline{x}\right|=\left|\int_D\nabla\times\underline{F}\cdot\mathrm d\underline{S}\right|\ge\frac{\epsilon}{2}\operatorname{Area}(D)>0$$
    Contradiction.
\end{proof}
\begin{example}
    Let $S_{\epsilon}$ be any sufficiently nice surface contained inside a disk with radius $\epsilon>0$ centered at $\underline{x}=\underline{x_0}$ with normal $\underline{k}$.
    If
    \begin{align*}
        \int_{S_\epsilon}\nabla\times\underline{F}\cdot\mathrm d\underline{S}&=\int_{S_\epsilon}\nabla\times\underline{F}(\underline{x_0})\cdot\mathrm d\underline{S}+\left( \int_{S_\epsilon}\nabla\times(\underline{F}-\underline{F}(\underline{x_0}))\cdot\mathrm d\underline{S} \right)
        \\
        &=\underline{k}\cdot\nabla\times\underline{F}(\underline{x_0})\operatorname{Area}(S_\epsilon)+\int_{S_\epsilon}\nabla\times(\underline{F}-\underline{F}(\underline{x_0}))\cdot\mathrm d\underline{S}
    \end{align*}
    Now we claim that the last term is $o(\operatorname{Area}(S_\epsilon))$ as $\epsilon\to 0$.
    Indeed,
    \begin{align*}
        \left| \int_{S_\epsilon}\nabla\times(\underline{F}-\underline{F}(\underline{x_0}))\cdot\mathrm d\underline{S} \right|
        &\le\int_{S_\epsilon}|\nabla\times(\underline{F}-\underline{F}(\underline{x_0}))|\cdot\mathrm d\underline{S}\\
        &\le\sup_{\underline{x}\in S_\epsilon}|\nabla\times(\underline{F}(\underline{x})-\underline{F}(\underline{x_0}))|\operatorname{Area}(S_\epsilon)\\
        &=o(\operatorname{Area}(S_\epsilon))
    \end{align*}
    As $\underline{F}$ is continuously differentiable.
    Therefore by Stokes' Theorem,
    \begin{align*}
        \frac{1}{\operatorname{Area}(S_\epsilon)}\oint_{\partial S_\epsilon}\underline{F}\cdot\mathrm d\underline{x}
        &=\frac{1}{\operatorname{Area}(S_\epsilon)}\int_{S_\epsilon}\nabla\times\underline{F}\cdot\mathrm d\underline{S}\\
        &=\underline{k}\cdot\nabla\times\underline{F}(\underline{x_0})+o(1)
    \end{align*}
    As $\epsilon\to 0$.
    So the curl is the infinitesimal circulation around the normal $\underline{k}$ per unit area.
\end{example}