\documentclass{tufte-handout}

\usepackage{xcolor}
\usepackage{graphicx}

% set hyperlink attributes
\hypersetup{colorlinks}

\usepackage{amsmath}

% set image attributes:
\usepackage{graphicx}
\graphicspath{ {images/} }

% create environment for bottom paragraph:
\newenvironment{bottompar}{\par\vspace*{\fill}}{\clearpage}

% ============================================================

% define the title
\title{SOC 4930/5050: Week 06 Equations Quick \\Reference}
\author{Christopher Prener, Ph.D.}
\date{October 2\textsuperscript{nd}, 2017}
% ============================================================
\begin{document}
% ============================================================
\maketitle % generates the title
% ============================================================

\vspace{5mm}
\section{Standard Error}
\begin{equation}
\scalebox{2} {$ { \sigma }_{ \bar{ X } } = \frac{ {\sigma}_{x} }{ \sqrt{ n } } $}
\end{equation}

\vspace{5mm}
\section{Z-Score for Sample Means}
\begin{equation}
\scalebox{2} {$ z = \frac{\bar{x}-\mu}{\frac{\sigma}{\sqrt{n}}} $}
\end{equation}

\vspace{5mm}
\section{Simple Power Analysis}\marginnote{Use any of the two-tailed critical values' z-scores depending on how wide you want your interval.}
\begin{equation}
\scalebox{2} {$ \left( \frac{1.96\sigma}{\Delta} \right)^{2} $}
\end{equation}

\vspace{5mm}
\section{Predictive Interval}\marginnote{Use any of the two-tailed critical values' z-scores depending on how wide you want your interval.}
\begin{equation}
\scalebox{2} {$ \left(\mu-1.96\sigma, \mu+1.96\sigma \right)  $}
\end{equation}

\vspace{5mm}
\section{Predictive Interval for Sample Mean}\marginnote{Use any of the two-tailed critical values' z-scores depending on how wide you want your interval.}
\begin{equation}
\scalebox{2} {$ \left(\mu-1.96\frac{\sigma}{\sqrt{n}}, \mu+1.96\frac{\sigma}{\sqrt{n}}  \right) $}
\end{equation}

\vspace{5mm}
\section{Confidence Interval for Sample Mean}\marginnote{Use any of the two-tailed critical values' z-scores depending on how wide you want your interval.}
\begin{equation}
\scalebox{2} {$ \left(\bar{x}-1.96\frac{\sigma}{\sqrt{n}}, \bar{x}+1.96\frac{\sigma}{\sqrt{n}}  \right) $}
\end{equation}

% ============================================================
\end{document}


\chapter{Growth of Functions}

\section{Asymptotic notation}

\begin{enumerate}

\item[3.1{-}1]{Let $f(n)$ and $g(n)$ be asymptotically nonnegative functions.
  Using the basic definition of $\Theta$-notation, prove that
  $\max(f(n), g(n)) = \Theta(f(n) + g(n))$.}

\begin{framed}
Since $f(n)$ and $g(n)$ are both asymptotically nonnegative,
\[
\Exists n_0 \mid f(n) \ge 0\;g(n) \ge 0\;\Forall n \ge n_0.
\]

From the definition of $\Theta(\cdot)$, we have
\[
\Exists c_1\;c_2\;n_0\in\mathbb{R}^+ \mid
c_1 f(n) + c_1 g(n) \le \max(f(n), g(n)) \le c_2 f(n) + c_2 g(n)\;
\Forall n \ge n_0.
\]

If $f(n) \ge g(n)$, we have
\[
c_1 f(n) + c_1 g(n) \le f(n) \le c_2 f(n) + c_2 g(n).
\]

The right-hand-side inequality is trivially satisfied with $c_2 = 1$. To find
\(c_1\), we notice that,
\[
f(n) + g(n) \le 2f(n),
\]

and say,
\[
c_1 = \frac{1}{2}.
\]

The demonstration is similar for $g(n) > f(n)$, with $c_1 = 1/2$ and $c_2 = 1$.
\end{framed}

\item[3.1{-}2]{Show that for any real constants $a$ and $b$, where
$b > 0$, $(n + a)^b = \Theta(n^b)$.}

\begin{framed}
From the definition of \(\Theta(\cdot)\), we have
\[
\Exists c_1\;c_2\;n_0\in\mathbb{R}^+ \mid c_1 n^b \leq (n+a)^b \leq c_2 n^b\;
\Forall n \ge n_0,
\]

and from the binomial theorem, we have
\[
(n + a)^b = \binom{b}{0} n^b a^0 + \binom{b}{1} n^{b - 1} a^1 + \cdots +
            \binom{b}{b - 1} n^1 a^{b - 1} + \binom{b}{b} n^0 a^b.
\]

To find \(c_1\), we notice that for $n$ big enough,
\[
\binom{b}{i} n^{b - i} a^i + \binom{b}{i + 1} n^{b - (i + 1)} a^{i + 1} \ge 0
\quad \forall\ i \in 0, 2, \dots ,b,
\]

which implies
\[
\binom{b}{0} n^b a^0 + \binom{b}{1} n^{b - 1} a^1 \le (n + a)^b,
\]

and also for $n$ big enough,

\[
\frac{n^b}{2} \le n^b + \binom{b}{1} n^{b - 1} a^1,
\]

which implies
\[
\frac{n^b}{2} \le (n + a)^b,
\]

and say
\[
c_2 = \frac{1}{2}.
\]

To find \(c_2\), we notice that for $n$ big enough,
\[
n^b = \binom{b}{0} n^b a^0
\geq \binom{b}{i} n^{b - i} a^i \quad \forall\ i \in 1, \dots ,b,
\]
which implies
\[
(n + a)^b \leq (b + 1) n^b,
\]

and say
\[
c_2 = b + 1.
\]
\end{framed}

\item[3.1{-}3]{Explain why the statement, ``The running time of algorithm $A$ is
  at least $O(n^2)$,'' is meaningless.}

\begin{framed}
Because the $O$-notation only bounds from the top, not from the bottom.
\end{framed}

\item[3.1{-}4]{Is $2^{n+1} = O(2^n)$? Is $2^{2n} = O(2^n)$?}

\begin{framed}
From the definition of $O(\cdot)$, we have
\[
\Exists c\;n_0\in\mathbb{R}^+ \mid 0 \leq 2^{n+1} \leq c \cdot 2^n\;
\Forall n \ge n_0.
\]

To find $c$, we notice that,
\[
2^{n+1} = 2 \cdot 2^n,
\]

and say $c = 2$ and $n_0 = 0$.

From the definition of $O(\cdot)$, we have
\[
\Exists c\;n_0\in\mathbb{R}^+ \mid 0 \leq 2^{2n} \leq c \cdot 2^n\;
\Forall n \ge n_0.
\]

To show that $2^{2n} \neq O(2^n)$, we notice that,
\[
2^{2n} = 2^n \cdot 2^n,
\]

which implies
\[
c \ge 2^n,
\]
which is not possible, since $c$ is a constant and $n$ is not.
\end{framed}

\item[3.1{-}5]{Prove Theorem 3.1.}

\begin{framed}
To prove
\[
f(n) = \Theta(g(n)) \iff f(n) = O(g(n)) \wedge f(n) = \Omega(g(n)).
\]

we need to show
\[
f(n) = O(g(n)) \wedge f(n) = \Omega(g(n)) \rightarrow f(n) = \Theta(g(n)),
\]

and
\[
f(n) = \Theta(g(n)) \rightarrow f(n) = O(g(n)) \wedge f(n) = \Omega(g(n)).
\]

From the definition of $O(\cdot)$, we have
\[
\Exists c_1\;n_1\in\mathbb{R}^+ \mid 0 \le f(n) \le c_1g(n)\;\Forall n \ge n_1,
\]

and from the definition of $\Omega(\cdot)$, we have
\[
\Exists c_2\;n_2\in\mathbb{R}^+ \mid 0 \le c_2g(n) \le f(n)\;\Forall n \ge n_2,
\]

which implies
\[
\Exists c_1\;c_2\in\mathbb{R}^+\;n_0 = \max(n_1, n_2) \mid
c_2g(n) \leq f(n) \leq c_1g(n)\;\Forall n \ge n_0 \iff f(n) = \Theta(g(n)).
\]

From the definition of $\Theta(\cdot)$, we have
\[
\Exists c_1\;c_2\;n_0\in\mathbb{R}^+ \mid c_2g(n) \leq f(n) \leq c_1g(n)\;
\Forall n \ge n_0,
\]

which implies
\[
\Exists c_1\;n_0\in\mathbb{R}^+ \mid 0 \leq f(n) \leq c_1 g(n)\;
\Forall n \ge n_0 \iff f(n) = O(g(n)),
\]

\[
\Exists c_2\;n_0\in\mathbb{R}^+ \mid c_2 g(n) \leq f(n) \leq 0\;
\Forall n \ge n_0 \iff f(n) = \Omega(g(n)).
\]

\end{framed}

\newpage

\item[3.1{-}6]{Prove that the running time of an algorithm is $\Theta(g(n))$ if
  and only if its worst-case running time is $O(g(n))$ and its best-case running
  time is $\Omega(g(n))$.}

\begin{framed}
Let $f_b(n)$ and $f_w(n)$ be the best and worst-case running times of algorithm
$A$, respectivelly.

If the running time of $A$ is $\Theta(g(n))$, we have
\[
f_b(n) = \Theta(g(n)),
\]

and
\[
f_w(n) = \Theta(g(n)).
\]

From Theorem 3.1,
\[
f_b(n) = \Theta(g(n)) \iff f_b(n) = O(g(n)) \wedge f_b(n) = \Omega(g(n)),
\]

and
\[
f_w(n) = \Theta(g(n)) \iff f_w(n) = O(g(n)) \wedge f_w(n) = \Omega(g(n)).
\]

\end{framed}

\item[3.1{-}7]{Prove that $o(g(n)) \cap \omega(g(n))$ is the empty set.}

\begin{framed}
From the definition of $o(\cdot)$, we have
\[
o(g(n)) = \{ f(n) : \Forall c_1 > 0\;\Exists n_1 \in \mathbb{R}^+ \mid
             0 \le f(n) < c_1g(n)\;\Forall n \ge n_1 \},
\]

and from the definition of $\omega(\cdot)$, we have
\[
\omega(g(n)) = \{ f(n) : \Forall c_2 > 0\;\Exists n_2\in\mathbb{R}^+ \mid
                  0 \le c_2g(n) < f(n)\;\Forall n \ge n_2 \}.
\]

Thus,
\[
o(g(n)) \cap \omega(g(n)) =
\{ f(n) : \Forall c_1 > 0\;\Forall c_2>0\;\Exists n_0\in\mathbb{R}^+\mid
   0 \le c_2g(n) < f(n) < c_1g(n)\;\Forall n \ge n_2 \},
\]
which is the empty set since, for very large $n$, $f(n)$ cannot be less than
$c_1 g(n)$ and greater than $c_2 g(n)$ for all $c_1, c_2 > 0$.

\end{framed}

\item[3.1{-}8]{We can extend our notation to the case of two parameters $n$ and
$m$ that can go to infinity independently at different rates. For a given
$g(n, m)$, we denote by $O(g(n, m))$ the set of functions
\[
O(g(n, m)) = \{{f(n, m) : \text{there exist positive constants } c, n_0,
\text{and\ } m_0 \text{\ such that\ } 0 \le f(n, m) \le c g(n, m)
\text{\ for all\ } n \ge n_0 \text{\ and\ } m \ge m_0}\}.
\]

Give corresponding definitions for $\Omega(g(n, m))$ and $\Theta(g(n, m))$.
}

\begin{framed}
We denote by $\Omega(g(n, m))$ the set of functions
\[
\Omega(g(n, m)) = \{f(n, m) : \Exists c\;n_0\;m_0 \in \mathbb{R}^+ \mid
                    0 \le cg(n, m)) \le f(n, m)\;
                    \Forall n \ge n_0\;\Forall m \ge m_0\}.
\]

We denote by $\Theta(g(n, m))$ the set of functions
\[
\Theta(g(n, m)) = \{f(n, m) : \Exists c_1\;c_2\;n_0\;m_0\in\mathbb{R}^+\mid
                    0 \le c_1g(n, m) \le f(n, m) \le c_2g(n, m)\;
                    \Forall n \ge n_0\;\Forall m \ge m_0\}.
\]
\end{framed}

\end{enumerate}

\newpage

\section{Standard notations and common functions}

\begin{enumerate}

\item[3.2{-}1]{Show that if $f(n)$ and $g(n)$ are monotonically increasing
functions, then so are the functions $f(n) + g(n)$ and $f(g(n))$, and if
$f(n)$ and $g(n)$ are in addition nonnegative, then $f(n) \cdot g(n)$ is
monotonically increasing.}

\begin{framed}
If $f(n)$ and $g(n)$ are both monitonically increasing and $n \le m$, we have
\[
f(n) \le f(m) \quad \text{and} \quad g(n) \le g(m),
\]

which implies that
\[
f(n) - f(m) \le 0 \quad \text{and} \quad g(n) - g(m) \le 0.
\]

Adding the above inequalities together, we have
\[
f(n) - f(m) + g(n) - g(m) \le 0 \rightarrow
f(n) + g(n) \le f(m) + g(m),
\]

which shows that $f(n) + g(n)$ is monitonically increasing.

Also, let $g(n) = p$ and $g(m) = q$.
Since $f(n) \le f(m)$ and $g(n) \le g(m)$, we have
\[
f(p) \le f(q) \rightarrow f(g(n)) \le f(g(m)),
\]
which shows that $f(g(n))$ is monitonically increasing.

If in addition, $f(\cdot) \ge 0$ and $g(\cdot) \ge 0$, we have
\[
f(n) \le f(m) \rightarrow f(n) g(n) \le f(m) g(n) \rightarrow f(n) g(n) \le f(m) g(m),
\]
which shows that $f(n) \cdot g(n)$ is monitonically increasing.
\end{framed}

\item[3.2{-}2]{Prove equation (3.16).}

\begin{framed}
For all real $a > 0, b > 0, c > 0$,
\[
a^{\log_b c} = a^{\frac{\log_a c}{\log_a b}}
             = \left(a^{\log_a c}\right)^{\frac{1}{\log_a b}}
             = c^{\frac{1}{\log_a b}}
             = c^{\log_b a}.
\]
\end{framed}

\item[3.2{-}3]{Prove equation (3.19).
Also prove that $n! = \omega(2^n)$ and $n! = o(n^n)$.}

\begin{framed}
Using the Stirling's approximation, we have
\begin{equation*}
\begin{split}
  \lg(n!) & \approx \lg\left(\sqrt{2 \pi n} \binom{n}{e}^n \left(1 + \Theta\left(\frac{1}{n}\right)\right)\right)\\
          & = \lg{(\sqrt{2 \pi n})} + \lg(\sqrt{n}) + \lg(n^n) - \lg(e^n) + \Theta(\lg(1/n))\\
          & = \Theta(1) + 1/2 \lg(n) + n \lg n - n \lg e + \Theta(\lg(1/n))\\
          & = \Theta(1) + \Theta(\lg n) + \Theta(n \lg n) - \Theta(n) + \Theta(\lg(1/n))\\
          & = \Theta(n \lg n),
\end{split}
\end{equation*}
which proves Equation (3.19).

We have
\[
  n! = n \cdot (n - 1) \cdot (n - 2) \cdots 2 \cdot 1
     < \underbrace{n \cdot n \cdot n \cdots}_\text{n times}
     = n^n\;\Forall n \ge 2,
\]
which implies
\[
  n! = o(n^n).
\]

We have
\[
  n! = n \cdot (n - 1) \cdot (n - 2) \cdots 2 \cdot 1
     > \underbrace{2 \cdot 2 \cdot 2 \cdots}_\text{n times}
     = 2^n\;\Forall n \ge 4,
\]
which implies
\[
  n! = w(2^n).
\]

\end{framed}

\newpage

\item[3.2{-}4]{($\star$) Is the function $\ceil{\lg n}{!}$ polynomially bounded?
Is the function $\ceil{\lg \lg n}{!}$ polynomially bounded?}

\begin{framed}
A function $f(n)$ is polynomially bounded if there are constants $c, k, n_0$
such that for all $n \ge n_0$, $f(n) \le c n^k$. Thus, $\lg(f(n)) \le c k \lg{n}$.

We have
\[
\lg(\ceil{\lg{n}}!) = \Theta(\ceil{\lg{n}} \lg(\ceil{\lg{n}})) = \Theta(\lg{n} \lg{\lg{n}}) = w(\lg{n}),
\]

which implies that $\lg(\ceil{\lg{n}}!) > c k \lg{n}$, i.e., $\ceil{\lg{n}}{!}$
is not polynomially bounded.

We have
\[
\lg(\ceil{\lg{\lg{n}}}!) = \Theta(\ceil{\lg{\lg{n}}}\lg{\ceil{\lg{\lg{n}}}})
                          = \Theta(\lg{\lg{n}} \lg{\lg{\lg{n}}})
                          = o(\lg^2{\lg{n}})
                          = o(\lg^2{n})
                          = o(\lg{n}),
\]
which implies that $\lg(\ceil{\lg{\lg{n}}}!) \le c k \lg{n}, i.e.,
\ceil{\lg{\lg{n}}}{!}$ is polynomially bounded.

\end{framed}

\item[3.2{-}5]{($\star$) Which is asymptotically larger:
$\lg(\lg^\star n)$ or $\lg^\star(\lg n)$}?

\begin{framed}
Let's assume that $\lg^*(x) = k$.

We have
\[
\lg(\lg^*{x}) = \lg{k},
\]
and
\[
\lg^*(\lg{x}) = k - 1,
\]
since the inner logarithm that is applied to $x$ will reduce the number of
iterations of the iterative logarithm by 1.

Thus, since $(k - 1)$ is asymptotically larger than $\lg(k)$, $\lg^*(\lg{x})$ is
also asymptotically larger than $\lg(\lg^*{x})$.
\end{framed}

\item[3.2{-}6]{Show that the golden ration $\phi$ and its conjugate $\hat\phi$}
both satisfy the equation $x^2 = x + 1$.

\begin{framed}
The demonstration follows directly from the formulas of $\phi$ and $\hat\phi$.
\[
\phi^2 = \left(\frac{1 + \sqrt{5}}{2}\right)^2 = \frac{1 + 2 \sqrt{5} + 5}{4}
       = \frac{2 \sqrt{5} + 6}{4} = \frac{\sqrt{5} + 3}{2}
       = \frac{1 + \sqrt{5}}{2} + 1 = \phi + 1.
\]

\[
\hat\phi^2 = \left(\frac{1 - \sqrt{5}}{2}\right)^2 = \frac{1 - 2 \sqrt{5} + 5}{4}
           = \frac{6 - 2 \sqrt{5}}{4} = \frac{3 - \sqrt{5}}{2}
           = \frac{1 - \sqrt{5}}{2} + 1 = \hat\phi + 1.
\]
\end{framed}

\item[3.2{-}7]{Prove by induction that the $i$th Fibonacci number satisfies the
equality
\[
F_i = \frac{\phi^i - \hat\phi^i}{\sqrt{5}},
\]
where $\phi$ is the golden ratio and $\hat\phi$ is its conjugate.
}

\begin{framed}
We have that
\[
F_0 = \frac{\phi^0 - \hat\phi^0}{\sqrt{5}} = \frac{1 - 1}{\sqrt{5}} = 0,
\]
and
\[
F_1 = \frac{\phi^1 - \hat\phi^1}{\sqrt{5}}
    = \frac{1 + \sqrt{5} - 1 + \sqrt{5}}{2 \sqrt{5}}
    = \frac{2 \sqrt{5}}{2 \sqrt{5}} = 1.
\]

which are the correct Fibonacci values for $i = 0$ and $i = 1$. Then we have the inductive step:
\begin{equation*}
\begin{split}
F_i + F_{i + 1} &= \frac{\phi^i + \hat\phi^i}{\sqrt{5}} +
                   \frac{\phi^{i + 1} + \hat\phi^{i + 1}}{\sqrt{5}}\\
                &= \frac{\phi^i + \phi^{i + 1} - (\hat\phi^i + \hat\phi^{i + 1})}{\sqrt{5}}\\
                &= \frac{\phi^i (1 + \phi) - \hat\phi^i (1 + \phi)}{\sqrt{5}}\\
                &= \frac{\phi^i \phi^2 - \hat\phi^i \hat\phi^2}{\sqrt{5}}\\
                &= \frac{\phi^{i + 2} - \hat\phi^{i + 2}}{\sqrt{5}} = F_{i + 2}.
\end{split}
\end{equation*}
\end{framed}

\item[3.2{-}8]{Show that $k \ln k = \Theta(n)$ implies $k = \Theta(n / \ln n)$.}

\begin{framed}
From the symmetry of $\Theta$, we have
\[
k \ln k = \Theta(n) \rightarrow n = \Theta(k \ln k),
\]
and
\[
\ln n = \Theta(\ln(k \ln k)) = \Theta(\ln k \ln \ln k) = \Theta(\ln k).
\]
Thus,
\[
\frac{n}{\ln n} = \frac{\Theta(k \ln k)}{\Theta(\ln k \ln \ln k)}
                = \Theta\left(\frac{k \ln k}{\ln k \ln \ln k}\right)
                = \Theta(k),
\]
which implies
\[
k = \Theta\left(\frac{n}{\ln n}\right).
\]
\end{framed}

\end{enumerate}

\newpage

\section*{Problems}
\addcontentsline{toc}{section}{\protect\numberline{}Problems}%

Skipped for later.


% This document was created while following the amzaing LaTeX beginner tutorial (https://www.youtube.com/playlist?list=PL542920k_cOoSTC23aG3jhczwHtrRKb9q) of the YouTube channel DorFuchs (https://www.youtube.com/channel/UC97dp7op_ZjSCNp3DRLGymQ)

\documentclass{article}

% packages to import:
\usepackage[ngerman]{babel} % Language package for usign for example umlauts
\usepackage[utf8]{inputenc} % Input encoding - rendering of for example umlauts
\usepackage[T1]{fontenc} % Font encoding to find later words with for example umlauts
\usepackage{amsmath} % Use advanced math in \begin{align}\end{align} sections
\usepackage{amssymb} % Use math symbols like the symbol for the real numbers
\usepackage{amsthm} % Create your own sections and use for example a proof section

% create a section named satz
\newtheorem{satz}{Satz}

% create custom commands ([#] is the number of arguments)
\newcommand{\R}{\mathbb{R}}
\newcommand{\vektor}[1]{\begin{pmatrix}#1\end{pmatrix}}

% remove indent
\setlength{\parindent}{0px}

\begin{document}

\title{Testdokument}
\author{DorFuchs}
\maketitle

% add table of contents to the document
\tableofcontents

% show text on the next side
\newpage

\section{Satz des Pythagoras}

\begin{satz}
	In einem rechtwinkligen Dreieck mit Katheten $a$ und $b$ und Hyphotenuse $c$ gilt
	\[ a^2 + b^2 = c^2 . \]
\end{satz}
\begin{proof}
	... \\
	... \\
	...
\end{proof}

\section{p-q-Formel}

Für die Gleichung $x^2 + px + q = 0$, wobei $p, q \in \mathbb{R}$ mit $\frac{p}{2}^2 - q > 0$, lautet die Lösungsformel
\begin{align}
	x &= \frac{p}{2} \pm \sqrt{ \left( \frac{p}{2} \right)^2 - q} \\
	&= - \frac{p}{2} \pm \sqrt{\frac{p^2}{4} - q}
\end{align}

\section{Euklidische Geometrie}

\subsection{Matrizen}

\begin{satz}
	Die Einheitsmatrix im $\R^3$ lautet
	\[ \begin{pmatrix}
	1 & 0 & 0 \\
	0 & 1 & 0 \\
	0 & 0 & 1
	\end{pmatrix} . \]
\end{satz}

\subsection{Vektoren}

\begin{satz}
	Der Betrag eines Vektors ist im $\R^3$ ist definiert durch
	\[ \left| \vektor{x\\y\\z} \right| = \sqrt{x^2 + y^2 + z^2} . \]
\end{satz}

\end{document}


\section{Basis, Dimension and Direct Sum}
\begin{definition}
    This cardinality is called the dimenion $\dim V$ of $V$.
\end{definition}
This is well-defined due to Corollary \ref{dim_well_defined}.
\begin{proposition}
    Let $U,W$ be subspaces of $V$.
    If they are finite dimensional, then so is $U+W$ and
    $$\dim(U+W)=\dim U+\dim V-\dim(U\cap W)$$
\end{proposition}
\begin{proof}
    Pick a basis $v_1,\ldots,v_l$ of $U\cap W$.
    Extend it to a basis $v_1,\ldots,v_l,u_1,\ldots,u_m$ of $U$ and a basis $v_1,\ldots,v_l,w_1,\ldots,w_n$ of $W$, then $\{v_i\}\cup\{u_i\}\cup\{w_i\}$ is easily a basis of $U+W$.
    The equality follows.
\end{proof}
\begin{proposition}
    If $V$ is a finite dimensional vectyor space and $U\le V$, then $U,V/U$ are both finite dimensional and $\dim V=\dim U+\dim V/U$.
    Furthermore,
    $$\dim V=\dim U+\dim V/U$$
\end{proposition}
\begin{proof}
    It is obvious that $U$ is finite dimensional.
    Choose a basis $u_1,\ldots,u_l$ and extend it to a basis $u_1,\ldots,u_l,w_{l+1},\ldots,w_n$ of $V$, then $w_{l+1}+U,\ldots,w_n+U$ can be verified to be a basis of $V/U$.
    The statement is immediate.
\end{proof}
\begin{remark}
    If $U$ is a proper subspace of $V$, written $U<V$ (meaning that $U\le V$ and $U\neq V$), then the proposition gives us $\dim V/U\neq 0$, so $\dim U<\dim V$.
\end{remark}
\begin{definition}
    Let $V$ be a vector space and $U,W\le V$.
    We say $V$ is the direct sum of $U,W$, written $V=U\oplus W$, if every element $v\in V$ can be written uniquely as $v=u+w$ for $u\in U,w\in W$.\\
    If this happens, then we say $W$ is a direct complement of $U$ in $V$.
\end{definition}
Note that direct complement is not unique in general.
\begin{example}
    Take $U=\mathbb R\times \{0\}$, then both $W=\{0\}\times \mathbb R$ and $W'=\langle\{(1,1)^\top\}\rangle$ are direct complements of $U$.
\end{example}
\begin{lemma}
    Let $U,W\le V$, then the followings are equivalent:\\
    1. $V=U\oplus W$.\\
    2., $V=U+W$ and $U\cap W=\{0\}$.\\
    3. For any basis $B_1$ of $U$ and $B_2$ of $W$, the union $B=B_1\cup B_2$ is a basis of $V$.
\end{lemma}
\begin{proof}
    Trivial.
\end{proof}
\begin{definition}
    Let $V_1,\ldots,V_l\le V$, then we define
    $$\sum_{i=1}^lV_i=\{v_1+\cdots+v_l:v_j\in V_j,1\le j\le l\}$$
    The sum is direct, i.e.
    $$\sum_{i=1}^lV_i=\bigoplus_{i=1}^lV_i$$
    iff $v_1+\cdots +v_l=v_1'+\cdots +v_l'$ implies $v_j=v_j'$ for any $1\le j\le l$ and $v_j\in V_j$.
    Equivalently,
    $$V=\bigoplus_{i=1}^lV_i\iff \forall v\in V,\exists!(v_1,\ldots,v_l)\in V_1\times\cdots\times V_l,v=\sum_{i=1}^lv_i$$
\end{definition}
\begin{proposition}
    The followings are equivalent:\\
    1.
    $$\sum_{i=1}^lV_i=\bigoplus_{i=1}^lV_i$$
    2. For any $i$,
    $$V_i\cap\left( \sum_{j\neq i}V_j \right)=\{0\}$$
    3. For any bases $B_i$ of $V_i$, the union $\bigcup_iB_i$ is a basis of $\sum_iV_i$.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}


\documentclass[12pt]{article}
\usepackage{Sweave}
% \SweaveOpts{echo=T,pdf=T,eps=F,eval=T,keep.source=T}
\usepackage{mathpazo}
\usepackage{hyperref,url}
\usepackage[a4paper,margin=1.5cm]{geometry}

\begin{document}

\title{Example Sweave document: estimating $\pi$}
\author{Stephen J Eglen}

\date{\today}
\maketitle

\section{Introduction}

This is an example document created using the Sweave system
(\url{http://www.statistik.lmu.de/~leisch/Sweave/}).  Sweave is a
tool for combining both \LaTeX\ documentation and R code within the
same file.  For this document, the master file is \url{estimate.Rnw}.
This is processed by the Sweave system in R, which runs the R code to
generate textual/graphical output, and also creates a \LaTeX\ document.
The \LaTeX\ document is then typeset to create the pdf document.  On
unix/macintosh, the following commands should recreate the pdf file:

\begin{verbatim}
$ R CMD Sweave estimate.Rnw
$ pdflatex estimate.tex
\end{verbatim}

\section{Task: estimate the value of $\pi$}

Our task is to estimate the value of $\pi$ by simulating darts being
thrown at a dartboard.  Imagine that the person throwing the darts is
not very good, and randomly throws each dart so that it falls
uniformly within a square of side length $2r$, with the dartboard of
radius $r$ centred within that square.  If the player throws $n$
darts, and $d$ of them hit the dartboard, then for large enough $n$,
the ratio $d/n$ should approximate the ratio of the area of the
dartboard to the enclosing square, $ \pi r^2 / 4 r^2 \equiv \pi/4$.
From this, we can estimate $\pi \approx 4d/n$.

We start with an example, using R to draw both the dartboard and the
surrounding square, together with $n=50$ darts.  The radius of the
dartboard here is 1 unit, although the value is not important.

\setkeys{Gin}{width=0.6\textwidth}
\begin{center}
<<fig=TRUE>>=
r <- 1
n <- 50
par(las=1)
plot(NA, xlim=c(-r,r), ylim=c(-r,r), asp=1, bty='n',
     xaxt='n', yaxt='n', xlab='', ylab='')
axis(1, at=c(-r,0,r)); axis(2, at=c(-r,0,r))
symbols(x=0, y=0, circles=r, inch=F, add=T)
x <- runif(n, -r, r);  y <- runif(n, -r, r)
inside <- x^2 + y^2 < r^2
d <- length(which(inside))
points(x, y, pch=ifelse(inside, 19, 4))
rect(-r, -r, r, r, border='blue', lwd=2)
@
\end{center}

A dart is drawn as a filled circle if it falls within the dartboard,
else it is drawn as a cross.  In this case the number of darts within
the circle is \Sexpr{d}, and so the estimated value is $\pi \approx
\Sexpr{4*d/n}$.

The estimate of $\pi$ should improve as we increase the number of darts
thrown at the dartboard.  To verify this, we write a short function
that, given the number of darts to throw, $n$, returns an estimate of
$\pi$.

<<>>=
estimate.pi <- function(n=1000) {
  ## Return an estimate of PI using dartboard method
  ## with N trials.
  r <- 1                                # radius of dartboard
  x <- runif(n, min=-r, max=r)
  y <- runif(n, min=-r, max=r)
  l <- sqrt(x^2 + y^2)
  d <- length(which(l<r))
  4*d/n
}
@

We can then test the procedure a few times, using the default number
of darts, 1000:

<<>>=
replicate(9, estimate.pi())
@

Finally, for a given value of $n$, we can show 99 estimates of $\pi$,
as clearly the estimate will vary from run to run.  In the following
plot, we compare the estimates of $\pi$ for three different values of $n$:

\begin{center}
<<fig=TRUE>>=
ns <- 10^c(2,3,4)
res <- lapply(ns, function(n) replicate(99, estimate.pi(n)))
par(las=1, bty='n')
stripchart(res, method="jitter", group.names=ns,
           xlab="number of darts",
           ylab=expression(paste('estimate of ', pi)),
           vert=T, pch=20, cex=0.5)
abline(h=pi, col='red')
@
\end{center}

As the number of darts increases, the estimate of $\pi$ gradually
converges onto the actual value of $\pi$ (shown by the solid red line).

\end{document}


\subsubsection{Secondary model}
\label{sec:methods_pipeline_secondary_model}

The metalabelling process allows to algorithmically determine the positiveness
of the labels. Note that metalabels should not be included as input features
of the model because they belong to the future of the label and one model doing
that will not be plausible to implement on a real time system.
Initially, all features in the feature engineering section \ref{sec:methods_features}
will be used to build this model.

This research project will use bagging trees. In chapter 6 of \cite{lopez_de_prado}
there is a discussion about ensemble models and three are compared: bagging
trees, random forests and boosting. Boosting is generally superior in terms of
bias and variance results with respect to the others but it comes with, in my
opinion, an important penalty: the training process should be done sequentially whereas
bagging can be parallelized. The author is explicit about the other positive and
negative aspects such as tendency to over-fitting or under-fitting, by stating
that those claims are relative to how careful the researcher is when developing
the training pipeline and curating data.

The focus of this research is not to get into the details of one machine
learning model or the other, but to explain the most relevant characteristics of
the one to be used and how its characteristics are used in favor of the research
problem at hand. Bagging ensembles rely on multiple, $B$, decision (or
regression but in this case we need decision) trees whose outcome is averaged
to reduce the high variance of each tree. Decision trees tend to be deep, in
favor of bias reduction, but each tree variance is high. Increasing the number
of $B$ trees does not imply an immediate shift towards over-fitting what makes
it an appropriate hyperparameter to adjust in favor of variance reduction. See
section 8.2.1 of \cite{intro_to_statistical_learning} for a better
description of how bagging ensembles work.

Provided that only one dataset is available, a bootstrap sample method is used.
Bootstrapping consists of sampling the base dataset with replacement to generate
new datasets that will be used to train each decision trees. Each new dataset is
expected to be biased and the variance between datasets will be diminished when
averaging, or bagging, the results of the trees. Bootstrapping can also be done
with the features of the dataset. See section 5.2 of \cite{intro_to_statistical_learning}
for a concise but comprehensive description to bootstrapping.

The general bootstrapping method for samples assumes that all samples are IID.
This is not our case when using the triple barrier method.
Note that some samples might co-occur when time windows expand from the label
timestamp to $h$ sample periods ahead. In high volatility events, or with high
values of $h$, we should expect events, or labels, to happen while another one
is being evaluated. Weighing these labels with respect to unique and without any
overlap events is important to get the most out of the bootstrapping sampling
procedure. Following Lopez de Prado's analysis, concurrent labels are defined as
those that share at least one return attribution in the triple barrier method.
Let the concurrency $c_t$ at time $t$ be defined as:

\[c_t = \sum_{i=1}^I l_{t,i}\]

Where $l_{t,i}$ is the i-th label value that occurs at $t$ and $I$ is the number of
label which co-occur at $t$. Labels start at a certain time $t$ but span a
number of sampling periods that could be $h$ or the time difference with respect
to one of the horizontal (price) barriers crossing events. Consequently, a label
will contribute to at least two different and consecutive timestamps and up to
$h$ consecutive timestamps.

Let the uniqueness of a label $i$ at time $t$ be defined as:

\[u_{t,i} = \frac{l_{t_i}}{c_t}\]

And the average uniqueness will be defined as the averaged uniqueness over the
label's lifespan:

\[\bar{u}_{t,i} = \frac{\sum_{t=1}^{T} u_{t,i}}{\sum_{t=1}^{T} l_{t,i}}\]

Label's average uniqueness allows a smarter bootstrap sampling method because it
can be used to prioritize events with higher uniqueness over the ones with less
uniqueness because the sole fact that they are weird in the dataset.

Lopez de Prado proposes a change to the above uniqueness sample weight. He
introduces the bet return also to account for an average of all the bets
simultaneously running. To do so, a new weight $w_i$ for label $i$ is proposed:

\[ \tilde{w}_i =\big| \sum_{t=t_{i,0}}^{t_{i,1}} \frac{r_{t-1,t}}{c_t} \big|\]

\[ w_i = \frac{\tilde{w}_i I}{\sum_{j=1}^{I} \tilde{w}_j}\]

Moving from $\tilde{w}_i$ to $w_i$ involves a scale factor to assure:
$\sum_{i=1}^{I} w_j = I$. When training the model, the training set / fold (when
using cross validation) will incorporate the weight with combined return and
uniqueness attribution to differentiate rare as well as rare high-return events
from the other concurrent and low-return events. Lopez de Prado also proposes a
sequential bootstrap method which updates the probability of each sample in the
series every time a row is drawn. This process yields train sets closer to IID
but this is not part of this research pipeline.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS630: Database Management Systems
% Copyright 2014 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 1}

Suppose you are given a relation $R$ with four attribute $ABCD$ and the set of functional dependencies as follows.

\begin{enumerate}
\item FD = \{ $AB\rightarrow C$, $BC\rightarrow D$ \}
\item FD = \{ $BC\rightarrow A$, $AB\rightarrow C$, $C\rightarrow DA$ \}
\end{enumerate}

For each set of functional dependencies,

\begin{enumerate}[label=(\alph*)]
\item Identify the candidate key(s) for R (recall that keys must be \textit{minimal})
\item Determine if $R$ is in $BCNF$, $3NF$, or none of the above.
If it is not in $BCNF$, decompose it into a set of $BCNF$ relations.
Does the decomposition you obtained preserve all initial dependencies?
\end{enumerate}

Recall that it is not sufficient to consider the set of functional dependencies that are given but also it's closure.

\subsection*{Solution}

Tables \ref{table1} (a) and \ref{table1} (b) are constructed to find candidate key(s) of first and second set of functional dependencies, respectively.
Based on attribute closures of each attribute combination, combinations whose closure is equivalent to relation $R$ are underlined as minimal candidate keys.

\begin{table}[H]
\centering\caption{Candidate keys of relation $R$ under different set of functional dependencies}\label{table1}
\begin{subtable}{.5\linewidth}
\centering\caption{FD = \{ $AB\rightarrow C$, $BC\rightarrow D$ \}}\label{table1a}
\begin{tabular}{|c|c||c|c|}
\hline
$X$ & $X^+$ & $X$ & $X^+$\\
\hline
A & A & BC & BCD\\
B & B & BD & BD\\
C & C & CD & CD\\
D & D & ABC & ABCD\\
\underline{\bf AB} & ABCD & ABD & ABCD\\
AC & AC & ACD & ACD\\
AD & AD & BCD & BCD\\
\hline
\end{tabular}
\end{subtable}%
\begin{subtable}{.5\linewidth}
\centering\caption{FD = \{ $BC\rightarrow A$, $AB\rightarrow C$, $C\rightarrow DA$ \}}\label{table1b}
\begin{tabular}{|c|c||c|c|}
\hline
$X$ & $X^+$ & $X$ & $X^+$\\
\hline
A & A & \underline{\bf BC} & ABCD\\
B & B & BD & BD\\
C & ACD & CD & ACD\\
D & D & ABC & ABCD\\
\underline{\bf AB} & ABCD & ABD & ABCD\\
AC & ACD & ACD & ACD\\
AD & AD & BCD & ABCD\\
\hline
\end{tabular}
\end{subtable}
\end{table}

\begin{enumerate}

\item $FD = \{ AB\rightarrow C, BC\rightarrow D \}$

	\begin{enumerate}[label=(\alph*)]
	\item As shown in Table \ref{table1}(a), set of minimal candidate keys are $\{AB\}$.
	\item To check if $R$ is in \textit{BCNF} or \textit{3NF}, closure of functional dependencies are constructed in Equation \ref{equation1}.
	\begin{equation}\label{equation1}
	F^+ = \{AB\rightarrow C, BC\rightarrow D\} \cup \{AB\rightarrow D\}
	\end{equation}
	Each functional dependency (FD) is now verified to conform to $BCNF$ and $3NF$ form, as given in Table \ref{table2}.

	\begin{table}[H]
	\centering\caption{Conformance verification of relation $R$}\label{table2}
	\begin{tabular}{|r||c|c|c|}
	\hline
	 & $AB\rightarrow C$ & $BC\rightarrow D$ & $AB \rightarrow D$\\
	\hline
	$BCNF$ & \checkmark & $\times$ & \checkmark \\
	$3NF$ & \checkmark & $\times$ & \checkmark \\
	\hline
	\end{tabular}
	\end{table}

	As shown in Table \ref{table2}, relation $R$ is not \textit{BCNF} since functional dependency $BC\rightarrow D$ violates \textit{BCNF} condition.
To resolve the violation, $R$ is decomposed into $ABC$ and $BCD$ whose closures are respectively $\{AB\rightarrow C, AB\rightarrow D\}$ and $\{BC\rightarrow D\}$ both conforming to \textit{BCNF} form.
Clearly, the decomposition preserves all initial functional dependencies.

	\end{enumerate}

\item FD = \{ $BC\rightarrow A$, $AB\rightarrow C$, $C\rightarrow DA$ \}

	\begin{enumerate}[label=(\alph*)]
	\item As shown in Table \ref{table1}(a), set of minimal candidate keys are $\{AB, BC\}$.
	\item To check if $R$ is in \textit{BCNF} or \textit{3NF}, closure of functional dependencies are constructed in Equation \ref{equation2}.
	\begin{equation}\label{equation2}
	F^+ = \{BC\rightarrow A, AB\rightarrow C, C\rightarrow DA\} \cup \{AB\rightarrow D, BC\rightarrow D\}
	\end{equation}
	Each functional dependency (FD) is now verified to conform to $BCNF$ and $3NF$ form, as given in Table \ref{table3}.
	\begin{table}[H]
	\centering\caption{Conformance verification of relation $R$}\label{table3}
	\begin{tabular}{|r||c|c|c|c|c|}
	\hline
	 & $BC\rightarrow A$ & $AB\rightarrow C$ & $C\rightarrow DA$ & $AB\rightarrow D$ & $BC\rightarrow D$ \\
	\hline
	$BCNF$ & \checkmark & \checkmark & $\times$ & \checkmark & \checkmark \\
	$3NF$ & \checkmark & \checkmark & $\times$ & \checkmark & \checkmark \\
	\hline
	\end{tabular}
	\end{table}
	As shown in Table \ref{table3}, relation $R$ is not \textit{BCNF} since functional dependency $C\rightarrow DA$ violates \textit{BCNF} condition ($C$ is not a minimal key).
	To resolve the violation, $R$ is decomposed into $BC$ and $CDA$.
	As neither of the decomposed sets have violations, new relations conform \textit{BCNF} form.
In this case, the decomposition does not preserves initial functional dependencies.
	\end{enumerate}

\end{enumerate}


\section{Kernel smoothers}
One of the reasons why the previous smoothers is wiggly is because
when we move from $x_i$ to $x_{i+1}$ two points are usually changed in
the group we average. If the new two points
are very different then  $s(x_i)$ and $s(x_{i+1})$ may be quite
different. One way to try and fix this is by making the transition
smoother. That's the idea behind kernel smoothers.

Generally speaking a kernel smoother defines a set of weights
$\{W_i(x)\}_{i=1}^{n}$ for each $x$ and defines
\[
s(x) = \sum_{i=1}^n W_i(x) y_i.
\]

We will see that most scatter plot smoothers can be considered to be
kernel smoothers in this very general definition.

What is called a kernel smoother in practice has a simple approach to
represent the weight sequence $\{W_i(x)\}_{i=1}^{n}$ by describing the
shape of the weight function $W_i(x)$ by a density function with a
scale parameter that adjusts the size and the form of the weights near
$x$. It is common to refer to this shape function as a {\it kernel}
$K$. The kernel is a continuous, bounded, and symmetric real function
$K$ which integrates to one,

\[
\int K(u)\,du = 1.
\]

For a given scale parameter $h$, the weight sequence is then defined
by
\[
W_{hi}(x) = \frac{K\left( \frac{x - x_i}{h} \right) }{ \sum_{i=1}^n K\left
  ( \frac{ x - x_i }{h} \right)}
\]

Notice: $\sum_{i=1}^n W_{hi} (x_i) = 1$

The kernel smoother is then defined for any $x$ as before by
\[
s(x) = \sum_{i=1}^n W_{hi}(x) Y_i.
\]

Notice: if we consider $x$ and $y$ to be observations of random
variables $X$ and $Y$ then one can get an intuition for why this would
work because
\[
E[ Y | X ] = \int y f_{X,Y}(x,y) \, dy / f_X(x),
\]
with $f_X(x)$ the marginal distribution of $X$ and $f_{X,Y}(x,y)$ the joint
distribution of $(X,Y)$, and
\[
s(x) = \frac{ n^{-1}\sum_{i=1}^n K\left( \frac{x - x_i}{h} \right) y_i }
  { n^{-1}\sum_{i=1}^n K\left   ( \frac{ x - x_i }{h} \right)}
\]

Because we think points that are close together are similar, a kernel
smoother usually defines weights that decrease in
a smooth fashion as one moves away from the target point.

Running mean smoothers are kernel smoothers that use a ``box'' kernel. A
natural candidate for $K$ is the standard Gaussian density. (This is
very inconvenient computationally because its never 0).  This smooth
is shown in Figure \ref{f2.5} for $h=1$ year.

\begin{figure}[htp]
\caption{\label{f2.5} CD4 cell count since seroconversion for HIV infected men.}
\centerline{\epsfig{figure=Plots/plot-02-05.ps,angle=270,width=.8\textwidth}}
\end{figure}

In Figure \ref{f2.6} we can see the weight sequence for the box and Gaussian
kernels for three values of $x$.

\begin{figure}[htp]
\caption{\label{f2.6} CD4 cell count since seroconversion for HIV infected men.}
\centerline{\epsfig{figure=Plots/plot-02-06.ps,angle=270,width=.8\textwidth}}
\end{figure}

\newpage

\subsection{An Asymptotic result}
For the asymptotic theory presneted here we will assume the stochastic
design model with a one-dimensional covariate.

For the first time in this Chapter we will set down a specific
stochastic model. Assume we have $n$ IID observations of the random
variables $(X,Y)$
and that
\begin{equation}
\label{simplemodel}
Y_i = f(X_i) + \varepsilon_i, i=1,\dots,n
\end{equation}
where $X$ has marginal distribution $f_X(x)$ and the $\varepsilon_i$ IID
errors independent of the $X$. A common extra assumption is that the
errors are normally distributed.
We are now going to
let $n$ go to infinity... What does that mean?

For each $n$ we define an estimate for $f(x)$ using the kernel
smoother with scale parameter $h_n$.

\begin{theorem}
\label{t2.1}
Under the following assumptions
\begin{enumerate}
\item $\int |K(u)| \, du < \infty$
\item $\lim_{|u| \rightarrow \infty} uK(u) = 0$
\item $\E(Y^2) \leq \infty$
\item $n \rightarrow \infty, h_n \rightarrow 0, nh_n \rightarrow
  \infty$
\end{enumerate}

Then, at every point of continuity of $f(x)$ and  $f_X(x)$ we have
\[
 \frac{ \sum_{i=1}^n K\left( \frac{x - x_i}{h} \right) y_i }
  { \sum_{i=1}^n K\left   ( \frac{ x - x_i }{h} \right)}
  \rightarrow f(x) \mbox{ in probability.}
\]
\end{theorem}

{\bf Proof:} Homework. Hint: Start by proving the fixed design model.


\section{Classifying Differential Equations}
\noindent
Below is a list of differential equations
\begin{enumerate}
	\item \begin{equation*}
		t = 7\frac{\mathrm{d}^2 x}{\mathrm{d} t^2} + x\frac{\mathrm{d} x}{\mathrm{d} t}
	\end{equation*}
	\item \begin{equation*}
		\frac{\mathrm{d}y}{\mathrm{d}x} = x^2 + 3xy -7y
	\end{equation*}
	\item \begin{equation*}
		\frac{\partial^2 u}{\partial x^2} + \frac{\partial^2 u}{\partial y^2} = 0
	\end{equation*}
	\item \begin{equation*}
		\frac{\partial^2 y}{\partial t^2} = 4\frac{\partial^2 y}{\partial x^2} + e^{t-x}
	\end{equation*}
	\item \begin{equation*}
		\frac{\mathrm{d} y}{\mathrm{d} x} + 5y = e^x
	\end{equation*}
	\item \begin{equation*}
		\frac{\mathrm{d} y}{\mathrm{d} x} = 3x^2 + y^2
	\end{equation*}
	\item \begin{equation*}
		\frac{\mathrm{d}^2 x}{\mathrm{d} t^2} = \frac{\mathrm{d}x}{\mathrm{d}t} + \cos{x}
	\end{equation*}
	\item \begin{equation*}
		\frac{\mathrm{d}^2 y}{\mathrm{d} x^2} - 2x\frac{\mathrm{d} y}{\mathrm{d} x} + 2y = 0
	\end{equation*}
	\item \begin{equation*}
		\frac{\partial^2 x}{\partial t \partial z} + xt = 5
	\end{equation*}
	\item \begin{equation*}
		\frac{\mathrm{d} x}{\mathrm{d}y} + xy + ty = \ln{y}
	\end{equation*}
\end{enumerate}
Let's think about some of the ways we can classify these equations.
% Order
\input{./basics/classification/order.tex}
% Linearity
\input{./basics/classification/linearity.tex}
% Ordinary / Partial
\input{./basics/classification/type.tex}
% Homogeneity
\input{./basics/classification/homogeneity.tex}


%&LaTeX

\section{Let's Catch Some Z's}

This lab covers the z-transform, used to convert arbitrary digital
signals to the frequency domain. It also exercises the relationship
between a filter's transfer function and impulse response and how the
operations of multiplication and convolution, respectively, can be
used to compute a filter's output.

\subsection{The z-transform, Transfer Function, \& Impulse Response}

A discrete signal $x[n]$ has a z-transform $X(z)$ defined by the
following equation:
\[
X(z)=\sum_{n=0}^{\infty}x[n]z^{-n}
\]
With this definition lets investigate a feed forward filter with ten
coefficients, $\{b_0, b_1,\cdots, b_9\}$.  Recall that the Matlab
\verb|filter| function allows us to specify a filter in terms of its
\emph{coefficients}, but we can also think of it as being defined in
terms of its \emph{transfer function}. Considering the $b_k$
coefficients of the above feed forward filter, the \verb|filter|
function implements the transfer function:
\begin{equation}
  H(z) = \sum_{k=0}^{9} b_k z^{-k}
\end{equation}
In previous labs we have computed the transfer function using the
delays of the \emph{defining function}.  Mathematically, we were
actually taking the z-transform of the \emph{impulse response}!  In
this example, the impulse response is:
\begin{equation}
  h[n] = \sum_{k=0}^{9} b_k \delta[n-k]
\end{equation}
where $\delta[k]$ is the unit impulse and only has a non-zero value at
$k=n$. $H(z)$ and $h[n]$ form a z-transform pair,
$h[n]\xleftrightarrow{z} H(z)$. It should now be obvious why
feedforward filters are also known as finite impulse response filters
--- their impulse response only has a \emph{finite} number of
values. To compute the output, $y[n]$, using the impulse response we
use \emph{convolution}. Namely, we \emph{convolve} the input, $x[n]$,
by the impulse response, $h[n]$,
\begin{equation}
  y[n] = x[n] \ast h[n]  = \sum_{k=0}^{9}x[k]h[n-k]
\end{equation}
And, indeed, Matlab has a \verb|conv| function to do this convolution.
Alternatively, we can compute a filter's output by multiplying the
transfer function by the z-transform of the input to yield the
z-transform of the output:
\begin{equation}
  Y(z) = H(z) X(z)
\end{equation}
From a practical point of view, of course, it makes more sense to
implement a filter in terms of its impulse response. However, for
filters with long impulse responses, it is sometimes more convenient
to represent them mathematically using the transfer function (which we
now know is just the z-transform of the impulse response!).

\subsection{Z-Transforms}

\paragraph{Step 1.1} On paper, compute the z-transform, $X(z)$, of
	\begin{equation}
		x[n] = \left\{
		\begin{array}{ll}
			(-1)^n & n \ge 0 \\
			0 & n < 0
		\end{array}\right.
	\end{equation}
	Note that this is an infinite geometric series.  What are the
        locations of any pole(s) (roots of the denominator polynomial)
        or zero(s) (roots of the numerator polynomial)?

\paragraph{Step 1.2} Evaluate the frequency response of $X(z)$
        from step~1.1, $X(z)\big{|}_{z=e^{j\hat{\omega}}}$, by
        sketching it by hand.  What kind of filter is this?

\paragraph{Step 1.3} Consider the z-transform:
	\begin{equation}
	  X(z) = 1 - 2z^{-1} + 3z^{-3} - z^{-5}
	\end{equation}
	Write the inverse z-transform, $x[n]$, as a table of values for
	corresponding $n$ values.

\subsection{Impulse Response}

\paragraph{Step 2.1} Consider a filter with a transfer function
	\begin{equation}
	H(z) = 1 + 5z^{-1} - 3z^{-2} + 2.5z^{-3} + 4z^{-8}
	\end{equation}
	What is the defining equation for this filter, $y[n] = F(x[n])$?

\paragraph{Step 2.2} What is the output sequence of the filter of
	Step~2.1 when the input is $x[n] = \delta[n]$? Verify this
        using Matlab.

\paragraph{Step 2.3} The impulse response of a filter is $h[n]
        = \delta[n] + 2\delta[n-1] + \delta[n-2] - \delta[n-3]$, or equivalently,
        $h[n]=\{1,2,1,-1\}$, $n=\{0, 1, 2, 3\}$. Determine the
        response of the system to the input signal $x[n]=\{1,2,3,1\}$,
        $n=\{0, 1,2,3\}$ by hand. Use Matlab to check your
        results. Include a figure that shows both the input and output
        signals; make sure the reader can clearly see what the signal
        values are (the \verb|stem| plot function should help to
        ensure this is the case).

\paragraph{Step 2.4} Change the input to the filter of
        Step~2.3 to be $\delta[n]$. What are the output values? How do
        they compare to the impulse response? Include plots of the
        filter input and output values in your report.

\paragraph{Step 2.5} Use Matlab to determine the output of the
        filter \{1/3,1/3,1/3\}, $n=\{0,1,2\}$ for the input:
	\begin{equation}
	  x[n] = 4 + \sin[0.25\pi(n-1)] - 3 \sin[(2\pi/3)n]
	\end{equation}
        Include a listing of your Matlab code and a figure with
        plots of the filter input and output in your report. Is the
        result expected? Why or why not?

\paragraph{Step 2.6} Create your own Matlab \emph{function},
        \verb|convolution|, to implement a convolution function.  To
        test your function, make sure it works exactly like the Matlab
        \verb|conv| and \verb|filter| functions by providing the same
        input to each and subtracting their outputs. Use the filter
        $\{1/3,1/3,1/3\}$, $n=\{0,1,2\}$ from step 2.5.

\subsection{Canceling Sinusoidal Components}

Filters can be designed to cancel sinusoids.  Implement a filter in
Matlab with the following impulse response:
\begin{equation}
h[n] = \delta[n] - 2\cos(\pi/4) \delta[n-1] + \delta[n-2]
\end{equation}

\paragraph{Step 3.1} Plot the frequency response for this filter. What
	are the zero locations?

\paragraph{Step 3.2} Use as an input to this filter the signal
        $x[n] = \sin\hat{\omega}n$, using the two frequencies
        $\hat{\omega} = \pi/2$ and $\hat{\omega} = \pi/4$. You will
        need to choose appropriate analog signals, with convenient
        frequencies and durations, and then sample them appropriately
        so they have the correct digital frequencies. Make sure to
        verify that you get the correct digital frequencies and that
        plots you make are convenient for the reader (for example,
        neither too many nor too few cycles)! Compute the filter in
        Matlab for each of these two inputs, plotting the input and
        output of each. When do you get cancellation?

\paragraph{Step 3.3} Can you modify the filter coefficients to cancel
	the other sinusoid? If so, show your work.

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\section{The \iotai algorithm}
\Label{sec:iotai}

The \iotai algorithm in the \cxx Standard Library \cite[\S 29.8.12]{cxx-17-draft} assigns sequentially
increasing values to a range,
where the initial value is user-defined.
%
Our version of the original signature
reads:

\begin{lstlisting}[style=acsl-block]

  void iota(value_type* a, size_type n, value_type v);
\end{lstlisting}

Starting at \inl{v}, the function assigns consecutive
integers to the elements of the range \inl{a}.
When specifying \iotai we must be careful to deal with possible overflows of the argument~\inl{v}.

\subsection{Formal specification of \iotai}

The specification of \iotai relies on the logic function \logicref{IotaGenerate}
that is defined in the following listing.

\input{Listings/IotaGenerate.acsl.tex}

The specification of \iotai is shown in the following listing.
It uses the logic function \logicref{IotaGenerate} in order to express the
postcondition~\inl{increment}.

\input{Listings/iota.h.tex}

The specification of \iotai refers to \inl{VALUE_TYPE_MAX} which is
the maximum value of the underlying integer type (see Listing~\ref{lst:value-type-limits}).
In order to avoid integer overflows
the sum  \inl{v+n} must not be greater than the constant \inl{VALUE_TYPE_MAX}.

\clearpage

\subsection{Implementation of \iotai}

The following listing shows an implementation of the \iotai function.

\input{Listings/iota.c.tex}

The loop invariant \inl{increment} describes that in each iteration of the loop the current
value~\inl{v} is equal to the sum of the value \inl{v} in state of function
entry and the loop index \inl{i}.
We have to refer here to \inl{\\at(v,Pre)} which is the value on entering \iotai.

\clearpage


\Lecture{Jayalal Sarma}{Oct 19, 2020}{18}{Introduction to Ramsey Numbers}{Shivlal Gangesh}{$\alpha$}{JS}
\section{Introduction}
Till now we have seen advanced versions of the discrete mathematics topics we already know. Now we are going to get into Extremal combinatorics. Here we are interested in questions of the form
\begin{itemize}
\item \textit{If this structure appears, then what is the minimum/maximum size of the object?}
\item \textit{If the size is at least this much , then what kind of structures appear in the object?}
\item \textit{What is the minimum size of the collection such that it is guaranteed to have certain property?}
\end{itemize}
In general we are interested in the extreme behaviors in combinatorics. The classic example we start with is an extension to an example that we have done in the beginning of the course as an Application of Pigeon Hole Principle.
\section{Starting Point} \label{R(3,3)}
\begin{theorem}
Six people meet in a party. Then either there exist three people who are friends with each other or there exist three people who are strangers with each other.(Note : Any two people can either be friends or strangers)
\end{theorem}
We are interested in proving the above statement. Lets look into two different approaches
\subsection{Model 1 (Using Cliques and Independent Sets)}
\begin{description}
   \item[Model] Let us represent the problem as a $6$ vertex graph $G(V,E) $with each person corresponding to a vertex. $(u,v) \in E$ if and only if person $u$ is a friend of person $v$.
In this Model the original statement can be reformulated as
\item[Statement]
\textit{Any graph on $6$ vertices must either have a clique on $3$ vertices or an independent set on $3$ vertices.
\item}
\begin{proof}
 Consider any vertex $v$ in the graph G, without loss of generality we can assume that the degree of $v$ is greater than or equal to $3$  because suppose it is not the case then consider $\overline{G}$ ; as $\textrm{Cliques in }G \leftrightarrow \textrm{Independent Sets in } \overline{G} $.\\
 Let the $3$ neighbours of $v$ be $a$, $b$ and $c$. Consider the two exhaustive cases :
 \begin{description}
    \item[Case 1 : There are no edges among $a$, $b$ and $c$]
    $ $ \newline
    Here we have $\{a, b, c\}$ as the 3-Independent Set
    \item[Case 2 : There is at least one edge among $a$, $b$ and $c$ ]
    $ $ \newline
    Let $(a,b) \in E$ be that edge, then we have $\{v, a, b\}$ as the 3-clique
 \end{description}
Therefore the given statement holds true.
\end{proof}
\item[Proof for tightness]
To prove that this is tight we need to show there is a graph with $5$ vertices such that it does not have 3-clique and 3-Independent Set. Given below is one such  example
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.2\linewidth]{images/r33counter_example.png}
    \caption{5-vertex graph with no 3-clique and no 3-Independent Set}
\end{figure}
\end{description}
\subsection{Model 2 (Using Graph Edge colouring)}
\begin{description}
   \item[Model]
   Let us represent the problem as 2-edge coloring of a $K_6$ graph with each vertex corresponding to a person. Color the edge $(u,v)$ with \textit{red} if $u$ and $v$ are friends, color it with \textit{blue} if $u$ and $v$ are strangers.
In this Model the original statement can be reformulated as
\item[Statement]
\textit{For any 2-edge colouring of $K_6$, there must exist either  a red $K_3$  or a blue $K_3$ }
\item
\begin{proof}
 Consider any Red,Blue-edge coloring of $K_6$. Consider any vertex $v$, the degree of $v$ is $5$ as the graph is a complete graph. By Pigeon Hole Principle , $v$ must have either $3$ red edges incident on it or $3$ blue edges incident on it. Consider the case when $v$ is incident on with $3$ red edges. Let the 3 neighbours of $v$ be $a$, $b$ and $c$.  Now there are 2 cases :
 \begin{description}
    \item[Case 1 : There is no red colored edge among $(a,b)$, $(b,c)$ and $(c,a)$ ]
    $ $ \newline
    Then all the three edges $(a,b)$, $(b,c)$ and $(c,a)$ are colored blue. Therefore $\{a, b, c\}$ forms a blue $K_3$
    \item[Case 2 : There is at least one red colored edge among $(a,b)$, $(b,c)$ and $(c,a)$ ]
    $ $ \newline
    Let $(a,b)$ be the red colored edge, then $\{v, a, b\}$ forms a red $K_3$
 \end{description}
 Therefore the given statement holds true.
\end{proof}
\item[Proof for tightness]
To prove that this is tight we need to show there is a 2-edge coloring of $K_5$ Such that it does not have red $K_3$ and blue $K_3$. Given below is one such  example
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.2\linewidth]{images/k5counter_example.png}
    \caption{2-edge coloring of $K_5$ with no red $K_3$ and no blue $K_3$}
\end{figure}
\end{description}

Generalizing the above problem with arbitrary red $k_p$ and blue $k_q$ has been extensively studied by Ramsey and has led to the definition of Ramsey numbers.
\section{Ramsey numbers}
\begin{definition}[Ramsey number]
The Ramsey number denoted by $R(p,q)$ is the minimum number of vertices say $n$ such that any 2-edge coloring of $K_n$ must have either a red $K_p$ or a blue $K_q$.\\
(\textbf{Or equivalently as})\\
The minimum number of vertices ($n$) such that any graph on $n$ vertices must either have a clique on $p$ vertices or an independent set on $q$ vertices.
\end{definition}

\subsection{Some Observations}
\begin{property}
$R(3,3)=6$
\end{property}
This is the direct formulation of the example we have done previously in \ref{R(3,3)}
\begin{property}
$R(p,q) = R(q,p)$
\end{property}
The colors \textit{red} and \textit{blue} are just placeholders for two colors, thus swapping the colors will still preserve the Ramsey number property. Therefore $R(p,q) = R(q,p)$.
\begin{property}
$\forall l \geq 1 \quad R(l,1) = 1$
\end{property}
The existence of a blue $K_1$ is nothing but the presence of single vertex and any graph with a single vertex satisfies this property. Therefore $R(l,1) = 1$

\section{Existence of R(p,q)}
The Proof for the existence of $R(p,q)$ is due to Erdős–Szekeres. The existence was proved by providing an upper bound as a recurrence relation as follows :
\begin{theorem}
$$\forall p,q \geq 2 \quad R(p,q) \; \leq \; R(p,q-1) + R(p-1,q) $$
\end{theorem}
\begin{proof}
 Let us prove this by mathematical induction on $n$ where $n=p+q$.
 \begin{description}
    \item[Idea] To show the upper bound for $R(p,q) \leq n$ , we must argue that for any 2-edge coloring of $K_n$ there exist a red $K_p$ or blue $K_q$

   \item[Base case] $p=q=2$
$$R(2,2) \leq  R(2,1) + R(1,2)  $$
$$2 \leq 1+1$$
Hence it holds true for the base case.
   \item[Induction Hypothesis]
Assume the recurrence relation is true for $n<l$. Then we need to prove it for $n=l$. Let $n=R(p-1,q)+R(p,q-1)$. Let $w$ be any vertex in $G$ ($K_n$) and consider any $2$ -edge coloring of $G$. Let $H_1$ be the subgraph of $G$ formed from the vertices sharing a red-edge with $v$ and $H_2$ be the the subgraph of $G$ formed from the vertices sharing a blue-edge with $v$.
\begin{description}
   \item[Case 1 : There are at least $R(p-1,q)$ many red edges incident on vertex $w$]
   $ $ \newline
   $H_1$ is a complete graph on $R(p-1,q)$ vertices with 2-edge coloring. By definition and Induction Hypothesis we have that there exist a red $K_{p-1}$ or blue $K_q$ in $H_1$. So in graph $G$ (along with vertex $w$) there exist a red $K_p$ or blue $K_q$
   \item[Case 2 : There are at least $R(p,q-1)$ blue edges incident on vertex $w$]
      $ $ \newline
      $H_2$ is a complete graph on $R(p,q-1)$ vertices with 2-edge coloring. By definition and Induction Hypothesis we have that there exist a red $K_p$ or blue  $K_{q-1}$ in $H_2$. So in graph $G$ (along with vertex $w$) there exist a red $K_p$ or blue $K_q$.
\end{description}
   \end{description}

\end{proof}

\Lecture{Jayalal Sarma}{Oct 21, 2020}{19}{Computing Ramsey Numbers and Multidimensional Ramsey numbers}{Shivlal Gangesh \& Reetwik Das}{$\alpha$}{JS}
\section{Generalizing Ramsey numbers}
\begin{definition}[3-dimensional Ramsey numbers]
$R_3(p,q,r)$ is the minimum number $n$, such that any 3-edge coloring $K_n$ must have either a red $K_p$ or a blue $K_q$ or a green $K_r$
\end{definition}
\begin{definition}[$k$-dimensional Ramsey numbers]
$R_k(s_1,s_2,\cdots ,s_k)$ is the minimum number of vertices $n$ such that for any $k$-edge coloring of $K_n$  there must exist an $i$ such that there is a $K_{s_i}$ of colour $i$
\end{definition}

\section{Some Observations}
\begin{property}
$R(2,p)=p$
\end{property}
\begin{proof}
$ $
 \begin{description}
    \item[Case1 : $R(2,p) \leq p$]
    $ $ \newline
    Any 2-coloring of $K_p$ must have either a red $K_2$ or blue $K_p$. This is true because either there can exist a red edge (red $K_2$) or no red edge (blue $K_p$) in $K_p$
    \item[Case 2 : $R(2,p) \geq p$]
    $ $ \newline
    There exist a 2-coloring of edges of $K_{p-1}$ such that no red $K_2$ exists and no blue $K_p$ exists. Coloring all the edges of $K_{p-1}$ with blue will result in no red $K_2$ and no blue $K_p$ in $K_{p-1}$
 \end{description}
\end{proof}
\begin{claim}
$$ R(p,q) \leq {p+q-2 \choose p-1} $$
\end{claim}
\begin{proof}
 \begin{align*}
     R(p,q) &\leq  R(p,q-1) + R(p-1,q) && \textrm{(Erdos-Szekeres recurrence relation)} \\
     &\leq {p+(q-1)-2 \choose p-1} + {p-1+q-2 \choose p-2} \\
     &\leq {p+q-3 \choose p-1} + {p+q-3 \choose p-2} \\
     &\leq {p+q-2 \choose p-1}  && ({n+1 \choose k+1} = {n \choose k+1} +{n \choose k})
 \end{align*}

\end{proof}
\section{Explicit Computation of R(3,4)}
We don't know the exact values of Ramsey numbers for higher values as their computation becomes very hard. There is this famous saying by Paul Erdos on the difficulty of computing Ramsey numbers that
\begin{description}
   \item[Paul Erdos on Ramsey numbers]
   $ $ \newline
\textit{   "Suppose aliens invade the earth and threaten to obliterate it in a year's time unless human beings can find the Ramsey number for red five and blue five. We could marshal the world's best minds and fastest computers, and within a year we could probably calculate the value. If the aliens demanded the Ramsey number for red six and blue six, however, we would have no choice but to launch a preemptive attack."}
\end{description}
 So let us now try to calculate the value of $R(3,4)$.
 \begin{claim}
 $$ R(3,4) = 9 $$
 \end{claim}
 \begin{proof}
 $ $
 Consider any 2-coloring of $K_9$ and call it as $G$. We need to prove that $G$ has either a red $K_3$ or a blue $K_4$. Any vertex in $G$ can have it's incident edges as one of the three cases below
  \begin{itemize}
  \item \textbf{Case 1 : }There are at least $4$ red edges going out of the vertex
  \item \textbf{Case 2 : }There are at least $6$ blue edges going out of the vertex
  \item \textbf{Case 3 : }There are exactly $3$ red edges and $5$ blue edges going out of the vertex
  \end{itemize}
 However note that not all vertices in $G$ come under \textbf{Case 3} because, if so then the total sum of degrees of all vertices becomes odd which is not possible. So let $v$ be a vertex in $G$ which does not fall under \textbf{Case 3}. Then
 \begin{description}
    \item[Case 1 : There are at least 4 red edges going out of $v$ ]
    $ $ \newline
    Let $H_1$ be the subgraph of $G$ formed from the four vertices which are sharing the red edge with $v$.  Since we know that $R(2,4)=4$, $H_1$ with 4 vertices must have a red $K_2$ or a blue $K4$, So along with vertex $v$, $G$ must have a red $K_3$ or a blue $K_4$.
    \item[Case 2 :There are at least 6 blue edges going out of $v$]
    $ $ \newline
    Let $H_2$ be the subgraph of $G$ formed from the six vertices which are sharing the blue edge with $v$. Since we know that $R(3,3)=6$, $H_2$ with 6 vertices must have a red $K_3$ or a blue $K3$, So along with vertex $v$, $G$ must have a red $K_3$ or a blue $K_4$.
    \item[Proof for tightness]
    $ $ \newline
    To prove that $9$ is tight, we need to show that there is a 2-coloring of $K_8$ such that it does not have red $K_3$ or blue $K_4$. Given below is one such  example
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.5\linewidth]{images/R34counter_example.png}
    \caption{2-coloring of $K_8$ with no red $K_3$ and no blue $K_4$}
\end{figure}
 \end{description}

 \end{proof}
 As the values $p$, $q$ increases we can only calculate the range of the Ramsey number. The following is a table with value or range of Ramsey numbers for the first few natural numbers.
 \begin{figure}[h!]
    \centering
    \includegraphics[width=1\linewidth]{images/RamseyTable.png}
    \caption{Table for $R(p,q)$}
\end{figure}

% Reetwik Started from here

\section{Multidimentional Ramsey numbers}
\begin{definition}
$R_k(S_1,S_2,..S_k)$ is mininum number $n$ such that any $k$-edge coloring of $K_n$ must have $K_{S_i}$ of color $i$ for some $i \in \{1,2..k\}$
\end{definition}
We need to show that why should exist $R_k(S_1,S_2,..S_k)$.
\begin{claim}
$$R_k(S_1,S_2,..S_k) \leq R_{k/2}(R(S_1,S_2), R(S_3,S_4).... R(S_{k-1},S_k))$$
\end{claim}
\begin{proof}
Let $n =  R_{k/2}(R(S_1,S_2), R(S_3,S_4).... R(S_{k-1},S_k))$\\
Consider $K_n$ and any K-edge coloring of the edges of $K_n$\\

We need to show that $\exists S_1$ clique of color 1 or $S_2$ clique of color 2.\\
Consider colors paired up and rename them $\{1,2\} = 1, \{3,4\} = 2, ... \{k-1,k\} = k/2$ \\
By the definition of $R_{k/2}$ we are guaranteed $\exists i$ such that $\exists$ a clique of size $R(S_{2i-1},S_{2i})$ of color $i$.\\

After we uninterpret the color $i$ as the original pair of colors we get a 2-coloring of the clique that we have $R(S_{2i-1},S_{2i})$\\
By the definition of $R_2$ we know that $\exists$ a $S_{2i-1}$ clique of color $2i-1$ or $S_{2i}$ clique of color $2i$.
\end{proof}

\section{Fermat's last theorem}
We know from Pythagoras theorem that $x^2 +y^2 = z^2$ has integral solutions. But we want to know if this equation has any integral solutions for any power greater than $2$.
\begin{theorem}
$x^n +y^n = z^n$ doesn't have any integral solutions $\forall n>2$.
\end{theorem}

\Lecture{Jayalal Sarma}{Oct 22, 2020}{20}{Finite fields}{Reetwik Das}{$\alpha$}{JS}

\subsection{Finite fields}
$x^n +y^n = z^n$ does have integeral solutions for finite fields such as for $Z_p$.\\
$Z_p = \{0,1,....p-1\}$ and addition and multiplication are $modulo$ $p$ within this field.\\

\begin{definition}
$Z_p^*$ is a cyclic group $\{1,2,3....p-1\}$
\end{definition}

\begin{claim}
If $p$ is a prime then $Z_p^*$ is generated by a single element, and the element is known as the generator.
\end{claim}

Fermat's last theorem is completely algebraic to connect it to coloring we need a tool.
\begin{theorem}
\textbf{Schur's theorem :} If $r \geq 0$ positive integer then. $\exists$ integer $S(r)$ such that if we color $\{1,2,....S(r)\}$ vertices with $r$ colors then $\exists x,y,z$ in the set and $x+y =z$
\end{theorem}
\begin{proof}
Given an $r$, Let $S(r) = R_r(3,3,3,...3)$\\
Consider $K_n$, $n = S(r)$ by the definition if we color the edges of $K_n$ using $r$ colors then we are guaranteed a monochromatic $K_3$.\\
We are given a coloring $\{1,2,...S(r)\}$\\
Define a coloring for edges in $K_n$.\\
Associate vertices of $K_n$ with elements in $\{1,2,...S(r)\}$ \\
$\forall a,b\in V$ the color of edge $(a,b) $ = color of $|a-b|$\\

Let $\{\alpha, \beta,\gamma\}$ be the vertices of the monochromatic triangle.
Let $x = \alpha - \beta$, $y = \beta - \gamma$ and $z = \alpha - \gamma$ then $x,y,z$ have the same color.
It also satifies the equation $x+y =z$.

\end{proof}

\begin{theorem}
$\forall m \exists q$ such that $\forall p\geq q$ in $Z_p$ ($p$ is a prime) \\
$x^m +y^m = z^m$ has a solution.
\end{theorem}
\begin{proof}
Given $m$ from $ x^m +y^m = z^m$\\
$p=q=S(m)+1$ by Schur's theorem any coloring of $\{1,2...q\}$ must have a triplet $a+b = c$.\\
$Z_p = {0,1,2...q-1}$\\
Let $g$ be the generator of $Z_p$ then every non-zero element in $Z_p = g^k$ for some $k$.\\

Assign the coloring $\{1,2...q\}$ as follows :\\
$\forall x \in Z_p^*,  x=g^{mi+j}$ and $color(x)= j = k (mod m)$\\

By Schur's theorem, $\exists a,b,c$ such that $a+b=c$ and all have the same color.\\
$$g^{mi_a+j} + g^{mi_b+j} = g^{mi_c+j}$$
$$(g^{i_a})^m + (g^{i_b})^m = (g^{i_c})^m$$
and we have the solution for $x^m +y^m = z^m$.
\end{proof}

\subsection{Lower bounds for Ramsey numbers}
\begin{claim}
$$\forall k, R(k,k) > 2^{k/2}$$
\end{claim}
\begin{proof}
Suffices to show that $n = 2^{k/2}$, $\exists$ a 2-coloring of the edges of $K_n$ such that there is no monochromatic $K_k$ in it.\\

Fix $m = 2^{k/2}$ there are $n\choose 2$ many edges.\\
A coloring is said to be bad if $\exists$ no monochromatic $K_k$ in it.\\

\textbf{Probabilistic method :}\\
For every edge, assign red/blue color with probability 1/2 each.\\
if we show that the probability[coloring is bad]$ > 0$ then this means $\exists$ a bad coloring.\\

Suffices to show that the Pr[coloring is good] $<1$\\
Pr[$\exists K_k$ which is monochromatic] $\leq \Sigma_{S \subseteq K_k,|S|=k}$ Pr[S is monochromatic]\\
$ = {n\choose k}$ Pr[S is monochromatic]
$$ = {n\choose k} \frac{2}{2^{k\choose 2}}$$
$$= {n\choose k} 2^{1-{k\choose2}}$$
$$ =\frac{n(n-1)...(n-k+1)}{k!} \frac{2^{1+k/2}}{2^{k^2/2}}$$
$$ \leq \frac{n^k}{k!} \frac{2^{1+k/2}}{2^{k^2/2}}$$
$$= \frac{2^{1+k/2}}{k!} < 1$$
\end{proof}


\lab{Convolution and Filtering}{Convolution and Filtering}
\objective{The Fourier transform reveals information in the frequency domain about signals and images that might not be apparent in the usual time (sound) or spatial (image) domain.
% It provides an extremely powerful tool to analyze difficult problems simply.
In this lab, we use the discrete Fourier transform to efficiently convolve sound signals and filter out some types of unwanted noise from both sounds and images.
This lab is a continuation of the Discrete Fourier Transform lab and should be completed in the same Jupyter Notebook.}

\section*{Convolution} % ======================================================

Mixing two sounds signals---a common procedure in signal processing and analysis---is usually done through a \emph{discrete convolution}.
% Recall that sound is modeled with discrete samples of a sound wave in rapid succession.
Given two periodic sound sample vectors $\f$ and $\g$ of length $n$, the discrete convolution of $\f$ and $\g$ is a vector of length $n$ where the $k$th component is given by
\begin{align}
(\f \ast \g)_k = \sum_{j=0}^{n-1} f_{k-j}g_j,\qquad k = 0,1,2,\dots,n-1.
\label{eq:naive-convolve}
\end{align}

Since audio needs to be sampled frequently to create smooth playback, a recording of a song can contain tens of millions of samples; even a one-minute signal has $2,646,000$ samples if it is recorded at the standard rate of $44,100$ samples per second ($44,100$ Hz).
The na\"ive method of using the sum in \eqref{eq:naive-convolve} $n$ times is $O(n^2)$, which is often too computationally expensive for convolutions of this size.

Fortunately, the discrete Fourier transform (DFT) can be used compute convolutions efficiently.
The \emph{finite convolution theorem} states that the Fourier transform of a convolution is the element-wise product of Fourier transforms:
\begin{align}
F_n(\f \ast \g) = n(F_n\f)\odot (F_n\g).
\label{eq:convolution-theorem}
\end{align}
In other words, convolution in the time domain is equivalent to component-wise multiplication in the frequency domain.
Here $F_n$ is the DFT on $\mathbb{R}^n$, $\ast$ is discrete convolution, and $\odot$ is component-wise multiplication. % (the \emph{Hadamard product}).
Thus, the convolution of $\f$ and $\g$ can be computed by
\begin{align}
\f \ast \g = nF_n^{-1}((F_n\f)\odot (F_n\g)),
\label{eq:fft-convolution}
\end{align}
where $F_n^{-1}$ is the \emph{inverse discrete Fourier transform} (IDFT).
The fast Fourier transform (FFT) puts the cost of \eqref{eq:fft-convolution} at $O(n\log{n})$, a huge improvement over the na\"ive method.

\begin{info} % Discard imaginaries after IDFT
Although individual samples are real numbers, results of the IDFT may have small complex components due to rounding errors.
These complex components can be safely discarded by taking only the real part of the output of the IDFT.

\begin{lstlisting}
>>> import numpy
>>> from scipy.fftpack import fft, ifft # Fast DFT and IDFT functions.

>>> f = np.random.random(2048)
>>> f_dft_idft = ifft(fft(f)).real      # Keep only the real part.
>>> np.allclose(f, f_dft_idft)          # Check that IDFT(DFT(f)) = f.
<<True>>
\end{lstlisting}
\end{info}

\begin{warn} % SciPy conventions
SciPy uses a different convention to define the DFT and IDFT than this and the previous lab, resulting in a slightly different form of the convolution theorem.
Writing SciPy's DFT as $\hat{F}_n$ and its IDFT as $\hat{F}_n^{-1}$, we have $\hat{F}_n = n F_n$, so \eqref{eq:fft-convolution} becomes
\begin{align}
\f \ast \g = \hat{F}_n^{-1}((\hat{F}_n\f)\odot (\hat{F}_n\g)),
\label{eq:fft-convolution-scipy}
\end{align}
without a factor of $n$.
Use \eqref{eq:fft-convolution-scipy}, not \eqref{eq:fft-convolution}, when using \li{fft()} and \li{ifft()} from \li{scipy.fftpack}.
\end{warn}

\subsection*{Circular Convolution} % ------------------------------------------

The definition \eqref{eq:naive-convolve} and the identity \eqref{eq:fft-convolution} require $\f$ and $\g$ to be periodic vectors.
However, the convolution $\f \ast \g$ can always be computed by simply treating each vector as periodic.
The convolution of two raw sample vectors is therefore called the \emph{periodic} or \emph{circular convolution}.
This strategy mixes sounds from the end of each signal with sounds at the beginning of each signal.

\begin{problem} % Circular convolution.
\item Implement the \li{__mul__()} magic method for the \li{SoundWave} class so that if \li{A} and \li{B} are \li{SoundWave} instances, \li{A * B} creates a new \li{SoundWave} object whose samples are the circular convolution of the samples from \li{A} and \li{B}.
If the samples from \li{A} and \li{B} are not the same length, append zeros to the shorter array to make them the same length before convolving.
Use \li{scipy.fftpack} and \eqref{eq:fft-convolution-scipy} to compute the convolution, and raise a \li{ValueError} if the sample rates from \li{A} and \li{B} are not equal.

A circular convolution creates an interesting effect on a signal when convolved with a segment of white noise: the sound loops seamlessly from the end back to the beginning.
To see this, generate two seconds of white noise (at the same sample rate as \texttt{tada.wav}) with the following code.
\begin{lstlisting}
>>> rate = 22050        # Create 2 seconds of white noise at a given rate.
>>> white_noise = np.random.randint(-32767, 32767, rate*2, dtype=np.int16)
\end{lstlisting}
Next, convolve \texttt{tada.wav} with the white noise.
Finally, use the \li{>>} operator to append the convolution result to itself.
This final signal sounds the same from beginning to end, even though it is the concatenation of two signals.
\end{problem}

\subsection*{Linear Convolution} % --------------------------------------------

Although circular convolutions can give interesting results, most common sound mixtures do not combine sounds at the beginning of one signal with sounds at the end of another.
Whereas circular convolution assumes that the samples represent a full period of a periodic function, \emph{linear convolution} aims to combine non-periodic discrete signals in a way that prevents the beginnings and endings from interacting.
Given two samples with lengths $n$ and $m$, the simplest way to achieve this is to pad both samples with zeros so that they each have length $n+m-1$, compute the convolution of these larger arrays, and take the first $n+m-1$ entries of that convolution.

\begin{problem} % Linear convolution.
\item Implement the \li{__pow__()} magic method for the \li{SoundWave} class so that if \li{A} and \li{B} are \li{SoundWave} instances, \li{A ** B} creates a new \li{SoundWave} object whose samples are the linear convolution of the samples from \li{A} and \li{B}.
Raise a \li{ValueError} if the sample rates from \li{A} and \li{B} are not equal.

Because \li{scipy.fftpack} performs best when the length of the inputs is a power of $2$, start by computing the smallest $2^a$ such that $2^a \ge n + m - 1$, where $a\in\mathbb{N}$ and $n$ and $m$ are the number of samples from \li{A} and \li{B}, respectively.
Append zeros to each sample so that they each have $2^a$ entries, then compute the convolution of these padded samples using \eqref{eq:fft-convolution-scipy}.
Use only the first $n + m - 1$ entries of this convolution as the samples of the returned \li{SoundWave} object.

To test your method, read \texttt{CGC.wav} and \texttt{GCG.wav}.
Time (separately) the convolution of these signals with \li{SoundWave.__pow__()} and with \li{scipy.signal.fftconvolve()}.
Compare the results by listening to the original and convolved signals.
\label{prob:fft-linear-convolution}
\end{problem}

\begin{problem} % Convolve the balloon pop with Chopin.
Clapping in a large room with an echo produces a sound that resonates in the room for up to several seconds.
This echoing sound is referred to as the \emph{impulse response} of the room, and is a way of approximating the acoustics of a room.
When the sound of a single instrument in a carpeted room is convolved with the impulse response from a concert hall, the new signal sounds as if the instrument is being played in the concert hall.

The file \texttt{chopin.wav} contains a short clip of a piano being played in a room with little or no echo, and \texttt{balloon.wav} is a recording of a balloon being popped in a room with a substantial echo (the impulse).
Use your method from Problem \ref{prob:fft-linear-convolution} or \li{scipy.signal.fftconvolve()} to compute the linear convolution of \texttt{chopin.wav} and \texttt{balloon.wav}.
% Listen to the new signal; there should be echo in the piano recording.
\end{problem}

\section*{Filtering Frequencies with the DFT} % ===============================

The DFT also provides a way to clean a signal by altering some of its frequencies.
Consider \texttt{noisy1.wav}, a noisy recording of a short voice clip.
The time-domain plot of the signal only shows that the signal has a lot of static.
On the other hand, the signal's DFT suggests that the static may be the result of some concentrated noise between about $1250$--$2600$ Hz.
Removing these frequencies could result in a much cleaner signal.

\begin{figure}[H]
\captionsetup[subfigure]{justification=centering}
\centering
\begin{subfigure}{.53\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/noisy1.pdf}
\end{subfigure}%
\begin{subfigure}{.47\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/noisy1_dft.pdf}
\end{subfigure}
\caption{The time-domain plot (left) and DFT (right) of \texttt{noisy1.wav}.}
\label{fig:fft-noisy-signal}
\end{figure}

To implement this idea, recall that the $k$th entry of the DFT array $\c = F_n \f$ corresponds to the frequency $v = k r / n$ in Hertz, where $r$ is the sample rate and $n$ is the number of samples.
Hence, the DFT entry $c_k$ corresponding to a given frequency $v$ in Hertz has index $k = v n / r$, rounded to an integer if needed.
In addition, since the DFT is symmetric, $c_{n-k}$ also corresponds to this frequency. %; any changes to $c_k$ must also be done to $c_{n-k}$ in order to alter the signal.
This suggests a strategy for filtering out an unwanted interval of frequencies $[v_\text{low},v_\text{high}]$ from a signal:
\begin{enumerate}
\item Compute the integer indices $k_\text{low}$ and $k_\text{high}$ corresponding to $v_\text{low}$ and $v_\text{high}$, respectively.
\item Set the entries of the signal's DFT from $k_\text{low}$ to $k_\text{high}$ and from $n-k_\text{high}$ to $n-k_{low}$ to zero, effectively removing those frequencies from the signal.
\item Take the IDFT of the modified DFT to obtain the cleaned signal.
\end{enumerate}
Using this strategy to filter \texttt{noisy1.wav} results in a much cleaner signal. %, shown in Figure \ref{fig:fft-noisy-cleaned}.
However, any ``good'' frequencies in the affected range are also removed, which may decrease the overall sound quality.
The goal, then, is to remove only as many frequencies as necessary.

\begin{figure}[H]
\captionsetup[subfigure]{justification=centering}
\centering
\begin{subfigure}{.53\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/noisy1_clean.pdf}
\end{subfigure}%
\begin{subfigure}{.47\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/noisy1_clean_dft.pdf}
\end{subfigure}
\caption{The time-domain plot (left) and DFT (right) of \texttt{noisy1.wav} after being cleaned.}
\label{fig:fft-noisy-cleaned}
\end{figure}

\begin{problem} % Clean up a noisy signal (The only thing we have to fear...)
\label{prob:fft-filter-frequencies}
Add a method to the \li{SoundWave} class that accepts two frequencies $v_\text{low}$ and $v_\text{high}$ in Hertz.
Compute the DFT of the stored samples and zero out the frequencies in the range $[v_\text{low}, v_\text{high}]$ (remember to account for the symmetry DFT).
Take the IDFT of the altered array and store it as the sample array.
% \\(Hint: be sure to sure the indices of the DFT array are integers.)

Test your method by cleaning \texttt{noisy1.wav}, then clean \texttt{noisy2.wav}, which also has some artificial noise that obscures the intended sound.
\\(Hint: plot the DFT of \texttt{noisy2.wav} to determine which frequencies to eliminate.)
\end{problem}

A digital audio signal made of a single sample vector with is called \emph{monoaural} or \emph{mono}.
When several sample vectors with the same sample rate and number of samples are combined into a matrix, the overall signal is called \emph{stereophonic} or \emph{stereo}.
This allows multiple speakers to each play one \emph{channel}---one of the original sample vectors---simultaneously.
``Stereo'' usually means there are two channels, but there may be any number of channels ($5.1$ surround sound, for instance, has five).

% \begin{warn}
Most stereo sounds are read as $n\times m$ matrices, where $n$ is the number of samples and $m$ is the number of channels (i.e., each column is a channel).
However, some functions, including Jupyter's embedding tool \li{IPython.display.Audio()}, receive stereo signals as $m\times n$ matrices (each row is a channel).
Be aware that both conventions are common.
% \end{warn}

% When a digital audio signal is played on a computer, the signal is sent to a speaker, which vibrates, producing sound waves.
% When multiple speakers are used, they can all produce the same signal or they can produce different signals.
% When there is only one signal, the sound is \emph{monoaural}, or \emph{mono}.  When speakers produce more than one signal, the overall signal is \emph{stereophonic}, or \emph{stereo}.
% Usually stereo means two, but there may be any number of signals ($5.1$ surround sound, for instance, has $5$).

% Most sounds are multi-dimensional, but can be treated similarly to mono sounds.
% The first and second columns of the array sample correspond to the signals for the left and right speaker respectively.

\begin{problem}
\label{prob:fft-vuvuzelas}
During the 2010 World Cup in South Africa, large plastic horns called vuvuzelas were blown excessively throughout the games.
Broadcasting organizations faced difficulties with their programs due to the incessant noise level.
Eventually, audio filtering techniques were used to cancel out the sound of the vuvuzela, which has a frequency of around $200$--$500$ Hz.

The file \li{vuvuzela.wav}\footnote{See \url{https://www.youtube.com/watch?v=g_0NoBKWCT8}.} is a stereo sound with two channels.
Use your function from Problem \ref{prob:fft-filter-frequencies} to clean the sound clip by filtering out the vuvuzela frequencies in each channel.
Recombine the two cleaned samples.
\end{problem}

% \begin{info} % Notch filtering and other filters
% The cleaning strategy of Problems \ref{prob:fft-filter-frequencies} and \ref{prob:fft-vuvuzelas} is called \emph{notch filtering} and is a special case of \emph{band-stop filtering}.
% \end{info}

\subsection*{The Two-dimensional Discrete Fourier Transform} % ----------------

The DFT can be easily extended to any number of dimensions.
Computationally, the problem reduces to performing the usual one-dimensional DFT iteratively along each of the dimensions.
For example, to compute the two-dimensional DFT of an $m \times n$ matrix, calculate the usual DFT of each of the $n$ columns, then take the DFT of each of the $m$ rows of the resulting matrix.
Calculating the two-dimensional IDFT is done in a similar fashion, but in reverse order: first calculate the IDFT of the rows, then the IDFT of the resulting columns.

\begin{lstlisting}
>>> from scipy.fftpack import fft2, ifft2

>>> A = np.random.random((10,10))
>>> A_dft = fft2(A)                 # Calculate the 2d DFT of A.
>>> A_dft_ifft = ifft2(A_dft).real  # Calculate the 2d IDFT.
>>> np.allclose(A, A_dft_ifft)
<<True>>
\end{lstlisting}

% The DFT for matrices is useful for a variety of image applications, including cleaning, compression, enhancement, and edge detection.
Just as the one-dimensional DFT can be used to remove noise in sounds, its two-dimensional counterpart can be used to remove ``noise'' in images.
The procedure is similar to the filtering technique in Problems \ref{prob:fft-filter-frequencies} and \ref{prob:fft-vuvuzelas}: take the two-dimensional DFT of the image matrix, modify certain entries of the DFT matrix to remove unwanted frequencies, then take the IDFT to get a cleaner version of the original image.
This strategy makes the fairly strong assumption that the noise in the image is periodic and corresponds to certain frequencies.
While this may seem like an unlikely scenario, it does actually occur in many digital images---for an example, try taking a picture of a computer screen with a digital camera.

\begin{figure}[H]
\captionsetup[subfigure]{justification=centering}
\centering
\begin{subfigure}{.4\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/blurry_face.png}
    \caption{The original blurry image.}
    \label{fig:blurry_face}
\end{subfigure}
\begin{subfigure}{.4\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/blurry_fft.png}
    \caption{The DFT of the original image.}
    \label{fig:blurry_fft}
\end{subfigure}
\begin{subfigure}{.4\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/improved_face.png}
    \caption{The improved image.}
    \label{fig:improved_face}
\end{subfigure}
\begin{subfigure}{.4\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/covered_fft.png}
    \caption{The DFT of the improved image.}
    \label{fig:covered_fft}
\end{subfigure}
\caption{To remove noise from an image, take the DFT of the image and replace the abnormalities with values more consistent with the rest of the DFT.
Notice that the new image is less noisy, but only slightly.
This is because only some of the abnormalities in the DFT were changed; in order to further decrease the noise, we would need to further alter the DFT.}
\label{fig:image_fft}
\end{figure}

To begin cleaning an image with the DFT, take the two-dimensional DFT of the image matrix.
Identify \emph{spikes}---abnormally high frequency values that may be causing the noise---in the image DFT by plotting the log of the magnitudes of the Fourier coefficients.
With \li{cmap="gray"}, spikes show up as bright spots.
See Figures \ref{fig:blurry_face}--\ref{fig:blurry_fft}.

\begin{lstlisting}
# Read the image.
>>> import imageio
>>> im = imageio.read("noisy_face.png")

# Plot the log magnitude of the image's DFT.
>>> im_dft = fft2(image)
>>> plt.imshow(np.log(np.<<abs>>(im_dft)), cmap="gray")
>>> plt.show()
\end{lstlisting}

Instead of setting spike frequencies to zero (as was the case for sounds), replace them with values that are similar to those around them.
There are many ways to do this, but one convention is to simply ``patch'' each spike by setting portions of the DFT matrix to some set value, such as the mean of the DFT array.
See Figure \ref{fig:covered_fft}.

Once the spikes have been covered, take the IDFT of the modified DFT to get a (hopefully cleaner) image.
Notice that Figure \ref{fig:improved_face} still has noise present, but it is a slight improvement over the original.
However, it often suffices to remove some of the noise, even if it is not possible to remove it all with this method.

\begin{problem} % Clean an image.
The file \texttt{license\_plate.png} contains a noisy image of a license plate.
The bottom right corner of the plate has is a sticker with information about the month and year that the vehicle registration was renewed.
However, in its current state, the year is not clearly legible.

Use the two-dimensional DFT to clean up the image enough so that the year in the bottom right corner is legible.
This may require a little trial and error.
\end{problem}

\begin{comment}
\newpage
\section*{Additional Material} % ==============================================

\subsection*{Other Convolution Functions} % -----------------------------------

Talk about the options in \li{np.covolve()}, etc. (circular by default).
Use \li{scipy.ndimage.convolve()} to do circular convolution if needed (but the results must be shifted to match what we do).

\subsection*{Other Sound Filters} % -------------------------------------------

What we did is called a notch filter, blah blah blah...

\subsection{Other Image Filters} % --------------------------------------------

Overview of the following comment?

\end{comment}

\begin{comment} % VERY OLD (and bad) image filtering section ==================

\section*{Image Filters}

Recall that a computer stores an image as a 2-D array of pixel values (i.e., a matrix of intensities).
An image filter is a function that transforms an image by operating on it locally.
That is, to compute the $(ij)$th pixel value in the new image, an image filter uses only the pixels in a small neighborhood around the $(ij)$th pixel in the original image.

In this lab, we use a filter derived from the gradient of an image to find edges in an image.

\subsection*{Convolutions}

One example of an image filter is to \emph{convolve} an image with a filter matrix.
A filter matrix is a matrix whose height and width are relatively small odd numbers.
If the filter matrix is
\[
F = \begin{pmatrix}
f_{-1,-1}&f_{-1,0}&f_{-1,1}\\
f_{0,-1}&f_{0,0}&f_{0,1}\\
f_{1,-1}&f_{1,0}&f_{1,1}
\end{pmatrix},
\]
then the convolution of an image $A$ with $F$ is $A \ast F = (C_{ij})$ where
\begin{equation}\label{equ:convolve}
C_{ij} = \sum_{k=-1}^1 \sum_{\ell=-1}^1 f_{k\ell}A_{i+k,j+\ell}.
\end{equation}
Say $A$ is an $m \times n$ matrix. Here, we take $A_{ij}=0$ when $i \not \in \{1, \ldots m\}$ or $j \not \in \{1, \ldots, n\}$.
The value of $C_{ij}$ is a linear combination of the nearby pixel values, with coefficients given by $F$ (see Figure \ref{fig:convolution}).
In fact, $C_{ij}$ equals the Frobenius inner product of $F$ with the $3 \times 3$ submatrix of $A$ centered at $ij$.

\begin{figure}
\centering
\begin{tikzpicture}
\node[draw, minimum width=12cm, minimum height=
    7cm, ultra thick](outer_rec)[]{};
\node[draw, minimum width=10cm, minimum height=
    5cm, ultra thick, fill=black!10!](inner_rec)[]{};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick, fill=black!30!](square1)at(-4.75,2.25){\textbullet};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick,fill=black!30!](square2)at(-1.75,-.75){\textbullet};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick, fill=black!30!](square3)at(4.75,.75){\textbullet};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick, fill=black!30!](square4)at(3.25,-1.25){\textbullet};
\draw[step=.5, ultra thin, color=black!50!](-6,-3.5)grid(6,3.5);

\node[draw, minimum width=10cm, minimum height=
    5cm, ultra thick, ][]{};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick]at(-4.75,2.25){};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick]at(-1.75,-.75){};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick]at(4.75,.75){};
\node[draw, minimum width=2.5cm, minimum height=2.5cm,
    ultra thick]at(3.25,-1.25){};
\end{tikzpicture}
\caption{This diagram illustrates how to convolve an image with a filter.
The light grey rectangle represents the original image $A$, and the dark grey squares are the filter $F$.
The larger rectangle is the image padded with zeros; i.e., all pixel values in the outer white band are 0.
To compute the entry of the convolution matrix $C$ located at a black dot, take the inner product of $F$ with the submatrix of the padded image centered at the dot.}
\label{fig:convolution}
\end{figure}

\subsubsection*{Implementation in NumPy}

Let us write a function that convolves an image with a filter.
You can test this function on the image \li{cameraman.jpg}, which appears in Figure \ref{fig:cameraman}.
The following code loads this image and plots it with matplotlib.

\begin{lstlisting}
>>> image = plt.imread('cameraman.jpg')
>>> plt.imshow(image, cmap = 'gray')
>>> plt.show()
\end{lstlisting}

Here is the function definition and some setup.

\begin{lstlisting}
1. def Filter(image, F):
2.     m, n = image.shape
3.     h, k = F.shape
\end{lstlisting}

To convolve \li{image} with the filter \li{F}, we must first \emph{pad} the array \li{image} with zeros around the edges.
This is because in \eqref{equ:convolve}, entries $A_{ij}$ are set to zero when $i$ or $j$ is out of bounds.
We do this by creating a larger array of zeros, and then making the interior part of the array equal to the original image (see Figure \ref{fig:convolution}).

For example, if the filter is a $3 \times 3$ matrix, then the following code pads the matrix with the appropriate number of zeros.

\begin{lstlisting}
 # Create a larger matrix of zeros
image_pad = np.zeros((m+2, n+2))
# Make the interior of image_pad equal to the original image
image_pad[1:1+m, 1:1+n] = image
\end{lstlisting}

We want to do this in general in our function.  Note that the number of zeros we need to pad our array depends on the size of the filter \li{F}.

\begin{lstlisting}
5.    image_pad = # Create an array of zeros of the appropriate size
6.   # Make the interior of image_pad equal to image
\end{lstlisting}

Finally, we iterate through the image to compute each entry of the convolution matrix.

\begin{lstlisting}
7.    C = np.zeros(image.shape)
8.    for i in range(m):
9.        for j in range(n):
10.            C[i,j] = # Compute C[i, j]
\end{lstlisting}

\subsubsection*{Gaussian Blur}

A \emph{Gaussian blur} is an image filter that operates on an image by convolving with the matrix
\[
G = \frac{1}{159}\begin{pmatrix}
2&4&5&4&2\\
4&9&12&9&4\\
5&12&15&12&5\\
4&9&12&9&4\\
2&4&5&4&2
\end{pmatrix}.
\]

Blurring an image can remove ``noise'', or random variation that is the visual analog of static in a radio signal (and equally undesirable).

\begin{problem}\label{prob:filter}
\leavevmode
Finish writing the function \li{Filter} by filling in lines 5, 6, and 10.  Hint: Note in \ref{equ:convolve}, $C_{ij}$ was calculated by summing from -1 to 1.  This is only the case if the filter \li{F} is $3 \times 3$. A slight modification is needed in the general case.  Test your function on the image \li{cameraman.jpg} using the Gaussian Blur. The result is in Figure \ref{fig:cameraman_blur}.
\end{problem}

\begin{figure}
\centering
\begin{subfigure}[b]{.49\textwidth}
\centering
% \includegraphics[width=\textwidth]{figures/cameraman.jpg}
\caption{Unfiltered image.}
\label{fig:cameraman}
\end{subfigure}
\begin{subfigure}[b]{.49\textwidth}
\centering
% \includegraphics[width=\textwidth]{figures/cameramanBlur.pdf}
\caption{Image after Gaussian blur is applied.}
\label{fig:cameraman_blur}
\end{subfigure}
\begin{subfigure}[b]{.49\textwidth}
\centering
% \includegraphics[width=\textwidth]{figures/edges.pdf}
\caption{Image after the Sobel filter is applied.}
\label{fig:cameraman_edges}
\end{subfigure}
\caption{Here is an example of a Gaussian blur and the Sobel filter applied to an image.
This photo, known as ``cameraman,'' is a standard test image in image processing.
A database of such images can be downloaded from \url{http://www.imageprocessingplace.com/root_files_V3/image_databases.htm}.}
\label{fig:cameraman1}
\end{figure}

\subsection*{Edge Detection}

Automatic detection of edges in an image can be used to segment or sharpen the image.
We find edges with the Sobel filter, which computes the gradient of the image at each pixel.
The magnitude of the gradient tells us the rate of change of the pixel values, and so large magnitudes should
correspond to edges within the image.
The Sobel filter is not a convolution, although it does use convolutions.

We can think of an image as a function from a $2 \times 2$ grid of points to $\mathbb{R}$.
The image maps a pixel location to an intensity.
It does not make sense to define the derivative of this function as a limit because the domain is discrete---a step size $h$ cannot take on arbitrarily small values.
Instead, we \emph{define} the derivative to be the centered difference quotient of the previous section.
That is, we define the derivative in the $x$-direction at the $ij$th pixel to be
\[
\frac{1}{2}A_{i+1, j} - \frac{1}{2}A_{i-1, j}.
\]

We can use a convolution to create a matrix $A_x$ whose $ij$th entry is the derivative of $A$ at the $ij$th entry, in the $x$-direction.
In fact, $A_x = A \ast S$, where
\[
S = \frac{1}{8}
\left[\begin{array}{ccc}
-1 & 0 & 1\\
-2 & 0 & 2\\
-1 & 0 & 1
\end{array}\right].
\]

Note that this convolution takes a weighted average of the $x$-derivatives at $(i, j)$, $(i, j+1)$, and $(i, j-1)$.
The derivative at $(i, j)$ is weighted by 2.
Using a weighted average instead of just the derivative at $(i, j)$ makes the derivative less affected by noise.

Now we can define the Sobel filter.
A Sobel filter applied to an image $A$ results in an array $B = (B_{ij})$ of 0's and 1's, where the 1's trace out the edges in the image.
By definition,
\[
B_{ij} = \left\{
     \begin{array}{ll}
       1 & \text{if}\; \;\|\nabla A(ij)\|_2 > M \\
       0 & \text{otherwise}.
     \end{array}
   \right.
\]
Here, $\nabla A(ij) = ((A \ast S)_{ij}, (A\ast S^T)_{ij})$ is the gradient of $A$ at the $ij$th pixel.
The constant $M$ should be ``sufficiently large'' enough to pick out those pixels with the largest gradient (i.e., those pixels that are part of an edge).
A good choice for $M$ is 4 times the average value of $\|\nabla A(ij)\|_2$ over the whole image $A$.

When the Sobel filter is applied to \li{cameraman.jpg}, we get the image in Figure \ref{fig:cameraman_edges}.
Here, the 1's in $B$ were mapped to ``white'' and the 0's were mapped to ``black.''

\begin{problem}
Write a function that accepts an image as input and applies the Sobel filter to the image.  Test your function on \li{cameraman.jpg}.  Hint: If you want to find the average of a matrix \li{A}, use the function \li{A.mean()}.
\end{problem}

\end{comment}