\newpage\section{Sets}

\subsection{Lemmas}

\lem{}{\label{lemma:sets_lemma_1}
    Let $ S $ be a set with $ n $ elements, and let $ F $ be a family of
    subsets of S such that for any pair $ A, B $ in $ F $, $ A \cap B \not=
    \varnothing $. Then $ |F| \leq 2^{n-1} $ .
}

% \begin{enumerate}
%     \iref{lemma:sets_lemma_1_1}{Iran TST 2008 D3P1}{}
% \end{enumerate}

\theo{https://en.wikipedia.org/wiki/Erdos-Ko-Rado_theorem}{Erdos Ko Rado theorem}{
    Suppose that $ A $ is a family of distinct subsets of $\{ 1 , 2 \dots n
    \}$ such that each subset is of size $ r $ and each pair of subsets has a
    nonempty intersection, and suppose that $ n \geq 2r $. Then the number of
    sets in $ A $ is less than or equal to the binomial coefficient \[\binom
    {n-1}{r-1}\]
}

\lem{}{Let $ S $ be a set with $ n $ elements, and let $ F $ be a family of subsets of $ S $ such that for any pair $ A, B $ in $ F $, $ S $ is not contained by $ A \cup B $. Then $ |F| \leq 2^{n-1} $.}\label{lemma:sets_lemma_2}

\lem{Kleitman lemma}{A set family $F$ is said to be downwards closed if the following holds: if $X$ is a set in $F$, then all subsets of $X$ are also sets in $F$. Similarly, $F$ is said to be upwards closed if whenever $X$ is a set in $F$, all sets containing $X$ are also sets in $F$. Let $F_1$ and $F_2$ be downwards closed families of subsets of $S = \{1, 2, ..., n\}$, and let $F_3$ be an upwards closed family of subsets of $S$. Then we have

    \begin{align}
        |F_1 \cap F_2| &\geq \frac{|F_1| \cdot |F_2|}{2^n}\\
        |F_1 \cap F_3| &\leq \frac{|F_1| \cdot |F_3|}{2^n}
\end{align}}\label{lemma:sets_lemma_3_Kleitman}

\lem{}{Let $ S $ be a set with $ n $ elements, and let $ F $ be a family of subsets of $ S $ such that for any pair $ A, B $ in $ F $, $ A \cap B \not= \varnothing $ and $ A \cap B \not= S $. Then $ |F| \leq 2^{n-2} $.}\label{lemma:sets_lemma_4}

\solu{
    Using the sets in \autoref{lemma:sets_lemma_1} and
    \autoref{lemma:sets_lemma_1}, defining upwards and downwards sets like in
    \autoref{lemma:sets_lemma_3_Kleitman}.
}

\theo{https://www.wikiwand.com/en/Sunflower_(mathematics)}
{The Sunflower Lemma}{
    A \textbf{sunflower} with $ k $ petals and a core $ X $ is a family of sets $ S_1,
    S_2,\dots, S_k $ such that $ S_i\cap S_j = X $ for each $ i \neq j $. (The
    reason for the name is that the Venn diagram representation for such a
    family resembles a sunflower.) The sets $ S_i \setminus X $ are known as
    petals and must be nonempty, though $ X $ can be empty. \\

    Show that if $ F $ is a family of sets of cardinality $ s $, and $ |F| >
    s!(k-1)^s $, then $ F $ contains a sunflower with $ k $ petals.
}\label{problem:induction_type1_20}\label{problem:extreme_object_10}

\solu{Applying induction and considering the best case where $ |X|=0 $}

\subsection{Extremal Set Theory}

\begin{myitemize}
    \item \href{http://math.mit.edu/~cb_lee/18.318/lecture8.pdf}{MIT 18.314 Lecture-8}
\end{myitemize}

\theo{}{Mirsky Theorem}{A set $ S $ with a chain of height $ h $ can’t be partitioned into $ t $ anti-chains if $ t < h $. In other words, the minimum number of sets in any anti-chain partition of $ S $ is equal to the maximum height of the chains in $ S $. (And Vice Versa)}\label{theorem:mirsky_theorem}

\theo{}{}{In any poset, the largest cardinality of an antichain is at most the smallest cardinality of a chain-decomposition of that poset.}

\theo{}{Dilworth's Theorem}{Let $ P $ be a poset. Then there exist an antichain $ A $ and a chain decomposition $ \mathcal{C} $ of $ P $ such that $ |A| = |\mathcal{C}| $}

\theo{http://mathworld.wolfram.com/Erdos-SzekeresTheorem.html}{Erdos-Szekeres Theorem}{Any sequence of $ ab+1 $ real numbers contains either a monotonically decreasing subsequence of length $ a+1 $ or a monotonically increasing subsequence of length $ b+1 $. The more useful case is when $ a=b=n $. }

\prob{}{}{E}{Let $ n\ge 1 $ be an integer and let $ X $ be a set of $ n^2+1 $ positive integers such that in any subset of $ X $ with $ n+1 $ elements there exist two elements $ x\neq y $ such that $ x|y $. Prove that there exists a subset $ \{x_1, x_2\dots x_{n+1} \in X $ such that $ x_i|x_{i+1} $ for all $ i=1, 2, \dots n $.}

\den{Sperner Family}{
    A family of sets in which none of the sets is a strict subset of another
    is called a \textbf{Sperner family}, that is, an antichain of sets.
}

\theo{https://www.wikiwand.com/en/Lubell–Yamamoto–Meshalkin_inequality}
{LYM Inequality}{
    Let $U$ be a set of $n$ elements and $\mathcal{S}$ is a Sperner family of
    subsets of $U$. If $a_k$ is the number of $k$ element subsets of
    $\mathcal{S}$, then
    \[\impeq{\sum^{n}_{k=0}\frac{a_k}{{n \choose k}}\le 1}\]
}

\begin{prooof}
    Rewrite the inequality by
    \[\sum^{n}_{k=0} a_k k!(n-k)! \le n!\]
    The left hand side equals to
    \[\sum^{}_{S\in \mathcal{S}} \left|S\right|!
    \left(n-\left|S\right|\right)!\]
    For a $S \in \mathcal{S}$, $\left|S\right|!\left(n-\left|S\right|\right)!$
    is the number of permutations of $[n]$ for which the first
    $\left|S\right|$ terms are the elements of $S$. Since the sets of
    $\mathcal{S}$ doesn't include each other, all of these permutations are
    different. And so if we count them together, we will get at most all the
    permutatios of $[n]$, which gives us our desired result.
\end{prooof}

\theo{https://www.wikiwand.com/en/Sperner's_theorem}
{Sperner's Theorem}{
    Sperner's theorem bounds the number of sets in any Sperner family. For any
    Sperner Family $\mathcal{S}$, if the union of those sets is a set of $n$
    elements, then
    \[\left|\mathcal{S}\right|< {n \choose \left\lfloor n/2 \right\rfloor}\]
    Where equality holds iff $\mathcal{S}$ is consisted of all subsets of
    $[n]$ with $\left\lfloor n/2 \right\rfloor$ or $\left\lceil
    n/2  \right\rceil $ elements.
}

\begin{prooof}
    Let $a_k$ be the number of $k$ element subsets of $\mathcal{S}$. Then
    since ${n\choose \left\lfloor n/2\right\rfloor} \le {n\choose k}$, we have
    \[\frac{a_k}{{n \choose \left\lfloor n/2\right\rfloor}} \le
    \frac{a_k}{{n\choose k}}\]
    And since the sets doesn't contain each other, we can sum them up and by
    \autoref{theorem:LYM Inequality}, we get
    \[\sum^{n}_{k=0} \frac{a_k}{{n\choose \left\lfloor
    n/2\right\rfloor}} \le \sum^{n}_{k=0} \frac{a_k}{{n\choose k}} \le 1\]
\end{prooof}

\newpage
\subsection{Problems}

\prob{https://artofproblemsolving.com/community/c6h148835p841269}{USA TST 2005 P1}{E}{
    Let $ n $ be an integer greater than $ 1 $. For a positive integer
    $ m $ , let $ S_{m}= \{ 1,2,\ldots, mn\} $. Suppose that there exists a $
    2n $ -element set $ T $ such that

    \begin{enumerate}
        \item each element of $ T $ is an $ m $ -element subset of $ S_{m} $
        \item each pair of elements of $ T $ shares at most one common element
        \item each element of $ S_{m} $ is contained in exactly two elements of $ T $
    \end{enumerate}

Determine the maximum possible value of $ m $ in terms of $ n$.
\index[cat]{Sets!Max-set with property!USA TST 2005 P1}
\index[strat]{Double Counting!USA TST 2005 P1}
}
\label{problem:double_counting_5}

\solu{We use double counting to find the ans, after that the rest is easy.}

\prob{https://artofproblemsolving.com/community/c6h206650p1136980}{Iran TST 2008 D3P1}{E}{
    Let $S$ be a set with $n$ elements, and $F$ be a family of subsets of $S$ with
    $ 2^{n-1}$ elements, such that for each $A,B,C\in F$, $A\cap B\cap C$ is not
    empty. Prove that the intersection of all of the elements of $F$ is not empty.

    \index[cat]{Sets!Local!Iran TST 2008 D3P1}
    \index[strat]{Induction!Iran TST 2008 D3P1}
}\label{lemma:sets_lemma_1_1}\label{problem:induction_type1_18}

\solu{Using Induction with \hrf{lemma:sets_lemma_1}{this} lemma.}

\prob{https://artofproblemsolving.com/community/c6t309f6h1538018}{Romanian TST
2016 D1P2}{EM}{
Let $n$ be a positive integer, and let $S_1, S_2,\dots S_n$ be
a collection of finite non-empty sets such that
\[\sum_{1\leq i<j\leq n}{\frac{|S_i \cap S_j|}{|S_i||S_j|}} <1\]

Prove that there exist pairwise distinct elements $x_1, x_2\dots x_n$ such that $x_i$ is a member of $S_i$ for each index $i$.

\index[cat]{Sets!Local!Romanian TST 2016 D1P2}
\index[strat]{Focus!Romanian TST 2016 D1P2}
\index[strat]{Induction!Romanian TST 2016 D1P2}
}
\label{problem:induction_type1_25}\label{problem:forget_and_focus_7}

\solu{The Inductive proof reduces the problem to
    \hrf{problem:induction_type1_26}{American Mathematical Monthly problem
E2309}}

\solu{The other approach is to focus on the given weird condition, and interpolate it to something nice, like probabilistic condition.}

\prob{https://artofproblemsolving.com/community/c6h225275p1252232}{American Mathematical Monthly E2309}{EM}{If $ A_1$, $ A_2$,\dots $ A_n$ are $ n$ nonempty subsets of the set $ \left\{1,2,...,n - 1\right\}$, then prove that

\[\sum_{1\leq i < j\leq n}\frac {\left|A_i\cap A_j\right|}{\left|A_i\right|\cdot\left|A_j\right|}\geq 1\]

\index[cat]{Sets!Subsets of $[N]$!American Mathematical Monthly E2309}
}
\label{problem:induction_type1_26}

\prob{https://artofproblemsolving.com/community/c6h362426p1986928}{CGMO 2010
    P1}{E}{Let $n$ be an integer greater than two, and let $A_1,A_2, \cdots ,
    A_{2n}$ be pairwise distinct subsets of $\{1, 2, \dots n\}$. Determine the maximum
    value of
    \[\sum_{i=1}^{2n} \dfrac{|A_i \cap A_{i+1}|}{|A_i| \cdot |A_{i+1}|}\]
    Where $A_{2n+1}=A_1$ and $|X|$ denote the number of elements in $X.$

    \index[cat]{Sets!Subsets of $[N]$!CGMO 2010 P1}
}

\prob{https://artofproblemsolving.com/community/c6h17340p119108}{ISL 2002 C5}{M}{
    Let $r\geq2$ be a fixed positive integer, and let $F$ be an infinite
    family of sets, each of size $r$, no two of which are disjoint. Prove that
    there exists a set of size $r-1$ that meets each set in $F$.

\href{https://artofproblemsolving.com/community/c6h17340p7934669}{HMMT 2016
Team Round}:
    Fix positive integers $r>s$, and let $\mathcal F$ be an infinite
    family of sets, each of size $r$, no two of which share fewer than $s$
    elements. Prove that there exists a set of size $r-1$ that shares at least $s$
    elements with each set in $F$.

\index[cat]{Sets!Local!ISL 2002 C5}
\index[strat]{Induction!ISL 2002 C5}
\index[strat]{Focus!ISL 2002 C5}
\index[strat]{Adding new stuffs!ISL 2002 C5}

}\label{problem:induction_type1_31}

\solu{[Focus on a set] If we take an arbitrary set, we can say that there
exists infinitely many sets $ \in \mathbb{F} $ which includes a fixed element
from our test set. If we do this argument for $ r-1 $ times, we get a set $ X
$ of $ r-1 $ elements, and an infinte family of sets that contains $ X $
completely. At this point the problem is trivial.}

\solu{[Adding Elements]Since it's tricky to work with one family, why not introduce another family, like the second monk. \hrf{http://artofproblemsolving.com/community/c6h17340p3251745}{This} solution generalizes the problem as such.}

\prob{https://artofproblemsolving.com/community/c6h57290p352698}{ISL 1988 P10}{M}{
    Let $ N = \{1,2 \ldots, n\}, n \geq 2. $

    A collection $ F = \{A_1, \ldots,A_t\} $ of subsets
    $ A_i \subseteq N, $  $ i = 1, \ldots, t, $ is said to be
    \textbf{separating}, if for every pair $ \{x,y\} \subseteq N, $ there is a set $ A_i
    \in F $ so that $ A_i \cap \{x,y\} $ contains just one element.

    $ F $ is said to be \textbf{covering}, if every element of $ N $ is contained
    in at least one set $ A_i \in F. $

    What is the smallest value $ f(n) $ of $ t, $ so there is a set $
    F = \{A_1, \ldots, A_t\} $ which is simultaneously separating and covering.

    \index[cat]{Sets!Subsets of $[N]$!ISL 1988 P10}
    \index[strat]{Bijection!Binary Representation!ISL 1988 P10}
} \label{problem:binary_2}

\solu{[Binary Representation]Using \hrf{binary}{Binary} Representations for the elements as in or not in, we get an easy bijection.}

\prob{https://artofproblemsolving.com/community/c6h530095p3024272}{Iran TST 2013 D1P2}{E}{
    Find the maximum number of subsets from $\left \{ 1,...,n \right \}$ such
    that for any two of them like $A,B$ if $A\subset B$ then $\left | B-A
    \right |\geq 3$. (Here $\left | X \right |$ is the number of elements of
    the set $X$.)
    \index[cat]{Sets!Max-set with property!Iran TST 2013 D1P2}
    \index[strat]{Roots of unity filtering!Iran TST 2013 D1P2}
    \index[strat]{Induction!$n$ is or isn't!Iran TST 2013 D1P2}
}

\solu{By partitioning the maximum set of subsets into groups which contain the number $n$ and which don't and \href{induction}{Induction} on $n$ we can show that the maximum number of subset is
\[\frac{2^{n}-(-1)^{n}}{3}\]. }

\prob{https://artofproblemsolving.com/community/c7h64445p383300}{Putnam 2005 B4}{E}{For positive integers $ m$ and $ n$, let $f\left(m,n\right)$ denote the number of $ n$-tuples $\left(x_1,x_2,\dots,x_n\right)$ of integers such that $\left|x_1\right| + \left|x_2\right| + \cdots + \left|x_n\right|\le m$. Show that $ f\left(m,n\right) = f\left(n,m\right)$.}

\solu{Try to show \href{bijection}{Bisection} between the result and choosing $m$ or $n$ objects from $m+n$ objects or show that the result is $\binom{m+n}{n}$.}

\newpage
\subsection{Hamming Distance}

\begin{myitemize}
    \item \href{https://dgrozev.wordpress.com/2019/11/09/hamming-distance-in-olympiad-problems/} {Blogpost by dgrozev}
\end{myitemize}

\vspace{1em}

\den{Hamming Distance}{
    The Hamming distance between two equal-length strings of symbols is the
    number of positions at which the corresponding symbols are different.
}

Occasionally we might be able to use the idea of Hamming distance to
``partition'' a set into sets of equivalence classes. For example consider the
following problem:

\prob{}
{}{}{
    Suppose we have some real numbers $\left(x_{n}\right), n=1,2, \ldots, N$
    and consider the sums
    \[ \sum_{i=1}^{N} b_{i} x_{i},\quad b_{i} \in\{0,1\}\]
    We want to estimate how many among those sums hit a fixed interval
    $\Delta$ with some length.

    \index[strat]{Hamming distance}
}

\begin{solution}
    Suppose a sum $\sum_{i=1}^{N} b_{i} x_{i}$ is inside $\Delta$. \\

    Then changing any bit $b_{i}, i=1,2, \ldots, N$ we leave $\Delta$ if this
    interval is smaller than the largest $\left|x_{i}\right|, i \in[1 \ldots
    N]$.\\

    So, for any $N$ -tuple of bits $b=\left(b_{1}, b_{2}, \ldots,
    b_{N}\right)$ with corresponding sum in $\Delta$, all the other tuples of
    bits that differ from $b$ in exactly one bit have sums outside $\Delta$.\\

    In fact the set of those bits is the unit sphere around $b$ with respect
    to the Hamming distance. So no other set on this sphere has the sum in
    $\Delta$.\\

    Now, we take the set $B$ of all tuples $b=\left(b_{1}, b_{2}, \ldots,
    b_{N}\right)$ with $\sum_{i=1}^{N} b_{i} x_{i} \in \Delta$. Then for any $b
    \in B$ the unit sphere
    {\color{solC} \[S(b):=\left\{b^{\prime}: b^{\prime} \text{differs from } b
    \text{ in exactly one bit}\right\}\]}
    consists of tuples not in $B$. If it happens for $b \in B$, $S(b)$ are
    disjoint, we can estimate the number of elements in $B$ as
    \[ |B| \leq \frac{2^{N}}{N+1} \]
\end{solution}

\newpage
\prob{https://artofproblemsolving.com/community/c6h1946999p13424041}
{CIIM 2019 P3}{M}{
    Let $\{x_n\}_{n\in{}\mathbb{N}}$ be a sequence of non-zero real numbers.
    For every positive integer $m$, we define:
    \[X_m=\left\{A\subseteq{}\{0,1,...,m-1\}\left|\quad\left|\sum_{a\in{A}}
    x_a\right| >\frac{1}{m}\right.\right\}\]
    Prove that: \[\lim_{n\to\infty} \frac{\vert{}X_n\vert{}}{2^n}=1\]

    \index[strat]{Hamming distance!CIIM 2019 P3}
}

\begin{Remark}
    The idea roughly speaking is as follows.

    {\color{impC}For sufficiently large $m$ we map each $A$ having small sum $\sum_{a\in
    A}x_a$ to a family $F(A)$ of many other sets with big sums.} If those
    families do not intersect, we obtain that the number of $A$'s with small
    sums is a small portion of the all subsets.

    The vague idea is to mark some finite set of distinct elements
    $x_n$ and for a fixed set $A$ we may add or remove any marked element,
    depending if it's in $A$ or not. Thus we obtain another sets, but those
    sets cannot have small sums if the marked elements are appropriately
    taken.
\end{Remark}

\begin{solution}[dgrozev]
    To apply this idea we consider two cases.\\

    \boxed{\textbf{1.}} There are infinitely many distinct numbers among
    $(x_n)_{n\ge 0}$. \\

    Let us fix some $N\in\mathbb{N}$ and $I$ be a set of $N$ non negative
    integers (indices) such $x_i, i\in I$ are all distinct. Denote by
    $\varepsilon$ any positive real number less than $\min \left\{|x_i-x_j|,\
    |x_i|\right\}, i, j\in I$. \\

    We take any $m$ large enough such that $2/m<\varepsilon$ and $m>\max\{i\in
    I\}$. Let $A$ is any subset of $\{0,1,\dots,m\}$ satisfying

    \begin{align*}
        \left|\sum_{a\in A}x_a\right |\leq \frac{1}{m}\qquad\qquad (1)
    \end{align*}
    For any $i\in I$ we construct the set $A_i$ as
    \[A_i:=\begin{cases} A\cup {i}& \text{if } i\not\in A\\
    A\setminus\{i\}&\text{if } i\in A \end{cases} \]
    It can be easily seen
    \[\displaystyle \left|\sum_{a\in A_i}x_a\right |> \frac{1}{m}\,,\,\forall
    i\in I\]

    By $F(A)$ we denote the family $\{A_i:i\in I\}$. For any two subsets $A,B$
    of $\{0,1,\dots,m\}$ satisfying $(1)$, {\color{impC}the corresponding families $F(A),
    F(B)$ do not meet}.\\

    Indeed, if $X\in F(A)$ and $X\in F(B)$, then $X$
    differs from $A, B$ only in one element of $I$. That is, $A$
    differs from $B$ only in two elements that are in $I$. Hence,
    \[\left|\sum_{a\in A}x_a-\sum_{a\in B}x_a\right|>2/m\]
    because of the choice of $m$. It means $A$ and $B$ cannot both satisfy
    $(1)$.\\

    To recap. Let $\mathcal{A}$ be the family of all subsets $A$ of $\left\{0,
    1,\dots m-1\right\}$ satisfying $(1)$. Then for any $A\in\mathcal{A}$ all
    sets in $F(A)$ are not in $\mathcal{A}$. Further, $F(A), A\in \mathcal{A}$
    are disjoint. Since $|F(A)|=N,\forall A\in\mathcal{A}$, we have
    \[|\mathcal{A}|\leq \frac{2^m}{N+1}\]and henceh
    \[\displaystyle \frac{|X_m|}{2^m}\geq \frac{N}{N+1}\]
    Since $N$ could be chosen arbitrary large, the result follows.\\

    \boxed{\textbf{2.}} Only finite values of $(x_n)_{n\ge 0}$ are distinct.\\

    So, suppose $x_n=a$ for infinitely many $n\in \mathbb{N}$. Let $I$ be a
    finite set of such indices with $|I|=N$ and $m\in \mathbb{N}$ satisfies
    $2/m<|a|$ and $m\geq \max\{i:i\in I\}$. For any fixed subset $A\subset
    [0..m]\setminus I$ there exists at most one $k\in\mathbb{N}$ with
    $\left|\sum_{i\in A}x_i+ka\right|\leq \frac{1}{m}$. Hence the number of
    subsets $A'\subset I$ with $\left|\sum_{i\in A\cup A'}x_i+ka\right|\leq
    \frac{1}{m}$ are at most $\dbinom{N}{\lfloor N/2\rfloor}$.\\

    It means the number of sets $A\subset [0..m]$ satisfying $(1)$ are at most
    $2^{m-N}\dbinom{N}{\lfloor N/2\rfloor}$, thus
    \[\displaystyle \frac{|X_m|}{2^m}\geq 1-\frac{\lfloor N/2\rfloor}{2^N}\]
    which proves the desired result.
\end{solution}

\begin{solution}[Sperner's theorem, IMD2]
    We call a subset ``bad'' if it doesn't satisfy the inequality.\\

    Now fix an integer $N$. Let $m$ be an integer such that
    \[\frac{2}{m}< \min \left\{\left|x_1\right|, \left|x_2\right|\dots
    \left|x_N\right|\right\}\]
    WLOG assume that there are as many positives in the first $N$ terms as
    there are negatives. Let $P$ be the positive elements subset of
    $\left\{x_1, x_2\dots x_N\right\}$.\\

    Define $S = \left\{x_1, x_2, \dots x_m\right\} - P$. Then every subset of
    the first $m$ terms can be written as ${A, B}$, where $A\subseteq P,
    B\subseteq S$ and the subset is $A\cup B$. \\

    Using this notation, notice that if $A\subset A' \subseteq P$, then both
    ${A, B}$ and ${A', B}$ can't be bad, because of our choice of $m$. So for
    a fixed $B\subseteq S$, the largest family of subsets of $P$ is a
    \emph{Sperner Family} of $P$. Which by Sperner's theorem has size at most
    \[t_p = {\left|P\right| \choose \left\lfloor\left|P\right|/2\right\rfloor}\]
    So the probability of a subset of $\left\{x_1, x_2,\dots x_m\right\}$
    being bad is $\frac{t_p}{2^p}$, which tends to $0$ for larger choice of
    $N$.
\end{solution}


\documentclass[]{report}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\graphicspath{ {images/} }

\title{CSCI 567 HW\#2}
\author{Mohmmad Suhail Ansari\\USC ID: 8518586692}

\begin{document}
\maketitle

\newpage

	\paragraph{Sol. 1.1}

		The negative of log likelihood can be written as:

		\[ \mathcal{L}(\textbf{w}) = - log\bigg( \prod_{i=1}^N P(Y= y_i|\textbf{X}=x_i) \bigg) \]
		\[ \mathcal{L}(\textbf{w}) =
			\left\{
				\begin{array}{ll}
					{y_n = 1}  & - \sum_{i = 0}^{N} log(\sigma(b + \textbf{w}^T \textbf{x}_n))\\
					{y_n = 0}  & - \sum_{i = 0}^{N} log(1 - \sigma(b + \textbf{w}^T \textbf{x}_n))
				\end{array}
			\right.
		\]
		combining the two parts, we get

		\begin{equation}
			\mathcal{L}(\textbf{w}) = - \sum_{i = 0}^{N} y_n [log(\sigma(b + \textbf{w}^T \textbf{x}_n))] + (1- y_{n})[log(1 - \sigma(b + \textbf{w}^T \textbf{x}_n))]
		\end{equation}

	\paragraph{Sol. 1.2}
		First we will transform the eq(1) above by appending $1$ to $\textbf{x}$ and $b$ to $\textbf{w}$ i.e.
		\[ \textbf{x} = [1 \quad x_i \quad x_2 \quad x_3 \quad ... \quad x_D ]\]
		\[ \textbf{w} = [b \quad w_1 \quad w_2 \quad w_3 \quad ... \quad w_D ]\]

		$\therefore$ the eq(1) can be written as

		\begin{equation}
			\mathcal{L}(\textbf{w}) = - \sum_{i = 0}^{N} y_n [log(\sigma(\textbf{w}^T \textbf{x}_n))] + (1- y_{n})[log(1 - \sigma(\textbf{w}^T \textbf{x}_n))]
		\end{equation}

		Now, we know that the derivative of $\sigma(a)$ is given as
		\[ \frac{d\,\sigma(a)}{d\,a} = \frac{1}{1 + e^{-a}} \bigg( 1 - \frac{1}{1 + e^{-a}} \bigg) \]
		\[ = \sigma(a)[1 - \sigma(a)]\]
		similarly we can write the derivative of $log(\sigma(a))$ w.r.t $a$
		\[ \frac{d \, log(\sigma(a))}{d \, \sigma(a)} = 1 - \sigma(a)\]

		Now, using the above definitions and we can write the derivative of the loss function i eq(2) as
		\[ \frac{\partial{\mathcal{L}(\textbf{w})}}{\partial{\textbf{w}}} = - \sum_{i = 0}^{N} y_n ({ 1 - \sigma(\textbf{w}^T \textbf{x}_n)})\textbf{x}_n + (1-y_n){\sigma(\textbf{w}^T \textbf{x}_n)}\textbf{x}_n \]

		\begin{equation}
			\frac{\partial{\mathcal{L}(\textbf{w})}}{\partial{\textbf{w}}}= \sum_{i = 0}^{N} \{ \sigma(\textbf{w}^T \textbf{x}_n) -y_n \}\textbf{x}_n
		\end{equation}

		from eq(3) we get the error as
		\[ e_n = {\sigma(\textbf{w}^T \textbf{x}_n ) - y_n} \]
		and the stationary point as
		\[ \sum_{i = 0}^{N} \sigma(\textbf{w}^T \textbf{x}_n ) \textbf{x}_n =  \sum_{i = 0}^{N} \textbf{x}_n y_n \]

		Now, let $\eta$ be the step size, then we can write the update rule for $\textbf{w}$ as
		\begin{equation}
			\textbf{w}^{(t + 1)}  = \textbf{w}^{(t)} - \eta \sum_{i = 0}^{N} \{ \sigma(\textbf{w}^T \textbf{x}_n) - y_n\}x_n
		\end{equation}

		Yes, it would converge to a global minimum. Since, the curve is linear and the local minimum would be the global minimum itself, which we can reach through gradient descent while following along the curve.

	\paragraph{Sol. 1.3}
		For multi-class classification we are given the posterior probability as
		\begin{equation}
			 P(Y = k | \textbf{X} = \textbf{x}) = \frac{exp(\textbf{w}_k^T \textbf{x})}{ 1 + \sum_{t=1}^{K-1} exp(\textbf{w}_t^T \textbf{x})} \quad for \quad k = 1,\, ...\, {K-1}
		\end{equation}
		\begin{equation}
			P(Y = k | \textbf{X} = \textbf{x}) = \frac{1}{ 1 + \sum_{t=1}^{K-1} exp(\textbf{w}_t^T \textbf{x})} \quad for \quad k = K
		\end{equation}

		since, $\textbf{w}_K = 0$, we can simply write
		\begin{equation}
			P(Y = k | \textbf{X} = \textbf{x}) = \frac{exp(\textbf{w}_k^T \textbf{x})}{1 + \sum_1^{K-1} exp(\textbf{w}_t^T \textbf{x})}
		\end{equation}

		Now, using eq(7) we can write the negative log-likelihood as
		\[ \mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K) = - \sum_n log(P(y_n | \textbf{x}_n = \textbf{x})) = - \sum_n log( \prod_k [P(y = k | \textbf{x}_n)]) \]
		\[ \mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K) = - \sum_n \sum_k log\, P(y = k | \textbf{x}_n)) \]

		using, eq(7), we can write the negative log-likelihood function as
		\begin{equation}
			 \mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K) = - \sum_n \sum_k \big[ \textbf{w}_k^T \textbf{x}_n - log(1 + \sum_{k=1}^{K-1} exp(\textbf{w}_t^T \textbf{x}_n )) \big]
		\end{equation}

	\paragraph{So. 1.4}
		To find the maximum log-likelihood, we use eq(8) and take partial derivative w.r.t $\textbf{w}_i$, we get
		\[ \frac{\partial{\mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K)}}{\partial{\textbf{w}_i}} = - \sum_n \big[ \textbf{x}_i - \frac{exp(\textbf{w}_i^T \textbf{x}_i)\textbf{x}_i}{ 1 + \sum_{k=1}^{K-1} exp(\textbf{w}_t^T \textbf{x}_n )} \big] \]
		\[ = - \sum_n \big[ 1 - \frac{exp(\textbf{w}_i^T \textbf{x}_i)}{ 1 + \sum_{k=1}^{K-1} exp(\textbf{w}_t^T \textbf{x}_n)} \big] \textbf{x}_i \]
		\[ = - \sum_n \big[ 1 - P(y = i| \textbf{x}_i) \big] \textbf{x}_i \]

		$\therefore$ we can define the error as
		\[ e_i = 1 - P(y = i | \textbf{x}_i )\]

		let $\eta$ be the step function, then we can write the update rule as
		\[ \textbf{w}_{(i + 1)} = \textbf{w}_{(i)} - \eta \sum_n \big[ 1 - P(y = i| \textbf{x}_i) \big] \textbf{x}_i \]

	%% Linear/Gaussian Discriminant

	\paragraph{Sol. 2.1}
		From the definition of $p(x_n, y_n)$ we can write the log-likelihood as
		\[ \mathcal{L}(D) =  \sum_{n=1, y_n = 1}^N log(p_1 \frac{1}{\sqrt{2\pi}\sigma_1} exp(- \frac{(x_n - \mu_1)^2}{2\sigma_1^2}))\] \[ + \sum_{n=1, y_n = 2}^N log(p_2 \frac{1}{\sqrt{2\pi}\sigma_2} exp(- \frac{(x_n - \mu_2)^2}{2\sigma_2^2}))\]

		\[ \mathcal{L}(D) = \sum_{n=1, y_n = 1}^N log(p_1) - log(\sqrt{2\pi}\sigma_1) - \frac{(x_n - \mu_1)^2}{2\sigma_1^2} \]
		\[ + \sum_{n=1, y_n = 2}^N log(p_2) - log(\sqrt{2\pi}\sigma_2) - \frac{(x_n - \mu_2)^2}{2\sigma_2^2} \]

		Now, we know that $p_1 + p_2 = 1$. Let, $N_1$ = no. of samples where $y_n = 1$ and $N_2$ = no. of samples where $y_n = 2$
		we'll first find the estimate of $p_1$ that minimizes $-\mathcal{L}(D)$

		\[ \mathcal{L}(D) = \sum_{n=1, y_n = 1}^N log(p_1) - log(\sqrt{2\pi}\sigma_1) - \frac{(x_n - \mu_1)^2}{2\sigma_1^2} \]
		\[ + \sum_{n=1, y_n = 2}^N log(1 - p_1) - log(\sqrt{2\pi}\sigma_2) - \frac{(x_n - \mu_2)^2}{2\sigma_2^2} \]

		taking partial derivative w.r.t. $p_1$, we get

		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{p_1}} = \sum_{n=1, y_n = 1}^N \frac{1}{p_1} - \sum_{n=1, y_n = 2}^N \frac{1}{1 - p_1} \]
		from definition of $N_1$ and $N_2$, we get
		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{p_1}} = 0 = \frac{N_1}{p_1} + \frac{N_2}{1 -p_1} \]
		\[ \hat{p_1} = \frac{N_1}{N_1 + N_2}\]
		similarly, $\hat{p_2}$ will be
		\[ \hat{p_2} = \frac{N_2}{N_1 + N_2}\]

		To, estimate $\mu_1$, we take the partial derivative of $\mathcal{L}(D)$ w.r.t $\mu_1$, we get
		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{\mu_1}} = \sum_{n=1, y_n = 1}^N (x_n - \mu_1) = 0\]
		\[ \hat{\mu_1} = \frac{1}{N_1} \sum_{n=1, y_n = 1}^N x_n\]
		similarly, estimate of $\mu_2$ will be
		\[ \hat{\mu_2} = \frac{1}{N_2} \sum_{n=1, y_n = 2}^N x_n\]

		To, estimate $\sigma_1$, we take the partial derivative of $\mathcal{L}(D)$ w.r.t $\sigma_1$, we get
		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{\sigma_1}} = \sum_{n=1, y_n = 1}^N - \frac{1}{\sigma_1} + \frac{(x_n - \mu_1)^2}{\sigma_i^3} = 0\]
		which gives us
		\[ \hat{\sigma_1^2} = \frac{\sum_{n=1, y_n = 1}^N (x_n - \mu_1)^2}{N_1} \]
		similarly, estimate of $\sigma_2^2$ will be
		\[ \hat{\sigma_2^2} = \frac{\sum_{n=1, y_n = 2}^N (x_n - \mu_2)^2}{N_2} \]

	\paragraph{Sol. 2.2}
		From Baye's formula we can write that
		\[ P(y = k | x) = \frac{P(x | y = k)\, P(y = k)}{\sum_k P(x | y = k)\, P(y = k)} \]
		Now, since $P(y = 1) + P(y = 2) = 1$, then let $P(y = 1) = \pi$, $\therefore P(y = 2) = 1 - \pi$.
		We'll first find the probability $P(y = 1| x)$

		\[ P(y = 1| x) = \frac{P(x|y = 1)\, P(y = 1)}{ [P(x | y = 1)\, P(y = 1)] + [P(x | y = 2)\, P(y = 2)]} \]
		\[ = \frac{\pi \mathcal{N}(\mu_1, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma) + (1 - \pi)\pi \mathcal{N}(\mu_2, \Sigma)} \]
		\[ = \frac{1}{1 + \frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)}}\]

		Now,
		\[ \frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)} = exp(log(\frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)}))\]

		Solving for $log(\frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)})$
		\[ log(\frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)}) = \]
		\[ = - log(\frac{\pi}{1 - \pi}) + log(\frac{\mathcal{N}(\mu_2, \Sigma)}{\mathcal{N}(\mu_1, \Sigma)})\]

		\[ = -log(\frac{\pi}{1 - \pi}) -\frac{1}{2}(x- \mu_2)^T \Sigma^{-1}(x-\mu_2) + \frac{1}{2}(x- \mu_1)^T \Sigma^{-1}(x-\mu_1) \]
		\[ = -log(\frac{\pi}{1 - \pi}) - \frac{1}{2}{x^T}{\Sigma^{-1}}{x} + {\mu_2^T}{\Sigma^{-1}}{x} - \frac{1}{2}{\mu_2^T}{\Sigma^{-1}}{\mu_2} + \frac{1}{2}{x^T}{\Sigma^{-1}}{x} - {\mu_1^T}{\Sigma^{-1}}{x} + \frac{1}{2}{\mu_1^T}{\Sigma^{-1}}{\mu_1} \]
		\[ = (\mu_2 - \mu_1)^T \Sigma^{-1} x - \frac{1}{2}{\mu_2^T}{\Sigma^{-1}}{\mu_2} + \frac{1}{2}{\mu_1^T}{\Sigma^{-1}}{\mu_1} - log(\frac{\pi}{1 - \pi})\]

		\[ = - \boldsymbol{\theta}^{T} \textbf{x} + b \]
		where,
		\[ \theta = \Sigma^{-1}(\mu_1 - \mu_2)\]
		and
		\[ b = -\frac{1}{2} \mu_2^T \Sigma^{-1} \mu_2  +\frac{1}{2} \mu_1^T \Sigma^{-1} \mu_1 - log \frac{\pi}{1 - \pi}\]

	\paragraph{Sol. 3.1}
		Data Analysis\\

		\begin{center}
			\includegraphics[scale=0.2]{hist_0}\\
			\includegraphics[scale=0.2]{hist_1}\\
			\includegraphics[scale=0.2]{hist_2}\\
			\includegraphics[scale=0.2]{hist_3}\\
			\includegraphics[scale=0.2]{hist_4}\\
			\includegraphics[scale=0.2]{hist_5}\\
			\includegraphics[scale=0.2]{hist_6}\\
			\includegraphics[scale=0.2]{hist_7}\\
			\includegraphics[scale=0.2]{hist_8}\\
			\includegraphics[scale=0.2]{hist_9}\\
			\includegraphics[scale=0.2]{hist_10}\\
			\includegraphics[scale=0.2]{hist_11}\\
			\includegraphics[scale=0.2]{hist_12}\\
		\end{center}

	\paragraph{Sol. 3.2}
		The after applying Linear Regression to the training set, we get the results

		\begin{center}
			\begin{tabular}{c|c}
				Test Data & MSE \\
				\hline
				test\_data & 28.4644 \\
				training\_data &20.9512 \\
				\hline
			\end{tabular}
		\end{center}

		%\newline

		After applying Ridge regression for different $\lambda$ we get the following results
		\begin{center}
		\begin{tabular}{c|c|c}
			\hline
			   	 $\lambda$ & Test Data    &     MSE \\
			\hline
			     0.01 & test\_data  & 28.4183 \\
			     0.01 & train\_data & 20.9501 \\
			     0.1  & test\_data  & 28.4217 \\
			     0.1  & train\_data & 20.9502 \\
			     1    & test\_data  & 28.4574 \\
			     1    & train\_data & 20.9539 \\
			\hline
		\end{tabular}
		\end{center}

		\begin{center}
		\begin{tabular}{c|c}
		\hline
		   $\lambda$ &     CVE \\
		\hline
		    0.001 & 9.99383 \\
		    0.01  & 9.99273 \\
		    0.02  & 9.9915  \\
		    0.1   & 9.98177 \\
		    0.1   & 9.98177 \\
		    1     & 9.8773  \\
		    2     & 9.77169 \\
		    3     & 9.67664 \\
		    4     & 9.59175 \\
		    5     & 9.51671 \\
		    6     & 9.45125 \\
		    7     & 9.39513 \\
		    8     & 9.34814 \\
		    9     & 9.31007 \\
		   10     & 9.28075 \\
		\hline
		\end{tabular}
		\end{center}

		As we see that the cross-validation error is minimum for $\lambda = 10$, therefore we choose $\lambda = 10$ for out MSE calculation.
		For \textbf{Test Data set we get MSE = 28.98456207} and for \textbf{Training Data set we get MSE = 21.28681485}.

		\paragraph{Sol. 3.3.a}
		First, we will calculate the Pearson's Coefficient for each attribute with target values and we get
		\begin{center}
		\begin{tabular}{c | c}
			\hline
			 Feature   &   abs(r) \\
			\hline
			 CRIM      & 0.387697 \\
			 ZN        & 0.362987 \\
			 INDUS     & 0.483067 \\
			 CHAS      & 0.2036   \\
			 NOX       & 0.42483  \\
			 RM        & 0.690923 \\
			 AGE       & 0.390179 \\
			 DIS       & 0.252421 \\
			 RAD       & 0.385492 \\
			 TAX       & 0.468849 \\
			 PTRATIO   & 0.505271 \\
			 B         & 0.343434 \\
			 LSTAT     & 0.73997  \\
			\hline
			\end{tabular}
		\end{center}

		We see that the attributes with highest Pearson's Coefficients are INDUS, PTRATIO, RM, LSTAT.\\

		Now, applying \textbf{Linear Regression}
	    \begin{center}
	    \begin{tabular}{c | c}
	        \hline
	        Test Data & MSE \\
	        \hline
	        test\_data & 31.4962 \\
	        train\_data & 26.4066
	    \end{tabular}
	    \end{center}

	    Now, applying Ridge Regression for different $\lambda$ we get

	    \begin{center}
	    \begin{tabular}{c|c|c}
			\hline
			   $\lambda$ & Test Data    &     MSE \\
			\hline
			     0.01 & test\_data  & 31.496  \\
			     0.01 & train\_data & 26.4066 \\
			     0.1  & test\_data  & 31.4944 \\
			     0.1  & train\_data & 26.4066 \\
			     1    & test\_data  & 31.4806 \\
			     1    & train\_data & 26.4094 \\
			\hline
		\end{tabular}
	    \end{center}

	    For K-Fold Ridge Regression we get the following values of CVE

	    \begin{center}
	    \begin{tabular}{c | c}
			\hline
			   $\lambda$ &    CVE \\
			\hline
			    0.001 & 8.3003 \\
			    0.01  & 8.2992 \\
			    0.1   & 8.2885 \\
			    1     & 8.1855 \\
			    2     & 8.0791 \\
			    3     & 7.9811 \\
			    4     & 7.8915 \\
			    5     & 7.8099 \\
			    6     & 7.7363 \\
			    7     & 7.67   \\
			    8     & 7.6125 \\
			    9     & 7.562  \\
			   10     & 7.5189 \\
			\hline
			\end{tabular}
	    \end{center}

	  	Choosing $\lambda = 10$, we get the MSE as

		\begin{center}
	    \begin{tabular}{c|c|c}
			\hline
			   $\lambda$ & Test Data    &     MSE \\
			\hline
			     10 & test\_data  & 31.5937  \\
			     10 & train\_data & 26.6775 \\
			\hline
		\end{tabular}
	    \end{center}

	\paragraph{Sol. 3.3.b}
	    From the previous exercise we know that the feature with the highest correlation coefficient with the target is LSTAT.

	    Now, calculating residue and correlation coefficient of the rest of the attributes with the residue we get

	    \begin{center}
	    \begin{tabular}{c|c}
			\hline
			 Feature & abs(r(residue, attr\_vals)  \\
			\hline
			 CRIM    & 0.426588158091 \\
			 ZN      & 0.398133315751 \\
			 INDUS   & 0.537842703653 \\
			 CHAS    & 0.175073939283 \\
			 NOX     & 0.491958859016 \\
			 RM      & 0.702473672384 \\
			 AGE     & 0.469369575525 \\
			 DIS     & 0.331853691887 \\
			 RAD     & 0.436013562838 \\
			 TAX     & 0.514184631153 \\
			 PTRATIO & 0.496710217811 \\
			 B       & 0.374589793897 \\
			\hline
			\end{tabular}
	    \end{center}

	    We get RM as the next feature with highest correlation coefficient. \\

	    After updating the residue, we get the correlation coeff. as

	    \begin{center}
	    \begin{tabular}{c|c}
			\hline
			 Feature & abs(r(residue, attr\_vals)  \\
			\hline
			 CRIM    & 0.129554112358  \\
			 ZN      & 0.0461901567406 \\
			 INDUS   & 0.0561934824677 \\
			 CHAS    & 0.249301273706  \\
			 NOX     & 0.0109129511978 \\
			 AGE     & 0.0159396362859 \\
			 DIS     & 0.134713549718  \\
			 RAD     & 0.087477994304  \\
			 TAX     & 0.13681944445   \\
			 PTRATIO & 0.297604744231  \\
			 B       & 0.156620202695  \\
			\hline
			\end{tabular}
	    \end{center}

	    We get PTRATIO as the next feature with highest correlation coefficient. \\

	    Again,
	    \begin{center}
	    \begin{tabular}{c|c}
			\hline
			 Feature & abs(r(residue, attr\_vals)  \\
			\hline
			 CRIM    & 0.0889293951571  \\
			 ZN      & 0.0323063735902  \\
			 INDUS   & 0.00214795182009 \\
			 CHAS    & 0.2195949638     \\
			 NOX     & 0.0198942311859  \\
			 AGE     & 0.0435891823287  \\
			 DIS     & 0.170389023828   \\
			 RAD     & 0.0210281033187  \\
			 TAX     & 0.0440426778355  \\
			 B       & 0.144133379832   \\
			\hline
		\end{tabular}
	    \end{center}

	  	Finally, we select the feature CHAS.\\

	  	So, finally our selected features (in-order) are LSTAT, RM, PTRATIO, CHAS. \\

	  	Now, calculating MSE from previously fitted data, we get

	  	\begin{center}
	  	\begin{tabular}{c|c}
			\hline
			 Input Data   &     MSE \\
			\hline
			 test\_data    & 34.5988 \\
			 train\_data   & 25.106  \\
			\hline
			\end{tabular}
	  	\end{center}

	  	\textbf{Brute Force}: The best combination is B, RM, LSTAT, PTRATIO with $MSE = 30.09226$ with test data set.

	  \paragraph{Sol. 3.4}
	  	The result after calculating the MSE using Linear Regression with the augmented data is

	  	\begin{center}
	  	\begin{tabular}{c|c}
			\hline
			 Input Data   &     MSE \\
			\hline
			 test\_data    & 14.5553 \\
			 train\_data   & 5.05978 \\
			\hline
			\end{tabular}
	  	\end{center}

\end{document}


%auto-ignore
\providecommand{\MainFolder}{..}
\documentclass[\MainFolder/Text.tex]{subfiles}
\newcommand{\TestF}{\mathcal{D}}
\begin{document}
\section{Schwartz form and smooth extension to blow-up}\label{Sec:SchwFrom}
\allowdisplaybreaks

For a smooth vector bundle $\pi: E\rightarrow M$, let $\TestF(E)$ be the space of ``test sections of $E$'', and let $\TestF'(E^*)$ be the space of distributions.  Let $\langle\cdot,\cdot\rangle: \TestF'(E^*)\otimes\TestF(E)\rightarrow \R$ be the natural pairing. Proper definitions can be formulated easily based on \cite[Section~6]{HormanderI}.
\begin{Proposition}[Schwartz kernel theorem, see {\cite[Theorem~5.2.1]{HormanderI}} for the local version]\label{Prop:SchwKer}
Let $E_1$ and $E_2$ be smooth vector bundles over smooth manifolds $M_1$ and $M_2$, respectively. Then there is a one-to-one correspondence between continuous operators $\TOp: \TestF(E_1)\rightarrow\TestF'(E_2)$ and elements $\KKer_\TOp\in\TestF'(E_1\boxtimes E_2^*)$ which is given by the equation
$$ \langle \TOp s_1, \psi_2 \rangle = \langle \KKer_\TOp, s_1 \boxtimes \psi_2 \rangle\quad\text{for all }s_1\in \TestF(E_1)\text{ and }\psi_2\in \TestF(E_2^*). $$
The distributional section $\KKer_{\TOp}$ is called the \emph{Schwartz kernel} of $\TOp$.
\end{Proposition}

An $L^1_{\text{loc}}$-integrable function $k: M\times M \rightarrow \Hom(E_1,E_2)=E_1^*\boxtimes E_2$ on an oriented Riemannian manifold $M$ defines the distribution $\KKer\in \TestF'(E_1\boxtimes E_2^*)$ by
$$ \langle\KKer,s_1\boxtimes \psi_2\rangle = \int_{x_1, x_2} \langle k(x_1,x_2)s_1(x_1),\psi_2(x_2)\rangle \Vol(x_1)\Vol(x_2) $$
%$$ \langle T s_1, \psi_2 \rangle = \int_{x_1} \langle t(s_1)(x_1),\psi_2(x_1)\rangle\Vol(x_1) $$
In the case of exterior bundles, we introduce an equivalent notion of a Schwartz form. This name was proposed by Dr.~A.~Hermann in a discussion in Potsdam.

\begin{Proposition}[Schwartz kernel and Schwartz form]\label{Prop:SchwForm}
In the setting of Proposition~\ref{Prop:SchwKer}, suppose that $M_1 = M_2 = M$ is a smooth oriented Riemannian manifold and $E_1 = E_2 = \Lambda T^* M$. We consider the isomorphism of vector bundles $\Psi:\Lambda T^*M\boxtimes(\Lambda T^*M)^*\rightarrow\Lambda T^*(M\times M)$ which is for every $x_1$, $x_2\in M$ given by
\begin{align*}
 \Psi: \Lambda T^*_{x_1}M\otimes\bigl(\Lambda T_{x_2}^*M\bigr)^* &\longrightarrow \Lambda T^*_{(x_1,x_2)}(M\times M) \\
 \omega_1\otimes\xi_2&\longmapsto\omega_1\wedge\sharp\xi_2.
\end{align*}
Here, $\sharp: \Lambda T^*M \rightarrow \bigl(\Lambda T^*M \bigr)^* $ denotes the musical isomorphism with respect to the natural pointwise inner product on $\Lambda T^*M$. We obtain the isomorphism
\begin{align*}
\Psi_*:\DR'(M\times M)&\longrightarrow\TestF'(E_1^*\boxtimes E_2)\\
\DR_\TOp&\longmapsto\KKer_\TOp,
\end{align*}
where $\DR'$ denotes the space of de Rham currents.

If $\TOp$ is homogenous of degree~$\Abs{\TOp}$, then the degree of $\DR_\TOp$ satisfies
$$ \Deg(\DR_\TOp) = \dim(M) + \Abs{\TOp}. $$

If $\KKer_T$ is represented by an $L^1_{\mathrm{loc}}$-integrable section of $\Hom(E_1,E_2)\simeq E_1^*\boxtimes E_2$, then $\DR_T$ is represented by an $L^1_{\mathrm{loc}}$-integrable form, and the other way round. In this case, for all $\omega_1 \in \DR(M)$, $\omega_2\in \DR_c(M)$, we have
\begin{equation}\label{Eq:SchwarzKerForm}
\begin{aligned}
\int_{x_2} \bigl((\TOp \omega_1)(x_2)\bigr)\bigl[\omega_2(x_2)\bigr] &= \int_{x_1, x_2} \KKer_T(x_1,x_2)[\omega_1(x_1)]\wedge \Vol(x_1)\wedge \omega_2(x_2)\\
&= \int_{x_1, x_2} \DR_T(x_1,x_2)\wedge\omega_1(x_1)\wedge\omega_2(x_2).
\end{aligned}
\end{equation}
\end{Proposition}
\begin{proof}
Straightforward computations similar to Example~\ref{Ex:IdFE} below.
\end{proof}

\begin{Definition}[Schwartz form]\label{Def:SDFDF}
The current $\DR_\TOp\in \DR'(M\times M)$ from Proposition~\ref{Prop:SchwForm} is called the \emph{Schwartz form} of $\TOp$.
\end{Definition}

We will consider pseudo-differential operators $\TOp:\DR(M)\rightarrow\DR(M)$ on a Riemannian manifold $M$; this class of operators generalizes differential operators and contains generalized inverses of elliptic operators (see \cite{Hormander} for thorough treatment).

\begin{Proposition}[Schwartz form of pseudo-differential operators] \label{Prop:ASD}
Let $\TOp: \DR(M) \rightarrow \DR(M)$ be a pseudo-differential operator on a smooth oriented Riemannian manifold $M$. Then the Schwartz form $\DR_{\TOp}$ restricts to a smooth form on $M\times M\backslash\Diag$.
\end{Proposition}
\begin{proof}
Well known fact, proof based on \cite{Hormander}.
\end{proof}

\begin{Example}[Schwartz form of $\Id$]\label{Ex:IdFE}The Schwartz form of the identity $\Id: \DR(\R^n)\rightarrow \DR(\R^n)$ reads
\begin{equation}\label{Eq:SchwartzFormOfId}
\DR_{\Id}(x,y) = \delta(x-y)(\Diff{x}^1 - \Diff{y}^1)\dotsb(\Diff{x}^n-\Diff{y}^n),
\end{equation}
where $\delta$ denotes the Dirac delta function on $\R^n$ centered at $0$. In order to prove this, we start by rewriting
\begin{align*}
(\Diff{x}^1 - \Diff{y}^1) \dotsb (\Diff{x}^n - \Diff{y}^n) &=\sum_{I}(-1)^{\Abs{I}} \varepsilon(I^c I \mapsto[n])\Diff{x}^{I^c} \wedge \Diff{y}^I \\
&=\sum_{I} (-1)^{n \Abs{I}} (\Star \Diff{x}^I) \wedge \Diff{y}^I.
\end{align*}
Here, the sum is over all multiindices $I\subset \{1,\dotsc,n\}$, and we use that $\Star(\Diff{x}^I) = \varepsilon(I, I^c) \Diff{x}^{I^c}$, where $\varepsilon(I,I^c)$ denotes the sign to order $I I^c$ to $\{1,\dotsc,n\}$. Now, for any $\omega\in \DR_c(\R^n)$, which we write as $\omega(x) = \sum_K \omega_K(x)\Diff{x}^K$, we compute using $\Diff{x}^K\wedge\Star(\Diff{x}^I)=(\Diff{x}^K,\Diff{x}^I)\Vol(x) = \delta^{KI}\Vol(x)$ the following:
\begin{align*}
&\int_x \delta(x-y)  (\Diff{x}^1-\Diff{y}^1)\dotsb(\Diff{x}^n-\Diff{y}^n) \omega(x)  \\
&\qquad=\sum_{I,K} \int_x \delta(x-y)(-1)^{n\Abs{I}}  \omega_K(x)(\Star \Diff{x}^I)\Diff{y}^I\Diff{x}^K \\
&\qquad=\sum_{I,K}\int_x \delta(x-y) (-1)^{n\Abs{I} + n\Abs{K}} \delta(x-y)  \omega_K(x) \Diff{x}^K \Star(\Diff{x}^I) \Diff{y}^I \\
&\qquad=\sum_{I} \int_x \delta(x-y) \omega_I(x)\Vol(x) \Diff{y}^I \\
&\qquad=\omega(y).
\end{align*}
This shows \eqref{Eq:SchwarzKerForm}, and \eqref{Eq:SchwartzFormOfId} follows. Notice that
$$ \Restr{\DR_\Id}{\R^n\times \R^n\backslash \Diag} = 0, $$
and thus the smooth part of $\DR_\Id$ does not recover the data of the operator $\Id$.
\end{Example}

%\begin{Definition}[Laplace Green kernel and standard Hodge propagator]\label{Def:GKerHPr}
%Let $M$ be a closed oriented Riemannian manifold, $\DR(M)=\Dd\DR(M)\oplus\CoDd\DR(M)\oplus\Harm(M)$ the Hodge decomposition, $\pi: \DR(M) \rightarrow \Harm(M)$ the harmonic projection and $\Laplace= \Dd\circ\CoDd + \CoDd\circ\Dd$ the Hodge de Rham Laplacian. We define the \emph{Laplace Green operator}  $\GOp: \DR(M)\rightarrow \DR(M)$ by
%$$ .$$
%We denote its Schwartz form by $\GKer$ and call it the \emph{Laplace Green kernel.}  We define the \emph{standard Hodge homotopy} by
%$$ \StdHtp \coloneqq - \CoDd \circ \GOp : \DR^\bullet(M) \longrightarrow \DR^{\bullet - 1}(M). $$
%It satisfies $\Dd\circ\StdHtp + \StdHtp\circ\Dd = - \Id$. We denote the Schwartz form of $\StdHtp$ by $\StdPrpg$ and call it the \emph{standard Hodge propagator.}
%\end{Definition}

We consider the Green operator $\GOp$ for the Laplacian $\Laplace$ (see \cite{Warner1983}) and the standard Hodge propagator $\HtpStd$. They are both pseudo-differential operators, and it holds $\HtpStd = - \CoDd \GOp$. We will study their Schwartz forms $\GKer$ and $\PrpgStd$, which are called the \emph{Green kernel} and the \emph{standard Hodge propagator,} respectively.

\begin{Proposition}[Basic facts about $\GKer$ and $\StdPrpg$]\label{Prop:BasicFactsGP}
The Green kernel $\GKer$ represents an $L^2$-integrable form.
The standard Hodge propagator $\StdPrpg$ represent an $L^1$-integrable form.
\end{Proposition}
\begin{proof}
The fact that $\GKer$ is $L^2$ is an exercise in \cite{Warner1983}. The fact that~$\StdPrpg$ defines an $L^1$-integrable form on $M\times M$ for a compact manifold $M$ was proved in \cite{Harris2004} using the heat kernel approximation (see the next section).
\end{proof}

A consequence of Proposition~\ref{Prop:BasicFactsGP} is that the Schwartz forms $\GKer$ and $\StdPrpg$ are determined by their smooth restrictions to $M\times M\backslash \Diag$. This follows from \eqref{Eq:SchwarzKerForm} because the integral does not depend on sets of zero measure. Therefore, we will write $\GKer$, $\StdPrpg \in \DR^{n-1}(M\times M\backslash\Diag)$.

\begin{Definition}[Smooth extension to the blow-up]\label{Def:Esdas}
Let $M$ be a smooth manifold, and let $\pi: \Bl_\Diag(M\times M)\rightarrow M\times M$ be the spherical blow-up of $M\times M$ at the diagonal $\Diag$.\footnote{Another name of this construction suggested to me by Dr.~Oliver Lindblad Petersen after explaining him our setting should be ``Melrose blow-up''.} Consider the blow-up diagram
\[\begin{tikzcd}
 & \Bl_\Diag(M\times M)\arrow[two heads]{d}{\pi} \\
 M\times M\backslash \Diag \arrow[hook]{r}{\iota}\arrow[hook]{ru}{\tilde{\iota}} & M\times M,
\end{tikzcd}\]
where $\iota$ is the inclusion and $\tilde{\iota}$ its unique smooth lift --- the embedding of the interior. We say that a smooth form $\omega\in\DR(M\times M\backslash\Diag)$ \emph{extends smoothly to the blow-up} if there is a smooth form $\tilde{\omega}\in \DR(\Bl_\Diag(M\times M))$ such that
$$ \tilde{\iota}^*\tilde{\omega}=\omega. $$
\end{Definition}

Note that the extension, if it exists, is necessarily unique.

\begin{Question}
Do $\GKer$ and $\StdPrpg$ extend smoothly to the blow-up?
We expect that $\GKer$ does not and $\StdPrpg$ does.
\end{Question}
\end{document}


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 10.4 Multi-Dimensional Real Fourier Transform

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

This subroutine computes Real Fourier Transforms for real data in up to six
dimensions using the fast Fourier transform. In ND dimensions, the relations
between the values $x$ and the Fourier coefficients $\xi $ have the form%
\begin{multline*}
x(j_1,j_2,...,j_{ND})=\sum_{k_1=0}^{N_1-1}\cdots\sum_{k_{ND}=0}^{N_{ND}-1}%
\xi (k_1,k_2,...,k_{ND})\\
\times W_1^{j_1k_1}\cdots W_{ND}^{j_{ND}k_{ND}},
\end{multline*}
where $N_\ell=2^{\text{M}(\ell)}$, $W_\ell=e^{2\pi i/N_\ell}$, 0 $\leq $ $j_\ell
\leq N_\ell-1$, $x$ is real and $\xi $ is complex.

\subsection{Usage}

\subsubsection{Program Prototype, Single Precision}

\begin{description}
\item[\bf REAL]  \ {\bf A}$(N_1$, $N_2$, ..., $\geq N_{ND})$\quad $%
[N_k=2^{\text{M}(k)}]$

\item[\bf REAL]  \ {\bf S}$(\geq \max (\nu _1$, $\nu _2$, ..., $\nu
_{ND})-1)$\  $[\nu _k=2^{\text{M}(k)}-^2]$

\item[\bf INTEGER]  \ {\bf M}$(\geq $ ND){\bf , ND, MS}

\item[\bf CHARACTER]  \ {\bf MODE}
\end{description}
On the initial call set MS to~0 to indicate the array S() does not yet
contain a sine table. Assign values to A(), MODE, M(), and ND.
$$
\fbox{{\bf CALL SRFT(A, MODE, M, ND, MS, S)}}
$$
On return A() will contain computed results. S() will contain the sine table
used in computing the Fourier transform. MS may have been changed.

\subsubsection{Argument Definitions}

\begin{description}
\item[A()]  [inout] If the argument MODE selects Analysis, A() contains
values $x$ on entry and the Fourier coefficients $\xi $ on exit. If MODE
selects Synthesis, A() contains the Fourier coefficients $\xi $ on entry and
the values $x$ on exit. The Functional Description below describes the way $x
$ and $\xi $ are stored in A().

\item[MODE]  [in] The character variable MODE selects Analysis or Synthesis.

'A' or 'a' selects Analysis, transforming $x$'s to $\xi ^{\prime }s.
$

'S' or 's' selects Synthesis, transforming $\xi $'s to $x^{\prime
}s.$

\item[M()]  [in] Defines $N_k=2^{\text{M}(k)}$, the number of real data points in
the $k^{th}$ dimension. Require $0\leq \text{M}(k)\leq 31$ for all $k$%
, and M($1)=0$ only if M($k)\equiv 0$ for all $k$. No action is taken with
respect to dimensions for which M($k)=0.$

\item[ND]  [in] Number of dimensions. Require 1 $\leq $ ND $\leq $ 6.

\item[MS]  [inout] Gives the state of the sine table in $S()$.  Let
$\text{MS}_{in}\text{ and MS}_{out}$ denote the values of MS on entry
and return respectively. If the sine table has not previously been
computed, set $\text{MS}_{in} = 0$ or $-$1 before the call. Otherwise
the value of $\text{MS}_{out}$ from the previous call using the same
S() array can be used as $\text{MS}_{in}$ for the current call.

Certain error conditions described in Section E cause the subroutine
to set $\text{MS}_{out} = -2$ and return.  Otherwise, with $\max _i
\{\text{M}(i)\} > 0$, the subroutine sets $\text{MS}_{out} = \max
($M(1), M(2), ..., M(ND), $\text{MS}_{in}).$

If $\text{MS}_{out} > \max (2, \text{MS}_{in}),$ the subroutine sets
NT = $2^{\text{MS}_{out}-2}$ and fills S() with NT $-$ 1 sine values.

If $\text{MS}_{in}=-1$, the subroutine returns after the above
actions, not transforming the data in A().  This is intended to allow
the use of the sine table for data alteration before a subsequent Fourier
transform, as discussed in Section G of Chapter~16.0.

\item[S()]  [inout] When the sine table has been computed, S($j)=\sin \pi
j/(2\times {\textstyle NT})$, $j=1$, 2, ..., NT $-$ 1, see MS above.
\end{description}

\subsubsection{Modifications for Double Precision}

Change SRFT to DRFT and the REAL type statements to DOUBLE PRECISION.

\subsection{Examples and Remarks}

A ``smooth'' function that approximates%
\begin{equation*}
A(i,j)=
\begin{cases}
0 & \text{if}\ |9-i|+|9-j|>4 \\
1 & \text{if}\ |9-i|+|9-j|\leq 4
\end{cases}
\ 1\leq i,\ j\leq 16.
\end{equation*}
is desired. The example at the end of the chapter does this by computing
the two dimensional transform of A, applying sigma factors (see Section
G of Chapter~16.0), and then transforming back. Results are printed only
for 1 $\leq $ $i,\ j\leq 9$ since, to within round-off limitations, A($%
9+m,9-n)={\textstyle A}(9-m,9-n)$, $1\leq 9\pm m\leq 16$ and $1\leq $ $9\pm
n\leq 16.$

\subsection{Functional Description}

The multi-dimensional real Fourier transform is done by changing it to a
problem in complex variables, doing a multi-dimensional complex Fourier
transform, and then adjusting the results to obtain the solution for the
original problem.

Taking complex conjugates in Eq.\,(1) and using the fact that $x$ is real, it
can be verified that%
\begin{equation*}
\xi (N_1-k_1,\;N_2-k_2,...,N_d-k_d)=\overline{\xi }(k_1,k_2,...,k_d),
\end{equation*}
where $N_i-k_i$ is interpreted modulo $N_i$. (Thus $N_i-k_i=0$ if $k_i$ is
0.) Using the above, it is possible to pack the nonredundant $\xi ^{\prime
}s $ in the same space as is required for $x$.

Storage of $\xi $ in A() for the case $d=2$ is illustrated in Table~1. The
rows and columns in the table correspond to rows and columns in the array
A(). Only the subscripts $k_1$, $k_2$ of the $\xi $'s are given.
The symbols $J_1$ and $J_2$ are used as abbreviations for $N_1/2$ and $N_2/2$
respectively. The coefficients with subscripts (0,0), $(J_1,0)$, $(0,J_2)$,
and $(J_1,J_2)$ are real and occupy single array elements as shown. For the
other coefficients, the subscript $``k_1,k_2"$ identifies the location of
the real part of $\xi (k_1$, $k_2)$, and an ``I'' immediately below such a
subscript gives the location of the imaginary part of $\xi (k_1$,\ $k_2).$

The first column (with the second subscript ignored) gives the storage
scheme for the case $d=1$. For $d>2$, and with $\nu =$ the smallest value of
$i(>1)$ for which $k_i\neq 0$ and $k_i\neq N_\nu /2$, the storage scheme
generalizes as follows.

If $1 < k_1 < N_1$ (A$(2k_1+1$, $k_2+1$, ..., $k_d+1),$ A$(2k_1+2$,
$k_2+1$, ..., $k_d+1))$ contains $\xi (k_1,k_2,...,k_d)$.  Else
(A(1, $k_2+1$, ..., $k_d+1),$ A(2, $k_2+1$, ..., $k_d+1))$ contains
\begin{alignat*}{2}
&\xi (0,k_2,...,k_d) &\ &1\leq k_\nu <N_\nu /2\\
&\xi (N_1/2,k_2,...,k_d)&\ \qquad &N_\nu /2<k_\nu <N_\nu
\end{alignat*}
and when $k_i$ is either 0 or $N_i/2$ for all i,
(A(1, $k_2+1$, ..., $k_d+1),$ A(2, $k_2+1$, ..., $k_d+1))$ contains $(\xi
(0,k_2,...,k_d),~\xi (N_1/2,k_2,...,k_d)).$ Note that in this last case both
$\xi $'s are real.

More details can be found in \cite{Krogh:1970:RFT}.

\bibliography{math77}
\bibliographystyle{math77}

\subsection{Error Procedures and Restrictions}

Require $0 \leq \text{M}(k)\leq 31$ and $1 \leq \text{ND} \leq $ 6.  MODE
must contain one of the allowed values.  If any of these conditions are
violated the subroutine will issue an error message using the error
processing procedures of Chapter~19.2 with a severity level of~2 to cause
execution to stop.  A return is made with $\text{MS}=-2$ instead of
stopping if the statement ``CALL\ ERMSET($-$1)'' is executed before
calling this subroutine.

If the sine table does not appear to have valid data, an error message is
printed, and the sine table and then the transform are computed.

\subsection{Supporting Information}

The source language is ANSI Fortran~77.

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DRFT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
 DFFT, DRFT, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
SRFT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
 ERFIN, ERMSG, IERM1, IERV1, SFFT, SRFT}\\
\end{tabular}

Subroutine designed and written by: Fred T. Krogh, JPL, October~1969,
revised January~1988.

\end{multicols}

\begin{table} \centering
  \begin{tabular}{cccccccc}
     0,0 & 0,1 & $\cdots$ & 0,$J_2-1$ & 0,$J_2$ & $J_1$,$J_2+1$ & $\cdots$ & $J_1$,$N_2 -1$ \\
     $J_1$, 0 & I & $\cdots$ & I & $J_1$,$J_2$ & I & $\cdots$ & I \\
      1,0 & 1,1 & $\cdots$ & 1,$J_2-1$ & 1,$J_2$ & 1,$J_2+1$ & $\cdots$ & 1,$N_2 -1$ \\
     I & I & $\cdots$ & I & I & I & $\cdots$ & I \\
    $\cdots$ & & & & $\cdots$ & & & $\cdots$ \\
     $J_1 -1$,0 & $J_1-1$,1 & $\cdots$ & $J_1-1$,$J_2-1$ & $J_1-1$, $J_2$ & $J_1-1$,$J_2+1$ & $\cdots$ & $J_1-1$,$N_2 -1$ \\
    I & I & $\cdots$ & I & I & I & $\cdots$ & I \\
  \end{tabular}
  \caption{Storage of $\xi$ in A() for Two Dimensions}
\end{table}

\begcodenm

\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRSRFT}\vspace{10pt}
\lstinputlisting{\codeloc{srft}}

\vspace{30pt}\centerline{\bf \large ODSRFT}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{srft}}
\end{document}


\section{Modeling Coral Reef Dynamics}

\begin{frame}
\frametitle{Coral Reef Dynamics}
\includegraphics[scale=.175]{./coral-reef-triangle.png}\\ \cite{Hastings}
\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}
The deterministic ordinary differential equation model:
$$\begin{cases}\begin{array}{rl}
\frac{dM}{dt}\hspace{-.8em}&=aMC - \frac{gM}{M+T} + \gamma M T, \\
\frac{dC}{dt}\hspace{-.8em}&=rTC - dC - aMC, \\
\frac{dT}{dt}\hspace{-.8em}&=\frac{gM}{M + T} - \gamma MT - rTC + dC.
\end{array}\end{cases}$$

where
\begin{itemize}\itemsep0pt
\item $r$ is the rate corals overgrow upon algal turfs\\
\item $d$ is the mortality rate of corals\\
\item $a$ is the rate that macroalgae overgrow upon corals\\
\item $\gamma$ is the rate that macroalgae spread over algal turfs\\
\item $g$ is the indiscriminate grazing rate of scarid
\end{itemize} We also assume $a<d<\gamma<r<2\gamma$ and $0<g<\gamma$. \vspace{1em}
\cite{Hastings}
\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}

\hspace{1.57em}

\begin{itemize}
\item $\frac{dT}{dt}=-\frac{dM}{dt}-\frac{dC}{dt}$ implies $M+C+T$ is
  constant.
\item We assume that $M+C+T=1$.
\item This limits our scope to regions entirely covered by coral,
  macroalgae, and algal turf.
\end{itemize}

Thus, we reduce our system to,
$$\begin{cases}
\begin{array}{rl}
\frac{dM}{dt}&= aMC-\frac{gM}{1-C} + \gamma M - \gamma M^2 -\gamma M C,\\
\frac{dC}{dt}&=rC - rC^2 - rCM - dC - aMC.
\end{array}
\end{cases}$$
\end{frame}

\begin{comment}\frametitle{Equilibria and Stability}
Equilibrium point

\end{comment}

\begin{frame}\frametitle{Equilibria and Stability}
To find equilibrium points analytically we first set the derivatives equal to zero to obtain the nullclines:
$$\begin{cases}
\begin{array}{rl}
0\hspace{-.8em}&=M(aC + \gamma-\gamma M-\gamma C - \frac{g}{1-C}),\\
0\hspace{-.8em}&=C(r-Mr-Cr - d - aM).
\end{array}
\end{cases}$$
\end{frame}

\begin{frame}\frametitle{Equilibria and Stability}

  \begin{itemize}
  \item $M'=0$ is satisfied when:
    \begin{itemize}
    \item $M=0$, or
    \item
      $aC + \gamma - \gamma M - \gamma C -
      \frac{g}{1-C}=0$.
    \end{itemize}
  \item $C'=0$ is satisfied when:
    \begin{itemize}
    \item$C=0$, or
    \item $r-Mr-Cr-d-aM=0$.
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}\frametitle{Phase Plane}

  %Our equilibrium points are located at the intersections of the nullclines:\\
  \includegraphics[scale=.325]{./nullclines.pdf}

\end{frame}

\begin{frame}
\frametitle{Time Delay in the Coral Ecosystem}

We can identify time delays in the coral ecosystem:
\begin{itemize}
\item the delay between overfishing of herbivores and growth of
  macroalgae \cite{Dinsdale},
\item the delay between the growth of algae and the effect of algae on
  coral, \cite{Mumby}
\item the delay between the grazing of macroalgae and growth of algal
  turf \cite{Hastings}.
\end{itemize} We focus on the third phenomena.

\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}
Scarid grazing has an impact for the macroalgae in the future:
$$\begin{cases}
  \begin{array}{rl}
    \frac{dM}{dt}\hspace{-.8em}&=aMC - \frac{gM(t-\tau)}{1-C(t-\tau)} + \gamma M (1-M-C),\\
    \frac{dC}{dt}\hspace{-.8em}&=rC(1-M-C) - dC - aMC,\\
  \end{array}
\end{cases}$$ where $\tau$ is a fixed time delay.
\end{frame}

\begin{frame}
\frametitle{Equilibria and Stability}
These are three equilibrium points of interest:
\begin{itemize}
\item $(0,0)$ (Unstable for all $\tau\geq0$)\\
\item $(0,1-\frac{g}{\gamma})$ (High Coral Cover)\\
\item $(1-\frac{d}{r},0)$ (High Algae Cover).
\end{itemize}
Notice that time delay does not affect the equilibria.
\end{frame}

\begin{frame}\frametitle{Jacobian Matrices}
We linearize our delay model:

\begin{eqnarray}
  \label{eqn:linearizedDelayModel}
  \begin{bmatrix}
    M'\\C'
  \end{bmatrix}=J_1
  \begin{bmatrix}
    M \\
    C
  \end{bmatrix} +
  J_2
  \begin{bmatrix}
    M(t-\tau) \\
    C(t-\tau)
  \end{bmatrix},
\end{eqnarray}

where

\begin{eqnarray*}
  J_1 & = & \begin{bmatrix}
    \gamma-2\gamma M^* +(a-\gamma)C^* & (a-\gamma)M^*\\
    -(a+r)C^* & r-d-(a+r)-2M^*C^*
  \end{bmatrix},  \\
  J_2 & = &
  \begin{bmatrix}
    \frac{-g}{1-C^*} & \frac{-gM^*}{(1-C^*)^2} \\
    0 & 0
  \end{bmatrix}.
\end{eqnarray*}

\end{frame}

\begin{comment}

\begin{frame}\frametitle{Putting Jacobians to Use}
{ Suppose $\begin{bmatrix} x\\y\end{bmatrix}=\overrightarrow{v_1}e^{\lambda t}$.}\\\vspace{2em}
{ Differentiating, setting equal to 1, and doing algebra stuff, we get $(J_1+J_2e^{-\lambda t} -\lambda I)\overrightarrow{v_1}=0$.}
\end{frame}
\end{comment}

\begin{frame}\frametitle{Putting Jacobians to Use}
Consider the Jacobians evaluated at the origin:

\begin{eqnarray*}
  J_1 & = &
            \begin{bmatrix}
              \gamma & 0\\
              0 & r-d
            \end{bmatrix}, \\
  J_2 & = &
            \begin{bmatrix}
              -g & 0\\
              0 & 0
            \end{bmatrix}.
\end{eqnarray*}

\end{frame}

\begin{frame}[c]\frametitle{Putting Jacobians to Use}
%Recall, $(J_1+J_2e^{-\lambda t} -\lambda I)\overrightarrow{v_1}=0$. \\\vspace{2em} Taking determinants, we find the characteristic polynomial of $(J_1+J_2e^{-\lambda t} -\lambda I)$ evaluated at $M=0$, $C=0$ to be

  The characteristic polynomial of
  $(J_1+J_2e^{-\lambda \tau})$ is,
$$(\lambda - r + d)(\lambda -\gamma + ge^{-\lambda\tau})=0.$$

\end{frame}

  \begin{frame}\frametitle{Get Them Eigenvalues}
Hence,
\begin{itemize}{\itemsep .5in}
\item $\lambda = r-d>0$ is an eigenvalue with positive real part
\item Other eigenvalues satisfy $\lambda=\gamma-ge^{-\lambda\tau}$.
\item For $\tau\geq0$, our system has two eigenvalues with positive
  real part. This implies our system is unstable at $(0,0)$ for all $\tau\geq0$.
\end{itemize}
\end{frame}

\begin{comment}
\begin{frame}\frametitle{Get Them Eigenvalues}
Now we consider positive $\tau$.
\begin{itemize}
\item Let $\lambda \tau=(\alpha+i\omega)\tau$.
\item Apply Euler's Formula: $\gamma-g(e^{-\lambda\tau})=\gamma-g(\cos(\lambda\tau)-i\sin(\lambda\tau))$.
\end{itemize}
\end{frame}\end{comment}

\begin{frame}\frametitle{Stochastic Coral Features}
The time delay model fails to account for stochastic features of the ecosystem:
\begin{itemize}
\item human activities (e.g. fishing and boating),
\item grazing habits of scarids,
\item hurricanes.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Stochastic Model}
To account for these stochastic features we add noise!
$$\begin{cases}
\begin{array}{rl}
dM\hspace{-.8em}&=(aMC - \frac{gM(t-\tau)}{1-C(t-\tau)+\gamma M T})dt+{\color{red}\beta M(1-M)dW},\\
dC\hspace{-.8em}&=(rTC  - dC - aMC)dt.\\
\end{array}
\end{cases}$$
\end{frame}


\section{Preliminaries}
All rings mentioned are commutative and have a multiplicative identity,
referred to as $1$.

We will try not to assume that the reader knows everything,
but we will assume general knowledge of algebra and some introductory commutative algebra like definitions of rings,
modules, algebras etc.

\begin{theorem}[Noether's normalization lemma]
Let $k$ be a field and $B$ a non-trivial finitely generated $k$-algebra.
Then there exists a non-negative integer $d$ and algebraically independent elements $y_1, ..., y_d$,
i.e. elements with no relations,
in $B$ such that $B$ integral over the polynomial ring $k[y_1, ..., y_d]$.
\end{theorem}
\begin{proof}
Note that the following proof is heavily based on the proof from Mumford's ``The red book''.

Since $B$ is finitely generated the proof is done by induction on the number of generators, $m$.

$m=0$:
In this case $A=k$ and we are done.

Assume $m>0$ and that the lemma holds for all $k$-algebras generated by $m-1$ or fewer elements.

Now, let $B$ be generated by $y_1, \ldots, y_{m}$.
If there is no relations on the generators,
i.e. they are algebraically independent, then
\begin{equation*}
    B \cong k[y_1, \ldots, y_{m}]
\end{equation*}
and we are done.
Hence we assume that $f$ is a relation on the $m$ generators,
i.e. a polynomial equation
\begin{equation*}
    f(y_1, \ldots y_m) = 0.
\end{equation*}
Let $r$ be an integer.
We are going to determine this later.
Set $z_1 = y_1$ and $z_i = y_i - y_1^{r^{i-1}}$ for $2\leq i \leq m$.
Note that the $z_i$'s also are generators of $B$.
Then we can rewrite the relation as
\begin{equation*}
    f(z_1, z_2+y_1^r, z_3+y_1^{r^2}, \ldots z_m+y_1^{r^{m-1}}) = 0.
\end{equation*}
For $a\in k$,
the highest term of $y_1$ appearing in $a\prod_1^m (z_i + z_1^{r^{i-1}})^{\alpha_i}$ looks like $ay_1^{\alpha_1+r\alpha_2+\ldots+r^{m-1}\alpha_m}$.
Hence, if $r$ is larger than all of the $\alpha_i$'s appearing in $f$,
then the highest term of $y_1$ appearing in $f(z_1, z_2+y_1^r, z_3+y_1^{r^2}, \ldots z_m+y_1^{r^{m-1}})$ also has the same form as the one above,
which means that $y_1$ is integral over $k[y_2, \ldots, y_m]$.

Since $y_{i}=z_{i}+y_{1}^{r^{i-1}}$ are also integral over the same ring,
the $z_i$'s generate $B$, and being an integral extension is a transitive property,
we get that B is integral over $k[y_2, \ldots, y_m]$.
Since $k[y_2, \ldots, y_m]$ is generated by $m-1$ elements,
we can apply the inductive hypothesis, and hence we are done.
\end{proof}

%Note that Noether's normalization lemma also can be formulated,
%and hence is equivalent to the statement that there exists a finite injective $k$-algebra homomorphism $k[y_1, \ldots y_d]\longrightarrow B$ for some $d\in \mathbb{N}$.

We want to apply Noether's normalization lemma to prove Zariski's lemma.
In the proof we present we use a result stating that integral ring extensions preserve Krull dimension.
We actually only use the fact that if a ring with Krull dimension zero is an integral ring extension of another ring,
then this ring also has Krull dimension zero.
The more general statement also is true,
but we only prove a more strict statement that we need.

We remind ourself what we mean by Krull dimension.
\begin{definition}[Krull dimension]
Let $A$ be a ring.
The Krull dimension of $A$ is defined as the supremum over the length of all chains of prime ideals in $A$.
\end{definition}

\begin{proposition}
Let $S$ be a ring and $T$ be an integral ring extension of $S$.
Then the Krull dimension of $S$ is lower or equal to the Krull dimension of $T$.
\end{proposition}
\begin{proof}
Let $\mathfrak{p}_0\subsetneq \cdots \subsetneq \mathfrak{p}_n$ be a chain of prime ideals in $S$.
By the Going-up theorem we can extend any chain of length less than $n$,
$\mathfrak{q}_0\subsetneq \cdots \subsetneq \mathfrak{q}_m$ such that $\mathfrak{q}_i\cap S = \mathfrak{p}_i$,
for $i=1,\ldots, m$,
to a chain of prime ideals $\mathfrak{q}_0\subsetneq \cdots \subsetneq \mathfrak{q}_n$ in $T$,
such that $\mathfrak{q}_i\cap S = \mathfrak{p}_i$ for $i=1,\ldots, n$.
Hence the Krull dimension of an integral ring extension can't get any lower than the Krull dimension of the ring it extends.
\end{proof}

Note that we here used the so called Going-up theorem.
This theorem proves exactly what is stated in the proof of the proposition,
i.e. that we can extend chains of prime ideals in integral extensions.
We will not prove this theorem,
but we refer the reader to the proof in Introduction to commutative algebra by Atiyah and MacDonald.

This immediately implies what we need,
i.e. that if a ring with Krull dimension zero is an integral ring extension of another ring,
then this ring also has Krull dimension zero.

\begin{corollary}[Zariski's lemma]
Let $k$ be a field, and $K$ be a field extension that is finitely generated as an algebra over $k$.
Then $K$ is a finite field extension,
i.e. a finite dimensional $k$-vectorspace.
\end{corollary}
\begin{proof}
By Noether's normalization lemma,
$K$ is integral over the ring $k[y_1, \ldots y_d]$,
where $y_1, \ldots y_d$ are the elements of $K$ that are algebraically independent over $k$.
Since $K$ is a field, it has Krull dimension $0$,
and since integral ring extensions preserve Krull dimension,
the polynomial ring $k[y_1, \ldots y_d]$ must have Krull dimension zero,
i.e. $d=0$. Hence $K$ is a finitely generated module over $k$,
i.e. a finite dimensional $k$-vectorspace.
\end{proof}

Now, before we tackle the actual proof of Hilbert's nullstellensatz,
we need to chose a formulation.
There are many formulations of the theorem,
all hopefully equivalent,
but we have chosen one of the more standard and iconic ones.
The formulation is based on some language from algebraic geometry,
hence we need some definitions.

\begin{definition}[Radical]
Let A be a ring and $\mathfrak{a}$ an ideal in A.
We define the radical of $\mathfrak{a}$ to be the set
\begin{equation*}
    \sqrt{\mathfrak{a}} = \{ f\in A \ | \ \exists n \in \mathbb{N} \text{ s.t. } f^n \in \mathfrak{a} \}.
\end{equation*}
We say an ideal $\mathfrak{a}$ is a radical ideal if $\mathfrak{a}=\sqrt{\mathfrak{a}}$.
\end{definition}

\begin{definition}[Zero locus]
Let $k$ be an algebraically closed field,
$A$ be the polynomial ring $A = k[t_1, \ldots, t_n]$ and let $S \subset A$ be a subset.
We define the zero locus of $S$ to be the set
\begin{equation*}
    Z(S) = \{ x\in k^n \ | \ f(x)=0, \ \forall f\in S\}.
\end{equation*}
\end{definition}
If a subset $V\subset k^n$ has the form $V=Z(S)$ for some $S\subset A$,
then we call $V$ an affine algebraic variety.

\begin{definition}[Vanishing set]
Let $V\subset k^n$. We define the vanishing set of $V$ to be the set
\begin{equation*}
    I(V) = \{ f\in A \ | \ f(x)= 0, \ \forall x\in V\}.
\end{equation*}
\end{definition}


\documentclass{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{hyperref}
\usepackage{parskip}

\newcommand{\KL}[2]{\mathrm{KL}(#1~\|~#2)}

\begin{document}

Suppose we wish to estimate $p(y \mid x)$ when we observe $(x_i, y_i) \sim p(y
\mid x) p_1(x) $ but care about achieving high accuracy under some other $p(x
\mid x) p_2(x)$ (the \emph{covariate shift} problem). Suppose further that we
have some information about $y$ under the target distribution---namely that we
can sample from $p_2(y) := \sum_x p_2(x) p(y \mid x)$. How can we use this
information to find some $p_\theta(y \mid x)$ that maximizes $E_{x \sim p_2(x)} p(x
\mid y)$?

Idea: optimize
\begin{align}
  \min_{\theta,\eta} ~~ &-\sum_i \log p_\theta(y_i \mid x_i) + \lambda~\KL{p_2}{p_\eta(x)
  p_\theta(y \mid x)}
\end{align}
This can be viewed as a form of posterior regularization and specifically
\textbf{expectation regularization} as in
\url{https://www.aclweb.org/anthology/D19-1004/}.
\begin{align}
  &= -\sum_i \log p_\theta(y_i \mid x_i) + \lambda H(p_2) - \lambda E_{y \sim p_2(y)}
  [\log E_{x \sim p_\eta} p_\theta(y \mid x)] \\
  &\leq -\sum_i \log p_\theta(y_i \mid x_i) + \lambda H(p_2) -  \lambda E_{y \sim p_2(y)}
  E_{x \sim p_\eta} \log p_\theta(y \mid x) \\
  \intertext{(Jensen)}
  \min_{\theta, \eta, \phi} ~~ &\leq -\sum_i \log p_\theta(y_i \mid x_i) + \lambda H(p_2) -  \lambda E_{y \sim p_2(y)}[
  E_{x \sim q_\phi(x \mid y)} [\log p_\theta(y \mid x)] -\KL{q_\phi(x \mid y)}
  {p_\eta(x)}]
\end{align}
(ELBO)

Important special case of this model: \textbf{backtranslation}, where we assume
that $q(x \mid y)$ is the same under $p_1$ and $p_2$ and estimate it directly
from training data. From this perpsective, backtranslation can be viewed not
just as amortizing a noisy channel translation model but actually providing
robustness to the covariate shift implied by any difference between $p(y)$ on
bilingual and monolingual data.

Claim: we can get lots of ``NLP-flavored'' inductive biases in this framework
just by setting $p(y)$ uniform over an appropriate set. Example:

\paragraph{Mutual exclusivity}
Let $\pi$ be a permutation on $[n]$ and $(x_i, y_i)$ be one-hot vectors $(e_i,
e_{\pi(i)}$ for $i \in [n-1]$. Prove that a regularized logistic regression
model has $p(y \mid e_{n-1})$ uniform over classes when trained via MLE but
mostly on $e_{\pi(n-1)}$ when trained as in Equation 1.

\paragraph{Compositionality}
As above, but let each $x_i$ be a concatenation of two one-hot vectors and $y_i$
be the concatenation of their permuted values. Prove that if we hold out $k$ of
these vectors, Equation 1 still recovers the correct values but MLE doesn't.

\paragraph{Productivity}
Something about learning a weighted string transducer.

\end{document}


\subsection{Branching Properties}
$\wp_\Lambda$ has a unique pole in $\mathbb C/\Lambda$ of order $2$.
The other ramification points are at the zeros of $\wp_\Lambda^\prime$.
Recall that
$$\wp_\Lambda^\prime(z)=\sum_{\omega\in\Lambda}\frac{-2}{(z-\omega)^3}$$
which is an odd function with degree $3$ and poles exactly at the lattice points.
For any $\omega\in\Lambda$, we have $\wp_\Lambda^\prime(\omega/2)=\wp_\Lambda^\prime(\omega/2-\omega)=\wp_\Lambda^\prime(-\omega/2)=-\wp_\Lambda^\prime(\omega/2)$ as $\wp_\Lambda^\prime$ is odd, so $\omega/2$ is either a zero or a pole.
So in the period parallelogram $P$, there are at least three zeros (up to $\Lambda$) namely $\omega_1/2,\omega_2/2,(\omega_1+\omega_2)/2$.
But $\deg\wp_\Lambda^\prime=3$, so these are all the zeros and all of them are simple.
\begin{remark}
    $\wp_\Lambda$ has $4$ ramification points in $\mathbb C/\Lambda$, namely $0,\omega_1/2,\omega_2/2,(\omega_1+\omega_2)/2$ each with multiplicity $2$.
    So by the valency theorem, they must have distinct images $\infty=\wp_\Lambda(0),e_1=\wp_\Lambda(\omega_1/2),e_2=\wp_\Lambda(\omega_2/2),e_3=\wp_\Lambda((\omega_1+\omega_2)/2)$.
\end{remark}
\begin{remark}
    By plugging in, this is consistent with Riemann Hurwitz as $\mathbb C/\Lambda$ has genus $1$.
\end{remark}


%---------------------------Stretch---------------------------
\section{Stretch}

The stretch is the ratio of the minimum edge length to the maximum diagonal, normalized
so that a unit cube has a value of 1:
\[
  q = \sqrt{3}\frac{L_{\min}}{D_{\max}}.
\]

Note that if $D_{\max} < DBL\_MIN$, we set $q = DBL\_MAX$.

\hexmetrictable{stretch}%
{$1$}%                                        Dimension
{$[0.25,1]$}%                                 Acceptable range
{$[0,1]$}%                                    Normal range
{$[0,DBL\_MAX]$}%                             Full range
{$1$}%                                        Cube
{Adapted from \cite{fimesh:xx}}%              Citation
{v\_hex\_stretch}%                            Verdict function name


\documentclass{article}

\usepackage{amsmath}

\def\r{\mathbf{r}}
\def\q{\mathbf{q}}
\def\i{\mathbf{i}}
\def\j{\mathbf{j}}
\def\k{\mathbf{k}}

\begin{document}

\section{Coordinate systems}

Let $x,y,z$ be a coordinate system such that the camera is at the
origin, the floor is some known constant $y$, and positive $z$ is the
current direction of motion of the robot. We write
\begin{equation}
  \r = x\i + y\j + z\k
\end{equation}
where $\i,\j,\k$ can be considered unit vectors or quaternions.  These
can be conveniently represented as the complex matrices
\begin{equation}
\i = \begin{pmatrix} 0 & i  \\ i & 0  \end{pmatrix} \qquad
\j = \begin{pmatrix} 0 & -1 \\ 1 & 0  \end{pmatrix} \qquad
\k = \begin{pmatrix} i & 0  \\ 0 & -i \end{pmatrix}
\end{equation}
but other representations are also possible.

The camera is at some orientation given by the quaternion $\q$ as
\begin{equation}
\r' = \q \, \r \, \q^*
\end{equation}
such that $z'$ is the forward direction of the camera.  Let $f$ be the
focal distance.  Then the camera-plane coordinates will be
\begin{equation}
  \begin{pmatrix} X' \\ Y' \end{pmatrix} =
  f \begin{pmatrix} x'/z' \\ y'/z' \end{pmatrix}
\end{equation}
The actual camera coordinates will, however, involve a nonlinear
calibration
\begin{equation}
  X,Y = K(X',Y')
\end{equation}

Consider two snapshots, between which the robot moves by $\Delta z$.
That is, the world coordinate system moves by $-\Delta z$.  This
results in visual vectors
\begin{equation}
  \begin{pmatrix} \Delta X \\ \Delta Y \end{pmatrix}
\end{equation}

\section{Lens-calibrated case}

We assume that $K$ has been calibrated out to the identity.  Then
\begin{equation}
  \Delta\r' = - \q \, \k \, \q^* \, \Delta z
\end{equation}

Suppose first that $\q=1$.  Then
\begin{equation}
   \begin{pmatrix} \Delta X \\ \Delta Y \end{pmatrix} =
   f \begin{pmatrix} x \\ y \end{pmatrix}
   \left(1/z-1/(z-\Delta z)\right)
\end{equation}
or (since $\Delta z$ is small)
\begin{equation}
   \begin{pmatrix} \Delta X \\ \Delta Y \end{pmatrix} =
-f \, \frac{\Delta z}z \begin{pmatrix} X \\ Y \end{pmatrix}
\end{equation}
In this expression, all the visual vectors point to the origin of the
camera plane.  If we now know $f$ and $\Delta z$, we have $z$.

If $\q\neq1$ the visual vectors will still all point to a single
point, namely the vanishing point corresponding to the direction of
motion in camera coordinates.  This fact suggests a simple strategy
for calibrating the pitch and yaw of the camera: find $\q$ so as to
move the vanishing point of the visual vectors to the origin.
Determining the roll (or the $\k$ component of $\q$) is harder, but
maybe not necessary.

\end{document}


\section{Proof of Theorem~\ref{thm:less_than_total_degree}}
\label{app:proof_thm_less_than_total_degree}
To prove Theorem~\ref{thm:less_than_total_degree} we need the following lemmas.
  \begin{lemma}\label{lem:difference_to_p-1}
    For all $(a,b)\in\Z^2$ we have:
    \[
      (a - b)^{p-1} = \sum_{i=0}^{p-1} a^i b^{p-1-i} \mod p.
    \]
  \end{lemma}
  \begin{proof}
    Using the binomial theorem we obtain
    \[
      (a - b)^{p-1} = \sum_{i=0}^{p-1} \binom{p-1}{i} a^i (-b)^{p-1-i}.
    \]
    Computing the binomial coefficient modulo $p$
    \begin{align*}
      \binom{p-1}{i} & = \frac{(p-1)!}{i! (p-1-i)!} \\
      & = \frac{(p-1)(p-2)\dots(i+1)}{1 \cdot 2 \dots (p-(1+i))} \\
      &= (-1)^{p-1-i} \mod p\,,
    \end{align*}
    we prove the lemma.
  \end{proof}

  \begin{lemma}\label{lem:sum_poly}
    Let $P(X)$ be a polynomial of degree $d$ less than $p-1$.
    For any prime number $p > 2$, it holds
    \[
      \sum_{a=0}^{p-1} P(a) = 0 \mod p.
    \]
  \end{lemma}
  \begin{proof}
    Since $\sum_{a=0}^{p-1} b = 0 \mod p$ for any $b \in \F_p$, it is enough to prove that the sum $\sum_{a=0}^{p-1} a^n = 0 \mod p$ for any $0 \leq n < p-1$.
    Since the case $n=0$ is straightforward, let us assume $n>0$.
    Let $g$ be a primitive element of $\F_p$.
    Since $p > 2$, we have $g \ne 1$.
    Thus, we can rewrite the above sum as follows.
    \begin{align*}
      \sum_{i=1}^{p-1} g^{in} = \frac{g^{pn} - g^n}{g^n - 1}.
    \end{align*}
    Since $g^{pn} \equiv g^n \mod p$, the sum turns into zero modulo $p$.
  \end{proof}
  Now, we have all the ingredients to prove Theorem~\ref{thm:less_than_total_degree}.
  \begin{proof}[Proof of Theorem \ref{thm:less_than_total_degree}]
    Assume that all computations are done modulo $p$.
    Using Lemma~\ref{lem:difference_to_p-1}, we obtain that $P_{\LT_\S}(X,Y)$ is equal to
    \begin{align*}
      \sum_{a = 0}^{p-2} \left(1-\sum_{i=0}^{p-1} X^i a^{p-1-i}\right) \sum_{b=a+1}^{p-1} \left(1-\sum_{j=0}^{p-1} Y^j b^{p-1-j}\right)
    \end{align*}
    Let us expand this expression distributively.
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} 1 - \sum_{i=0}^{p-1} X^i a^{p-1-i} - \sum_{i=0}^{p-1} Y^i b^{p-1-i} \\
      + \sum_{i=0}^{p-1} \sum_{j=0}^{p-1} X^i Y^j a^{p-1-i} b^{p-1-j}.
    \end{align*}
    Let us compute individual polynomial coefficients.
    The constant term is equal to
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} 1 - a^{p-1} - b^{p-1} + a^{p-1} b^{p-1} \\
      = \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} 1 - a^{p-1} - 1 + a^{p-1} = 0\,.
    \end{align*}
    Coefficients by $X^i$ with $i > 0$ can be computed as follows
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} \left(-a^{p-1-i}\right) + a^{p-1-i}b^{p-1} = 0\,.
    \end{align*}
    Next, we compute coefficients by $Y^i$ with $i > 0$.
    \begin{align*}
      -\sum_{a = 0}^{p-2} &\sum_{b=a+1}^{p-1} b^{p-1-i} - a^{p-1} b^{p-1-i} \\
      &= -\sum_{b=1}^{p-1} b^{p-1-i} - \sum_{a = 1}^{p-2} \sum_{b=a+1}^{p-1} b^{p-1-i} - b^{p-1-i} \\
      &= -\sum_{b=1}^{p-1} b^{p-1-i}.
    \end{align*}
    If $i = p-1$, this sum is equal to $1$.
    According to Lemma~\ref{lem:sum_poly}, it is $0$ if $i < p-1$.

    To compute coefficients by $X^i Y^j$ with $i, j > 0$, we will use Faulhaber's formula below
    \begin{align*}
      \sum_{k=1}^n k^e = \frac{1}{e+1} \sum_{i=1}^{e+1} (-1)^{\delta_{ie}} \binom{e+1}{i} B_{e+1-i} \cdot n^i\,,
    \end{align*}
    where $\delta_{ie}$ is the Kronecker delta and $B_{i}$ is the $i$th Bernoulli number.
    This implies that there exist a polynomial $P(X) \in \F_p[X]$ of degree $e+1$ such that
    \begin{align}\label{eq:faulhaber}
      \sum_{k=1}^n k^e = P(n).
    \end{align}
    Note that $P(0) = 0$.
    The coefficient by $X^i Y^j$ for some positive $i$ and $j$ is equal to
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} a^{p-1-i} b^{p-1-j} &= \sum_{b = 1}^{p-1} b^{p-1-j} \sum_{a=0}^{b-1} a^{p-1-i}\,. \\
    \end{align*}
    According to~(\ref{eq:faulhaber}), there exist a polynomial $P_i(X)$ of degree $p-i$ such that $\sum_{a=0}^{b-1} a^{p-1-i} = P_i(b)$.
    Since $Q_{ij}(X) = X^{p-1-j} P_i(X)$ has degree $2p-1-i-j$, Lemma~\ref{lem:sum_poly} implies that if $i+j > p$, then
    \begin{align*}
      \sum_{b = 1}^{p-1} b^{p-1-j} &\sum_{a=0}^{b-1} a^{p-1-i} = \sum_{b = 1}^{p-1} b^{p-1-j} P_i(b) \\
      &= \sum_{b=1}^{p-1} Q_{ij}(b) = \sum_{b=0}^{p-1} Q_{ij}(b) - Q_{ij}(0) = 0.
    \end{align*}
    Thus, all the coefficient $X^i Y^j$ with $i + j > p$ are zero, which means that the total degree of $P_{\LT_\S}(X,Y)$ is at most $p$.

    In addition, we consider the case when $i = j$ and $i,j \le (p-1)/2$.
    Let us consider the following sum
    \begin{align*}
      \sum_{a=0}^{p-1} a^{p-1-i} \sum_{b=0}^{p-1} b^{p-1-i} = 0\,.
    \end{align*}
    We can rewrite it as follows
    \begin{align*}
      \sum_{a=0}^{p-1} & a^{p-1-i} \sum_{b=0}^{p-1} b^{p-1-i} \\
      &= 2\sum_{a=0}^{p-2} a^{p-1-i} \sum_{b=a+1}^{p-1} b^{p-1-i} + \sum_{a=0}^{p-1} a^{2(p-1-i)}\,.
    \end{align*}
    This implies that
    \begin{align*}
      \sum_{a=0}^{p-2} a^{p-1-i} \sum_{b=a+1}^{p-1} b^{p-1-i} = -\frac{1}{2}\sum_{a=0}^{p-1} a^{2(p-1-i)}\,.
    \end{align*}
    Note that the inverse of $2$ is well defined modulo $p$ since $p$ is an odd prime.
    If $i < (p-1)/2$, then Lemma~\ref{lem:sum_poly} says the sum on the right side is zero.
    Thus, the coefficient by $X^i Y^i$, which is exactly the sum on the left side, is equal to zero.
    If $i = (p-1)/2$, the above equality yields that the coefficient by $(XY)^{(p-1)/2}$ is equal to $-(p-1)/2$.
  \end{proof}

\section{Decomposition of $f(X,Y)$ for $3\leq p \leq 7$}
\label{app:decomposition-f}
Let $Z = Y(X-Y)$.
One non-scalar multiplication is needed to compute $Z$.

\textbf{p=3.}
$$f(X,Y) = 2\,.$$

Since the polynomial $f(X,Y)$ is constant, it can be computed without any homomorphic multiplication.

\textbf{p=5.}
$$f(X,Y) = 4X^2 + 4X + Z\,.$$

Two non-scalar multiplications are needed to compute $X^2$ and $Z$.

\textbf{p=7.}
$$f(X,Y) = 1 + 4X(X+1) + 6[X(X+1)]^2 + (X^2+3X)Z + 6Z^2$$

In this case, four non-scalar multiplications are needed (indicated in bold) when rewritten as follows
$$f(X,Y) = 1 + 2(\bm{X}^2+X)\bm{\cdot}[2 + 3(X^2+X)] + \bm{Z}\bm{\cdot}[(X^2+3X) + 6Z]\,.$$

\section{Proof of Theorem~\ref{th:univariate}}
\label{app:proof-lem-univariate}
   Let $Z = X-Y$.
    Thus we can rewrite $Q_{\LT_\S}(X,Y)$ as the univariate function $\chi_{\F_p^-}$, namely
    \begin{align*}
      Q_{\LT_\S}(X,Y) = \chi_{\F_p^-}(Z) = \sum_{a=-\frac{p-1}{2}}^{-1} 1 - (Z - a)^{p-1}.
    \end{align*}
    Thanks to Lemma~\ref{lem:difference_to_p-1}, we can expand $(Z-a)^{p-1}$ and obtain
    \begin{align*}
      \sum_{a=-\frac{p-1}{2}}^{-1} 1 - \sum_{i=0}^{p-1} Z^i a^{p-1-i}
      = \sum_{i=1}^{p-1} Z^i \sum_{a=-\frac{p-1}{2}}^{-1} (-a^{p-1-i}).
    \end{align*}
    If $i$ is even and $i < p-1$, then the $i$th coefficient is equal to
    \begin{align*}
      -\sum_{a=-\frac{p-1}{2}}^{-1} a^{p-1-i} &= -\sum_{a=1}^{\frac{p-1}{2}} a^{p-1-i} = -\frac{1}{2}\sum_{a=-\frac{p-1}{2}}^{\frac{p-1}{2}} a^{p-1-i}.
    \end{align*}
    This coefficient is equal to $0$ for any even $0<i < p-1$ thanks to Lemma~\ref{lem:sum_poly}.
    The $(p-1)$-th coefficient is equal to $-(p-1)/2 = (p+1)/2 \bmod p$.
    If $i$ is odd, then we can rewrite the $i$th coefficient in the following way
    \begin{align*}
      -\sum_{a=-\frac{p-1}{2}}^{-1} a^{p-1-i} &= \sum_{a=1}^{\frac{p-1}{2}} a^{p-1-i},
    \end{align*}
    which finishes the proof.

%%%Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\section{Solution approach using a CNN}
\label{sec:ansatz}
The most successful approach to image recognition based an machine-learing algorithms is the use of \textit{Convolutional Neural Networks} (CNN).
CNNs can use multidimensional input arrays which is their main advantage for working with image data compared to \enquote{normal} fully-connected neural networks,
because the information about the position of each pixel compared to its neighbors is not lost by flattening the pixel-values in an one dimensional array.
This is done by repeatedly applying weight-matrices of size $N_\text{kernel}\times M_\text{kernel}$ called \textit{kernels} on the image, while moving the kernel from one edge of the image to the other,
whereby the steps size (referring to the number of pixel-rows/ -columns that the kernel is moved by in each step) is called \textit{stride}.
This allows CNNs to learn geometric shapes within the images it is trained on.
As we want to classify the images into the eleven flower species, we also add a small fully-connected neural network after the CNN structure, so that the final output of our
neural network are eleven scores for each image which describe how likely the image shows a flower(s) of each of the eleven species.
A subsequent final assignment to one of the species is done by choosing the species with the highest score.

Our problem represents a classification problem with $k = 11$ target classes.
For this type of problem the \textit{categorical cross-entropy} is a good loss function
\begin{align}
    CE = -\frac{1}{n} \sum_{i=1}^n \sum_{k=1}^{11} p_{i,k} \ln{y_{i,k}},
\end{align}
where $n$ is the number of images used for training, $p_{i,k}$ the target vector with $p_{i,k} = 0$ for all classes but the true class $k'$ of the image $i$ ($p_{i,k'} = 1$) and
$y_{i,k}$ the prediction vector with the predicted probabilities of the image being of class $k$.
As our goal is to correctly classify as many images as possible, the \textit{accuracy} of the classification is the most fitting performance metric
\begin{align}
    \text{accuracy} = \frac{n_\text{correctly classified}}{n}.
\end{align}

\subsection{First network and manual hyperparameter tuning}
For constructing, training and applying a neural network we use the \texttt{keras} API for Googles \texttt{TensorFlow} deep-learning library \cite{keras},
while some helper functions from the \texttt{scikit-learn} library are used \cite{scikit-learn}.
Due to our preprocessing, the shape of the input data is $200\times 200\times 3$, where the last dimension represents the three RGB color channels.

We start by trying different structures for the network to get a basic of idea of which type of network works for our problem.
This leads to a network with three convolutional layers, each followed by a MaxPooling layer to stabilize the performance of the network against slight displacement or rotation of certain shapes
in different images.
MaxPooling layers take the maximal value of a $N_\text{pool}\times N_\text{pool}$ subspace of each of the two-dimensional arrays (/image-channels) of the three-dimensional input and
return it as the single output for each of those subspaces.
After the convolutional part of the network we add two hidden fully-connected layers before the output layer (a second hidden layer improves the accuracy compared with only one hidden layer).

As \textbf{activation functions} we use the \textit{ReLU} function for all layers but the output layer, for which we use the \textit{softmax} function.
This choice of activation functions makes it possible to interpret the output scores of the network as actual probabilities of the image showing a flower of the given species.

Regarding the \textbf{number of kernels} in each convolutional layer, an increasing number of kernels seems to work best and we choose to double the number of kernels in each layer
starting with 16 kernels in the first layer.
For the \textbf{MaxPooling} layers we choose $N_\text{pool} = 2$, as any larger number decreases the image size too quickly.
The combination of an increasing number and kernels with pooling layers after each convolutional layer effectively leads to a compression of the first two dimensions (height and width of the image)
of the data while more information gets transformed into the third dimension (number of kernels (RGB channels) in the (input) layer).

For an image size of $200\times 200$ the kernel must also be square ($N_\text{kernel} = M_\text{kernel}$) and a \textbf{kernel size} of $N_\text{kernel} = 4$ seems to work well.
We tried using non-unit \textbf{stride}, but the best result is achieved by using the (default) unit stride which means moving the kernel one pixel-row/ -column at a time across the image.

An overview of the network structure we find based on these manual hyperparameter variations can be found in \autoref{fig:first_model}.

For \textbf{training} the network we use the \textit{Adam} algorithm \cite{adam} with the (default) initial learing rate of 0.001 and a batch size of 128.
We test initial learing rates of 0.01 and 0.0001 and batch sizes of 256 and 64, but these settings do perform significantly worse or slow the training down a lot.

Using the hyperparameter values described so far we achieve an accuracy of 0.753 in training epoch 17 and the loss and accuracy curves can be seen in \autoref{fig:first_curves}.
To validate the performance of the model we split another $\SI{15}{\percent}$ of the images into a validation set.
The whole data is hereby split into $\SI{70}{\percent}$ training set, $\SI{15}{\percent}$ validation set and $\SI{15}{\percent}$ test set.

\subsection{Hyperparameter tuning via sequential grid searches}
After finding a network structure which performs reasonably well, we perform four sequential grid searches.
The first search covers the number of nodes in the two hidden fully-connected layers.
\begin{align}
    N_\text{dense nodes} \in \{64, 96, 128, 160, 192, 224, 256\}
\end{align}
The best accuracy on the validation set (0.770 in training epoch 22) is achieved by using 224 nodes in both layers and the loss and accuracy curves of the three best performing models
(in regards to accuracy on the validation set) can be seen in \autoref{fig:GS1}

In the second search we use more convolutional layers and vary the number of kernels in each layer (still doubling the number from layer to layer).
\begin{align}
    N_\text{conv. layers} &\in \{3, 4, 5, 6\}, &
    \text{num}_\text{kernels} &\in \{8, 16, 24, 32, 40, 48, 64, 80, 96\}
\end{align}
For $N_\text{conv. layers} = 3, 4, 5$ we still use a MaxPooling layer after each layer, but for $N_\text{conv. layers} = 6$ we only include a MaxPooling layer every two layers and
also double the number of kernels only every two layers.
This search is done in three parts, because the first two tries result in the model with the largest $\text{num}_\text{kernels}$ performing the best.
Therefore, it is necessary to increase the range of $\text{num}_\text{kernels}$ to test.
$N_\text{conv. layers} = 5$ is also not present in the first part of the search, but in the second part it performs better than $N_\text{conv. layers} = 4$ (which performed best in the first part)
for all tested number of kernels, which is why only $N_\text{conv. layers} = 5$ is tested in the third part.
The performance of the four best models in the third part of the search can be seen in \autoref{fig:GS2_ext2}.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/GS2_ext2.pdf}
    \caption{The loss and accuracy curves for the three best performing models in the third part of the second grid search ($N_\text{conv. layers} = 5$).}
    \label{fig:GS2_ext2}
\end{figure}
Even though the model with $\text{num}_\text{kernels} = 96$ performs the best we choose $\text{num}_\text{kernels} = 48$ as our optimal number of kernels (in the first layer),
because it performs only slightly worse while representing a much less complex model.
On one hand less complex models are generally to be preferred over more complex ones, if they perform equally well (\textit{Occam's razor}) and on the other hand training a model
with as big a capacity as the one with $\text{num}_\text{kernels} = 96$ is so slow, that it would make further hyperparameter tuning extremely difficult, because of our limited computing resources.
In the end, the model with $N_\text{conv. layers} = 5$ and $\text{num}_\text{kernels} = 48$ achieves a validation accuracy of 0.829 in epoch 32.

The third gird search covers the kernel sizes
\begin{align}
    N_\text{kernel} \in \{2, 3, 4, 5, 6\}.
\end{align}
$N_\text{kernel} = 6$ is the biggest kernel size that can be applied on the images in the fifth convolutional layer, because the image resolution is reduced to $6\times 6$ in this layer
due to the five previous MaxPooling layers.
Considering that we use the same kernel size in all convolutional layers, this search covers the whole parameter space for this hyperparameter given the restrictions of our network structure,
as chosen so far.
The performance of the three best models can be seen in \autoref{fig:GS3} and we conclude, that our previous choice of $N_\text{kernel} = 4$ still performs the best
with a validation accuracy of 0.820 in epoch 32.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/GS3.pdf}
    \caption{The loss and accuracy curves for the three best performing models in the third gird search.}
    \label{fig:GS3}
\end{figure}
Even though the best model in this search ends up using the exact same hyperparameter values as the optimal model in the previous search, its accuracy is $\SI{0.9}{\percent}$ worse.
This shows, that it is not possible (or at least within the scope of this project) to make the training results completely reproducible, even though we use fixed seeds for functions using
random numbers wherever possible.
It would make sense to use \textit{cross-validation} to mitigate these random fluctuations, however we do not have the computing resources to do so effectively.

By now the structure of the convolutional part of our network has changed so much, that we deem it necessary to reexamine the structure of the fully-connected part of the network,
as it effectively receives quite different inputs compared to when we performed the first gird search.
For this reason, we once again try out different values for $N_\text{dense nodes}$ and check if two hidden layers still provide a better accuracy than one.
\begin{align}
    N_\text{dense layers} &\in \{1, 2\}, &
    N_\text{dense nodes} &\in \{64, 128, 192, 256\}
\end{align}
We omitted a single layer with $N_\text{dense nodes} = 64$ to speed up the search, because this combination performed much worse than others during the manual hyperparameter search.
The three models achieving the best validation accuracies can be seen in \autoref{fig:GS4}.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/GS4.pdf}
    \caption{The loss and accuracy curves for the three best performing models in the fourth gird search.}
    \label{fig:GS4}
\end{figure}
The model with a single hidden layer containing $N_\text{dense nodes} = 128$ nodes performs the best with a validation accuracy of 0.824 in epoch 44.

\subsection{Regularization}
Although the accuracy curves do not show signs of significant overfitting (diminishing validation accuracy in later training epochs), the loss curves show, that the validation loss
increases again in later epochs.
Therefore, we now try introducing some regularization to our network in an afford to further improve the networks expected performance on new data by creating redundancy within the
network and forcing it to generalize more.

After implementing one \textit{Dropout} layer between the fifth convolutional and the hidden fully-connected layer and trying different dropout rates $r_\text{dropout}$, we do not see an
improved validation accuracy compared to no dropout.
No dropout layer is implemented between the hidden layer and the output layer, as there are too few weights between those two layers.
Dropout is ineffective in small layers, because too much information gets lost, when some nodes are deactivated.
The next step is using \textit{L2-regularization} in every layer instead and testing different L2-regularization rates
\begin{align}
    r_\text{L2} \in \{0, \num{1e-6}, \num{1e-5}, \num{1e-4}, \num{1e-3}\}.
\end{align}
This shows the best result when using $r_\text{L2} = \num{1e-5}$ as can be seen in \autoref{fig:l2}.

Finally we try combining dropout and L2 regularization by fixing $r_\text{L2} = \num{1e-5}$ and testing the dropout rates
\begin{align}
    r_\text{dropout} \in \{0, 0.1, 0.2, 0.3, 0.4\}.
\end{align}
The loss and accuracy curves for the three best models can be seen in \autoref{fig:l2_dropout}.
The model with $r_\text{dropout} = 0.3$ achieves the best validation accuracy of 0.844 in epoch 57.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/dropout_l2.pdf}
    \caption{The loss and accuracy curves for the three best performing models during the search for a combination of L2 and dropout regularization.}
    \label{fig:l2_dropout}
\end{figure}