\section{Sets and Functions}
\subsection{Naive Set Theory}
\begin{definition}[Naive definition of sets]
    A set is any
    \footnote{Not quite, see later.}
    collection of (mathematical) objects.
\end{definition}
\begin{example}
    $\mathbb N,\mathbb Q,\mathbb R,\mathbb C,\mathbb Z,[0,1],\{1,2,3\},\ldots$
\end{example}
A set is determined by its members: $(a\in A\iff a\in B)\iff(A=B)$.
Hence, the set is not ordered per se: $\{1,3,7\}=\{7,1,3\},\{1,3,3,7\}=\{1,3,7\}$.
From the second example, we also know that there is no multiple membership.\\
Of course, we would want to make new sets from old.
\begin{definition}
    A set $A$ is a subset of another set $B$ iff $x\in A\implies x\in B$.
    We write it as $A\subset B$ or $A\subseteq B$.
\end{definition}
So $A=B$ is equivalent to $A\subset B$ and $B\subset A$.
\begin{example}
    $\{1,7\}\subset \{1,3,7\}\subset\mathbb N\subset\mathbb Z\subset \mathbb Q\subset\ldots, [0,1]\subset\mathbb R$.
\end{example}
\begin{definition}
    Given a set $A$ and a property $P$ on $A$, we can form the set of all elements in $A$ having property $P$, $\{x\in A:P(x)\}$
\end{definition}
\begin{example}
    We can construct the primes by $\{n\in\mathbb N:\text{$n$ is prime}\}$.
\end{example}
\begin{definition}
    Given sets $A,B,U$ such that $A,B\subset U$, we can construct their union by
    $$A\cup B=\{x\in U:x\in A\lor x\in B\}$$
    and intersection by
    $$A\cap B=\{x\in U:x\in A\land x\in B\}$$
    We say $A,B$ are disjoint if $A\cap B=\varnothing$.\\
    We also have the minus action
    $$A\setminus B=\{x\in A:x\notin B\}$$
\end{definition}
We can view $A\cap B$ as a subset selection, since $A\cap B=\{x\in A:x\in B\}$.
Unions and intersections are commutative and associatives.
Also, the union is distributive over the intersection: $A\cup (B\cap C)=(A\cup B)\cap (A\cup C)$.
The intersection is distributive over the union as well: $A\cap (B\cup C)=(A\cap B)\cup (A\cap C)$.
One can easily check them.\\
We can also have arbitrary unions and intersections.
\begin{definition}
    For $A_i\subset U$ where $i\in I$ for some index set $I$, the union of all $A_i$'s can be defined by
    $$\bigcup_{i\in I}A_i=\{x\in U:\exists i\in I,x\in A_i\}$$
    the intersection by
    $$\bigcap_{i\in I}A_i=\{x\in U:\forall i\in I,x\in A_i\}$$
\end{definition}
It coincides with our previous definition in the finite case due to the associativity and commutativity of unions and intersections.
\begin{definition}
    In $\mathbb R$, let $A_n=[1-1/n,1+1/n],n\in\mathbb N$, so $\bigcup_{n\in\mathbb N}A_n=[0,2],\bigcap_{n\in N}A_n=\{1\}$, note that there is no limiting operation going on here.\\
    Let let $A_n=(1-1/n,1+1/n),n\in\mathbb N$, so $\bigcup_{n\in\mathbb N}A_n=(0,2),\bigcap_{n\in N}A_n=\{1\}$
\end{definition}

\begin{definition}
    Given two objects $a,b$, you can form the ordered pair $(a,b)$,
    And that $(a,b)=(c,d)$ iff $a=c,b=d$.
    For sets $A,B$, one can form the collection $A\times B=\{(a,b):a\in A,b\in B\}$, which is called the product, or Cartesian product of $A,B$.
\end{definition}
\begin{example}
    The plane $\mathbb R^2$ can be viewed as $\mathbb R\times\mathbb R$.
\end{example}
Similarly, we can construct things like $\mathbb R^n$ by recognising the collection of all ordered $n$-tuples.\\
Note that if we wish to do it, we could define $(a,b)$ by the set $\{\{a\},\{a,b\}\}$.
One can check that $(a,b)=(c,d)\iff a=c,b=d$ under this notion.
\footnote{Quoting the lecturer, ``You're completely nuts if you really think of this in that way'}
\begin{definition}
    For any set $A$, we can form the power set of $A$, written as $\mathbb P(A)$ or $2^A$,
    \footnote{Different from the lecturer, the author of this set of notes prefers the latter notation.}
    which is set of all subsets of $A$.
\end{definition}
\begin{example}
    Let $X=\{1,2\}$, then $2^X=\{\varnothing, \{1\},\{2\},\{1,2\}\}$.
\end{example}
\begin{remark}
    Warning: Given a set $A$, we know we can form $\{x\in A:P(x)\}$, but you should not form $\{x:P(x)\}$.
    Suppose we could form $X=\{x:x\notin x\}$, then do we have $X\in X$?
    Indeed, this gives a contradiction.\\
    This is called the Russel's Paradox.
\end{remark}
Similarly, there is not an universal set $U$ such that $\forall x,x\in V$, because it would induce the Russel's Paradox.
We can only guarantee that a given set exists if it is obtained, in some way, from known sets.
\subsection{Finite Sets and Their Sizes}
\begin{definition}
    A set $A$ has size $n$ where $n\in\mathbb N_0$ if we can write $A=\{a_1,a_2,\ldots,a_n\}$ such that $a_i$'s are distinct.
\end{definition}
\begin{example}
    1. $\{1,3,7\}$ has size $3$.\\
    2. $\varnothing$ has size $0$.
\end{example}
\begin{proposition}
    If a set has size $n$ and size $m$, then $m=n$.
\end{proposition}
\begin{proof}
    It's trivial but whatever.\\
    Indeed, if $A$ has size $n$ and size $m$ with $n>m$, then if $m=0$ it is trivial, otherwise, removing an element from $A$ gives a set of sizes $n-1,m-1$, so it is done by induction on $m$.
\end{proof}
\begin{proposition}\label{2power}
    If $A$ has size $n$, then $2^A$ has size $2^n$.
\end{proposition}
It is obvious when $n=0$, so we assume henceforth it is not.
\begin{proof}
    We can relabel the element in $A$ by $\{1,2,\ldots,n\}$.
    So to specify a subset $S$, we must specify if $1\in S$, $2\in S$, and so on, so the size of $2^A$ is $2\times 2\times 2\times\cdots\times 2$ $n$ times, which is $2^n$.
\end{proof}
\begin{proof}[Alternative proof]
    Induction on $n$.
\end{proof}
The alternative proof can be viewed as a more formal version of the first proof.\\
We are tired of saying ``the size of blah blah blah'' so we write $|A|$ to denote the size of $A$.\\
A set of size $n$ is sometimes called an $n$-set.
\begin{definition}[Binomial Coefficients]
    Let $A=\{1,2,\ldots,n\}$ for $n\ge 1$, then the binomial coefficient is defined as
    $$\binom{n}{k}=\{S\subset A:|S|=k\}$$
    is the number of ways to choose a $k$-set from an $n$-set.
\end{definition}
\begin{example}
    $$\binom{4}{2}=6$$
    by listing.
\end{example}
We always have $\binom{n}{n}=\binom{n}{0}=1,\binom{n}{1}=n$ and $\binom{n}{k}=\binom{n}{n-k}$.
Also, by Proposition \ref{2power}, we instantly have
$$\binom{n}{0}+\binom{n}{1}+\cdots+\binom{n}{n}=2^n$$
In addition,
$$\binom{n}{k}=\binom{n-1}{k-1}+\binom{n-1}{k}$$
since both size counted the number of $k$-sets in an $n$-set, that is, the number of $k$-sets which include some specified element and the number of $k$ sets which do not include.
Hence we have the Pascal's triangle.
\begin{proposition}
    $$\binom{n}{k}=\frac{n(n-1)(n-2)\cdots (n-k+1)}{k!}$$
\end{proposition}
\begin{proof}
    We first count the number of ordered $k$-sets is $n(n-1)(n-2)\cdots (n-k+1)$.
    But here we have overcounted each $k$-sets by $k(k-1)(k-2)\cdots 1=k!$, therefore the formula.
\end{proof}
\begin{corollary}
    When $n$ is big, $\binom{n}{k}\sim n^k/k!$.
\end{corollary}
An appplication of the binomial coefficient is the binomial theorem.
\begin{theorem}[Binomial Theorem]
    $$(a+b)^n=\sum_{k=0}^n\binom{n}{k}a^{n-k}b^k$$
\end{theorem}
\begin{proof}
    When we expand $(a+b)^n=(a+b)(a+b)\cdots (a+b)$ where there are altogether $n$ brackets, we obtain terms of the form $a^{n-k}b^{k}$ where $0\le k\le n$.
    But the number of terms that we get in each is the number of ways to choose $k$ (or equivalently, $n-k$) brackets from the $n$ brackets, so it is $\binom{n}{k}$, thus the theorem.
\end{proof}
In particular, $(1+x)^n=1+nx+\binom{n}{2}x^2+\cdots +x^n$, hence for given $n$, $(1+x)^n\sim 1+nx$ when $x$ is small, and we can get better approximations when we take more term(s).\\
Now, how do sizes of unions and intersections (for finite sets) relate to each other?
\begin{example}
    We have $|A\cup B|=|A|+|B|-|A\cap B|$, and $|A\cup B\cup C|=|A|+|B|+|C|-|A\cap B|-|B\cap C|-|C\cap A|+|A\cap B\cap C|$.
\end{example}
\begin{theorem}[Inclusion/Exclusion Principle]
    For finite sets $(S_i)_{i=1}^n$, $|S_1\cup S_2\cup\cdots\cup S_n|$ equals
    $$\sum_i|S_i|-\sum_{i<j}|S_{ij}|+\sum_{i<j<k}|S_{ijk}|-\cdots+(-1)^{n+1}\sum_{i_1<i_2<\ldots <i_n}S_{i_1i_2\ldots i_n}$$
    Where $S_{k_1k_2\ldots k_r}=S_{k_1}\cap S_{k_2}\cap \cdots\cap S_{k_r}$
\end{theorem}
\begin{proof}
    Consider $x\in |S_1\cup S_2\cup\cdots\cup S_n|$, suppose we can choose the maximal $k$ such that there is a $k$-set $A$ contained in $\{1,2,\ldots, n\}$ such that $x\in S_a$ for each $a\in A$.
    Then, the number of times that $x$ is counted is
    $$k-\binom{k}{2}+\binom{k}{3}-\cdots+(-1)^{k+1}\binom{k}{k}=1-(1-1)^k=1$$
    by Binomial Theorem.
    So each element is counted exactly once, hence the theorem is proved.
\end{proof}
\subsection{Functions}
\begin{definition}[Intuitive Definition of Functions]
    For sets $A,B$, a function $f:A\to B$ is a ``rule'' that assigns each element $a$ in $A$ exactly one element, called $f(a)$ in $B$.
\end{definition}
\begin{definition}[Rigourous Definition of Functions]
    A function $f:A\to B$ is a set $f\subset A\times B$ such that if $(a,b),(a,c)\in f$ for $a\in A$ and $b,c\in B$, then $b=c$ (equivalently $(a,b)=(a,c)$).
    In other words, $\forall a\in A,\exists!b\in B,(a,b)\in f$.
    We write $f(a)=b$.
\end{definition}
$A$ is called the domain of $f$ and $B$ is called the range (or codomain) of $f$.
The set $\{f(a):a\in A\}$ is called the image of $f$.
\begin{example}
    1. We can have a function $f:\mathbb R\to\mathbb R$ given by $f(x)=x^2$.\\
    2. (non-example) But $f:\mathbb R\to\mathbb R$ by $f(x)=1/x$ is not a function since it does not have value at $0$.\\
    3. (non-example) The function $f:\mathbb R\to\mathbb R$ by $f(x)=\pm\sqrt{x^2}$ is not a function due to multiple value.\\
    4. $A=\{1,2,3,4,5\},B=\{1,2,3,4\}$ and $f:A\to B$ by $1\mapsto 1,2\mapsto 3,3\mapsto 4,4\mapsto 3,5\mapsto 5$ is a function.\\
    5. $A=B=\{1,2,3\}$ and $f:A\to B$ by $1\mapsto 2,2\mapsto 1,3\mapsto 3$ is a function.\\
    6. $A=B=\{1,2,3,4\}$ and $f:A\to B$ by $1\mapsto 1,2\mapsto 2,3\mapsto 4,4\mapsto 4$.\\
    7. $A=\{1,2,3,4,5\}, B=\{1,2,3,4\}$ and $f:A\to B$ by $1\mapsto 2,2\mapsto 1,3\mapsto 3,4\mapsto 4,5\mapsto 4$.
\end{example}
Note that it can be ambiguous if we just say $f(x)=x^2$ since we may not know about the domain (and range).
We can only define a function with knowing about the domain and the range.
Note also that we do not have to have a ``closed form'' of a function, which is frankly ridiculous to do so.
\begin{definition}
    A function $f:A\to B$ is called injective if $\forall x,y\in A,x\neq y\implies f(x)\neq f(y)$, or equivalently, $\forall x,y\in A,f(x)=f(y)\implies x=y$.\\
    It is called surjective if $\forall b\in B,\exists a\in A,f(a)=b$, or equivalently, $B$ is the image of $f$.
\end{definition}
One can easily classify the above examples by injective/non-injective and surjective/non-surjective.
\begin{definition}
    If a function $f$ is both injective and surjective, we say it is bijective, or that it is a bijection.
\end{definition}
Note that a bijection pair up elements in $A$ and $B$.
It is also called a 1-1 correspondence.
Example 5 above is an example of a bijection.
Note that injectivity and surjectivity depend strongly on the domain and range of the function.
\begin{remark}
    If $A,B$ are finite and $f:A\to B$.
    If $|A|>|B|$, $f$ cannot be injective.
    If $|A|<|B|$, $f$ cannot be surjective.\\
    If $|A|=|B|$, then surjectivity, injectivity and bijectivity are equivalent.
    Thus any $f:A\to A$ cannot biject $A$ to a proper subset $A'\subsetneq A$.
    But this is not necessarily true for infinite (i.e. not finite) sets, for example, the ``adding-$1$'' function is a bijection $\mathbb N\to\mathbb N\setminus\{1\}$ (assuming $\mathbb N$ starts with $1$), this also shows that injectivity does not imply surjective.
    Also the function $\mathbb N\to\mathbb N$ by $1\mapsto 1$ and $n\mapsto n-1$ for $n\ge 2$ is surjective but not injective.
\end{remark}
We have more examples of functions.
\begin{example}
    1. For any set $A$, we have the identity function $A\to A$ which sends each element to itself
    It is a bijection.\\
    2. Given a set $X$ and a subset $A\subset X$, the function $\chi_A:X\to\{0,1\}$ by $\chi_A(x)=1$ if $x\in S$ and $\chi_A(x)=0$ otherwise is a function.
    This is called the indicator (or characteristic) function of $A$.\\
    3. A sequence on a set $X$ is a function $\mathbb N\to X$.\\
    4. The addition and multiplication, say on $\mathbb N$, are functions $\mathbb N\times\mathbb N\to\mathbb N$.\\
    5. A finite set $A$ has set $m$ if and only if there is a bijection $\{1,2,3,\ldots,m\}\to A$.
\end{example}
\begin{definition}
    Let $f:A\to B,g:B\to C$, then the composition $g\circ f$ is a function $A\to C$ defined by $(g\circ f)(a)=g(f(a))$.
\end{definition}
Note that function compositions are not necessarily commutative (if $A=C$).
\begin{example}
    If $A=B=C=\mathbb R$ and $f: x\mapsto 2x,g: x\mapsto x+1$, then $f\circ g: x\mapsto 2x+2,g\circ f:x\mapsto 2x+1$, which are indeed different.
\end{example}
However, function compositions are associative.
\begin{proposition}
    If $f:A\to B,g:B\to C,h:C\to D$, then $h\circ (g\circ f)=(h\circ g)\circ f$.
\end{proposition}
\begin{proof}
    $\forall x\in A, (h\circ (g\circ f))(x)=h(g(f(x)))=((h\circ g)\circ f)(x)$.
\end{proof}
\begin{definition}
    $f:A\to B$ is invertible if and only if there is some other function $g:B\to A$ such that $f\circ g=\operatorname{id}_B,g\circ f=\operatorname{id}_{A}$.
    We say $g$ is the inverse of $f$.
    Note that $g$ is invertible as well with inverse $f$.
\end{definition}
\begin{example}
    Consider $f:\mathbb R\to\mathbb R$ by $f(x)=2x+1$, consider $g:\mathbb R\to\mathbb R$ by $g(x)=(x-1)/2$, then $\forall x\in\mathbb R,(f\circ g)(x)=(g\circ f)(x)=x$, thus $f$ is invertible and $g$ is the inverse of $f$.
\end{example}
\begin{remark}
    Note that we have to check both sides.
    Consider $f_0:\mathbb N\to\mathbb N$ by $f_0(x)=x+1$ and $f_1:\mathbb N\to\mathbb N$ by $f_1(x)=x-1$ for $x>1$ and $f_1(1)=1$, then $f_1\circ f_0=\operatorname{id}_{\mathbb N}$ but $f_0\circ f_1\neq\operatorname{id}_{\mathbb N}$.
\end{remark}
\begin{proposition}
    A function is invertible if and only if it is a bijection.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
\subsection{Equivalence Relations}
\begin{definition}
    Let $A$ be a set, then a relation $R$ is a subset of $A\times A$.
    We say $x,y\in A$ are related, or $xRy$, if $(x,y)\in R$.
\end{definition}
\begin{example}
    1. On $\mathbb N$, the relation $aRb\iff a\equiv b\pmod{7}$ is a relation.\\
    2. On $\mathbb N$, $aRb\iff a|b$ is a relation.\\
    3. On any set, $aRb\iff a\neq b$ is a relation.\\
    4. On $\mathbb N$, $aRb\iff a=b\pm 1$.\\
    5. On $\mathbb N$, $aRb\iff |a-b|\le 2$.\\
    6. On $\mathbb N$, $aRb\iff (a,b<5\lor a,b\ge 5)$.
\end{example}
\begin{definition}
    A relation $R$ is reflexive iff $\forall x\in A, xRx$.
\end{definition}
The first, second, fifth and sixth examples above are reflexive relations.
\begin{definition}
    A relation $R$ is symmetric iff $\forall x,y\in A, xRy\implies yRx$
\end{definition}
The first, third, fourth, fifth and sixth examples above are symmetric relations.
\begin{definition}
    A relation $R$ is transitive iff $\forall x,y,z\in A, (xRy\land yRz)\implies xRz$
\end{definition}
The first, second and sixth exampels above are transitive relations.
\begin{definition}
    We say $R$ is an equivalence relation if it is reflexive, symmetric and transitive.
\end{definition}
The first and sixth relations above are equivalence relations.
\begin{example}\label{partition}
    Let $X$ be a set, then consider a partition $\{C_i\}_{i\in I}$ of $X$.
    That is, $C_i\neq \varnothing$, $i\neq j\implies C_i\cap C_j=\varnothing$ and $X=\bigcup_{i\in I}C_i$.
    Then $xRy\iff \exists i\in I, x,y\in C_i$ is a equivalence relation.
\end{example}
\begin{proposition}\label{eq_class}
    All equivalence relations on $X$ can be written in the form of Example \ref{partition}.
\end{proposition}
\begin{definition}
    For an equivalence relation $R$ on a set $X$ and $x\in X$, the equivalence class containing $x$, written as $C_x$ or $[x]$, is $\{y\in X:xRy\}=\{y\in X:yRx\}$.
\end{definition}
\begin{example}
    In the first example, $[2]=6+7\mathbb Z=[16]=[23]=\ldots$.
    Note that we have $7$ equivalence classes in total, namely $i+7\mathbb Z,i\in\{0,1,2,3,4,5,6\}$.
\end{example}
\begin{proof}[Proof of Proposition \ref{eq_class}]
    let $X$ be the set with the equivalence relation $R$.
    We shall show that the equivalence classes partitions $X$.
    Note that $x\in [x]$, so their union is $X$, so it remains to show that different equivalence classes are disjoint.
    If $y\in [x]$ and $y\in [z]$, then for any $w\in [x]$, we know $wRx,xRy$, so $wRy$, but $yRz$, so $wRz$, therefore $w\in [z]$, hence $[x]\subset [z]$.
    Similarly $[z]\subset [x]\implies [x]=[z]$.
\end{proof}
Hence equivalence relation is just a partition of the set.
\begin{definition}
    The collection $X/R=\{[x]:x\in X\}$ is called the quotient of $X$ with respect to an equivalence relation $R$.
\end{definition}
\begin{example}
    1. $xRy\iff x\equiv y\pmod{7}$ is an equivalence relation on $\mathbb Z$.
    It gives the quotient $\mathbb Z/R=\{i+7\mathbb Z:i\in \{0,1,2,3,4,5,6\}\}$.\\
    2. $(a,b)R(c,d)\iff ad=bc$ is an equivalence relation on $\mathbb Z\times\mathbb N$.
    So we could construct $\mathbb Q$ by $\mathbb Q=(\mathbb Z\times\mathbb N)/R$.
    The equivalence classes are like, for example, $[(1,2)]=\{(1,2),(2,4),(4,8),\ldots\}$.
    In that way we can recognize $a/b=[(a,b)]$.
\end{example}
\begin{definition}
    The quotient map (or projection map) $q:X\to X/R$ is defined by $q(x)=[x]$.
\end{definition}
This is well-defined since equivalence classes partitions the set $X$.


\lab{Applications}{Wave Phenomena}{Wave Phenomena}
\label{lab:waveeqn}

\section*{Advection Equation}
The advection equation (or transport equation) is given by $u_t + s u_x = 0$, where $s$ is a nonzero constant.
Consider the Cauchy problem
\begin{align*}
	& u_t + su_x = 0, \quad -\infty < x < \infty,\\\
	& u(x,0) = f(x).
\end{align*}
The function $f(x)$ may be thought of as an initial wave or signal.
The general solution of this initial boundary value problem is $u(x,t) = f(x-st)$ (check this!).
The solution $u(x,t)$ is a travelling wave that takes the signal $f(x)$ and moves it along at a constant speed $s$ - to the right if $s > 0$, and to the left if $s < 0$.

\section*{Wave Equation}
Many different wave phenomena can be described using a hyperbolic PDE called the wave equation.
These wave phenomena occur in fields such as electromagnetics, fluid dynamics, and acoustics.
This equation is given by
\begin{align}
	u_{tt} &= s^2 \triangle u.
\end{align}
The 1D equation can be derived in the context of many physical models; a common derivation describes the motion of a string vibrating in a plane.
Another nice derivation uses Hooke's law from the theory of elasticity.

After making the change of variables $(\xi,\eta) = (x-st, x + st)$ and using the chain rule, we find that the 1D wave equation $u_{tt} = s^2 u_{xx}$ is equivalent to $u_{\xi \eta} = 0$.
The general solution of this last equation is
\[u(\xi, \eta) = F(\xi) + G(\eta)\]
for some scalar functions $F$ and $G$.
In $(x,t)$ coordinates the solution is
\[u(x,t) = F(x-st) + G(x+st)\]
Thus the general solution of the wave equation is the sum of two parts: one is a signal travelling to the right with constant speed $|s|$, and the other is a signal travelling to the left with speed $|s|$.

The wave equation is usually seen in the context of an initial boundary value problem.
This takes the form
\begin{align*}
	u_{tt} &= s^2 u_{xx}, \quad 0 < x < l, \quad t > 0,\\
	u(0,t) &= u(l,t) = 0, \\
	u(x,0) &= f(x),\\
	u_t(x,0) &= g(x).
\end{align*}

\subsection*{Numerical solution of the wave equation}
We look to approximate $u(x,t)$ on a grid of points $(x_j,t_m)_{j=0,m=0}^{J,M}$.
Denote the approximation to $u(x_j,t_m)$ by $U_{j}^{m}$.
Recall that the centered approximations in space and time are
\begin{align*}
D_{tt} U_{j}^{m} = \frac{U_{j}^{m+1} -2 U_{j}^{m} + U_{j}^{m-1}}{(\triangle t)^2} ,\\
D_{xx} U_{j}^{m} = \frac{U_{j+1}^{m} -2 U_{j}^{m} + U_{j-1}^{m}}{(\triangle x)^2} .
\end{align*}
% \[u_{tt}(x_j,t_m) = \frac{u(x_j,t_m+k) -2 u(x_j,t_m) + u(x_j,t_m-k)}{k^2} + \mathcal{O}(k^2)\]
%
% \[u_{xx}(x_j,t_m) = \frac{u(x_j+h,t_m) -2 u(x_j,t_m) + u(x_j-h,t_m)}{k^2} + \mathcal{O}(h^2)\]
The resulting method is given by
\begin{align*}
	&\frac{U_{j}^{m+1} -2 U_{j}^{m} + U_{j}^{m-1}}{(\triangle t)^2} = s^2 \frac{U_{j+1}^{m} -2 U_{j}^{m} + U_{j-1}^{m}}{(\triangle x)^2}, \\
	% &U_{j,m+1} = 2 U_{j,m} - U_{j,m-1} + \lambda ^2 (U_{j+1,m} -2 U_{j,m} + U_{j-1,m}), \\
	&U_{j}^{m+1} =  - U_{j}^{m-1} + 2 (1-\lambda^2) U_{j}^{m} + \lambda ^2 (U_{j+1}^{m} + U_{j-1}^{m}),
\end{align*}
where $ \lambda  =  s(\triangle t)/(\triangle x)$.
This method may be written in matrix form as
\[U^{m+1} = AU^{m} - U^{m-1} \]
where
\[A =
\left[\begin{array}{cccc}2(1-\lambda^2) & \lambda^2 &  &  \\ \lambda^2 & 2(1-\lambda^2) & \lambda^2 &  \\ \ddots & \ddots & \ddots &  \\ & \lambda^2 & 2(1-\lambda^2) & \lambda^2 \\  &  & \lambda^2 & 2(1-\lambda^2)\end{array}\right]\]
and
\[U^m = \left[\begin{array}{c}U_{1}^{m} \\U_{2}^{m} \\\vdots \\U_{J-1}^{m}\end{array}\right]\]

% \[\left[\begin{array}{c}U_{1,j+1} \\U_{2,j+1} \\\vdots \\U_{m-1,j+1}\end{array}\right] =
% \left[\begin{array}{cccc}2(1-\lambda^2) & \lambda^2 &  &  \\ \lambda^2 & 2(1-\lambda^2) & \lambda^2 &  \\ \ddots & \ddots & \ddots &  \\ & \lambda^2 & 2(1-\lambda^2) & \lambda^2 \\  &  & \lambda^2 & 2(1-\lambda^2)\end{array}\right]
% \left[\begin{array}{c}U_{1,j} \\U_{2,j} \\\vdots \\U_{m-1,j}\end{array}\right]  -
% \left[\begin{array}{c}U_{1,j-1} \\U_{2,j-1} \\\vdots \\U_{m-1,j-1}\end{array}\right]\]
In the matrix equation above, we have already used the boundary conditions to determine that $U_{0}^{m} = U_{J}^{m} = 0$ at each time $t_m$.
Note that, to obtain the approximation $U_{j}^{m+1}$ of $u(x_j,t_{m+1})$, the method uses the value of the approximation at \emph{the previous two time steps}.
We can find the solution for the first two time steps by using the initial conditions.
Using the initial conditions directly gives an approximation at $t = t_0 = 0:$
\[U_{j}^{0} = f(x_j), \quad 1 \leq j \leq J-1\]

To obtain an approximation at the second time step, we consider the Taylor expansion
\begin{align*}
	u(x_j,t_1) &= u(x_j, 0) + u_t(x_j,0) \triangle t + u_{tt}(x_j,0) \frac{\triangle t^2}{2} + u_{ttt}(x_j,t_1^*) \frac{\triangle t^3}{6}.
\end{align*}
Recalling that the solution $u(x,t)$ satisfies the wave equation, we substitute in expressions from our initial conditions:
\begin{align*}
	u(x_j,t_1) &= u(x_j, 0) +  g(x_j) \triangle t+ s^2 f''(x_j)\frac{\triangle t^2}{2} +  u_{ttt}(x_j,t_1^*) \frac{\triangle t^3}{6}.
\end{align*}
Ignoring the third order term, we obtain a second order approximation for the second time step:
\[U_{j}^{1}= U_{j}^{0} + g(x_j) \triangle t+ s^2 f''(x_j) \frac{\triangle t^2}{2}, \quad 1 \leq j \leq J-1\]
or if $f$ is not readily differentiable,
% & U_{j,1}= U_{j,0} + g(x_j) \triangle t+ c^2  \frac{f(x_{j+1}) -2 f(x_{j}) + f(x_{j-1})}{(\triangle x)^2}\frac{\triangle t^2}{2}.
\[U_{j}^{1}= U_{j}^{0} + g(x_j) \triangle t+ \frac{\lambda^2}{2} (f(x_{j+1}) -2 f(x_{j}) + f(x_{j-1}))\]
This method is conditionally stable; the CFL condition is that $\lambda \leq 1$.

\begin{problem}
Consider the initial boundary value problem
\begin{align*}
	u_{tt} &= u_{xx}, \\
	u(0,t) &= u(1,t) = 0, \\
	u(x,0) &= \sin(2 \pi x),\\
	u_t(x,0) &= 0.
\end{align*}
Numerically approximate the solution $u(x,t)$ at $t = .5$.
Use $J=5$ subintervals in the $x$ dimension and $M=5$ subintervals in the $t$ dimension.
Compare your results with the analytic solution $u(x,t) = \sin{(2 \pi x)} \cos{(2 \pi t)}$.
This function is known as a standing wave.
\end{problem}

\begin{problem}
Consider the initial boundary value problem
\begin{align*}
	u_{tt} &= u_{xx}, \\
	u(0,t) &= u(1,t) = 0, \\
	u(x,0) &= .2e^{-m^2(x-1/2)^2}\\
	u_t(x,0) &= -.4m^2(x-1/2)e^{-m^2(x-1/2)^2}.
\end{align*}
The solution of this problem is a Gaussian pulse.
It travels to the right at a constant speed.
This solution models, for example, a wave pulse in a stretched string.
Note that the fixed boundary conditions reflect the pulse back when it meets the boundary.

Numerically approximate the solution $u(x,t)$ at $t = 1$ (set $m=20$).
Use 200 subintervals in space and 220 in time, and animate your results.
Then use 200 subintervals in space and 180 in time.
Note that the stability condition is not satisfied for the second mesh.
% Compare your results with the analytic solution $u(x,t) = \sin{(2 \pi x)} \cos{(2 \pi t)}$.
\end{problem}

\begin{problem}
Consider the initial boundary value problem
\begin{align*}
	u_{tt} &= u_{xx}, \\
	u(0,t) &= u(1,t) = 0, \\
	u(x,0) &= .2e^{-m^2(x-1/2)^2}\\
	u_t(x,0) &= 0.
\end{align*}
The initial condition separates into two smaller, slower-moving pulses, one travelling to the right and the other to the left.
This solution models, for example, a plucked guitar string

Numerically approximate the solution $u(x,t)$ at $t = 2$ (set $m=20$).
Use 200 subintervals in space and 440 in time, and animate your results.
It is rather easy to see that the solution to this problem is the sum of two travelling waves, one travelling to the left and the other to the right, as described earlier.
How can the solution to the first problem also be shown to be the sum of two travelling waves?
\end{problem}

\begin{problem}
Consider the initial boundary value problem
\begin{align*}
	u_{tt} &= u_{xx}, \\
	u(0,t) &= u(1,t) = 0, \\
	u(x,0) &= \begin{cases} 1/3 & \text{if } 5/11 < x < 6/11,\\
	0 & \text{otherwise}
	\end{cases}\\
	u_t(x,0) &= 0.
\end{align*}

Numerically approximate the solution $u(x,t)$ at $t = 2$.
Use 200 subintervals in space and 440 in time, and animate your results.
Even though the method is second order and stable for this discretization, since the initial condition is discontinuous there are large dispersive errors.
The finite volume method can be used to smooth out the numerical solution.
\end{problem}

\section*{Travelling Wave Solutions of an Evolution Equation}
Recall that the advection (transport) equation with initial conditions, given by
\begin{align*}
	&{ }u_t + su_x  = 0, \quad -\infty < x < \infty, \\
	&{ }u(x,0) = f(x),
\end{align*}
has as its general solution $u(x,t) = f(x -st)$.
Consider a general evolutionary PDE of the form
\begin{align}
u_t = G(u,u_x, u_{xx}, \ldots)
\label{evol_pde}
\end{align}
An interesting question to ask is whether \eqref{evol_pde} has travelling wave solutions: is there a signal or wave profile $f(x)$, so that $u(x,t) = f(x-st)$ is a solution of \eqref{evol_pde} that carries the signal at a constant speed $s$?
These travelling waves are often significant physically.
For example, in a PDE modelling insect population dynamics a travelling wave could represent a swarm of locusts; in a PDE describing a combustion process a travelling wave could represent an explosion or detonation.

% Travelling wave solutions of a partial differential equation are solutions of the form $u(x,t) = u(x-st),$ where $s$ is the speed of the traveling wave.
% Thus a travelling wave solution is a solution that is a function of one variable, $\xi= x-st$.
% This new frame of reference corresponds to an observer moving along with the wave, so that the wave appears stationary as the observer studies it.

\subsection*{Burgers' equation}
We will examine the process of studying travelling wave solutions using Burgers' equation, a nonlinear PDE from gas dynamics.
It is given by
\begin{align}
	u_t + \left( \frac{u^2}{2} \right)_x = \nu u_{xx}, \label{eqn:Burgers_pde}
\end{align}
where $u$ and $\nu$ represent the velocity and viscosity of the gas, respectively.
It models both the process of transport with the nonlinear advection term $(u^2/2)_x = u u_x$, as well as diffusion due to the viscosity of the gas ($\nu u_{xx}$).

Let us look for a travelling wave solution $u(x,t) = \hat{u}(x-st)$ for Burgers equation.
We transform \eqref{eqn:Burgers_pde} into the moving frame $(x,t) \to (\bar{x},\bar{t}) = (x-st, t)$. In this frame \eqref{eqn:Burgers_pde} becomes
\begin{align}
	u_{\bar{t}} - s u_{\bar{x}}+ \left(\frac{u^2}{2} \right)_{\bar{x}} = \nu u_{\bar{x}\bar{x}}
	\label{eqn:Burgers_pde_moving_frame}
\end{align}
% The coordinate system $(\bar{x},\bar{t})$ is called the moving frame because the travelling wave is stationary in this coordinate system.
This new frame of reference corresponds to an observer moving along with the wave, so that the wave appears stationary as the observer studies it.
Thus, $\hat{u}_{\bar{t}} = 0$, so that the wave profile $\hat{u}$ satisfies the ordinary differential equation
\begin{align}
	 -s u_{\bar{x}}+ \left(\frac{u^2}{2} \right)_{\bar{x}} = \nu u_{\bar{x}\bar{x}}.
	\label{eqn:Burgers_ode}
\end{align}

From here on we will drop the bar notation for simplicity.
We seek a travelling wave solution with asymptotically constant boundary conditions; that is,  $\lim_{x \to \pm \infty}\hat{u}(x) = u_{\pm}$
% \begin{align*}
% \lim_{x \to -\infty}u(x) &= u_-,\\
% \lim_{x \to +\infty}u(x) &= u_+
% \end{align*}
both exist, and  $\lim_{x \to \pm \infty} \hat{u}'(x) = 0$.
We will suppose that $u_- > u_+ > 0$.

% A travelling wave solution $u(\xi)$ of Burgers' equation will satisfy the ordinary differential equation
% \[ -s u' + u u' = \nu u''.\]

Note that to this point we still don't know the speed of the travelling wave.
Integrating both sides of this differential equation, and then taking the limit as $x \to +\infty$, we obtain
\begin{align*}
-s\int_{-\infty}^x u' + \int_{-\infty}^x \left(\frac{u^2}{2}\right)' &= \nu \int_{-\infty}^x u'',\\
-s(u(x) - u_-) + \frac{u^2(x)}{2} - \frac{u_-^2}{2} &= \nu (u'(x) - u'(-\infty)), \\
-s(u_+ - u_-) + \frac{u_+^2}{2} - \frac{u_-^2}{2} &= 0.
\end{align*}
Thus given boundary conditions $u_{\pm}$ at $\pm \infty$, the speed of the travelling wave must be $s = \frac{u_- + u_+}{2}$.

Usually at this point, the travelling wave must be numerically solved using the profile ODE (\eqref{eqn:Burgers_ode} for Burgers equation).
However, the profile ODE for Burgers is simple enough that it is possible to obtain an analytic solution.
The travelling wave is  given by
% If we continue by solving the first order equation
% \[-\frac{u_- + u_+}{2}(u(x) - u_-) + \frac{u^2(x)}{2} - \frac{u_-^2}{2} = \nu u'(x)\]
% we obtain a one-parameter family of solutions
\[\hat{u}(x) = s - a \tanh \left(\frac{ax }{2\nu} + \delta\right)\]
where $a = (u_- - u_+)/2$ and $\delta$ is fixed real number.
We get a family of solutions because any translation of a travelling wave solution is also a travelling wave solution.

\subsection*{Stability of travelling waves}
Suppose that an evolutionary PDE
\begin{align}
u_t = G(u,u_x, u_{xx}, \ldots).
\label{eqn:evol_pde_repeat}
\end{align}
has a travelling wave solution $u(x,t) = \hat{u}(x-st)$.
An interesting question to consider is whether the mathematical solution, $\hat{u}$, has a physical analogue.
In other words, does the travelling wave show up in real life?
This question is the start of the mathematical study of stability of travelling waves.

We begin by translating \eqref{eqn:evol_pde_repeat} into the moving frame $(x,t) \to (\bar{x},\bar{t}) = (x-st, t)$.
In this frame the PDE becomes
\begin{align*}
u_t - su_x = G(u,u_x, u_{xx}, \ldots).
\end{align*}
In these coordinates the travelling wave is stationary.
Thus, the solution of
\begin{align*}
\begin{split}
u_t - su_x &= G(u,u_x, u_{xx}, \ldots), \\
u(x,t = 0) &= \hat{u}(x),
\end{split}
\end{align*}
is given by $u(x,t) = \hat{u}(x)$.
We say that the travelling wave $\hat{u}$ is asymptotically orbitally stable if whenever $v(x)$ is a small perturbation of $\hat{u}(x)$, the general solution of
\begin{align*}
\begin{split}
u_t - su_x &= G(u,u_x, u_{xx}, \ldots), \\
u(x,t = 0) &= v(x),
\end{split}
\end{align*}
converges to some translation of $\hat{u}$ as $t \to \infty$.
Using this definition to prove stability of a travelling wave is a nontrivial task.

\subsection*{Visualizing stability of the travelling wave solution of Burgers' equation}
The travelling wave solution of Burgers' equation is a stable wave.
To view this numerically, we discretize the PDE
\[u_t -su_x + uu_x = u_{xx}\]
using the second order centered approximations
% \begin{align*}
% &{ } u_t(x_j,t_{n+1/2}) \approx \frac{u_j^{n+1}-u_j^n}{\triangle t}, \quad
% u_x(x_j,t_{n+1/2}) \approx \frac{1}{2} \left( \frac{u_{j+1}^{n+1}-u_{j-1}^{n+1}}{2 \triangle x} +  \frac{u_{j+1}^{n}-u_{j-1}^{n}}{2 \triangle x}\right) ,\\
% &{ } u_{xx}(x_j,t_{n+1/2}) \approx \frac{1}{2} \left( \frac{u_{j+1}^{n+1}- u_{j}^{n+1}+u_{j-1}^{n+1}}{(\triangle x)^2} + \frac{u_{j+1}^{n}- u_{j}^{n}+u_{j}^{n+1}}{(\triangle x)^2}\right).
% \end{align*}
\begin{align*}
&{ } D_t U_j^{n+1/2} = \frac{U_j^{n+1}-U_j^n}{\triangle t}, \quad
D_{xx}U_j^{n+1/2} = \frac{1}{2} \left( \frac{U_{j+1}^{n+1}-U_{j-1}^{n+1}}{2 \triangle x} +  \frac{U_{j+1}^{n}-U_{j-1}^{n}}{2 \triangle x}\right),\\
&{ } D_{xx}U_j^{n+1/2} = \frac{1}{2} \left( \frac{U_{j+1}^{n+1}- U_{j}^{n+1}+U_{j-1}^{n+1}}{(\triangle x)^2} + \frac{U_{j+1}^{n}- U_{j}^{n}+U_{j-1}^{n}}{(\triangle x)^2}\right)
\end{align*}

Substituting these expressions into the PDE we obtain a second-order, implicit Crank-Nicolson method
% \begin{align*}
% &{ }u_j^{n+1} + k_1 (u_j^{n+1}-s)(u_{j+1}^{n+1} - u_{j-1}^{n+1})
% - k_2(u_{j+1}^{n+1} - 2u_j^{n+1}+ u_{j-1}^{n+1}) = \\
% &{ } u_j^n - k_1(u_j^n-s) (u_{j+1}^n - u_{j-1}^n)
% + k_2 (u_{j+1}^n -2u_j^n + u_{j-1}^n),
% \end{align*}
\begin{align*}
U_j^{n+1} - U_j^n &= K_1 \big[(s - U_j^{n+1})(U_{j+1}^{n+1} - U_{j-1}^{n+1})
+ (s - U_j^n) (U_{j+1}^n - U_{j-1}^n) \big] \\
&{ }  \quad
+ K_2 \big[(U_{j+1}^{n+1} - 2U_j^{n+1}+ U_{j-1}^{n+1}) + (U_{j+1}^n -2U_j^n + U_{j-1}^n) \big],\\
&{ }  \quad
\end{align*}
where $K_1 = \frac{ \triangle t }{4 \triangle x}$ and $K_2 = \frac{ \triangle t}{2(\triangle x)^2}$.

\begin{problem}
Numerically solve the initial value problem
\begin{align*}
	&{ } u_t -su_x + uu_x = u_{xx}, \quad x \in (-\infty,\infty),\\
	&{ } u(x,0) = v(x),
\end{align*}
for $t \in [0,1]$.
Let the perturbation $v(x)$ be given by
\[v(x) = 3.5(\sin{(3x)} + 1)\frac{1}{\sqrt{2\pi}} \exp{(-x^2/2)}\]
And let the initial condition be $u(x, 0) = \hat{u}(x) + v(x)$
Approximate the $x$ domain,$(-\infty, \infty)$, numerically by the finite interval $[-20,20]$, and fix $u(-20) = u_-$, $u(20) = u_+$. Let $u_- = 5$, $u_+ = 1$.
Use 150 intervals in space and 350 steps in time.
Animate your results.
You should see the solution converge to a translate of the travelling wave $\hat{u}$.

Hint: This difference scheme is no longer a linear equation.
We have a nonlinear equation in $U^{n+1}$.
We can still solve this function using Newton's method or some other similar solver.
In this case, use \li{scipy.optimize.fsolve}.
\end{problem}


\documentclass[a4paper]{article}

\usepackage{amsmath}
\usepackage{amssymb,amsthm,commath,mathtools}
\usepackage[margin=1in]{geometry}
\usepackage[english]{babel}
\usepackage{subcaption}
\usepackage{graphicx}

\usepackage{opexpl}
\usepackage{parskip}

\usepackage{hyperref}

\author{Vincent Kuhlmann}
\title{Opexpl demonstration}
\begin{document}
	\maketitle

\section{In center of mass frame}
	In the center of mass there is
\begin{align*}
	&m_1v_{1,m} + m_2v_{2,m} = 0\\
	&m_1v_{1,m}^2 + m_2v_{2,m}^2 = 2\dfrac{m_1m_2^2gd + m_1^2m_2gd}{(m_1 + m_2)^2}.
\end{align*}

	From this we solve
\begin{align*}
	&\hphantom{{}\implies{}}{}m_2^2v_{2,m}^2/m_1 + m_2v_{2,m}^2 = 2\dfrac{m_1m_2^2gd + m_1^2m_2gd}{(m_1 + m_2)^2}\\
	&\implies v_{2,m}^2 = \dfrac{2}{m_2^2/m_1 + m_2}\dfrac{m_1m_2^2gd + m_1^2m_2gd}{(m_1 + m_2)^2}\\
	%		&\implies v_{2,m} = \dfrac{\sqrt{m_1^2m_2^2 + m_1^3m_2}\sqrt{2gd}}{\sqrt{m_2^2 + m_2m_1}\left(m_1 + m_2\right)}\\
	&\optag{op:posRootv2m}\implies v_{2,m} = \dfrac{m_1}{m_1 + m_2}\sqrt{2gd}\dfrac{\sqrt{m_1m_2^2 + m_1^2m_2}}{\sqrt{m_1m_2^2 + m_1^2m_2}} = \dfrac{m_1}{m_1 + m_2}\sqrt{2gd}\\
	&\implies v_{2,m} - v_{1,m} = \left(1 + m_2/m_1\right)\dfrac{m_1}{m_1 + m_2}\sqrt{2gd}\\
	&\implies v_{2,m} - v_{1,m} = \sqrt{2gd}.
\end{align*}
\opexpl{op:posRootv2m}{We can't have $ v_{2,m} < 0 $, because conservation of momentum would imply $ v_{1,m} \geq 0 > v_{2,m} $, meaning particle~1 goes right through particle~2. This is not allowed here.}

%	\begin{OpExplMult}
%		\ExplItem aa
%	\end{OpExplMult}

%	\begin{align*}
%		&\implies v_{2,m} - v_{1,m} = \dfrac{m_1 + m_2}{m_1}\dfrac{\sqrt{m_1^2m_2^2 + m_1^3m_2}\sqrt{2gd}}{\sqrt{m_2^2 + m_2m_1}\left(m_1 + m_2\right)}\\
%		&\implies v_{1,m} + v_{2,m} = \left(m_1 - m_2\right)\dfrac{\sqrt{m_2 + m_1}}{\sqrt{m_2 + m_1}}\dfrac{\sqrt{2gd}}{m_1 + m_2}\\
%		&\implies v_{1,m} + v_{2,m} = \dfrac{m_1 - m_2}{m_1 + m_2}\sqrt{2gd}.
%	\end{align*}

This is the relative speed between the particles.

\section{In restframe of particle 2 before collision}
We see
\begin{align*}
	&m_1v_1^2 + m_2v_2^2 = 2m_1gd\\
	&m_1v_1 + m_2v_2 = m_1\sqrt{2gd}\\
	&\implies m_2v_2^2 + \dfrac{1}{m_1}\left(m_1\sqrt{2gd} - m_2v_2\right)^2 = 2m_1gd\\
	&\implies m_1m_2v_2^2 - 2m_1m_2\sqrt{2gd}v_2 + m_2^2v_2^2 = 0\\
	&\implies (v_2 = 0)\mbox{ of }\left(v_2(m_1 + m_2) = 2m_1\sqrt{2gd}\right)\\
	&\optag{op:posv2}\implies v_2(m_1 + m_2) = 2m_1\sqrt{2gd}.
\end{align*}
\opexpl{op:posv2}{We need $ v_2 > 0 $ because of conservation of momentum, and the fact particle~1 can't go right through particle~2.}

	This yields
\begin{align*}
	&v_2 - v_1 = (1 + m_2/m_1)v_2 - \dfrac{1}{m_1}\left(m_1v_1 + m_2v_2\right)\\
	&\implies v_2 - v_1 = (1 + m_2/m_1)v_2 - \sqrt{2gd}\\
	&\implies %(v_2 - v_1 = -\sqrt{2gd})\mbox{ of }
	v_2 - v_1 = 2\sqrt{2gd} - \sqrt{2gd} = \sqrt{2gd}.
\end{align*}

\end{document}


\chapter{Feature Selection \label{chapter:featureselection}}

Modern clinical datasets tend to suffer from an overabundance of features. In fact, there are often more available features than there are training examples. Not all of these features will contribute equal information about the outcome. Including dozens or hundreds of predictors in a supervised learning model does not guarantee higher accuracy; in fact, it is more likely to lead to models that are unnecessarily complex and overfit (see Chapter~\ref{chapter:biasvariance}). Even when features are related to the outcome, they may contribute information that is redundant with other features in the study.

In cases like these, the model designer will either need to choose a subset of features manually or incorporate some form of \textbf{feature selection}: a process that automatically or semi-automatically decides which features are most relevant to the model and discards the others. The goal of feature selection is to remove useless and redundant features in a principled way.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Example: The Pima Indians Dataset}

The so-called ``Pima Indians diabetes dataset'' was collected in the 1980s. It includes information on 768 women from the Pima people, who live near Phoenix, Arizona. The Pima were, as of the late 1980s, under continuous study by the National Institute of Diabetes and Digestive and Kidney Diseases because of their high incidence of diabetes\footnote{The causative factors behind this high diabetes rate are not clear. Some scholars believe that it was driven by a sudden shift in diet during the last century from traditional agricultural crops to processed foods, together with a decline in physical activity \cite{schulz2006effects}.}. There are eight predictors in the dataset and one outcome. The predictors are:
\begin{center}
\texttt{ \small
\begin{tabular}{lp{0.6\textwidth}}
\toprule
Predictor & Description \\
\midrule
Pregnancies & Number of times pregnant \\
Glucose & Plasma glucose concentration in a two-hour oral glucose tolerance test \\
BloodPressure & Diastolic blood pressure (mm Hg) \\
SkinThickness & Triceps skin fold thickness (mm) \\
Insulin & Two-hour serum insulin ($\mu$U/mL) \\
BMI & Body mass index (weight in kg/(height in m)$^2$) \\
DiabetesPedigreeFunction & Diabetes pedigree function (developed by research team; described in paper) \\
Age & Age in years \\
\bottomrule
\end{tabular}
}
\end{center}

The outcome is whether or not the woman went on to develop type II diabetes within $5$~years from the time of the survey.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Correlograms}

For datasets with a manageable number of features, A good way to alert oneself to the presence of highly correlated predictors is to create a \textbf{correlogram}, or scatterplot matrix, which looks at associations between all pairs of variables. A correlogram for the Pima dataset is below.

\begin{center}
\includegraphics[width=\textwidth]{img/pima-ggpairs.png}
\end{center}

\begin{question}{}
This correlogram quantifies correlation using a metric called the \textbf{Pearson correlation coefficient}. Which pairs of predictors are the most tightly correlated? Are they positively or negatively correlated? How might you modify your dataset to eliminate redundancies in the information contributed by the different predictors?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Univariate vs. Multivariate Models \label{section:redund}}

The presence of correlations will affect different types of models in different ways, and some suffer more than others. In the clinical research literature, the standard approach to assessing and accounting for correlations is to start with \textbf{univariate} models, in which each predictor's association with the outcome is studied on its own. Those predictors that display some association with the outcome are then incorporated into larger \textbf{multivariate} models.

Here are the results of eight univariate logistic regression models that capture the effect of each predictor in the Pima dataset on the outcome of diabetes vs. no diabetes. The coefficients on each predictor are called the \textbf{unadjusted} coefficients, and the p-values on the predictor-specific hypothesis tests are called unadjusted $p$-values. The exponentiated coefficients are called unadjusted odds ratios\footnote{See Chapter~\ref{chapter:logreg} if you don't understand why you're exponentiating or where the term ``odds ratio'' comes from. The odds ratio compares the odds of having a positive outcome among two groups separated by a one unit difference of the predictor in question, all else being the same.}.
\vspace{-3mm}

\begin{center}
\texttt{ \small
\begin{tabular}{llll}
\toprule
Predictor & Unadjusted  & Unadjusted  & Unadjusted  \\
& Coefficient & Odds Ratio & P-value \\
\midrule
Pregnancies & 0.137 & 1.147 & $<$0.001 \\
Glucose & 0.038 & 1.039 & $<$0.001 \\
BloodPressure & 0.007 & 1.007 & 0.073 \\
SkinThickness & 0.010 & 1.010 & 0.039 \\
Insulin & 0.002 & 1.002 & $<$0.001 \\
BMI & 0.094 & 1.100 & $<$0.001 \\
DiabetesPedigreeFunction & 1.083 & 2.953 & $<$0.001 \\
Age & 0.042 & 1.043 & $<$0.001 \\
\bottomrule
\end{tabular}
}
\end{center}

Now, here is a multivariate logistic regression model that includes all eight predictors. The coefficients, exponentiated coefficients, and p-values are often called \textbf{adjusted}.

\vspace{-3mm}
\begin{center}
\texttt{ \small
\begin{tabular}{llll}
\toprule
Predictor & Adjusted  & Adjusted  & Adjusted  \\
& Coefficient & Odds Ratio & P-value \\
\midrule
Pregnancies & 0.123 & 1.131 & $<$0.001 \\
Glucose & 0.035 & 1.036 & $<$0.001 \\
BloodPressure & -0.013 & 0.987 & 0.011 \\
SkinThickness & 0.001 & 1.001 & 0.929 \\
Insulin & -0.001 & 0.999 & 0.186 \\
BMI & 0.090 & 1.094 & $<$0.001 \\
DiabetesPedigreeFunction & 0.945 & 2.573 & 0.002 \\
Age & 0.015 & 1.015 & 0.111 \\
\bottomrule
\end{tabular}
}
\end{center}

Alternatively, one might say that the odds ratios here measure the effect of each predictor, \textbf{controlling for} the effects of the other predictors.

\begin{question}{}
How can the odds ratio for Insulin be so close to 1.0 yet its p-value so low? (Hint: See Section~\ref{section:sehyp}.)
\end{question}

\begin{question}{}
Why might the coefficient and p-value for SkinThickness change so much in the shift from unadjusted to adjusted?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Filter Methods}

The approach of creating univariate models and then incorporating the best-performing predictors into multivariate models is part of a broader class of feature selection methods called \textbf{filter methods}. Filter methods select subsets of variables as a preprocessing step, independently of the  supervised learning model that will eventually be implemented. These methods use \textbf{proxy measures} to rank variables; the proxy measure is often chosen to be computationally fast so that large numbers of features can be sifted through quickly.

A predetermined threshold of the proxy measure is usually used to determine which features pass to the multivariate modeling stage. Alternatively, the modeler may decide on a fixed number of features to include. Some examples of filter methods include:

\begin{itemize}
\item Any kind of univariate model (e.g. univariate logistic or linear regression)
\item Any kind of hypothesis test (e.g. t-test, chi-squared test; see Chapter~\ref{chapter:hypothesistesting})
\item Any kind of correlation coefficient (e.g. Pearson, Spearman)
\item Mutual information\footnote{The mutual information, in another format, is the most common splitting criterion used for decision trees; see Chapter~\ref{chapter:decisiontrees}. In the case of continuous variables, the sums are replaced by integrals.}
$$ MI(X_i,Y) = \sum_x \sum_y P(X_i = x, Y = y) \log \frac{P(X_i = x, Y = y)}{P(X_i = x) P(Y = y)} $$
\item Variance thresholding (simply remove features with low variance)
\end{itemize}

\begin{question}{}
If you wanted to use the univariate logistic regression models above in Section~\ref{section:redund} as a filter for a downstream model (potentially not even multivariate logistic regression - it could be a decision tree, etc.), how would you rank them and how would you decide on an appropriate cutoff?
\end{question}

\begin{question}{}
How would you apply a filter-based selection method in a case where you had dozens of different predictors of different types (e.g. some categorical, some binary, some numeric)?
\end{question}

\begin{question}{}
How might you choose the appropriate threshold for a filter-based method in a data-driven way?
\end{question}

\begin{question}{}
What is problematic about testing each potential feature, one at a time?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Wrapper Methods}

Filter methods are just one possible approach to feature selection. An alternative to filter methods are \textbf{wrapper methods}. These methods use a search algorithm to traverse the space of possible features, evaluating each subset by running the chosen model using that subset. They are generally computationally intensive (e.g., imagine trying to find the optimal subset of 10,000 features, or even 50) so \textbf{heuristics} generally have to be used to pare down the search space. Some examples of wrapper methods include:

\begin{itemize}
\item \textbf{Exhaustive search.} Try all possible subsets of features. If there are $m$ features, this means trying $2^m$ possible subsets.
\item \textbf{Forward selection.} Start with a baseline (e.g., intercept only) model. Add in each of $m$ possible predictors individually and take the best one based on some performance criterion. Repeat, adding one predictor at each step, until the performance criterion stops getting better or you run out of predictors.
\item \textbf{Backward elimination.} Start with a complete model (all predictors included). Try removing each predictor and take the one whose removal causes the performance criterion to increase the most. Repeat, removing one predictor at each step, until the performance criterion stops getting better or you are left with no predictors (null model).
\item \textbf{Forward-backward selection.} A combination of forward selection and backward elimination.
\item \textbf{Simulated annealing.} Add or remove predictors with some probability depending on how well the model is doing. At each stage, if the new model is better, accept it; it becomes the new baseline. If the new model is worse, accept it with some probability, $p$, that decreases over time according to a ``cooling schedule''. This helps prevent the variable selection process from getting stuck in local optima.
\end{itemize}

\begin{question}{}
Why is exhaustive search problematic for almost any reasonably sized $m$?
\end{question}

\vspace{2mm}

\begin{question}{}
Here is the output of forward selection for the Pima example, using R's \emph{MASS} package and the \textbf{Akaike Information Criterion (AIC)} as the model performance metric.
{\footnotesize
\begin{verbatim}
Start:  AIC=995.48
Outcome ~ 1
                           Df Deviance    AIC
+ Glucose                   1   808.72 812.72
+ BMI                       1   920.71 924.71
+ Age                       1   950.72 954.72
+ Pregnancies               1   956.21 960.21
+ DiabetesPedigreeFunction  1   970.86 974.86
+ Insulin                   1   980.81 984.81
+ SkinThickness             1   989.19 993.19
+ BloodPressure             1   990.13 994.13
<none>                          993.48 995.48

Step:  AIC=812.72
Outcome ~ Glucose

                           Df Deviance    AIC
+ BMI                       1   771.40 777.40
+ Pregnancies               1   784.95 790.95
+ DiabetesPedigreeFunction  1   796.99 802.99
+ Age                       1   797.36 803.36
<none>                          808.72 812.72
+ SkinThickness             1   807.07 813.07
+ Insulin                   1   807.77 813.77
+ BloodPressure             1   808.59 814.59

Step:  AIC=777.4
Outcome ~ Glucose + BMI

                           Df Deviance    AIC
+ Pregnancies               1   744.12 752.12
+ Age                       1   755.68 763.68
+ DiabetesPedigreeFunction  1   762.87 770.87
+ Insulin                   1   767.79 775.79
+ BloodPressure             1   769.07 777.07
<none>                          771.40 777.40
+ SkinThickness             1   770.20 778.20

Step:  AIC=752.12
Outcome ~ Glucose + BMI + Pregnancies

                           Df Deviance    AIC
+ DiabetesPedigreeFunction  1   734.31 744.31
+ BloodPressure             1   738.43 748.43
+ Age                       1   742.10 752.10
<none>                          744.12 752.12
+ Insulin                   1   742.43 752.43
+ SkinThickness             1   743.60 753.60

Step:  AIC=744.31
Outcome ~ Glucose + BMI + Pregnancies +
          DiabetesPedigreeFunction

                Df Deviance    AIC
+ BloodPressure  1   728.56 740.56
+ Insulin        1   731.51 743.51
<none>               734.31 744.31
+ Age            1   732.51 744.51
+ SkinThickness  1   733.06 745.06

Step:  AIC=740.56
Outcome ~ Glucose + BMI + Pregnancies +
          DiabetesPedigreeFunction +
          BloodPressure

                Df Deviance    AIC
+ Age            1   725.46 739.46
+ Insulin        1   725.97 739.97
<none>               728.56 740.56
+ SkinThickness  1   728.00 742.00

Step:  AIC=739.46
Outcome ~ Glucose + BMI + Pregnancies +
          DiabetesPedigreeFunction +
          BloodPressure + Age

                Df Deviance    AIC
+ Insulin        1   723.45 739.45
<none>               725.46 739.46
+ SkinThickness  1   725.19 741.19

Step:  AIC=739.45
Outcome ~ Glucose + BMI + Pregnancies +
          DiabetesPedigreeFunction +
          BloodPressure + Age + Insulin

                Df Deviance    AIC
<none>               723.45 739.45
+ SkinThickness  1   723.45 741.45
\end{verbatim}
}
What does the final model look like? Which predictor is missing from the final model? Note: AIC is an estimate of out-of-sample prediction error and depends on the likelihood; thus it does not work for models that do not calculate some form of likelihood.
\end{question}

\vspace{2mm}

\begin{question}{}
Here is the output of backward selection for the Pima example, again using R's \emph{MASS} package and AIC as the model performance metric.
{\footnotesize
\begin{verbatim}
Start:  AIC=741.45
Outcome ~ Pregnancies + Glucose + BloodPressure + SkinThickness +
    Insulin + BMI + DiabetesPedigreeFunction + Age

                           Df Deviance    AIC
- SkinThickness             1   723.45 739.45
- Insulin                   1   725.19 741.19
<none>                          723.45 741.45
- Age                       1   725.97 741.97
- BloodPressure             1   729.99 745.99
- DiabetesPedigreeFunction  1   733.78 749.78
- Pregnancies               1   738.68 754.68
- BMI                       1   764.22 780.22
- Glucose                   1   838.37 854.37

Step:  AIC=739.45
Outcome ~ Pregnancies + Glucose + BloodPressure + Insulin + BMI +
    DiabetesPedigreeFunction + Age

                           Df Deviance    AIC
<none>                          723.45 739.45
- Insulin                   1   725.46 739.46
- Age                       1   725.97 739.97
- BloodPressure             1   730.13 744.13
- DiabetesPedigreeFunction  1   733.92 747.92
- Pregnancies               1   738.69 752.69
- BMI                       1   768.77 782.77
- Glucose                   1   840.87 854.87
\end{verbatim}
}
What does the final model look like? How does it compare to the model obtained through forward selection?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Embedded Methods}

The third and final class of feature selection methods are called \textbf{embedded methods}. Embedded methods perform feature selection during the process of model training. They are usually specific to a particular type of model.

\subsection{Decision Trees}

One example of an embedded method is a decision tree (see Chapters~\ref{chapter:decisiontrees}, \ref{chapter:randomforests}, and \ref{chapter:boosting}), which implicitly performs feature selection by placing the most informative predictors at the top of the tree and ignoring those that are unassociated with the outcome.

\vspace{2mm}

\begin{question}{}
Here is the decision tree produced by CART, using information gain/mutual information as the splitting criterion as usual:
\begin{center}
\includegraphics[width=0.6\textwidth]{img/pima-decision-tree.png}
\end{center}
Which features were selected for this tree and which were ignored? How were the features transformed from their original forms in the dataset?
\end{question}

\subsection{Regularized Models}

Another example of an embedded method is \textbf{regularization}. The easiest way to understand regularization is through our discussion of maximum likelihood estimation for GLMs in Chapter~\ref{chapter:glms}. The goal of maximum likelihood estimation is to find the set of model coefficients, $\beta$s, that maximize the joint probability (likelihood) of our observed data given the model. The trouble with this is that more complex models, with more parameters, will generally fit the data better: i.e. produce a higher likelihood.

Regularization addresses this by introducing a penalty term on the likelihood that is proportional to the size of the parameters. In $L_1$ regularization, a.k.a. \textbf{Lasso}, the penalty term is proportional to the absolute values of the coefficients. It looks like this:
$$ \lambda \sum_{j=1}^p \vert \beta_j \vert $$
where $p$ is the number of predictors. This creates a tradeoff in the model between the likelihood and the number of parameters. During optimization, the model will set the coefficients on predictors to zero if including those predictors does not sufficiently improve the likelihood. The relative importance of the penalty term and likelihood is adjusted using the parameter $\lambda$. We will see regularized regression methods in much greater detail in Chapter~\ref{chapter:lassoridge}.

\vspace{2mm}

\begin{question}{}
Here is the raw model output from the multivariate logistic regression model that includes all eight predictors:
\begin{center}
\includegraphics[width=0.7\textwidth]{img/cor-example-multivar.png}
\end{center}
Now let's consider what happens when we use a $L_1$ regularized logistic regression model, produced using the R package \emph{glmnet}. Here is what happens to the model's error (assessed using $10$-fold cross validation; measured using a metric called \textbf{binomial deviance}) when we vary $\lambda$:
\begin{center}
\includegraphics[width=0.8\textwidth]{img/pima-glmnet-plot.png}
{\small
\begin{verbatim}
Measure: Binomial Deviance

      Lambda Measure      SE Nonzero
min 0.004468  0.9686 0.02647       7
1se 0.028723  0.9922 0.02118       5
\end{verbatim}
}
\end{center}
We choose $\lambda$ to be equal to the value that produces the minimum deviance. Here are the coefficients of the final model:
\begin{center}
\includegraphics[width=0.6\textwidth]{img/pima-glmnet-output.png}
\end{center}
Compare this output to the results of models obtained through forward and backward selection methods, as well as to the full (unregularized) logistic regression model. What are the advantages and disadvantages of the regularization approach vs. wrappers and filters?
\end{question}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Written By Michael Brodskiy
% Class: Analytic Geometry & Calculus III (Math-292)
% Professor: V. Cherkassky
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[12pt]{article}
\usepackage{alphalph}
\usepackage[utf8]{inputenc}
\usepackage[russian,english]{babel}
\usepackage{titling}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{enumitem}
\usepackage{amssymb}
\usepackage[super]{nth}
\usepackage{everysel}
\usepackage{ragged2e}
\usepackage{geometry}
\usepackage{fancyhdr}
\geometry{top=1.0in,bottom=1.0in,left=1.0in,right=1.0in}
\newcommand{\subtitle}[1]{%
  \posttitle{%
    \par\end{center}
    \begin{center}\large#1\end{center}
    \vskip0.5em}%

}
\usepackage{hyperref}
\hypersetup{
colorlinks=true,
linkcolor=blue,
filecolor=magenta,
urlcolor=blue,
citecolor=blue,
}

\urlstyle{same}

\title{Lecture XV Notes}
\date{\today}
\author{Michael Brodskiy\\ \small Professor: V. Cherkassky}

% Mathematical Operations:

% Sum: $$\sum_{n=a}^{b} f(x) $$
% Integral: $$\int_{lower}^{upper} f(x) dx$$
% Limit: $$\lim_{x\to\infty} f(x)$$

\begin{document}

\maketitle

\section{Double Integrals over General Regions $-$ 15.2}

First of all, $f$ must be continuous on a region $D$:

$$\{(x,y)|a\leq x\leq b, g_1(x)\leq y\leq g_2(x)\}$$

Such regions are categorized as Type I, and may be found using the following:

$$\iint_D f(x,y)\,dA=\int_a^b\int_{g_1(x)}^{g_2(x)} f(x,y)\,dy\,dx$$

A Type II equation must meet the same criteria as Type I, but with functions of $x$:

$$\{(x,y)|c\leq y\leq d, h_1(x)\leq x\leq h_2(x)\}$$

Such regions may be solved using the following:

$$\iint_D f(x,y)\,dA=\int_c^d\int_{h_1(y)}^{h_2(y)} f(x,y)\,dx\,dy$$

If the region, $D$, is neither Type I nor II, then the regions may be broken into two parts, $D_1$ and $D_2$:

$$\iint_D f(x,y)\,dA=\iint_{D_1} f(x,y)\,dA+\iint_{D_2} f(x,y)\,dA$$

Furthermore, if the function being integrated is a constant, $c$, then:

$$\iint_D c\,dA=cA(D)$$

Where $A(D)$ represents the area of the region $D$

This property may be used if $m\leq f(x,y)\leq M$ to achieve:

$$mA(D) \leq \iint_D f(x,y)\,dA \leq MA(D)$$

\end{document}


\documentclass[10pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{mhchem}
\usepackage{stmaryrd}
\usepackage{graphicx}
\usepackage[export]{adjustbox}
\graphicspath{ {./images/} }
\usepackage{bbold}

\begin{document}
\section{Contents}
1 Convolutional Neural Networks $\ldots \ldots \ldots \ldots \ldots \ldots \ldots$

$1.1$ Variational problems $\ldots . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .$

$1.2$ Introduction $\ldots . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .$

$1.3$ Finite element methods $\ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots$

1.3.1 Linear finite element $\ldots \ldots \ldots \ldots \ldots \ldots \ldots .$

$1.3 .2 \quad$ Bilinear element $\ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots$

References $\ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots \ldots$

\includegraphics[max width=\textwidth]{2022_01_06_c2f144dbff0f0a17cc7dg-2}

\section{Convolutional Neural Networks}
\subsection{Variational problems}
Lemma 1. Assume that $u$ is continuous in $(0,1)$, then the following statements are equivalent

(1) $u(x)=0$.

(2) $\int_{0}^{1} u(x) v(x) d x=0$ for any smooth (compactly supported) function $v$ in $(0,1)$.

Define function $v:[0,1] \rightarrow R$ and define space
$$
V=\{v: v \text { is continuous and } v(0)=v(1)=0\}
$$
Given any $f:[0,1] \rightarrow R$, consider
$$
J(v)=\frac{1}{2} \int_{0}^{1}\left|v^{\prime}\right|^{2} d x-\int_{0}^{1} f v d x
$$
Find $u \in V$ such that
$$
u=\underset{v \in V}{\arg \min } J(v)
$$
which is equivalent to: Find $u \in V$ such that
$$
\left\{\begin{array}{l}
-u^{\prime \prime}=f, 0<x<1, \\
u(0)=u(1)=0 .
\end{array}\right.
$$
Proof. For any $v \in V, t \in R$, let $g(t)=J(u+t v)$. Since $u=\arg \min _{v \in V} J(v)$ means $g(t) \geq g(0) .$ Hence, for any $v \in V, 0$ is the global minimum of the function $g(t)$. Therefore $g^{\prime}(0)=0$ implies
$$
\int_{0}^{1} u^{\prime} v^{\prime} d x=\int_{0}^{1} f v d x \quad \forall v \in V
$$
By integration by parts, which is equivalent to
$$
\int_{0}^{1}\left(-u^{\prime \prime}-f\right) v d x=0 \quad \forall v \in V \text {. }
$$
By variational principal Lemma 1, we obtain
$$
\left\{\begin{array}{l}
-u^{\prime \prime}=f, 0<x<1, \\
u(0)=u(1)=0 .
\end{array}\right.
$$
$\square$

Let $V_{h}$ be finite element space and $\left\{\varphi_{1}, \varphi_{2}, \cdots \varphi_{n}\right\}$ be a nodal basis of the $V_{h} .$ Let $\left\{\psi_{1}, \psi_{2}, \cdots, \psi_{n}\right\}$ be a dual basis of $\left\{\varphi_{1}, \varphi_{2}, \cdots \varphi_{n}\right\}$, namely $\left(\varphi_{i}, \psi_{j}\right)=\delta_{i j} .$
$$
J\left(v_{h}\right)=\frac{1}{2} \int_{0}^{1}\left|v_{h}^{\prime}\right|^{2} d x-\int_{0}^{1} f v_{h} d x .
$$
Let
$$
u_{h}=\sum_{i=1}^{n} v_{i} \varphi
$$
then
$$
u_{h}=\underset{v h \in V_{h}}{\arg \min } J\left(v_{h}\right)
$$
is equivalent to: Find $u_{h} \in V_{h}$
$$
a\left(u_{h}, v_{h}\right)=\left\langle f, v_{h}\right\rangle \quad \forall v_{h} \in V_{h} .
$$
where
$$
a\left(u_{h}, v_{h}\right)=\int_{0}^{1} u_{h}^{\prime} v_{h}^{\prime} d x
$$
Which is equivalent to: Find $u_{h} \in V_{h}$
$$
a\left(u_{h}, v_{h}\right)=\left\langle f, v_{h}\right\rangle \quad \forall v_{h} \in V_{h},
$$
which is equivalent to solving $\underline{A} \mu=b$, where $\underline{A}=\left(a_{i j}\right)_{i j}^{n}$ and $a_{i j}=a\left(\varphi_{j}, \varphi_{i}\right)$ and $b_{i}=\int_{0}^{1} f \varphi_{i} d x .$ Namely
$$
\frac{1}{h}\left(\begin{array}{ccccc}
2 & -1 & & & \\
-1 & 2 & -1 & & \\
& \ddots & \ddots & \ddots & \\
& & -1 & 2 & -1 \\
& & & -1 & 2
\end{array}\right)\left(\begin{array}{c}
\mu_{1} \\
\mu_{2} \\
\vdots \\
\mu_{n}
\end{array}\right)=\left(\begin{array}{c}
b_{1} \\
b_{2} \\
\vdots \\
b_{n}
\end{array}\right) .
$$
Which can be rewritten as
$$
\frac{-\mu_{i-1}+2 \mu_{i}-\mu_{i+1}}{h}=b_{i}, \quad 1 \leq i \leq n, \quad \mu_{0}=\mu_{n+1}=0
$$
Using the convolution notation, (1.9) can be written as
$$
A * \mu=b
$$
where $A=\frac{1}{h}[-1,2,-1]$

\subsection{Introduction}
Let us first briefly describe finite difference methods and finite element methods for the numerical solution of the following boundary value problem
$$
-\Delta u=f, \text { in } \Omega, \quad u=0 \text { on } \partial \Omega, \quad Q=(0,1)^{2} .
$$
For the $x$ direction and the $y$ direction, we consider the partition:
$$
\begin{aligned}
&0=x_{0}<x_{1}<\cdots<x_{n+1}=1, \quad x_{i}=\frac{j}{n+1}, \quad(i=0, \cdots, n+1) \\
&0=y_{0}<y_{1}<\cdots<y_{n+1}=1, \quad y_{j}=\frac{j}{n+1}, \quad(j=0, \cdots, n+1)
\end{aligned}
$$
Such a uniform partition in the $x$ and $y$ directions leads us to a special example in two dimensions, a uniform square mesh $\mathbb{R}_{h}^{2}=\{(i h, j h) ; i, j \in \mathbb{Z}\}$ (Figure 1.2). Let $\Omega_{h}=\Omega \cap \mathbb{R}_{h}^{2}$, the set of interior mesh points and $\partial \Omega_{h}=\partial \Omega \cap \mathbb{R}_{h}^{2}$, the set of boundary mesh points.\\

\includegraphics[max width=\textwidth]{2022_01_06_c2f144dbff0f0a17cc7dg-5}

Fig. 1.1. Two-dimensional uniform grid for finite element and finite difference

\subsection{Finite element methods}
We consider two finite elements: continuous linear element and bilinear element. These two finite element methods find $u_{h} \in V_{h}$ such that
$$
\left(\nabla u_{h}, \nabla v_{h}\right)=\left(f, v_{h}\right), \forall v_{h} \in V_{h} .
$$
The above formulation can be written as
$$
\underline{A u}=\underline{f}
$$
with $\underline{A}_{(j-1) n+i,(l-1) n+k}=\left(\nabla \phi_{k l}, \nabla \phi_{i j}\right), f_{(j-1) n+i,(l-1) n+k}=\left(f, \phi_{i j}\right) .$

Basis functions $\phi_{i j}$ satisfy
$$
\phi_{i j}\left(x_{k}, y_{l}\right)=\delta_{(i, j),(k, l)}
$$

\subsubsection{Linear finite element}
Continuous linear finite element discretization of (1.11) on the left triangulation in Fig 1.2. The discrete space for linear finite element is
$$
\mathcal{V}_{h}=\left\{v_{h}:\left.v_{h}\right|_{K} \in P_{1}(K) \text { and } v_{h} \text { is globally continuous }\right\}
$$
Denote $E_{i, j}=\left[x_{i}, x_{i+1}\right] \times\left[y_{i}, y_{i+1}\right]=K_{i, j}^{U} \cup K_{i, j}^{D} .$ For linear element case,

\includegraphics[max width=\textwidth]{2022_01_06_c2f144dbff0f0a17cc7dg-6}

It is easy to verify that the formulation for the linear element method is

(1.16) $4 u_{i, j}-\left(u_{i+1, j}+u_{i-1, j}+u_{i, j+1}+u_{i, j-1}\right)=f_{i, j}, \quad u_{i, j}=0$ if $i$ or $j \in\{0, n+1\}$,

where
$$
f_{i, j}=\int_{\Omega} f(x, y) \phi_{i, j}(x, y) \mathrm{d} x \mathrm{~d} y \approx h^{2} f\left(x_{i}, y_{j}\right)
$$
Proposition 1. The mapping A* has following properties

    \begin{enumerate}
      \item A is symmetric, namely
    \end{enumerate}
$$
(A * u, v)_{l^{2}}=(u, A * v)_{l^{2}} .
$$

    \begin{enumerate}
      \setcounter{enumiii}{2}
      \item $(A * v, v)_{F}>0$, if $v \neq 0$.

      \item $A * u=f$ if and only if

    \end{enumerate}
$$
u \in \underset{v \in \mathcal{V}_{h}}{\arg \min } J(v)=\frac{1}{2}(A * v, v)-(f, v)
$$

    \begin{enumerate}
      \setcounter{enumiii}{4}
      \item The eigenvalues $\lambda_{k l}$ and eigenvectors $u^{k l}$ of A are given by
    \end{enumerate}
$$
\begin{gathered}
\lambda_{k l}=4\left(\sin ^{2} \frac{k \pi}{2(n+1)}+\sin ^{2} \frac{l \pi}{2(n+1)}\right), \\
u_{i j}^{k l}=\sin \frac{k i \pi}{n+1} \sin \frac{l j \pi}{n+1}, 1 \leq i \leq n, 1 \leq j \leq n,
\end{gathered}
$$
and $\rho(A)<8 .$ Furthermore,
$$
\lambda_{n, n}=8 \cos ^{2} \frac{\pi}{2(n+1)} \approx 8\left(1-\left(\frac{\pi}{2(n+1)}\right)^{2}\right) \approx 8-\frac{2 \pi^{2}}{(n+1)^{2}}
$$

\subsubsection{Bilinear element}
Continuous bilinear finite element discretization of (1.11) on the right mesh in Fig. 1.2. The discrete space for linear finite element is
$$
\mathcal{V}_{h}=\left\{v_{h}:\left.v_{h}\right|_{K} \in\{1, x, y, x y\} \text { and } v_{h} \text { is globally continuous }\right\}
$$
For bilinear element case, we have
$$
\begin{aligned}
\left(\nabla \mathbf{u}_{h}, \nabla \mathbf{v}_{h}\right)=& \sum_{i, j=1}^{n} \int_{E_{i, j}} \nabla \mathbf{u}_{h}, \nabla \mathbf{v}_{h} d x d y \\
=& \sum_{i, j=1}^{n} \int_{E_{i, j}}\left(\frac{\left(u_{i+1, j}-u_{i, j}\right)\left(y_{j+1}-y\right)}{h^{2}}+\frac{\left(u_{i, j+1}-u_{i+1, j+1}\right)\left(y-y_{j}\right)}{h^{2}}\right) \\
&\left(\frac{\left(v_{i+1, j}-v_{i, j}\right)\left(y_{j+1}-y\right)}{h^{2}}+\frac{\left(v_{i, j+1}-v_{i+1, j+1}\right)\left(y-y_{j}\right)}{h^{2}}\right) \\
&+\left(\frac{\left(u_{i, j+1}-u_{i, j}\right)\left(x_{i+1}-x\right)}{h^{2}}+\frac{\left(u_{i+1, j}-u_{i+1, j+1}\right)\left(x-x_{i}\right)}{h^{2}}\right) \\
=&\left(\frac{\left(v_{i, j+1}-v_{i, j}\right)\left(x_{i+1}-x\right)}{h^{2}}+\frac{\left(v_{i+1, j}-v_{i+1, j+1}\right)\left(x-x_{i}\right)}{h^{2}}\right) d x d y
\end{aligned}
$$
where $A=\left(\begin{array}{ccc}-1 & -1 & -1 \\ -1 & 8 & -1 \\ -1 & -1 & -1\end{array}\right)$ and $A * u$ is given by $(1.20)$

And we have

(1.20) $8 u_{i j}-\left(u_{i+1, j}+u_{i-1, j}+u_{i, j+1}+u_{i, j-1}+u_{i+1, j+1}+u_{i-1, j-1}+u_{i-1, j+1}+u_{i+1, j-1}\right)=f_{i, j}$, and $u_{i, j}=0$ if $i$ or $j \in\{0, n+1\} .$ References

\end{document}


\subsection{Finding neighbours}

Say we have a distance function and a sample. How can we identify the \(k\)-nearest neighbours?

We can find the distance for all points, sort this and take the top \(k\) observations.


% Copyright 2018 Melvin Eloy Irizarry-Gelpí
\chapter{Hyper Perplex Numbers}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
An exo-2 split binion has the form
\begin{equation}
    a_{0} + a_{1} S + a_{2} W + a_{3} SW + a_{4} X + a_{5} SX + a_{6} WX + a_{7} SWX
\end{equation}
These follow from a parabolic Cayley-Dickson construct on the exo-1 split binions.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Arithmetic Operations}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conjugate Operations}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Asterisk Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quadrance}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cloak Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dagger Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hodge Star}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Differential Operators}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...