i\documentclass[10pt]{article}
\title{Math 180A: Introduction to Probability}
\author{Suhas Arehalli}
\usepackage{amsmath}
\usepackage{amsfonts}

\begin{document}
\maketitle
\newcommand{\indefint}{\int_{-\infty}^{\infty}}

\section{Preliminaries}

First, fundamental definitions:
\begin{description}
    \item[Experiment] A procedure that can lead to certain outcomes.
    \item[Sample Space] A set $\Omega$ of all possible outcomes of an
    experiment. The "image" of an experiment.
    \item[Event] A subset of the sample space $\Omega$.
\end{description}

Now, using this, we can define \textbf{Kolmogorov's Axioms of Probability}, which tell us that a function
$ P: \Omega \rightarrow [0, 1] $ is a probability if and only if
\begin{enumerate}
    \item $\forall A \subset \Omega, 0 \leq P(A) \leq 1$. Probabilities must be positive and less that 1.
    \item $P(\Omega) = 1$. The probability of having an outcome must be 1.
    \item Given a disjoint $A$ and $B$, $P(A \cup B) = P(A) + P(B)$. The
        probability of either 2 mutually exclusive events occurring is the sum of
        their individual probabilities.
    \item Given an infinite sequence $A_1, A_2, ...$ of pairwise disjoint ($A_i \cap A_j = \emptyset $) events,
             \[P\left(\bigcup_{i = 0}^{\infty} A_i\right) =  \sum_{i =0}^{\infty} P(A_i) \]
         This is extends Axiom 3 to the infinite case.
\end{enumerate}

We also have several useful properties that come out of this:
\begin{description}
    \item[Complementary Probabilities] $P(A^c) = 1 -P(A)$. \\
        The probability of the complement of
        an event $A$ is $P(A^c) = P(\Omega \setminus A)$, and since
        $(\Omega \setminus A) \cup A = \Omega$, $P(A^c) + P(A) = 1$ and thus
        $P(A^c) = 1 -P(A)$.
        Calculating complementary probabilities can sometimes be easier that
        calculating probabilities directly.
    \item[Inclusion-Exclusion] $P(A \cup B) = P(A) + P(B) + P(A \cap B)$.\\
        We know
        \begin{align*}
            P(A) = P(A \cap B) + P(A \cap B^c) \\
            P(B) = P(A \cap B) + P(A^c \cap B)
        \end{align*}
        Thus
        \[P(A) + P(B) = 2P(A \cap B) + P(A \cap B^c) + P(A^c \cap B)\]
        and since
        \[P(A \cap B^c) + P(A^c \cap B) + P(A \cap B) = P(A \cup B)\]
        we can rearrange to obtain the desired equality.
    \item[Monotonicity] If $A \subset B$, the $P(A) \leq P(B)$. \\
        if $A \subset B$, then $B \cap A^c \neq \emptyset$ and $A \cap B = A$, and thus
        $P(B) - P(A) = P(A^c \cap B) \geq 0$, which shows the necessary inequality.
\end{description}

Now we can discuss independence:
\begin{description}
    \item[Independence] Two events, $A$ and $B$, are independent if and only if
        \[ P(A \cap B) = P(A)P(B) \]
\end{description}

That's pretty quick. Now to random variables and distributions.
\begin{description}
    \item[Random Variable] A random variable is a function $X: \Omega \rightarrow \mathbb{F}$ for some field
        $\mathbb{F}$ ($\mathbb{Z}$ or $\mathbb{N}$ for discrete random variables and $\mathbb{R}$ for continuous)
        that maps numbers to outcomes of an experiment. Common choices are the sums of the faces of
        2 rolled dice, then number of heads after a series of coin flips, etc. You're really never
        \textit{that} formal about it.
    \item[Distribution (Discrete)] A distribution of a random variable X is a
        probability $P(X = x)$ for all $x$ in the image of $X$. These also obey the
        probability axioms, with the sample space being the image
        of $X$.
    \item[Expectation (Discrete)] The expectation, or expected value of a discrete random variable $X$ is
        \[ E[X] = \sum_{x} x P(X = x) \]
        consider it an average of outcomes (made numerical by our random variable)  weighted by their
        probabilities. \\\\
        \textbf{Expectation is Linear}. Thus, for scalars ${a_i}$ and random vars ${X_i}$
        \[ E\left[\sum_i a_iX_i\right] = \sum_i a_iE[X_i] \]
    \item[Moments] The expectation is a specific case of a moment, particularly the 1st moment. the kth moment
        of a random variable is defined as
        \[ E[X^k] = \sum_x x^k P(X = x) \]
    \item[Variance] We define the variance as the centered 2nd moment, namely
        \[ var[X] = E[(X - E[X])^2] = E[X^2] - E[X]^2 \]
\end{description}

And that's the basics.

\section{Combinatorial Probability}

The following rules allow you to count the outcomes of nearly any experiment

\begin{description}
    \item[Multiplication Rule] Given $m$ experiments with the $i$th experiment having $n_k$ outcomes, we have
        $\prod_{i = 0}^{m} n_k$ total outcomes
    \item[Ordering] Given a set of $n$ objects, we can arrange them in $n!$ unique orders.
    \item[Combinations] The number of ways to choose $k$ objects from a set of $n$ is the binomial coefficient
        \[ \binom{n}{k} = \frac{n!}{k!(n-k)!} \]
    \item[Partitioning] The number of ways to partition a set into $m$ groups of sizes $n_1, ..., n_m$ with
        $\sum_i n_i = n$ is the multinomial coefficient
        \[ \binom{n}{n_1, ..., n_m} = \frac{n!}{\prod_i (n_i!)} \]
\end{description}

\par
With these counting tools, we can calculate probabilities of complex events

\begin{description}
    \item[Urn Problems] The probability of choosing  $k$ items from a subset of size $m$
        from a set of $n$ items is
        \[ \frac{\binom{m}{k}}{\binom{n}{k}} \]
\end{description}

\begin{description}
    \item[Binomial Distribution]
        Suppose a random variable $X \sim binom(n, p)$, or is distributed according to a binomial distribution
        with parameters $n$ and $p$. Then
        \[ P(X = k) = \binom{n}{k}p^k(1-p)^{n-k} \]
        which represents the probability of having exactly $k$ successes after $n$ trials of an experiment
        with a success rate of $p$. \\\\
        Consider that there are $\binom{n}{k}$ ways of choosing where the $k$ successes occur, and for each
        of these trials, the trial occurs with probability $p^k(1-p)^{n-k}$. Thus the total probability is
        as given. \\\\

        \textbf{Expected Value:}
            Rephrase in terms of indicator random variables $X_i$, which are 1 if the $i$th trial succeeds and
            0 otherwise.:
            \begin{align*}
                X &= \sum_{k = 1}^n X_i \\
                E[X] &= E\left[\sum_{k=1}^n X_i\right] = \sum_{k=1}^n E[X_i]
            \end{align*}
            and since $E[X_i]$ is simply $p$,
            \begin{align*}
                E[X] &= \sum_{k=1}^n p = np
            \end{align*}
        \textbf{Variance:}
            As above,
            \begin{align*}
                var[X] &= var\left[\sum_{k=1}^n X_i\right] = \sum_{k=1}^n var[X_i] = n(var[X_i])
            \end{align*}
            to find $var[X_i]$,
            \begin{align*}
                E[X_i] &= (1)(p) + (0)(1-p) \\
                E[X_i^2] &= (1^2)p + (0^2)(1-p) \\
                E[X_i^2] &= p \\
                var[X_i] &= E[X_i^2] - E[X_i]^2 \\
                var[X_i] &= p - p^2 = p(1-p)
            \end{align*}
            thus,
            \[ var[X] = np(1-p) \]

\end{description}
\par
These can easily extend to experiments with more than one outcome (using the multinomial distribution), but this
distrbution is higher dimensional, so don't worry too much about the distribution.

The binomial distribution isn't always easy to calculate, especially when $n$ becomes large. However, we can use the \textbf{Poisson Distribution} to approximate the binomial distribution for large $n$

\begin{description}
    \item[Poisson Distribution]
        Suppose $ X \sim poisson(\lambda) $. Then
        \[ P(X = k) = \frac{\lambda ^k}{k!} e^{-\lambda}, k \in \mathbb{N} \]
        \\\\
        \textbf{Expected Value:}
        Note that $e^x = \sum_{k=0} \frac{x^k}{k!}$. Taylor series.
        \begin{align*}
            E[X] &= \sum_{k=0}^{\infty}k\frac{\lambda ^k}{k!} e^{-\lambda} = e^{-\lambda}\sum_{k=0}^{\infty}k\frac{\lambda ^k}{k!} \\
            &= e^{-\lambda}\sum_{k=0}^{\infty}\frac{\lambda ^k}{(k-1)!} \\
            &= \lambda e^{-\lambda}\sum_{k=1}^{\infty}\frac{\lambda ^{k-1}}{(k-1)!} \\
            &= \lambda e^{-\lambda}e^{\lambda} = \lambda \\
        \end{align*}
        \textbf{Variance:}
        Here, note that $E[X^2] = E[X(X-1)] + E[X]$
        \begin{align*}
            E[X(X-1)] &= e^{-\lambda}\sum_{i=0}^{\infty}(k)(k-1) \frac{\lambda^k}{k!}\\
            &= \lambda ^2 e^{-\lambda}\sum_{i=2}^{\infty}\frac{\lambda^{k-2}}{(k-2)!} \\
            &= \lambda ^2 \\
            var[X] &= E[X^2] - E[X]^2 = E[X(X-1)] + E[X] - E[X]^2 \\
            &= \lambda ^2 + \lambda - \lambda ^2 = \lambda
        \end{align*}

    \item[Poisson Approximation Theorem]
        Suppose $S_n \sim binom(n,p_n)$ and $X \sim poisson(\lambda)$. If,
        $\lim_{n\rightarrow \infty} p_n = 0$ and $\lim{n \rightarrow \infty} np_n = \lambda$, then
        \[\lim_{n \rightarrow \infty} P(S_n = k) = P(X = k) \]
\end{description}
\par
Thus, if we have a large $n$ in a binomial distribution, we can allow $\lambda = np$ and approximate it using
the Poisson distribution.

\section{Conditional Probability}
\begin{description}
    \item[Conditional Probability] The probability of $A$ given $B$ is defined as
        \[ P(A|B) = \frac{P(A \cap B)}{P(B)} \]
    Note that if we let $F(A) = P(A|B)$, $F: \Omega \rightarrow [0,1]$ is a probability as defined by
    Kolmogorov's axioms. It can also be rephrased as
        \[ P(A \cap B) = P(A|B)P(B) = P(B|A)P(A) \]
    Which tells us that A and B are independent if and only if
        \[ P(A|B) = P(A) \]
    \item[Law of Total Probability] If we create a \textbf{partition} of $\Omega$ $B_1, ..., B_n$, meaning
        $\bigcup_i B_i = \Omega$ and $B_i \cap B_j = \emptyset, \forall i \neq j$, then
        \[ P(A) = \sum_{i=0}^n P(A|B_i)P(B_i) \]
    \item[Bayes' Rule] Using the above, we can obtain
        \[ P(B_i|A) = \frac{P(A|B_i)P(B_i)}{P(A)} = \frac{P(A|B_i)P(B_i)}{\sum_{j = 1}^n P(A|B_j)P(B_j)} \]
\end{description}

Now we can consider joint probabilities: the probability that 2 or more random variable both equal specific values.

\begin{description}
    \item[Joint Probability Distribution] We define a joint probability distribution for random variables
        $X$ and $Y$ as a distribution
        \[P(X = x,..., X_n = x_n) = P(X = x \cap ... \cap  X_n = x_n)\]
    \item[Marginal Distribution] Given a joint distribution $P(X_1, ... X_n)$, we can find a marginal
        distribution $P(X_i = k)$ since
        \[ P(X_i = k) = \sum_{x_1}...\sum_{x_{i-1}}\sum_{x_{i+1}}...\sum_{x_n} P(X_1 = x_1,...,X_i = k,...,X_n = x_n) \]
        Basically, sum over everything \textit{but} the random variable you want a distribution for. With 2
        random variables.
        \[ P(X = k) = \sum_y P(X = k, Y = y) \]
    \item[Independence Redux] We can also say 2 random variables are independent if
        \[P(X = x, Y = y) = P(X = x)P(Y = y), \forall x,y \]
\end{description}

We can now talk about conditional distributions. Again, exactly what you'd expect.

\begin{description}
    \item[Conditional Distribution]
        \[P(X = x | Y = y) = \frac{P(X = x, Y = y)}{P(Y = y)} = \frac{P(X = x, Y = y)}{\sum_u P(X = x, Y = y)} \]
    \item[Conditional Expectation]
        \[E[X | Y = y] = \sum_x xP(X = x | Y = y) \]
\end{description}

\section{Continuous Distributions}

So far we've discussed distributions that take discrete values. Now we can move to distributions of random
variables that take continuous values.

\begin{description}
    \item[Continuous Random Variable] A function from $X: \Omega \rightarrow \mathbb{R}$ that maps outcomes
        to continuous, real values. See above for more info on Random Variables.
    \item[Probability Density Function] A function $f: \mathbb{R} \rightarrow \mathbb{R}$ such that
        \begin{align*}
            f(x) &\geq 0 \\
            \int_{-\infty}^{\infty} f(x)dx &= 1
        \end{align*}
        is a Probability Density Function for a continuous random variable $X$ if
        \[ P(a \leq X \leq b) = \int_{a}^{b} f(x)dx \]
    \item[Expected Value (Continuous)] For a continuous random variable $X$, the Expected Value of $X$ is
        \[ E[X] = \int_{-\infty}^{\infty} xf(x)dx \]
        And further, for some function $r: \mathbb{R} \rightarrow \mathbb{R}$
        \[ E[r(X)] = \int_{-\infty}^{\infty} r(x)f(x)dx \]
    \item[Cumulative Distribution Function] We define the distribution function $F: \mathbb{R} \rightarrow \mathbb{R}$
        for continuous random variable $X$ as
        \[ F(x) = P(X \leq x) = \int_{-\infty}^{x} f(y)dy \]
        Note the parallels to the antiderivative. Namely
        \[ P(a \leq X \leq b) = \int_{a}^{b} f(x)dx = F(b) - F(a) \]
\end{description}

And now let's give examples of common continuous distributions
\newpage
\begin{description}
    \item[Uniform Distribution]
        let $ X \sim uniform(a,b)$.  \\
        \textbf{PDF}
        \[ f(x) = \begin{cases}
                    \frac{1}{b-a} & x \in [a,b] \\
                    0 & otherwise
                  \end{cases} \]
        \textbf{Expected Value}\\
        \begin{align*}
            E[X] &=  \indefint xf(x)dx \\
                 &= \int_{a}^{b} (x)\left( \frac{1}{b-a} \right)dx \\
                 &= \left( \frac{1}{b-a} \right) \int_a^b xdx  \\
                 &= \left( \frac{1}{b-a} \right) \left[\frac{x^2}{2}\right]_a^b   \\
                 &= \frac{b^2 - a^2}{2(b-a)} \\
                 &= \frac{b + a}{2}
        \end{align*}
        \textbf{CDF}\\
        \begin{align*}
            F(x) &= \int_{-\infty}^{x} f(y)dy \\
                 &= \int_{a}^{x} \left(\frac{1}{b-a} \right)dy, x \in [a,b] \\
                 &= \frac{x - a}{b -a}\\
            F(x) &= \begin{cases}
                     0 & x \leq a  \\
                     \frac{x-a}{b-a} & x \in [a,b] \\
                     1 & x \geq b
                    \end{cases}
        \end{align*}
    \newpage
    \item[Exponential Distribution]
        Let $ X \sim exp(\lambda) $. \\
        \textbf{PDF}
        \[ f(x) = \begin{cases}
                    0 & x < 0 \\
                    \lambda e^{-\lambda x} & x \geq 0
        \end{cases} \]
        \textbf{Expected Value}
        \begin{align*}
            E[X] &= \indefint f(x)dx \\
                 &= \int_0^{\infty} \lambda e^{-\lambda x}dx \\
                 &= \lambda \int_0^{\infty} e^{-\lambda x}dx \\
                 &= \lambda \left[ - \frac{1}{\lambda} e^{-\lambda x}dx \right]_0^{\infty} \\
                 &= 0 - (-1) = 1
        \end{align*}
        \textbf{CDF}
        \begin{align*}
            F(x) &= \int_{-\infty}^{x} f(x)dx \\
                 &= \int_0^x \lambda e^{- \lambda x} dx \\
                 &=  \left[ - e^{- \lambda x} \right]_0^x \\
                 &= - e^{- \lambda x} - (-1) \\
                 &= 1 - e^{- \lambda x}
        \end{align*}
    \newpage
\end{description}
\par
Now lets discuss a few transformations that are useful theoreticall and practically.
\begin{description}
    \item[Reduction to the Uniform]. Let $X$ have some arbitrary continuous distribution. Then
        $ Y = F(X)$ is uniform on $(0,1)$.
        \\\\
        Define the inverse of $F$, $F^{-1}$, as
        \[ F^{-1} = min\{x: F(x) \geq y\} \]
        Since a distribution is not necessarily bijective, we just use the minimum satisfying value.
        Now we have,
        \begin{align*}
            P(F(X) < y) &= P(X < F^-1(y)) \\
                        &= F(F^-1(y)) = y
        \end{align*}
        Which is the CDF of the uniform distribution on $[0,1]$, thus,
        \[Y \sim uniform(0,1)\]
    \item[Construction from Uniform] Let $X \sim uniform(0,1)$. Then $Y = F^{-1}(X)$ has distribution
        function $F$.
        \\\\
        Note that, as defined, $F^{-1}(y) \leq x$ iff $F(x) \geq y$. Then
        \begin{align*}
            P(F^{-1}(X) \leq y) &= P(U \leq F(y)) \\
                                &= F(x)
        \end{align*}
        (\textit{Note that $P(X \leq x)$ for $X \sim Uniform(0,1)$})
\end{description}

And now for general functions of random variables

\begin{description}
    \item[PDFs of r(X)] Given a random variable $X$ with PDF $f_X(x)$, the distribution
        function of $Y = r(X)$, $f_Y(y)$ is
        \[ f_Y(y) = f(r^{-1}(y))\frac{d}{dy}\left[r^{-1}(y)\right] \]
        This follows from the chain rule.
        Calculate using the CDF and the theorems above, then derive and use the chain rule.
        \begin{align*}
            F_Y(y) &= P(r(X) < y) \\
                   &= P(X < r^{-1}(y)) \\
                   &= F(r^{-1}(y)) \\
            \int_{-\infty}^{y} f_Y(t)dt &= \int_{-\infty}^{y} f_X(r^{-1}(t))dt \\
            \frac{d}{dy}\left[ \int_{-\infty}^{y} f_Y(t)dt\right] &= \frac{d}{dy}\left[ \int_{-\infty}^{y} f_X(r^{-1}(t))dt\right] \\
            f_Y(y) &= f_X(r^{-1}(y))\frac{d}{dy}\left[r^{-1}(y)\right]
        \end{align*}
\end{description}

Now we discuss joint and conditional distributions in terms of Continuous Random Variables.

\begin{description}
    \item[Joint Density Function] A joint density function $f_{X,Y}$ for continuous random variables $X,Y$ if for
        all $A \subset \mathbb{R}^2$
        \[ P((x,y) \in A)  = \iint_A f_{X,Y}(x,y)dxdy \]
        Also note that
        \[ \iint_{\mathbb{R}^2} f_{X,Y}dxdy = 1 \]
    \item[Marginal Distributions (Continuous)] Given a joint density function $F_{X,Y}$ for continuous random variables $X,Y$, we can calculate
        \begin{align*}
            f_X(x) = \indefint f_{X,Y}(x,y)dy \\c{d}{dy}\left[r^{-1}(y)\right]
            f_Y(y) = \indefint f_{X,Y}(x,y)dx \\
        \end{align*}
    \item[Independence] As you should expect, $X$ and $Y$ are independent iff
        \[ f_{X,Y}(x,y) = f_X(x)f_Y(y), \forall x,y \in \mathbb{R}\]
    \item[Conditional Distributions (Continuous)] We define
        \[ f_X(x | Y = y) = \frac{f_{X,Y}(x,y)}{f_Y(y)} \]
        and using the marginal distributions,
        \[ f_X(x | Y = y) = \frac{f_{X,Y}(x,y)}{\indefint f_{X,Y}(x,y)dx} \]
\end{description}

\section{Limit Theorems}
Here we describe the Normal Distribution and the convergence of sums to distributions to it.

\begin{description}
    \item[Sums of Random Variables (Discrete)] The distributions of the sums of
        independent discrete random variables can be calculated as the sum over
        all values of the 2 random variables that sum to the requested value.
        \[ P(X + Y = k) = \sum_x P(X = x)P(Y = (k -x))  \]
        \\
        \textbf{Binomial} \\
        Let $X \sim binom(n,p)$ and $Y \sim binom(m,p)$ and independent. then \[X + Y \sim binom(n+m, p).\] \\
        \textbf{Poisson} \\
        Let $X \sim poisson(\lambda)$ and $Y \sim poisson(\mu)$. Then \[X + Y \sim poisson(\lambda + \mu) \]
        \begin{align*}
            P(X + Y = k) &= \sum_{x = 0}^{\infty} P(X = x)P(Y = (k -x)) \\
                         &= \sum_{x = 0}^{\infty} \left( e^{-\lambda}\frac{\lambda^x}{x!}\right)\left( e^{-\mu}\frac{\mu^{k-x}}{(k-x)!}\right) \\
                         &= e^{-(\mu+\lambda)} \frac{1}{k!}\sum_{x = 0} \left(\frac{k!}{x!(k-x)!}\right) \lambda^x \mu^{k-x} \\
                         &= e^{-(\mu+\lambda)} \frac{(\mu + \lambda)^k}{k!} \\
            X + Y &\sim poisson(\lambda + \mu)
        \end{align*}
    \item[Sums of Random Variables (Continuous)] As above, but integrating rather than summing.
        \[ f_{X+Y}(z) = \indefint f_X(x)f_Y(z - x)dx \]
        Examples are painful, so figure them out yourself (It's left as an exercise to the reader!).
\end{description}
    Now let's talk about the mean and variance of these sums.

\begin{description}
    \item[Expected Value of Sums]
        Since expectation is linear,
        \begin{align*}
            E[X + Y] &= E[X] + E[Y] \\
            E\left[\sum_{i=0}^n X_i\right] &= \sum_{i=0}^n E[X_i]
        \end{align*}
    \item[Expected Value of Products]
        If $X$ and $Y$ are independent,
        \begin{align*}
            E[XY] &= \sum_{x,y} xyP(X = x, Y = y)  \\
                  &= \sum_{x,y} xyP(X = x)P(Y = y) \\
                  &= \sum_{x} \left(xP(X = x) \left( \sum_{y} yP(Y = y)\right) \right)  \\
                  &= \left(\sum_{y} yP(Y = y) \right) \left( \sum_{x} xP(X = x) \right) \\
                  &= E[X]E[Y]
        \end{align*}
    \item[Covariance]
        The covariance, a measure of how the variance of one variable interacts with the variance of the
        other, is defined as
        \begin{align*}
            cov[X,Y] &= E[(X - E[X])(Y - E[Y])] \\
                     &= E[XY - E[X]y - xE[Y] - E[X]E[Y]] \\
                     &= E[XY] - E[X]E[Y] - E[X]E[Y] - E[X]E[Y] \\
                     &= E[XY] - E[X]E[Y]
        \end{align*}
        Which shows that if $X$ and $Y$ are independent,
        \[ cov[X,Y] = E[X]E[Y] - E[X]E[Y] = 0 \]
    \item[Variance of Sums]
        By definition
        \begin{align*}
            var[X+Y] &= E[((X+Y) - E[X+Y])^2] \\
                     &= E[((x - E[X]) + (Y - E[Y]))^2] \\
                     &= E[(x - E[X])^2] + 2E[(x - E[X])(y - E[Y])] + E[(y - E[Y])^2] \\
                     &= var[X] + var[Y] + 2cov[X,Y]
        \end{align*}
        And if $X$ and $Y$ are independendent.
        \[ var\left[ \sum_{i=0}^n X_i \right] = \sum_{i=0}^n var[X_i] \]
\end{description}

Now a few more related pieces

\begin{description}
    \item[Chebyshev's Inequality] Given $y \geq 0$.
        \[ P(|Y - E[Y]| \geq y) \leq \frac{var[Y]}{y^2} \]
        Note that
        \begin{align*}
            E[Z] = \int_{\mathbb{R}}zf_Z(z)dz &\geq \int_k^{\infty} xf_Z(z)dz \\
                                              &\geq \int_k^{\infty} kf_Z(z)dz  \\
                                              &\geq k\int_k^{\infty} f_Z(z)dz = kP(k \leq Z)
        \end{align*}
        If we let $Z = (Y - E[Y])^2$ and $k = y^2$,
        \begin{align*}
            var[Y] = E[(Y - var[Y])^2] &\geq y^2 P((Y - E[Y])^2 \geq y^2) \\
                                         &\geq y^2 P(|Y-var[Y]| \geq y) \\
        \end{align*}
        and with rearrangement,
        \[P(|Y-E[Y]| \geq y) \leq \frac{var[Y]}{y^2} \]
\end{description}
Now lets move to the 2 capstone results

\begin{description}
    \item[Law of Large Numbers] Given a series of independent and identically distributed (i.i.d.) random
        variables $X_1,X_2,...$ with partial means \\ $\bar{X}_n = \frac{1}{n}\sum_{i=0}^n X_i$, and
        $var[X] = \sigma ^2$ and $E[X] = \mu$.
    \[ \lim_{n \rightarrow \infty} P(|\bar{X}_n - E[X]|\geq \varepsilon) = 0 \]
        \begin{align*}
            var[\bar{X}_n] &= var\left[\frac{1}{n}\sum_{i=0}^n X_i \right] \\
                           &= \frac{1}{n^2} \cdot n(var[X_i]) \\
                           &= \frac{\sigma ^2}{n} \\
            P(|\bar{X}_n - E[X]| \geq \varepsilon) &\leq \frac{var[\bar{X}_n]}{\varepsilon ^2} \\
            P(|\bar{X}_n - \mu| \geq \varepsilon) &\leq \frac{\sigma ^2}{n \varepsilon ^2} \\
            \lim_{n \rightarrow \infty} P(|\bar{X}_n - \mu| \geq \varepsilon) &\leq
                \lim_{n \rightarrow \infty}\frac{\sigma ^2}{n \varepsilon ^2} = 0\\
            \lim_{n \rightarrow \infty} P(|\bar{X}_n - \mu| \geq \varepsilon) &= 0
        \end{align*}
    \item[Normal Distribution] Given $X \sim N(\mu,\sigma)$ and $Y \sim N(\upsilon,\nu)$,
        \[ f_X(x) = \frac{1}{\sqrt{2\pi}\sigma}e^{\frac{-(x-\mu)^2}{2\sigma^2}} \]
        and
        \[ X + Y \sim N(a + b, u + v) \]
        Proofs of these are trivial but painful.

    \item[Central Limit Theorem] Suppose $X_1, X_2, ...$ are i.i.d. with $E[X_i] = \mu$ and
        $var[X_i] = \sigma ^2 < \infty$. Let $S_n = \sum_{i = 1}^n X_i$ and $\chi \sim N(0,1)$.
        As $n \rightarrow \infty$,
        \[ \frac{S_n - \mu}{\sigma\sqrt{n}} \approx \chi \]
    \item[Histogram Correction] Since the Central Limit Theorem is meant for continuous random variables,
        we offset this by treating an integer value $k$ as the interval $[k - 0.5, k + 0.5]$. Thus, for
        a discrete random variable $X$,
        \[P\left(\frac{X - \mu}{\sigma \sqrt{n}} \leq k\right) \approx P\left(\chi \leq k + 0.5\right) \]
        \[P\left(\frac{X - \mu}{\sigma \sqrt{n}} \geq k\right) \approx P\left(\chi \geq k - 0.5\right) \]
\end{description}
And now we done, fam.
\end{document}


\subsection{Gradient descent}

\subsection{What is gradient descent?}

Rather than solve a normal equation, gradient descent takes the loss function, and takes the derivative of the loss function with respect to each parameter.

Small adjustments are then made to the parameters, in the direction of the steepest derivative, resulting in better parameters.

As derivative term gets smaller, convergance happens. The largest changes to the parametres occurs early on in the algorithm.

Can stop if not lowering by much

\subsection{Local minima}

Gradient descent is not guaratneed to arrive at a global minimum. For some loss functions, there will be multiple local minima, and gradient descent can end up in the wrong one.

Linear regression does not have this issue.

As a result, when we create functions with loss functions, convextity is very important. If the loss space is convex, then we will not get stuck in a local minima.


\section*{Exercises}

\begin{ex}
  Consider the following functions $T:\R^3\rightarrow \R^2$.
  For each of these functions $T$, show that it is not linear either
  by showing that $T$ does not preserve addition, or by showing that it
  does not preserve scalar multiplication, or by showing that it does
  not preserve the zero vector.
  \begin{enumerate}
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      x+2y+3z+1 \\
      2y-3x+z
    \end{mymatrix}$.
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      x+2y^2+3z \\
      2y+3x+z
    \end{mymatrix}$.
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      \sin x+2y+3z \\
      2y+3x+z
    \end{mymatrix}$.
  \end{enumerate}
  \begin{sol}
    \begin{enumerate}
    \item We have $T\begin{mymatrix}{c} 0 \\ 0 \\ 0 \end{mymatrix}
      = \begin{mymatrix}{c} 1 \\ 0 \end{mymatrix}$, so $T$ does
      not preserve the zero vector.
    \item We have
      $T\begin{mymatrix}{c} 0 \\ 1 \\ 0 \end{mymatrix}
      = \begin{mymatrix}{c} 2 \\ 2 \end{mymatrix}$ and
      $T\begin{mymatrix}{c} 0 \\ 2 \\ 0 \end{mymatrix}
      = \begin{mymatrix}{c} 8 \\ 4 \end{mymatrix}$, so $T$ does not
      preserve scalar multiplication.
    \item We have
      $T\begin{mymatrix}{c} \pi/2 \\ 0 \\ 0 \end{mymatrix}
      = \begin{mymatrix}{c} 1 \\ \pi/2 \end{mymatrix}$ and
      $T\begin{mymatrix}{c} \pi \\ 0 \\ 0 \end{mymatrix}
      = \begin{mymatrix}{c} 0 \\ \pi \end{mymatrix}$, so $T$ does not
      preserve scalar multiplication.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the following functions $T:\R^3\rightarrow \R^2$. For
  each function $T$, show that $T$ is a linear transformation. Do this
  by showing that $T$ is a matrix transformation, i.e., find a matrix
  $A$ such that $T(\vect{v})=A\vect{v}$.
  \begin{enumerate}
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      x+2y+3z \\
      2y-3x+z
    \end{mymatrix}$.
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      7x+2y+z \\
      3x-11y+2z
    \end{mymatrix}$.
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      3x+2y+z \\
      x+2y+6z
    \end{mymatrix}$.
  \item $T\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c}
      2y-5x+z \\
      x+y+z
    \end{mymatrix}$.
  \end{enumerate}
\end{ex}

\begin{ex}
  Which of the following functions are linear transformations?
  \begin{enumerate}
  \item The function $f:\Seq_K\to\Seq_K$ defined by
    $f((a_0,a_1,a_2,\ldots)) = (-a_1,-a_3,-a_5,\ldots)$. In words, the
    function $f$ removes the even-numbered elements, and negates the
    odd-numbered elements.
  \item The function $f:\Mat_{2,2}\to\Mat_{2,2}$ defined by $f(A) =
    AB$. Here, $\Mat_{2,2}$ is the vector space of $2\times
    2$-matrices with real entries, and $B$ is a fixed matrix.
  \item The function $f:\Mat_{2,2}\to\Mat_{2,2}$ defined by $f(A) =
    A+B$, where $B$ is a fixed matrix.
  \item The function $f:\Mat_{2,2}\to\Mat_{2,2}$ defined by $f(A) =
    A^T$.
  \end{enumerate}
  \begin{sol}
    Yes, yes, no, yes.
  \end{sol}
\end{ex}

\begin{ex}
  Recall the vector space $\Poly$ of polynomials with coefficients in
  a field $K$. Consider the function $M:\Poly\to\Poly$ defined by
  $M(p(x)) = xp(x)$.
  \begin{enumerate}
  \item Compute $M(x^3)$, $M(2x^2+x)$, and $M(ax^2+bx+c)$.
  \item Show that $M$ is a linear transformation.
  \end{enumerate}
\end{ex}

\begin{ex}
  Recall the vector space $\Poly$ of polynomials with coefficients in
  a field $K$. Consider the function $S:\Poly\to\Poly$ defined by
  $S(p(x)) = p(x+1)$.
  \begin{enumerate}
  \item Compute $S(x^3)$, $S(2x^2+x)$, and $S(ax^2+bx+c)$.
  \item Show that $S$ is a linear transformation.
  \end{enumerate}
\end{ex}

\begin{ex}
  Consider the shift function $\shift:\Seq_K\to\Seq_K$ from
  Example~\ref{exa:shift-unshift}. Find a basis for the solution space
  of each of the following equations:
  \begin{enumerate}
  \item $\shift(a) = a$.
  \item $\shift(a) = -a$.
  \item $\shift(\shift(a)) = \shift(a) + 2a$.
  \end{enumerate}
  \begin{sol}
    \begin{enumerate}
    \item Let $a=(a_0,a_1,a_2,\ldots)$. Then the equation $\shift(a) =
      a$ is equivalent to
      \begin{equation*}
        (a_1,a_2,a_3,\ldots) = (a_0,a_1,a_2,\ldots),
      \end{equation*}
      which translates to the recurrence $a_{n+1}=a_n$. The only free
      variable is $a_0$, and the only solutions are the constant
      sequences. They form a 1-dimensional space with basis
      $\set{(1,1,1,\ldots)}$.
    \item The equation $\shift(a) = -a$ is equivalent to
      \begin{equation*}
        (a_1,a_2,a_3,\ldots) = (-a_0,-a_1,-a_2,\ldots),
      \end{equation*}
      which translates to the recurrence $a_{n+1} = -a_n$. The only
      free variable is $a_0$, and the solutions form a 1-dimensional
      space with basis $\set{(1,-1,1,-1,\ldots)}$.
    \item The equation $\shift(\shift(a)) = \shift(a) + 2a$ is
      equivalent to
      \begin{equation*}
        (a_2,a_3,a_4,\ldots) = (a_1+2a_0,a_2+2a_1,a_3+2a_2,\ldots),
      \end{equation*}
      which translates to the recurrence $a_{n+2} = a_{n+1} + 2a_n$.
      The only free variables are $a_0$ and $a_1$. The most obvious
      basis vectors are obtained by letting $(a_0,a_1)=(1,0)$ and
      $(a_0,a_1)=(0,1)$, giving the sequences
      $(1,0,2,2,6,10,22,\ldots)$ and $(0,1,1,3,5,11,21,\ldots)$. Thus,
      \begin{equation*}
        \set{(1,0,2,2,6,10,22,\ldots), (0,1,1,3,5,11,21,\ldots)}
      \end{equation*}
      is a basis for the solutions. Another basis, which is slightly
      more convenient, is
      \begin{equation*}
        \set{(1,-1,1,-1,1,-1,\ldots), (1,2,4,8,16,32,\ldots)}.
      \end{equation*}
    \end{enumerate}
  \end{sol}
\end{ex}


\chapter{Falling Bodies}

Because of gravity, if you throw a hammer straight up in the air, from
the moment it leaves your hand until it hits the ground, it is
accelerating toward the center of the earth at a constant rate.

\emph{Acceration} is the change in velocity. If the hammer leaves your
hand with a velocity of 12 meters per second upward, one second later
it will be rising, its velocity will have slowed to 2.2 meters per
second. One second after that, the hammer will be falling at a rate of
7.6 meters per second. Every second the hammer's velocity is changing by
9.8 meters per second, and that change is always toward the center of
the earth. When the hammer is going up, gravity is slowing it down by
9.8 meters per second each second.  When the hammer is coming down,
gravity is speeding it up by 9.8 meters per second each second.\index{acceleration}

Acceleration due to gravity on earth is a constant negative 9.8 meters per second per second:
\begin{equation*}
a = -9.8
\end{equation*}
(Why is it negative? We are talking about height, which increases as
you go away from the center of the earth. Acceleration is changing the
velocity in the opposite direction.)

\section{Calculating the Velocity}

Given that the acceleration is constant, it makes sense that the
velocity is a straight line. Assuming once again that the hammer
leaves your hand at 12 meters per second, then the upwards velocity at
time $t$ is given by:
\begin{equation*}
  v = 12 - 9.8t
\end{equation*}

Note that the velocity of the hammer is being given as a function. Here is its graph:

\begin{tikzpicture}
    \begin{axis}[
        xmin=-0.25,xmax=2.75,
        ymin=-13,ymax=13,
        axis x line=middle,
        axis y line=middle,
        axis line style=<->,
        xlabel={$t$},
        ylabel={$v$},
        ]
        \addplot[no marks,sdkblue] expression[domain=0:2.25,samples=100]{x * (-9.8) + 12} node[left] {$12 - 9.8t$};
    \end{axis}
\end{tikzpicture}

\begin{Exercise}[title={When is the apex of flight?}, label=vapex]
  Given the hammer's velocity is given by $12 - 9.8t$, at what time (in seconds)
  does it stop rising and begin to fall?
\end{Exercise}
\begin{Answer}[ref=vapex]
  Solve for when the velocity is zero.

  $t = \frac{12}{9.8} = 1.22$ seconds after release.
\end{Answer}

At this point, we need to say something about air resistance. Gravity
is not the only force on the hammer; as it travels through the air,
the air tries to slow it down. This force is called \emph{air resistance},
and for a large, fast moving object (like an airplane) it is very big force. For a
dense object (like a hammer) moving at a slow speed (what you generate
with your hand), air resistance doesn't significantly affect acceleration.

\section{Calculating Position}

If you let go of the hammer when it is 2 meters
above the ground, the height of the hammer is given by:
\begin{equation*}
  p = -\frac{9.8}{2}t^2 + 12t + 2
\end{equation*}

Here is a graph of this function:

\begin{tikzpicture}
    \begin{axis}[
        xmin=-1.2,xmax=3.5,
        ymin=-13,ymax=13,
        axis x line=middle,
        axis y line=middle,
        axis line style=<->,
        xlabel={$t$},
        ylabel={$p$},
      ]
      \addplot[no marks,sdkblue,dashed,<-] expression [domain=-0.7:0,samples=100] {(-4.9)*(x^2) + 12 * x + 2};
      \addplot[no marks,sdkblue] expression [domain=0:2.58,samples=100] {(-4.9)*(x^2) + 12 * x + 2};
      \addplot[no marks,sdkblue,dashed,->] expression [domain=2.58:3,samples=100] {(-4.9)*(x^2) + 12 * x + 2};
    \end{axis}
\end{tikzpicture}

How did I figure this out? \textbf{The change in position between time
  $0$ and any time $t$ is equal to the area under the velocity graph
  between $x = 0$ and $x = t$.}

Let's use the velocity graph to figure out how much the position has
changed in the first second of the hammer's flight. Here's the
velocity graph with the area under the graph for the first second filled
in:

\usepgfplotslibrary{fillbetween}

\begin{tikzpicture}
    \begin{axis}[
        xmin=-0.25,xmax=2.75,
        ymin=-13,ymax=13,
        axis x line=middle,
        axis y line=middle,
        axis line style=<->,
        xlabel={$t$},
        ylabel={$v$},
      ]
      \addplot[no marks,sdkblue, name path=f] expression[domain=0:2.25,samples=100]{x * (-9.8) + 12} node[left] {$12 - 9.8t$};
      \path[name path=xaxis] (axis cs:0,0) -- (axis cs:1,0);
      \addplot[
        thick,
        color=sdkblue,
        fill=sdkblue,
        fill opacity=0.05
    ]
    fill between[
        of=f and xaxis,
        soft clip={domain=0:1},
    ];
    \addplot[dashed,gray] coordinates {(0,12)(1,12)};
    \addplot[dashed,gray] coordinates {(1,12)(1,0)};
    \end{axis}
\end{tikzpicture}

The blue filled region is the area of the dashed rectangle minus that
empty triangle in its upper left.  The height of the rectangle is
twelve and its width is the amount of time the hammer has been in
flight ($t$).  The triangle is $t$ wide and and $9.8t$ tall. Thus, the
area of the blue region is given by $12t - \frac{1}{2}9.8 t^2$.

That's the change in position. Where was it originally? 2 meter off
the ground.  So the height is given by $p = 2 + 12t - \frac{1}{2}9.8t^2$.
We usually write terms so that the exponent decreases, so:

$$p = - \frac{1}{2}9.8t^2 + 12t + 2$$

Finding the area under the curve like this is called
\textit{integration}.  We say ``To find a function that gives the
change in position, we just integrate the velocity function.''  A lot
of the study of calculus is learning to integrate different sorts of
functions.

One important note about integration: Any time the curve drops under
the $x$-axis, the area is considered negative. (Which makes sense,
right? If the velocity is negative, the hammer's position is
decreasing.)

\begin{tikzpicture}
    \begin{axis}[
        xmin=-0.25,xmax=2.75,
        ymin=-13,ymax=13,
        axis x line=middle,
        axis y line=middle,
        axis line style=<->,
        xlabel={$t$},
        ylabel={$v$},
      ]
      \addplot[no marks,sdkblue, name path=f] expression[domain=0:2.25,samples=100]{x * (-9.8) + 12} node[left] {$12 - 9.8t$};
      \path[name path=xaxis] (axis cs:0,0) -- (axis cs:2.25,0);
      \addplot[
        thick,
        color=sdkblue,
        fill=sdkblue,
        fill opacity=0.05
      ]
      fill between[
        of=f and xaxis,
        soft clip={domain=0:1.2245},
      ];
      \addplot[
        thick,
        color=red,
        fill=red,
        fill opacity=0.07
      ]
      fill between[
        of=f and xaxis,
        soft clip={domain=1.2245:2.1},
      ];
    \end{axis}
\end{tikzpicture}

A natural question at this point is ``When will the hammer hit the
ground?''  That is, when does $p = 0$. The values of $t$ where $p = 0$
are known as the \textit{roots} of the quadratic function. In the next
chapter, you will get the trick for finding the roots of any quadratic
function.


%\subsection{Nodal bases and dual bases on multilevel spaces}
%\noindent\textbf{Nodal bases and dual bases on multilevel spaces}

Here each $\mathcal V_\ell$ consists of all piecewise linear (or bilinear)
functions with respect to the grid \eqref{grids} and \eqref{mn-ell}.
Each $\mathcal V_\ell $ has a set of basis functions:
$\phi_{i,j}^\ell\in \mathcal V_\ell$ satisfying:
\begin{equation}
\label{NodalBases-mul}
\mathbf\phi_{i,j}^\ell(x_p^\ell,y_q^\ell)=\delta_{(i,j), (p,q)} =
\begin{cases}
1 \quad &\text{if} \quad (p,q) = (i,j), \\
0 \quad &{\text{if}} \quad (p,q)\neq (i,j).
\end{cases}
\end{equation}
\begin{figure}[!ht]
\begin{center}
\begin{tikzpicture}[xscale=2,yscale=2]
%\tikzstyle{every node}=[font=\Large,scale=0.9]
\draw[-] (0,0) -- (2,0);
\draw[-] (0,0.5) -- (2,0.5);
\draw[-] (0,1) -- (2,1);
\draw[-] (0,1.5) -- (2,1.5);
\draw[-] (0,2) -- (2,2);

\draw[-] (0,0) -- (0,2);
\draw[-] (0.5,0) -- (0.5,2);
\draw[-] (1,0) -- (1,2);
\draw[-] (1.5,0) -- (1.5,2);
\draw[-] (2,0) -- (2,2);

\draw[-] (0,0) -- (2,2);
\draw[-] (0,0.5) -- (1.5,2);
\draw[-] (0,1) -- (1,2);
\draw[-] (0,1.5) -- (0.5,2);
\draw[-] (0.5,0) -- (2,1.5);
\draw[-] (1,0) -- (2,1);
\draw[-] (1.5,0) -- (2,0.5);

\node at (1.35,1.2) {$K_1$};
\node at (1.1,1.3) {$K_2$};
\node at (0.8,1.1) {$K_3$};
\node at (0.7,0.9) {$K_4$};
\node at (0.9,0.7) {$K_5$};
\node at (1.2,0.8) {$K_6$};

\node at (1.18, 1) {$(i,j)$};

\fill(1,1) circle(0.8pt);

\draw[-] (3,0) -- (5,0);
\draw[-] (3,0.5) -- (5,0.5);
\draw[-] (3,1) -- (5,1);
\draw[-] (3,1.5) -- (5,1.5);
\draw[-] (3,2) -- (5,2);

\draw[-] (3,0) -- (3,2);
\draw[-] (3.5,0) -- (3.5,2);
\draw[-] (4,0) -- (4,2);
\draw[-] (4.5,0) -- (4.5,2);
\draw[-] (5,0) -- (5,2);

\node at (4.2,1.2) {$K_1$};
\node at (3.8,1.2) {$K_2$};
\node at (3.8,0.8) {$K_3$};
\node at (4.2,0.8) {$K_4$};
\node at (4.18,1) {$(i,j)$};
\fill(4,1) circle(0.8pt);
\end{tikzpicture}
\end{center}
\end{figure}

For the piecewise linear finite element space, the nodal basis function $\phi^\ell_{i,j}$  associate with each $(x_i^\ell,y_j^\ell)$
(satisfying \eqref{NodalBases-mul}) is given by
\begin{equation}
  \label{LinearNodalBasis}
  \phi_{i,j}^\ell(x,y)=\left\{
  \begin{array}{ll}
\frac{x^\ell_{i+1}-x}{h}, & (x,y)\in K_1, \\
\frac{y^\ell_{j+1}-y}{h}, &(x,y)\in K_2,\\
\frac{x-x^\ell_{i-1}-(y-y^\ell_j)}{h}, &(x,y)\in K_3,\\
\frac{x-x^\ell_{i-1}}{h}, &(x,y)\in K_4,\\
\frac{y-y^\ell_{j-1}}{h}, &(x,y)\in K_5,\\
\frac{x^\ell_{i+1}-x+y-y^\ell_j}{h}, &(x,y)\in K_6\\
0, & \mbox{ elsewhere.}
\end{array}
\right.
\end{equation}
shown in Fig. \ref{fig:nodallinear}.
\begin{figure}
\centering
\includegraphics[width=5.5cm,height=5cm]{figures/nodalbasis.pdf}
\caption{\footnotesize{Nodal basis for linear element.}}
\label{fig:nodallinear}
\end{figure}

For bilinear element, it is easy to see that the nodal basis function $\phi^\ell_{i,j}$  associated with each $(x^\ell_i,y^\ell_j)$
(satisfying \eqref{NodalBases-mul}) is given by
\begin{equation}
  \label{BilinearNodalBasis}
  \phi^\ell_{i,j}(x,y)=\left\{
  \begin{array}{ll}
\frac{(x^\ell_{i+1}-x)(y^\ell_{j+1}-y)}{h^2}, \quad& (x,y)\in K_1, \\
\frac{(x-x^\ell_{i-1})(y^\ell_{j+1}-y)}{h^2}, \quad &(x,y)\in K_2,\\
\frac{(x-x^\ell_{i-1})(y-y^\ell_{j-1})}{h^2},\quad &(x,y)\in K_3,\\
\frac{(x^\ell_{i+1}-x)(y-y^\ell_{j-1})}{h^2},\quad &(x,y)\in K_4,\\
0 \quad & \mbox{ elsewhere.}
\end{array}
\right.
\end{equation}

Associated with the above nodal basis functions $\phi^{\ell}_{i,j}(x,y)\subset \mathcal V_{\ell}$,
we define the corresponding dual basis functions $\psi^{\ell}_{i,j}(x,y)\subset \mathcal V_{\ell}$
satisfying
\begin{equation}
  \label{dual-basis}
(\psi^{\ell}_{i,j}(x,y), \phi^{\ell}_{p,q}(x,y))_{L^2(\Omega)}=\delta_{(i,j), (p,q)}.
\end{equation}
The existence of dual basis functions is obvious, but the exact expression of the dual basis functions are
in general difficult to obtain. In fact, \eqref{dual-basis} is the only property that is needed in the application
of dual basis.

We write $\mathbf u_h(x,y)=\sum\limits_{i,j=1}^{n} u_{i,j}\phi_{i,j}(x,y), \mathbf v_h(x,y)=\sum\limits_{i,j=1}^{n} v_{i,j}\phi_{i,j}(x,y)$.

\begin{lemma}
For bilinear functions, we have
\begin{equation}\label{basis:plongation}
\begin{split}
\phi_{i,j}^{\ell+1}(x,y)&=\phi_{2i,2j}^{\ell}(x,y)+\frac{1}{2}\left(\phi_{2i-1,2j}^{\ell}(x,y)+\phi_{2i,2j-1}^{\ell}(x,y)
+\phi_{2i+1,2j}^{\ell}(x,y)+\phi_{2i,2j+1}^{\ell}(x,y)\right)\\
&+\frac{1}{4}\left(\phi_{2i-1,2j-1}^{\ell}(x,y)+\phi_{2i+1,2j-1}^{\ell}(x,y)
+\phi_{2i+1,2j+1}^{\ell}(x,y)+\phi_{2i-1,2j+1}^{\ell}(x,y)\right).
\end{split}
\end{equation}
For linear functions, we have
\begin{equation}\label{basis:plongation2}
\begin{split}
\phi_{i,j}^{\ell+1}(x,y)&=\phi_{2i,2j}^{\ell}(x,y)
+\frac{1}{2}\left(\phi_{2i-1,2j-1}^{\ell}(x,y)+\phi_{2i+1,2j+1}^{\ell}(x,y)\right)\\
&+\frac{1}{2}\left(\phi_{2i-1,2j}^{\ell}(x,y)+\phi_{2i,2j-1}^{\ell}(x,y)
+\phi_{2i+1,2j}^{\ell}(x,y)+\phi_{2i,2j+1}^{\ell}(x,y)\right).
\end{split}
\end{equation}
\end{lemma}
Thus, for each $\mathbf v^{\boldsymbol {\ell}} \in \mathcal V_{\ell}, \mathbf f^{\boldsymbol {\ell}} \in \mathcal V'_{\ell}=\mathcal V_{\ell}$, we have
\begin{equation}\label{expand}
\mathbf v^{\boldsymbol \ell}(x,y)=\sum_{i=1}^{m_\ell}\sum_{j=1}^{n_\ell}v^\ell_{i,j}\phi_{i,j}^\ell(x,y),
~~ \mathbf f^{\boldsymbol \ell}(x,y)=\sum_{i=1}^{m_\ell}\sum_{j=1}^{n_\ell}f^\ell_{i,j}\psi_{i,j}^\ell(x,y),
\end{equation}
where
\begin{equation}
  \label{vf}
  v^\ell_{i,j}=\mathbf v^\ell(x_i^\ell,y_j^\ell),~~ f_{i,j}^\ell= (\mathbf f^\ell, \phi^\ell_{i,j})_{L^2(\Omega)}.
\end{equation}
Let us introduce the following tensors:
\begin{equation}
  \label{v}
  v^\ell=(v^\ell_{i,j}),~~f^\ell=(f^\ell_{i,j}),~~\phi^\ell=(\phi^\ell_{i,j}),~~\psi^\ell=(\psi^\ell_{i,j}).
\end{equation}
The following identities obviously hold:
$$
\mathbf v^\ell=(v^\ell, \phi^\ell)_{l^2},~~\mathbf f^\ell=(f^\ell, \psi^\ell)_{l^2},~~(\mathbf f^\ell,\mathbf v^\ell)_{L^2(\Omega)}=(f^\ell, v^\ell)_{l^2}.
$$

497-6DL/8 Convolutional Multigrid Method/8.10-2d_basis.tex


\section*{Exercises}

\begin{ex} Find the eigenvalues and eigenvectors of the matrix
\begin{equation*}
\begin{mymatrix}{rrr}
15 & -24 & 7 \\
-6 & 5 & -1 \\
-58 & 76 & -20
\end{mymatrix}
\end{equation*}
One eigenvalue is $-2. $ Diagonalize if possible. \textbf{Hint:\ }
This one has some complex eigenvalues.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Find the eigenvalues and eigenvectors of the matrix
\begin{equation*}
\begin{mymatrix}{rrr}
15 & -25 & 6 \\
-13 & 23 & -4 \\
-91 & 155 & -30
\end{mymatrix}
\end{equation*}
One eigenvalue is $2$. Diagonalize if possible. \textbf{Hint:\ }
This one has some complex eigenvalues.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Find the eigenvalues and eigenvectors of the matrix
\begin{equation*}
\begin{mymatrix}{rrr}
-11 & -12 & 4 \\
8 & 17 & -4 \\
-4 & 28 & -3
\end{mymatrix}
\end{equation*}
One eigenvalue is $1$. Diagonalize if possible. \textbf{Hint:\ }
This one has some complex eigenvalues.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Find the eigenvalues and eigenvectors of the matrix
\begin{equation*}
\begin{mymatrix}{rrr}
14 & -12 & 5 \\
-6 & 2 & -1 \\
-69 & 51 & -21
\end{mymatrix}
\end{equation*}
One eigenvalue is $-3$. Diagonalize if possible. \textbf{Hint:\ }
This one has some complex eigenvalues.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Suppose $A$ is an $n\times n$-matrix consisting entirely of real
entries but $a+ib$ is a complex eigenvalue having the eigenvector, $X+iY$ Here $X$ and $Y$ are real vectors. Show
that then $a-ib$ is also an eigenvalue with the eigenvector, $X-iY$. \textbf{Hint: }You should remember that the conjugate of a
product of complex numbers equals the product of the conjugates. Here $a+ib$
is a complex number whose conjugate equals $a-ib$.
\begin{sol}
 $AX=(
a+ib)X$. Now take conjugates of both sides. Since $A$ is
real,
\[
A\conjugate{X}=(a-ib) \conjugate{X}
\]
\end{sol}
\end{ex}


\documentclass[12pt]{pylatex}
\usepackage{examples}

\begin{document}

\section*{Quadratic convergence of Newton-Raphson iterations}

This is a simple example that uses Python and {\tt\small sympy} to demonstrate the quadratic converegnce of Newton-Raphson iterations to the exact root of a non-linear equation.

\vspace{10pt}

\begin{python}
   from sympy import *

   x = Symbol('x')

   f    = Lambda (x,x-exp(-x))
   df   = Lambda (x,diff(f(x),x))
   Step = Lambda (x,x-f(x)/df(x))

   Digits = 200  # use 200 decimal digits for all numerical computations

   x_new = Float('0.5',Digits)
   f_new = N (f(x_new),Digits)

   # pyBeg (table)

   print ('\RuleA {:2d} & {: .25f} & {: .10e} &\\\\'.format(0,x_new,f_new))

   for n in range (1,7):
      x_old = x_new
      x_new = N (Step(x_new),Digits)
      f_old = N (f(x_old),Digits)
      f_new = N (f(x_new),Digits)
      ratio = N (f_new / f_old**2,Digits)
      print ('\RuleA {:2d} & {: .25f} & {: .10e} & {: .5f}\\\\'.format(n,x_new,f_new,ratio))

   # pyEnd (table)
\end{python}

\clearpage

Note the clear quadratic convergence in the iterations -- the last column settles to approximately $-0.11546$ independent of the number of iterations. This behaviour would not be seen using normal floating point computations as they are normally limited to no more than 18 decimal digits. This computation used 200 decimal digits.

\def\RuleA{\vrule depth0pt  width0pt height14pt}
\def\RuleB{\vrule depth8pt  width0pt height14pt}
\def\RuleC{\vrule depth10pt width0pt height16pt}

\setlength{\tabcolsep}{0.025\textwidth}%

\begin{center}
   \begin{tabular}{cccc}%
      \noalign{\hrule height 1pt}
      \multicolumn{4}{c}{\RuleC\rmfamily\bfseries%
      Newton-Raphson iterations \quad%
      $x_{n+1} = x_n - f_n/f'_n\ ,\quad f(x) = x-e^{-x}$}\\
      \noalign{\hrule height 1pt}
      \RuleB$ n$&$ x_n$&$ \epsilon_{n} =  x_{n} - e^{-x_{n}}$&$\epsilon_{n}/\epsilon_{n-1}^2$\\
      \noalign{\hrule height 0.5pt}
      \py{table}
      \noalign{\hrule height 1pt}
   \end{tabular}
\end{center}

\vspace{20pt}

\begin{latex}
   \def\RuleA{\vrule depth0pt  width0pt height14pt}
   \def\RuleB{\vrule depth8pt  width0pt height14pt}
   \def\RuleC{\vrule depth10pt width0pt height16pt}

   \setlength{\tabcolsep}{0.025\textwidth}%

   \begin{center}
      \begin{tabular}{cccc}%
         \noalign{\hrule height 1pt}
         \multicolumn{4}{c}{\RuleC\rmfamily\bfseries%
         Newton-Raphson iterations \quad%
         $x_{n+1} = x_n - f_n/f'_n\ ,\quad f(x) = x-e^{-x}$}\\
         \noalign{\hrule height 1pt}
         \RuleB$ n$&$ x_n$&$ \epsilon_{n} =  x_{n} - e^{-x_{n}}$&$\epsilon_{n}/\epsilon_{n-1}^2$\\
         \noalign{\hrule height 0.5pt}
         \py{table}
         \noalign{\hrule height 1pt}
      \end{tabular}
   \end{center}
\end{latex}

\end{document}


\section{Methods of Counting}

\subsection{package au.id.cxd.math.count}

The count package contains a series of modules dedicated to methods of counting.

\subsubsection{Factorial}

The factorial operation is provided as $n!$ implementing:
$$
\prod_{i=1}^{n-1} (n-i)
$$
The Factorial implementation will memoize results, allowing for efficient reuse during runtime.

\subsubsection{Choose}

The choose module implements $n \choose m$, how many ways can m items be selected with replacement from a set of n items. \\
Determined as:
$$
\frac{n!}{m!n!}
$$

\subsubsection{Permutation}

The method of selecting m ordered items from a set of n ordered items $P {n \choose m}$.\\
$$
\frac{n!}{(n-m)!}
$$


\subsection{Test 1 Answers}
\begin{enumerate}[label=\arabic*.]
	\item
		Using implicit differentiation,
		\begin{equation*}
			\dd{y}{x}-\frac{1}{y}\dd{y}{x} = 2x
		\end{equation*}
		so
		\begin{equation*}
			\dd{y}{x} = \frac{2x}{1-1/y} = \frac{2xy}{1-y}
		\end{equation*}
		So, $y - \ln{y} = x^2 + 1$ is an implicit solution to $\dd{y}{x} = \frac{2x}{y-1}$ as we have demonstrated by differentiating and checking.
	\item
		We will use the integrating factor method. Rewriting the equation so all terms involving $y$ are on the left,
		\begin{equation*}
			y' + 4y = e^{-x}
		\end{equation*}
		In this case,
		\begin{equation*}
			a(x) = 4 \text{ and } b(x) = e^{-x}
		\end{equation*}
		so,
		\begin{equation*}
			\mu(x) = e^{\int{a(x) \mathrm{d}x}} = e^{4x}
		\end{equation*}
		Multiplying both sides by $\mu(x)$,
		\begin{equation*}
			\left(e^{4x}y\right)' = e^{3x}
		\end{equation*}
		Integrating both sides,
		\begin{equation*}
			e^{4x}y = \frac{1}{3}e^{3x} + C_1
		\end{equation*}
		Solving for $y$,
		\begin{equation*}
			y = \frac{1}{3}e^{-x} + C_1e^{-4x}
		\end{equation*}
		Plugging in $x = 0$ and $y=\frac{4}{3}$ to solve for $C_1$,
		\begin{equation*}
			\frac{4}{3} = \frac{1}{3} + C_1 \implies C_1 = 1
		\end{equation*}
		So, our answer to the IVP is
		\begin{equation*}
			y = \frac{1}{3}e^{-x} + e^{-4x}
		\end{equation*}
	\item
		The equation is separable and can be rewritten as
		\begin{equation*}
			\frac{x+2}{x} \mathrm{d}x = (t-3)^2 \mathrm{d}t \text{, } x\neq 0
		\end{equation*}
		We'll come back later to see if $x = 0$ is a solution.
		Integrating both sides,
		\begin{equation*}
			x + 2\ln{\abs{x}} = \frac{(t-3)^3}{3} + C_1 \text{, } x\neq 0
		\end{equation*}
		Plugging in $x = -1$ and $t = 3$ to solve for $C_1$,
		\begin{equation*}
			1 + 2\ln{1} = 0 + C_1 \implies C_1 = -1 \text{, } x\neq 0
		\end{equation*}
		So, our solution to the IVP is
		\begin{equation*}
			x + 2\ln{\abs{x}} = \frac{(t-3)^3}{3} - 1 \text{, } x\neq 0
		\end{equation*}
		Checking if $x = 0$ is a solution,
		\begin{equation*}
			0 = 0\frac{(t-3)^2}{2}
		\end{equation*}
		So, $x = 0$ is a general solution. However for our solution $x(t) = 0$, $x(3) \neq -1$, to $x = 0$ is not a solution to the IVP. So, our solution to the IVP remains.
	\item
		This equation is linear and homogeneous, so we can find the general solution using the auxiliary equation. Extracting the auxiliary equation and finding the roots,
		\begin{equation*}
			r^3 + r^2 - 2 = 0 \implies r = 1, -1 \pm i
		\end{equation*}
		So, the general solution is
		\begin{equation*}
			y = C_1e^{x} + C_2e^{-x}\cos{x} + C_3e^{-x}\sin{x}
		\end{equation*}
	\item
		\begin{enumerate}[label=(\alph*)]
			\item
				As stated in the problem, 90kg of salt is in the tank initially.
			\item
				So find the amount of salt in the tank after 2 hours we'll need to set up a differential equation that models the situation.
				Let $y(t)$ be the number of kgs of salt in the tank after $t$ minutes. Let $V(t)$ be the volume of brine in the tank after $t$ minutes.
				Modeling the salt,
				\begin{equation*}
					\dd{y}{t} = \text{salt rate in} - \text{salt rate out} = 0 - \text{salt rate out}
				\end{equation*}
				Modeling the volume,
				\begin{equation*}
					\dd{V}{t} = \text{brine rate in} - \text{brine rate out} = 3\frac{L}{min} - 6\frac{L}{min} = -3\frac{L}{min}
				\end{equation*}
				We're also given that $V(0) = 2000L$, so be can find that
				\begin{equation*}
					V(t) = 2000 - 3t
				\end{equation*}
				Now we can write an equation to find $\text{salt rate out}$.
				\begin{equation*}
					\text{salt rate out} = \frac{y \text{kg of salt}}{V \text{L of brine}}*\frac{6L}{\text{min}} = \frac{6y}{2000-3t}
				\end{equation*}
				This allows us to write a differential equation for $y(t)$.
				\begin{equation*}
					\dd{y}{t} = -\text{salt rate out} = \frac{-6y}{2000-3t} = \frac{6y}{3t-2000}
				\end{equation*}
				This eqation is seperable and can be rewritten as
				\begin{equation*}
					\frac{\mathrm{d}y}{y} = \frac{6\mathrm{d}t}{3t-200}
				\end{equation*}
				Integrating both sides,
				\begin{equation*}
					\ln{\abs{y}} = 2\ln{\abs{3t-2000}}+C_1
				\end{equation*}
				We know that $y > 0$ always because we can't have a negative amount of salt. Further, we're concerned with the time between $t= 0$ and $t = 120 < \frac{2000}{3}$, so $3t-2000 < 0$. This means we can rewrite our equation as
				\begin{equation*}
					\ln{y} = 2\ln{(2000-3t)} + C_1
				\end{equation*}
				Exponentiating both sides,\footnote{$C_1$ is different from the $C_1$ previously, but it's still a constant.}
				\begin{equation*}
					y = C_1(2000-3t)^2
				\end{equation*}
				Applying our initial condition of $y(0)=90$ to solve for $C_1$,
				\begin{equation*}
					90 = C_1(2000)^2 \implies C_1 = \frac{2000^2}{90}
				\end{equation*}
				Plugging our solution for $C_1$ back into our general solution,
				\begin{equation*}
					y = 90\left(\frac{2000-3t}{2000}\right)^2
				\end{equation*}
		\end{enumerate}
	\item
		Newton's law of cooling is
		\begin{equation*}
			\dd{T}{t} = k\left(T_e - T\right)
		\end{equation*}
		where $k$ is some constant, $T_e$ is the temperature of the environment, and $T$ is the current temperature of the object. In the detective's case, $T_e = 16$, so
		\begin{equation*}
			\dd{T}{t} = k(16-T)
		\end{equation*}
		with initial conditions $T(12) = 34$ and $T(13) = 32$. One could then solve this equation and apply the inital conditions to solve for $k$ and the constant of integration $C$. Then one would need to solve for $x$ in $T(x) = 37$. This value of $x$ will tell you the number of hours after midnight one the same day that the murder took place\footnote{For those that do solve the equation, $x \approx 10.691$, so the murder should have took place at about 10:41am.}.

\end{enumerate}


\textbf{The function $|x-i|$  is analytic for $x\in[-1,1]$. This means it can be analytically continued to an analytic function $f(z)$ in a neighborhood of $[-1,1]$ in the complex z-plane. The formula $|z-i|$ itself does not define an analytic function in any complex neighborhood. Find another formula for $f$ that does, and use it to explain what singularities $f$ has in the complex plane.}
\newline

For a complex function $f(z) = u+vi$ to be analytic, it has to satisfy the Cauchy-Riemann Equations:

\begin{align*}
\frac{\partial u}{\partial x} &= \frac{\partial v}{\partial y},\\
\frac{\partial u}{\partial y} &= -\frac{\partial v}{\partial x}.
\end{align*}
In this case we have that
\begin{align*}
f(z) = |z-i| &= |x+yi-i| \\
&= |x+i(y-1)| \\
&= \sqrt{x^2+(y-1)^2},
\end{align*}
where we have used that $z=x+yi$. Hence we observe that this function is not analytic anywhere since $v=0$ and $u_x$,$u_y$ are not. Therefore we can define
\begin{align*}
f(z) = f(x) = |x-i| = \sqrt{x^2+1},
\end{align*}
which is analytic within a neighborhood of $x\in [-1,1]$ in the complex plane and has singularities at $z=\pm i$. Hence, like in the previous problem, its Bernstein ellipse is defined by
\begin{align*}
\rho = 1+\sqrt{2},
\end{align*}
and the exponential convergence is shown in the next figure.
\begin{figure}[H]
\centering
\includegraphics[scale=0.75]{sqrt(x^2+1).png}\caption{Convergence as $n\rightarrow\infty$ of the Chebyshev interpolant to $f(x)=|x-i|$.}
\end{figure}

\subsection*{Matlab code for this section}
\begin{verbatim}
%% Problem 2 - 8.7 ATAP

rho=1+sqrt(2);
orderAcuracy('sqrt(x^2+1)',50,2,rho)
\end{verbatim}


\newpage
\part{Regression Diagnostics}
\section{Gauss-Markov Theorem}
The Gauss-Markov theorem states that in a linear regression model, where the errors
\begin{itemize}
	\item have expectation 0
	\item are uncorrelated
	\item have equal variances
\end{itemize}
the \textbf{Best Linear Unbiased Estimator (BLUE)} of the coefficients is given by the \textbf{Ordinary Least Square (OLS)} estimator.
\\ \ \\
Properties of the BLUE:
\begin{itemize}
	\item Unbiased: $E(\hat{\beta}) = \beta$
	\item Consistent: $n \uparrow$, $var(\hat{\beta}) \downarrow$
	\item Efficient: $var(\hat{\beta}) < var(\tilde{\beta})$, it gives the \textbf{lowest variance} compared to other linear unbiased estimators.
\end{itemize}

\section{Gauss-Markov Assumptions/Requirements}
The OLS estimator is the best linear unbiased estimator (BLUE), iff
\begin{figure}[H]
	\centering
	\includegraphics[width=0.9\textwidth]{gauss-markov.png}
\end{figure}
\subsection{Linearity}
\begin{itemize}
	\item Definition: Linear relationship in \textbf{coefficients} $\beta_i$.
	\item Why: core assumption of linear regression.
	\item Solution to non-linearity: Transform the \textbf{predictors or response}(logarithmic, interval-wise)
	\item Influence Factor: \textbf{outliers}.

	Reasons for outlier:
	\begin{itemize}
		\item error in recording the value
		\item point doesn't belong to the sample
		\item no error, it's an valid observation
	\end{itemize}
	Solution to outlier:
	\begin{itemize}
		\item identify outliers $\rightarrow$ exclude
		\item apply ''robust'' regression
	\end{itemize}
\end{itemize}

\subsection{No Multicollinearity}
\begin{itemize}
	\item Definition: \textbf{No} linear dependency between the \textbf{predictors} $X_i$

	$\rightarrow$ rank(X) = p (the data matrix has \textbf{full rank} = number of columns)

	$\rightarrow$ \textbf{No high correlation} between predictors (though full rank).

	\item Testing:
	\begin{itemize}
		\item \textbf{Correlation Coefficient} between predictor \textbf{pairs}
		\item \textbf{Variance Inflation Factor (VIF)}: correlation among \textbf{multiple predictors}
		$$VIF = \frac{1}{1-R^2_k}$$
		\paragraph{Interpretation of VIF}

		When the predictor $X_k$ is set as the dependent variable, $R^2_k$ of the variance in the predictor $X_k$ can be explained by the rest of other predictors.

		eg: VIF = 10. $R^2_k$ = 90\%. 90\% of the variance in the predictor $X_k$ can be explained by the rest of other predictors.

		\paragraph{Rule} VIF $\uparrow$, Multicollinearity $\uparrow$. Remove predictor $X_k$ if $\mathbf{VIF >10}$

	\end{itemize}

	\item Consequences of Multicollinearity: Non-significance of the coefficients

	The coefficient has a small t-value/large p-value (can't reject $H_0$)
	\begin{itemize}
		\item small VIF: predictor $X_i$ is not related to response

		$\rightarrow$ remove the variable $X_i$
		\item large VIF: predictor $X_i$ is highly correlated to some other predictors.

		$\rightarrow$ \textbf{correlation matrix}: remove one of the highly correlated variables (near 1 or -1)
	\end{itemize}

	\item Example R-Interpretation:
	\begin{figure}[H]
		\centering
		\includegraphics[width=0.6\textwidth]{vif1.png}
	\end{figure}
	\begin{figure}[H]
		\centering
		\includegraphics[width=0.6\textwidth]{vif2.png}
	\end{figure}
	large p-value, large VIF $\rightarrow$ high correlation among predictors. Call correlation matrix.

	Find out and remove one of the highly correlated predictors.
	\begin{figure}[H]
		\centering
		\includegraphics[width=0.6\textwidth]{vif3.png}
	\end{figure}
\end{itemize}

\subsection{Homoscedasticity}
\begin{itemize}
	\item Definition: \textbf{each residual $\sigma_i$} of predictor $X_i$ exhibit \textbf{constant} variance.

	$\rightarrow$ the spread of residual in different predictors remains nearly the same.

	$\rightarrow$ no systematic development(grows larger/smaller) of residuals -- \textbf{Heteroscedasticity}

	\item Testing:
	\begin{itemize}
		\item \textbf{Breusch-Pagan Test}
		\item \textbf{White Test}
		\begin{itemize}
			\item $H_0$: all variances $\sigma_i$ are equal (homoscedasticity)

			$H_1$: heteroscedasticity
			\item Distribution: $\chi^2$-Distribution
			\item reject $H_0$ if $p < \alpha$
		\end{itemize}
	\end{itemize}

	\item Consequence of Heteroscedasticity:
	\begin{itemize}
		\item estimated variance of coefficients $Var(\hat{\beta})$ is \textbf{biased}.
		\item OLS Estimator no longer efficient.
		\item Some predictors has more ''weight'' than others $\rightarrow$ higher sensitivity
	\end{itemize}
\end{itemize}

\subsection{No Autocorrelation}
\begin{itemize}
	\item Definition: no correlation between the $i^{th}$ and $j^{th}$ \textbf{overall residual}

	$\rightarrow$ $Cor(\varepsilon_i, \varepsilon_j) = 0$

	$\rightarrow$ \textbf{no pattern} of residuals should be observed over time, in case of \textbf{time series data}.

	\item Testing:

	The significance test of coefficients might say they are significant from 0. However, Autocorrelation detected.
	\begin{itemize}
		\item visualize residuals against time
		\item \textbf{Durbin-Watson statistic [0,4]}: test for first-order autocorrelation
		$$DW = \frac{\Sigma_{i=2}^n (e_i - e_{i-1})^2}{\Sigma_{i=1}^n e_i^2}$$
		\paragraph{Interpretation of DW-statistic}
		\begin{itemize}
			\item DW = 2: \textbf{no} autocorrelation

			Rule of thumb: DW $\in$ [1.5, 2.5] $\rightarrow$ no serial correlation
			\item DW = 0: perfect \textbf{positive} autocorrelation
			\item DW = 4: perfect \textbf{negative} autocorrelation
		\end{itemize}
	\end{itemize}

	\item Consequences for autocorrelation:
	\begin{itemize}
		\item an important predictor is omitted (which explains the pattern over time)
		\item functional misfit
		\item measurement error in predictors
	\end{itemize}

	\item Solution to autocorrelation: Model the missing predictor
	\begin{itemize}
		\item overall trend in time: t
		\item dummy variable for seasonal indexes $Q_1$, $Q_2$, $Q_3$

		number of dummy variables: \textbf{number of choices -1} $\rightarrow$ avoid multicollinearity
	\end{itemize}

	Example Model:
	$$y = \beta_0 + \beta_1\cdot t + \beta_2 \cdot Q_1 + \beta_3 \cdot Q_2 + \beta_4 \cdot Q_3$$
\end{itemize}

\subsection{Exogeneity}
\begin{itemize}
	\item Definition: the expected value of \textbf{residual vector} given all predictors is 0.

	$\rightarrow$ $E(\varepsilon|X) = 0$, $Cov(\varepsilon, X) = 0$

	\item Consequences for Endogeneity(not exogene):
	\begin{itemize}
		\item measurement error
		\item predictors and response effect each other mutually
		\item \textbf{important predictors are omitted} $\rightarrow$ bias in estimation of coefficients
	\end{itemize}

	\item Testing for individual effects: \textbf{Lagrange Multiplier Test}, in R ''plmtest(model)''
	\begin{itemize}
		\item $H_0$: No individual effects
	\end{itemize}
	\item Testing for fixed or random effect model when Lagrange Multiplier Test fails: \textbf{Hausman Test}
	\begin{itemize}
		\item $H_0$: random effect estimator is consistent \& efficient $\rightarrow$ \textbf{random effect model}

		$H_1$: \textbf{fixed effect model} needed.
	\end{itemize}

	\item Solution to endogeneity due to omitted variable bias:

	according to types of data:
	\paragraph{cross-section data} data observing many objects at the same time

	difficult to find out the confounding variables $\rightarrow$ \textbf{no solution}
	\paragraph{panel data} repeated observations on the same objects over time. Mostly unbalanced panel data, where some individuals are not recorded in all time period.

	$\rightarrow$ individual-specific panel data structure

	$\rightarrow$ find out the omitted individual-specific effects on the response.

	Solution:
	\begin{itemize}
		\item \textbf{Fixed Effects Model}:
		Individual/Entity-specific effects are \textbf{correlated} to other predictors

		$\rightarrow \lambda_i$ is constant, can be seen as \textbf{an additional intercept} for each individual $i$ in regression model.

		$$y_{it} = (\beta_0 + \lambda_i) + \beta_1 x_{1it} + \beta_2 x_{2it} + \dots + \beta_k x_{kit} + \varepsilon_{it}$$

		Estimators for the fixed effect models: first differences, within, least square dummy variable
		\item \textbf{Random Effects Model}:
		Individual/Entity-specific effects are \textbf{uncorrelated} to other predictors

		$\rightarrow \lambda_i$ is drawn independently, can be seen as \textbf{an element of residual} for each individual in regression model.
		$$y_{it} = \beta_0 + \beta_1 x_{1it} + \beta_2 x_{2it} + \dots + \beta_k x_{kit} + \underbrace{(\lambda_i + u_{it})}_{\varepsilon_{it}}$$
	\end{itemize}
\end{itemize}


\lab{1-d Optimization}{1-d Optimization}
\label{lab:1-dOpt}
\objective{To help students be comfortable with all the techniques of 1-d optimization. This will prepare them for later line-search methods in higher-dimensional optimization.}

\section*{Overview of 1-d Optimization}
Imagine you are out hiking on a mountain, and you lose track of the trail. Thick fog
gathers around, reducing visibility to just a couple of feet. You decide it is time
to head back home, which is located in the valley located near the base of the mountain.
How can you find your way back with such limited visibility? The obvious way might be to
pick a direction that leads downhill, and follow that direction as far as you can, or
until it starts leading upward again. Then you might choose another downhill direction,
and take that as far as you can, repeating the process. By always choosing a downhill
direction, you hope to eventually make it back to the bottom of the valley, where you live.

This is the basic approach of line search algorithms for numerical optimization.
Suppose we have a real-valued function $f$ that we wish to minimize. Our goal is to find the
point $x^*$ in the domain of $f$ such that $f(x^*)$ is the smallest value in the range of
$f$. For some functions, we can use techniques from calculus to analytically obtain this
minimizer. However, in practical applications, this is often impossible, especially when
we need a system that works for a wide class of functions. A line search algorithm starts with
an initial guess at the minimizer, call it $x_0$, and iteratively produces a sequence of
points $x_1, x_2, x_3, \ldots$ that hopefully converge to the minimizer $x^*$. The basic
iteration to move from $x_k$ to $x_{k+1}$ involves two steps: first, choosing a search direction $p_k$
in which to proceed from the current point, and second, specifying a step size $\alpha_k$ to travel
in this direction. The next point is determined by the formula
$$
x_{k+1} = x_k + \alpha_kp_k.
$$
This procedure is called a line search because at each iteration, we are simply examining the
function in a particular linear direction. The choice of the step size $\alpha_k$ is often
chosen by solving a one-dimensional optimization problem in the given direction. In this lab,
we will discuss approaches to choosing the step size and the search direction.

Line search procedures are an integral part of many nonlinear optimization techniques.
In some sense, they represent the simplest nontrivial case in general optimization, as
we only have to worry about one parameter. And yet far more sophisticated optimization
algorithms really crucially on the effectiveness and efficiency of line searches, since
higher-dimensional problems are often broken down into one-dimensional optimizations.
There are many different line search methods, and their effectiveness depends very much
on the nature of the optimization problem. Although the line search procedure is often
only a subroutine of the optimization algorithm at hand, understanding the basics of
the line search is necessary for understanding the robustness of the entire algorithm.
\section*{Optimizing Functions on the Real Numbers}
\subsection*{Derivative versus Derivative-Free Methods}
As you have seen in calculus classes, the derivative of a function gives information
about how the value of the function changes at each point, and can be used to determine
local optima. However, not all objective functions are differentiable, so we need other
techniques at our disposal. Line search methods may be broadly separated into two groups
based on whether they use the derivative of the objective function. We discuss two
simple examples to illustrate this distinction.

\subsection*{Golden Section Search}
This method is appropriate when minimizing a real-valued function on the reals over a
closed interval. The function must further satisfy the \emph{unimodal} property, i.e.
it has just one local minimum, and is monotonic to the left and right of the minimum.
The goal, of course, is to find the
global minimum. We do this by making a sequence of guesses that we hope will converge
quickly to the minimum. Although we may not end up with the exact minimum, this method
will allow us to pin down the true minimum within an interval of any given width in a
finite number of steps.

For the Golden Section Search, each step consists of evaluating the function at two
points within the current interval, comparing these values, and then reducing the size
of the interval for the next step. Let us consider a typical step in the algorithm. At
the outset, we have our function $f$ and a closed interval $[a, b]$ over which we seek
to minimize $f$. Choose two points $a'$ and $b'$ within the interval, and assume that
$a' < b'$. Now calculate $f(a')$ and $f(b')$, and assume that $f(a') \geq f(b')$.
Because of the unimodal condition, we now know that the minimizer must be in the
interval $[a', b]$, for otherwise the function $f$ would have a local minimum in
both $[a, a']$ and $[a', b]$. In the next step, we repeat the process over the interval
$[a', b]$. If instead we had $f(b') \geq f(a')$, then we choose the interval $[a, b']$
for the next step, and if the two values are equal, then it does not matter which
interval is chosen.

We now have the basic description of the algorithm, but how do we choose the two test
points $a'$ and $b'$? There is in fact an optimal choice, which reduces the amount of
work we have to do. Given an interval $[a, b]$, choose $a'$ and $b'$ satisfying
\begin{align*}
a' &= a + \rho(b - a) \\
b' &= a + (1 - \rho)(b - a),
\end{align*}
where $\rho = \frac{1}{2}(3 - \sqrt{5}) \approx 0.382$. By choosing these particular
points, we need to only evaluate the function at one additional point in the next step.
To demonstrate this fact, the reader may verify that, within the interval $[a, b']$,
the point $a'$ already satisfies the equation
\begin{equation*}
a' = a + (1 - \rho)(b' - a),
\end{equation*}
and so we need only evaluate the function at the point $c$ satisfying
\begin{equation*}
c = a + \rho(b' - a).
\end{equation*}
(The constant $\rho$ is not difficult to derive, and is related to the famous Golden Ratio, hence the name of this algorithm.)

At each step, the interval is reduced by a factor of $1-\rho$, which means that after
$n$ steps, we have pinned down the minimizer to within an interval approximately
$(0.61803)^n$ times the length of the original interval. Note that this convergence is
independent of the objective function.

\begin{problem}
Implement Golden Section Search as described above. Use this to minimize $e^x - 4x$
on the interval $\lbrack 0, 3 \rbrack$. How many steps do you need to take to get
within $.001$ of the true minimizer? Check that with the sentence preceding this
problem.
\end{problem}

\subsection*{One-Dimensional Newton's Method}
Let us first start out with a basic task: minimizing a function of one variable.
We will use a popular approach known as Newton's Method, which is a basic line search
algorithm that uses the derivatives of the function to select a direction and
step size.

To use this method, we need a real-valued function of a real variable that is twice
differentiable. The idea is to approximate the function with a quadratic polynomial and
then solve the trivial problem of minimizing the polynomial. Doing so in an iterative
manner can lead us to the actual minimizer. Let $f$ be a function satisfying the
appropriate conditions, and let us make an initial guess, $x_0$. The relevant quadratic
approximation to $f$ is
\begin{equation*}
q(x) = f(x_0) + f'(x_0)(x-x_0) + \frac{1}{2}f''(x_0)(x-x_0)^2,
\end{equation*}
or just the second-degree Taylor polynomial for $f$ centered at $x_0$. The minimum
for this quadratic function is easily found by solving $q'(x) = 0$, and we take the
obtained $x$-value as our new approximation. The formula for the $(n+1)$-th
approximation, which the reader can verify, is
\begin{equation*}
x_{n+1} = x_n - \frac{f'(x_n)}{f''(x_n)}.
\end{equation*}
In the one dimensional case, there are only two search directions: to the right ($+$)
or to the left ($-$). Newton's method chooses the search direction
$\text{sign}(-f'(x_n)/f''(x_n))$ and the step size $|f'(x_n)/f''(x_n)|$.

As is typical with optimization algorithms, Newton's Method generates a sequence of
points or successive approximations to the minimizer. However, the convergence
properties of this sequence depend heavily on the initial guess $x_0$ and the function
$f$. Roughly speaking, if $x_0$ is sufficiently close to the actual minimizer, and if
$f$ is well-approximated by parabolas, then one can expect the sequence to converge
quickly. However, there are cases when the sequence converges slowly or not at all.
See Figure \ref{linesearch:newton}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{newton.pdf}
\caption{The results of Newton's Method using two
different initial guess. The global minimizer was
correctly found with initial guess of 1. However,
an initial guess of 4 led to only a local minimum.}
\label{linesearch:newton}
\end{figure}

\begin{problem}
Implement Newton's Method as described using the following function declaration.
\begin{lstlisting}
def newton1d(f, df, ddf, x, niter=10):
    '''
    Perform Newton's method to minimize a function from R to R.

    Parameters
    ----------
    f : callable function object
        The objective function (twice differentiable)
    df : callable function object
        The first derivative
    ddf : callable function object
        The second derivative
    x : float
        The initial guess
    niter : integer
        The number of iterations

    Returns
    ------
        min : float
            The approximated minimizer
    '''
    pass
\end{lstlisting}

Use this function to minimize $x^2 + \sin(5x)$ with an initial guess of $x_0 = 0$.
Now try other initial guesses farther away from the true minimizer, and note when the
method fails to obtain the correct answer.
\end{problem}

\section*{General Line Search Methods}
\subsection*{Step Size Calculation}
We now examine Line Search methods in more generality. Given a differentiable function
$f : \mathbb{R}^n \rightarrow \mathbb{R}$ that we wish to minimize, and assuming that
we already have a current point $x_k$ and direction $p_k$ in which to search, how do we
choose our step size $\alpha_k$? If our step size is too small, we will not make good progress
toward the minimizer, and convergence will be slow. If the step size is too large, however,
we may overshoot and produce points that are far away from the solution.
A common approach to pick an appropriate step size involves the \emph{Wolfe conditions}:

\begin{align*}
&f(x_k + \alpha_kp_k) \leq f(x_k) + c_1\alpha_k\nabla f_k^Tp_k, &(0 < c_1 < 1),
\\ &\nabla f(x_k + \alpha_kp_k)^Tp_k \geq c_2\nabla f_k^Tp_k, &(c_1 < c_2 < 1).
\end{align*}

Here, we use the shorthand notation $\nabla f_k$ to
mean the gradient of $f$ evaluated at the point $x_k$. The search direction $p_k$ is
often required to satisfy $p_k^T \nabla f_k < 0$, in which case it is called a
\emph{descent direction}, since the function is guaranteed to decrease in
this direction. Generally speaking, choosing a step size $\alpha_k$ satisfying these conditions
ensures that we achieve sufficient decrease in the function and also that we do not
terminate the search at a point of steep decrease (since then we could achieve even
better results by choosing a slightly larger step size). The first condition is known
as the \emph{Armijo} condition.

Finding such a step size satisfying these conditions is not always an easy task, however.
One simple approach, known as \emph{backtracking}, starts with an initial step size
$\alpha$, and repeatedly scales it down until the Armijo condition is satisfied.
That is, choose $\alpha >0, \rho \in (0, 1), c\in (0, 1)$, and while
$$
f(x_k + \alpha p_k) > f(x_k) + c\alpha\nabla f_k^Tp_k,
$$
re-scale $\alpha := \rho\alpha$. Once the loop terminates, set $\alpha_k = \alpha$. Note that the value
$\nabla f_k^Tp_k$ remains fixed for the duration of the backtracking algorithm, and hence need only
be calculated once at the beginning.

\begin{problem}
Implement this backtracking algorithm using the following function declaration.
\begin{lstlisting}
def backtracking(f, slope, x, p, a=1, rho=.9, c=10e-4):
    '''
    Perform a backtracking line search to satisfy the Armijo condition.

    Parameters
    ----------
    f : callable function object
        The objective function
    slope : float
        The value of grad(f)^T p
    x : ndarray of shape (n,)
        The current iterate
    p : ndarray of shape (n,)
        The current search direction
    a : float
        The intial step length (set to 1 in Newton and quasi-Newton methods)
    rho : float
        A number in (0,1)
    c : float
        A number in (0,1)

    Returns
    -------
        alpha : float
            The computed step size satisfying the Armijo condition.
    '''
    pass
\end{lstlisting}
\end{problem}

\begin{problem}
Implement the Gradient Descent algorithm and Newton's Method using the following function declarations.
In each function, you should call your backtracking function with
values $\alpha = 1, \rho = .9$, and $c = 10^{-4}$. The \li{scipy.linalg} module
may be useful when computing the search direction in Newton's Method.
\begin{lstlisting}
def gradientDescent(f, df, x, niter=10):
    '''
    Minimize a function using gradient descent.

    Parameters
    ----------
    f : callable function object
        A differentiable real-valued function
    df : callable function object
        The gradient of the function
    x : ndarray of shape (n,)
        The initial point
    niter : integer
        The number of iterations to run.

    Returns
    -------
        pts: list of ndarrays
            The sequence of points generated
    '''
    pass

def newtonsMethod(f, df, ddf, x, niter=10):
    '''
    Minimize a function using Newton's method.

    Parameters
    ----------
    f : callable function object
        Real-valued, twice-differentiable function
    df : callable function object
        The gradient of the function
    ddf : callable function object
        The Hessian of the function
    x : ndarray of shape (n,)
        The initial point
    niter : integer
        The number of iterations

    Returns
    -------
    pts : list of ndarrays
        The sequence of points generated
    '''
    pass
\end{lstlisting}

\end{problem}

\subsection*{Line Search in SciPy}
The SciPy module \li{scipy.optimize} contains implementations of various optimization algorithms,
including several line search methods. In particular, the module provides a useful routine for
calculating a step size satisfying the Wolfe Conditions described above, which is more robust
and efficient than our simple backtracking approach. We recommend its use for the remainder of
this lab. The function is called \li{line_search}, and accepts several arguments. We can typically
leave the keyword arguments at their default values, but we do need to pass in the objective
function, its gradient, the current point, and the search direction. The following code gives
an example of its usage, using the objective function $f(x, y) = x^2+4y^2$.
\begin{lstlisting}
>>> import numpy as np
>>> from scipy.optimize import line_search
>>>
>>> def objective(x):
>>>     return x[0]**2 + 4*x[1]**2
>>>
>>> def grad(x):
>>>     return 2*x*np.array([1, 4])
>>>
>>> x = np.array([1., 3.]) #current point
>>> p = -grad(x)           #current search direction
>>> a = line_search(objective, grad, x, p)[0]
>>> print a
0.125649913345
\end{lstlisting}
Note that the function returns a tuple of values, the first of which is the step size. We have illustrated
the very basic use of this function. See the documentation for further uses.


%================================
\section{Boundaries}
%================================

%--------------------------------
\begin{definition}
	[boundaries]
	\label{def: boundaries}
	Let $A$ be any set, the \textit{boundary} of $A$, denoted $\partial A$, is defined to be the complement of the interior of $A$ in the closure of $A$; i.e.,
	$$
	\partial A = \overline A \setminus A^\circ.
	$$
\end{definition}
%--------------------------------

%--------------------------------
\begin{proposition}
	[properties of boundaries]
	\label{prop: properties of boundaries}
	Let $(X, \mathcal T)$ be a topological space, and let $A \subseteq X$.
	\begin{enumerate}[(i)]
		\item
		$\partial A$ is closed.

		\item
		$A^\circ \cap \partial A = \emptyset$.

		\item
		$\overline A = A^\circ \cup \partial A$.

		\item
		$A$ is closed iff $\partial A \subseteq A$.

		\item
		$\partial A$ is nowhere dense.

		\item
		$\partial \overline A \subseteq \partial A \subseteq \partial A^\circ$.

		\item
		$\partial A = \partial (X \setminus A)$.

		\item
		$A$ is dense iff $\partial A = X \setminus A^\circ$.

	\end{enumerate}

	\begin{proof} \
		\begin{enumerate}[(i)]
			\item
			$\overline A$ is closed, and $X \setminus A^\circ$ is also closed. Thus
			$$
			\partial A = \overline A \setminus A^\circ = \overline A \cap (X \setminus A)
			$$
			is closed.

			\item
			By Definition \ref{def: boundaries}, we have
			$$
			\begin{aligned}
				\partial A = \overline A \setminus A^\circ &\iff \partial A \cap A^\circ = \overline A \setminus A^\circ \cap A^\circ = \overline A \cap \emptyset = \emptyset.
			\end{aligned}
			$$

			\item
			We have
			$$
			\begin{aligned}
				\partial A = \overline A \setminus A^\circ &\iff \partial A \cup A^\circ = \overline A \setminus A^\circ \cup A^\circ = \overline A \cap (X \setminus A^\circ \cup A^\circ) \\
				&\iff \partial A \cup A^\circ = \overline A \cap X |_\text{for $A^\circ \subseteq X$} = \overline A.
			\end{aligned}
			$$

			\item As $A$ is closed, $A = \overline A$ (this can be straightly proved by Definition \ref{def: closure}). By Definition \ref{def: boundaries}, it is clear that $\partial A \subseteq \overline A$, thus $\partial A \subseteq A$.

			\item
			By Definition \ref{def: nowhere dense sets}, $\partial A$ is nowhere dense iff $\overline{\partial A}^\circ$ is empty. We have
			$$
			\begin{aligned}
				\overline{\partial A}^\circ &= \overline{\overline A \setminus A^\circ}^\circ \\
				&= (\overline A \setminus A^\circ) \cup (\overline A \setminus A^\circ) \setminus (\overline A \setminus A^\circ) \\
				&= \emptyset.
			\end{aligned}
			$$

			\item
			$\overline A \supseteq A^\circ$ implies $\overline A^\circ \supseteq (A^\circ)^\circ = A^\circ$, then we have,
			$$
			\begin{aligned}
				\partial \overline A &= \overline{\overline A} \setminus  \overline A^\circ \subseteq \overline A \setminus A^\circ = \partial A.
			\end{aligned}
			$$

			$A^\circ \subseteq A$ implies $ \overline{A^\circ} \subseteq \overline A$, then we have,
			$$
			\begin{aligned}
				\partial A^\circ = \overline{A^\circ} \setminus (A^\circ)^\circ \supseteq \overline A \setminus A^\circ.
			\end{aligned}
			$$

			\item
			We have
			$$
			\begin{aligned}
				\partial (X \setminus A) &= \overline{X \setminus A} \setminus (X \setminus A)^\circ \\
				&= X \setminus A^\circ \setminus (X \setminus \overline A) \\
				&= X \setminus A^\circ \cap \overline A \\
				&= \overline A \setminus A^\circ \\
				&= \partial A.
			\end{aligned}
			$$

			\item
			By Definition \ref{def: dense sets}, $A$ is dense  in $X$ iff $\overline A = X$. Then we have,
			$$
			\begin{aligned}
				\overline A = X &\iff \overline A \setminus A^\circ = X \setminus A^\circ \\
				&\iff \partial A = X \setminus A^\circ.
			\end{aligned}
			$$
		\end{enumerate}
	\end{proof}
\end{proposition}
%--------------------------------


\section{Some Point Set Topology}

\subsection{Naive Set Theory}

\subsubsection{Theorem 1}
We only prove (10) to illustrate the technique; the other parts are either immediate
from definitions or can be proved similarly.

(10)
\begin{align*}
        &x \in \bigg(\cup_{S_i \in \mathcal{S}_1} S_i\bigg) \cap \bigg(\cup_{S_i \in \mathcal{S}_2} S_i\bigg) \\
        \implies &\exists i, j \: | \: x \in S_i \cap S_j \implies x \in \cup_{S_i \in \mathcal{S}_1, S_j \in \mathcal{S}_2} (S_i \cap S_j) \\
                 &x \in \cup_{S_i \in \mathcal{S}_1, S_j \in \mathcal{S}_2} (S_i \cap S_j) \implies \exists i, j \: | \: x \in S_i \cap S_j \\
        \implies &x \in \bigg(\cup_{S_i \in \mathcal{S}_1} S_i\bigg) \cap \bigg(\cup_{S_i \in \mathcal{S}_2} S_i\bigg)
\end{align*}


\chapter{Differentials}

\section{Differentiator}

\index{differentiator}%
The \emph{differentiator}
\(d : (\Real \to \Real) \to (\Real \to \Real)\)
is \(d(f) = \lim_{h\to 0} \frac{f(x+h)-f(x)}{h}\).

If \(d\) is the differentiator and \(f\) is a function,
we will abuse the notation \(df\) to mean \(d(f)\).

Write \(d^{n+1}\) to mean \(d \circ d^n\)
where \(\circ\) is function composition
and \(n \in \Nat\).
Thus \(d^{n+1}(f) = d(d^n(f))\).

\section{Properties}

\index{derivative!linearity}%
\paragraph{Linearity}
If \(c\) is a constant function, then \(d(c \cdot f) = c \cdot df\).
Also, \(d(f+g) = df + dg\).

\index{derivative!product rule}%
\index{product rule}%
\paragraph{Product rule}
\(d(f \cdot g) = df \cdot g + f \cdot dg\).

\index{derivative!chain rule}%
\index{chain rule}%
\paragraph{Chain rule}
\(d(f \circ g) = dg \cdot (df \circ g)\).

\index{derivative!power rule}%
\index{power rule}%
\paragraph{Power rule}
\(d(x \to x^n) = n x^{n-1}\).

\section{Vector differential calculus}

\index{directional derivative}%
\index{derivative!directional}%
\paragraph{Directional derivative}
The derivative of \(f : \Real^n \to \Real^m\) at direction \(v\) at point \(x\) is
\((d_v f)(x) = \lim_{h\to 0}\frac{f(x+h v)-f(x)}{h}\)
where \(h : \Real\).

\paragraph{Partial derivative}
Let \(f : \Real^n \to \Real^m\) and
\(f_k(x) = (f(x))_k\).
The
\index{partial derivative}%
\emph{partial derivative of \(f\) with respect to the \(k\)th variable} is
\((d_k f)(x) = \lim_{h\to 0}\frac{f(x+he_k)-f(x)}{h}\)
where \(h : \Real\) and
\((e_k)_i = [k = i]\) is a standard basis vector.

\index{gradient}%
\paragraph{Gradient}
The \emph{gradient} of \(f\) is the \(\nabla f\)
that satisfies \((\nabla f)(x) \cdot v = (d_v f)(x)\).
It is also written \(\fgrad(f)\).

The symbol \(\nabla\) is called \emph{del} or \emph{nabla}.
Its type is \((\Real^n \to \Real) \to (\Real^n \to \Real^n)\).

\section{Jacobian operator}

The
\index{Jacobian operator}%
\emph{Jacobian operator} \(J : (\Real^n \to \Real^m) \to (\Real^n \to \Real^{m \times n})\)
satisfies:
\[
    [(Jf)(x)]_{ij} = (d_j f_i)(x)
\]
The matrix \((Jf)(x)\) is called the
\index{Jacobian matrix}%
\emph{Jacobian matrix of \(f\) at \(x\)}.

\section{Differential equation}

\index{differential equation}%
A differential equation is something like \(d^2f = -cf\).

\paragraph{Fixed point of \(d\)}
\index{derivative!fixed point}%
Iff \(f(x) = e^x\), then \(df = f\)
where \(e\) is Euler's constant, the base of natural logarithm.


\section*{Problem 4: Naive Bayes: Theory [20 pts] (Brynn)}

Recall the difference in the modeling assumptions in a discriminative and a generative classifier. In a generative model, $P(X,Y)$ is estimated by initially modeling the conditional $P(X|Y)$, since, $P(X,Y) = P(Y)P(X|Y)$. The joint probability with Bayes rule is then used to calculate $P(Y|X)$ for each class label. On the other hand, a discriminative classifier directly estimates $P(Y|X)$. In this problem, we will explore the relation between Naive Bayes and logistic regression classifiers, by focusing on the class conditional $P(Y|X)$.

When Y is Boolean and $X = \langle{X_{1}...X_{n}}\rangle$ is a vector of continuous variables, and each $P(X_i | Y= y_k)$ is modelled with a Gaussian distribution, then the assumptions of the Gaussian Naive Bayes classifier imply that $P(Y\mid{X})$ is given by the logistic function with
appropriate parameters $w_0, w_1, .. w_n$. In particular:

\begin{align*}
    P(Y=1\mid{X})=\frac{1}{1+exp(w_0+\sum_{i=1}^{n} w_iX_i)}
\end{align*}
and
\begin{align*}
    P(Y=0\mid{X})=\frac{exp(w_0+\sum_{i=1}^{n} w_iX_i)}{1+exp(w_0+\sum_{i=1}^{n} w_iX_i)}
\end{align*}

\begin{enumerate}

\item Consider instead the case where Y is Boolean and ${X = \langle{X_{1}...X_{n}}}\rangle$ is a vector of Boolean variables. Show that the derived expression of $P(Y|X)$ has the same form as that in the Logistic Regression classifier model (by getting an appropriate expression, which can be substituted for weights in the $P(Y|X)$ equation for Logistic Regression).

\emph{Hints}
\begin{enumerate}
    \item Simple notation will help. Since the $X_{i}$ are Boolean variables, you need only one parameter to define $P(X_{i}\mid{Y} = y_k)$. Define $\theta_{i1} \equiv P(X_{i} = 1\mid{Y = 1})$, in which case $P(X_{i} = 0\mid{Y = 1})$ = (1-$\theta_{i1}$). Similarly, use $\theta_{i0}$ to denote $P(X_{i} = 1|Y = 0)$.
    \item Notice with the above notation you can represent $P(Xi\mid{Y = 1})$ as follows
\begin{align*}
       P(X_{i}\mid{Y = 1}) = \theta_{i1}^{(X_i)}(1-\theta_{i1})^{(1-X_i)}
\end{align*}
    Note when $X_{i}$ = 1 the second term is equal to 1 because its exponent is zero. Similarly, when $X_{i}$ = 0 the first term is equal to 1 because its exponent is zero.
\end{enumerate}
\end{enumerate}

\newpage