\graphicspath{{sec02/images/}{sec02/code/}}
\lstset{inputpath=sec02/code/}

\begin{frame}{Path}

     Path is the main TikZ essence.

    %  \twocolImg{
\lstinputlisting[linerange={8-8, 11-13}]{pathmy.tex}
% }{skipsmy}

\inclassFrag{what do you think ``(0,0)'', ``(0,1)'' for?}

``(0,0)'', ``(0,1)'' is the simplest coordinate assignment. The (x, y) coordinate in units (typically 1cm)

\inclassFrag{Try to run the code}[4]
The \ccol\path\ is not draw anything by itself!
\end{frame}

\begin{frame}{Draw}\relax
\twocolImg{
\lstinputlisting[linerange={8-8, 14-16}]{drawmy1.tex}
}{drawmy1}

\begin{columns}
\begin{column}{0.45\textwidth}
     \lstinputlisting[linerange={8-8, 14-16}]{drawmy2.tex}
\end{column}
 \begin{column}{0.45\textwidth}
 %nothing
\end{column}
\end{columns}

Use \ccol{\draw} or \ocol\path[draw] to draw
\end{frame}

\begin{frame}{Fill}\relax
\twocolImg{
\lstinputlisting[linerange={8-8, 14-16}]{fillmy.tex}
}{fillmy}

Use \ccol{\fill} or \ocol\path[fill] to fill.

And {\csk \verb|--cycle| to make the path close}
\end{frame}

\begin{frame}{Options}

Let us add some color!

\twocolImg{
\lstinputlisting[linerange={8-8, 14-17}]{fillcolormy.tex}
}{fillcolormy}

new command \ccol\filldraw.

Notice the \textit{key=value} syntax!

\end{frame}

\begin{frame}{Options}

Draw another line

\twocolImg{
\lstinputlisting[linerange={8-8, 14-16}]{drawoptmy.tex}
}{drawoptmy}

\end{frame}

\begin{frame}{Other figures}

\twocolImg{
\lstinputlisting[linerange={9-9, 15-18}]{circlerectmy.tex}
}{circlerectmy}

\end{frame}

\begin{frame}{Nodes}\relax

\twocolImg{
\lstinputlisting[linerange={9-9, 15-17}]{nodemy.tex}
}{nodemy}

\ccol\node\ or \ocol\path[node]. Without optional arguments a node has no border

\end{frame}

\inclassFrame{
\begin{frame}{Draw a snowman!}
\centering
\begin{tikzpicture}

\draw (0, 0) circle[radius=1.5cm];
\draw (0, 2.4) circle[radius=1cm];
\draw (0, 4) circle[radius=0.7cm];
\filldraw[fill=gray] (-0.5, 4.5) rectangle (0.5, 5.5);
\filldraw[fill=orange, draw=orange!50!red] (0.3, 4.1) -- (1.5, 4) -- (0.3, 3.9) -- cycle;
\draw[very thick, ->] (-4, 4.2) -- (-1, 3);
\node at (-4, 4.5) {snowman!};
\path (1, 3) -- (4, 5);
\end{tikzpicture}

\end{frame}
}

\begin{frame}{Vertical and horizontal}\relax
\twocolImg{
\lstinputlisting[linerange={9-9, 15-17}]{curveverthor.tex}
}{curveverthor}

Use {\csk -|} for ``first horizontal, than vertical''. Use {\csk |-} for ``first vertical, than horizontal''

\skfootnote{\tikzc{14.2.2}[152]}

\end{frame}

\begin{frame}{Curves}\relax
\vspace{-0.5cm}
\twocolImg{
\lstinputlisting[linerange={9-9, 15-20}]{curveinout.tex}
}{curveinout}
\footnotesize
\ccol{to [out=.., in=..]} the angle on which curve flows out and the angle on which curve flows in.

\twocolImg{
\lstinputlisting[linerange={9-9, 15-19}]{curveibez.tex}
}{curveibez}

\ccol{ .. controls <coord> and <coord> .. }

\skfootnote{\url{https://en.wikipedia.org/wiki/Bezier_curve}}

\end{frame}

\subsection{Coordinates}

\begin{frame}{``standart''}{x,y}\relax
\twocolImg{
\lstinputlisting[linerange={15-21}]{coordinatesSimple.tex}
}{coordinatesSimple}
\ccol{(<x-coord>, <y-coord>)}
\skfootnote{\tikzc{13.2}[133]}
\end{frame}
\begin{frame}{``standart''}{x,y,z}\relax
\twocolImg{
\lstinputlisting[linerange={15-21}]{coordinatesSimple2.tex}
}{coordinatesSimple2}
\ccol{(<x-coord>, <y-coord>, <z-coord>)}
\skfootnote{\tikzc{13.2}[133]}
\end{frame}
\begin{frame}{``standart''}{$\theta$,r}\relax
\twocolImg{
\lstinputlisting[linerange={15-21}]{coordinatesSimple3.tex}
}{coordinatesSimple3}
\ccol{(<$\theta$-coord>:<r-coord>)}
\skfootnote{\tikzc{13.2}[133]}
\end{frame}

\begin{frame}[fragile]{``++'' and ``+=+'' cordinates}\relax
\cprotect\inclassFrag{Write

\begin{lstlisting}
\begin{tikzpicture}
\draw (-3,-3) rectangle (3,3);
\draw (2,0) node{1} -- (1,0) node{2}  -- (0,1)  node{3} -- (-1,0)  node{4} -- cycle;
\end{tikzpicture}
\end{lstlisting}

\begin{lstlisting}
\begin{tikzpicture}
\draw (-3,-3) rectangle (3,3);
\draw (2,0) node{1} -- ++(1,0) node{2}  -- ++(0,1)  node{3} -- ++(-1,0)  node{4} -- cycle;
\end{tikzpicture}
\end{lstlisting}

\begin{lstlisting}
\begin{tikzpicture}
\draw (-3,-3) rectangle (3,3);
\draw (2,0) node{1} -- +(1,0) node{2}  -- +(0,1)  node{3} -- +(-1,0)  node{4} -- cycle;
\end{tikzpicture}
\end{lstlisting}
}[1]
\twocolImg{
\lstinputlisting[linerange={14-21}]{coordinatesPlus.tex}
}{coordinatesPlus}
 \begin{itemize}
     \item ``++'' use relative coordinate and set this new coordinate as ``current''
     \item ``+'' use relative coordinate and DOESN't set this new coordinate as ``current''

 \end{itemize}
\end{frame}

\begin{frame}{``Node''-based}\relax
\twocolImg{
\lstinputlisting[linerange={16-22}]{coordinatesNode.tex}
}{coordinatesNode}

\begin{enumerate}
    \item label node \verb|\node| \ccol{(<label>)}
    \item refer to the node as \ccol{(node cs:name=<label>)}
    \item you can also use things like <label>.west or <label>.right

\end{enumerate}

\end{frame}

\begin{frame}{Coordinate calculation\magicPage}\relax
\twocolImg{
\lstinputlisting[linerange={9-10, 15-20}]{coordinatesCalc.tex}
}{coordinatesCalc}

\begin{enumerate}
    \item \ccol\usetikzlibrary\{calc\}
    \item syntax: {\csk \$<coord1>!fraction!<coord2>\$}
    \item in this slide you can also see \ccol\foreach!

\end{enumerate}

\end{frame}

\begin{frame}{Coordinate intersection\magicPage}\relax
\twocolImg{
\lstinputlisting[linerange={9-10, 15-22}]{coordinatesInter.tex}
}{coordinatesInter}

\end{frame}


\section{AMP: Adversarial Model Perturbation}

\begin{frame}{From Empirical Risk to AMP Loss}

\begin{columns}

\column{0.66\textwidth}

In this paper, we optimize an alternative ``AMP loss''.

\vspace{1em}

\begin{equation*}
\mathcal{L}_\mathrm{ERM}(\boldsymbol{\theta}):=\frac{1}{|D|}\sum_{(\boldsymbol{x},\boldsymbol{y})\in\mathcal{D}}\ell(\boldsymbol{x},\boldsymbol{y};\boldsymbol{\theta})
\end{equation*}

\vspace{1em}

\begin{equation*}
\mathcal{L}_\mathrm{AMP}(\boldsymbol{\theta}):=\max_{\Delta:\Vert\Delta\Vert\le\epsilon}\mathcal{L}_\mathrm{ERM}(\boldsymbol{\theta}+\Delta)
\end{equation*}

\vspace{2em}

\column{0.34\textwidth}

\begin{figure}
\includegraphics[width=.8\textwidth]{figs/loss_example_a.pdf}
\end{figure}
\vspace{-1.5em}
\begin{figure}
\includegraphics[width=.8\textwidth]{figs/loss_example_b.pdf}
\end{figure}
\end{columns}

\vspace{2em}

\end{frame}

\begin{frame}{Training Algorithm}

A mini-batch SGD is used for solving the ``min-max'' problem.

\vspace{-0.5em}

\begin{equation*}
\min_{\boldsymbol{\theta}}\max_{\Delta:\Vert\Delta\Vert\le\epsilon}\mathcal{L}_\mathrm{ERM}(\boldsymbol{\theta}+\Delta)
\end{equation*}

\begin{algorithm}[H]
\SetAlgoVlined
\While{$\boldsymbol{\theta}$ not converged}{
Initialize perturbation $\Delta$ with $\boldsymbol{0}$\;
\For{$n\gets1\text{ to }N$}{
Update $\Delta$ to maximize $\!\mathcal{L}_\mathrm{ERM}(\boldsymbol{\theta}\!+\!\Delta)\!$ via gradient ascent with learning rate $\zeta$\;
\If{$\Vert\Delta\Vert_2>\epsilon$}{
Normalize $\Delta$ to restrict its norm $\Vert\Delta\Vert_2$ to $\epsilon$;
}
}
Update $\boldsymbol{\theta}$ to minimize $\mathcal{L}_\mathrm{ERM}(\boldsymbol{\theta}\!+\!\Delta)$ via gradient descent with learning rate $\eta$\;
}
\caption{Adversarial Model Perturbation Training}
\end{algorithm}

\vspace{1em}

\end{frame}

\begin{frame}[fragile]{Implementation}
Source code: \url{https://github.com/hiyouga/AMP-Regularizer}

\begin{minted}[linenos]{python}
from amp import AMP
optimizer = AMP(model.parameters(), lr=0.1, epsilon=0.5, momentum=0.9)
for inputs, targets in dataset:
    def closure():
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = loss_fn(outputs, targets)
        loss.backward()
        return outputs, loss
    outputs, loss = optimizer.step(closure)
\end{minted}
\end{frame}


\lab{Public Key Cryptography}{RSA}
\objective{Understand the fundamentals of RSA public key cryptography.}

How do you exchange information over unsecure, public channels fo communcations?
Historically, the answer has been to use some sort of cipher to obscure the meaning of a message.
Many ciphers use a secret key which was used to encrypt and decrypt any sent messages.
However, the problem then became how to keep the key secret and known only to the authorized recipients
If the key was obtained by any attackers, then the whole cipher was compromised.

Public key cryptography provides a way for each person to have their own keys.
There is no need to transmit any keys.
There are several public key crypto systems, but one of the most popular is RSA.
In the RSA system, each person has a public key and a private key.
The security of the system is founded in the difficulty of factoring large numbers.

To make RSA feasible, we need the following two results.
\section*{Fast Modular Exponentiation}
How would we compute $8^{3039} \pmod{38}$?
We understand that one possible way of computing it would be
\[
8*8*8*8*\dots (3034 \text{ more times}) *8 \pmod{38}
\]
Calculating $8^8 = 16777216$!
Multiplying by $8$ each time leads to an enormous number which will finally be modded by $38$.
This will yield the answer $8^{3039} \equiv 18 \pmod{38}$.
However, we end up taking the mod of a number that is $2746$ digits!
Imagine if our exponent was hundreds of digits!
We could quickly run out of memory.

We can avoid dealing such huge numbers by doing the mod after each multiplication.
Doing the mod after each multiplication means that the largest number we have to deal with in this problem is 240.
In general, we wouldn't have to work with anything larger than $37^{2}$.
However, if we still have large exponent, the loop doing the multiplication would still have to run $3039$ iterations.

\begin{problem}
Write a function that will naively calculate $b^{e} \pmod{m}$.
Do not use Python's built-in power function.
\label{prob:naivepower}
\end{problem}

We need a faster better way.
The most common method of modular exponentiation is the \emph{right-to-left binary method}.
The essence of this method is to use the binary representation of the exponent.
We can quickly calculate the power up to the nearest power of two of the exponent.
This we simply calculate the remaining part the naieve way.

Calculating a modular exponent in this way leads to a very desirable $O(\log_2 e)$ runtime performance, where $e$ is the exponent.

\begin{problem}
Write a function that will perform modular exponentiation using the right-to-left binary method.
Compare the performance of \ref{prob:naivepower} and your new implementation.
Note that Python's built-in function, \li{pow()}, performs this type of fast modular exponentiation.
We expect you to use it instead of your own implementation in the future.
\label{prob:rlbpower}
\end{problem}

\section*{Fermat's Little Theorem}
An integral part of RSA relies on working with large prime numbers.
Prime numbers are numbers that are only divisible by 1 and the number itself.
In the RSA algorithms, we need to start with two prime numbers, $p$ and $q$.
We multiply these numbers to get $n=pq$ which is our modulus.
However, if $n$ is not that big, we can easily factor it, defeating the whole purpose of RSA.
The security of RSA largely reduces down to how well we choose starting primes.
If we pick primes that are too small, we sacrifice security.
How do we pick large primes?
We can use sieves to generate primes, but doing so would take lots of time and lots of memory.
It would be better if we could choose a number and determine if it is prime or not.

Fortunately, mathematics has developed a few tools that can help us out.
One of the first steps was given by Fermat.
\begin{theorem}[Fermat's Little Theorem]
If $p$ is prime, then for a natural number $a$, $a^{p-1} - 1 \equiv 0 \pmod{p}$
\end{theorem}
But, to use this we have start with a prime number.
However, if we have a candidate for $p$ and it satisifies the condition can we say anything about $p$?
The answer is yes, but we cannot say that $p$ is prime.
We can only say that $p$ might be prime.
\begin{example}
Suppose $p=37$.
We want to know if $p$ is prime.  We choose $a=8$.
\[
8^{36} - 1 \equiv 0 \pmod{37}
\]
We call $8$ the witness number.  Think of as, \emph{$8$ says that $37$ is prime}.
This is the probabilistic part of Fermat's theorem.
We try enough witness numbers until we are convinced that our candidate number is prime.
Trying enough prime numbers, we will eventually be convinced that $37$ is prime, which it is.
\end{example}

\begin{example}
Let's check the primality of $1729$.
We will use $a=145$ as a witness number.
\[
145^{1728} \equiv 1 \pmod{1729}
\]
It appears that $1729$ is prime.
Let's try another witness number, $a=583$.
\[
583^{1728} \equiv 1 \pmod{1729}
\]
Let's try one last witness number, $a=945$.
\[
945^{1728} \equiv 742 \not\equiv 1 \pmod{1729}
\]
We have a problem.
Our candidate is not behaving like we would expect a prime to behave.
But several witnesses verified that our number was prime.  What happened?
Our number, $1729$ is not really prime.  Its prime factors are $7$, $13$, and $19$.
It is a special number called a Carmichael number.
Because of these numbers, we have to be careful with Fermat's Little Theorem.
More stringent primality tests exist, but are beyond the scope of this lab.
\end{example}
We have to remember that the Fermat test is really a test for compositeness, not primality.

\begin{problem}
For any candidate prime number, $n$, all the integers $2 \geq a < n$ can serve as a witness number.
Each of these possible witness numbers will either prove the compositeness of $n$ or give no information.
If we randomly choose a witness number, we will select a number in one of these two sets.
Supposing we chose a good random number generator, we can become convinced to a very high degree that $n$ is likely prime with just a few witness numbers.

Write a function that will return the number of witnesses that prove the compositeness of $n$.
What is the ratio of these witness numbers to all the numbers $2 \geq a < n$?
Suppose we were to choose witness numbers at random.
How many random witness numbers must we choose to be $99$\% sure that $n$ is prime?
Remember that each new witness number increases the confidence that $n$ is probably prime.
\label{prob:prime_confidence}
\end{problem}

\section*{RSA Algorithm}
Bob wants to send a secret message to Alice.
Eve is trying to intercept this message.
Bob and Alice agree to use RSA to exchange the secret message.
Bob and Alice need to generate their keys.
Each generate a pair of keys, a private and a public one.
Both Bob and Alice need to keep their private keys secret.
Anyone with Alice's public key can send her a message that only she can decrypt.
Bob obtains a copy of Alice's pulic key and encrypt his message using it.
Alice decrypts Bob's message using her private key, and reads the message.

RSA involves generating a private key and a public key.

We begin by finding two large prime numbers.
By large, we mean several hundred digits.

RSA can be succinctly summarised as the following operations for encryption and decryption respectively
\[
c \equiv m^e \pmod{n}
\]
\[
m \equiv c^d \pmod{n}
\]

\begin{problem}
Generate a public/private key pair.

We do this by first multiplying the two primes, $p$ and $q$, together to get $n=pq$.
This will act as the modulus for future operations.
Next we need to choose an encryption exponent, $e$, such that $1 < e < \phi(n)$ and $\gcd(e, \phi(n)) = 1$.
Euler's totient function is
Then our decryption exponent, $d$, must naturally be an inverse of $e$ mod $\phi(n)$.
In other words, we need $de \equiv 1 \pmod{\phi(n)}$.
You will need to write your own function that finds the inverse of $e$ mod $\phi(n)$ using the Extended Euclidean Algorithm.
We distribute $(e, n)$ as the public key and need to make sure that $(d, n)$ remains private.
After the keys are created, we have no further need of $p$ or $q$.
\end{problem}

Before we can encrypt and decrypt a text message, we need to convert it to a number first.
It is useful to understand the code below.
\begin{lstlisting}
# Recipe from itertools module in standard library
def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return izip_longest(fillvalue=fillvalue, *args)

def a2i(msg):
    '''Convert an ASCII message to an integer.'''
    # bytearray will give us the ASCII values for each character
    if not isinstance(msg, bytearray):
        msg = bytearray(msg)
    binmsg = []
    # convert each character to binary
    for c in msg:
        binmsg.append(bin(c)[2:].zfill(8))
    return int(''.join(binmsg), 2)

def i2a(msg):
    '''Convert an integer to an ASCII message'''
    # convert to binary first
    binmsg = bin(msg)[2:]
    # we need to pad the message so length is divisible by 8
    binmsg = "0"*(8-(len(binmsg)%8)) + binmsg
    msg = bytearray()
    for block in grouper(binmsg, 8):
        # convert block of 8 bits back to ASCII
        msg.append(int(''.join(block), 2))
    return msg
\end{lstlisting}

\begin{problem}
Encrypt the message: \emph{Simon sells seashells by the seashore.}
Let $p=83285677$ and $q=2848968679$.
Choose $e=65593$.
Verify your encryption is correct by decrypting the message and checking the result.
\end{problem}

\begin{warn}
The security of RSA depends on keeping the private key secret.
If an attacker is able to obtain the private key, then the security of the entire algorithm is compromised.
When sharing keys, be absolutely sure that you are making only the public keys available and securely storing the private keys.
This is especially important with digital signatures since an attacker with a private key can impersonate the owner of that private key.
If a private key is compromised, generating a new RSA key is the only remedy.
\end{warn}

\subsection*{Digital Signatures}
Suppose Alice wants to confirm that Bob was really the sender of a message.
How would she do this?
Using the RSA algorithm, you can easily sign a message to verify authenticity.
This is done by encrypting the message using Alice's public key and then decrypting the result with Bob's private key.
Order is very important.
Switching the two steps will result in the same message, but there are security vulnerablilities doing that way.

Another, more commonly used, method for signing messages is to sign a hash of the message.
Using a known good hash function, we hash the message (before encrypting), sign the hash, and send it along with the encrypted message.
The exchange would be as follows for the message \emph{Hi Alice! Bob.}
Suppose Bob's private and public keys respectively are $(d, n) = (127890504297793865743357757627627807059, 618783157710153965827030630189807545377)$ and $(e, n)=(65323, 618783157710153965827030630189807545377)$.
Alice's private and public keys respectively are $(d,n)=(16630354644972796093082980345041785609, 58576444815385640869376382633234713033)$ and $(e,n)=(54905, 58576444815385640869376382633234713033)$.
First Bob would hash the message.  We will use MD5, but in practice we should a hash algorithm which is considered cryptographically secure.
The hash of the message (including the period symbol) in hexidecimal is: \verb|\x02\xd2\x0b\xe2\x00Vnd:\xbdY\x8c\x1d\x08/M|.
Bob would encrypt the message using Alice's public key obtaining: $1243069665756046897997663464501811921$.
Bob can then sign the hash of the message which results in: $378011416081119417777777120819271568111$.
To sign the hash of the message, he decrypts the hash with his private key (calculates $m^{d} \mod n$).

When Alice recieves Bob's message and signature she can decrypt the message using her private key.
Alice decrypts the message: $1468662431139370234409485782180398$ which translates to \emph{Hi Alice! Bob.}
She can then verify that the message came from Bob by encrypting the signature with Bob's public key.
This yeilds: $3749079344037002869620996734462734157$.
Now she must hash the message that she decrypted (she would need to know that Bob used MD5 for the signature hash).
She compares the hash of the message she decrypted with the hash she obtained by encrypting Bob's purported signature.
If the two hashes are equal, she can be reasonably certain that the message she decrypted indeed came from Bob (since we assume that only Bob has his private key).
She verifies the hashes and they are equal.
If the hashes differed, then Alice would know that either the message was signed with someone else's private key, or the message itself was tampered with after it was signed.
In either event, Alice can choose to ignore the message because she can verify its authenticity.

\section{PyCrypto}
RSA encryption in Python can be accomplished very easily with a library called PyCrypto.
This library contains many secure hash functions, random number generators, and encryption classes.
Many programs use PyCrypto for their security needs.
\begin{warn}
Make certain that you are using the latest version of PyCrypto.
Security software is updated often to fix security vulnerablilities and bugs.
The current version of PyCrypto at the time of writing is version 2.6.1.
\end{warn}

The RSA module in PyCrypto is located in the PublicKey module.
Let's try a small example using PyCrypto.
The library allows us to explicity construct a key, or generate a key automatically.
\begin{lstlisting}
>>> from Crypto.PublicKey import RSA
>>> from Crypto import Random
>>> keypair = RSA.generate(2048) #generate a 2048-bit RSA key
>>> publickey = keypair.publickey()
>>> share_this = publickey.exportkey()
\end{lstlisting}

The RSA encryption and decryption methods on these keys are textbook approaches.
However, to increase security, we will want to pad the messages so every message encrypted with a particular will become exactly as large (in bits) as the key itself.
A commonly used padding algorithm is implemented in PyCrypto in the \li{Crypto.Cipher.PKCS1_OAEP} module.
\begin{lstlisting}
>>> from Crypto.Cipher import PKCS1_OAEP as oaep

# generate a new key from the original RSA key.
# This key can encrypt and decrypt
>>> paddedkey = oaep.new(keypair)
>>> encrypted = paddedkey.encrypt('hello world')
>>> paddedkey.decrypt(encrypted)
'hello world'
\end{lstlisting}

To sign the message we use PyCrypto's \li{Crypto.Signature.PKCS1_PSS} module.
We first need to hash the message using a cryptographically secure hash, such as SHA.
Just as only private keys can decrypt, only private keys can sign.
Public keys can decrypt and verify.
\begin{lstlisting}
>>> from Crypto.Signautre import PKCS1_PSS as pss
>>> from Crypto.Hash import SHA
>>> sigkey = pss.new(keypair)
>>> mhash = SHA.new("Hello World")
>>> sigkey.sign
\end{lstlisting}

\begin{problem}[Group Project]
Split into several groups of at least three individuals.
Each group member should generate a private and public keypair using PyCrypto.
Share your public keys with everyone in the group.

You might find it useful to transmit your messages as strings.
Here are two functions that will read and write specially formatted strings for your messages.
\begin{lstlisting}
def print_msg(m_encrypted):
    #print the encrypted message as a specially formatted string
    out = ("----BEGIN MESSAGE----",
           m_encrypted[0],
           "----END MESSAGE----",
           "----BEGIN SIGNATURE----",
           m_encrypted[1],
           "----END SIGNATURE----")
    return '\n'.join(out)

def read_msg(m_string):
    d1 = m_string.find('----BEGIN MESSAGE----') + 21
    d2 = m_string.find('----END MESSAGE----', d1)

    d3 = m_string.find('----BEGIN SIGNATURE----', d2) + 23
    d4 = m_string.find('----END SIGNATURE----', d3)
    message  = m_string[d1:d2]
    sign = m_string[d3:d4]
    return message.strip(), sign.strip()
\end{lstlisting}

\begin{enumerate}
\item Encrypt and sign a message for someone in the group and send it to the entire group.
Attach the sender's name to the encrypted message (a claim that the message came from a particular sender).
In this exercise we will verify that the actual origin of the message is indeed the claimed origin.
Each group member should do the following:
\begin{enumerate}
\item Decrypt and verify the message addressed for you.
\item Attempt to decrypt and verify message addressed for someone else in the group.
\item Report the message and verified signature.
\end{enumerate}

\item Encrypt and sign a different message for someone in the group and send it, anonymously, to the entire group.
For example, if Bob encrypts a message and sends it to Alice, he should claim the message came from another individual.
Each group member should do the following:
\begin{enumerate}
\item Decrypt and verify the message address to you.
\item Report the message and verified signature.
\end{enumerate}
\end{enumerate}
\end{problem}

The security of RSA relies on the assumption that certain mathematical operations are not easily done.
Breaking RSA reduces down to being able to factor the modulus into its two primes.  This is hard to do for large moduli.
There is no proof of security.  RSA can be broken depending on the laziness of the keyholder, or the cleverness of the attacker.
To keep ahead of advances in computing speed, the RSA key size is increased beyond the capability of existing hardware.
However, larger keys mean that messages will take longer to encrypt and decrypt.
There are a new family of methods that rely on elliptic curves.
These methods can acheive the same security of RSA using smaller key sizes.
For example a $224$ bit elliptic curve key gives comparable security to a $2048$ bit RSA key.


\section{Conditional Probability}

\subsection{Conditional Probability}

\begin{theorem}
    \begin{equation}
        \probability{E} = \begin{cases}
            \displaystyle \sum_y \probability{E|Y = y} p(Y=y) \\
            \displaystyle \int_{-\infty}^\infty f_Y(y) \dif{y}
        \end{cases}
    \end{equation}
\end{theorem}

\subsection{Conditional Expectation}

\begin{definition}
    The \cindex{conditional probability mass function} of $X$ given $Y=y$ is
    \begin{equation}
        p_{X|Y}(x|y) = \dfrac{p(x,y)}{p_Y (y)}
    \end{equation}

    The \cindex{conditional probability density function} of $X$ given $Y=y$ (where $f_Y (y) > 0$) is
    \begin{equation}
        f_{X|Y}(x|y) = \dfrac{f(x,y)}{f_Y (y)} = \dfrac{f(x,y)}{\displaystyle \int_x f(x,y) \dif{x}}
    \end{equation}
\end{definition}

\begin{definition}
    The \cindex{conditional expectation} of $X$ given $Y=y$ is
    \begin{equation}
        \displaystyle \expect{X|Y=y} = \begin{cases}
            \displaystyle \sum_x x \cdot p_{X|Y}(x|y) \\
            \displaystyle \int_{-\infty}^\infty x f_{X|Y} (x|y) \dif{s}
        \end{cases}
    \end{equation}
\end{definition}

Note: $\expect{X|Y}$ is a random variable of $Y$. $Y$ here may be an expression of $X$ as the following example shows.

\begin{example}
    If $X$ and $Y$ are independent variable, calculate the conditional expectation of $X$ given $X+Y=n$.
\end{example}

\begin{theorem}
    \begin{equation}
        \begin{aligned}
            \expect{X} &= \expect{ \expect{X|Y}} \\
            &= \begin{cases}
                \displaystyle \sum_y \expect{X|Y = y} \probability{Y=y} \\
                \displaystyle \int_{-\infty}^\infty \expect{X|Y=y} f_Y (y) \dif{y}
            \end{cases}
        \end{aligned}
    \end{equation}
\end{theorem}

% conditional variance
\subsection{Conditional Variance}

\begin{theorem}
    \begin{equation}
    \begin{aligned}
        \variance{X|Y=y} &= \expect{(X - \expect{X|Y=y})^2 | Y=y} \\
        &= \expect{X^2|Y=y}- \left(\expect{X|Y=y} \right)^2
    \end{aligned}
    \end{equation}
\end{theorem}

\begin{theorem}
 \begin{equation}
        \variance{X} = \expect{\variance{X|Y}} + \variance{\expect{X|Y}}
    \end{equation}
\end{theorem}

% example
\subsection{Example}

\begin{example}
    A miner is trapped in a mine containing 3 doors. The first door needs to a tunnel that takes him to safety after 2 hours. The second door take him back after 3 hours and the third door takes him back after 5 hours. What is the expected length of time until the miner reaches out when he chooses the door equally?
    \qed
\end{example}

\begin{example}
    $n$ men through their hat into the room and randomly select one. They will leave the game if they have selected their own hat.
    \begin{enumerate}
        \item \label{hatq1} what is the probability of no match in first round?
        \item \label{hatq2} what is the probability of k match in first round?
        \item \label{hatq3} what is the expected match in first round?
        \item \label{hatq4} what is the variance of match in first round?
        \item \label{hatq5} what is the expected number of rounds?
        \item \label{hatq6} what is the variance of rounds?
        \item \label{hatq7} what is the expected total number of selection?
        \item \label{hatq8} what is the expected number of wrong selection for each man?
    \end{enumerate}
\end{example}
\begin{proof}
    define these variables:
    \begin{itemize}
        \item $R_n$: the number of rounds necessary for $n$ men.
        \item $S_n$: the total number of all selection.
        \item $C_i$: the number of all wrong selection by $i$th man.
        \item $X_n$: the number of matches in the first round.
    \end{itemize}

    For question (\ref{hatq1}) let $E_n$ be the event that no match occurs in $n$ men scenario, $M$ be the event that the first man select its own hat. We have
    \begin{equation*}
        \probability{E_n} = \probability{E_n|M} \probability{M} + \probability{E_n|M^c} \probability{M^c}
    \end{equation*}

    Because $\probability{E|M} = 0$, $\probability{E_n} = \dfrac{n-1}{n} \probability{E_n|M^c} $.

    In $M^c$ case, the first man selected the hat of another person. there are two possibilities on whether the other person select the hat of the first man:
    \begin{enumerate}
        \item the "another" man did not select the hat of first man. In this case, $\probability{E_{n}|?} = \probability{E_{n-1}}$.
        \item the "another" man did select the hat of the first man. In this case, $\probability{E_{n}|?} = \probability{E_{n-2}} \probability{?} = \dfrac{1}{n-1} \probability{E_{n-2}}$.
    \end{enumerate}

    So
    \begin{equation*}
        \probability{E_n} = \probability{E_{n-1}} + \frac{1}{n-1} \probability{E_{n-2}}
    \end{equation*}

    and $\probability{E_n} = \displaystyle \sum_{i=2}^n \frac{(-1)^i}{n!}$

    For question (\ref{hatq2}), for any fixed group of $k$ men that select their own hat, the probability is
    \begin{equation*}
        \frac{1}{n} \frac{1}{n-1} \cdots \frac{1}{n-(k-1)} \probability{E_{n-k}} = \frac{(n-k)!}{n!} \probability{E_{n-k}}
    \end{equation*}

    Because there are $\binom{n}{k}$ choices of $k$ men, the result is
    \begin{equation*}
        \frac{(n-k)!}{n!} \probability{E_{n-k}} \binom{n}{k} = \frac{1}{k!} \sum_{i=2}^{n-k} (-1)^{n-k} \frac{1}{i!}
    \end{equation*}

    Another way is to calculate the length $C$ of the cycle that contains the first man. we have

    \begin{equation*}
    \begin{aligned}
        \probability{E_n} &= \sum_{k=1}^n \probability{E_n|C=k} \probability{C=k} \\
        &= \sum_{k=1}^n \probability{E_{n-k}} \probability{C=k}
    \end{aligned}
    \end{equation*}

    We have
    \begin{equation*}
        \probability{C=k} = \frac{n-1}{n} \frac{n-2}{n-1} \cdots \frac{n-k+1}{c-k+2} \frac{1}{n-k+1} = \frac{1}{n}
    \end{equation*}

    So $\displaystyle \probability{E_n} = \frac{1}{n} \sum_{k=2}^n \probability{E_{n-k}}$.

    For equestion question (\ref{hatq3}), the probability $i$th man select its hat is $\probability{H_i} = \dfrac{1}{n}$, so
    \begin{equation*}
        \begin{aligned}
            \expect{\sum_i H_i} &= \sum_i \expect{H_i} \\
            &= \sum_i \frac{1}{n} \\
            &= 1
        \end{aligned}
    \end{equation*}

    For question (\ref{hatq4}), the $\variance{ \sum_i H_i} = 1$

    For question (\ref{hatq5}),
    \begin{equation*}
    \begin{aligned}
        \expect{R_n} &= \sum_{i=0}^n \expect{R_n | X_n = i} \probability{X_n = i} \\
        &= \sum_{i=0}^n (1 + \expect{R_{n - i}}) \probability{X_n = i} \\
        &= 1 + \expect{R_n} \probability{X_n = 0} + \sum_{i=0}^n \expect{R_{n-i}} \probability{X_n=i} \\
        &= \expect{R_n} \probability{X_n = 0} + n(1 - \probability{X_n = 0} )
    \end{aligned}
    \end{equation*}

    So the only solution is $\expect{R_n} = n$.

    For question (\ref{hatq6}), for the variance, we have
    \begin{equation*}
        \expect{R_n|X} = 1 + \expect{R_{n-X}} = 1 + n - X
    \end{equation*}

    Also we have $\variance{R_n|X} = \variance{R_{n-X}}$. So we have
    \begin{equation*}
        \begin{aligned}
            \variance{R_n} &= \expect{\variance{R_n|X}} + \variance{\expect{R_n|X}} \\
            &= \expect{\variance{R_{n-X}}} + \variance{X} \\
            &= \sum_{i=0}^n \variance{R_{n-j}} \probability{X=j} + \variance{X} \\
            &= \variance{R_n}\probability{X = 0} + \sum_{j=1}^n \variance{R_{n-j}} \probability{X=j} + \variance{X}
        \end{aligned}
    \end{equation*}

    The solution is $\variance{R_n} = n$.

    For question (\ref{hatq7}), for $S_n$, we have
    \begin{equation*}
        \expect{S_n} = n + \expect{S_{n - X_n}}
    \end{equation*}
    And the solution is $\expect{S_n} = n + \dfrac{n^2}{2}$.

    For question (\ref{hatq8}),
    \begin{equation*}
        \begin{aligned}
            \sum_{j=1}^n (C_j + 1) & = S_n \\
            \expect{C_j} &= \expect{C_j + 1} -1 \\
            &= \dfrac{\expect{S_n}}{n}  -1\\
            &= \left( 1 + \frac{n}{2} \right) - 1 \\
            &= \frac{n}{2}
        \end{aligned}
    \end{equation*}
\end{proof}

\begin{example}
what is the number of necessary trial to get $k$ consecutive success? Each trial has probability $p$ of being successful.
\end{example}
\begin{proof}
    Let $N_k$ the number of $k$ consecutive success. $A_{k-1, k}$ be the additional trial from $k-1$ success to $k$ success, so
    \begin{equation*}
        \expect{N_k} = \expect{N_{k-1}} + \expect{A_{k-1, k}}
    \end{equation*}

    We have
    \begin{equation*}
        \expect{A_{k-1,k}} = 1 \times p + (1 + \expect{N_k}) (1-p) = 1 + (1-p) \expect{N_k}
    \end{equation*}
    So
    \begin{equation*}
        \expect{N_k} = \frac{1}{p} + \frac{\expect{N_{k-1}}}{p}
    \end{equation*}
    with
    \begin{equation*}
        \expect{N_1} = \frac{1}{p}
    \end{equation*}
    So
    \begin{equation*}
        \expect{N_k} = \sum_{i=1}^k \frac{1}{p^i}
    \end{equation*}
\end{proof}

\begin{example}
    Analyse the quick sort algorithm
\end{example}
\begin{proof}
    Suppose there are $n$ numbers and all permutation are of equally likely. Let $M_n$ be the expected number of comparison needed for $n$ numbers. So
    \begin{equation*}
    \begin{aligned}
        M_n &= \sum_{j=1}^n \expect{\text{number of comparison} | \text{pivot is the } i \text{th smallest}} \frac{1}{n} \\
        &= \sum_{j=1}^n (n - 1 + M_{j-1} + M_{n - j}) \frac{1}{n} \\
        &= n - 1 + \frac{2}{n} \sum_{k=1}^{n-1} M_k
    \end{aligned}
    \end{equation*}

    So
    \begin{equation*}
        \begin{aligned}
            M_{n+1} &= 2(n+2) \sum_{i=1}^n \frac{i}{(i+1)(i+2)} \\
            &\approx 2(n+2) \log{(n+2)}
        \end{aligned}
    \end{equation*}
\end{proof}

\begin{example}[compound random variable]
    Let $X_i$ be identical independent random variable with mean $\mu$ and variance $\sigma^2$. Let $N$ be random variable. The random variable $\displaystyle \sum_{i=1}^N X_i$ is called \cindex{compound random variable}. Its expectation is $\displaystyle \expect{\sum_{i=1}^N X_i} = \expect{N} \expect{X}$, and the variance is $\variance{\sum_{i=1}^N X_i} = \sigma^2 \expect{N} + \mu^2 \variance{N}$
\end{example}
\begin{proof}
    Let $\displaystyle S = \sum_{i=1}^N X_i $  be the random variable.
    \begin{equation*}
        \begin{aligned}
            \displaystyle \expect{S| N = n} &= \expect{\sum_{i=1}^n X_i | N = n} \\
            &= \expect{\sum_{i=1}^n X_i} \\
            &= n \expect{X}
        \end{aligned}
    \end{equation*}
    Thus
    \begin{equation*}
        \expect{S} = \expect{N \expect{X}} = \expect{N} \expect{X}
    \end{equation*}

    \begin{equation*}
        \begin{aligned}
            \displaystyle \variance{S|N=n} &= \variance{\sum_{i=1}^n X_i | N = n} \\
            &= \variance{\sum_{i=1}^n X_i} \\
            &= n \variance{X}
        \end{aligned}
    \end{equation*}
    So $\variance{S|N} = N \variance{X}$, $\expect{S|N} = N \expect{X}$, and
    \begin{equation*}
        \begin{aligned}
            \variance{S} &= \expect{\variance{S|N}} + \variance{\expect{S|N}} \\
            &= \expect{N} \variance{S} + \variance{N}\left(\expect{S}\right)^2
        \end{aligned}
    \end{equation*}
\end{proof}

\begin{example}
    For $n$ distinct value, select the biggest one using the following rules:
    \begin{enumerate}
        \item determine the maximum of first $k$ value. reject all of them.
        \item select the first one that is larger than the maximum value just found.
    \end{enumerate}
    What is the probability that the rule select the maximum among all $n$ values?
\end{example}
\begin{proof}
    Let $X$ be the position of largest value and $P_k(\text{best})$ be the probability that the best value is elected using the rule. We have
    \begin{equation*}
        \begin{aligned}
            P_k (\text{best}) &= \sum_{i=1}^n P_k (\text{best} | X = i) p(X=i) \\
            &= \frac{1}{n} \sum_{i=1}^n P_k (\text{best}|X=i)
        \end{aligned}
    \end{equation*}
    Because the largest will be selected if the largest of first $k$ is also the largest of first $i-1$, we have
    \begin{equation*}
        P_k(\text{best} | X=i) = \frac{k}{i-1}
    \end{equation*}
    So
    \begin{equation*}
        \begin{aligned}
            P_k(best) &= \frac{1}{n} \sum_{i=1}^n P_k (\text{best}|X=i) \\
            &= \frac{k}{n} \sum_{i=k+1}^n \frac{1}{k-1} \\
            &\approx \frac{k}{n} \int_k^{n-1} \frac{1}{x} \dif{x} \\
            &= \frac{k}{n} \log{\frac{n-1}{k}}
        \end{aligned}
    \end{equation*}
    The best $k$ is $\dfrac{1}{e} n$.
\end{proof}

\begin{example}[The Ballot Problem]
    In an election candidate A received $n$ notes and B received $m$ notes where $n > m$. What is the probability that A is always ahead of B?
\end{example}
\begin{proof}
    Let $P_{n,m}$ denote the probability. By conditioning on who receive the last vote, we have:
    \begin{equation*}
        P_{n,m} = \frac{n}{n+m} P_{n-1,m} + \frac{m}{n+m}P_{n,m-1}
    \end{equation*}
    The solution is $P_{n,m} = \dfrac{n-m}{n+m}$
\end{proof}

\begin{example}
    A coin has $p$ of being head. What is the probability that the total number of head equals tail after $2n$ flip?
\end{example}
\begin{proof}
    \begin{equation*}
        \begin{aligned}
            \probability{\text{first time } = 2n} &= \probability{\text{first time } = 2n | n \text{ heads in first } 2n} \binom{2n}{n} p^n (1-p)^n \\
            &= P_{n,n-1} \binom{2n}{n} p^n (1-p)^n \\
            &=  \frac{\displaystyle \binom{2n}{n} p^n (1-p)^n}{2n - 1}
        \end{aligned}
    \end{equation*}
\end{proof}

\begin{example}
    What is the probability that the first time there are $i$ more heads than tails occurs after the $2n+i$   flip?
\end{example}
\begin{proof}
    it occurs $\iff$ starting from the final flip and working backwards, the head is always in the lead.
\end{proof}


\chapter{Relationship between SA and SGD (Minibatch)}
\section{From SGD (minibatch) to SA}
We first consider the following lost function
$$
F(x) = \frac{1}{N} \sum_{i=1}^N f_i(x),
$$
where $N$ is the number of data point. The Minibatch SGD is written
as
\begin{equation} \label{equ:minibatch}
x^{n+1} = x^n - \Delta t_{n} \nabla F_{B_n}(x^n),
\end{equation}
where
$$
F_{B_n}(x) = \frac{1}{|B_n|} \sum_{j \in B_n} f_j(x).
$$
Now, we rewrite \eqref{equ:minibatch} as
\begin{equation} \label{equ:minibach-SA}
X^{n+1} = X^n - \Delta t_{n} \nabla F(X^n) + \Delta {t}_n
\underbrace{[\nabla F(X^n) - \nabla F_{B_n}(X^n)]}_{Y^n}.
\end{equation}

\begin{equation} \label{equ:minibach-SA}
X^{n+1} = X^n - a_{n} \nabla F(X^n) + a_n
[\nabla F(X^n) - \nabla F_{B_n}(X^n)-\frac{b_n}{a_n}] + b_nW_n
\end{equation}
\begin{equation} \label{equ:minibach-SA}
X^{n+1} = X^n - a_{n} [\nabla F(X^n) + \xi_n] + b_nW_n
\end{equation}
where
$$
a_n=\Delta t_n, \xi_n=\nabla F(X^n) - \nabla F_{B_n}(X^n)-\frac{b_n}{a_n}W_n.
$$

Let $p_i$ is the probability of choosing the $i$-th component to
the minibatch. We note here that each component may be in the
minibatch for more than once. Then, we have
$$
\begin{aligned}
\mathbb{E}(Y^n) &= \nabla F(X^n) - \mathbb{E} [\nabla F_{B_n}(X^n)] \\
&= \frac{1}{N} \sum_{i=1}^N \nabla f_i(X^n) - \frac{1}{|B_n|}
\mathbb{E}[\sum_{j\in B_n} \nabla f_j(X^n)] \\
&= \frac{1}{N} \sum_{i=1}^N \nabla f_i(X^n) - \mathbb{E}[\nabla
f(X^n)] \qquad (\text{independence}) \\
&= \frac{1}{N} \sum_{i=1}^N \nabla f_i(X^n) - \sum_{i=1}^N p_i \nabla
f_i(X^n).
\end{aligned}
$$
Therefore, we have the following condition
\begin{equation} \label{equ:mean-condition}
\frac{1}{N}\sum_{i=1}^N \nabla f_i(x^n) = \mathbb{E}[\nabla f(X^n)].
\end{equation}
From the above condition, a simple choice is $p_i = \frac{1}{N}$,
namely uniform distribution. Condition \eqref{equ:mean-condition}
can also be written as $\mathbb{E}(Y^n) = 0$.

Note that ${\rm Var}(Y^n) = \mathbb{E}[Y^n (Y^n)^T] - \mathbb{E}(Y^n)
\mathbb{E}(Y^n)^T = \mathbb{E}[Y^n (Y^n)^T]$. For simplicity,
we remove the dependence of $X^n$ in the following. Then,
$$
\begin{aligned}
{\rm Var}(Y^n) &= \mathbb{E}[ (\nabla F - \nabla F_{B_n})
(\nabla F - \nabla F_{B_n})^T ]
\\
&= \mathbb{E}[\nabla F_{B_n} (\nabla F_{B_n})^T ] - \nabla F (\nabla
    F)^T \\
&= \frac{1}{|B_n|^2} \mathbb{E}_{i_1} \mathbb{E}_{i_1} \cdots
\mathbb{E}_{i_{|B_n|}}[ (\sum_{j=1}^{|B_n|} \nabla f_{i_j})
  (\sum_{j=1}^{|B_n|} \nabla f_{i_j})^T ] - \nabla F (\nabla F)^T \\
&= \frac{1}{|B_n|^2} \mathbb{E}_{i_1} \mathbb{E}_{i_2} \cdots
\mathbb{E}_{i_{|B_n|}}[ \sum_{j,r=1 , i\neq j}^{|B_n|} \nabla f_{i_j}
(\nabla f_{i_r})^T] \\
& ~~ + \frac{1}{|B_n|^2} \mathbb{E}_{i_1} \mathbb{E}_{i_2} \cdots
\mathbb{E}_{i_{|B_n|}} [\sum_{j=1}^{|B_n|} \nabla
f_{i_j} (\nabla f_{i_j})^T]- \nabla F (\nabla F)^T \\
& = (1 - \frac{1}{|B_n|}) \mathbb{E}(\nabla f) [\mathbb{E}(\nabla
    f)]^T + \frac{1}{|B_n|} \mathbb{E}[ (\nabla f) (\nabla f)^T] -
\nabla F (\nabla F)^T \\
&= \frac{1}{|B_n|} \left( \mathbb{E}[(\nabla f) (\nabla f)^T] -
    \mathbb{E}(\nabla f) [\mathbb{E}(\nabla f)^T]
    \right) \\
&= \frac{1}{|B_n|} {\rm Var}(\nabla f).
\end{aligned}
$$

\begin{remark}
Since the index here can be repeated, there is NO need that ${\rm
Var}(Y^n) = 0$ when $|B_n| = N$.
\end{remark}

In summary, we have
\begin{equation} \label{equ:summary}
\mathbb{E}(Y^n) = 0, \qquad {\rm Var}(Y^n) = \frac{1}{|B_n|} {\rm
  Var}(\nabla f).
\end{equation}
It is conceivable that the result can be extended to the expectation
version of the lost function, namely,
$$
F(x) = \mathbb{E}_{\xi} f(x, \xi).
$$

\section{Convergence result}
In the SA scheme,
\begin{equation} \label{equ:SA}
X^{n+1} = X^n - \Delta t_n \nabla F(X^n) + \sigma_n W_{\Delta t_n},
\end{equation}
the convergence is proven under the following condition:
$$
\Delta t_n = \mathcal{O}(n), \qquad \sigma_n^2 =
\mathcal{O}(\frac{1}{n \log\log n})
$$
Compare the variance of $\sigma_n W_{\Delta t_n}$ and $\Delta t_n
Y^n$, we have
$$
\sigma_n^2 \Delta t_n \approx \Delta t_n^2 {\rm Var}(Y^n),
$$
which implies that
$$
\frac{1}{|B_n|} {\rm Var}(\nabla f) =
\mathcal{O}(\frac{\sigma_n^2}{\Delta t_n}) =
\mathcal{O}(\frac{1}{\log\log n}).
$$
We conclude that, if the SGD is stagnated, namely ${\rm Var}(\nabla
f)$ stays the almost the same, we need to slowly increase the size of
minibatch in the way that
\begin{equation} \label{equ:batch-size}
|B_n| = \mathcal{O}(\log\log n).
\end{equation}

\begin{remark}
The above argument requires that $\nabla f(x, \xi)$ is Gaussian on
$\xi$ for any $x$. This assumption holds for linear Gaussian
regression.
\end{remark}


\section{Probability}
This section is mostly an introduction (or a refresher?) to some tools and ideas from probability theory. One of the main reasons for including it is that it lets us give backround to the final sub-section on Brownian motion, but we'll also use some of the methods from here in the final section of the course on complex networks.

Physical variables can be either continuous or discrete. Continuous variables, like the spatial position of a particle ${\bf r} = (x,y,z);$ $x\rightarrow x+dx$ allow us to apply tools from differential calculus. Discrete variables (like the spin of an electron, the face of a playing card, or the energy level of a quantum particle) require a different approach. As the number of discrete states gets gets large, it is possible to approximate a discrete system by a continuous one. However, there are often statistical reasons why this is not always the best approach. (For an example of the issue of approximating a discrete distribution with a continuous one, see the article  Clauset, Shalizi, and Newman. ``Power-Law Distributions in Empirical Data.'' SIAM Review (2009) doi:10.1137/070710111)

[It is probably a good idea to add a bibliography to the notes and use LaTex citations, (along with naming the reference in the text to make it more readable.)]

Here we are going to look at some simple discrete distributions, including one of the simplest --- the binomial distribution which describes the probability of events that can take one of only two possible outcomes. The tool we will employ to study these distributions is \emph{probability generating functions} (or generating functions for short). Although PGFs can be somewhat cumbersome, they provide a very powerful general tool that can applied to a wide range of situations involving discrete probabilities. In this sense they are like the Taylor series of discrete probability.

\subsection{Probability generating functions}
A probability generating function (PGF) is a polynomial whose coefficients are the probabilities associated with each outcome of some random process.
For example, the PGF for an equal probability die is
$$\calG(x) = 0x^0+\frac16x^1+\frac16x^2+\frac16x^3+\frac16x^4+\frac16x^5+\frac16x^6.$$
As an example of how PGFs are a handy general tool, say we now wanted to find the PGF for the sum when we roll two such dice. We can just multiply the PGFs to get $\calG'(x) = \calG(x)\calG(x) = 0x^0 + 0x^1+\frac{1}{36}x^1x^1+\frac{2}{36}x^1x^2+\cdots+\frac{1}{36}x^6x^6.$

More generally, the set of probabilities associated with any discrete distribution can be used as coefficients of $\calG(x)$:
$$\calG(x) = P(X=r_0)x^0+P(X=r_1)x^1+\cdots = \sum_{n=0}^\infty P(X=r_n)x^n.$$

\subsection{Properties of generating functions}
\begin{eqnarray*}
	\calG(0)&=&P(X=0)\\
	\calG(1)&=&\text{sum over all probabilities}=1
\end{eqnarray*}
The two properties above are easy to see and are not unexpected. More interesting is the fact that derivatives of PGFs can be used to obtain expressions for the mean, and higher order moments, of the distribution associated with the PGF.

\begin{eqnarray*}
	\frac{d}{dx}\calG(x)=\calG'(x)&=&\sum_r rP_r x^{r-1},\qquad\text{and}\\
	\calG''(x)&=&\sum_r r(r-1)P_rx^{r-2}.
\end{eqnarray*}

If we evaluate the first derivative of the PGF at 1 we get
$$\calG'(1) = \sum_r rP_r.$$
The expected value of an arbitrary function $f(r)$ is given by $\langle f(r)\rangle = \sum_r f(r) P(X=r)$, therefore $\calG'(1) = \langle r\rangle$ --- the expected value, or mean, of the distribution.

Similarly,  Evaluating $\calG''(x)$ at 1 gives
$$\calG''(1) = \sum_r r(r-1)P(X=r) = \langle X(X-1)\rangle.$$

Why would we care about being able to find $\langle X(X-1)\rangle = \langle X^2 -X\rangle$? Because the variance of X, is given by
$$ V(X) = \langle X^2\rangle -\langle X\rangle^2.$$ We are therefore able to construct $V(X)$ from combinations of $\calG$, $\calG'$, and $\calG''$.
We can use the fact that $\langle X^2 -X\rangle =\langle X^2\rangle-\langle X\rangle$ I.e. $\sum(r^2-r)P(X=r)=\sum r^2P(X=r)-\sum rP(X=r)$ to get

\begin{eqnarray}
V(X)&=&\langle X^2\rangle - \langle X\rangle^2\\
&=&\underbrace{\langle X^2\rangle -  \langle X \rangle}_{\calG''(1)} + \underbrace{\langle X\rangle}_{\calG'(1)} - \underbrace{\langle X \rangle^2}_{\calG'(1)^2}\\
&=& \calG''(1) +\calG'(1)-\calG'(1)^2 \label{eq:binvar}
\end{eqnarray}

%Generating functions may not always be the shortest way to find the properties of some distribution, but they are a powerful way.

\subsection{The binomial distribution}
The binomial distribution describes the probability of getting $r$ identical outcomes from a sequence of $N$ events where each event has only two possible states, e.g. number of heads from a sequence of coin tosses, number of Au atoms from deposition of Au and Ag atoms in some sputtering process, or number of spin up electrons in an uncorrelated system.

The probabilities for the binomial distribution are given by
$$P(X=r) = \binom{N}{r}p^rq^{N-r}, ~\text{ where } \binom{N}{r} = \frac{N!}{r!(N-r)!}, ~~r=0,1,\ldots,N$$
and where (mostly) $p+q=1$.
The PGF for the binomial distribution is therefore given by $\calG(x) = \sum_{r=0}^N\binom{N}{r}p^rq^{N-r}x^r$.
The coefficients in the binomial distribution are (not surprisingly) the coefficients from the  binomial expansion theorem. This means that instead of writing the PGF as a sum, we can easily express it (and its derivatives) as a polynomial:
\begin{eqnarray*}
	\calG(x)  &=& (xp +q)^N \\
	\calG'(x) &=& N(xp+q)^{N-1}p\\
	\calG''(x) &=& N(N-1)(xp+q)^{N-2}p^2.
\end{eqnarray*}

Evaluating the above at $x=1$ gives:
	\begin{eqnarray*}
	\calG(1) &=& (p+x)^N = 1 ~\text{ if } p+q=\\
	\calG'(1) &=& N(p+q)^{N-1}P = Np  ~\text{ if } p+q=1\\
	\calG''(1) &=& N(N-1)(p+q)^{N-2}p^2 \\
		&=&N(N-1)p^2  ~\text{ if } p+q=1.
\end{eqnarray*}
As expected, we have $\langle X\rangle =Np$ for the event associated with the probability $p$.
Similarly, we can calculate the variance of the binomial distribution using equation \ref{eq:binvar}
\begin{eqnarray*}
	V(X) &=&\calG''(1) - \calG'(1) -\calG'(1)^2\\
		&=& N^2p^2-Np^2+Np-N^2p^2\\
		&=& N(p-p^2)\\
		&=& Np(1-p)= Npq.
\end{eqnarray*}

If you are feeling enthusiastic, compare the calculation above with the usual method for finding the variation of the binomial distribution without using a PGF.

\subsection{The Poisson distribution}
Since the Poisson distribution shows up quite often in stat mech, we will have a look at how its properties can be derived via PGFs.
The Poisson distribution describes the probability of $r$ distinct events occur in a time interval, given that the long-run average rate is $\lambda$ per time interval. It has probabilities given by $P(X=r) = \frac{\lambda^r}{r!}\exp(-\lambda)$. The PGF for the Poisson distribution is given by
$$G(x) = \sum_r=0^\infty \frac{\lambda^r}{r!}\exp(-\lambda)x^r = \exp(-\lambda)\sum\frac{(\lambda x)^r}{r!}.$$
Note that the sum in the last term of the expression above is the power series expansion for the exponential function. Handy --- we can now write:
\begin{eqnarray*}
\calG(x) &=& \exp(-\lambda)\exp(\lambda x)\\
\calG'(x) &=& \lambda\exp(-\lambda)\exp(\lambda x)\\
\calG''(x)&=&\lambda^2\exp(-\lambda)\exp(\lambda x)
\end{eqnarray*}

So we have $\langle X \rangle = \calG'(1) = \lambda$, as per the definition, and
$$V(X) = G''(1) + G'(1) - G'(1)^2 = \lambda^2+\lambda-\lambda^2 = \lambda.$$

\subsection{The expected long-run outcome of a binary process}
What is the net outcome we can expect from a sequence of $N$ binary events? What is the variance in this net outcome? How long can we expect the net outcome to remain positive?

Think of a sequence of coin tosses where heads implies $+1$ and tails implies $-1$. We will denote the number of heads from $N$ events as $n_h$ and the number of tails as $n_t = N-n_h$, then the net outcome is $d = n_t-n_h = N-2n_h$.

If $p=q=\frac12$ then the expected value of $d$ is $\langle d\rangle= 0$. (It's easy to check this: $\langle d\rangle = \langle N-2n\rangle = N-2\langle n\rangle = N-\frac{2}{2}N=0.$)

We want to find the variation in $d$, or rather, the root-mean-squared deviation
$$d_{rms} = \left(\langle d^2\rangle - \langle d\rangle^2\right)^{\frac12} =  \langle d^2\rangle^{\frac12}.$$

At this point we'll drop the subscript in $n$ since we only have $n_h$ and since the probabilities are equal. From $d=N-2n$ we have $$ \langle d^2\rangle = \langle (N-2n)^2\rangle = \langle N^2 -4Nn+4n^2\rangle.$$
We know that $\langle n\rangle = \calG'(1)=Np=N/2$ and from the expressions for $\calG''(1)$ and $\calG'(1)$ we get $\langle n^2 \rangle = \calG''(1) + \calG'(1)$ where $\calG''(1) = N^2p^2 = N^2/4-N/4$. We can now calculate $\langle d^2 \rangle$.

\begin{eqnarray*}
	\langle d^2 \rangle &=& N^2 -4N\frac{N}{2}+4\left(\frac{N^2}{4}-\frac{N}{4}+\frac{N}{2}\right)\\
		&=& N^2-2N^2+N^2+N\\
		&=& N.
\end{eqnarray*}
So, $d_{rms}=\sqrt{N}$ and the variance in the net expected outcome scales like $\sqrt{N}$ as $N$ increases. I.e. after $N$ steps we expect to be a rms distance of $\pm \sqrt{N}$ away from the mean of zero.

[Draw/add a figure of this with N as the independent variable.]

We can now look at how this would change if the binary process had some small bias $\alpha = |p-q| >0$. In this case, at each step, the expected outcome is $\alpha$ and the bounds for the rms distance from the mean are $\alpha N\pm\sqrt{N}$.

[Draw/add this too.]

\subsection{Brownian motion}
Brownian motion is a model that describes the behaviour of particles in suspension (e.g. of dust in air, fine silt in dirty water) that have some velocity but which are constantly changing direction as they are buffeted by other particles. Their motion can be understood through a calculation similar to that for the net outcome of the binary process above, but in 3D.

The conceptual model that is often used when discussing Brownian motion is that of a \emph{random walk}. Consider a random walker who after $n$ steps is at the position ${\bf r} = (x,y,z)$. Each step is given by ${\bf s}_{n+1} = {\bf r}_{n+1}- {\bf r}_n$ and has step length $s_n$ in a random and uncorrelated direction ${\bf\hat{s}}_n$. Again (by symmetry) $\langle {\bf s}\rangle =0$. We want to find how far a particle might have gone after $N$ steps. We will define $\lambda^2= \langle {\bf s}^2\rangle$ --- the average square step-length. We write ${\bf r}_N$ for the position of the walker (or the particle) after $N$ steps, then
\begin{eqnarray*}
	\langle {\bf r}_N^2 \rangle &=& \bigg\langle \left(\sum_{n=1}^N{\bf s}_n\right)^2 \bigg\rangle\\
		&=& \bigg\langle \sum_{n=1}^N{\bf s}_n^2 +2\sum_{n=1}^{N-1}\sum_{m=n+1}^N{\bf s}_n\cdot{\bf s}_m \bigg\rangle\\
		&=& \sum\underbrace{\langle {\bf s}_n\rangle}_{=\lambda^2} + 2\sum\sum \underbrace{\langle {\bf s}_n \cdot {\bf s}_m \rangle}_{=\langle {\bf s}_n\rangle \cdot \langle{\bf s}_m \rangle=0}
\end{eqnarray*}

The first term in the expression is the mean squared step-length while the second term is zero since there is no correlation between the directions of each of the steps. Hence $\langle {\bf r}_N\rangle = N\lambda^2$ and the rms displacement is $\sqrt{N}\lambda$. If the number of steps is proportionate to the length of time that the particles move for, then we have that the mean-square displacement scales like $\sqrt{t}$. This is one of the key features of diffusive processes.


\section{Basic Topology}

\subsection{Exercise 1}
The empty set has no elements, so all of its elements are vacuously also elements
of every set.

\subsection{Exercise 2}
The roots of complex polynomials with integer coefficients can be expressed
as elements of the countable cross product of $\mathbb{N}$ with itself
(cross $\mathbb{N}$ with itself $n$ times for the coefficients, and then once
more to indicate which root).

\subsection{Exercise 3}
If all real numbers were algebraic, then the set of algebraic numbers would be
uncountable (thus contradicting Exercise 2).

\subsection{Exercise 4}
The set of irrational numbers is $\mathbb{R} / \mathbb{Q}$, which
must be uncountable as otherwise $\mathbb{R}$ would be countable.

\subsection{Exercise 5}
We can use $\big(\frac{n}{n+1}\big)_{n \in \mathbb{N}} \cup \big(\frac{2n}{n+1}\big)_{n \in \mathbb{N}} \cup \big(\frac{3n}{n+1}\big)_{n \in \mathbb{N}} $ to get the three limit points $1, 2, 3$.

\subsection{Exercise 6}
If $p$ is a limit point of $E'$, then every neighborhood of $p$ contains
a limit point $q$ of $E$, and every neighborhood of $q$ contains a point
of $E$ thereby implying that $p$ is a limit point of $E$. $E$ and $E'$
do not need to have the same limit points, since $E'$ could be finite
and thus have no limit points.

\subsection{Exercise 7}
(a) If $p$ is a limit point of $\overline{B_n}$, then every neighborhood
of $p$ contains a point $q \in A_i$. Since there are only finitely many $A_i$,
$p$ must be a limit point for at least one of the $A_i$, as an infinite number
of neighborhoods of $p$ must have non-zero intersection with some of the $A_i$.

(b) If we take $A_i = \big(\frac{i n}{(i + 1)n+1}\big)_{n \in \mathbb{N}}$, then
1 is a limit point of $B_n$ despite not being a limit point of any of the $A_i$.

\subsection{Exercise 8}
Every point of an open set in $\mathbb{R}^2$ is by definition a limit point of
the set, since the point must have a neighborhood contained in the set. The same
is not true for closed sets, since we can just take a finite set.

\subsection{Exercise 10}
Every set in $X$ is open, since any set containing $p$ also contains $N_r(p)$ for $r < 1$.
No set in $X$ is closed, since $N_r(p) = p$ for $r < 1$. All infinite sets in $X$ are not
compact, since we can take balls of radius $r < 1$ around each point as an open cover.

\subsection{Exercise 12}
Take any open cover of $K$. There must be some open set in this cover containing 0,
which means that the same set contains all but a finite number of the elements of $K$
(since 0 is the only limit point of $K $). Take a union of this set as well as the finitely
many other sets containing the aforementioned points to get a finite subcover.

\subsection{Exercise 13}
Take $\cup_{k = 1}^\infty \{0, \big(\frac{n}{kn+1}\big)_{n \in \mathbb{N}}, \frac{1}{k}\}$.
This set is closed and bounded, so it is compact by Heine-Borel. Its limit points
are 0 and $\big(\frac{1}{k}\big)_{n \in \mathbb{N}}$.

\subsection{Exercise 14}
We can use $\cup_{n \in \mathbb{N}} (0, \frac{n}{n + 1})$, which has no finite subcover
(since we could choose $x \in (0, 1)$ larger than the largest endpoint in the finite subcover).

\subsection{Exercise 15}
For closed, we can take $K_i = \mathbb{N} / {0, ..., i - 1}$, since any $x \in K_i$ will not
be in $K_j$ if $j > x$. For bounded, we can take $K_i = (0, \frac{1}{i})$.

\subsection{Exercise 16}
$E$ is by definition bounded, and $E$ is closed since $q^2 \neq 3$ ($q$ is rational), and
$q^2 > 3 \implies \exists \epsilon \: | \: p  \in N_{\epsilon}(q) \implies p^2 > 3$. Same
logic gives that $E$ is also open in $\mathbb{Q}$. $E$ is, however, not compact, since
we can construct an open cover consisting of $G_n = \{x \: | \: 2 < x^2 < 2 + \frac{n}{n+1}\}$.

\subsection{Exercise 17}
$E$ is not countable by diagonalization. $E$ is not dense in $[0, 1]$, since $E \cap [0, 0.1] = \emptyset$.
$E$ is not perfect, consider $N_{0.001}(0.77)$. $E$ is closed and therefore compact by Heine-Borel.
To see closed, suppose a limit point $q$ had a non-4/7 digit in the $i^{th}$ decimal spot. Then we could
take a neighborhood of size $10^{-(i + 1)}$.

\subsection{Exercise 18}
Originally I thought this was no, but this can actually be done with a modified version of the Cantor set
construction. See \href{https://math.stackexchange.com/questions/1064/perfect-set-without-rationals}{here} for
a discussion.


\subsection{Moving Average models (MA)}

We add previous error terms as input variables

MA(\(q\)) has \(q\) previous error terms in the model

Unlike AR models, the effects of any shocks wear off after \(q\) terms.

This is harder to fit the OLS, the error terms themselves are not observed.