\documentclass[10pt,letter]{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[makeroom]{cancel}
\usepackage{graphicx}
\usepackage{setspace}
\usepackage{stmaryrd}
\def\fatbar{\talloblong}
\onehalfspacing
\usepackage{fullpage}
\newcommand{\R}{\mathbb{R}}
\newcommand{\inner}{\langle\cdot,\cdot\rangle}
\newcommand{\inr}[2]{\langle #1, #2\rangle}
\newcommand\norm[1]{\left\lVert#1\right\rVert}

\graphicspath{ {images/} }

\begin{document}

\title{CS142 Homework Set \#6 Solutions}

\author{Timur Kuzhagaliyev}

%\date{12th November, 2017}

\maketitle

\section*{Problem 1}

\paragraph{a)} The statement is true. Proof:

\begin{align*}
& P\; \textrm{\textbf{ensures}}\; \neg P \textrm{ in } F
\\ \equiv \quad & \qquad \{\ \textrm{Definition of ensures}\ \} \\
& ( ( P \land \neg\neg P )\; \textrm{\textbf{next}}\; (P \lor \neg P)) \land \textrm{\textbf{transient}}\; (P \land \neg\neg P) \textrm{ in } F
\\ \equiv \quad & \qquad \{\ \textrm{Predicate calculus}\ \} \\
& ( P\; \textrm{\textbf{next}}\; true) \land \textrm{\textbf{transient}}\; P \textrm{ in } F
\\ \equiv \quad & \qquad \{\ [ P\; \textrm{\textbf{next}}\; true \equiv true ] \textrm{  as shown in previous homework sets, $true$ is the identity of $\land$ }\ \} \\
& \textrm{\textbf{transient}}\; P \textrm{ in } F
\\ \equiv \quad & \qquad \{\ \textrm{Definition of transient}\ \} \\
& ( \exists\, a : a \in F : \{P\}\; a\; \{\neg P\} )
\\ \Rightarrow \quad & \qquad \left\{ \;
\begin{aligned}
&\textrm{Actions of $F \fatbar G$ are a union of actions in $F$ and $G$. If there exists at least one action}\\
& \textrm{that take us from $P$ to $\neg P$ in $F$, the same action must also exist in $F \fatbar G$, for any $G$.}
\end{aligned}
\; \right\} \\
& ( \exists\, a : a \in F \fatbar G : \{P\}\; a\; \{\neg P\} ) \textrm{ for any } G
\\ \equiv \quad & \qquad \{\ \textrm{Definition of transient}\ \} \\
& \textrm{\textbf{transient}}\; P \textrm{ in } F \fatbar G \textrm{ for any } G
\\ \equiv \quad & \qquad \{\ [ P\; \textrm{\textbf{next}}\; true \equiv true ] \textrm{  as shown in previous homework sets, $true$ is the identity of $\land$ }\ \} \\
& ( P\; \textrm{\textbf{next}}\; true) \land \textrm{\textbf{transient}}\; P \textrm{ in } F \fatbar G \textrm{ for any } G
\\ \equiv \quad & \qquad \{\ \textrm{Predicate calculus and definition of ensures}\ \} \\
& P\; \textrm{\textbf{ensures}}\; \neg P \textrm{ in } F \fatbar G \textrm{ for any } G
\end{align*}

\pagebreak

\paragraph{b)} The statement is false. We can produce a counter-example. Consider the following two programs:
\begin{align*}
\begin{aligned}
\textrm{\textbf{Program}} \qquad & F
\\
\textrm{\textbf{var}} \qquad & x\; \textrm{:\; integer}
\\
\textrm{\textbf{initially}} \qquad & x = 0
\\
\textrm{\textbf{assign}} \qquad &
x := x + 1
\end{aligned}
\qquad \qquad \qquad \qquad \qquad
\begin{aligned}
\textrm{\textbf{Program}} \qquad & G
\\
\textrm{\textbf{var}} \qquad & x\; \textrm{:\; integer}
\\
\textrm{\textbf{initially}} \qquad & x = 0
\\
\textrm{\textbf{assign}} \qquad &
x := 0
\end{aligned}
\end{align*}
Both $F$ and $G$ have a single action, let's denote these by $f$ and $g$ respectively. Note that $x = 0 \leadsto x = 2$ in $F$ and $x = 0$ is stable in $G$. The union of the two programs, $F \fatbar G$, will have both actions $f$ and $g$. Now, consider the sequence of actions $(f, g)^\omega$, where $\omega$ stands for infinite repetition. This sequence satisfies weak fairness since every action gets called infinitely often, but $x = 2$ never holds because $x$ gets reset to $0$ after every increment. Therefore, $x = 0 \leadsto x = 2$ does not hold in $F \fatbar G$.

\paragraph{c)} This statement is also false, which can be shown using a counter-example. Consider the following two programs:
\begin{align*}
\begin{aligned}
\textrm{\textbf{Program}} \qquad & F
\\
\textrm{\textbf{var}} \qquad & z\; \textrm{:\; integer}
\\ & p, q\; \textrm{:\; booleans}
\\
\textrm{\textbf{initially}} \qquad & z = 0
\\ \land \quad & p = true
\\
\textrm{\textbf{assign}} \qquad &
p \longrightarrow z := z + 1
\\
\fatbar \quad & q := false
\end{aligned}
\qquad \qquad \qquad \qquad \qquad
\begin{aligned}
\textrm{\textbf{Program}} \qquad & G
\\
\textrm{\textbf{var}} \qquad & z\; \textrm{:\; integer}
\\ & p, q\; \textrm{:\; booleans}
\\
\textrm{\textbf{initially}} \qquad & z = 0
\\ \land \quad & q = true
\\
\textrm{\textbf{assign}} \qquad &
q \longrightarrow z := z + 1
\\
\fatbar \quad & p := false
\end{aligned}
\end{align*}

I'll refer to actions of $F$ as $f_1, f_2$ and actions of $G$ as $g_1, g_2$. Note that $z = 0 \leadsto z = 2$ holds in $F$ and $G$ on their own, but does not hold in $F \fatbar G$. Action $f_2$ deactivates the guard on $g_1$ and action $g_2$ deactivates the guard on $f_1$. Consider the sequence of actions $(f_2, g_2, f_1, f_2)^\omega$. This sequence satisfies weak fairness since every action gets called infinitely often, but $z$ never gets incremented and just stays $0$ forever. Therefore, $z = 0 \leadsto z = 2$ does not hold in $F \fatbar G$.

\pagebreak

\section*{Problem 2}

\paragraph{a)} The idea behind superposition is to extend an existing program, while preserving all of its original actions and not modifying any of its original variables. To count how many times the actions of $GCD$ were executed, we can define a program $SuperGCD$ by superposition from $GCD$.

I will assume that the action was "executed" when it has been picked and its guard was activated.

\begin{align*}
\textrm{\textbf{Program}} \qquad & SuperGCD
\\
\textrm{\textbf{var}} \qquad & x, y, count\; \textrm{:\; integers}
\\ & claim\; \textrm{:\; boolean}
\\
\textrm{\textbf{initially}} \qquad & count = 0
\\ \land \quad & claim = false
\\
\textrm{\textbf{transform}} \qquad &\textrm{each statement $s$ in $GCD$ to}
\\
s \quad \Vert \quad & count := count + 1
\\
\textrm{\textbf{add}} \qquad &
\\
x = y & \longrightarrow claim := true
\end{align*}

In my \textbf{transform} section, I assume that actions used to augment actions of $GCD$ inherit the same guard. If augmented actions can have 2 different guards, one could add the $x \neq y$ guard to the right hand side in the \textbf{transform} section, which would achieve the same logic.

Clearly, for $claim$ to become $true$, its guard must be activated. This implies $x = y$, so no other action can be executed by definition of $GCD$ and our augmented actions. Therefore we consider $x = y \land claim$ to be the fixed point of $SuperGCD$.

\paragraph{b)} To prove the correctness of $SuperGCD$, we need to prove that it satisfies the safety and progress properties. Let $C$ be the actual action execution count and $terminated$ be boolean which holds iff $GCD$ has terminated. I'll define 2 invariants that guarantee safety: ${\textbf{\textrm{invariant}}(count \leq C)}$ and ${\textbf{\textrm{invariant}}(claim \Rightarrow terminated)}$.

We can use the definitions of actions in $SuperGCD$ to prove ${\textbf{\textrm{invariant}}(claim \Rightarrow terminated)}$. Initially, $claim$ is false, so the property holds. As stated before, $claim$ only becomes $true$ when the guard on the relevant command is activated, what implies $x = y$. To show that this condition only holds when $GCD$ has terminated, we can use the definition of $GCD$. We know that actions of $GCD$ are only applied when $x \neq y$. When $x = y$, all guards in $GCD$ are deactivated and no assignments occur, hence $termianted$ holds (see \textit{aside} for details). Therefore ${\textbf{\textrm{invariant}}(claim \Rightarrow terminated)}$ holds.

\textit{Aside:} One could argue that $GCD$ could reach fixed point when one of the variables is zero and the other is greater than zero (e.g. $x = 10, y = 0$), hence $x \neq y$. This cannot occur since both $x$ and $y$ start with values strictly greater than zero, and the smaller number gets subtracted from the bigger number until equality is reached. Before equality is reached, subtracting the smaller value from the bigger one cannot generate a $0$.

We can also show that ${\textbf{\textrm{invariant}}(count \leq C)}$ holds. Initially, $count$ is zero and since $C$ is bounded below by zero, the property holds. The actions of $SuperGCD$ that increment $count$ were created by augmenting actions of $GCD$. Additionally, these actions inherit the same guards. This implies that the only time that $count$ is incremented when an action of $GCD$ is actually executed, hence $count$ cannot exceed the true count $C$. Therefore ${\textbf{\textrm{invariant}}(count \leq C)}$.

Now that we have proved safety, we can use $count$ as our metric. Using the invariant, we've shown that the metric is bounded above by $C$. We also know that the metric cannot decrease, since any action changing $count$ increments it by $1$. The metric is also guaranteed to increase eventually: we know that until $x = y$ holds, one of the two actions in $GCD$ can be picked to increment $count$ by one. Moreover, the count will increment only until it is equal to $C$, since when $count = C$, underlying $GCD$ would have finished executing actions (by definition of $C$), so $x = y$ will hold and guards on both actions of $GCD$ would be deactivated.

Above I have (implicitly) shown that $terminated \leadsto claim$ and that when $GCD$ terminates, $count$ is equal to the actual count. Therefore the program $SuperGCD$ is correct.

\section*{Problem 3}

\paragraph{a)} I would use definition of priority \#3, as it is the only one that guarantees safety (graph is acyclic) and progress (everyone gets their turn eventually). The first proposed definition guarantees safety since the "greater than" relation is transitive and the agent ID is used to break ties, but it does not guarantee progress since the process with highest time can just keep using the resources repeatedly. The second definition does not guarantee safety since it has no way of breaking ties and the behaviour in case of a tie is undefined.

\paragraph{b)} We can prove safety by showing that "priority graph is acyclic" property is invariant with our definition of priority.

Initially, all agents are thinking. In this case, the [agent's current clock value, agent id] pair is used to determine priority. Not that this relationship is transitive. Let's denote this pair as $p$: ${(p_t < p_u \land p_u < p_v) \Rightarrow p_t < p_v}$. The transitivity of this relationship, and the fact that is not symmetric (${p_t < p_u \not\equiv p_u < p_t}$) guarantees that the graph is initially acyclic.

Now we can show that the "priority graph is acyclic" property is stable. The same idea of transitivity applies here too: for any action in the system, the agent will adjust the first value in the tuple (the clock value). If the agent has just begun waiting or is inside CS, it's pair will show the clock time corresponding to the moment when it begun waiting (since an agent has to go through waiting before entering CS). If the agent has left CS, it will set the clock value in its pair to its current time. In both cases, the pair still contains the unique agent ID, meaning that the pairs can still be sorted, even in case of ties. The same fact that the relationship is transitive and not symmetric as in paragraph above applies here, guaranteeing that the graph is acyclic.

Therefore the "priority graph is acyclic" property is invariant, and hence the safety property is satisfied. The same idea can be expressed by showing that with out definition of priority, the graph is topologically sorted and hence must be acyclic.

\paragraph{c)} In our case, lower numbers mean higher priority. To prove that the progress property is satisfied, we need show that every hungry agent gets to eat eventually. Let's define a metric $M$ as the number of agents that have higher priority than the current (waiting) agent.

$M$ is clearly bounded below by $0$ (and above by the total number of agents minus one). It can be shown that $M$ is guaranteed to not increase. This is the consequence of the fact that the clocks always tick forward - if some other agent has a clock time higher than the current agent (and hence does not contribute to $M$), there is no action it can perform to get the clock time to be lower than ours and increase $M$ (remember that current agent is waiting).

$M$ is also guaranteed to decrease eventually if it's not equal to $0$. If $M$ is equal to $0$, the current agent is at the top of the (topologically sorted) graph, so it can execute, and the progress property is satisfied. If the agent is not at the top of the graph, it means that there are $k = M$ agents above it. For all of the agents that are not waiting, the time will continue increasing and it will eventually exceed the time at which the current process begun waiting, decreasing $M$ to some $k^* \leq k$. Now, we know that $k^*$ processes above the current process are waiting. Additionally, one of them must be at the top of the graph. This process will enter and exit CS, meaning that its clock in the pair will now be its current clock reading, which will also eventually overtake current process' time, decreasing $M$ to $k^* - 1$. We can repeat this logic for all of the other processes above the current process until $M$ reaches $0$ and the current process can enter CS.

Therefore the progress property is also satisfied.

\end{document}


\documentclass[12pt]{article}
% page layout
\usepackage[margin=1in]{geometry}
\usepackage[doublespacing]{setspace}

% useful package
\def\studynote{}
\usepackage{../cmd}

% only for this document
\setcounter{section}{-1}

% document starts here
\title{Probability and Statistics Study Note}
\author{Long Wang}
\begin{document}
\maketitle
\tableofcontents

\newpage

\section{Preliminary}

\subsection{Random Variable}
\begin{itemize}
	\item Statistics
	\begin{itemize}
		\item Mean: $ \E{X} $
		\begin{itemize}
			\item Cons: sensitive (outlier)
		\end{itemize}
		\item Median
		\begin{itemize}
			\item Pros: robust (outlier)
		\end{itemize}
		\item Skewness: $ \tilde{\mu}_3 = \E{\paren{\frac{X - \mu}{\sigma}}^3} $
		\begin{itemize}
			\item Positive skew: mode $<$ median $<$ mean
			\item Negative skew: mean $<$ median $<$ mode
		\end{itemize}
	\end{itemize}
	\item Distribution
	\begin{itemize}
		\item \textbf{KL divergence:} $ D_{KL}(P||Q) = \E{\log \frac{P(x)}{Q(x)}} = \sum_x P(x) \log \frac{P(x)}{Q(x)} $
		\begin{itemize}
			\item Information gain: $ Q \Rightarrow P $
			\item Bayesian: $Q$ (prior) $\Rightarrow$ $P$ (posterior)
		\end{itemize}
	\end{itemize}
\end{itemize}

\subsection{Distribution}
\textbf{Geometric:} $ X \sim Geo(p) $ number of tries until 1 success (with prob $p$)
\begin{itemize}
	\item $ \Prob{X = k} = (1-p)^{k-1} p $
	\item $ \E{X} = 1/p, \Var(X) = (1-p)/p^2 $
\end{itemize}

\textbf{Negative Binomial:} $ X \sim NB(r, p) $ number of failures until $r$ success with success prob $p$
\begin{itemize}
	\item $ P(X = k) = \binom{k + r - 1}{r - 1} (1-p)^k p^r $
	\item $ \E{X} = pr / (1-p), \Var(X) = pr / (1-p)^2 $
	\item $ NB(1, 1-p) = Geo(p) $
\end{itemize}

\subsection{Integrable}
\begin{itemize}
	\item \textbf{Uniformly integrable:} $ \E{\abs{X}} \leq M $
\end{itemize}

\newpage
\section{Inequality}
\subsection{Math}
\begin{itemize}
	\item \textbf{Arithmetic mean-Geometric mean}
	\begin{itemize}
		\item If $ a_i \geq 0 $, then $ \sqrt[n]{a_1 \dots a_n} \leq \frac{a_1 + \cdots + a_n}{n} $
		\item Weighted: If $ a_i \geq 0 $, then $ \prod a_i^{w_i} \leq \sum w_i a_i $
		\item \textbf{Power Mean:} If $ a_i > 0, p < q $, then $ (\frac{1}{n} \sum a_i^p)^{1/p} \leq (\frac{1}{n} \sum a_i^q)^{1/q} $
		\item \textbf{Harmonic mean:} $ \frac{n}{1/a_1 + \cdots + 1/a_n} \leq \sqrt[n]{a_1 \dots a_n} \leq \frac{a_1 + \cdots + a_n}{n} \leq \sqrt{\frac{a_1^2 + \cdots + a_n^2}{n}} $
	\end{itemize}
	\item \textbf{Cauchy-Schwarz}
	\begin{itemize}
		\item $ \sum a_i b_i \leq \norm{a}_2 \norm{b_2} $
		\item If $ 0 < m \leq \frac{a_i}{b_i} \leq M $, then $ (\sum a_i^2) (\sum b_i^2) \leq \frac{(m + M)^2}{4mM} (\sum a_i b_i)^2 $
		\item \textbf{Holder's:} $ \sum_i a_i^{w_a} b_i^{w_b} \cdots z_i^{w_z} \leq (\sum_i a_i)^{w_a} (\sum_i b_i)^{w_b} \cdots (\sum_i z_i)^{w_z} $
	\end{itemize}
	\item \textbf{Rearrangement}
	\begin{itemize}
		\item For all permutations $ \sigma $, $ \sum a_{\sigma_i}b_i \leq \sum a_{(i)} b_{(i)} $
		\item \textbf{Chebyshev's:} $ (\frac{1}{n} \sum a_i) (\frac{1}{n} \sum b_i) \leq \frac{1}{n} \sum a_{(i)} b_{(i)} $
	\end{itemize}
	\item \textbf{Minkowski:} $ \norm{X + Y}_p \leq \norm{X}_p + \norm{Y}_p $
	\item \textbf{Schur's:} If $ a, b, c > 0 $, then $ a^r (a-b) (a-c) + b^r (b-a) (b-c) + c^r (c-a) (c-b) \geq 0 $
\end{itemize}

\subsection{Random Variable}
\textbf{Classic}
\begin{itemize}
	\item \textbf{Markov:} If $ X \geq 0, a > 0 $, then $ \Prob{X \geq a} \leq \frac{\E{X}}{a} $
	\begin{itemize}
		\item If $ \varphi $ is monotonically increasing, then $ \Prob{|X| \geq c} \leq \frac{\E{\varphi(|X|)}}{\varphi(c)} $
		\begin{itemize}
			\item $ \Prob{|X| \geq c} \leq \frac{\E{|X|^n}}{c^n} $
		\end{itemize}
		\item \textbf{Chebyshev:} $ \Prob{|X - \E{X}| \geq k \sigma} \leq \frac{1}{k^2} $
	\end{itemize}
	\item \textbf{Jensen's:} If $ f $ is convex, then $ f(\E{X}) \leq \E{f(X)} $
	\item \textbf{Cauchy-Schwarz}
	\begin{itemize}
		\item $ |\E{XY}| \leq \E{|XY|} \leq (\E{|X|^2})^{1/2} (\E{|Y|^2})^{1/2} $
		\item \textbf{Holder:} If $ 1/p + 1/q = 1 $, then $ \E{|XY|} \leq (\E{|X|^p})^{1/p} (\E{|Y|^q})^{1/q} $
	\end{itemize}

\end{itemize}

\textbf{Bounded Variable}
\begin{itemize}
	\item \textbf{Berstein:} If $ \E{X_i} = 0, |X_i| \overset{\astext}{\leq} M $, then $ \Prob{\sum X_i \geq t} \leq \exp(- \frac{t^2}{2 \sum E{X_i}^2 + 2Mt/3}) $
	\item \textbf{Kantorovich:} If $ 0 < m \le X \le M < \infty $, then $ 1 \le \E{X} \E{\frac{1}{X}} \leq \frac{(m+M)^2}{4mM} $
\end{itemize}

\newpage
\section{Convergence}
\subsection{Definition}
$ X_n \toas X: \Prob{\lim_{n\to\infty} X_n = X} = 1 $
\begin{itemize}
	\item \textbf{Borel-Cantelli:} $ \sum_n \Prob{\abs{X_n - X} > \varepsilon} < \infty \Rightarrow X_n \toas X $
	\item \textbf{Fatou's Lemma:} $ X_n \toas X \Rightarrow X_n \toprob X \Rightarrow X_n \todist X $
	\item \textbf{Dominated convergence:} $ X_n \toas X $ and $ X_n $ is u.i. $ \Rightarrow X_n \toL{1} X $
\end{itemize}

$ X_n \toprob X: \lim_{n\to\infty} \Prob{|X_n - X| > \varepsilon} = 0 $
\begin{itemize}
	\item \textbf{Continuous mapping:} $ X_n \toprob X \Rightarrow f(X_n) \toprob f(X) $ for continuous $f$
	\item $ X_n \toprob X $ and $ X_n $ is u.i. $ \iff X_n \toL{p} X $
\end{itemize}

$ X_n \toL{p} X: \lim_{n\to\infty} \E{\abs{X_n - X}^p} = 0 $
\begin{itemize}
	\item $ X_n \toL{r} X \Rightarrow X_n \toL{s} X $ for $ r > s \geq 1 $
	\item $ X_n \toL{p} X \Rightarrow \lim_{n\to\infty} \E{\abs{X_n}^p} = \E{\abs{X}^p} $
	\item \textbf{Markov:} $ X_n \toL{p} X \Rightarrow X_n \toprob X $
\end{itemize}

$ X_n \todist X: \lim_{n\to\infty} F_n(x) = F(x) $ at which $ F $ is continuous
\begin{itemize}
	\item $ X_n \todist X \iff \lim_{n\to\infty} \E{f(X_n)} = \E{f(X)} $ for all bounded and continuous $f$
	\item \textbf{Continuous mapping:} $ X_n \todist X \Rightarrow f(X_n) \todist f(X) $ for continuous $f$
	\item \textbf{Levy's continuity:} $ X_n \todist X \iff \varphi_{X_n}(t) \to \varphi_X(t) $ pointwise
	\item $ X_n \todist c \Rightarrow X_n \toprob c $ for a constant $c$
\end{itemize}

\newpage
\section{Estimation and Testing}
\subsection{Estimation}
\subsubsection{Method of Moments (MOM)}

\textbf{Algo:} $ \hat{\mu}_p = \frac{1}{n} \sum_i X_i^p $

\textbf{Pros:} consistent, asym unbiased

\textbf{Cons:} could be wrong, not efficient

\subsubsection{Maximum Likelihood Estimator (MLE)}

\textbf{Algo:} $ \hat{\thetabm} = \argmax_{\thetabm} \log \ell(\thetabm; X_{1:n}) $

\textbf{Pros:} consistent, asym normal, asym efficient (unbiased + CRLB)

\textbf{Cons:} sensitive (outlier), computationally expensive

\textbf{Asym normal}
\begin{itemize}
	\item Fisher: $ \Fbm_n(\thetabm) = \E{\fracpartial{\log\ell(\thetabm; X_{1:n})}{\thetabm} \fracpartial{\log\ell(\thetabm; X_{1:n})}{\thetabm^T}} = -\E{\fracpartialtwo{\log\ell(\thetabm; X_{1:n})}{\thetabm}{\thetabm^T}} $
	\item Normal: $ \sqrt{n}(\hat{\thetabm}-\thetabm_0) \sim \Ncal(\zerobm, \bar{\Fbm}^{-1}) $ where $ \bar{\Fbm} = \limntoinfty \frac{1}{n} \Fbm_n(\thetabm^*) $
	\item \textbf{CRLB:} $ \Var(\hat{\theta}) > \frac{1}{F_n(\theta)} $ for unbiased $ \hat{\theta} $
\end{itemize}

\textbf{EM algo}
\begin{itemize}
	\item Ideal obs: $ X \sim p(x, \thetabm) $ where $ l_p(\thetabm; x) $ is easy to optimize
	\item Actual obs: $ S(X) \sim q(s, \thetabm) $ where $ l_{q}(\thetabm; s) $ is difficult to optimize
	\item E-Step: $ J(\thetabm | \thetabm_{old}) = \E{\log \frac{p(X|\thetabm)}{p(X|\thetabm_{old})} | S(X) = s} $
	\item M-Step: $ \thetabm_{new} = \argmax J(\thetabm | \thetabm_{old}) $
\end{itemize}

\subsection{General Testing}

\subsubsection{Likelihood Ratio Test (LRT)}

$ H_0: \mu = \mu_0 $ vs. $ H_A: \mu = \mu_A $
\begin{itemize}
	\item Test stat: $ \Lambda = \frac{f(X|H_0)}{f(X|H_A)} $
	\item Reject: small value of $ \Lambda(X) $
	\item \textbf{Most powerful:} \textbf{simple} null vs. \textbf{simple} alternative
\end{itemize}

\subsubsection{Generalized Ratio Test}

$H_0:$ \textbf{composite} vs. $H_A:$ \textbf{composite}
\begin{itemize}
	\item Test stat: $ \Lambda = \frac{\max_{\theta\in H_0} f(X|\theta)}{\max_{\theta\in H_0\cup H_A} f(X|\theta)} \Rightarrow -2\log\Lambda \sim \chi^2_{\dim\Omega - \dim\omega_0} $ as $ n \rightarrow \infty $
	\item Reject: small value of $ \Lambda(X) $ or large value of $ -2\log \Lambda $
\end{itemize}

\textbf{Poisson Dispersion Test:}
\begin{itemize}
	\item $ X_i \sim \text{Poi}(\lambda_i)$, $ H_0: \lambda_i = \lambda $
	\item Test stat: $ -2\log\Lambda \sim \chi^2_{n-1} $; Reject when $ -2\log\Lambda > \chi^2_{n-1}(\alpha) $
\end{itemize}

\subsection{One-sample}

\subsubsection{Parametric}

\textbf{Bernoulli:} $ X_i \sim \Ber(p) $
\begin{itemize}
	\item $ \hat{p} = \bar{X} \Rightarrow \frac{\hat{p} - p}{\sqrt{p(1-p)/n}} \sim \Ncal(0,1) $
	\item $ H_0: p = p_0 $; Test stat: $ Z = \frac{\hat{p} - p_0}{\sqrt{p_0(1-p_0)/n}} $
\end{itemize}

\textbf{Normal mean:} $ X_i \sim N(\mu, \sigma^2) $
\begin{itemize}
	\item Known variance: $ \hat{\mu} = \bar{X} \Rightarrow \frac{\bar{X} - \mu}{\sigma/\sqrt{n}} \sim \Ncal(0,1) $
	\item Unknown variance: $ \hat{\mu} = \bar{X} \Rightarrow \frac{\bar{X} - \mu}{s/\sqrt{n}} \sim t_{n-1} $
	\begin{itemize}
		\item Proof: $ \frac{\bar{X} - \mu}{s/\sqrt{n}} = \frac{\bar{X} - \mu}{\sigma/\sqrt{n}} / \sqrt{s^2/\sigma^2}  = \frac{Z}{\sqrt{\chi_{n-1}^2 / (n - 1)}} $
	\end{itemize}
	\item $ H_0: \mu = \mu_0 $; Test stat: $ Z = \frac{\bar{X} - \mu_0}{\sigma/\sqrt{n}} $ or $ T = \frac{\bar{X} - \mu_0}{s/\sqrt{n}} $
\end{itemize}

\textbf{Normal variance:} $ X_i \sim N(\mu, \sigma^2) $
\begin{itemize}
	\item $ \hat{\sigma}^2 = s^2 \Rightarrow \frac{(n-1)s^2}{\sigma^2} \sim \chi^2_{n-1} $
	\begin{itemize}
		\item Proof: $ \frac{(n-1)s^2}{\sigma^2} + \frac{(\bar{X} - \mu)^2}{\sigma^2/n} = \sum_{i=1}^n \frac{(X_i-\mu)^2}{\sigma^2} \sim \chi^2_{n} $
	\end{itemize}
	\item $ H_0: \sigma^2 = \sigma_0^2 $; Test stat: $ X^2 = \frac{(n-1)s^2}{\sigma_0^2} $
\end{itemize}

\subsubsection{Non-parametric}

\textbf{Shapiro-Wilk (SW) Test:} $ H_0: X \sim \Ncal(\cdot, \cdot) $
\begin{itemize}
	\item Test stat: $ W = \frac{(\sum_i a_i X_{(i)})^2}{\sum_i (X_i - \bar{X})^2} $
	\item Pros: best power for normality test
	\item Cons: bad if many identical values
\end{itemize}

\textbf{Anderson-Darling (AD) Test:} $ H_0: X \sim F(x) $
\begin{itemize}
	\item Test stat: $ A^2 = -n - \sum_i \frac{2i - 1}{n} [\log F(X_{(i)}) - \log (1 - F(X_{(n+1-i)}))] $
	\item Cons: more weight on tails of distribution
	\item \textbf{Normality test:} $ H_0: \frac{X - \hat{\mu}}{\hat{\sigma}} \sim \Ncal(0,1) $, second best power
\end{itemize}

\textbf{Kolmogorov–Smirnov (KS) Test:} $ H_0: X \sim F(x) $
\begin{itemize}
	\item Test stat: $ D_n = \sup_x |F_n(x) - F(x)| $
	\begin{itemize}
		\item \textbf{Kolmogorov theorem:} $ \sqrt{n} D_n \todist \sup_t |B(F(t))| $ (Kolmogorov distribution)
	\end{itemize}
	\item \textbf{Normality test (Lilliefors):} $ H_0: \frac{X - \hat{\mu}}{\hat{\sigma}} \sim \Ncal(0,1) $
	\item \textbf{Homogeneity test:} $ X = Y $ (in distribution sense)
	\begin{itemize}
		\item Test stat: $ D_{n,m} = \sup_x |F_{1,n}(x) - F_{2,m}(x)| $
	\end{itemize}
\end{itemize}

\subsection{Two-sample}

\subsubsection{Parametric}

\textbf{Bernoulli:} $ X_i \sim \Ber(p_X), Y_j \sim \Ber(p_Y) $
\begin{itemize}
	\item $ \hat{p}_X - \hat{p}_Y = \bar{X} - \bar{Y} \Rightarrow \frac{\hat{p}_X-\hat{p}_Y - (p_X-p_Y)}{\sqrt{p_X(1-p_X) / n + p_Y(1-p_Y) / m}} \sim \Ncal(0,1) $
	\item $ H_0: p_X = p_Y $; Test stat: $ Z = \frac{\hat{p}_X-\hat{p}_Y}{\sqrt{\hat{p} (1-\hat{p}) / n + \hat{p} (1-\hat{p}) / m}} $, $ \hat{p} = \frac{n\bar{X} + m\bar{Y}}{n + m} $
	\item $ H_0: p_X - p_Y = D_0 $; Test stat: $ Z = \frac{\hat{p}_X-\hat{p}_Y - D_0}{\sqrt{\hat{p}_X (1-\hat{p}_X) / n + \hat{p}_Y (1-\hat{p}_Y) / m}} $
\end{itemize}

\textbf{Normal mean:} $ X_i \sim N(\mu_X, \sigma_X^2) $, $ Y_j \sim N(\mu_Y, \sigma_Y^2) $
\begin{itemize}
	\item  Known variances: $ \hat{\mu}_X - \hat{\mu}_Y = \bar{X} - \bar{Y} \Rightarrow \frac{\bar{X} - \bar{Y} - (\mu_X - \mu_Y)}{\sqrt{\sigma_X^2/n + \sigma_Y^2/m}} \sim \Ncal(0,1) $
	\item Equal variances: $ \hat{\mu}_X - \hat{\mu}_Y = \bar{X} - \bar{Y} \Rightarrow \frac{\bar{X} - \bar{Y} - (\mu_X - \mu_Y)}{\sqrt{s_p^2 / n + s_p^2 / m}} \sim t_{n+m-2} $
	\begin{itemize}
		\item $ s_p^2 = \frac{(n-1)s_X^2 + (m-1)s_Y^2}{n+m-2} $
		\item \textbf{Practical rule:} $ 1 / 3 < s_X^2 / s_Y^2 < 3 $
	\end{itemize}
	\item Unequal variances: $ \hat{\mu}_X - \hat{\mu}_Y = \bar{X} - \bar{Y} \Rightarrow \frac{\bar{X} - \bar{Y} - (\mu_X - \mu_Y)}{\sqrt{s_X^2 / n + s_Y^2 / m}} \sim t_{\nu} $
	\begin{itemize}
		\item $ \nu = \frac{(s_X^2/n + s_Y^2/m)^2}{(s_X^2/n)^2 / (n-1) + (s_Y^2/m)^2 / (m-1)} $
	\end{itemize}
	\item $ H_0: \mu_X - \mu_Y = D_0 $
	\item Test stat: $ Z = \frac{\bar{X} - \bar{Y} - D_0}{\sqrt{\sigma_X^2/n + \sigma_Y^2/m}} $ or $ T = \frac{\bar{X} - \bar{Y} - D_0}{\sqrt{s_p^2 / n + s_p^2 / m}} $ or $ T = \frac{\bar{X} - \bar{Y} - D_0}{\sqrt{s_X^2 / n + s_Y^2 / m}} $
\end{itemize}

\textbf{Normal variance:} $ X_i \sim N(\mu_X, \sigma_X^2) $, $ X_j \sim N(\mu_Y, \sigma_Y^2) $
\begin{itemize}
	\item $ \frac{\hat{\sigma}_X^2}{\hat{\sigma}_Y^2} = \frac{s_X^2}{s_Y^2} \Rightarrow \frac{s_X^2/\sigma_X^2}{s_Y^2/\sigma_Y^2} \sim F_{n-1, m-1} $
	\item $ H_0: \frac{\sigma_X^2}{\sigma_Y^2} = r_0 $; Test stat: $ F = \frac{s_X^2}{s_Y^2}\frac{1}{r_0} $
\end{itemize}

\subsubsection{Non-parametric}

\textbf{Pearson's chi-squared test:} goodness of fit; homogeneity; independence
\begin{itemize}
	\item Test stat: $ \chi^2 = \sum_i \frac{(O_i - E_i)^2}{E_i} $ or $ \chi^2 = \sum_{i,j} \frac{(O_{i,j} - E_{i,j})^2}{E_{i,j}} $
\end{itemize}

\textbf{Fisher exact test:} homogeneity in contingency tables
\begin{itemize}
	\item $ H_0: X \overset{\Dcal}{=} Y $ with binary $ X $ and $ Y $
	\item P-value: $ p = \binom{X_1 + Y_1}{X_1} \binom{X_2 + Y_2}{X_2} / \binom{n_X + n_Y}{n_X} $
\end{itemize}

\textbf{Wilcoxon rank-sum test:} $ X \overset{\Dcal}{\geq} Y $
\begin{itemize}
	\item Test stat: $ U = \sum_i \sum_j S(X_i, Y_j) $ where $ S(X, Y) = \{1, 0.5, 0\} $ by comparing $ X $ and $ Y $
\end{itemize}

\textbf{Wilcoxon signed-rank test:} $ X \overset{\Dcal}{=} Y $
\begin{itemize}
	\item Test stat: $ W = \sum_i \sgn(X_i - Y_i) R_i $
	\item \textbf{Cons:} discard identical pairs
\end{itemize}

\subsection{Multi-sample}

\subsubsection{One-factor ANOVA}
\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|}
		\hline
		\textbf{Data} &\\
		\hline
		Factor 1 & $ Y_{11}, \cdots, Y_{1J} $\\
		\hline
		\vdots & \vdots\\
		\hline
		Factor I & $ Y_{I1}, \cdots, Y_{IJ} $\\
		\hline
	\end{tabular}
\end{table}

\textbf{Goal:} Test if factor means are different

\textbf{Normal:} $ Y_{ij} \sim N(\mu+\alpha_{i}, \sigma^2) $ with $ \sum_{i=1}^{I} \alpha_i = 0 $
\begin{itemize}
	\item $ SSB = \sum_{i}n_i(\bar{Y}_{i\cdot}-\bar{Y}_{\cdot\cdot})^2 $, $ SSE = \sum_{i,j}(Y_{ij}-\bar{Y}_{i\cdot})^2 $, $ SST = \sum_{i,j} (Y_{ij}-\bar{Y}_{\cdot\cdot})^2 $
	\item $ SSB + SSE = SST $
	\item d.f.: $ (I-1) + I(J-I) = n - 1 $
\end{itemize}

\textbf{Estimation}
\begin{itemize}
	\item $ \hat{\mu} = \bar{Y}_{\cdot\cdot}$, $ \hat{\alpha}_i = \bar{Y}_{i\cdot} - \bar{Y}_{\cdot\cdot} $, $ \hat{\sigma}^2 = s_p^2 = MSE = SSE/(I(J-1)) $
	\item \textbf{C.I.} for $ \mu + \alpha_i $: $\bar{Y}_{i\cdot} \pm t_{n-I}(\alpha/2)\sqrt{\frac{MSE}{n_i}} $
	\item \textbf{C.I.} for $ \alpha_{i} - \alpha_{j} $ (Bonferroni): $ (\bar{Y}_{i\cdot}-\bar{Y}_{j\cdot}) \pm t_{n-I}((\alpha/2)/\left(\begin{smallmatrix}
		I\\2\end{smallmatrix}\right))\sqrt{\frac{MSE}{n_i}+\frac{MSE}{n_j}} $
	\item \textbf{C.I.} for $ \alpha_{i} - \alpha_{j} $ (Tukey):
	$ (n_1 = \cdots = n_I = J) $: $ (\bar{Y}_{i\cdot} - \bar{Y}_{j\cdot}) \pm q_{I, n-I}(\alpha)\sqrt{\frac{MSE}{J}} $
\end{itemize}

\textbf{Testing}
\begin{itemize}
	\item $ H_0: \alpha_1 = \cdots = \alpha_I = 0 $
	\begin{itemize}
		\item Test stat: $ F = \frac{SSB/(I-1)}{SSE/(I(J-1))} = \frac{MSB}{MSE} \sim F_{I-1, I(J-1)} $
		\item Reject: $ F > F_{I-1, I(J-1)}(\alpha) $ (only reject for \textbf{large} values)
	\end{itemize}
\end{itemize}

\subsubsection{Randomized Block Design}
\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|c|}
		\hline
		\textbf{Data} & Factor B1 & $\cdots$ & Factor BJ\\
		\hline
		Factor A1 & $ Y_{11} $ & $\cdots$ & $ Y_{1J} $\\
		\hline
		\vdots & \vdots & \vdots & \vdots \\
		\hline
		Factor AI & $ Y_{I1} $ & $\cdots$ & $ Y_{IJ} $\\
		\hline
	\end{tabular}
\end{table}

\textbf{Normal:} $ Y_{ij} \sim N(\mu + \alpha_i + \beta_j, \sigma^2) $ with $ \sum_i \alpha_i = \sum_j \beta_j = 0 $
\begin{itemize}
	\item $ SSA = \sum_{i} J(\bar{Y}_{i\cdot} - \bar{Y}_{\cdot\cdot})^2 $, $ SSB =  \sum_{j} I(\bar{Y}_{\cdot j} - \bar{Y}_{\cdot\cdot})^2 $, $ SST =  \sum_{i,j} (Y_{ij} - \bar{Y}_{\cdot\cdot})^2 $
	\item $ SSE =  \sum_{i,j} (Y_{ij} - \bar{Y}_{i\cdot} - \bar{Y}_{\cdot j} + \bar{Y}_{\cdot\cdot})^2 $; $ SSA + SSB + SSE = SST $
	\item d.f.: $ (I-1) + (J-1) + (I-1)(J-1) = (IJ - 1) $
\end{itemize}

\textbf{Estimation}
\begin{itemize}
	\item $ \hat{\mu} = \bar{Y}_{\cdot\cdot}$, $ \hat{\alpha}_i = \bar{Y}_{i\cdot} - \bar{Y}_{\cdot\cdot} $, $ \hat{\sigma}^2 = s_p^2 = MSE = SSE/((I-1)(J-1)) $
	\item \textbf{C.I.} for $ \mu + \alpha_i $: $\bar{Y}_{i \cdot} \pm t_{(I-1)(J-1)}(\alpha/2)\sqrt{\frac{MSE}{J}} $
	\item \textbf{C.I.} for $ \mu + \beta_j $: $\bar{Y}_{\cdot j} \pm t_{(I-1)(J-1)}(\alpha/2)\sqrt{\frac{MSE}{I}} $
	\item \textbf{C.I.} for $ \alpha_{i} - \alpha_{j} $ (Two-sample): $ (\bar{Y}_{i\cdot} - \bar{Y}_{j\cdot}) \pm t_{(I-1)(J-1)}(\alpha/2)\sqrt{\frac{MSE}{J} + \frac{MSE}{J}} $
	\item \textbf{C.I.} for $ \alpha_{i} - \alpha_{j} $ (Tukey): $ (\bar{Y}_{i\cdot} - \bar{Y}_{j\cdot}) \pm q_{I, (I-1)(J-1)}(\alpha)\sqrt{\frac{MSE}{J}} $
	\item \textbf{C.I.} for $ \beta_{i} - \beta_{j} $ (Two-sample): $ (\bar{Y}_{\cdot i} - \bar{Y}_{\cdot j}) \pm t_{(I-1)(J-1)}(\alpha/2)\sqrt{\frac{MSE}{I}+\frac{MSE}{I}} $
	\item \textbf{C.I.} for $ \beta_{i} - \beta_{j} $ (Tukey): $ (\bar{Y}_{\cdot i} - \bar{Y}_{\cdot i}) \pm q_{J, (I-1)(J-1)}(\alpha)\sqrt{\frac{MSE}{I}} $
\end{itemize}

\textbf{Testing}
\begin{itemize}
	\item $ H_0: \alpha_1 = \cdots = \alpha_I = 0 $
	\begin{itemize}
		\item Test stat: $ F = \frac{SSA/(I-1)}{SSE/((I-1)(J-1))} = \frac{MSA}{MSE} \sim F_{I-1, (I-1)(J-1)} $
	\end{itemize}
	\item $ H_0: \beta_1 = \cdots = \beta_J = 0 $
	\begin{itemize}
		\item Test stat: $ F = \frac{SSB/(J-1)}{SSE/((I-1)(J-1))} = \frac{MSB}{MSE} \sim F_{J-1, (I-1)(J-1)} $
	\end{itemize}
\end{itemize}

\subsubsection{Two-factor ANOVA}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|c|c|}
		\hline
		\textbf{Data} & Factor B1 & $\cdots$ & Factor BJ\\
		\hline
		Factor A1 & $ Y_{111}, \cdots, Y_{11K} $ & $\cdots$ & $ Y_{1J1}, \cdots, Y_{1JK} $\\
		\hline
		\vdots & \vdots & \vdots & \vdots \\
		\hline
		Factor AI & $ Y_{I11}, \cdots, Y_{I1K} $ & $\cdots$ & $ Y_{IJ1}, \cdots, Y_{IJK} $\\
		\hline
	\end{tabular}
\end{table}

\textbf{Normal:} $ Y_{ijk} \sim N(\mu+\alpha_{i}+\beta_{j}+\delta_{ij}, \sigma^2) $ with $ \sum_i \alpha_i = \sum_{j}\beta_j = \sum_{i} \delta_{ij} = \sum_{j} \delta_{ij} = 0 $
\begin{itemize}
	\item $ SSA = \sum_{i} JK(\bar{Y}_{i\cdot\cdot} - \bar{Y}_{\cdot\cdot\cdot})^2 $, $ SSB =  \sum_{j} IK(\bar{Y}_{\cdot j\cdot} - \bar{Y}_{\cdot\cdot\cdot})^2 $, $ SST =  \sum_{i,j,k} (Y_{ijk} - \bar{Y}_{\cdot\cdot\cdot})^2 $,
	\item $ SSAB =  \sum_{i,j} K(Y_{ij\cdot} - \bar{Y}_{i\cdot\cdot} - \bar{Y}_{\cdot j\cdot} + \bar{Y}_{\cdot\cdot\cdot})^2 $, $ SSE = \sum_{i,j,k}(Y_{ijk} - \bar{Y}_{ij\cdot})^2 $
	\item $ SSA + SSB + SSAB + SSE = SST $
	\item d.f.: $ (I-1) + (J-1) + (I-1)(J-1) + IJ(K-1) = (IJK - 1) $
\end{itemize}

\textbf{Estimation}
\begin{itemize}
	\item $ \hat{\sigma}^2 = s_p^2 = MSE = SSE/(IJ(K-1)) $
	\item \textbf{C.I.} (Tukey with significant interaction): $ (\bar{Y}_{ij\cdot} - \bar{Y}_{i'j'\cdot}) \pm q_{IJ, J(K-1)}(\alpha)\sqrt{\frac{MSE}{K}} $
\end{itemize}

\textbf{Testing}
\begin{itemize}
	\item $ H_0: \alpha_1 = \cdots = \alpha_I = 0 $
	\begin{itemize}
		\item Test stat: $ F = \frac{SSA/(I-1)}{SSE/(IJ(K-1))} = \frac{MSA}{MSE} \sim F_{I-1, IJ(K-1)} $
	\end{itemize}
	\item $ H_0: \beta_1 = \cdots = \beta_J = 0 $
	\begin{itemize}
		\item Test stat: $ F = \frac{SSB/(J-1)}{SSE/(IJ(K-1))} = \frac{MSB}{MSE} \sim F_{J-1, IJ(K-1)} $
	\end{itemize}

	\item $ H_0: \delta_{11} = \cdots = \delta_{IJ} = 0 $
	\begin{itemize}
		\item Test stat: $ F = \frac{SSAB/((I-1)(J-1)}{SSE/(IJ(K-1))} = \frac{MSAB}{MSE} \sim F_{(I-1)(J-1), IJ(K-1)} $
	\end{itemize}
\end{itemize}

\newpage

\section{Bootstrap}
\textbf{Bootstrap C.I.}: bootstrap samples: $ \theta^* $
\begin{itemize}
	\item Percentile.: $ (\theta^*_L, \theta^*_U) $
	\item Normal approx: $ (\hat{\theta} - z(\alpha/2)s_{\theta^*}, \hat{\theta} + z(\alpha/2) s_{\theta^*}) $
	\item Dist approx: $ (2\hat{\theta}-\theta^*_U, 2\hat{\theta}-\theta^*_L) $
	\begin{itemize}
		\item $ P(\Delta_L\leq\hat{\theta}-\theta_0\leq\Delta_U) = 1 - \alpha \Rightarrow $ C.I.: $ (\hat{\theta}-\Delta_U, \hat{\theta}-\Delta_L) $
		\item $ \Delta = \hat{\theta}-\theta_0 \approx \theta^* - \hat{\theta} \Rightarrow \Delta_L \approx \theta^*_L - \hat{\theta}, \Delta_U \approx \theta^*_U - \hat{\theta} \Rightarrow $ C.I.: $ (2\hat{\theta}-\theta_U^*, 2\hat{\theta}-\theta_L^*) $
		\item If $ \hat{\theta} - \theta^*_L = \theta^*_U - \hat{\theta} $, then $ 2\hat{\theta}-\theta^*_U = \theta^*_L $ and $ 2\hat{\theta}-\theta^*_L =\theta^*_U $
	\end{itemize}
\end{itemize}

\newpage
\section{Time Series}

\subsection{Autoregression (AR)}
$ AR(p): X_t = c + \sum_{i=1}^p \varphi_i X_{t-i} + \varepsilon_t $

\textbf{Variation}
\begin{itemize}
	\item Vector autogression (VAR)
\end{itemize}

\subsection{Moving Average (MA)}
$ MA(q): X_t = c + \sum_{i=1}^q \theta_i \varepsilon_{t-i} + \varepsilon_t $

\subsection{ARMA}
$ ARMA(p, q): X_t = c + \sum_{i=1}^p \varphi_i X_{t-i} + \sum_{i=1}^q \theta_i \varepsilon_{t-i} + \varepsilon_t $

\subsection{ARIMA}
Model: differences is stationary

\subsection{SARIMA}

\newpage
\section{Survival Analysis}
\textbf{Survival function:} $ S(t) = \Prob{T > t} $

\subsection{Kaplan-Meier Estimator}
$ \hat{S}(t) = \prod_{i: t_i \leq t} (1 - \frac{d_i}{n_i}) $
\begin{itemize}
	\item $ t_i $: event time before $ t $
	\item $ d_i $: deaths at time $ t_i $
	\item $ n_i $: patients known to have survived up to time $ t_i $
\end{itemize}

\subsection{Cox Propotional Hazards Model}
$ \lambda(t) $

\newpage
\section{Sampling Method}

\subsection{Important Sampling}
\textbf{Algo:} $ \Esub{X \sim f}{H(X)} \approx \frac{1}{n} \sum H(x_i) \frac{f(x_i)}{g(x_i)} $ with $ x_i \sim g(x) $

\subsection{Inverse Transformation}
Inverse CDF

\subsection{Accept-Reject}

\subsection{Markov chain Monte Carlo (MCMC)}
\textbf{Effective sample size:} $ \hat{\mu} = \sum_{i=1}^{n} X_i \Rightarrow \var(\hat{\mu}) = \frac{\sigma^2}{n_{\text{eff}}} $
\begin{itemize}
	\item $ n_{\text{eff}} = \frac{n}{1 + 2 \sum_{k=1}^{\infty} \rho_k} $ with $ \rho_k $ being the $k$-lag autocorrelation $ \Rightarrow $ large correlation is bad
	\item Weight samples to counter correlation
\end{itemize}

\subsection{Metropolis-Hasting Algorithm}
\textbf{Algorithm}
\begin{itemize}
	\item Propose $ \xbm' \sim \pi(\xbm' | \xbm) \Rightarrow $ accept with $ \alpha(\xbm, \xbm') = \min \cu{\frac{p(\xbm')}{p(\xbm)} \frac{\pi(\xbm | \xbm')}{\pi(\xbm' | \xbm)}, 1} $
	\item Reward for $ p(\xbm') $ and penalize for $ \pi(\xbm' | \xbm) $
	\item \textbf{Reversible:} $ p(\xbm) K(\xbm' | \xbm) = p(\xbm') K(\xbm | \xbm') $
\end{itemize}

\textbf{Pros:} No need to know normalizing constant

\textbf{Random walk:} $ \xbm' \sim \Ncal(\xbm, \sigma^2\Ibm) $
\begin{itemize}
	\item Small $ \sigma^2 \Rightarrow $ small jump w/ high accept rate and high autocorrelation
	\item Large $ \sigma^2 \Rightarrow $ large jump w/ low accept rate
\end{itemize}
\textbf{Metropolis:} symmetric proposal

\begin{pyminted}
	def target_pdf(x):
		return 0 if x < 0 else np.exp(-x)

	x = np.random.normal()
	samples = [x]
	for i in range(1000):
		x_cand = x + np.random.normal() # symmetric proposal
		ratio = target_pdf(x_cand) / target(x)
		if np.random.uniform() < ratio:
			x = x_cand
			samples.append(x)
\end{pyminted}

\subsection{Gibbs Sampling}
\textbf{Algorithm}
\begin{itemize}
	\item Sample $ x_1' \sim \pi(x_1' | x_2, \cdots, x_d) \Rightarrow $ sample $ x_2' \sim \pi(x_2' | x_1', x_3, \cdots, x_d) $
\end{itemize}

\textbf{Pros:} 100\% acceptance, good for high dimension, can deal mixed variables

\textbf{Cons:} Require all conditional probabilities

\subsection{Hamiltonian Monte Carlo}
\textbf{Hamiltonian total energy:} $ H(\xbm, \vbm)  = U(\xbm) + K(\vbm) $
\begin{itemize}
	\item Potential energy: $ U(\xbm) = - \log p(\xbm) $; Kinetic energy: $ K(\vbm) = -\log q(\vbm) $
	\item Trajectory: $ \xbm^{(k)} \to \wbm $
	\begin{itemize}
		\item Dynamics: $ \begin{cases}
			\frac{dx}{dt} = \partial_\vbm H(\xbm, \vbm) = \partial_\vbm K(\vbm)\\
			\frac{dv}{dt} = \partial_\xbm H(\xbm, \vbm) = \partial_\xbm U(\vbm)\\
		\end{cases} $
		\item Numerical: Leapfrog method (symplectic method for preserving total energy)
	\end{itemize}
\end{itemize}

\textbf{Algorithm}
\begin{itemize}
	\item Sample $ \vbm \sim q(\vbm) $
	\item Numerically find trajectory $ \xbm \to \xbm' $
	\item Accept with $ \alpha(\xbm, \xbm') = \min \cu{\frac{e^{-H(t=T)}}{e^{-H(t=0)}}, 1} $
\end{itemize}

\textbf{Pros:} High accept rate, little or no auto-correlation

\textbf{Cons:} need derivatives of $ p(\xbm) $ and $ q(\vbm) $, only for continuous distribution

\subsection{Reversible-Jump MCMC}
Parameter space: $ (d, \xbm) $

\textbf{Dimension matching:} $ \dim(\xbm, \wbm) = \dim(\xbm', \wbm') $

\textbf{Algorithm}: $ (d, \xbm, \wbm) \to (d', \xbm') $
\begin{itemize}
	\item Sample jump $ d' \sim \pi(d' | d) $
	\item Sample $ \xbm', \wbm' \sim \pi(\xbm' | d', d, \xbm)$
	\item Set $ (\xbm, \wbm) = g(\xbm', \wbm') $
	\item Accept with $ \alpha(\xbm, \xbm') = \min \cu{\frac{p(\xbm')}{p(\xbm)} \frac{\pi(d | d')}{\pi(d' | d)} \det \absbig{\fracpartial{(\xbm', \wbm')}{(\xbm, \wbm)}}, 1} $
\end{itemize}

\newpage

\section{Bayesian}
\subsection{Conjugate Priors}
Likelihood ($k$-param): $ p(\xbm|\thetabm) = h(\xbm) \exp(\etabm(\thetabm)^T \Tbm(\xbm) - B(\thetabm)) $

Prior ($(k+1)$-param): $ \pi(\thetabm|\tbm) = \exp(\sum_{j=1}^{k} \eta_j(\thetabm) t_j - t_{k+1} B(\thetabm) - \log \omega(\tbm)) $

\textbf{Example}
\begin{itemize}
	\item $ \pi(p) \sim \Beta(\alpha,\beta) + \Bin(n,p) \to \Beta(\alpha+N,\beta+n-N) $
	\item $ \pi(\lambda) \sim \Gamma(a,b) + \Poi(\lambda) \to \Gamma(a+S,b+n) $
	\item $ \pi(\lambda) \sim  \Gamma(a,b) + \Exp(\lambda) \to \Gamma(a+n,b+s) $
	\item $ \mu \sim \mathcal{N}(a,b^2) + \Ncal(\mu, \sigma^2) \to \Ncal(\mu^*, \sigma^{2*}) $
\end{itemize}

\end{document}


\subsection{Monoalphabetic substitution ciphers and keys}

(key plus algorithm encrpyts and decrypts)


\section{Expressions}

\subsection{Syntax}

\begin{frame}{Introduction}
  Let's start with a very simple language.
\end{frame}

\begin{frame}
  \frametitle{Language \lggeE}
  \input{language-e-syntax}
\end{frame}

% \note{A very simple language}

\begin{frame}
  \frametitle{The limits of syntax}
  \label{fr:syntax-e}
  Do all syntactically correct terms make sense?
  %
  \pause

  \bigskip

  Examples:
  \begin{itemize}[<+->]
  \item $\eplus{\enum{3}}{\enum{4}}$
  \item $\elet{4}{x}{\eplus{\var{x}}{15}}$
  \item $\ecat{\enum{42}}{\estr{hello}}$
  \item $\elet{4}{x}{\eplus{\var{y}}{15}}$
  \end{itemize}

\end{frame}

\subsection{Typing system}

\begin{frame}
  \frametitle{Judgements}

  We will use judgements of the form:
  %
  {\Huge
    \[
    {\tyjudge{\Gamma}{e}{\tau} }
    \]
  }
  which means ``$e$ has type $\tau$ under the hypotheses in $\Gamma$''.
  %
  \pause
  %
  \begin{itemize}
  \item $\Gamma$ is a \emph{context} which either empty (written $\varnothing$) or of the form:
    $\var{x}_1 : \tau_1, \ldots, \var{x}_n : \tau_n$ where
    $\forall i\neq j \qst \var{x}_i \neq \var{x}_j$.
    %
  \item $e$ is an expression in $\lggeE$, e.g., $e = \enum{n}$,
    $e = \eplus{e_1}{e_2}$, etc.
    %
  \item $\tau$ is a type in $\lggeE$, i.e., $\tau = \enum{}$ or
    $\tau = \estr{}$.
  \end{itemize}
  %
\end{frame}

\note{

  \begin{itemize}
  \item   We write
    %
    $\dom{\Gamma}$ for the \emph{domain} of $\Gamma$, i.e., if
    $\Gamma = \var{x}_1 : \tau_1, \ldots, \var{x}_n : \tau_n$, then
    $\dom{\Gamma} = \{ \var{x}_1  , \ldots, \var{x}_n \}$.
    %
    %
    %
  \item We take the convention that when we write ``$\Gamma, \var{x} : \tau$'',
    we implicitly assume that $\var{x} \notin\dom{\Gamma}$.
  \end{itemize}

}

\begin{frame}
  \frametitle{Statics}

  The type system of $\lggeE$ is inductively defined by the following rules:

  \[
  \tyrule
  {\etyrulename{var}}
  {\,}
  {\tyjudge{\Gamma_1, \var{x}: \tau, \Gamma_2}{\var{x}}{\tau} }
  \]

  \pause
  \[
  \tyrule
  {\etyrulename{str}}
  {\,}
  {\tyjudge{\Gamma}{\estr{s}}{\estr{}}}
  \qquad\qquad\qquad
  \tyrule
  {\etyrulename{num}}
  {\,}
  {\tyjudge{\Gamma}{\enum{n}}{\enum{}}}
  % \end{array}
  \]

  \pause
  \[
  \tyrule
  {\etyrulename{plus}}
  {
    \tyjudge{\Gamma}{e_1}{\enum{}}
    \and
    \tyjudge{\Gamma}{e_2}{\enum{}}
  }
  {\tyjudge{\Gamma}{\eplus{e_1}{e_2}}{\enum{}}}
  \]

  \pause
  \[
  \tyrule
  {\etyrulename{times}}
  {
    \tyjudge{\Gamma}{e_1}{\enum{}}
    \and
    \tyjudge{\Gamma}{e_2}{\enum{}}
  }
  {\tyjudge{\Gamma}{\etimes{e_1}{e_2}}{\enum{}}}
  %
  \]
\end{frame}

\begin{frame}
  \frametitle{Statics}
  \[
  \tyrule
  {\etyrulename{cat}}
  {
    \tyjudge{\Gamma}{e_1}{\estr{}}
    \and
    \tyjudge{\Gamma}{e_2}{\estr{}}
  }
  {\tyjudge{\Gamma}{\ecat{e_1}{e_2}}{\estr{}}}
  \]

  \bigskip
  \pause

  \[
  \tyrule
  {\etyrulename{len}}
  {
    \tyjudge{\Gamma}{e}{\estr{}}
  }
  {\tyjudge{\Gamma}{\elen{e}}{\enum{}}}
  \]

  \bigskip
  \pause

  \[
  \tyrule
  {\etyrulename{let}}
  {
    \tyjudge{\Gamma}{e_1}{\tau_1}
    \and
    \tyjudge{\Gamma, \var{x} : \tau_1 }{e_2}{\tau}
  }
  {\tyjudge{\Gamma}{\elet{e_1}{x}{e_2}}{\tau}}
  \]
\end{frame}

\begin{frame}
  \frametitle{Well-typed}
  \label{fr:types-e}

  Term $e$ is \emph{well-typed} iff there is $\tau$ such that
  $\tyjudge{\varnothing}{e}{\tau}$ is derivable according the static
  rules of the language.

  \pause
  \bigskip

  Are these $\lggeE$ phrases well-typed?
  \begin{itemize}[<+->]
  \item $\ecat
    {\ecat
      {\estr{\text{hello}}}
      {\estr{\text{world}}}
    }
    {\estr{\text{!}}}$
  \item $\etimes
    {\elen{\estr{\text{hello}}}}
    {\eplus{\enum{1}}{\elen{\estr{\text{world}}}}}$
    %
  \item $\elet{\estr{\text{mystring}}}{x}{\etimes{\elen{\var{x}}}{\enum{0}}}$
    %
  \item $\elet
    {\estr{\text{anotherstring}}}
    {y}
    {\eplus{\var{y}}{\etimes{\enum{1}}{\enum{2}}}}$
    %
  \item $\elet
    {\ecat{\estr{\text{anotherstring}}}{\var{y}}}
    {y}
    {\var{y}}$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Type systems: summary}

  \begin{itemize}
  \item Type systems restrict the set of ``allowed'' programs.
    %
  \item Type systems are defined through a set of (inductively
    defined) typing rules.
    %
  \item Type systems do not say anything about the \emph{meaning} of
    programs.
    %
  \item Type systems enforce \emph{static} properties of programs.
  \end{itemize}
\end{frame}

% \begin{frame}
%   \frametitle{Housekeeping}
%   \begin{itemize}[<+->]
%   \item Recording of lecture \#2: I'll upload last year's
%     lecture.
%   \item Temperature in KLT5
%   \end{itemize}
%   \pause
%   \includegraphics[scale=0.7]{roomtemp.png}
% \end{frame}

\begin{comment}
\begin{frame}
  \frametitle{Last week}
  {\Huge
    \[
    \tyjudge{\Gamma}{e}{\tau}
    \]
  }
  \pause
  \begin{itemize}
  \item $\Gamma$ is an \emph{environment} that keeps track of declared
    variables and their types.
    %
  \item At the start of a type checking derivation, no variable is
    declared, hence $\Gamma = \varnothing$ (i.e., $\Gamma$ is empty).
    %
  \item In language $\lggeE$:
    \begin{itemize}
    \item only rule {\etyrulename{let}} adds elements in $\Gamma$
    \item only rule {\etyrulename{var}} queries $\Gamma$.
    \end{itemize}
  \item The letter $\Gamma$ is a \emph{meta-variable}, it should never
    appear in a type derivation (just like $e_i$ and $\tau$).
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Last week}
  {\small
    \[
    \tyrule{{let}}
    {
      \tyrule{{num}}
      {\,}
      {
        \tyjudge{\varnothing}
        {\enum{42}}
        {\enum{}}
      }
      \\
      %
      \tyrule{{plus}}
      {
        \tyrule{{var}}
        {\,}
        {
          \tyjudge{\var{z} : \enum{}}
          {\var{z}}
          {\enum{}}
        }
        \\
        %
        \tyrule{{num}}
        {\,}
        {
          \tyjudge{\var{z} : \enum{}}
          {\enum{1}}
          {\enum{}}
        }
      }
      {
        \tyjudge{\var{z} : \enum{}}
        {\eplus{\var{z}}{\enum{1}}}
        {\enum{}}
      }
    }
    {
      \tyjudge{\varnothing}
      {\elet{\enum{42}}{z}{\eplus{\var{z}}{\enum{1}}}}
      {\enum{}}
    }
    \]
  }

  \bigskip

  \begin{tabular}{l}
    NB: \;
    ${\elet{\enum{42}}{z}{\eplus{\var{z}}{\enum{1}}}}$
    \\
    \qquad   \qquad    stands for: \;  $ {\clet{\enum{42}}{z}{\eplus{\var{z}}{\enum{1}}}}$
  \end{tabular}
\end{frame}
\end{comment}

\begin{frame}
  \frametitle{Abstract Binding Tree}
  \begin{itemize}[<+->]
  \item \textbf{Concrete syntax} is concerned with how phrases are
    entered and displayed on a computer, e.g., $\clet{e_1}{x}{e_2}$.
    %
  \item \textbf{Abstract syntax} is concerned with the \emph{structure} of
    phrases, specifically how they are composed from other phrases,
    $\eplus{e_1}{\etimes{e_2}{e_3}}$.
    %
  \item At the ``abstract level'' a phrase is a tree (data structure),
    called an \emph{abstract syntax tree}, whose nodes are operators
    that combine several phrases to form another phrase.
    \begin{center}
      E.g., see the \texttt{Term} data type in
      \href{https://github.com/rgrig/plad/blob/master/code/Lambda.hs}{Radu's
        $\lambda$-calculus interpreter}.
    \end{center}
    %
  \item \textbf{Abstract binding trees} (ABT) enrich abstract syntax
    trees with the concepts of \emph{binding} and \emph{scope}, e.g., $\elet{e_1}{x}{e_2}$.
    \begin{itemize}
    \item Binding: introduction of a new variable/parameter/name.
    \item Scope: the sub-tree where an identifier may be used.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Binders and Closed / Open terms}
  \begin{itemize}[<+->]
  \item Construct $\elet{e_1}{x}{e_2}$ is a \emph{binder}, it binds $\var{x}$ in $e_2$.
    \[
    \begin{tikzpicture}[
      edge from parent/.style={draw,-latex},
      sibling distance=5em,
      level distance=2.5em,
      every node/.style = {shape=rectangle, rounded corners,
        draw, align=center,
        top color=white}]]
      \node {$\kw{let}$}
      child { node {$e_1$} }
      child { node {$\var{x}$}
        child { node {$e_2$} edge from parent[thick, densely dotted,->]
        }
      };
    \end{tikzpicture}
    \]
  \item \emph{Bound variable:} a variable under a binder, e.g.,
    $\elet {\estr{\text{hello}}} {x} {\ecat{\var
        x}{\estr{\text{world}}}}$
    %
  \item A variable that is not bound is \emph{free}. A term with
    free-variables is \emph{open}, e.g., in
    $\elet{\estr{\text{hello}}}{x}{\ecat{\var x}{{{\var{y}}}}}$,
    $\var{y}$ is \emph{free}.
    %
  \item A consequence of $e$ being well-typed is that it is
    \emph{closed}, i.e., $e$ has no free variable.
    %
  \end{itemize}
\end{frame}

\subsection{Semantics}

\begin{frame}
  \frametitle{Semantics}
  The semantics of a programming language define what a program
  \emph{means}, i.e., how a program \emph{evaluates}.

  \bigskip

  The semantics of a programming language define ``dynamic''
  properties of the language, i.e., how programs ``move''.

\end{frame}

\begin{frame}
  \frametitle{Dynamics}
  \label{fr:value-e}

  The judgement ``$\valjudge{e}$'' states that $e$ is a value (i.e.,
  cannot be reduced further, it is the final result of a computation).
  %
  %
  %
  It is inductively defined by the following two rules:
  \[
  \semrule{}
  {\,}
  {\valjudge{\enum{n}}}
  \qquad \qquad  \qquad
  \semrule{}
  {\,}
  {\valjudge{\estr{s}}}
  \]

  \bigskip
  \pause

  Are these values (i.e., does  $\valjudge{e}$ hold)?
  \begin{itemize}[<+->]
  \item $\estr{\text{hello}}$
  \item $\elet
    {\estr{\text{anotherstring}}}
    {y}
    {\eplus{y}{\etimes{\enum{1}}{\enum{2}}}}$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Dynamics}

  Now, we define the actual semantics of language $\lggeE$, it is
  defined via judgement of the form: $\jtrans{e}{e'}$.
  %

  \bigskip

  The transition judgement $\jtrans{e}{e'}$ between states is
  inductively defined by the following rules:

  \pause

  \[
  \semrule
  {\esemrulename{plval}}
  {n_1 + n_2 = n}
  {\jtrans{\eplus{\enum{n_1}}{\enum{n_2}}}{\enum{n}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{pll}}
  {\jtrans{e_1}{e'_1}}
  {\jtrans{\eplus{e_1}{e_2}}{\eplus{e'_1}{e_2}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{plr}}
  { \valjudge{e_1}
    \and
    \jtrans{e_2}{e'_2}}
  {\jtrans{\eplus{e_1}{e_2}}{\eplus{e_1}{e'_2}}}
  \]

\end{frame}

\begin{frame}
  \frametitle{Dynamics}

  \[
  \semrule
  {\esemrulename{tmval}}
  {n_1 \times n_2 = n}
  {\jtrans{\etimes{\enum{n_1}}{\enum{n_2}}}{\enum{n}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{tml}}
  {\jtrans{e_1}{e'_1}}
  {\jtrans{\etimes{e_1}{e_2}}{\etimes{e'_1}{e_2}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{tmr}}
  { \valjudge{e_1}
    \and
    \jtrans{e_2}{e'_2}}
  {\jtrans{\etimes{e_1}{e_2}}{\etimes{e_1}{e'_2}}}
  \]

\end{frame}

% \note{Exercise: give are the rules for $\etimes{e_1}{e_2}$?}

\begin{frame}
  \frametitle{Dynamics}
  \[
  \semrule
  {\esemrulename{catval}}
  {s_1 \concat s_2 = s}
  {\jtrans{\ecat{\estr{s_1}}{\estr{s_2}}}{\estr{s}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{catl}}
  { \jtrans{e_1}{e'_1}}
  {\jtrans{\ecat{e_1}{e_2}}{\ecat{e'_1}{e_2}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{catr}}
  {\valjudge{e_1}
    \and
    \jtrans{e_2}{e'_2}}
  {\jtrans{\ecat{e_1}{e_2}}{\ecat{e_1}{e'_2}}}
  \]

\end{frame}

\begin{frame}
  \frametitle{Dynamics}
  \begin{itemize}[<+->]
  \item  Call-by-name:
    \[
    \semrule
    {\esemrulename{letn}}
    {\,}
    {
      \jtrans{\elet{e_1}{x}{e_2}}{\subs{e_2}{e_1}{x}}}
    \]
  \item   Call-by-value:

    \[
    \semrule
    {\esemrulename{letm}}
    {\jtrans{e_1}{e'_1}}
    {
      \jtrans{\elet{e_1}{x}{e_2}}{\elet{e'_1}{x}{e_2}}}
    \]
    \[
    \semrule
    {\esemrulename{letv}}
    {\valjudge{e_1}}
    {
      \jtrans{\elet{e_1}{x}{e_2}}{\subs{e_2}{e_1}{x}}}
    \]
  \end{itemize}

  \bigskip

  \pause

  Is call-by-name semantics equivalent to laziness
  (like the default behaviour in Haskell)?
  %
  What kind of strategy does OCaml use?

\end{frame}

\begin{frame}
  \frametitle{Evaluation strategies}
  \begin{itemize}[<+->]
  \item \emph{Call-by-value:} arguments are evaluated to values
    \emph{before} being substituted in for variables. Main advantage:
    program's behavior (easy to determine when an expression is
    evaluated) and performance (each expression is evaluated at most
    once). Main disadvantage: evaluate everything, even if not needed.
    %
  \item \emph{Call-by-name:} arguments are substituted without being
    computed. Main advantage: expressions whose value is never needed
    are not evaluated (evaluate as late as possible). Main
    disadvantage: requires a lot of memory.
    %
  \item \emph{Call-by-need:} arguments are not evaluated until their
    values are needed, but once they are evaluated, the value is
    \emph{recorded} so that it can be reused. Main advantage:
    expressions are evaluated at most once.
    %
    Good trade-off between CBV and CBN but harder to implement
    correctly and efficiently.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Call-by-name vs.\ call-by-value}
  %
  \label{fr:srategies-e}
  %
  What are the advantages/disadvantages of call-by-name vs.\ call-by-value?

  \bigskip

  Consider the following programs:
  \begin{itemize}
  \item $\elet{\etimes{1852}{2017}}{x}{\eplus{\var{x}}{\var{x}}}$
  \item $\elet{\etimes{1852}{2017}}{x}{\eplus{\var{1}}{\var{x}}}$
  \item $\elet{\etimes{1852}{2017}}{x}{\eplus{\var{1}}{\var{13}}}$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Examples}
  \label{fr:evaluate-e}

  Evaluate these $\lggeE$ phrases:
  \begin{itemize}
  \item $\ecat
    {\ecat
      {\estr{\text{hello}}}
      {\estr{\text{world}}}
    }
    {\estr{\text{!}}}$
  \item $\etimes
    {\elen{\estr{\text{hello}}}}
    {\eplus{\enum{1}}{\elen{\estr{\text{world}}}}}$
    %
  \item $\elet{\estr{\text{mystring}}}{x}{\etimes{\elen{\var{x}}}{\enum{0}}}$
    %
  \item $\elet
    {\estr{\text{anotherstring}}}
    {\var{y}}
    {\eplus{\var{y}}{\etimes{\enum{1}}{\enum{2}}}}$
  \end{itemize}
\end{frame}

\input{recap}

\begin{frame}
  \label{fr:ite-e}
  \frametitle{Exercise}
  Add the following construct to language $\lggeE$:
  \[
  \site{e}{e_1}{e_2}
  \qquad\qquad \text{i.e.,} \quad
  \csite{e}{e_1}{e_2}
  \]

  \bigskip

  Think of what you will need to add wrt.\ syntax, types, and
  semantics.
\end{frame}

\subsection{Type safety}

\begin{frame}
  \frametitle{Type safety}
  \begin{itemize}
  \item We want \emph{type safe} programming languages
    %
  \item i.e., certain kinds of mismatches cannot happen at runtime,
    e.g., we would like to avoid computation like: \texttt{``one'' == 123}.
    %
  \item \emph{Type safety} expresses the \emph{coherence} between
    statics (types) and dynamics (semantics).
    %
  \item A consequence of type safety is that evaluation cannot get stuck.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Type safety}\label{fr:ty-safety}

  Hereafter, we often write $\tyjudge{\,}{e}{\tau}$ for
  $\tyjudge{\varnothing}{e}{\tau}$.

  \bigskip

  \pause

  \begin{theorem}[Type safety]
    \begin{enumerate}[<+->]
    \item If $\tyjudge{}{e}{\tau}$ and $\jtrans{e}{e'}$, then $\tyjudge{}{e'}{\tau}$.
    \item If $\tyjudge{}{e}{\tau}$, then either $\valjudge{e}$, or there exists $e'$ such that $\jtrans{e}{e'}$.
    \end{enumerate}
  \end{theorem}

  The first part of the theorem is often referred to as \emph{type
    preservation}, while the second part is referred to as
  \emph{progress}.
\end{frame}

\begin{frame}
  \frametitle{Runtime errors}

  Suppose we extend $\lggeE$ with a division operator:
  $\ediv{e_1}{e_2}$.

  \begin{itemize}
  \item  A natural typing rule is:
    \pause
    \[
    \tyrule
    {\etyrulename{div}}
    {
      \tyjudge{\Gamma}{e_1}{\enum{}}
      \and
      \tyjudge{\Gamma}{e_2}{\enum{}}
    }
    {\tyjudge{\Gamma}{\ediv{e_1}{e_2}}{\enum{}}}
    %
    \]

  \item    Main semantics rule:
    \pause
    %
    %
    \[
    \semrule
    {\esemrulename{dval}}
    {n_1 \div n_2 = n}
    {\jtrans{\ediv{\enum{n_1}}{\enum{n_2}}}{\enum{n}}}
    \]

  \end{itemize}

  \pause

  \bigskip

  \emph{Problem:} the expression $\ediv{\enum{3}}{\enum{0}}$ is
  \emph{well-typed}, yet ``\emph{stuck}''!
  %

  %
\end{frame}
\begin{frame}
  \frametitle{Runtime errors}

  \emph{Two possible solutions:}

  \bigskip

  \begin{itemize}[<+->]
  \item \emph{Enhanced the type-system}, so that no well-typed program
    may divide by zero.
    %
  \item Add \emph{dynamic checks}, so that division-by-zero signals an error
    (of evaluation).
  \end{itemize}

  \pause

  \bigskip

  The first approach may rule out too many ``good'' programs.
  %
  In general, we cannot predict \emph{statically} whether an
  expression will be non-zero when evaluated.

  \bigskip

  Hence, the second approach is generally preferred.
  %
\end{frame}

\begin{frame}
  \frametitle{Checked vs. Unchecked errors}
  We need to distinguish between \emph{checked} and \emph{unchecked errors}.
  \bigskip
  \begin{description}
    \setlength\itemsep{2em}
    %
  \item[Unchecked error:] ruled out by the type system. No run-time
    checking performed because the type system rules out the
    possibility of such errors arising.
    %
  \item[Checked error:] not ruled out by the type system, hence
    run-time check \emph{must} occur, e.g., a zero divisor.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Dealing with run-time error}
  %
  \[
  \semrule
  {\esemrulename{dval}}
  {n_1 / n_2 = n \and n_2 > 0}
  {\jtrans{\ediv{\enum{n_1}}{\enum{n_2}}}{\enum{n}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{dval}}
  {\,}
  {\jtrans{\ediv{\enum{n_1}}{\enum{0}}}{\eerror}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{dl}}
  {\jtrans{e_1}{e'_1}}
  {\jtrans{\ediv{e_1}{e_2}}{\ediv{e'_1}{e_2}}}
  \]

  \pause

  \[
  \semrule
  {\esemrulename{dr}}
  { \valjudge{e_1}
    \and
    \jtrans{e_2}{e'_2}}
  {\jtrans{\ediv{e_1}{e_2}}{\ediv{e_1}{e'_2}}}
  \]

  \pause
  \bigskip

  Note $\eerror$ is a new term in language $\lggeE$.
  \pause
  %
  \[
    \tyrule
    {\etyrulename{err}}
    {
      \,
    }
    {\tyjudge{\Gamma}{\eerror}{\tau}}
      %
    \]

\end{frame}

\begin{frame}
  \frametitle{Dealing with run-time error}

  How do we preserve \emph{type safety} with run-time errors?

  \bigskip
  \pause

  \emph{Idea:} use expression $\eerror$ (i.e., a \emph{checked} error)
  and define a judgement $\errjudge{e}$ stating that $e$ incurs a
  run-time checked error.
  %
  % \[
  % \anorule
  % {\valjudge{e_1}}
  % {\errjudge{\ediv{e_1}{\enum{0}}}}
  % \quad
  % \anorule
  % {\errjudge{e_1}}
  % {\errjudge{\ediv{e_1}{e_2}}}
  % \quad
  % \anorule
  % {\valjudge{e_1}
  % \and \errjudge{e_2}}
  % {\errjudge{\ediv{e_1}{e_2}}}
  % \]

  \[
  \anorule{\,}{\errjudge{\eerror}}
  \]

  \begin{theorem}[Progress with error]
    %
    If $\tyjudge{}{e}{\tau}$, then either $\errjudge{e}$, $\valjudge{e}$, or there
    exists $e'$ such that $\jtrans{e}{e'}$.
    %
  \end{theorem}
\end{frame}

\begin{frame}
  \frametitle{Discussion}

  Do we need to update the \textbf{type preservation} statement?

  \pause
  \bigskip

  Can you think of other \textbf{examples} of checked / unchecked errors?
\end{frame}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
%
\documentclass[
]{article}
\usepackage{lmodern}
\usepackage{amssymb,amsmath}
\usepackage{ifxetex,ifluatex}
\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
\hypersetup{
  hidelinks,
  pdfcreator={LaTeX via pandoc}}
\urlstyle{same} % disable monospaced font for URLs
\usepackage[margin=1in]{geometry}
\usepackage{graphicx,grffile}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{-\maxdimen} % remove section numbering

\author{}
\date{\vspace{-2.5em}}

\begin{document}

setwd(``D:/Udemy/Linear Regression GLMs and GAMs with R'') \#\#\#\# So
how old is the universe? library(gamair) \# contains `hubble'
data(hubble) hubble hub.mod \textless-
lm(y\textasciitilde x-1,data=hubble) summary(hub.mod) head(hub.mod, 25)
\#\#\#\# Plot the residuals against the fitted values
plot(fitted(hub.mod),residuals(hub.mod),xlab=``fitted
values'',ylab=``residuals'')

\hypertarget{omit-offending-points-and-produce-new-residual-plot}{%
\paragraph{Omit offending points and produce new residual
plot}\label{omit-offending-points-and-produce-new-residual-plot}}

hub.mod1 \textless- lm(y\textasciitilde x-1,data=hubble{[}-c(3,15),{]})
summary(hub.mod1)
plot(fitted(hub.mod1),residuals(hub.mod1),xlab=``fitted
values'',ylab=``residuals'')

\hypertarget{estimate-hubbles-constant}{%
\paragraph{Estimate Hubble's Constant}\label{estimate-hubbles-constant}}

hubble.const \textless- c(coef(hub.mod),coef(hub.mod1))/3.09e19 age
\textless- 1/hubble.const age age/(60\^{}2\emph{24}365)

\hypertarget{adding-a-distributional-assumption}{%
\paragraph{1.1.3 Adding a distributional
assumption}\label{adding-a-distributional-assumption}}

\hypertarget{testing-hypotheses-about}{%
\paragraph{\texorpdfstring{Testing Hypotheses about
\beta}{Testing Hypotheses about }}\label{testing-hypotheses-about}}

cs.hubble \textless- 163000000
t.stat\textless-(coef(hub.mod1)-cs.hubble)/summary(hub.mod1)\$coefficients{[}2{]}
pt(t.stat,df=21)*2

\hypertarget{confidence-intervals}{%
\paragraph{Confidence intervals}\label{confidence-intervals}}

sigb \textless- summary(hub.mod1)\$coefficients{[}2{]}
h.ci\textless-coef(hub.mod1)+qt(c(0.025,0.975),df=21)\emph{sigb h.ci
h.ci\textless-h.ci}60\^{}2\emph{24}365.25/3.09e19 \# convert to 1/years
sort(1/h.ci)

\end{document}

Linear Regression GLMs and GAMs with R/test2.tex


\documentclass{beamer}

\mode<presentation>
{
  \usetheme{Hawke}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{times}
\usepackage[T1]{fontenc}
\usepackage{multimedia}

%%%%%%
% My Commands
%%%%%%

\newcommand{\ml}{{\sc matlab}}
\newcommand{\bb}{{\boldsymbol{b}}}
\newcommand{\bx}{{\boldsymbol{x}}}
\newcommand{\by}{{\boldsymbol{y}}}
\newcommand{\bfm}[1]{{\boldsymbol{#1}}}

%%%%

\title[Lecture 5] %
{Lecture 5 - Iterative Methods}

\author[I. Hawke]{I.~Hawke}

\institute[University of Southampton]%
{
  School of Mathematics, \\
  University of Southampton, UK
}

\date[Semester 1]{MATH3018/6141, Semester 1}

\subject{Numerical methods}

\pgfdeclareimage[height=0.5cm]{university-logo}{mathematics_7469}
\logo{\pgfuseimage{university-logo}}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\section{Iterative Methods}

\subsection{Iterative Methods}

\begin{frame}
  \frametitle{Iterative Methods}

   We (still) want to solve the linear system
   \begin{equation*}
     A \bx = \bb.
   \end{equation*}
   All direct methods require ${\cal O}(n^3)$ operations. For large
   matrices this takes time and each operation introduces error that
   may accumulate. Other methods may be preferable. \pause

   \vspace{1ex}

   Iterative methods build a \emph{sequence} of approximate
   solutions. Start from ``guess'' $\bx^{(0)}$ and compute
   approximations $\bx^{(1)}, \bx^{(2)}, \dots, \bx^{(N)}$ which
   converge to the exact solution.

   \vspace{1ex}

   Sequence can be truncated to save time or when solution found to
   sufficient accuracy. \pause

   \vspace{1ex}

   Key elements of an iterative method:
   \begin{enumerate}
   \item A rapid convergence to the correct solution,
   \item A fast algorithm for computing each successive approximation.
   \end{enumerate}

\end{frame}

\begin{frame}
  \frametitle{A general framework}

  Standard ``notation'' is to split the matrix as
  \begin{equation*}
    A = N - P,
  \end{equation*}
  where $N$ and $P$ are (as yet unknown) $n \times n$ matrices.
  The linear system becomes
  \begin{equation*}
    N \bx = P \bx + \bb.
  \end{equation*} \pause

  Then \emph{assume} that we have an approximate solution, or guess,
  $\bx^{(0)}$, and define the sequence
  \begin{equation*}
    N \bx^{(i)} = P \bx^{(i - 1)} + \bb, \quad i = 1, 2, \dots
  \end{equation*} \pause

  Obviously consistent with the original equation.  The algorithm will
  only be useful if
  \begin{enumerate}
  \item $N$ is nonsingular, and
  \item The linear system $N \bfm{y} = \bfm{z}$ is ``easy'' to solve.
  \end{enumerate}

\end{frame}

\subsection{Jacobi's Method}

\begin{frame}
  \frametitle{General assumptions}

  \vspace{2ex}

  In what follows we shall always assume that all diagonal entries are
  1. \pause

  \vspace{2ex}

  If this is not true, we can always arrange it to be so by dividing
  each row by its diagonal element. \pause

  \vspace{2ex}

  If the diagonal element of any row is zero, permute the rows. It
  must be possible to arrange a \emph{nonsingular} matrix such that
  every diagonal element is non-zero just by row operations.

\end{frame}

\begin{frame}
  \frametitle{Jacobi's Method}

  One key requirement on the split
  \begin{equation*}
    A = N - P
  \end{equation*}
  was that the $N \bfm{y} = \bfm{z}$ should be easy to solve. The
  simplest possible problem would be $N = I$. \pause

  \vspace{1ex}

  This means (as $A$ is 1 on the diagonal) that
  \begin{equation*}
    P = A_L + A_U
  \end{equation*}
  where $A_L$ and $A_U$ are the ``triangular'' parts of $-A$ (note
  sign!). The iteration scheme is
  \begin{equation*}
    \bx^{(i)} =  (A_L + A_U) \bx^{(i - 1)} + \bb, \quad i = 1, 2, \dots
  \end{equation*}
  and the convergence matrix is $M = P$.

\end{frame}

\begin{frame}
  \frametitle{Example of Jacobi's Method}

  We look at the solution of the simple linear system
  \begin{equation*}
    \begin{pmatrix}
      3 & 1 \\
      1 & 3
    \end{pmatrix}
    \begin{pmatrix}
      x_1 \\ x_2
    \end{pmatrix} =
    \begin{pmatrix}
      5 \\ 7
    \end{pmatrix}
  \end{equation*}
  using Jacobi's Method. \pause  Ensure the diagonal
  elements are all 1:
  \begin{equation*}
    A =
    \begin{pmatrix}
      1 & \tfrac{1}{3} \\
      \tfrac{1}{3} & 1 \\
    \end{pmatrix},
    \quad \bb =
    \begin{pmatrix}
      \tfrac{5}{3} \\ \tfrac{7}{3}
    \end{pmatrix}.
  \end{equation*} \pause
  Check convergence by computing
  \begin{equation*}
    M = P =
    \begin{pmatrix}
      0 & -\tfrac{1}{3} \\
      -\tfrac{1}{3} & 0
    \end{pmatrix}
  \end{equation*}
  which implies, as $\varrho(M) = 1 / 3 < 1$, that the method will
  converge.

\end{frame}

\begin{frame}
  \frametitle{Example of Jacobi's Method 2}

  \begin{columns}
    \begin{column}{0.475\textwidth}
      The sequence has entries
      \begin{tabular}{|c|c c|}
        $i$ & $x_1$ & $x_2$ \\ \hline
        $0$ & $0$ & $0$ \\
        $1$ & $5 / 3$ & $7 / 3$ \\
        $2$ & $0.888889$ & $1.777778$ \\
        $5$ & $1.008230$ & $2.004115$ \\
        $10$ & $0.999983$ & $1.999966$ \\
        $100$ & $1.000000$ & $2.000000$
      \end{tabular}
    \end{column}
    \begin{column}{0.525\textwidth}
      \begin{center}
        \includegraphics[width=\textwidth]{figures/Jacobi1}
      \end{center}
    \end{column}
  \end{columns}

  \vspace{1ex}

  A starting ``guess'' of $\bx = \bfm{0}$ appears to converge to $\bx
  = (1, 2)^T$. The convergence is exponential with a slope $\sim 1$.

\end{frame}

\subsection{Convergence}

\begin{frame}
  \frametitle{Convergence}

  To work, the sequence
  \begin{equation*}
    \bx^{(i)} = N^{-1} \left( P \bx^{(i - 1)} + \bb \right), \quad i =
    1, 2, \dots
  \end{equation*}
  must have a limit.  Intuitively clear that the matrix $M =
  N^{-1} P$ is central to the convergence or otherwise of
  this sequence.

  \vspace{1ex}

  {\bf Theorem}: The iterative method described converges if and only
  if $M = N^{-1} P$ exists and
  \begin{equation*}
    \varrho (M) \equiv \max_i | \lambda_i | < 1.
  \end{equation*}

  \begin{overlayarea}{\textwidth}{0.3\textheight}
    \only<2|handout:0>
    {
      View $\bx^{(i)}$ as a vector in $n$-dimensional space: ensures
      that repeatedly multiplying by $M$ does not cause result to
      diverge.
    }
    \only<3>
    {
      The \emph{spectral radius} $\varrho (M)$ can be hard
      to compute. Often use the weaker condition
      \begin{equation*}
        \| M \| < 1 \Rightarrow \text{Method converges}.
      \end{equation*}
    }
  \end{overlayarea}

\end{frame}

\section{Summary}

\subsection{Summary}

\begin{frame}
  \frametitle{Summary}

  \begin{itemize}
  \item Iterative methods are often based on the split
    \begin{equation*}
      A = N - P.
    \end{equation*}
  \item Convergence depends on $M = N^{-1}P$.
  \item Jacobi chooses $N = I$ and converges slowly.
  \end{itemize}

\end{frame}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End: