\section{Multi-class Case}
We now consider the situation where there are $k > 2$ classes.

We now define the analogous notion of `margin,' which will be important when we introduce the SVM model later.
\begin{definition}\label{margin_k_classes}
 Suppose that $k$ subsets of $\mathbb{R}^n$, $A_1,...,A_k$ are linearly separable by the matrix $W$ and vector $b$,
 and let $d$ be a metric on $\mathbb{R}^n$ as before. The margin of separation with respect to
 $d$ is given by
 \begin{equation}
  m(W,b; A_1,..., A_k) = \sup \{\epsilon:\text{$A^{\epsilon}_1,..., A^\epsilon_k$ are still linearly separated by $W$ and $b$}\}
 \end{equation}
where, as before, the $\epsilon$-enlargement of $A_i$ is given by
\begin{equation}
 A^\epsilon_i = \{y:\text{there is an $x\in A_i$ such that $\|x-y\| < \epsilon$}\}
\end{equation}
\end{definition}
This notion of margin measures how much we can perturb each data point without triggering a misclassification. This is
quite a useful notion especially given the freedom to choose the metric $d$.

We close this section with a technical lemma relating compactness and the margin of separation, which can safely be skipped.
\begin{lemma}{\label{kclass_margin}}
 Suppose that the sets $A_1,...,A_k$ are separated by the matrix $W$ and vector $b$ and that $A_1,...,A_k$ are compact
 with respect to the topology induced by a metric $d$. Then
 \begin{equation}
  m(W,b; A_1,...,A_k) > 0
\end{equation}

\end{lemma}

\begin{proof}
	According to Lemma \ref{Interplation}, we only need to prove that for each $A_i$, there exists a positive number $\epsilon_i$ such that $A_i^{\epsilon_i}$ are still in $\Gamma_i(W,b)$. In fact, $\Gamma_i(W,b)$ is an open set, so $(\Gamma_i(W,b))^C$ and $A_i$ are two disjoint closed set. Thus we have
	\begin{equation}
	d_i = d(A_i,(\Gamma_i(W,b))^C) > 0
	\end{equation}
	We can take $\epsilon_i = d_i$. Actually, $d_i$ is the largest value that $\epsilon_i $ can take. And we have
	\begin{equation}
	m(W,b; A_1,...,A_k) =  \min_{i}~d_i > 0
	\end{equation}
\end{proof}

\subsection{k-class hard-margin Case}
In our examination of the $k$-class case, we first consider the situation where our data is linearly separable.

Assuming that our dataset is finite, and thus compact, we see that there exists an $\epsilon > 0$ such that
\begin{equation}
 w_i x+b_i \geq w_jx+b_j + \epsilon
\end{equation}
for all $x\in A_i$ and $j\neq i$, and a rescaling of $W$ and $b$ by $2\epsilon^{-1}$ means that we can assume without loss of
generality that
\begin{equation}
 w_i x+b_i \geq w_jx+b_j + 2
\end{equation}
for all $x\in A_i$ and $j\neq i$. Or we can formulate it as
\begin{equation}\label{separation_constraint_k}
w_i x+b_i \geq \max_{j\neq i} (w_jx + b_j) + 2,~~\forall x\in A_i.
\end{equation}

%In order to simplify notation, we index our data points $x_1,...,x_N$ and introduce labels $y_1,...,y_N$, where
%\begin{equation}
% y_i = e_j~\text{if $x_i\in A_j$}
%\end{equation}
%i.e. $y_i$ is the standard basis vector corresponding to the class to which $x_i$ belongs. A clever and compact way of writing
%the linear separation constraint is now
%\begin{equation}\label{separation_constraint_k}
 %(y_i - e_j)\cdot (Wx_i+b) \geq \|y_i - e_j\|_1
%\end{equation}
%for all $i = 1,...,N$ and $j = 1,...,k$.

We note that the condition (\ref{separation_constraint_k}) is a collection of $Nk$ linear constraints on the matrix $W$ and
vector $b$. This means that we can determine whether our data is linearly separable by checking whether a given linear
program is feasible.

Of course, if our data is linearly separable, then we would like to find a matrix $W$ and vector $b$ which maximizes the
margin of separation as defined in Definition \ref{margin_k_classes}. To this end, we have the following lemma.
\begin{lemma}
 Assume that our data is separated by the matrix $W$ and vector $b$, and that (\ref{separation_constraint_k}) holds.
 Then the margin of separation is bounded below by
 \begin{equation}
  m(W,b;A_1,...,A_k)\geq \frac{1}{\max_{1\leq i\leq k}\|w_i\|}
 \end{equation}
 where $\|w_i\|$ is the dual norm of the $i$-th row of $w$.
\end{lemma}

\begin{proof}
	Use the notations in the proof of lemma \ref{kclass_margin}. Because $m(W,b, A_1,...,A_k) =  \min_{i}~d_i$, we only need to prove for each $i$, $d_i = d(A_i,\partial\Gamma_i(W,b)) \geq 1/\max_{1\leq i\leq k}\|w_i\|$.\\
	Take a point $x$ from $A_i$ and a point $y$ from $\partial\Gamma_i(W,b)$ arbitrarily. There exists $j\neq i$ such that
	$y \in H_{ij}$. So we have
	\begin{equation}
	\|x-y\| \geq d(x^i,H_{ij}) = \frac{|(w_i-w_j)x+(b_i-b_j)|}{\|w_i-w_j\|}\geq \frac{2}{\|w_i-w_j\|}\geq
	\frac{1}{\max_{1\leq i\leq k}\|w_i\|}.
	\end{equation}
    So we have
	\begin{equation}
	d_i = \inf_{y\in \partial\Gamma_i(W,b)} \|x-y\| \geq \frac{1}{\max_{1\leq i\leq k}\|w_i\|}.
	\end{equation}
	Above all, we can obtain
	\begin{equation}
	m_d(W,b;A_1,...,A_k) = \min_i~d_i \geq  \frac{1}{\max_{1\leq i\leq k}\|w_i\|}.
	\end{equation}

\end{proof}

This leads to the following convex optimization problem
\begin{align}{\label{kclass_hard_op}}
 \min_{W,~b~}~~&\max_{1\leq i\leq k}\|w_i\|\\
 s.t.~~&w_i x+b_i \geq \max_{j\neq i} (w_jx + b_j) + 2,~~\forall x\in A_i.
\end{align}
Naturally, we are interested in the relation between (\ref{kclass_hard_op}) for $k=2$ and $(\ref{2class_hard_op})$. It seems that we have done some relaxation in (\ref{kclass_hard_op}), but actually it is equivalent to $(\ref{2class_hard_op})$ in 2-class situations.
\begin{lemma}
The optimization problem (\ref{kclass_hard_op}) is equivalent to the problem $(\ref{2class_hard_op})$ when $k=2$.
\end{lemma}
\begin{proof}
We'll set $k=2$ in the following statement. When $k=2$, the constraint (\ref{separation_constraint_k}) is equivalent to the constraint (\ref{separation_condition_2}), So we use (\ref{separation_condition_2}) to denote the constraint of both two problems. \\
We define three sets $\Omega,\Omega_1,\Omega_2$ as
\begin{gather}
\Omega = \{w~|~\exists b\in \mathbb{R},~\text{s.t.}~wx+b\geq 2, \forall x \in A_1~\text{and}~wx+b\leq -2, \forall x\in A_2\}.\\
\Omega_1 = \{(w_1,w_2)~|~w_1-w_2\in\Omega\}\\
\Omega_2 = \{(w_1,w_2)~|~w_1-w_2\in\Omega~\text{and}~w_1 = -w_2\}
\end{gather}
The constraint (\ref{separation_condition_2}) can be written as
\begin{equation}
(w_1,w_2)\in \Omega_1.
\end{equation}
Observing that $\Omega_2 \subset\Omega_1$, we'll show that for problem (\ref{kclass_hard_op}), nothing will change if we use the subset $\Omega_2$ to replace $\Omega_1$ in the constraint.\\
Take $(w_1,w_2)\in \Omega_1$ arbitrarily, and denote
\[
\tilde{w}_1 = \frac{w_1-w_2}{2},~\tilde{w}_2 = -\frac{w_1-w_2}{2},
\]
then we have $(\tilde{w}_1,\tilde{w}_2)\in \Omega_2$ and
\begin{equation}
\max_{1\leq i\leq 2}\|\tilde{w}_i\| = \frac{1}{2} \|\tilde{w}_1-\tilde{w}_2\| = \frac{1}{2}\|w_1-w_2\| \leq \max_{1\leq i\leq 2}\|w_i\|.
\end{equation}
The inequalities above show that the element in $\Omega_1\backslash\Omega_2$ won't affect the solution of (\ref{kclass_hard_op}) at all. So we can use  $\Omega_2$ to replace $\Omega_1$ in problem (\ref{kclass_hard_op}), which means (\ref{kclass_hard_op}) is equivalent to
\begin{equation}\label{proof_op1}
	\min_{(w_1,w_2)\in\Omega_2} \max_{1\leq i\leq 2}\|w_i\|.
\end{equation}
Also, easy to observe that the problem (\ref{separation_condition_2}) is equivalent to
\begin{equation}\label{proof_op2}
\min_{(w_1,w_2)\in \Omega_2} \| w_1-w_2\|.
\end{equation}
For all $(w_1,w_2)\in \Omega_2$, we have
\begin{equation}
	\max_{1\leq i\leq 2}\|w_i\| = \frac{1}{2} \| w_1-w_2\|,
\end{equation}
which implies that the problem (\ref{proof_op1}) is equivalent to the problem (\ref{proof_op2}). \\
Consequently, the problem (\ref{2class_hard_op}) is equivalent to the problem (\ref{kclass_hard_op}) when $k=2$.
\end{proof}

\subsection{k-class soft-margin Case}
If the data is not max-mat linearly separable, then we replace the hard constraint (\ref{separation_constraint_k})
by a penalty term. A common choice of penalty is the hinge or ReLU loss, which only penalizes those constraints which are
not satisfied and leads to an optimization problem of the form
\begin{equation}{\label{kclass_soft_op}}
 \min_{W,~b}\max_{1\leq i\leq k}\|w_i\| + C\displaystyle\sum_{i=1}^k\displaystyle\sum_{x\in A_i}\text{ReLU}\Big( \max_{j\neq i} (w_jx + b_j) + 2 - (w_ix+b_i)\Big)
\end{equation}
where $C$ is a parameter controlling how much the penalty term is weighted.

When $k =2$, denoting $w_1-w_2$ as $w$, $b_1-b_2$ as $b$, we can write the 2-class soft margin optimization as
\begin{equation}{\label{2-class soft-margin optimization}}
\min_{w,b} \|w\| + C\sum_{i=1}^2 \sum_{x\in A_i} ReLU((-1)^i (wx+b)+2)
\end{equation}

\section{Optimizaiton Problem in SVM}
From the discussion above, we transformed the classification problem into some optimization problems. In this section, we'll use Lagrange duality to transform the primal  problem into dual problem.  That will provide convinience for introducing the kernel method in SVM.\\
We first take a look at the hard-margin case as a transition to the soft-margin case.  Trivially, the problem (\ref{kclass_hard_op}) is equivalent to the following problem:
\begin{equation}
\begin{aligned}
\min_{W,~b} ~~& \max_{1\leq i\leq k}\|w_i\|,\\
s.t.               ~~& \|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)\leq 0,~\forall i,j.\\
\end{aligned}
\end{equation}
For the introduction of inner product, here we take $\|\cdot\|=\|\cdot\|_2$, and use $\displaystyle\sum_{l=1}^k \|w_i\|_2^2$ to replace $\max_{1\leq i\leq k}\|w_i\|$. We can obtain a convex quadratic optimization problem for hard-margin case:
\begin{equation}{\label{kclass_hard_CQP}}
\begin{aligned}
\min_{W,~b} ~~& \frac{1}{2}\displaystyle\sum_{l=1}^k \|w_l\|_2^2,\\
s.t.               ~~& \|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)\leq 0,~\forall i,j.\\
\end{aligned}
\end{equation}

Now let's turn to the soft-margin case. Similarly, we take $\|\cdot\|=\|\cdot\|_2$, and use $\displaystyle\sum_{l=1}^k \|w_i\|_2^2$ to replace $\max_{1\leq i\leq k}\|w_i\|$.

\begin{equation}{\label{kclass_soft_OP}}
\min_{W,~b}  \frac{1}{2}\displaystyle\sum_{i=1}^k \|w_i\|_2^2 + C\displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k\text{ReLU}\Big(\|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)\Big)
\end{equation}

\begin{theorem}[Representer Theorem]
The solution of (\ref{kclass_soft_OP}) must belong to $span\{x_1,\cdots,x_N\}$, which means for each $i = 1,2,\cdots,k$, if $w_i^*$ is the optimal $w_i$ of (\ref{kclass_soft_OP}), then there must exist $\alpha_1^i,\cdots,\alpha_N^i\in \mathbb{R}$, such that
\[
w_i^* = \sum_{j = 1}^N \alpha_j^i x_j.
\]
\end{theorem}

\begin{proof}
Denote $span\{x_1,\cdots,x_N\}$ as $X$ and the orthogonal projection operater on $X$ in $\mathbb{R}^n$ as $P$. Denote the objective function as $L(w_1,\cdots,w_k,b)$.\\
In fact, for each $i = 1,\cdots,k$, we have
\begin{align}
\|w_i\|_2^2 = ~&\|Pw_i + (w_i-Pw_i)\|_2^2 = \|Pw_i\|_2^2+\|w_i-Pw_i\|_2^2\geq \|Pw_i\|_2^2,\\
w_ix+b_i &= (Pw_i) x + b_i,~~\forall x \in\{x_1,\cdots,x_N\}.
\end{align}
So easy to observe that
\begin{align}
\sum_{i=1}^k &\|Pw_i\|_2^2\leq \sum_{i=1}^k \|w_i\|_2^2,\\
\max_{j\neq i} &((Pw_j)x + b_j) + 2 - ((Pw_i)x+b_i) = \max_{j\neq i} (w_jx + b_j) + 2 - (w_ix+b_i).
\end{align}
So we have
\begin{equation}
L(Pw_1,\cdots,Pw_k,b)\leq L(w_1,\cdots,w_k,b),
\end{equation}
which means the optimal $w_i$ for each $i\in\{1,\cdots,k\}$ must belong to $X$.
\end{proof}

We can obtain a similar convex quadratic optimization problem to (\ref{kclass_hard_CQP}) by introducing some slack variables $\xi$.

\begin{equation}{\label{kclass_soft_CQP}}
\begin{aligned}
\min_{W,b,\xi} ~~& \frac{1}{2}\displaystyle\sum_{j=1}^k \|w_j\|_2^2+C\displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k \xi_{ij},\\
s.t.               ~~& \|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)-\xi_{ij} \leq 0,~\forall i,j.\\
                        & \xi_{ij} \geq 0,~\forall i,j.
\end{aligned}
\end{equation}

\begin{lemma}
The problem (\ref{kclass_soft_CQP}) is equivalent to the probelm (\ref{kclass_soft_OP}).
\end{lemma}
\begin{proof}
Denote the objetive functions as $f_1,f_2$:
\begin{equation}
\begin{split}
&f_1(W,b) = \frac{1}{2}\displaystyle\sum_{j=1}^k \|w_j\|_2^2 + C\displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k\text{ReLU}\Big(\|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)\Big)\\
&f_2(W,b,\xi) = \frac{1}{2}\displaystyle\sum_{j=1}^k \|w_j\|_2^2+C\displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k \xi_{ij}.
\end{split}
\end{equation}
Easy to observe that
\begin{equation}
	f_2(W,b,\xi)\leq f_1(W,b),
\end{equation}
equality holds iff
\begin{equation}
    \xi_{ij} = \text{ReLU}\Big(\|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)\Big).
\end{equation}
\end{proof}

It's easy to observe that the Slater's condition holds in the problem (\ref{kclass_soft_CQP}). That means we can obtain the optimal points of problem (\ref{kclass_soft_CQP}) by solving its dual problem.\\
The Lagrangian function of (\ref{kclass_soft_CQP}) is
\begin{equation}
\begin{aligned}
	&L(W,b,\xi,\lambda,\mu) \\
	= &\frac{1}{2}\displaystyle\sum_{j=1}^k \|w_j\|_2^2+C\displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k \xi_{ij} + \displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k \lambda_{ij}\Big(\|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)-\xi_{ij}\Big) - \displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k \mu_{ij} \xi_{ij}\\
	= &\frac{1}{2}\displaystyle\sum_{j=1}^k \|w_j\|_2^2 + \displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k (C-\lambda_{ij}-\mu_{ij})\xi_{ij} + \displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k \lambda_{ij}\Big(\|y_i - e_j\|_1 - (y_i - e_j)\cdot (Wx_i+b)\Big)\\
	= &\frac{1}{2}\displaystyle\sum_{j=1}^k \|w_j\|_2^2 + \displaystyle\sum_{i=1}^N\displaystyle\sum_{j=1}^k (C-\lambda_{ij}-\mu_{ij})\xi_{ij} + \displaystyle\sum_{i=1}^N\Big(\delta_i-t_i\cdot (Wx_i)-t_i\cdot b\Big)
\end{aligned}
\end{equation}
where $\delta_i = \sum_{j=1}^k\lambda_{ij}\|y_i - e_j\|_1$, $t_i = \sum_{j=1}^k\lambda_{ij}(y_i - e_j)$, and $\lambda_{ij},\mu_{ij}\geq 0$.\\
The dual problem of (\ref{kclass_soft_CQP}) is
\begin{equation}{\label{dual_problem}}
\max_{\lambda,\mu\geq 0} \min_{W,b,\xi} L(W,b,\xi,\lambda,\mu).
\end{equation}
According to KKT conditions, we have
\begin{equation}
\begin{aligned}
\nabla_{W}L(W^*,b^*,\xi^*,\lambda,\mu) &= W_* - \displaystyle\sum_{i=1}^N t_ix_i^{T} = 0,\\
\nabla_{b}L(W^*,b^*,\xi^*,\lambda,\mu)  &= -\displaystyle\sum_{i=1}^N t_i = 0,\\
\nabla_{\xi_{ij}}L(W^*,b^*,\xi^*,\lambda,\mu)  &= C-\lambda_{ij}-\mu_{ij} = 0,~\forall i,j..
\end{aligned}
\end{equation}
Take the equations above into dual problem (\ref{dual_problem}), we have
\begin{equation}
\begin{aligned}
&\min_{W,b,\xi} L(W,b,\xi,\lambda,\mu) \\
= &L(W^*,b^*,\xi^*,\lambda,\mu)\\
= &\frac{1}{2} Tr\Big((\displaystyle\sum_{i=1}^N t_i x_i^T)(\displaystyle\sum_{l=1}^N x_l t_l^T)\Big) -\displaystyle\sum_{i=1}^N t_i\cdot\Big((\displaystyle\sum_{l=1}^N t_l x_l^T)x_i\Big)+ \displaystyle\sum_{i=1}^N \delta_i\\
= &-\frac{1}{2}\displaystyle\sum_{i=1}^N\sum_{l=1}^N (t_i\cdot t_l)(x_i\cdot x_l)+ \displaystyle\sum_{i=1}^N \delta_i\\
= &-\frac{1}{2}\displaystyle\sum_{i=1}^N\sum_{l=1}^N\Big(\displaystyle\sum_{j=1}^k\sum_{m=1}^k \lambda_{ij}\lambda_{lm}(y_i-e_j)\cdot(y_l-e_m)\Big)(x_i\cdot x_l)+ \displaystyle\sum_{i=1}^N\sum_{j=1}^k\lambda_{ij}\|y_i - e_j\|_1
\end{aligned}
\end{equation}
So our dual problem can be transformed into the following form:
\begin{equation}{\label{SVM_dual}}
\begin{aligned}
\min_{\lambda}&~~ -\frac{1}{2}\displaystyle\sum_{i=1}^N\sum_{l=1}^N\Big(\displaystyle\sum_{j=1}^k\sum_{m=1}^k \lambda_{ij}\lambda_{lm}(y_i-e_j)\cdot(y_l-e_m)\Big)(x_i\cdot x_l)+ \displaystyle\sum_{i=1}^N\sum_{j=1}^k\lambda_{ij}\|y_i - e_j\|_1,\\
s.t.&~~\displaystyle \sum_{i=1}^N\sum_{j=1}^k\lambda_{ij}(y_i - e_j) = 0,\\
     &~~0\leq \lambda_{ij}\leq C,~\forall i,j.
\end{aligned}
\end{equation}

\section{Kernel Methods}
We denote the input space as $X$. Recalling that in the beginning of this chapter, we mentioned that a common way of obtaining a nonlinear classification model is to set $\mathscr{H} = \{\bm{f}: \bm{f}(x) = W\bm{\phi}(x)+b\}$, where $\bm{\phi}$ is a feature mapping from $X$ to a feature space $\mathcal{H}$. In other words, we use two steps to obtain a nonlinear classification model. First, we map input space $X$ to a feature space $\mathcal{H}$. Second, use linear classifier to do classification on $\bm\phi(X)\subset \mathcal{H}$.\\
Notice that if we use SVM on feature space, the optimization problem (\ref{SVM_dual}) turns to be
\begin{equation}{\label{SVM_dual}}
\begin{aligned}
\min_{W,~b}&~~ -\frac{1}{2}\displaystyle\sum_{i=1}^N\sum_{l=1}^N\Big(\displaystyle\sum_{j=1}^k\sum_{m=1}^k \lambda_{ij}\lambda_{lm}(y_i-e_j)\cdot(y_l-e_m)\Big)\langle\bm\phi(x_i),\bm\phi(x_l)\rangle+ \displaystyle\sum_{i=1}^N\sum_{j=1}^k\lambda_{ij}\|y_i - e_j\|_1,\\
s.t.&~~\displaystyle \sum_{i=1}^N\sum_{j=1}^k\lambda_{ij}(y_i - e_j) = 0,\\
&~~0\leq \lambda_{ij}\leq C,~\forall i,j.
\end{aligned}
\end{equation}
An interesting thing is that we actually only need to use the inner product of feature space in computation. That inspires us to focus on the inner product in feature space instead of the feature mapping itself.  We regard the inner product in feature space as a binary function on $X\times X$ and define it as a kernel function. In other words, a kernel function is $k: X\times X \rightarrow \mathbb{R}$ which
\begin{equation}
k(x,x) = \langle\bm\phi(x),\bm\phi(x')\rangle.
\end{equation}

\begin{definition}
Let $X$ be a non-empty set. Then a function $k:X\times X \rightarrow\mathbb{K}$ is
called a \textbf{kernel} on $X$ if there exists a $\mathbb{K}$-Hilbert space $\mathcal{H}$ and a map $\bm{\phi}:X\rightarrow \mathcal{H}$ such that for all $x \in X$ we have
\begin{equation}
k(x,x) = \langle\bm\phi(x),\bm\phi(x')\rangle.
\end{equation}
We call $\bm{\phi}$ a feature map and $\mathcal{H}$ a feature space of k.
\end{definition}

\begin{lemma}
Let $X$ be a non-empty set and $f_n:X\times X \rightarrow \mathbb{K}, n\in \mathbb{N}$, be functions such that $(f_n(x))\in l_2$ for all $x\in X$. Then
\begin{equation}
k(x,x'):= \sum_{n=1}^{\infty} f_n(x) \overline{f_n(x')},~x,x'\in X,
\end{equation}
defines a kernel on $X$.
\end{lemma}

\begin{definition}
A function $k:X\times X \rightarrow \mathbb{R}$ is called \textbf{positive definite} if for all $n\in \mathbb{N}, ~\alpha_1,\cdots,\alpha_n\in \mathbb{R}$ and all $x_1,\cdots,x_n \in X$, we have
\begin{equation}{\label{positive_definite_kernel}}
\displaystyle\sum_{i=1}^n\sum_{j=1}^n\alpha_i \alpha_j k(x_j,x_i)\geq 0.
\end{equation}
Furthermore, $k$ is said to be \textbf{strictly positive definite} if, for mutually distinct $x_1,\cdots,x_n \in X$, equality in (\ref{positive_definite_kernel}) only holds for $\alpha_1=\cdots=\alpha_n=0$. Finnaly, $k$ is called \textbf{symmetric} if $k(x,x')=k(x',x)$ for all $x,x' \in X$.
\end{definition}

\begin{theorem}
A function $k:X\times X \rightarrow \mathbb{R}$ is a kernel if and only if it is symmetric and postive definite.
\end{theorem}
\begin{proof}
It is trivial that a kernel is a symmetric and postive definite function. We will mainly show the reverse implication.\\
Assuming $k$ is a positive definite kernel defined on $X$, we'll proceed to construct a feature mapping $\bm\phi$ into a Hilbert space for which $k$ is the kernel. We'll suppose that $X$ is an infinite set in the following discussion.
\begin{enumerate}
	\item Define the feature mapping $\bm\phi$ and construct a vector space $H_{pre}$.\\
	We define
	$$\bm\phi:x\longmapsto k(\cdot,x)$$
	According to this mapping, we span the image set to a vector space
	$$H_{pre}=\Big\{\sum_{i=1}^{n}\alpha_{i}k(\cdot,x_{i}):n\in\mathbb{N},\ \bm x_{i}\in X,~\alpha_{i}\in\mathbf{R},~i=1,\cdots,N \Big\}$$
	\item Make $H_{pre}$ an inner product space by defining inner product on $H_{pre}$.\\
	We define a binary function "$<\cdot,\cdot>$" on $H_{pre}$: for any $f,g\in \bm F$
	$$ f(\cdot)=\sum_{i=1}^{n}\alpha_{i}k(\cdot,x_{i}) $$
	$$ g(\cdot)=\sum_{j=1}^{m}\beta_{i}k(\cdot,x_{j}) $$
	$$ <f,g>=\sum_{i=1}^{n}\sum_{j=1}^{m}\alpha_{i}\beta_{j}k(x'_{j},x_{i}) $$
	The bilinearity and symmetry are easy to prove. Because the function$k$ is positive definite, we have
	$$ <f,f>=\sum_{i,j=1}^{m}\alpha_{i}\alpha_{j}k(x_{i},x_{j}),\forall\ f\in H_{pre} $$\\
	Use the symetric and positive definite property of Gram matrix, we can prove the Cauchy-Schwarz inequality
	\begin{equation}
	<f,g>^2\leqslant<f,f><g,g>
	\end{equation}
	Notice that
	\begin{equation}\label{reproducing property}
	<f, k(\cdot,x)>=\sum_{i=1}^{m}\alpha_{i}k(x_{i},x)=f(x)
	\end{equation}
	Take $g(\cdot)=k(\cdot,x)$ into (45), we have
	$$ <f,k(\cdot,x)>^2=|f(x)|^2\leq <f,f>k(x,x) $$
	So $<f,f>=0$ implies $f(x)\equiv 0$.
	Above all, binary function "$\bm\cdot$" satisfies bilinearity, symmetry and positive definiteness which suffices to show that "$<\cdot,\cdot>$" is an inner product on $H_{pre}$. \\
	Now vector space $H_{pre}$ equipped with inner product "$<\cdot,\cdot>$" is an inner product space.
	\item Complete the inner product space $H_{pre}$, we can get a Hilbert space $H$.\\
	Let $H$ be a completion of $H_{pre}$ and $I:H_{pre} \rightarrow H$ be the corresponding isometric embedding. Then $H$ is a Hilbert space and we have
	$$
	<Ik(\cdot,x'),Ik(\cdot,x)>_H=<k(\cdot,x'),k(\cdot,x)>_{H_{pre}}=k(x,x')
	$$
	for all $x,x'\in X, i.e., x\mapsto Ik(\cdot,x), x\in X$, defines a feature map of $k$.
	This Hilbert space is called $\textbf{reproducing\ kernel\ Hilbert\ space\ (RKHS)}$ because it satisfies equation (\ref{reproducing property}) which is called \textbf{reproducing property}.
\end{enumerate}
\end{proof}

\section{Relationship between SVM and Logistic Regression}
\includepdf[pages=-]{6DL/SVM-LR.pdf}


\section{Principles of Data Reduction}

In this chapter, we explore how we can use functions of a sample $\vec{X}$ to make inferences about an unknown parameter (of the distribution of the sample) $\theta$.

\begin{definition}[Statistic]
    A statistic is any function of the data.
\end{definition}

A statistic $T$ forms a partition of the sample space $\X$ according to its image, $\mathcal{T} = \{t: \exists \vec{x} \in \X{} \text{ s.t. } t = T(\vec{x})\}$. In this way a statistic provides a method of data reduction. An experimenter who observes only $T$ will treat as equal two samples $\vec{x}, \vec{y}$ for which $T(\vec{x}) = T(\vec{y})$.

\subsection{The Sufficiency Principle}

\begin{definition}[Sufficient Statistic]
    A statistic $T(\vec{X})$ is a \emph{sufficient statistic} for $\theta$ if the conditional distribution of the sample $\vec{X}$ given $T(\vec{X})$ does not depend on $\theta$.
\end{definition}

\begin{remark}
    We ignore the fact that all points have 0 probability for continuous distributions.
\end{remark}

\begin{definition}[The Sufficiency Principle]
    If $T(\vec{X})$ is a sufficient statistic for $\theta$, then any inference about $\theta$ should depend on the sample $\vec{X}$ only through $T(\vec{X})$.
\end{definition}

\begin{theorem}
    If $p(\vec{x}\vert{}\theta)$ is the pmf/pdf of the sample $\vec{X}$ and $q(t\vert{} \theta)$ is the pmf/pdf of $T(\vec{X})$, then $T(\vec{X})$ is a sufficient statistic for $\theta$ if $\forall \vec{x} \in \X{}$, $p(\vec{x}\vert{}\theta) / q(t\vert{}\theta)$ is constant as  a function of $\theta$.
\end{theorem}

\begin{remark}[Niceness of the exponential family]
    It turns out that outside of the exponential family, it is rare to have a sufficient statistic that is of smaller dimension than the size of the sample.
\end{remark}

\begin{theorem}[Factorisation Theorem]
    Let $f(\vec{x} \vert{} \theta)$ denote the pmf/pdf of a sample $\vec{X}$. A statistic $T(\vec{X})$ is a sufficient statistic for $\theta$ if and only if there exists functions $g(t \vert{} \theta)$ and $h(\vec{x})$ such that $\forall \vec{x} \in \X{}$, $\forall \theta \in \Theta$
    \[
        f(\vec{x} \vert{} \theta) = g(T(\vec{x}) \vert{} \theta)h(\vec{x}).
    \]
\end{theorem}

\begin{remark}
    This theorem shows that the identity is a sufficient statistic. It is straightforward to show from this that any bijection of a sufficient statistic is a sufficient statistic.
\end{remark}

\begin{theorem}
    Let $X_1, \dots, X_n$ be iid observations from a pmf/pdf $f(x \vert{} \theta)$ from an exponential family
    \[
        f(x|\vec{\theta}) = h(x)c(\vec{\theta})\exp\left( \sum_{i=1}^k w_i(\vec{\theta}) t_i(x) \right),
    \]
    where $\vec{\theta} = (\theta_1, \dots, \theta_d)$, $d \leq k$. Then $\vec{T}(\vec{X})$ defined by
    \[
        T_i(\vec{X}) = \sum_{j=1}^k t_i(\vec{X}_j)
    \]
    is a sufficient statistic for $\vec{\theta}$.
\end{theorem}

\begin{definition}[Minimal sufficient statistic]
    A sufficient statistic $T(\vec{X})$ is called a \emph{minimal sufficient statistic} if, for any other sufficient statistic $T'(\vec{X})$, $T$ is a function of $T'$.
\end{definition}

\begin{remark}
    By `function of' we mean that if $T'(\vec{x}) = T'(\vec{y})$ then $T(\vec{x}) = T(\vec{y})$ -- $T$ varies with respect to $X$ only insofar as it varies with $T'$. This means that each tile of the partition of the sample space according to the image of $T'$ is a subset of some tile in the partition according to $T$. This means that minimal sufficient statistics provide the coarsest possible tiling of the sample space and thus are the sufficient statistics that provide the greatest data reduction.
\end{remark}

\begin{theorem}
    Let $f(\vec{x} \vert{} \theta)$ be the pmf/pdf of a sample $\vec{X}$. Suppose there exists a function $T(\vec{X})$ such that $\forall \vec{x}, \vec{y} \in \X{}$ the ratio $f(\vec{x}\vert{}\theta)/f(\vec{y}\vert{}\theta)$ is constant as a function of $\theta$ if an only if $T(\vec{x}) = T(\vec{y})$. Then $T(\vec{X})$ is a minimal sufficient statistic for $\theta$.
\end{theorem}

\begin{definition}[Necessary Statistic]
    A statistic is \emph{necessary} if it can be written as a function of every sufficient statistic.
\end{definition}

\begin{theorem}
    A statistic is minimal sufficient if and only if it is a necessary and sufficient statistic.
\end{theorem}

\begin{definition}[Ancillary Statistic]
    A statistic $S(\vec{X})$ whose distribution does not depend on the parameter $\theta$ is called an \emph{ancillary statistic}.
\end{definition}

\begin{definition}[First Order Ancillary]
    A statistic $V(\vec{X})$ is \emph{first order ancillary}  if $\E{}[V(\vec{X})]$ is independent of $\theta$.
\end{definition}

\begin{definition}[Complete Statistic]
    Let $f(t \vert{} \theta)$ be a family of pdfs or pmfs for a statistic $T(\vec{X})$. The family of probability distributions is called \emph{complete} if for every (measurable) function $g$
    \[
        \E{}[g(T)] = 0  \,\, \forall \theta \implies \P{}(g(T) = 0) = 1 \,\, \forall \theta.
        \]
    Equivalently, $T(\vec{X})$ is called a $\emph{complete statistic}$.
\end{definition}

(It is left unsaid that the function $g$ must be independent of $\theta$.)

\begin{theorem}[Basu's Theorem]
    If $T(\vec{X})$ is a complete and minimal sufficient statistic, then $T(\vec{X})$ is independent of every ancillary statistic.
\end{theorem}

\begin{theorem}[Complete statistics in the exponential family]
    Let $X_1, \dots, X_n$ be iid observations from a pmf/pdf $f(x \vert{} \theta)$ from an exponential family
    \[
        f(x|\vec{\theta}) = h(x)c(\vec{\theta})\exp\left( \sum_{i=1}^k w_i(\vec{\theta}) t_i(x) \right),
    \]
    where $\vec{\theta} = (\theta_1, \dots, \theta_d)$, $d \leq k$. Then $\vec{T}(\vec{X})$ defined by
    \[
        T_i(\vec{X}) = \sum_{j=1}^k t_i(\vec{X}_j)
    \]
    is complete if $\{(w_1(\vec{\theta}), \dots, w_n(\vec{\theta}))\}$ contains an open set in $\R{}^k$.
\end{theorem}

\begin{remark}
    The open set criteria excludes curved exponential families.
\end{remark}

\begin{theorem}
    If a minimal sufficient statistic exists, then every complete statistic is minimal sufficient.
\end{theorem}

\subsection{The Likelihood Principle}

\begin{definition}[Likelihood]
    Let $f(\vec{x}\vert{} \theta)$ denote the pmf/pdf of the sample $\vec{X}$. Then the \emph{likelihood function}, given an observation $\vec{X} = \vec{x}$, is
    \[
        L(\theta \vert{} \vec{x}) = f(\vec{x} \vert{} \theta)
    \]
    as a function of $\theta$.
\end{definition}

\begin{definition}[Likelihood Principle]
    If $\vec{x}$ and $\vec{y}$ are two sample points such that $L(\theta \vert{} \vec{x})$ is proportional to $L(\theta \vert{} \vec{y})$, that is, there exists a constant $C(\vec{x}, \vec{y})$ such that
    \[
        L(\theta \vert{} \vec{x}) = C(\vec{x}, \vec{y})L(\theta \vert{} \vec{y}) \quad \forall \theta,
    \]
    then the conclusions drawn from $\vec{x}$ and $\vec{y}$ should be identical.
\end{definition}

\subsection{A Slightly More Formal Construction}
\begin{definition}[Experiment]
    We define an \emph{experiment} $E$ to be a triple $(\vec{X}, \theta, f(\vec{x}\vert{}\theta))$, where $\vec{X}$ is a random vector with pmf/pdf $f$.\\

    Knowing what experiment $E$ was performed, an experimenter will observer $\vec{X} = \vec{x}$. The conclusions they draw about $\theta$ will be denoted $\text{Ev}(E, \vec{x})$, which stands for \emph{the evidence about $\theta$ arising from $E$ and $\vec{x}$}.
\end{definition}

\begin{definition}[Formal Sufficiency Principle]
    Consider an experiment $E = (\vec{X}, \theta, f(\vec{x}\vert{}\theta))$ and suppose that $T(\vec{X})$ is a sufficient statistic for $\theta$. If $\vec{x}$ and $\vec{y}$ are sample points satisfying $T(\vec{x}) = T(\vec{y})$, then $\text{Ev}(E, \vec{x}) = \text{Ev}(E, \vec{y})$.
\end{definition}

\begin{definition}[Conditionality Principle]
    Suppose that $E_1 = \{X_1, \theta, f_1(x_1\vert{} \theta)\}$ and $E_2 = \{X_2, \theta, f_2(x_2\vert{} \theta)\}$ are two experiments, where only the unknown parameter $\theta$ need be common between the two experiments. Consider the mixed experiment in which the random variable $J$ is observed, where $\P{}(J = 1) = \P{}(J = 2) = \frac12$ (independent of $\vec{X}_1, \vec{X}_2, \theta$), and then the experiment $E_J$ is performed. Formally, the experiment performed is $E^* = (\vec{X}^*, \theta, f(\vec{x}^* \vert{} \theta))$, where $\vec{X}^* = (j, \vec{X})j$ and $f^*(\vec{x}^* \vert{} \theta) = f^*((j, \vec{x}_j)\vert{} \theta) = \frac12 f_j(\vec{x}_j \vert{}\theta)$. Then
    \[
        \text{Ev}(E^*, (j, \vec{x}_j)) = \text{Ev}(E_j, \vec{x_j}).
    \]
    That is, information about $\theta$ depends only on the experiment run (not on the fact that the particular experiment was chosen).
\end{definition}

\begin{definition}{Formal Likelihood Principle}
    Suppose that we have two experiments, $E_1 = (\vec{X}_1, \theta, f_1(\vec{x}_1 \vert{} \theta))$ and $E_2 = (\vec{X}_2, \theta, f_2(\vec{x}_2 \vert{} \theta))$ where the unknown parameter $\theta$ is the same in both experiments. Suppose that $\vec{x}_1^*$ and $\vec{x}_2^*$ are sample points from $E_1$ and $E_2$ respectively, such that
    \[
        L(\theta \vert{} \vec{x}_2^*) = CL(\theta \vert{} \vec{x}_1^*)
    \]
    for all $\theta$ and for some constant $C(\vec{x}_1^*, \vec{x}_2^*)$ that is independent of $\theta$. Then
    \[
        \text{Ev}(E_1, \vec{x}_1^*) = \text{Ev}(E_2, \vec{x}_2^*).
    \]
\end{definition}

\begin{remark}
    Note that this is more general from the other likelihood principle since it concerns two experiments (that we can of course set to be equal).
\end{remark}

\begin{corollary}[Likelihood Principle Corollary]
    If $E = (\vec{X}, \theta, f(\vec{x} \vert{} \theta))$ is an experiment then $\text{Ev}(E, \vec{x})$ should depend on $E$ and $\vec{x}$ only through $L(\theta \vert{} \vec{x})$.
\end{corollary}

\begin{theorem}[Birnbaum's Theorem]
    The Formal Likelihood Principle follows from the Formal Sufficiency Principle and the Conditionality Principle. The converse is also true.
\end{theorem}

\begin{remark}
    Many common statistical procedures violate the Formal Likelihood Principle -- the topic of the applicability of these principles is not settled. For instance, checking the residuals of a model (to grade the model) violates the Sufficiency Principle. These notions are model dependent, so may not be applicable until \emph{after} we have decided on a model.
\end{remark}

\subsection{The Equivariance Principle}

\begin{definition}[Measurement Equivariance]
    Inferences should not depend on the measurement scale used.
\end{definition}

\begin{definition}[Formal Invariance]
    If two inference problems have the same formal structure, in terms of the mathematical model used, then the same inference procedure should be used, regardless of the physical realisation.
\end{definition}

\begin{definition}[Equivariance Principle]
    If $\vec{Y} = g(\vec{X})$ is a change of measurement scale such that the model $\vec{Y}$ has the same formal structure  as the model $\vec{X}$, then an inference procedure should be both measurement equivariant and formally invariant.
\end{definition}

\begin{definition}
    Let $\mathcal{F} = \{f(\vec{x} \vert{} \theta): \theta \in \Theta\}$ be a set of pdfs or pmfs for $\vec{X}$, and let $\mathcal{G}$ be  group of transformations on the sample space $\X{}$. Then $\mathcal{F}$ is \emph{invariant under the group} $\mathcal{G}$ if for every $\theta \in \Theta$ and $g \in \mathcal{G}$ there exists a unique $\theta' \in \Theta$ such that $\vec{Y} = g(\vec{X})$ has the distribution $f(\vec{y} \vert{} \theta')$ if $\vec{X}$ has the distribution $f(\vec{x} \vert{} \theta')$.
\end{definition}


\documentclass[12pt]{cdblatex}
\usepackage{exercises}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% --------------------------------------------------------------------------------------------
\section*{Exercise 2.1 Using Cadabra's own product rule}

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u#}::Indices(position=independent).

   \nabla{#}::Derivative.
   \partial{#}::PartialDerivative.

   # templates for covariant derivatives

   deriv1 := \nabla_{a}{A?^{b}} -> \partial_{a}{A?^{b}}
                                 + \Gamma^{b}_{c a} A?^{c}.

   deriv2 := \nabla_{a}{A?_{b}} -> \partial_{a}{A?_{b}}
                                 - \Gamma^{c}_{b a} A?_{c}.

   # create an object

   uv := \nabla_{a}{v_{b} u^{b}}
       - \partial_{a}{v_{b} u^{b}}.      # cdb (ex-0201.101,uv)

   # apply the rules, then simplify

   product_rule   (uv)                   # cdb (ex-0201.102,uv)
   substitute     (uv,deriv1)            # cdb (ex-0201.103,uv)
   substitute     (uv,deriv2)            # cdb (ex-0201.104,uv)
   distribute     (uv)                   # cdb (ex-0201.105,uv)
   sort_product   (uv)                   # cdb (ex-0201.106,uv)
   rename_dummies (uv)                   # cdb (ex-0201.107,uv)
\end{cadabra}

\begin{align}
   \cdb{ex-0201.101} &= \Cdb{ex-0201.102}\\
                     &= \Cdb{ex-0201.103}\\
                     &= \Cdb{ex-0201.104}\\
                     &= \Cdb{ex-0201.105}\\
                     &= \Cdb{ex-0201.106}\\
                     &= \Cdb{ex-0201.107}
\end{align}

\clearpage

% --------------------------------------------------------------------------------------------
\section*{Exercise 2.1 Using hand crafted product rules}

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u#}::Indices(position=independent).

   \nabla{#}::Derivative.
   \partial{#}::PartialDerivative.

   # templates for covariant derivatives

   deriv1 := \nabla_{a}{A?^{b}} -> \partial_{a}{A?^{b}}
                                 + \Gamma^{b}_{c a} A?^{c}.

   deriv2 := \nabla_{a}{A?_{b}} -> \partial_{a}{A?_{b}}
                                 - \Gamma^{c}_{b a} A?_{c}.

   # tempaltes for product rules

   deriv3 := \nabla_{a}{A?_{b} B?^{c}} -> B?^{c} \nabla_{a}{A?_{b}}
                                        + A?_{b} \nabla_{a}{B?^{c}}.

   deriv4 := \partial_{a}{A?_{b} B?^{c}} -> B?^{c} \partial_{a}{A?_{b}}
                                          + A?_{b} \partial_{a}{B?^{c}}.

   # create an object

   uv := \nabla_{a}{v_{b} u^{b}}
       - \partial_{a}{v_{b} u^{b}}.      # cdb (ex-0201.201,uv)

   # apply the rules, then simplify

   substitute     (uv,deriv3)            # cdb (ex-0201.202,uv)
   substitute     (uv,deriv4)            # cdb (ex-0201.203,uv)
   substitute     (uv,deriv1)            # cdb (ex-0201.204,uv)
   substitute     (uv,deriv2)            # cdb (ex-0201.205,uv)
   distribute     (uv)                   # cdb (ex-0201.206,uv)
   sort_product   (uv)                   # cdb (ex-0201.207,uv)
   rename_dummies (uv)                   # cdb (ex-0201.208,uv)
\end{cadabra}

\begin{align}
   \cdb{ex-0201.201} &= \Cdb{ex-0201.202}\\
                     &= \Cdb{ex-0201.203}\\
                     &= \Cdb{ex-0201.204}\\
                     &= \Cdb{ex-0201.205}\\
                     &= \Cdb{ex-0201.206}\\
                     &= \Cdb{ex-0201.207}\\
                     &= \Cdb{ex-0201.208}
\end{align}

\end{document}


\section{Abstract}

A \emph{Support Vector Machine} is a learning model used both for \emph{classification} and \emph{regression} tasks whose goal is to construct a \emph{maximum margin separator}, i.e., a decision boundary with the largest distance from the nearest training data points.

The aim of this report is to compare the \emph{primal}, the \emph{Wolfe dual}~\cite{fletcher2009support} and the \emph{Lagrangian dual} formulations of this model in terms of \emph{complexity}.

Firstly, a detailed mathematical derivation of the model for all these formulations is given, then three algorithms are described to solve the optimization problem in case of \emph{primal}, \emph{Wolfe dual} or \emph{Lagrangian dual} formulation of the problem, explaining their theoretical properties, i.e., \emph{convergence rate} and \emph{complexity}.

Finally, some experiments are shown for \emph{linearly} and \emph{nonlinearly} separable generated datasets to compare the performance with different \emph{hyperparameters} and different \emph{kernels}, also comparing the \emph{custom} results with \emph{liblinear}~\cite{fan2008liblinear} for the \emph{primal} formulations, \emph{libsvm}~\cite{chang2011libsvm} and \emph{cvxopt}~\cite{vandenberghe2010cvxopt} for the \emph{dual} ones.


\lab{Algorithm}{The Knapsack Problem}{The Knapsack Problem}
\label{Ch:Knapsack}

\objective{This section teaches about NP-hard problems using the knapsack problem as an example.}

\section*{The Knapsack Problem}

%Lab \ref{Knapsack}

The two dimensional Knapsack problem is you are given a knapsack that can only hold a certain weight. You have a plethera of items that have value and weight. The goal is to find the combination of items that have the most value with out exceeding the weight.

A three dimensional version the items would have volume and the knapsack can only hold an certain volume as well as weight.

This problem is has many different applications for disecion making in a wide variety of fields. Two examples are finding the least wasteful way to cut raw materials and the selection of capital investments and financial portfolios.

\begin{problem}
Write a function that solves the Knapsack problem by testing all the combination and choosing the one with the most value that meets the weight constraint. Only test it up to 15 objects.
\end{problem}

This way finds the optimal solution, but there is a problem. Given $n$ objects the number of combinations is $2^n$. The complecity of the problem grows fast. For example, if you had 50 items, it would take more than 10 years to compute the optimal solution. For all the following timing problems let your items have vaules between 1 and 100 and the weights between 1 and $\frac{Capacity}{10}$.

\begin{problem}
Time the Knapsack problem for $11-20$ items with a carrying capacity of $10,000$. Plot the times. What is the complexity of the algorithm for increasing the number of items?
\end{problem}

\begin{problem}
Time that algorithm for the carrying capacities $10,000-90,000$ every mutiple of $10,000$ with $15$ items. Plot the times. What is the complexity of the algorithm for increasing the weight?
\end{problem}

\section*{NP-Hard}

Any problem that is not polynomial time is know as NP-hard. The Knapsack Problem is a NP-hard problem. Using the branch and bound approach we can find the optimal solution in psuedo-polynomial time.

\section*{Branch and Bound method}
We calculate the optimal solution using only the first i items for weights 0 through W (where W is the max weight the knapsack can hold)

We do this by intializing 0th row to being empty.
For i$>$0 row we go through j (being the weight) from 1 thorugh W. If the ith element's weight is more than j then the optimal combination is the same as it was for i-1. If ith element's weight is less than j, then we compare the combination at i-1 at wieght j to the combination i-1 at the weight j-the ith element weight plus the ith element. Whichever one has a higher value is the optimal combination using the first ith elements less than or equal to the wieght of j. You continue this until you have done all n elements. The combination using n items with a weight of W is guaranteed to be the optimal combination.

This only does $W*n$ checks, so it is a lot faster. This works by eliminating combinations that could not be the optimal solution

\begin{problem}
Write a function that solves the Knapsack problem using the branch and bound method.
\end{problem}

\begin{problem}
Time the Knapsack problem for $11-20$ items with a carrying capacity of $10,000$. Plot the times. What is the complexity of the algorithm for increasing the number of items? For the items use the same specifications as in the previous timing problems.
\end{problem}

\begin{problem}
Time that algorithm for the carrying capacities $10,000-90,000$ every mutiple of $10,000$ with $15$ items. Plot the times. What is the complexity of the algorithm for increasing the weight?
\end{problem}


\section*{Exercises}

\label{LU}

\begin{ex} \label{p7.1}Find an $LU$ factorization of $\begin{mymatrix}{rrr}
1 & 2 & 0 \\
2 & 1 & 3 \\
1 & 2 & 3
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{ccc}
1 & 2 & 0 \\
2 & 1 & 3 \\
1 & 2 & 3
\end{mymatrix} = \begin{mymatrix}{ccc}
1 & 0 & 0 \\
2 & 1 & 0 \\
1 & 0 & 1
\end{mymatrix} \begin{mymatrix}{rrr}
1 & 2 & 0 \\
0 & -3 & 3 \\
0 & 0 & 3
\end{mymatrix}
\]

\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of $\begin{mymatrix}{rrrr}
1 & 2 & 3 & 2 \\
1 & 3 & 2 & 1 \\
5 & 0 & 1 & 3
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{cccc}
1 & 2 & 3 & 2 \\
1 & 3 & 2 & 1 \\
5 & 0 & 1 & 3
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
1 & 1 & 0 \\
5 & -10 & 1
\end{mymatrix} \begin{mymatrix}{rrrr}
1 & 2 & 3 & 2 \\
0 & 1 & -1 & -1 \\
0 & 0 & -24 & -17
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix $\begin{mymatrix}{rrrr}
1 & -2 & -5 & 0 \\
-2 & 5 & 11 & 3 \\
3 & -6 & -15 & 1
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{rrrr}
1 & -2 & -5 & 0 \\
-2 & 5 & 11 & 3 \\
3 & -6 & -15 & 1
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
-2 & 1 & 0 \\
3 & 0 & 1
\end{mymatrix} \begin{mymatrix}{rrrr}
1 & -2 & -5 & 0 \\
0 & 1 & 1 & 3 \\
0 & 0 & 0 & 1
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix $\begin{mymatrix}{rrrr}
1 & -1 & -3 & -1 \\
-1 & 2 & 4 & 3 \\
2 & -3 & -7 & -3
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{rrrr}
1 & -1 & -3 & -1 \\
-1 & 2 & 4 & 3 \\
2 & -3 & -7 & -3
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
-1 & 1 & 0 \\
2 & -1 & 1
\end{mymatrix} \begin{mymatrix}{rrrr}
1 & -1 & -3 & -1 \\
0 & 1 & 1 & 2 \\
0 & 0 & 0 & 1
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix $\allowbreak \allowbreak
\begin{mymatrix}{rrrr}
1 & -3 & -4 & -3 \\
-3 & 10 & 10 & 10 \\
1 & -6 & 2 & -5
\end{mymatrix}$.
\begin{sol}
\[
 \begin{mymatrix}{rrrr}
1 & -3 & -4 & -3 \\
-3 & 10 & 10 & 10 \\
1 & -6 & 2 & -5
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
-3 & 1 & 0 \\
1 & -3 & 1
\end{mymatrix} \begin{mymatrix}{rrrr}
1 & -3 & -4 & -3 \\
0 & 1 & -2 & 1 \\
0 & 0 & 0 & 1
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix
$\begin{mymatrix}{rrrr}
1 & 3 & 1 & -1 \\
3 & 10 & 8 & -1 \\
2 & 5 & -3 & -3
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{rrrr}
1 & 3 & 1 & -1 \\
3 & 10 & 8 & -1 \\
2 & 5 & -3 & -3
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
3 & 1 & 0 \\
2 & -1 & 1
\end{mymatrix} \begin{mymatrix}{rrrr}
1 & 3 & 1 & -1 \\
0 & 1 & 5 & 2 \\
0 & 0 & 0 & 1
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix $\begin{mymatrix}{rrr}
3 & -2 & 1 \\
9 & -8 & 6 \\
-6 & 2 & 2 \\
3 & 2 & -7
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{rrr}
3 & -2 & 1 \\
9 & -8 & 6 \\
-6 & 2 & 2 \\
3 & 2 & -7
\end{mymatrix} = \begin{mymatrix}{rrrr}
1 & 0 & 0 & 0 \\
3 & 1 & 0 & 0 \\
-2 & 1 & 1 & 0 \\
1 & -2 & -2 & 1
\end{mymatrix}  \begin{mymatrix}{rrr}
3 & -2 & 1 \\
0 & -2 & 3 \\
0 & 0 & 1 \\
0 & 0 & 0
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix $\begin{mymatrix}{rrr}
-3 & -1 & 3 \\
9 & 9 & -12 \\
3 & 19 & -16 \\
12 & 40 & -26
\end{mymatrix}$.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Find an $LU$ factorization of the matrix $\begin{mymatrix}{rrr}
-1 & -3 & -1 \\
1 & 3 & 0 \\
3 & 9 & 0 \\
4 & 12 & 16
\end{mymatrix}$.
\begin{sol}
\[
\begin{mymatrix}{rrr}
-1 & -3 & -1 \\
1 & 3 & 0 \\
3 & 9 & 0 \\
4 & 12 & 16
\end{mymatrix} = \begin{mymatrix}{rrrr}
1 & 0 & 0 & 0 \\
-1 & 1 & 0 & 0 \\
-3 & 0 & 1 & 0 \\
-4 & 0 & -4 & 1
\end{mymatrix} \begin{mymatrix}{rrr}
-1 & -3 & -1 \\
0 & 0 & -1 \\
0 & 0 & -3 \\
0 & 0 & 0
\end{mymatrix}
\]
\end{sol}
\end{ex}

\begin{ex} Find the $LU$ factorization of the coefficient matrix using Dolittle's
method and use it to solve the system of equations.
\begin{equation*}
\begin{array}{c}
x+2y=5 \\
2x+3y=6
\end{array}
\end{equation*}
\begin{sol}
An $LU$ factorization of the coefficient matrix is
\[
\begin{mymatrix}{cc}
1 & 2 \\
2 & 3
\end{mymatrix} =  \begin{mymatrix}{cc}
1 & 0 \\
2 & 1
\end{mymatrix} \begin{mymatrix}{cc}
1 & 2 \\
0 & -1
\end{mymatrix}
\]
First solve
\[
\begin{mymatrix}{cc}
1 & 0 \\
2 & 1
\end{mymatrix} \begin{mymatrix}{c}
u \\
v
\end{mymatrix} =\begin{mymatrix}{c}
5 \\
6
\end{mymatrix}
\]
which gives $\begin{mymatrix}{c}
u \\
v
\end{mymatrix} =$ $\begin{mymatrix}{r}
5 \\
-4
\end{mymatrix}$. Then solve
\[
\begin{mymatrix}{rr}
1 & 2 \\
0 & -1
\end{mymatrix} \begin{mymatrix}{c}
x \\
y
\end{mymatrix} =\begin{mymatrix}{r}
5 \\
-4
\end{mymatrix}
\]
which says that $y=4$ and $x=-3$.
\end{sol}
\end{ex}

\begin{ex} Find the $LU$ factorization of the coefficient matrix using Dolittle's
method and use it to solve the system of equations.
\begin{equation*}
\begin{array}{c}
x+2y+z=1 \\
y+3z=2 \\
2x+3y=6
\end{array}
\end{equation*}
\begin{sol}
An $LU$ factorization of the coefficient matrix is
\[
\begin{mymatrix}{rrr}
1 & 2 & 1 \\
0 & 1 & 3 \\
2 & 3 & 0
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 & 1 & 0 \\
2 & -1 & 1
\end{mymatrix} \begin{mymatrix}{rrr}
1 & 2 & 1 \\
0 & 1 & 3 \\
0 & 0 & 1
\end{mymatrix}
\]
First solve
\[
 \begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 & 1 & 0 \\
2 & -1 & 1
\end{mymatrix} \begin{mymatrix}{c}
u \\
v \\
w
\end{mymatrix} =\begin{mymatrix}{c}
1 \\
2 \\
6
\end{mymatrix}
\]
which yields $u=1,v=2,w=6$. Next solve
\[
\begin{mymatrix}{rrr}
1 & 2 & 1 \\
0 & 1 & 3 \\
0 & 0 & 1
\end{mymatrix} \begin{mymatrix}{c}
x \\
y \\
z
\end{mymatrix} =\begin{mymatrix}{c}
1 \\
2 \\
6
\end{mymatrix}
\]
This yields $z=6,y=-16,x=27$.
\end{sol}
\end{ex}

\begin{ex} Find the $LU$ factorization of the coefficient matrix using Dolittle's
method and use it to solve the system of equations.
\begin{equation*}
\begin{array}{c}
x+2y+3z=5 \\
2x+3y+z=6 \\
x-y+z=2
\end{array}
\end{equation*}
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Find the $LU$ factorization of the coefficient matrix using Dolittle's
method and use it to solve the system of equations.
\begin{equation*}
\begin{array}{c}
x+2y+3z=5 \\
2x+3y+z=6 \\
3x+5y+4z=11
\end{array}
\end{equation*}
\begin{sol}
An $LU$ factorization of the coefficient matrix is
\[
\begin{mymatrix}{rrr}
1 & 2 & 3 \\
2 & 3 & 1 \\
3 & 5 & 4
\end{mymatrix} = \begin{mymatrix}{rrr}
1 & 0 & 0 \\
2 & 1 & 0 \\
3 & 1 & 1
\end{mymatrix} \begin{mymatrix}{rrr}
1 & 2 & 3 \\
0 & -1 & -5 \\
0 & 0 & 0
\end{mymatrix}
\]
First solve
\[
 \begin{mymatrix}{rrr}
1 & 0 & 0 \\
2 & 1 & 0 \\
3 & 1 & 1
\end{mymatrix} \begin{mymatrix}{c}
u \\
v \\
w
\end{mymatrix} =\begin{mymatrix}{c}
5 \\
6 \\
11
\end{mymatrix}
\]
Solution is: $\begin{mymatrix}{c}
u \\
v \\
w
\end{mymatrix} =$ $\begin{mymatrix}{c}
5 \\
-4 \\
0
\end{mymatrix}$. Next solve
\[
\begin{mymatrix}{rrr}
1 & 2 & 3 \\
0 & -1 & -5 \\
0 & 0 & 0
\end{mymatrix} \begin{mymatrix}{c}
x \\
y \\
z
\end{mymatrix} =\begin{mymatrix}{c}
5 \\
-4 \\
0
\end{mymatrix}
\]
Solution is: $\begin{mymatrix}{c}
x \\
y \\
z
\end{mymatrix} =\begin{mymatrix}{c}
7t-3 \\
4-5t \\
t
\end{mymatrix} ,t\in \R$.
\end{sol}
\end{ex}

\begin{ex} Is there only one $LU$ factorization for a given matrix? \textbf{
Hint:\ }Consider the equation
\begin{equation*}
\begin{mymatrix}{rr}
0 & 1 \\
0 & 1
\end{mymatrix} =\begin{mymatrix}{rr}
1 & 0 \\
1 & 1
\end{mymatrix} \begin{mymatrix}{rr}
0 & 1 \\
0 & 0
\end{mymatrix} .
\end{equation*}
Look for all possible $LU$ factorizations.
\begin{sol}
Sometimes there is more than one $LU$ factorization as is the case in this
example. The given equation clearly gives an $LU$ factorization. However, it
appears that the following equation gives another $LU$ factorization.
\[
\begin{mymatrix}{cc}
0 & 1 \\
0 & 1
\end{mymatrix} =\begin{mymatrix}{cc}
1 & 0 \\
0 & 1
\end{mymatrix} \begin{mymatrix}{cc}
0 & 1 \\
0 & 1
\end{mymatrix}
\]
\end{sol}
\end{ex}


\section{Computing Limits: Graphically}\label{sec:LimitsGraphically}

In this section we look at an example to illustrate the concept of a limit \ifont{graphically}.

The graph of a function $f(x)$ is shown below.  We will
analyze the behaviour of $f(x)$ around $x=-5$, $x=-2$, $x=-1$ and $x=0$, and $x=4$. % and also as $x\to\pm\infty$.

$$\includegraphics[width=5.5in]{images/limits-graphically}$$

Observe that $f(x)$ is indeed a function (it passes the vertical line test). We now analyze the function at each point separately.

$\ifont{x=-5:}$ Observe that at $x=-5$ there is no closed circle, thus $f(-5)$ is undefined.
From the graph we see that as $x$ gets closer and closer to $-5$ from the left, then $f(x)$ approaches $2$, so
$$\lim_{x\to -5^-}f(x)=2.$$
Similarly, as $x$ gets closer and closer $-5$ from the right, then $f(x)$ approaches $-3$, so
$$\lim_{x\to -5^+}f(x)=-3.$$
As the right-hand limit and left-hand limit are not equal at $-5$, we know that
$$\lim_{x\to -5}f(x)\quad\mbox{does not exist.}$$

$\ifont{x=-2:}$ Observe that at $x=-2$ there is a closed circle at $0$, thus $f(-2)=0$.
From the graph we see that as $x$ gets closer and closer to $-2$ from the left, then $f(x)$ approaches $3.5$, so
$$\lim_{x\to -2^-}f(x)=3.5.$$
Similarly, as $x$ gets closer and closer $-2$ from the right, then $f(x)$ again approaches $3.5$, so
$$\lim_{x\to -2^+}f(x)=3.5.$$
As the right-hand limit and left-hand limit are both equal to $3.5$, we know that
$$\lim_{x\to -2}f(x)=3.5.$$

Do not be concerned that the limit does not equal 0. This is a discontinuity, which is completely valid, and will be discussed in a later section.

We leave it to the reader to analyze the behaviour of $f(x)$ for $x$ close to $-1$ and $0$.

Summarizing, we have:
\hspace{-2.3cm}{
$$\begin{array}{|c|c|c|c|}
\hline&~&~&~\\
f(-5)~\mbox{is undefined} & f(-2)=0 & f(-1)=-2 & f(0)=-2  \\
~&~&~&~\\
\ds{\lim_{x\to -5^-}f(x)=2} & \ds{\lim_{x\to -2^-}f(x)=3.5} & \ds{\lim_{x\to -1^-}f(x)=0} & \ds{\lim_{x\to 0^-}f(x)=-2}  \\
~&~&~&~\\
\ds{\lim_{x\to -5^+}f(x)=-3} & \ds{\lim_{x\to -2^+}f(x)=3.5} & \ds{\lim_{x\to -1^+}f(x)=-2} & \ds{\lim_{x\to 0^+}f(x)=-2}\\
~&~&~&~\\
\ds{\lim_{x\to -5}f(x)=\mbox{DNE}} & \ds{\lim_{x\to -2}f(x)=3.5} & \ds{\lim_{x\to -1}f(x)=\mbox{DNE}} & \ds{\lim_{x\to 0}f(x)=-2}\\
~&~&~&~\\
\hline
\end{array}$$
}
%\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for Section \ref{sec:LimitsGraphically}}

\begin{enumialphparenastyle}

%%%%%%%%%%
\begin{ex}
Evaluate the expressions by reference to this graph:
$$\includegraphics[width=3.5in]{images/limit-exercise-graph}$$
\begin{multicols}{3}
\begin{enumerate}
	\item	$\ds \lim_{x\to 4} f(x)$
	\item	$\ds \lim_{x\to -3} f(x)$
	\item	$\ds \lim_{x\to 0} f(x)$
	\item	$\ds \lim_{x\to 0^-} f(x)$
	\item	$\ds \lim_{x\to 0^+} f(x)$
	\item	$\ds f(-2)$
	\item	$\ds \lim_{x\to 2^-} f(x)$
	\item	$\ds \lim_{x\to -2^-} f(x)$
	\item	$\ds \lim_{x\to 0} f(x+1)$
	\item	$\ds f(0)$
	\item	$\ds \lim_{x\to 1^-} f(x-4)$
	\item	$\ds \lim_{x\to 0^+} f(x-2)$
\end{enumerate}
\end{multicols}
\begin{sol}
\begin{multicols}{3}
\begin{enumerate}
	\item	$8$
	\item	$6$
	\item	dne
	\item	$-2$
	\item	$-1$
	\item	$8$
	\item	$7$
	\item	$6$
	\item	$3$
	\item	$-3/2$
	\item	$6$
	\item	$2$
\end{enumerate}
\end{multicols}
\end{sol}
\end{ex}

\end{enumialphparenastyle}


\documentclass[cyan,normal,en]{elegantnote}

\title{Computer Graphic Note}
\author{Jiechang Shi}
\version{1.1}
\date{\today}

\begin{document}

\maketitle

\section{Class 1: Overview}
\begin{enumerate}
	\item Frame Buffer
	\begin{itemize}
		\item Pixel: One element of a frame buffer
		\item Pixel depth: Number of bytes per-pixel in the buffer
		\item Resolution: Width $\times$ Height
		\item Buffer size: Total memory allocated for frame buffer
		\item \textbf{Exam Question}: Given Resolution and Pixel depth, Asked Buffer size
		\item Z Value: For solving the hidden-surface removal(HSR) problem
	\end{itemize}
\end{enumerate}
\newpage
\section{Class 2,3: Rasterization}
\begin{enumerate}
	\item For each pixel on screen, the sample point of that pixel is the center point of that pixel, which have \textbf{integer} coordinates. For examples, $(3,2)$.
	\item A simple problem: Rasterizing Lines \\
	\textbf{Program Description}: Given two endpoints, $P = (x_0,y_0)$ , $R=(x_1,y_1)$ find the pixels that make up the line. \\
	\textbf{Note that}: Lines are infinitely thin so they rarely fall on pixel sample point.\\
	\textbf{A Feasible Description}: Rasterize lines as \textbf{closest} pixels to actual lines, with 2 requirement
	\begin{itemize}
		\item No Gap
		\item Minimize error(distance to true line)
	\end{itemize}
	\textbf{To make this question simplify}: Only consider situation that $|x_1 - x_0| \geq |y_1 - y_0| \geq 0 \land |x_1-x_0| \neq 0$, which means the $-1 \leq slope \leq 1$. Otherwise we just exchange $x$ and $y$. \\
	\textbf{A basic Algorithm}: $k=\frac{y_1-y_0}{x_1-x_0}$, $d=y_0-kx_0$, for each $x_0 \leq x \leq x_1$, $y=ROUND(kx+d)$. This method by brute force is inefficient because of the multiplication and the function $ROUND()$.\\
	\textbf{Basic Incremental Algorithm}: for each $x_0 \leq x_i < x_{i+1} \leq x_1$, $y_{i+1} = y_i +k$, However, the successsive addition of a real number can lead to a \textbf{cumulative error buildup}!\\
	\textbf{Midpoint Line Algorithm}:
	\begin{itemize}
		\item For $0 \leq k \leq 1$
		\item For one approximate point $P=(x,y)$, we only have 2 choices for the next point $E=(x+1,y)$ and $NE=(x+1,y+1)$, we should choose the one which is closer to $k(x+1)+d$
		\item Calculate the middle point $M=(x+1,y+\frac{1}{2})$
		\item If the \textbf{Intersection point} $Q$ is below $M$, take $E$ as next, otherwise take $NE$ as next.
		\item Note that: we consider this equation:
		$$ f(x,y)=ax+by+c=(y_1-y_0)x - (x_1-x_0)y + (x_1y_0-y_1x_0)$$
		We assume $a>0$
		\item For a point $(x,y)$\\
		if $f(x,y)=0$, $(x,y)$ lies on the line.\\
		if $f(x,y)<0$, $(x,y)$ lies upon the line.\\
		if $f(x,y)>0$, $(x,y)$ lies below the line.
		\item So we have to test $f(M)=a(x+1)+b(y+\frac{1}{2})+c=f(Former)+a+\frac{b}{2}$
		\item Assum $a>0$, if $f(M) > 0$ choose $NE$ otherwise choose $E$
		\item Update $f(Former)$:\\
		If we choose $E$, $f(Former)=f(Former)+a$\\
		If we choose $NE$, $f(Former)=f(Former)+a+b$\\
		Note that:$a$ and $b$ are \textbf{constant integer}, so here is no \textbf{cumulative error issuse}
	\end{itemize}
	\item A harder problem: Triangles Rasterization\\
	\textbf{Why Triangle}:
	\begin{itemize}
		\item Triangles (\textit{tris}) are a simple explicit 3D surface representation.
		\item Convex and concave polygons (\textit{polys}) can be decomposed into triangles.
		\item Tris are planar and unambiguously defined by three vertex(\textit{verts}) coordinates (\textit{coords}).
	\end{itemize}
	\textbf{Definition}: Find and draw \textbf{pixel} samples \textbf{inside} \textit{tri} edges and interpolate parameters defined at \textit{verts}
	\item \textbf{Rasterizxation and Hidden Surface Removal(HSR) Algorithm Classes}:
	\begin{itemize}
		\item Image order rasterization: ray tracing/ ray casting \\
		\underline{traverse} pixel, process each in world-space\\
		\underline{transform} rays from image-space to world-space
		\item Object order rasterization: scan-line / LEE \\
		\underline{traverse} triangles, process each in image-space\\
		\underline{transform} objects from model-space to image-space
	\end{itemize}
	\item \textbf{LEE Linear Expression Evaluation Algorithm}:
	\begin{itemize}
		\item We already discussed in \textit{Midpoint Line Algorithm} that how to determine a point is on the left(up) or right(below) the line, just \textit{a quick review here}:\\
		Assume the lien have a positive $slope$\\
		For an Edge Equation $E$, for point $(x,y)$: $E(x,y)=dY(x-X)-dX(y-Y)$\\
		if $E(x,y)=0$, $(x,y)$ lies on the line.\\
		if $E(x,y)<0$, $(x,y)$ lies right(below) the line.\\
		if $E(x,y)>0$, $(x,y)$ lies left(up) the line.
		\item For \textbf{Rasterization}:\\
		Compute LEE result for all three edges.\\
		Pixels with \textbf{consistent sign} for all three edges are inside the \textit{tri}.\\
		Include \textbf{edge pixels} on left or right edges.
		\item LEE need to check every pixel in the bounding box.
		\item LEE is very good in parallel(SIMD) system.
		\item Furthermore: Given 3 random \textit{verts} how to find CW edge cycle\\
		Determine L/R and Top/Bot edges for edge-pixel ownership
	\end{itemize}
	\item \textbf{Scan Line Rasterizer}:
	\begin{itemize}
		\item Sort vets by Y
		\item Setup edge DDAs for edges
		\item Sort edges by L or R(The long edge on left or right)
		\item Start from Top Vertice, and switch DDA when hit the middle vertice.
	\end{itemize}
	\item \textbf{Interpolate Z}:\\
	A general 3D plane equation has 4 terms: $Ax+By+Cz+D=0$\\
	$(A,B,C)$ is the normal of that plane, so $(X,Y,Z)_0 \times (X,Y,Z)_1 = (A,B,C)$\\
	Then plug any vertex coord into equation and solve for $D$.\\
	Given $(A,B,C,D)$ and any point $(x,y)$ can solve $z$
	\item \textbf{Used $Z$-buffer to remove hidden surfaces}\\
	Initial $Z$-buffer to MAXINT at the begining of every frame\\
	Interpolate vertex $Z$ values to get $Z_{pix}$\\
	Only write new pixel to the buffer if $Z_{pix}<Z_{buffer}$\\
	Notice that $Z$ should always \textbf{bigger or equal to zero!}
	\item Hidden Line Removal(HLR):\\
	Simple z-buffer does not work when the render only draws edge(outlines of polygons).\\
	Need edge-crossing and object sorting methods.
	\item \textbf{Painter's Algorithm}: render in order front to back\\
	A object is in front of another object means:\\
	Z of all verts of one object is less than the other.\\
	This algorithm not work if Z-sort is ambiguous.
	\item \textbf{Warnock Algorithm}:\\
	Subdivide screen untril a leaf region has a simple front/back relationship.\\
	Leaf regions have one or zero surfaces visible, and the smallest region is usually a pixel\\
	Usually use quad tree subdivision
	\item \textbf{BSP-Tree}:\\
	View-Independent binary tree(pre-calculated) allows a view-dependent front-to-back or back-to-front traversal of surfaces.\\
	Use Painter Algorithm to do back-to-front traversal.\\
	Useful for \textbf{transparency} - full depth-sort of all surface.
	\item \textbf{Culling}:
	\begin{itemize}
		\item Culling with portals: pre-compute the invisible part.
		\item Culling by View Frustum: Skip a triangle iff all its vertices are beyond \textbf{the same screen edge}!\\
		\textbf{Pitfall:} If the vertices are beyond different edge, some part of the \textit{tri} might still in the screen. Image a giant \textit{tri} that cover the whole screen.
		\item Backface Culling: For \textbf{closed(water-tight)} objects, surfaces with \textbf{oriented-normals facing away} from the camera are never visible.\\
		\textbf{Pitfall}: BF Culling only work for water-tight object!
		\item Frustum: Only visible triangles are drawn into the frame buffer.
	\end{itemize}
\end{enumerate}
\newpage
\section{Class 4,5,6,7:Transformations}
\begin{itemize}
	\item Linear transformations (\textit{Xforms}) define a mapping of coordinates (\textit{coords}) in one coordinate frame to another.
	$$V_b=X_{ba}V_a$$
	\item \textbf{Homogeneous Vector} ($V$) is $4\times 1$ columns $(x,y,z,w)^T$
	\item \textbf{Homogeneous Transforms} ($X$) is $4 \times 4$ matrix
	\item From Homogeneous Vector to 3D Vector:
	$$x=\frac{x}{w} , y=\frac{y}{w},z=\frac{z}{w}$$
	\item \textbf{Why} we use Homogeneous Vector?\\
	We want to uniform the transform matrix including translation, scaling, rotation in the same form of matrix.
\end{itemize}
\subsection{Transformation Matrix}
\begin{enumerate}
	\item \textbf{Translation}:
	$$T(t_x,t_y,t_z)\Rightarrow
	\begin{bmatrix}
	1 & 0 & 0 & t_x \\
	0 & 1 & 0 & t_y \\
	0 & 0 & 1 & t_z \\
	0 & 0 & 0 & 1
	\end{bmatrix},
	T^{-1}(t_x,t_y,t_z)=T(-t_x,-t_y,-t_z)
	$$
	\item \textbf{Scaling}:
	$$S(s_x,s_y,s_z)\Rightarrow
	\begin{bmatrix}
	s_x & 0 & 0 & 0 \\
	0 & s_y & 0 & 0 \\
	0 & 0 & s_z & 0 \\
	0 & 0 & 0 & 1
	\end{bmatrix},
	S^{-1}(s_x,s_y,s_z)=S(\frac{1}{s_x},\frac{1}{s_y},\frac{1}{s_z})
	$$
	\item \textbf{Rotation, CCW}:
	$$R_x(\theta)\Rightarrow
	\begin{bmatrix}
	1 & 0 & 0 & 0 \\
	0 & cos\theta & -sin\theta & 0 \\
	0 & sin\theta & cos\theta & 0 \\
	0 & 0 & 0 & 1
	\end{bmatrix},
	R^{-1}_x(\theta)=R^T_x(\theta)
	$$
	$$R_y(\theta)\Rightarrow
	\begin{bmatrix}
	cos\theta & 0 & sin\theta & 0 \\
	0 & 1 & 0 & 0 \\
	-sin\theta & 0 & cos\theta & 0 \\
	0 & 0 & 0 & 1
	\end{bmatrix},
	R^{-1}_y(\theta)=R^T_y(\theta)
	$$
	$$R_z(\theta)\Rightarrow
	\begin{bmatrix}
	cos\theta & -sin\theta & 0 & 0 \\
	sin\theta & cos\theta & 0 & 0 \\
	0 & 0 & 1 & 0 \\
	0 & 0 & 0 & 1
	\end{bmatrix},
	R^{-1}_z(\theta)=R^T_z(\theta)
	$$
	\item Pitfall: \underline{commutative} property is for S,R only.\\
	Here assume \textbf{uniform scaling} in all dimensions. \\
	If S is not an uniform scaling matrix. S,R don't have commutative property.
\end{enumerate}
\subsection{Spaces Transformation}
\begin{enumerate}
	\item \textbf{NDC to Output Device}
	$$X_{sp}\Rightarrow
	\begin{bmatrix}
	\frac{xs}{2} & 0 & 0 & \frac{xs}{2} \\
	0 & -\frac{ys}{2} & 0 & \frac{ys}{2} \\
	0 & 0 & MAXINT & 0 \\
	0 & 0 & 0 & 1
	\end{bmatrix}
	$$
	Note that:\\
	Output Device is \textbf{RH coords} and origin in \textbf{upper left}.$X\in [0,xs), Y\in [0,ys), Z\in [0,MAXINT]$\\
	NDC is \textbf{LH coords} and origin at screen center. $X,Y \in [-1,1],Z \in[0,1]$
	\item \textbf{Perspective Projection}
	$$X_{pi}\Rightarrow
	\begin{bmatrix}
	1 & 0 & 0 & 0 \\
	0 & 1 & 0 & 0 \\
	0 & 0 & \frac{1}{d} & 0 \\
	0 & 0 & \frac{1}{d} & 1
	\end{bmatrix}
	$$
	\textbf{What} is $d$ And \textbf{Why} there are two \textbf{$\frac{1}{d}$}?
	\begin{itemize}
		\item Assume camera is on $(0,0,-d)$, perspective(also image) plane is $z=0$, a object in world space is $(X,Y,Z)$
		\item Defined FOV(field of view) as the angle the camera can see.
		\item Note: $X\in [-1,1]$, so the distant($d$) from Forcus point to view plane can be calculate by this equation:
		$$\frac{1}{d}=tan(\frac{FOV}{2})$$
		\item Futher More: The object project to view plane can be calculate by these equations:
		$$\frac{X}{Z+d}=\frac{x}{d} \Rightarrow x=\frac{X}{\frac{Z}{d}+1}$$
		$$\frac{Y}{Z+d}=\frac{y}{d} \Rightarrow y=\frac{Y}{\frac{Z}{d}+1}$$
		$$\frac{Z}{Z+d}=\frac{z}{d} \Rightarrow z=\frac{Z}{\frac{Z}{d}+1}$$
		$$(x,y,z)=(\frac{X}{\frac{Z}{d}+1},\frac{Y}{\frac{Z}{d}+1},\frac{Z}{\frac{Z}{d}+1})$$
		\item Write this 3D vector to Homogeneous Vector:
		$$(x,y,z,w)=(X,Y,Z,\frac{Z}{d}+1)$$
		\item Futher: We forcus on the \textbf{range} of Z now is $z\in (-\infty,d)$.\\
		But in NDC we hope $z \in [0,1)$ So:\\
		We delete all vector that $Z<0$, because they cannot project to the view plane.\\
		For $z>=0$, we define $z'=\frac{z}{d}$
		\item $(x,y,z',w)=(X,Y,\frac{Z}{d},\frac{Z}{d}+1)=X_{pi}*(X,Y,Z,1)$
	\end{itemize}
	\textbf{Pitfall}: \textbf{Do Z interpolation in Perspective Plane!}\\
	\textbf{Why} we need the farest plane?\\
	Asymptotic curve of $Z$ $vs.$ $z$, that z increase slower when Z is large.\\
	It might map different $Z$ to the same $z$
\end{enumerate}
\subsection{Camera Matrix}
\begin{enumerate}
	\item Assume camera position is $c$, camera look-at point is $l$, here $c$ and $l$ are both in world coordinate. And the world up vector is $\vec{up}$
	\item Camera Z-axis \textbf{in world coordinate} is $$\vec{Z}=\frac{\vec{cl}}{||\vec{cl}||}$$
	\item Camera Y-axis \textbf{in world coordinate} is the orthogonal(vertical) part of world-up vector to Z-axis which is $$\vec{up'}=\vec{up}-(\vec{up}\cdot \vec{Z})\vec{Z}$$
	$$\vec{Y}=\frac{\vec{up'}}{||\vec{up'}||}$$
	\item Camera X-axis \textbf{in world coordinate} is orthogonal to both Y and Z axises. So:
	$$\vec{X}=\vec{Y}\times\vec{Z}$$
	\item Build the $X_{wi}$ from camera space to world space:\\
	X-axis vector $[1,0,0]$ in camera space should be $\vec{X}$ in world space, also for Y,Z-axis vectors, So:
	$$X_{wi}\Rightarrow
	\begin{bmatrix}
	X_x & Y_x & Z_x & 0 \\
	X_y & Y_y & Z_y & 0 \\
	X_z & Y_z & Z_z & 0 \\
	0 & 0 & 0 & 1
	\end{bmatrix}
	$$
	\item Also we need to add the translation of the camera to the Matrix:
	$$X_{wi}\Rightarrow
	\begin{bmatrix}
	X_x & Y_x & Z_x & c_x \\
	X_y & Y_y & Z_y & c_y \\
	X_z & Y_z & Z_z & c_z \\
	0 & 0 & 0 & 1
	\end{bmatrix}
	$$
	\item Now we can get the inverst matrix $X_{iw}$:
	$$X_{iw}\Rightarrow
	\begin{bmatrix}
	X_x & X_y & X_z & -X\cdot c \\
	Y_x & Y_y & Y_z & -Y\cdot c \\
	Z_x & Z_y & Z_z & -Z\cdot c \\
	0 & 0 & 0 & 1
	\end{bmatrix}
	$$
	\item In this proof we know that: If we know the X,Y,Z-axis in world coordinate for a specific space, we can easily build and inverst the translation from or to that space.\\
	This method can also be used to \textbf{proof the general rotation matrix}.
	\item Orbit a Model about a Point\\
	The idea is the same as place camera.\\
	Need to care about \textbf{which space} you current in!
\end{enumerate}
\newpage

\section{Class8-10 Illumination and Shading}
\begin{enumerate}
	\item Global vs. Local Illumination\\
	Global: Models indirect illumination and occlusions
	Local: Only models direct illumination
	\item Irradiance
	$$E=\int_{\Omega} I(x,\omega) cos \theta dx$$
	Note: $I(x,\omega)$ is the light intensity arriving from all directions and entering the hemisphere $\Omega$ over unit serface area.\\
	Also we only care the vertical(normal) part of the light, we dismiss all lights parallel to the surface by using $\cos \theta$
	\item Simplified lighting:\\
	Assume all lights are distant-point light.
	\begin{itemize}
		\item Source have \textbf{uniform} intensity distribution
		\item Neglect distance fallout
		\item Direction to source is constant within scene
		\item Using 2 parameters to define a light:\\
		$direction(x,y,z)$ vector from surface to light source\\
		$intensity(r,g,b)$ of the light
	\end{itemize}
	\item specular reflection and diffuse reflection
	\begin{itemize}
		\item Color shift by attenuation of RPG components for all reflection
		\item Specular Reflection Model(View-Dependent):\\
		$$L_j(V) = L_e \cdot K_s \cdot (V\cdot R)^{spec}$$
		$$R=2(N\cdot L)N-L$$
		Note: $V$ and $R$ should be normalized.\\
		Direction: Reflection occurs mainly in the "mirror" R direction, but there is some spread in similar directions $V$.\\
		$spec$ controls the distribution of intensity about R. Higher value of $spec$ make the surface smoother\\
		$K_s$ controls the color attenuation of Surface.
		\item Diffuse Reflection Models(View-Independent):\\
		$$L_j = L_e \cdot K_d (L\cdot N)$$
		Direction: All \textbf{output} directions are the same. But we only care vertical \textbf{input} light.\\
		$L$ and $N$ should be normalized.\\
		$K_d$ is the surtface attenuation component.
		\item Ambient Light\\
		$$L_j = L_a \cdot K_a$$
		Direction: All input and output directions are the same.\\
		Only one ambient light is needed and allowed.
		\item Complete Shading Equation:
		$$Color=(K_s\sum{ L_e \cdot (V\cdot R)^{spec}}) + (K_d\sum{ L_e \cdot (L\cdot N)}) + (L_a \cdot K_a)$$
	\end{itemize}
	\item Detail about HW4(Lighting Implementation)
	\begin{itemize}
		\item $\vec{L}$ denotes the direction to a infinity-far point-light source
		\item $\vec{E}$ denotes the camera direction. If camera is far away, $\vec{E}$ is constant(In HW4.)
		\item $\vec{N}$ is specified at triangle vertices.
		\item $\vec{R}$ must be computed for each lighting calculation (at \textbf{a point}).\\
		Calculation of $\vec{R}$:\\
		$$\vec{R}=2(\vec{N}\cdot \vec{L})\vec{N}-\vec{L}$$\\
		Avoiding sqrt-root in this calculation
		\item Choosing a Shading Space: Wee need all $\vec{L},\vec{E},\vec{N},\vec{R}$ in some affine(pre-perspective) space \\
		Suggest use \textbf{Image Space} for HW4. \\
		\textbf{Model space} is also a reasonable choice since Normal vectors are already in that space. This is most \textbf{efficient}!
		\item \textbf{Image Space Lighting (ISL)}\\
		Create a Transformation stack from model space to image space.\\
		Need to \textbf{normalized the Scale and delete translation} for each matrix, \textbf{only maintain the rotation}, before push into this stack!
		\item \textbf{Check} the sign of \textbf{$\vec{N}\cdot\vec{E}$} and \textbf{$\vec{N}\cdot\vec{L}$}:\\
		Both positive: Compute lighting model.\\
		Both negative: \textbf{Flip normal($\vec{N}$)} and compute lighting model.\\
		Different sign: Skip it.
		\item \textbf{Check} the sign of \textbf{$\vec{R}\cdot\vec{E}$}: If negative, set to 0.
		\item \textbf{Check} color overflow($>1.0$): Set to 1.
		\item \textbf{Compute Color} at all pixels:\\
		\textbf{Per Face} - flat shading\\
		\textbf{Per Vertex} - interpolate vertex colors, Gouraud Shading(specular highlights are undersampled, aliased).\\
		\textbf{Per Pixel} - interpolate normals, Phong Shading (Expensive computation, but better sampling)\\
		Set \textbf{Shading Modes Parameter} for different lighting calculation.
		\item \textbf{Pitfall} in \textbf{Phong} Interpolation:\\
		Need to \textbf{normalize} the interpolation normal vector.
	\end{itemize}
\end{enumerate}
\subsection{Class10: Something More About Shading}
\begin{enumerate}
	\item Non-Uniform Scaling: \\
	\textbf{A non-uniform scaling alters the relationship between the surface orientation and the Normal Vector.}\\
	So we \textbf{cannot} use the same matrix M for transformation of the Normals and the vertex coordinates.\\
	We can fix this by using a different transformation $Q=f(M)$ for transforming the Normals.
	\item How to create a matrix for Normals:\\
	In HW4, We create a matrix \textbf{dismiss all} scale matrix.\\
	For Detail:\\
	As the definition of Normals:
	$$\vec{N}^T\cdot\vec{P}=0$$
	After include the transform matrix:
	$$(Q\vec{N})^T\cdot(M\vec{P})=0$$
	By Definition of Matrix Multiplyer:
	$$\vec{N}^T \cdot Q^T \cdot M \cdot \vec{P}=0$$
	Since we already know $\vec{N}^T\cdot\vec{P}=0$ we only need the inner part equal to identity matrix:
	$$Q^T \cdot M= I,Q=(M^{-1})^T$$
	Note that: If we only used uniform scaling: $S=I$ after normalization.\\
	If we compute $Q$ for each $M$ pushed on the $X_{im}$ transform stack, the resulting $X_n$ stack has $Q$ and therefore allows non-uniform scaling.
	\item Model Space Lighting(MSL): \\
	Only need to transform Global lighting parameters once per models.\\
	Also need to transform Eye/camera direction into model space.
\end{enumerate}
\newpage

\section{Class 11-13: Texture Mapping}
\subsection{Screen-Space Parameter Interpolation}
\begin{enumerate}
	\item In Z-buffer interpolation, we know that linear interpolation for $z$ is \textbf{wrong in image space}, we need to interpolate in \textbf{perspective space}.
	\item Accurate interpolation of RGB color or Normal vectors should also take perspective into account.\\
	But we can ignore the color and normal interpolation error.
	\item Interpolation for \textbf{Texture Function}: checkerboard Example: Using Linear Interpolation for $u\&v$ is also wrong!
	\item How to compute perspective-correct interpolation of $u,v$ at each pixel.
	\begin{itemize}
		\item For each parameter $P$, we used $P^s$ to denote the value in perspective space.
		\item Note that: For Z interpolation $V_z^s=\frac{V_z}{\frac{V_z}{d}+1}=\frac{V_z \cdot d}{V_z+d}$
		\item Rescale $V_z^s$ to $V_z^s \in [0,Z_{max}]$
		$$V_z^s=\frac{V_z \cdot d}{V_z+d} \cdot (\frac{Z_{max}}{d})=\frac{V_z \cdot Z_{max}}{V_z+d}$$
		\item We can also get the invert equation:
		$$V_z=\frac{V_z^s \cdot d}{Z_{max} - V_z^s}$$
		\item For parameter from image space to perspective space:
		$$P^s=\frac{P}{\frac{V_z}{d}+1}=\frac{Pd}{V_z+d}$$
		\item Also we can get inver equation:
		$$P=\frac{P^s(V_z+d)}{d}$$
		\item We don't have $V_z$ but we already calculated $V_z^s$ in HW2, so we can used that:
		$$P^s=\frac{P}{(\frac{V_z^s}{Z_{max} - V_z^s}+1)}$$
		$$P=P^s \cdot \frac{V_z^s}{Z_{max} - V_z^s}+1$$
		\item Note that we only have $V_z^s$ and $Z_{max}$ in this equation that we already know the value, we don't need to care $d$ and some other parameter.
		\item We used $V_z' = \frac{V_z^s}{Z_{max} - V_z^s}$ to simplify the equation:
		$$P^s=\frac{P}{V_z' +1}$$
		$$P=P^s \cdot(V_z'+1)$$
	\end{itemize}
	\item The Step for Parameter interpolation:\\
	Get $V_z^p$ for each vertex.\\
	Transform $P$ to perspective space $P^s$ for each vertex.\\
	Interpolate $V_z^p$ for each pixel.\\
	Interpolate $P^s$ for each pixel. \\
	Transform $P^z$ back to $P$ by using $V_z^p$ for each pixel.
\end{enumerate}
\newpage
\subsection{Texture}
\begin{enumerate}
	\item Scale $u,v$ to Texture Image Size:\\
	$(u,v)$ coords range over $[0,1]$\\
	2D Image is a pixel array of $xs-1,ys-1$\\
	But $u*(xs-1)$ might not be Integer so we need to interpolate the color for non-Integer $(u,v)$ coordinate from nearest 4 Integer point.
	$$Color(p)=(1-s)(1-t)A+s(1-t)B+stC+(1-s)tD$$
	\item For Phong Shading, using texture function $f(u,v)$ to replace $k_d$ and $k_a$
	\item For Gouraud Shading, using $f(u,v)$ to replace all $k_s$, $k_d$ and $k_a$
	\item Procedural Texture
	\item Bump Texture:
	\begin{itemize}
		\item Alter normals at each pixel to create bump.
		\item Normal Perturbation($\vec{P}$): $N'=N+P$, $\vec{P}$ should be in the same space as N. But $\vec{P}$ should not be in model space.
		\item Better spaces for $\vec{P}$ are Surface Coordinates, Tangent Space.
	\end{itemize}
	\item Surface Coordinates:
	\item Normal Space encodes the surface normal rather than perturbation.
	\item Noise Texture:
	\begin{itemize}
		\item Perlin Noise:(Ref(Chinese): \href{https://www.cnblogs.com/leoin2012/p/7218033.html}{https://www.cnblogs.com/leoin2012/p/7218033.html}
		\item Input: $(x,y,z)$ for 3D and $(u,v)$ in 2D
		\item Output: double value between 0 and 1
		\item We have 2 Pseudo Random Grid for each Integer Point(x,y,z are integers):
		\begin{itemize}
			\item Noise Matrix($d$): The color of Point for noise
			\item Gradient Matrix($g$): A random unit vector for each Point
		\end{itemize}
		\item For each input vector$(u,v)$, if $(u,v)$ isn't Integer, we found 4-corners Integer Point: $(i,j),(i+1,j),(i,j+1),(i+1,j+1)$
		\item For each Integer Point, we use \textbf{dot product} of distant vector(from $(u,v)$ to Integer Point) and gradient vector to get the noise value.
		\item In perlin noise every interpolation is in 1-D. So 2-D need first interpolate y-axis(twice) and then interpolate x-axis. 3-D need 7 interpolation.
		We used linear-interpolation in slides but we can use \textbf{Fade} function(easy curves) for better interpolation.
		\item Turbulence: Sum noise with diminishing ampitude:
		$$turbulence(x)=\sum_{k}^{i=0} \frac{1}{2^i}|noise(2^i x)|$$
	\end{itemize}
	\item Evironment(Reflection) Mapping:
	\begin{itemize}
		\item Basic Idea: During rendering, compute the reflection of Eye vector(not the light vector)
		\item \textbf{Ignore the position of surface point} in scene. We assume all points are on center point of the scene.
		\item Light and scenery are all merge into environment texture.
		\item No object inter-reflection or shadow
		\item Blur texture to simulate diffuse reflection
		\item Sharp texture to simulate specular reflection
	\end{itemize}
	\item Cube Map:
	\begin{itemize}
		\item Transform each Eye reflection vector R back to world space
		\item Find Max component: indicated which face of cube it would intersect
		\item Compute intersection of R with cube face:\\
		Move all Reflection ray tail to center of cube.
		Rescale the max component(for example$y$) of vector R to 1.0.\\
		The other 2 component($x,z$) indicate the texture-pixel.
	\end{itemize}
	\item Refraction Map:
	\begin{itemize}
		\item Use Snell's law to compute refraction vector
		\item Color aberration simulated with $f(\lambda)$ refraction angle for multiple color bands
	\end{itemize}
\end{enumerate}
\subsection{Implementation Of Texutre(HW5)}
\begin{enumerate}
	\item Step1: Texture coordinates: surface point $\rightarrow$ $(u,v)$\\
	Input: vertex in image space\\
	Output: $(u,v)$
	\item Step2: $(u,v)$ $\rightarrow$ RGB color\\
	Input: $(u,v)$
	Output: RGB color from image LUT
	\item Interpolation of $(u,v)$ need to be in perspective space.
	\item Interpolation of 4-corner for non-Integer $(u,v)$ is needed.
\end{enumerate}
\newpage

\section{Class14-16 Antialiasing}
\subsection{The Source of Aliasing}
\begin{enumerate}
	\item Quantization error arise from insufficient accuracy of sample
	\item Aliasing error arise from insufficient samples
	\item Nyquist Theorem: Sample at least twice the rate of highest frequency present in the signal.\\
	f(t) filtered for cutoff freq $\omega_F$(Remove high frequencies before sampling)\\
	Sample Rate $\frac{1}{T_0}$ is greater than $2\omega_F$\\
	Reconsturct(interpolate) with $sinc$ function
	\item Solution: Band-limit the input signal before sampling.
\end{enumerate}
\subsection{Implement Antialiasing(HW6)}
\begin{enumerate}
	\item Antialiasing by jitter supersampling
	\item Sample a pixel several with different center and weight
\end{enumerate}
\subsection{Texture Antialiasing}
\begin{enumerate}
	\item Sample Rate Mismatch: Texture sampling rate generally does not match screen pixel sample rate (Texel:Pixel ratio)
	\item Projected texture in screen image should sample near same rate(1:1) to texture map.\\
	1 Texel: Many Pixel: No aliasing problem, But blur. Fix by using higher resolution textures.\\
	1 Pixel: Many Texel: Aliasing problem. Fix by sample rate is twice highest freq in texture.
	\item Mip Map: Pre-compute filtered version of texture image at octave scale/size intervals.\\
	Using average color of $2 \times 2$ texels.\\
	The space cost is only $33\%$ more.\\
	Scale for each level of Mip Map:
	$$Scale=\frac{dU}{dX}=\frac{dV}{dY}$$
	Pixel Scale is more complex since it is non-axis-aligned(after rotation and projection):
	$$PixelScale=(\frac{dU}{dX},\frac{dU}{dY},\frac{dV}{dX},\frac{dV}{dY})$$
	An approach to match Scale and PixelScale is choosing the highest PixelScale component.(Blur is better than aliasing!)
	\item 3D Interpolation: If the Pixel Scale is between 2 Texel Scales, we need to interpolate between 2 texture samples.
	\item Anisotropic Interpolation: Combine more than $2 \times 2$ pixel in each texture samples.
	\item Summed-Area Table(SAT):Compute a texture table T so that each texel has sum of \textbf{all texels above and left}\\
	SAT provide an approximation approach to get the avagerage color in O(1).\\
	Note that: the texels sample might not be axis-aligned since	 the pixel to texel projection. But the different is strictly less than $\frac{1}{2}$.
\end{enumerate}
\newpage

\section{Final Exam Review}
\begin{enumerate}
	\item Shading Equation:
	$$Color=(K_s\sum{ L_e \cdot (E\cdot R)^{spec}}) + (K_d\sum{ L_e \cdot (L\cdot N)}) + (L_a \cdot K_a)$$
	Know the meaning for every terms:
	\begin{itemize}
		\item $K_s,K_a,K_d$
		\item $L_e,L_a$
		\item $s$
		\item $N,L,R,E$
		\item Equation1: $R=2(N\cdot L) N-L$
	\end{itemize}
	\item Shading Mode:(Flat, Gouraud, Phong)
	\item Texture
	\item Calculate the normal: With Non-translate and Non-scale Matrix To Image Space
	\item Other Topic:
	\begin{itemize}
		\item Enviroment Shading
	\end{itemize}
\end{enumerate}
\newpage

\section{BRDF}

Reference for this section(In Chinese): \href{https://blog.csdn.net/yjr3426619/article/details/81098626}{https://blog.csdn.net/yjr3426619/article/details/81098626}
\subsection{What is BRDF}
\begin{itemize}
	\item Basic Idea of BRDF: The reflect rate for a surface is based on input vector and camera vector.
	$$f(l,v)=\frac{dL_0(v)}{dE}(ForNonPointLight)=\frac{L_o(v)}{E_L cos\theta_i}(ForPointLight)$$
	\begin{itemize}
		\item $l$: input light vector
		\item $v$: camera vector
		\item $L_0$: Output Radiance(Color)
		\item $E_l$: Input Irradiance(Color*$\pi$ in Unity)
		\item $\theta_i$: angle between input vector and surface normal
	\end{itemize}
	\item Properties of $f(l,v)$
	\begin{itemize}
		\item $f(l,v)\geq 0$
		\item $f(l,v)=f(v,l)$
		\item $R(l)=\int_\phi f(l,v) cos\theta_0 d\omega \leq 1$, Discussed later
	\end{itemize}
	\item How to calculate color based on BRDF:
	\begin{itemize}
		\item In Point Light, Point Camera:
		$$L_0(v)=\sum_k f(l_k)\times E_l cos\theta_{ik}$$
	\end{itemize}
	\item Directional-hemispherical reflectance:
	\begin{itemize}
		\item Definition:
		$$R(l)=\int_\phi f(l,v) cos\theta_0 d\omega$$
		\begin{itemize}
			\item $\phi$ is the hemispherical in surface normal half
			\item $\theta_0$ is the angel between camera view and surface normal
			\item $\omega$ is the space angle
		\end{itemize}
		\item Basically, $R(l)$ is \textbf{the sum of the (directional) energy} of all output light in hemisphere.
		\item The output energy \textbf{should be equal or less than} the input energy so:
		$$R(l)\leq 1$$
	\end{itemize}
\end{itemize}
\subsection{How to calculate BRDF}
\subsubsection{From Lambertian Model}
\begin{itemize}
	\item \textbf{Lambertian Model}: Only Diffuse Color, No Specular Color. All output direction are the Same.
	\item In this model, we assume the output energy is part of input energy. So We can build this equation:
	$$R(l)=C_{diff}$$
	So:
	$$C_{diff}=R(l)=\int_\phi f(l,v) cos\theta_0 d\omega$$
	We know $f(l,v)$ and $C_{diff}$ are both constant number:
	$$f(l,v) \cdot \int_\phi cos\theta_0 d\omega=C_{diff}$$
	$$f(l,v)=\frac{C_{diff}}{\pi}$$
\end{itemize}
\subsubsection{A little bit harder: Phong Model}
\begin{itemize}
	\item We assume the ambient light equal to 0 then the shading equation is:
	$$L_0=(cos\theta_i * C_{diff} + (cos\alpha_r)^{spec}*C_{spec})\times B_L$$
	\begin{itemize}
		\item first part is \textbf{Lambertian model}
		\item $\alpha_r$ is the angel between reflection vector and camera vector
	\end{itemize}
	\item And we have shading equation in BRDF format:
	$$L_0(v)= f(l,v)\times E_l cos\theta_{ik}$$
	\item So we have a basic equation for $f(l,v)$ that:
	$$f(l,v)=\frac{C_{diff}}{\pi}+\frac{(cos\alpha_r)^{spec}*C_{spec}}{\pi * cos\theta_i}$$
	\item Notices that: If we need to normalize this equation, let $R_{spec}(l) = C_{spec}$
	\item First: When $\theta_i=\frac{\pi}{2}$, $f(l,v)=+\inf$, which is implausible, so we times $cos\theta_i$ and a constant $k$:
	$$f_{spec}(l,v)=k * \frac{(cos\alpha_r)^{spec}*C_{spec}}{\pi}$$
	\item Second:
	$$C_{spec}=R(l)=\int_\phi k*f_{spec}(l,v) cos\theta_0 d\omega=k*C_{spec}/\pi * \int_\phi(cos\alpha_r)^{spec} cos\theta_0d\omega$$
	\item It's very hard to do this calculation, by experience we can get:
	$$k=\frac{spec+2}{2}$$
	\item So in Phong Model:
	$$f(l,v)=\frac{C_{diff}}{\pi} + \frac{(spec+2)C_{spec}*(cos\alpha_r)^{spec}}{2\pi}$$
\end{itemize}
\subsubsection{A little bit more harder: Blinn-Phong Model}
\begin{itemize}
	\item We define $\vec{H}$ is the Halfway-Vector(Normalize the vector of the middle point) of input vector and camera vector.
	$$H=\frac{L+V}{|L+V|}$$
	\item Shading equation:
	$$L_0=(cos\theta_i * C_{diff} + (cos\beta)^{spec}*C_{spec})\times B_L$$
	$\beta$ is the angel between $H$ and surface normal vector.
	\item Follow the step in Phong Model:
	$$k=\frac{spec+8}{8}$$
	\item And:
	$$f(l,v)=\frac{C_{diff}}{\pi} + \frac{(spec+8)C_{spec}*(cos\beta)^{spec}}{8\pi}$$
\end{itemize}
\subsubsection{Microfacet Model in \href{https://disney-animation.s3.amazonaws.com/library/s2012_pbs_disney_brdf_notes_v2.pdf}{Disney Paper}}
\begin{itemize}
	\item Equation First:
	$$f(l,v)=diffuse + \frac{D(\theta_h)F(\theta_d)G(\theta_l,\theta_v)}{4cos\theta_l cos\theta_v}$$
	\item Microfacet Model: The surface is an aggregate of many microfacet, each microfacet have different normal and you can only see part of the microfacet, and some microfacet might be blocked by others.
	\item How many microfacets you can see is define by $D(\theta_h)$
	\item How many microfacets is blocked by other microfacets is define by $G(\theta_l,\theta_v)$
	\item The fraction of reflaction energy in the total input energy is define by $F(\theta_d)$
	\item $\frac{1}{4cos\theta_l cos\theta_v}$ is a normalize factor.
\end{itemize}
\subsubsection{BRDF With Physics Based Render}
\begin{itemize}
	\item In Disney's paper Section 5.3~5.6, it present how \textbf{roughness} can affect diffuse function and G function.
	\item Diffuse Function:
	$$diffuse=\frac{baseColor}{\pi}(1+(F_{D90}-1)(1-cos\theta_l)^5)(1+(F_{D90}-1)(1-cos\theta_v)^5)$$
	$$F_{D90}=0.5+2*cos\theta^2_d*roughness$$
	\item D Function:
	$$D_GTR=\frac{c}{(\alpha^2cos^2\theta_h +sin^2\theta_h)^\gamma}$$
	$$\alpha=roughness^2$$
	\item F Function:
	$$F_{Schlick}=F_0 + (1-F_0)(1-cos\theta_d)^5$$
	\item G Function: Using GGX Function\\
	Reference: \textit{Microfacet Models for Refraction through Rough Surfaces }
	$$\alpha_g=(0.5+roughness/2)^2$$
\end{itemize}
\section{Next Step:}
\begin{itemize}
	\item Build the BRDF shading function in unity.
	\item Find a better diffuse,D,F,G Function
	\item Put some new variable into diffuse,D,F,G
\end{itemize}
\newpage
\section{finalEquation}
$$
\begin{aligned}
f_{disneyBRDF}(\vec{L},\vec{N},\vec{V})&=(1-metallic)(\frac{basecolor}{\pi}(f_d*(1-subsurface)+f_{ss}*subsurface)+f_{sh})\\
&+\frac{F_s*G_s*D_s}{4*\vec{N}\cdot\vec{L}*\vec{N}\cdot\vec{V}}+\frac{clearcoat}{4}*\frac{F_c*G_c*D_c}{4*\vec{N}\cdot\vec{L}*\vec{N}\cdot\vec{V}}
\end{aligned}
$$
$$
\begin{aligned}
f_d&=(1+(F_{D90}-1)(1-\vec{N}\cdot\vec{L})^5)(1+(F_{D90}-1)(1-\vec{N}\cdot\vec{V})^5)\\
F_{D90}&=0.5+2*\cos^2(\vec{N}\cdot\vec{H})*roughness
\end{aligned}
$$
$$
\begin{aligned}
f_{ss}&=1.25(((1+(F_{ss90}-1)(1-\vec{N}\cdot\vec{L})^5)(1+(F_{ss90}-1)(1-\vec{N}\cdot\vec{V})^5))(\frac{1}{\vec{N}\cdot\vec{L}+\vec{N}\cdot\vec{V}}-0.5)+0.5)\\
F_{ss90}&=\cos^2(\vec{N}\cdot\vec{H})*roughness
\end{aligned}
$$
$$
\begin{aligned}
f_{sh}&=(white*(1-sheenTint)+\frac{baseColor}{lum(baseColor)}*sheenTint)*sheen*(1-\vec{N}\cdot\vec{H})^5\\
C_{tint}&=\frac{baseColor}{lum(baseColor)}
\end{aligned}
$$
$$
\begin{aligned}
F_{s}&=C_{s}+(1-C_{s})(1-\vec{N}\cdot\vec{H})^5\\
C_{s}&=(1-metallic)*0.08specular((1-specularTint)white+specularTint\frac{baseColor}{lum(baseColor)})+metallic*baseColor
\end{aligned}
$$
$$
\begin{aligned}
G_{s}&=\frac{1}{\vec{N}\cdot\vec{V}+\sqrt{\alpha_{G}^2+(\vec{N}\cdot\vec{V})^2-\alpha_{G}(\vec{N}\cdot\vec{V})}}\\
\alpha_{G}&=\frac{1+roughness}{2}^2
\end{aligned}
$$
$$
\begin{aligned}
D_{s}&=\frac{1}{\pi*roughness^4*((\frac{\vec{H}\cdot\vec{X}}{roughness^2/a}+\frac{\vec{H}\cdot\vec{Y}}{roughness^2*a})^2+(\vec{N}\cdot\vec{H})^2)^2}\\
a&=\sqrt{1-0.9*anisotropic}
\end{aligned}
$$
$$
F_{c}=0.04+0.96*(1-\vec{V}\cdot\vec{H})^5
$$
$$
D_{c}=\frac{(0.1-0.09clearCoatGloss)^2-1}{2\pi\ln(0.1-0.09clearCoatGloss)*((0.1-0.09clearCoatGloss)^2*(\vec{N}\cdot\vec{V})^2+(1-(\vec{N}\cdot\vec{H})^2))}
$$
$$
G_{c}=\frac{1}{\vec{N}\cdot\vec{V}+\sqrt{(0.5+roughness*0.5)^4+({\vec{N}\cdot\vec{V})^4-(0.5+roughness*0.5)^2*({\vec{N}\cdot\vec{V})^2}}}}
$$
\newpage
\subsection{FullyExpent}
$
f_{disneyBRDF}(\vec{L},\vec{N},\vec{V})=(1-metallic)(\frac{basecolor}{\pi}(((1+((0.5+2*\cos^2(\vec{N}\cdot\vec{H})*roughness)-1)(1-\vec{N}\cdot\vec{L})^5)(1+((0.5+2*\cos^2(\vec{N}\cdot\vec{H})*roughness)-1)(1-\vec{N}\cdot\vec{V})^5))*(1-subsurface)+(1.25(((1+(\cos^2(\vec{N}\cdot\vec{H})*roughness-1)(1-\vec{N}\cdot\vec{L})^5)(1+(\cos^2(\vec{N}\cdot\vec{H})*roughness-1)(1-\vec{N}\cdot\vec{V})^5))(\frac{1}{\vec{N}\cdot\vec{L}+\vec{N}\cdot\vec{V}}-0.5)+0.5))*subsurface)+((white*(1-sheenTint)+\frac{baseColor}{lum(baseColor)}*sheenTint)*sheen*(1-\vec{N}\cdot\vec{H})^5))+(1-metallic)*0.08specular((1-specularTint)white+specularTint\frac{baseColor}{lum(baseColor)})+metallic*baseColor+(1-(1-metallic)*0.08specular((1-specularTint)white+specularTint\frac{baseColor}{lum(baseColor)})+metallic*baseColor)(1-\vec{N}\cdot\vec{H})^5*\frac{1}{\vec{N}\cdot\vec{V}+\sqrt{\frac{1+roughness}{2}^4+(\vec{N}\cdot\vec{V})^2-\frac{1+roughness}{2}^2(\vec{N}\cdot\vec{V})}}*(\frac{1}{\pi*roughness^4*((\frac{\vec{H}\cdot\vec{X}}{roughness^2\sqrt{1-0.9*anisotropic}}+\frac{\vec{H}\cdot\vec{Y}}{roughness^2\sqrt{1-0.9*anisotropic}})^2+(\vec{N}\cdot\vec{H})^2)^2})/(4*\vec{N}\cdot\vec{L}*\vec{N}\cdot\vec{V})+\frac{clearcoat}{4}*(0.04+0.96*(1-\vec{V}\cdot\vec{H})^5)/(\vec{N}\cdot\vec{V}+\sqrt{(0.5+roughness*0.5)^4+({\vec{N}\cdot\vec{V})^4-(0.5+roughness*0.5)^2*({\vec{N}\cdot\vec{V})^2}}})*((0.1-0.09clearCoatGloss)^2-1)/(2\pi\ln(0.1-0.09clearCoatGloss)*((0.1-0.09clearCoatGloss)^2*(\vec{N}\cdot\vec{V})^2+(1-(\vec{N}\cdot\vec{H})^2))))/(4*\vec{N}\cdot\vec{L}*\vec{N}\cdot\vec{V})
$
\end{document}