\chapter{February}

\section{Standard deviation} \index{Standard deviation}
\textbf{Standard deviation} $\sigma$ is defined as the square root of \textbf{variance}. That is,
\begin{align}
	\sigma(X) &= \sqrt{var(X)}
\end{align}
The definition for variance:
\begin{align}
var(X) = E[(X - \mu_X)^2]
\end{align}
The definition for \textbf{covariance}:
\begin{align}
cov(X,Y) = E[(X - \mu_X)(Y- \mu_Y)]
\end{align}
A property:
\begin{align}
	cov(X,X) = var(X) = \sigma(X)^2
\end{align}
Definition for \textbf{correlation}:
\begin{align}
	\begin{split}
	\rho(X,Y) & = corr(X, Y) 		\\
			  & = \frac{cov(X,Y)}{\sigma_X \sigma_Y}
	\end{split}
\end{align}

\section{Determinant}\index{Determinant}
Today when I try to compute the determinant of the covariance matrix in the multivariate Gaussian, I come across the problem of overflow. In fact, I only
need to know the logarithm of the determinant. Therefore, I apply the following solution:
\begin{align}
	\begin{split}
	\log(\det A) &= \log(\Pi_{i=1}^{N} \lambda_i)  \\
				 & = \sum_{i=1}^{N} \log(\lambda_i)
	\end{split}
\end{align}

\section{Matplotlib}\index{Matplotlib}
Sample code to plot figures with Python:

\begin{minted}[frame=lines, framesep=2mm,]
	       {python}
x = np.linspace(-10, 4, 500, endpoint=True)
y = (3 * x + 12) / 4

fig = plt.figure()
fig.suptitle('Problem 3', fontsize=20, fontweight='bold')
ax = fig.add_subplot(111)
ax.plot(x, y)

ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data',0))
ax.yaxis.set_ticks_position('left')
ax.spines['left'].set_position(('data',0))

plt.annotate(r'$(-4,0)$',
xy=(-4, 0), xycoords='data',
xytext=(-20, +60), textcoords='offset points', fontsize=16,
arrowprops=dict(arrowstyle="->", connectionstyle="arc3,rad=.2"))

plt.annotate(r'$(0,3)$',
xy=(0, 3), xycoords='data',
xytext=(+10, -30), textcoords='offset points', fontsize=16,
arrowprops=dict(arrowstyle="->", connectionstyle="arc3,rad=.2"))

plt.annotate(r'positive',
xy=(-6, 2), xycoords='data',
xytext=(+10, +30), textcoords='offset points', fontsize=16)

ax.set_xlabel("x")
ax.set_ylabel("y")

plt.savefig('3.png', dpi = 100)
plt.show()
\end{minted}

\section{Different types of machine learning methods:}
\begin{enumerate}
\item \textbf{Parametric methods}: a family of distributions that can be described using a finite
number of paramters, such as \textit{GMM, Naive Bayes, SVM}.
\item \textbf{Non-parametric methods}: methods that are not based on parametrized families of
probability distributions, such as \textit{decision trees, KNN}.
\end{enumerate}

The difference between parametric models and non-parametric models is that the former
has a fixed number of parameters, while the latter grows the number of parameters with
the amount of training data.
\begin{remark}
Note that the non-parametric model does not have no parameters: parameters are determined by the training data, not the model.
\end{remark}

Parametric models can also be categorized into \textbf{generative models} and
\textbf{discriminative models}.
\begin{enumerate}
\item Generative models: Fit a probability distribution like a multivariate Gaussian to each class.
\item Approximate the boundaries between classes by simple functions.
\end{enumerate}

\section{Representation Learning}

\paragraph{Dimensionality Reduction and Denoising}
Given data in high-dimensional Euclidean space, project to a
low-dimensional linear subspace while retaining as much of the
signal as possible.

\paragraph{Embedding and manifold learning}
Given data that lie in a non-Euclidean space, find an embedding into
Euclidean space that preserves as much of the geometry as possible.

\paragraph{Metric learning}
A metric $d$ should satisfy the following properties:
\begin{enumerate}
	\item $d(x, y) \geq 0$, non-negativity
	\item $d(x, y) = 0$ if and only if $x = y$
	\item $d(x, y) = d(y, x)$, symmetry
	\item $d(x, z) \leq d(x, y) + d(y, z)$, triangle inequality
\end{enumerate}

\section{Fast Nearest Neighbor Search}
\textbf{Locality sensitive hashing.}\hspace{0.2cm} Grouping points in space into \textit{buckets} based on some distance metric operating on the points. Points that are close to each other under the chosen metric are mapped to the same bucket with high probability.

\textbf{K-d tress.}\hspace{0.2cm} A space-partitioning data structure for organizing points in a k-dimensional space.

\begin{remark}
	Nearest neighbor is sensitive to noise.
\end{remark}

\section{Decision Trees}
\textbf{Node split.}\hspace{0.2cm} When to split a node, we need to decide which
node to split. We have the following methods to measure the \textbf{uncertainty in prediction} $u(S)$:
\begin{itemize}
	\item \textit{Misclassification rate}: $min(p, 1-p)$, where $p$ is the fraction of positive points.
	\item \textit{Gini index}: $2p(1-p)$
	\item \textit{Entropy}: $-p\log p - (1-p)\log(1-p)$
\end{itemize}
We select the node split that can mostly reduce uncertainty (see Figure~\ref{fig:feb-dt-split}).
\begin{figure}[h]
	\centering{
		\includegraphics[width = 0.9\textwidth]{./images/feb/dt_split.PNG}
	}
	\label{fig:feb-dt-split}
	\caption{Benefit of a split.}
\end{figure}

\textbf{When to stop?}\hspace{0.2cm} Keep splitting until leaves are pure.

\textbf{Pruning.}\hspace{0.2cm} After split stops, we do pruning to avoid over-fitting. Consider all pairs of leave nodes,
any pair whose elimination yields a satisfactory (small)
increase in impurity is eliminated, and the common
antecedent node is declared as leaf node.

\section{Classification with generative models}

\textbf{Bayes-optimal prediction.}\hspace{0.2cm}
\begin{align}
	\begin{split}
	Pr(y= j | x) &= \frac{Pr(y= j) Pr(x|y=j)}{Pr(x)} \\
		&= \frac{\pi_j P_j(x)}{\sum_{i=1}^{k} \pi_i P_i(x)}
	\end{split}
\end{align}

\textbf{Laplace smoothing.}\hspace{0.2cm}
We want to estimate $p_{ji} = Pr(x_i = 1 | y=j)$. Then the maximum likelihood estimate of $p_{ji}$ is $p_{ij} = \frac{n_{ji}}{n_j}$, where $n_{ji}$ is the number of instances of class $j$ with $x_i = 1$, and $n_j$ is the number of
instances of class $j$.
\begin{remark}
This causes the problem if $n_{ji} = 0$. Instead, we use \textit{Laplace smoothing}: $p_{ji} = \frac{n_{ji} + 1}{n_j + 2}$
\end{remark}

\textbf{Multinomial naive Bayes.}\hspace{0.2cm}
Classify document $x$ as $\argmax_j \pi_j \prod_{i=1}^{N} p_{ji} ^ {x_i}$.
To improve the performance, we may adopt the following heuristics:
\begin{itemize}
	\item Compensating for burstiness. Problem: once a word has appeared in a document, it has a much higher chance of appearing again.
	Solution: Instead of the number of occurrences $f$ of a word, use
	$\log(1 + f )$.

	\item Downweighting common words. Problem: common words can have a unduly large influence on classification. Solution: weight each word $w$ by \textit{inverse document frequency}:
	\[
		\log\frac{\# \text{docs}}{\#(\text{docs containing w})}
	\]
\end{itemize}

\section{Gaussian}
\textbf{Multivariate Gaussian.}\hspace{0.2cm} mean: $\mu \in R^p$. covariance $p\times p$
matrix $\sum$:
\begin{align}
\begin{split}
	p(x) &= \frac{1}{(2\pi)^{p/2} |\sum|^{1/2}} \exp(-\frac{1}{2} (x-\mu)^T \sum {}^{-1} (x-\mu))
\end{split}
\end{align}

\textbf{Spherical Gaussian.}\hspace{0.2cm} $\sum = diag(\sigma^2, ..., \sigma^2)$

Here we call it \textit{spherical Gaussian} because the contour of points with equal density is a sphere:
\begin{align}
\begin{split}
||x - \mu||^2 &= r^2
\end{split}
\end{align}

\begin{figure}[h]
\centering{
	\includegraphics{./images/feb/sg.PNG}
}
\caption{The contour means that points on it have the same density. For spherical Gaussians, density at a point only depends on its distance from $\mu$.}
\end{figure}

\textbf{Diagonal Gaussian.}\hspace{0.2cm} $\sum = diag(\sigma_1^2, ..., \sigma_p^2)$

\begin{figure}[h]
	\centering{
		\includegraphics[width=0.9\textwidth]{./images/feb/dg.PNG}
	}
	\caption{Diagonal Gaussian: the contours are axis-aligned ellipsoids.}
\end{figure}

\myheader{General Gaussian .}

\begin{figure}[h]
	\centering{
		\includegraphics[width=0.9\textwidth]{./images/feb/gg.PNG}
	}
	\caption{General Gaussian: the contours are ellipsoids, with axis being the eigen-vectors
		of covariance matrix $\sum$.}
\end{figure}

\myheader{Binary classifications with Gaussians}
Suppose we have two Gaussians:
\begin{align}
\begin{split}
	P_1 &= N(\mu_1, \sum{}_1) \\
	P_2 &= N(\mu_2, \sum{}_2)
\end{split}
\end{align}

The classification boundary is:
\begin{align}
\begin{split}
	x^T M x + 2w^Tx & \geq \theta	\\
	\text{where } M & = \frac{1}{2}(\sum{}_2^{-1} - \sum{}_{1}^{-1}) \\
				w & = \sum{}_1^{-1} \mu_1 - \sum{}_{2}^{-1}\mu_2
\end{split}
\end{align}
\begin{remark}
	From this we know that, when $\sum_1 = \sum_2$, it has a linear decision boundary. Otherwise, it has quadratic boundary.
\end{remark}

\begin{remark}
	When calculate the inverse of covariance matrix $\sum$, to avoid the problem that $\sum$
	is singular, we need to regularize it: $\sum = \sum + \sigma^2 I$.
\end{remark}

\section{New commands in LaTex}
\begin{minted}[frame=lines, framesep=2mm,]
{cpp}
\newcommand{\plusbinomial}[3][2]{(#2 + #3)^#1}

To save some time when writing too many expressions
with exponents is by defining a new command to make simpler:

\[ \plusbinomial{x}{y} \] -> $(x+y)^2$

And even the exponent can be changed

\[ \plusbinomial[4]{y}{y} \] -> $(y+y)^4$

\plusbinomial
This is the name of the new command.
[3]
The number of parameters the command will take, in this case 3.

[2]
Is the default value for the first parameter. This is what makes the first
 parameter optional, if not passed it will use this default value.

(#2 + #3)^#1
This is what the command does. In this case it will put the second and third
parameters in a "binomial format" to the power represented by the first
parameter.
\end{minted}

\section{OpenGL}

\subsection{Window Management in OpenGL .}
The OpenGL doesn't specify any API in order to create and manipulate windows. Modern windowing systems that support OpenGL include a sub-system that provides the binding between an OpenGL context and the windowing system. The library we use here is called \textit{OpenGL utility library}, or \textit{GLUT}. It provides a simplified API for window mangement as well
as envent handling, IO control and few other services.

\myheader{State in OpenGL .}
Rendering is such a complex tas that it cannot be treated as a function call that receives a few paramters,
and correctly designed functions never receive a lot of parameters. You need to specify a lot of flags that affect
how rendering will take place. In addition, you would often want to keep the same piece of configuration across
several rendering operations.  \textbf{That is why most of the configuration of rendering operations is done by setting
flags and values in the OpenGL state machine. After calling a state changing function that particular configuration
remains intact until the next call to the same function with a different value}.

\myheader{glutMainLoop .}
This call passes control to GLUT which now begins its own internal loop. In this loop it listens to events from
the windowing system and passes them via callbacks that we configured.

\subsection{Hello dot}
\myheader{GLEW .} GLEW is short for \textbf{OpenGL Extension Wrangler Library}. GLEW helps you deal with
headache that can accompany the management of extensions in OpenGL.

\myheader{Vertex buffer objects .} VBOs are used to store vertices. They are the most efficient
way to load vertices into the GPU. The are buffers that can be stored in video memory and provide
the shortest access time to the GPU so they are definitely recommended.

\myheader{Rasterizer .} Rasterizer maps coordinates to screen space. And finally it draws the
primitives according to the topology which is specified in the draw call.

\begin{minted}[frame=lines, framesep=2mm,]
{cpp}
Vector3f Vertices[1];
Vertices[0] = Vector3f(0.0f, 0.0f, 0.0f);
GLuint VBO;
glGenBuffers(1, & VBO);
glBindBuffer(GL_ARRAY_BUFFER, VBO);
glBufferData(GL_ARRAY_BUFFER, sizeof(Vertices), Vertices, GL_STATIC_DRAW);
glEnableVertexAttribArray(0);
\end{minted}

\myheader{glGen* .}
OpenGL defines serveral \textit{glGen*} functions for generating objects of various
types. They take two parameters:
\begin{itemize}
	\item First parameter: the number of objects you want to create
	\item Second parameter: the address of an array of GLuints to store the handles that the driver
	allocates for you (make sure the array is large enough to handle your request). Future calls to this function will not generate the same object handles unless you delete them first with glDeleteBuffers.
\end{itemize}

\myheader{Handles .}
In computer programming, a handle is an abstract reference to a resource. Handles are usually
used when application software references blocks of memory or objects managed by another
system, such as database or an operating system.

\section{Fisher's Linear Discriminant}
\myheader{What is Fisher LDA ?} Fisher LAD is a framework for linear classification without Gaussian assumptions. It is a linear classifier projects all data onto a direction $w$.

\begin{figure}[h]
	\centering{
		\includegraphics[width=0.45\textwidth]{./images/feb/LDA_1.PNG}
		\includegraphics[width=0.45\textwidth]{./images/feb/LDA_2.PNG}
	}
	\caption{Fisher LDA.}
\end{figure}

\section{Logistic Regression}
\myheader{Why Logistic Regression ?} See Figure~\ref{fig:logistic}.
\begin{figure}[t]
	\centering{
		\includegraphics[width=0.45\textwidth]{./images/feb/logistic.PNG}
		\includegraphics[width=0.45\textwidth]{./images/feb/logistic_1.PNG}
	}
	\caption{Logistic regression.}
	\label{fig:logistic}
\end{figure}

\myheader{How to optimize $L(w)$ ?}
\begin{itemize}
\item \myheader{Gradient descent .} Gradient descent is a first-order optimization algorithm. To find a local minimum of a function using gradient descent, one takes steps proportional to the \textbf{negative} of the gradient (or of the approximate gradient) of the function at the current point. If instead one takes steps proportional to the positive of the gradient, one approaches a local maximum of that function; the procedure is then known as gradient ascent.

\item \myheader{Newton's method .} Newton's method is a method for finding successively better approximations to the roots (or zeroes) of a real-valued function: $f(x) = 0$. If the function satisfies the assumptions made in the derivation of the formula and the initial guess is close, then a better approximation $x_1$ is: $x_1 = x_0 - \frac{f(x_0)}{f'(x_0)}$.

To apply Newton's method for minimization and maximization problem, considering the fact that the derivative is zero at a minimum or maximum, so minima and maxima can be found by applying Newton's method to the derivative. The iteration becomes:
\[
x_{n+1} = x_n - \frac{f'(x_n)}{f''(x_n)}
\]
\end{itemize}

\myheader{How to determine step size ?} A method like Newton's method chooses a step, but the validity of that step only goes as far as the Newton quadratic model for the function really reflects the function. The idea of a line search is to use the direction of the chosen step, but to control the length, by solving a one-dimensional problem of minimizing
\[
\phi(\alpha) = f(\alpha p_k + x_k)
\]
where $p_k$ is is the search direction chosen from the position $x_k$.

\myheader{Backtracking line search .}
First, fix a parameter $0 < \beta < 1, 0 < \alpha < 0.5$, then at each iteration, start with $t = 1$, and while
\myequ{
	f(x - \nabla f(x)) &> f(x) -  \alpha t||\nabla f(x)||^2
}
update $t = \beta t$.

\section{Matrix Calculus}
\myheader{First order derivatives}
\myequ{
	\frac{\partial AX}{\partial X} & = A^T \\
	\frac{\partial X^T a}{\partial X} &= a \\
	\frac{\partial a^T X b}{\partial X} &= ab^T \\
	\frac{\partial a^T X^T b}{\partial X} &= ba^T
}

\myheader{Jacobian matrix and Hessian matrix}
Jacobian matrix is the matrix of all \textbf{first-order partial derivatives} of a vector-valued function.
Hessian matrix is a square matrix of \textbf{second-order partial derivatives} of a scalar-valued function, or scalar field.

\section{Feature scaling}
Feature scaling is a method used to standardize the range of independent variables or features of data. In data processing, it is also known as data normalization and is generally performed during the data preprocessing step.

\myequ{
	x' &= \frac{x - \min(x)}{\max(x) - \min(x)}
}

Since the range of values of raw data varies widely, in some machine learning algorithms, objective functions will not work properly without normalization. For example, the majority of classifiers calculate the distance between two points by the Euclidean distance. If one of the features has a broad range of values, the distance will be governed by this particular feature. Therefore, the range of all features should be normalized so that each feature contributes approximately proportionately to the final distance.

Another reason why feature scaling is applied is that gradient descent converges much faster with feature scaling than without it.

\section{Likelihood function}
A likelihood function is a function of the parameters of a statistical model. The likelihood
of a set of parameters values $\theta$, given outcomes $x$, is equal to the probability of those
outcomes given those parameter values, that is,
\myequ{
	\ell (\theta | x) &= P(x  | \theta)
}

For many applications, the natural logarithm of the likelihood function, called
\textbf{log-likelihood} is more convenient to work with.

\section{Keyword Static in Programming}
The keyword \verb|static| has multiple uses in programming:
\begin{itemize}
	\item Use \verb|static| inside a function: it means that once the variable
	has been initialized, it remains in memory until \textit{the end of the program}.
\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
#include <stdio.h>
void func() {
  	static int x = 0;
  	x++;
  	printf("%d\n", x);
}

int main() {
	func(); // prints 1
	func(); // prints 2
	func(); // prints 3
	func(); // prints 4
	func(); // prints 5
	return 0;
}
\end{minted}

	\item Use \verb|static| inside a loop to prevent it from being reinitialized
	inside a loop
\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
for(int x=0; x<10; x++) {
	for(int y=0; y<10; y++) {
		static int number_of_times = 0;
		number_of_times++;
	}
}
\end{minted}

	\item Use \verb|static| inside a class definition. A static member
    variable inside a class have the same value in any instance in the class
    and doesn't even require an instance of the class to exist.
    \begin{remark}
        When debugging or implementing a program using a static class member is that you cannot initialize the static class member inside of the class. In fact, if you decide to put your code in a header file, you cannot even initialize the static variable inside of the header file; do it in a .cpp file instead. Moreover, you are required to initialize the static class member or it will not be in scope. (The syntax is a bit weird: \verb|type class_name::static_variable = value"|).
    \end{remark}

   \item \verb|static member function|. Static member functions are functions that do not require an instance of the class, and are called the same way you access static member variables -- with the class name rather than a variable name. Static member functions can only operate on static members, as they do not belong to specific instances of a class.
\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
class user{
private:
    static int next_id;
public:
    static int next_user_id(){
        next_id++;
        return next_id;
    }
};
int user::next_id = 0; // initialize static variable
\end{minted}

    \item \verb|static function|. Static functions are functions that are only visible to other functions in the same file.L

    \item Use \verb|static| to define a global variable inside a file of code.
    \begin{remark}
        In this case, the use of static indicates that source code in other files that are part of the project cannot access the variable. Only code inside the single file can see the variable.
    \end{remark}
\end{itemize}


\section{Probability}

\subsection{Exercise 1}
If $x \in B_i$, then $x \in A_i$ and $x \notin A_j \forall j < i$, so $x \notin B_j \forall j < i$.
Thus, all of the $B_i$ are disjoint. Furthermore, if $x \in A_n$, then
$x \in A_i$ for some $i \leq n$, so $x \in \cup_{i = 1}^n B_i$. The reverse direction
is also true by definition of $B_i$, so $\cup_{i = 1}^n A_i = \cup_{i = 1}^n B_i$.

To prove the monotonically decreasing case, we first note that
$A_n \setminus A = \cup_{i = n}^\infty A_i \setminus A_{i + 1}$. This can be
seen from the fact that $x \in A_n \setminus A \implies x \notin \cap_{i = n}^\infty A_i$,
so there exists $i$ such that $x \in A_i$ and $x \notin A_{i + 1}$, which gives
$x \in A_i \setminus A_{i + 1}$. The reverse direction follows from the
definition of monotonically decreasing, so we have equality.

Since all of the $A_i \setminus A_{i + 1}$ are disjoint, we have
\begin{align*}
        \lim_{n \to \infty} P(A_n) &= P(A) + \lim_{n \to \infty} P(A_n \setminus A) \\
                                   &= P(A) + \lim_{n \to \infty} \sum_{i = n}^\infty P(A_i \setminus A_{i + 1}) \\
                                   &= P(A)
\end{align*}
as desired.

\subsection{Exercise 5}
The sample space consists of all strings that end in an H and otherwise contain only Ts
except for a single other H (i.e. ``TTTTHTTTH''). If we stop at $k$ tosses, then toss $k$
must have been an H and there must also have been exactly one H in the first $k - 1$ tosses.
Since we have $k - 1$ options for the other H, we have that the probability is
$\frac{k - 1}{2^k}$.

\subsection{Exercise 6}
Consider the partition of $\Omega$ into singletons. Assigning any non-zero probability to
these singletons would cause the probability of $\Omega$ (which is the sum of the probabilities
of the singletons) to diverge.

\subsection{Exercise 8}
Applying the results from exercises 4 and 7, we have
\begin{align*}
        P\big(\cap_{i = 1}^\infty A_i\big) &= P\bigg(\big(\cup_{i = 1}^\infty A_i^c\big)^c\bigg) \\
                                             &= 1 - P\big(\cup_{i = 1}^\infty A_i^c\big) \\
                                             &\geq 1 - \sum_{i = 1}^\infty P(A_i^c) \\
                                             &= 1
\end{align*}
Where the last line follows from the fact that $P(A_i) = 1$ was given.

\subsection{Exercise 10}
We define the sample space $\Omega$ to consist of the doubles $(A, B)$ where
$A, B \in \{1, 2, 3\}$ and $A, B$ correspond to the door containing the prize
and the door shown by Monty respectively. We can then define the event that
door $i$ contains the prize as $X_i = \{(A, B) \: | \: A = i\}$. Similarly,
we can define the event that door $i$ is shown as $Y_i = \{(A, B) \: | \: B = i\}$.

Now we can compute the probability $P(X_2 Y_3)$ using Bayes theorem as
$P(X_2 Y_3) = P(Y_3 | X_2) P(X_2) = 1 * \frac{1}{3}$. The calculation for
$P(X_3 Y_2)$ is identical. Furthermore, $P(X_1 Y_2) = P(X_1 Y_3) = \frac{1}{6}$,
so our likelihood of winning is always increased by switching.

\subsection{Exercise 11}
\begin{align*}
        P(A^c) P(B^c) &= (1 - P(A)) (1 - P(B)) \\
                      &= 1 - P(A) - P(B) + P(A)P(B) \\
                      &= 1 - P(A) - P(B) + P(AB) \\
                      &= 1 - P(A \cup B) \\
                      &= P(A^c B^c)
\end{align*}

\subsection{Exercise 12}
Let the sample space $\Omega$ consist of the doubles $(C, S)$ where $C \in \{1, 2, 3\}$
(card number) and $S \in \{1, 2\}$ (card side). Let $A$ denote the event that we see
a green side, and let $B$ denote the event that we see the double green card. Then
$P(B | A) = \frac{P(AB)}{P(A)} = \frac{1}{3} / \frac{1}{2} = \frac{2}{3}$.

\subsection{Exercise 14}
If $P(A) = 0$, then we have that $P(AB) = 0 = P(A)P(B)$ for all $B$ since $AB \subset A$.
If $P(A) = 1$, then we have $P(A \cup B) = P(A) + P(B) - P(A \cap B) = 1 + P(B) - P(A \cap B)$
which implies $P(A \cap B) = P(B) = P(A)P(B)$.
If $A$ is independent of itself, then $P(A)^2 = P(A)$, which is only possible if
$P(A) = 1$ or $P(A) = 0$.

\subsection{Exercise 17}
$P(ABC) = \frac{P(ABC)}{P(BC)} \frac{P(BC)}{P(C)} P(C) = P(A | BC) P(B | C) P(C)$.

\subsection{Exercise 18}
We have that $\sum_{i} P(B | A_i) P(A_i) = P(B)$. If $P(A_1 | B) < P(A_1)$, then
applying Bayes' Theorem gives $P(B | A_1) < P(B)$. Now assume $P(B | A_k) \leq P(B)$
for all $k > 1$. Then $\sum_{i} P(B | A_i) P(A_i) < \sum_{i} P(B) P(A_i) = P(B)$,
which is a contradiction.


% !TeX root = ../solution.tex

\hypertarget{he22.21}{%
\chapter{[HE22.21] Textbook}\label{he22.21}}

\begin{marginfigure}
	\includegraphics[width=49mm]{level5/challenge21.jpg}
\end{marginfigure}
\subsection{Intro}
I've got the source code and the output of a simple cipher.

\noindent Can you calculate the flag from it?

\noindent File \verb+textbook.zip+.

\subsection{Hint}

As usual, the flag starts with \verb+he2022+.

\section{Solution}\label{hv22.21solution}
The code generator is very simple, each character in the flag is coded as
\verb+pow(ord(c), e, n)+ -- take the ASCII value of the character to the power
$e$ and then store the modulo $n$.  Since we have a crib (``he2022´´) and know
the exponent, we can also calculate the real value of \verb+pow(ord(c),e)+.
Subtract the coded value, we get a number that is known to be a multiple of
$n$, so taking two such numbers, we can calculate $n$ as the GCD of the two
numbers.  Once $n$ is known, we can calculate a rainbow table for all printable
characters and look-up the flag from the encrypted flag:

\begin{minted}{python}
import math

def multi_n(c, cr):
    tmp = pow(ord(c), e)
    return (tmp - cr)

with open('output','r') as inF:
    s = inF.read()[1:-2]
    crypt = [int(x) for x in s.split(', ')]

n = math.gcd(multi_n(crib[0], crypt[0]),
            multi_n(crib[1], crypt[1]))

rainbow = {}
for i in range(32,128):
    rainbow[pow(i,e,n)] = chr(i)

res = ''
for i in crypt:
    res += rainbow[i]

print(res)
\end{minted}
Flag \verb+he2022{!!t3xtb00k_crypt0!!}+.


\chapter{Automorphisms}
\section {Specific Groups}
{\bf Theorem 1:}
If $G= \langle a \rangle $ and $|a|=n$ then $Aut(G)= \langle \rho \rangle $, $|\rho|= \phi(n)$.
$Aut(G) \cong {{\mathbb Z}_n}^*$.
\\
\\
{\bf Theorem 2:}
If $G= E(p^n)$ then $Aut(G)= L_n(p)$.
\\
\\
{\bf Theorem 3:}
If $G= S_n$ then $Aut(G)= S_n$ if $n \ne 6$ and a covering group with factor $2$ of $S_6$ for
$S_6$ (an outer automorphism).
\\
\\
{\bf Theorem 4:}
If $G$ is simple, $1 \rightarrow G \rightarrow Aut(G)$.
\\
\\
{\bf Definition:} $Q_8 =
\langle
\left(\begin{array} {cc}
i & 0\\
0 & i\\
\end{array}\right),
\left(\begin{array} {cc}
0 & -1\\
1 & 0\\
\end{array}\right),
\left(\begin{array} {cc}
0 & 1\\
-1 & 0\\
\end{array}\right)
\rangle$. $Q_8 \in S_2(SL_2(5))$.
\\
\\
{\bf Theorem 5:}  The automorphism group of $Q_8$ is $S_4$.
\\
\\
Automorphisms of Graphs.


\subsection{Riemann manifolds and pseudo-Riemann manifolds}

\subsubsection{Riemann manifolds}

Metric is positive definite.

\subsubsection{Pseudo-Riemann manifolds}

The metric isn't necessarily positive definite.

\subsection{Metric tensor field}

metric tensor field assigns a metric tensor to each point. metric tensor is defined on the tangent bundle.
so we have metric on each tangent bundle, but the metric can change thoughout the manifold


\section{Sequences and Series of Functions}

\subsection{Exercise 1}
We have that $\abs{f_n(x)} \leq M_n$. From uniform convergence, we get that there exists $N$ such that
\begin{align*}
        n, m \geq N \implies \abs{f_n(x) - f_m(x)} < \epsilon \implies \abs{M_n - M_m} < \epsilon
\end{align*}
So $\lim_{n \to \infty} M_n = M$ for some $M$. Thus, we can choose $N$ such that $n \geq N \implies M_n < M + 1$
, so we can set $M_u = \max\{M_1, M_2, ..., M_{N-1}, M + 1\}$. By construction, $M_u$ must be a uniform
bound for all of the $f_n$.

\subsection{Exercise 2}
Since $f_n, g_n$ are both uniformly convergent, we can choose $N_1, N_2$ such that
\begin{align*}
        n, m \geq \max\{N_1, N_2\} \implies \abs{f_n(x) - f_m(x)} < \frac{\epsilon}{2}, \: \abs{g_n(x) - g_m(x)} < \frac{\epsilon}{2} \\
        \abs{f_n(x) + g_n(x) - f_m(x) - g_m(x)} \leq \abs{f_n(x) - f_m(x)} + \abs{g_n(x) - g_m(x)} < \epsilon
\end{align*}
So $f_n + g_n$ converges uniformly as well.

Suppose $f_n \to f$ and $g_n \to g$, with $f_n(x) \leq M_f$ and $g_n(x) \leq M_g$. Then $g(x) f_n(x)$
converges uniformly since $M_g f_n(x)$ converges uniformly, and likewise for $g_n(x) f(x)$. Thus,
\begin{align*}
        \frac{\epsilon}{3} &> \abs{(f_n(x) - f(x))(g_n(x) - g(x))} \\
                 &> \abs{f_n(x)g_n(x) - f_n(x)g(x) - f(x)g_n(x) + f(x)g(x)} \\
                 &> \abs{f_n(x)g_n(x) - f(x)g(x)} - \abs{f(x)g(x) - f_n(x)g(x)} - \abs{f(x)g(x) - f(x)g_n(x)} \\
                 &> \abs{f_n(x)g_n(x) - f(x)g(x)} - \frac{\epsilon}{3} - \frac{\epsilon}{3} \\
        \implies \epsilon &> \abs{f_n(x)g_n(x) - f(x)g(x)}
\end{align*}
So $f_n g_n \to fg$ uniformly.

\subsection{Exercise 3}
Let $f_n(x) = \frac{1}{x} + \frac{1}{n}$ and $g_n(x) = \frac{1}{x}$ on $E = (0, 1)$.
Both $f_n$ and $g_n$ converge uniformly, and $f_n(x) g_n(x)$ converges pointwise to $\frac{1}{x^2}$. However,
\begin{align*}
        \abs{f_n(x) g_n(x) - \frac{1}{x^2}} = \abs{\frac{1}{xn}}
\end{align*}
So the convergence of $f_n(x) g_n(x)$ is not uniform on  $E$.

\subsection{Exercise 4}
Since $1 + n^2 x = 0$ whenever $x = -\frac{1}{n^2}$, $f(x)$ is not defined/does not converge for
$x = 0, -\frac{1}{n^2}$. For all other values of  $x$, however, $f(x)$ converges absolutely since
$\sum_{n = 1}^\infty \frac{1}{n^2}$ converges. $f(x)$ converges uniformly on any intervals of the form
$(a, -1)$ and $(b, c)$ with $b > 0$, since such intervals do not contain points of the form $-\frac{1}{n^2}$
and $\frac{1}{x}$ is bounded on all such intervals. $f$ fails to converge uniformly on all other intervals.
$f(x)$ is continuous on the intervals that it converges uniformly on.
$f$ is not bounded since $\frac{1}{x}$ can be made arbitrarily large.

\subsection{Exercise 5}
For every positive real number $x$, there exists $N$ such that $\frac{1}{N} < x$ (Archimedean property). Hence,
choosing $n \geq N \implies f_n(x) = 0$. Since every non-positive real number is less than $\frac{1}{n + 1}$
for all $n \in \mathbb{N}$, we have that $f_n$ converges pointwise to the function $f(x) = 0$.
As such, it is clear that $\sum_{n = 1}^\infty \abs{f_n(x)} = \sum_{n = 1}^N \abs{f_n(x)}$ converges for all $x$
, and yet we do not have uniform convergence.

\subsection{Exercise 6}
We can rewrite the series as
\begin{align*}
        \sum_{n = 1}^\infty (-1)^n \frac{x^2 + n}{n^2} = \sum_{n = 1}^\infty (-1)^n \frac{1}{n} + \sum_{n = 1}^\infty (-1)^n \frac{x^2}{n^2}
\end{align*}
so the series converges uniformly since the alternating harmonic series converges and the series
$\sum_{n = 1}^\infty \frac{M}{n^2}$ converges (where $x^2 \leq M$, which is possible on every bounded interval).
However, this series also clearly does not converge absolutely for any value of $x$ since the harmonic
series does not converge.

\subsection{Exercise 7}
We posit that $f_n$ converges uniformly to the function $f(x) = 0$. For this to be true, there needs to be a
single $N$ such that $n \geq N$ gives
\begin{align*}
        \abs{\frac{x}{1 + nx^2}} < \epsilon \implies \frac{\abs{x} - \epsilon}{\epsilon x^2} < n
\end{align*}
for all $x$. In other words, the lefthand side of the last expression above must have a maximum. Since the
lefthand side is differentiable for all $x \neq 0$, we can differentiate and equate to 0 to find that
\begin{align*}
        \dv{x} \frac{x - \epsilon}{\epsilon x^2} = \frac{2\epsilon^2 x - \epsilon x^2}{(\epsilon x^2)^2} \\
        2\epsilon^2 x - \epsilon x^2 = 0 \implies x = 2 \epsilon
\end{align*}
giving that $n > \frac{1}{4\epsilon^2}$ is sufficient for all $x$. Additionally, since $f_n(0) = 0$ for all
$n$, we have that $f_n \to f$ uniformly. Differentiating $f_n$ also shows that $f_n'(x) \to 0$ for all
$x \neq 0$ and that $f_n'(0) = 1 \neq f'(0)$.

\subsection{Exercise 8}
Since $\sum \abs{c_n}$ converges, there exists $N$ such that $n, m \geq N$ gives
\begin{align*}
        \epsilon > \sum_{i = n}^m \abs{c_i} \geq \sum_{i = n}^m \abs{c_i I(x - x_i)} \geq \abs{\sum_{i = n}^m c_i I(x - x_i)}
\end{align*}
so $f(x)$ converges uniformly as $x$ was arbitrary ($0 \leq I(x - x_i) \leq 1$ for all $x$). If $x \neq x_n$
and $x$ is not a limit point of $x_n$, then $f(x)$ is clearly continuous since there is a neighborhood
$B_{\delta}(x)$ that contains none of the $x_n$ which implies that $f(x)$ is constant in this neighborhood.
If $x$ is a limit point of the sequence $x_n$, then for any $\epsilon > 0$ we can choose $N$ such that
$n \geq N \implies \abs{c_n} < \epsilon$, so $f $ is continuous at $x$.

\subsection{Exercise 9}
Since $f_n$ is a sequence of continuous functions that converges uniformly to $f$, $f$ is continuous. By
uniform convergence of $f_n$, we have that there exists
$N_1 \: | \: n  \geq N_1 \implies \abs{f_n(x) - f(x)} < \frac{\epsilon}{2}$. By continuity of $f$, we have that there
exists $N_2 \: | \: n \geq N_2 \implies \abs{f(x_n) - f(x)} < \frac{\epsilon}{2}$.
Choosing $n \geq \max(N_1, N_2)$ then gives
\begin{align*}
        \abs{f_n(x_n) - f(x)} &= \abs{f_n(x_n) - f(x_n) + f(x_n) - f(x)} \\
                              &\leq \abs{f_n(x_n) - f(x_n)} + \abs{f(x_n) - f(x)} \\
                              &< \epsilon
\end{align*}
so $\lim_{n \to \infty} f_n(x_n) = f(x)$. The converse need not be true. Consider
\begin{align*}
        f_n(x) =
        \begin{cases}
                \frac{1}{n} & x = \frac{1}{n} \\
                0 & x \neq \frac{1}{n}
        \end{cases} \\
\end{align*}
which satisfies $\lim_{n \to \infty} f_n(x_n) = 0$ for all sequences $x_n$ despite none of the $f_n$ being
continuous.

\subsection{Exercise 10}
$f(x)$ converges uniformly since $0 \leq (nx) < 1$. Since $(nx)$ is discontinuous only when $nx$ is an integer,
$f(x)$ is discontinuous only at rational values of $x$ (because we can choose $n = b$ so that
$n\frac{a}{b} = a$). The continuity of $f(x)$ at irrational values of $x$ comes from the fact that the partial
sums are continuous at all such $x$ and converge uniformly to $f(x)$. Since
$f_n(x) = \sum_{i = 1}^n \frac{(ix)}{i^2}$ contains only finitely many discontinuities in any bounded interval
(there are only finitely many rational $x = \frac{z}{i}$ such that $z \in [a, b]$), we have that
$f_n \in \mathscr{R}$. Thus, applying Theorem 7.16, we get that $f \in \mathscr{R}$.


\chapter{CNN test for MNIST}

	\section{Benchmark Dataset}
	\subsection{MNIST}
	The MNIST database of handwritten digits, has a training set of 60,000 examples, and a test set of 10,000 examples. It is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. Each example is a $28\times 28$ grey-scale image with a label that show witch number the image is.
%	\subsection{to be continued}

	\section{The math representation of data and function}
	Let $\mathcal{M}$ is the number of examples in training data, $\mathcal{C}$ is the number of the variety of label.

	\subsection{Label} Each image has a label in $\{0,1,\cdots,9\}$ to show what's the number it is. We transform the label $n$ to a 10-dimension vector which the $n$-th element is $1$ and others are $0$, i.e.
	$$
	n \rightarrow \mathbf y
	$$
	where $y_i = 1 $ iff $i=n$.
	Then the label data can be present as a matrix $Y = (\mathbf y_1,\cdot,\mathbf{y}_{\mathcal M})$.
	\subsection{Grey-scale Image}
	In this note, we only focus on the CNN of 2D image. A 2D Grey-scal image can regard as a matrix $X =(x_{m,n})_{M\times N}$ where $x_{m,n}$ is the grey-level of pixel $(m,n)$.
	\subsection{Image with multiple channels}
	If we put more than one value on each pixel, the image can be think as a 3-rank tensor $\mathbf X = (x_{m,n,l})_{M\times N\times L}$ which $x_{m,n,l}$ is the $l$-th value on pixel $(m,n)$. For example, a colorful image usually has three channels: red, blue and green. More over, we can consider that a channel is a feature of the image, so we usually create more channels in the convolution layer so that we can get more features.
%	\subsection{Data between two layer}
%	The transport data $\mathbf X$ between two layer can be describe as a 3-rank tensor, i.e. $\mathbf X = (x_{m,n,l})_{M\times N\times L}$. We can regard it as a 2D image with L real number in each pixel.
	\subsection{Convolution Kernel}
	A 2D-convolution kernel for a 2D image can be represent as a matrix, i.e. $K = (k_{i,j})_{I\times J}, i= -I',-I'+1,...,I', j = -J',-J'+1,...,J'$.(Suppose $I$ and $J$ is odd integer and $I' = \dfrac{I-1}2, J' = \dfrac{J-1}2 $.) which $K_{i,j}$ is the parameter on pixel $(i,j)$.

	The effect of a $I\times J$ kernel $K$ acting on a $M\times N$  image $X$ is a 2D image
	$$
	Y =	 K ( X) = (y_{m,n})_{M\times N}
	$$
	where
	\begin{equation}
	y_{m,n} =\sum_{\alpha = - I'}^{I'} \sum_{\beta = - J'}^{J'}k_{\alpha,\beta} x_{m+\alpha,n+\beta}
	\end{equation}

	A 2D-convolution kernel for a 2D image with $L$ channels can be represent as a 3-rank tensor, i.e. $\mathbf{K} = (k_{i,j,l})_{I\times J\times L}, i= -I',-I'+1,...,I', j = -J',-J'+1,...,J'$.(Suppose $I$ and $J$ is odd integer and $I' = \dfrac{I-1}2, J' = \dfrac{J-1}2 $.) which $K_{i,j,l}$ is the parameter of $l$-th channel on pixel $(i,j)$.

	 The effect of a $I\times J\times L$ kernel $\mathbf K $ acting on a $M\times N \times L$ multiple channel image $\mathbf X$ is also a 2D image
	$$
		Y =	\mathbf K (\mathbf X) = (y_{m,n})_{M\times N}
	$$
	where
	\begin{equation}
	y_{m,n} =\sum_{l=1}^L\sum_{\alpha = - I'}^{I'} \sum_{\beta = - J'}^{J'}k_{\alpha,\beta,l} x_{m+\alpha,n+\beta,l}
	\end{equation}

	 A convolution layer usually has more than one kernels, if we write them together, we can get a 4-rank tensor $\mathcal{K} = (k_{i,j,l,p})_{I\times J\times L\times P}$ where $\mathbf K_p = (k_{i,j,l,p})_{I\times J\times L}$ is the $p$-th kernel.
	Then we can easily get the effect of a $I\times J\times L\times P$ kernel $\mathcal K $ acting on a $M\times N \times L$ multiple channel image $\mathbf X$ is a 2D image with $P$ channels
	$$
	\mathbf Y =	\mathcal K (\mathbf X) = (y_{m,n,p})_{M\times N\times P}
	$$
	where
	\begin{equation}
	y_{m,n,p} =\sum_{l=1}^L\sum_{\alpha = - I'}^{I'} \sum_{\beta = - J'}^{J'}k_{\alpha,\beta,l,p} x_{m+\alpha,n+\beta,l}
	\end{equation}
	\subsection{Pooling function}
	Suppose $X$ is a $S \times T$ matrix, we have pooling function:
	\begin{itemize}
		\item max-pooling:
		$$
			P^{max}_{S\times T}(X) = \max x_{s,t}
		$$
		\item mean-pooling:
		$$
		P^{mean}_{S\times T}(X) = \dfrac 1{ST} \sum_{s,t}x_{s,t}
		$$
	\end{itemize}
	Now let X is a $M \times N$ matrix and $M=S\times A, N=T\times B$, we write $X$ in block matrix
	\begin{equation}
	X=\left[
	\begin{matrix}
	X_{1,1}&\cdots	&X_{1,B}\\
	\vdots &	  	&\vdots\\
	X_{A,1}&\cdots	&X_{A,B}
	\end{matrix}
	\right]
	\end{equation}. Each block $X_{a,b}$ is a $S\times T$ matrix.

	After we selected a pooling function $P_{S\times T}$, then
	$$
		Y = P_{S\times T}(X) =(y_{a,b})_{A\times B}
	$$
	where
	\begin{equation}
	 y_{a,b} = P_{S\times T}(X_{a,b})
	\end{equation}

	\subsection{Loss function}\label{Loss}
	\paragraph{Cross-entropy} Suppose $\mathbf y$ is the training data transformed by a example's label and $\hat {\mathbf {y}}$ is calculate by our model, the cross-entropy loss function is:
	\begin{equation}
	L(\mathbf y,\hat {\mathbf {y}}) =- \sum y_n \ln \hat{y}_n
	\end{equation}

	So the loss function of training data is
	\begin{equation}
	\mathcal L(Y,\hat Y) = \sum L(\mathbf y_i,\hat {\mathbf {y}}_i)
	\end{equation}

 	\section{Introduction of several kinds of Layer}
	\subsection{Densely Connect Layer}\label{Dense}
	This kind of layer is same with DNN. The input is a vector $\mathbf x$ and the parameter is a matrix $W$ and vector $\mathbf b$ called bias. After we choose a active function $\sigma$, the output is
	\begin{equation}
	\mathbf u = \sigma(W \mathbf x + \mathbf b)
	\end{equation}
	\subsection{Convolution Layer}\label{Convolution}
	Suppose input data is a $M \times N \times L $ multi-channel image $\mathbf X$. Parameter of this layer is a $I\times J\times L\times P$ tensor $\mathcal K$ and a P-dimension vector $\mathbf b = (b_1,...,b_p)$ called bias. After we choose a active function $\sigma$. Then the output of this layer is a $M\times N \times P$ tensor
	$$
		\mathbf U = \sigma(\mathcal{K}(\mathbf X) + \mathbf B)
	$$
	where $\mathbf B =(B_{m,n,p})_{M\times N\times P}$ and $B_{m,n,p} = b_p$
	\subsection{Pooling Layer}\label{Pooling}
	Suppose input data is a $M \times N \times L $ multi-channel image $\mathbf X$ and $M=S\times A, N=T\times B$. The $l$-th channel of $X$ is a $M\times N$ matrix $X_p$. The output of this layer is a $A\times B\times P$ multi-channel image:

	$$
	\mathbf U =P_{S\times T}(\mathbf X)=(U_p)_{P}
	$$
	where
	\begin{equation}
	U_p = P_{S\times T}(X_p)
	\end{equation}
	\subsection{Dropout Layer}\label{Dropout}
	For a input $N$-element vector $\mathbf x$, and a number $\gamma \in [0,1]$, the output is
	$$
	\mathbf u = (u_n)_N = \text{dropout}_{\gamma} (\mathbf x)
	$$
	where
	\begin{equation}
	u_n = \left \{
	\begin{split}
	x_n &\ \text{with probility}\ \gamma\\
	0   &\ \text{with probility}\ 1-\gamma
	\end{split}
	\right.
	\end{equation}
	\subsection{Classifier Layer}
	\paragraph{Softmax Classifier}\label{Softmax} For a input vector $x$ and the parameter is a matrix $W$ and vector $\mathbf b$ called bias. Then
	$$
	\mathbf y = W\mathbf x + \mathbf b
	$$
	and the output is
	\begin{equation}
		\mathbf u = (u_c)_{\mathcal C} =\text{softmax}(\mathbf y)
	\end{equation}
	where
	\begin{equation}
	u_c = \dfrac{e^{y_c}}{\sum_{c=1}^{\mathcal{C}} e^{y_c}}
	\end{equation}
\iffalse
	\begin{table}[h]
		\begin{tabular}{ccc}
			Kind of Layer &\\ \hline
			Convolution Layer 		&\\
			Pooling Layer 			&\\
			Densely Connected Layer &\\
			Dropout Layer			&\\
			Classifier Layer		&\\
		\end{tabular}
	\end{table}
\fi
	\section{A Example of CNN}
	This CNN totally contains 7 hidden layers. The describe of each layer is listed in follow table.
	\newpage
	\begin{table}
		\begin{tabular}{|c|c|c|}\hline
		Layer 						&Input and			&Output	\\
									&Parameter			&\\ \hline
		0 Input Layer    			& 					& $X[28,28]$\\\hline
		1 Convolution Layer 		&$\mathbf X[28,28,1] = X$			&$\mathbf U_1[28,28,32]$\\
        &$\mathcal K_1[5,5,1,32], \mathbf b_1[32]$	&$\mathbf U_1 = \sigma (\mathcal K_1(\mathbf X)+ \mathbf B_1)$ (\ref{Convolution})\\\hline
		2 Pooling Layer 			&$\mathbf U_1[28,28,32]$	&$\mathbf U_2[14,14,32]$\\
		&							&$\mathbf U_2 = P_{2\times 2}^{\max}(\mathbf U_1)$ (\ref{Pooling})\\\hline
		3 Convolution Layer			&$\mathbf U_2$[14,14,32]		&$\mathbf U_3[14,14,64]$\\
		&$\mathcal K_2[5,5,32,64], \mathbf b_2[64]$&$\mathbf U_3 = \sigma (\mathcal K_2(\mathbf U_3)+ \mathbf B_2)$(\ref{Convolution})\\	\hline
		4 Pooling Layer 			&$\mathbf U_3[14,14,64]$		&$\mathbf U_4[7,7,64]$\\
		& 							&$\mathbf U_4 = P_{2\times 2}^{\max}(\mathbf U_3)$(\ref{Pooling})\\\hline
		5 Densely Connected Layer 	&$\mathbf x[3136] = \text{vec}(\mathbf U_4)$ &$\mathbf u_1[1024]$\\
		&$W[3136,1024],\mathbf b[1024]$		&$\mathbf u_1 = \sigma (W\mathbf x +\mathbf b)$(\ref{Dense})\\\hline
		6 Dropout Layer				&$\mathbf u_1[1024]$ 				&$\mathbf u_2[1024]$\\
		(Only be used in training)&					&$\mathbf u_2 = \text{dropout}_{0.5}(\mathbf u_1)$(\ref{Dropout})\\\hline
		7 Softmax Layer				&$\mathbf u_2[1024]$				&$\hat{\mathbf y}[10]$\\
		&$W_s[1024,10],\mathbf b_s[10]$	&$\hat{\mathbf y} = \text{softmax}(W_s\mathbf x +\mathbf b_s)$(\ref{Softmax}) \\\hline
		8 Output Layer				&									&\\ \hline
		\end{tabular}
	\end{table}

	In this table $\mathbf X[a,b,c]$ means $\mathbf X$ is a $a\times b\times c$ tensor.
\iffalse
	\begin{table}
	\begin{tabular}{ccc}
		Layer or Output & &Detail\\ \hline
	Layer 0 & Input a picture $\mathbf x$& $\mathbf x$ is a 28$\times$28 matrix \\\hline
	Layer 1 & Convolution Layer 		& \\
	Output of Layer 1 &  		&$\mathbf{u}_1$ is a 28$\times$28$\times$32 tensor\\ \hline
	Layer 2 & Pooling Layer 			&\\
	Output of Layer 2 &  		&$\mathbf{u}_2$ is a 14$\times$14$\times$32 tensor\\ \hline
		3 & Convolution Layer		&\\
	Output of Layer 3 &  		&$\mathbf{u}_3$ is a 14$\times$14$\times$64 tensor\\ \hline
		4 & Pooling Layer 			&\\
	Output of Layre 4&&$\mathbf{u}_4$ is a 7$\times$7$\times$64 tensor\\\hline
		5 & Densely Connected Layer &\\
	Output of Layer 5 &  		&$\mathbf{u}_5$ is a 1024 vector\\ \hline
		6 & Dropout Layer			&\\
	Output of Layer 6 &  		&$\mathbf{u}_6$ is a 1024 vector\\ \hline
		7 & Softmax Layer			&\\
	Output of Layer 5 &  		&$\mathbf{y}$ is a 10 vector\\ \hline
		8 & Output Layer			&output $\mathbf{y}$\\

	\end{tabular}
\end{table}
\fi

	\section{Optimizer and Numerical Result}
	For a input image $X_m$ in training data, now we can get $\hat{\mathbf y}_m = \hat{\mathbf y}(X_m,\bm \theta)$, where $\bm \theta = (\mathcal K_1, \mathbf b_1,\mathcal K_2, \mathbf b_2,W,\mathbf b,W_s,\mathbf b_s)$, Let $\hat{Y} = (\hat{\mathbf y}_1,\cdots,\hat{\mathbf y}_{\mathcal{M}})$
	Now we get the optimize question: Find
	$\bm \theta$ s.t.
	$$
	 \min	\mathcal{L}(Y,\hat{Y}) = \sum_{m=1}^{\mathcal M} L(\mathbf y_m, \hat{\mathbf{y}}_m)
	$$
	($L$ is in \ref{Loss}).

	The optimizers for this question is the same as optimizers in DNN. The numerical result of this CNN is showing below: The first figure show the accuracy rate, second figure is the value of loss function $\mathcal L$ and the third figure show the value of $-\lg \mathcal{L}$. Besides.

\begin{figure}
	\includegraphics[width = \textwidth,height=0.5\textheight]{SGD_relu_200}
	\caption{ Active function is relu, algorithm is SGD, step = 200,learning rate is 0.1}
	\includegraphics[width = \textwidth,height=0.5\textheight]{Momentum_relu_200}
	\caption{active function is relu, algorithm is Momentum, step = 200, learning rate is 0.03, momentum rate is 0.9}
\end{figure}

\begin{figure}
	\includegraphics[width = \textwidth,height=0.5\textheight]{Adagrad_relu_200}
	\caption{ Active function is relu, algorithm is Adagrad, step = 200,learning rate is 0.1}
	\includegraphics[width = \textwidth,height=0.5\textheight]{Adadelta_relu_200}
	\caption{active function is relu, algorithm is Adadelta, step = 200,learning rate is 0.1}
\end{figure}
\begin{figure}

	\includegraphics[width = \textwidth,height=0.5\textheight]{Adam_relu_200}
	\caption{active function is relu, algorithm is Adam, step = 200,learning rate is 0.0001 }
\end{figure}

\subsection{Logistic regression }
\subsubsection{Model}
\begin{itemize}
\item X: the input image of a handwritten digit
\item Y: the true value of the digit
\item W,b: weight and bias
\item Y$\_$pred=softmax(Wx+b)
\item Loss=cross$\_$entropy(Y, Y$\_$pred)+ regularization term
\item Optimization method: Adam
\end{itemize}

\subsubsection{Implementation }
\begin{itemize}
\item Parameter
\begin{enumerate}
\item learning rate = 0.001
\item training epochs=50
\item batch size =100
\item regularization coefficient =0.0001
\end{enumerate}
\item Result
\begin{enumerate}
\item no regularization: training accuracy=93$\%$; test accuracy =92$\%$
\item regularization coefficient =0.0001: training accuracy=93$\%$; test accuracy =92$\%$
\item regularization coefficient =0.001: training accuracy=92$\%$; test accuracy =92$\%$
\item no regularization+SGD: training accuracy= 88$\%$; test accuracy =89$\%$
\end{enumerate}

\end{itemize}

\subsection{One hidden layer }
\subsubsection{Model}
\begin{itemize}
\item X: the input image of a handwritten digit
\item Y: the true value of the digit
\item Hidden layer size=500
\item Loss=cross$\_$entropy(Y, Y$\_$pred)+ regularization term
\item Optimization method: Adam
\end{itemize}

\subsubsection{Implementation }
\begin{itemize}
\item Parameter
\begin{enumerate}
\item learning rate = 0.001
\item training epochs=50
\item batch size =100
\item regularization coefficient =0.0001
\end{enumerate}
\item Result
\begin{enumerate}
\item no regularization: training accuracy=100$\%$; test accuracy =98.36$\%$
\item regularization coefficient =0.0001: training accuracy=99.00$\%$; test accuracy =98.04$\%$
\item regularization coefficient =0.001: training accuracy=98.00$\%$; test accuracy =97.67$\%$
\item no regularization+SGD: training accuracy= 90$\%$; test accuracy =90.54$\%$
\end{enumerate}

\end{itemize}

\section{MgNet and iResNet results}
\subsection{MgNet}
\begin{description}
	\item[Model] The hyperparameters:
\begin{itemize}
	\item $J$: the number of grids. we choose $J = 4$, that makes $m_4 = n_4 = 5$.
	\item $\nu_\ell$:  the number of smoothings in each grids,
	just take $\nu_\ell = 2$.
	\item $c_u=c_f = 64$: the number of feature and data channels.
	\item $A^\ell = \xi^{\ell} $ and $B_{\ell,i} = \sigma \circ \eta^{\ell,i} \circ \sigma $ are all convolution with multichannel with
	$\mathbb{R}^{3\times3\times c\times c}$
	which need to be trained.
	\item $R_{\ell}^{\ell+1}$ and $\Pi_{\ell}^{\ell+1}$: the interpolation
	and restriction operator in MgNet.
	Here we choose it as a convolution with stride $2$ which need to be trained.
\end{itemize}

\item[Result] The result is: \\
model size: 557514, acc: 99.56, acc best: 99.60

\end{description}

\subsection{iResNet}
\begin{description}
	\item[Model] The hyperparameters:
	\begin{itemize}
	\item $J$: the number of grids. we choose $J = 4$, that makes $m_4 = n_4 = 5$.
\item $\nu_\ell$:  the number of smoothings in each grids,
in iResNet, because of the definition of $f^{\ell+1,0}$, so $(\nu_1, \cdots, \nu_{4}) = (2,1,1,1)$.
\item $c_f$: the number of channels.  Some suggestions:
\blue{\begin{itemize}
		\item The original iResNet-18: $(c_{f,1}, \cdots, c_{f,4}) = (64,128,256,512)$, but too many parameters.
		\item You can try $(16,32, 64, 128)$, less parameters.
		\item You can also try $(64,64,64,64)$, or any set up for channels.
\end{itemize}}
\item $\xi^{\ell,i} $ and $\eta^{\ell,i} $ are all convolution with multichannel with
$\mathbb{R}^{3\times3\times c\times c}$
which need to be trained.
\item $R_{\ell}^{\ell+1}$ and $\eta^{\ell,0}$: the restriction operator.
Here we choose it as a convolution with stride $2$ which need to be trained.
	\end{itemize}

	\item[Result] The result for $(16,32, 64, 128)$ is: \\
	model size: 605898,  acc: 99.52,  acc best : 99.56.

\end{description}


%\input{head.tex}
%\begin{document}

\section{Action and Conjugation}

\subsection{Action}

\defn{symmetric group, $S_n$}
\defn{
A group G \term{acts} on $\Omega$, if to every pair $(\alpha,g)\in\Omega\times G$, an element $\alpha^g\in\Omega$ is assigned s.t.\begin{enumerate}[i.]
\item{$\alpha^1=\alpha$ for $1=1_G$ and all $\alpha\in\Omega$,}
\item{$(\alpha^x)^y=\alpha^{xy}$ for all $x,y\in G$ and all $\alpha\in\Omega$.}
\end{enumerate}}

The mapping\\
\centerline{
$g^\pi:\Omega\to\Omega$ with $\alpha\mapsto\alpha^g$}
describes the action of $g\in G$ on $\Omega$. In particular $g^\pi$ is a bijection and thus a permutation on $\Omega$. \\

Now \romannumeral2. implies that\\
\centerline{
$\pi:G\to S_\Omega$ with $g\mapsto g^\pi$}
is a homomorphism.

%\end{document}


\section{Norms and Norm Dependent Linear Regression}

\subsection{Motivation}

\begin{frame}
  \frametitle{Motivation}

  \begin{itemize}
    \item Different norms and similarity measures play an important role in machine learning and pattern recognition. \\[.5cm]
    \item In this chapter we summarize important definitions and facts on norms. \\[.5cm]
    \item We consider the problem of linear regression for different norms. \\[.5cm]
    \item We will briefly look into associated optimization problems.
  \end{itemize}
\end{frame}

\subsection{Inner Product}

\begin{frame}
  \frametitle{Inner Product}

  \begin{citeblock}{Definition}

    The \structure{\emph{inner product of vectors}} $\vec x, \vec y\in \real^d$ is defined by
    \begin{displaymath}
      \langle\vec x, \vec y\rangle = \vec x^T \vec y = \sum_{i=1}^d x_iy_i \quad .
    \end{displaymath}
  \end{citeblock}
  \pspread

  \begin{ovalblock}{Example}
    The \structure{\emph{Euclidean norm} ($L_2$-norm)} can be written in terms of an inner product:
    \begin{displaymath}
      \| \vec x \|_2 = \sqrt{\langle \vec x, \vec x \rangle } =
      \sqrt{ \vec x^T \vec x} =
      \sqrt{\sum_{i=1}^d x_i^2 }\quad .
    \end{displaymath}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{Inner Product \cont}

  \begin{citeblock}{Definition}

    The \structure{\emph{inner product of matrices}} $\mat X, \mat Y\in \real^{m\times n}$ is defined by
    \begin{displaymath}
       \langle \mat X, \mat Y \rangle =
       \mbox{tr}(\mat X^T \mat Y) =
       \sum_{i=1}^m \sum_{j=1}^n x_{i,j} y_{i,j} \quad .
    \end{displaymath}
  \end{citeblock}
  \pspread

  \begin{ovalblock}{Example}
    The \structure{\emph{Frobenius norm}} can be written in terms of an inner product:
    \begin{displaymath}
      \| \vec X \|_F =
      \sqrt{ \langle \vec X, \vec X \rangle } =
      \sqrt{ \mbox{tr}(\mat X^T \mat X)} =
      \sqrt{ \sum_{i=1}^m \sum_{j=1}^n x_{i,j}^2} \quad .
    \end{displaymath}
  \end{ovalblock}
\end{frame}

\subsection{Norms}

\begin{frame}
  \frametitle{Norms}

  \begin{citeblock}{Definition}

    The function $\lVert\cdot\rVert$ %: \real^d\rightarrow \real$
    is called a \structure{\emph{norm}} if it
    \pause

    \begin{enumerate}
      \item is \structure{nonnegative}: $\forall \vec x:~ \lVert\vec x\rVert\geq 0$ \\[.3cm] \pause
      \item is \structure{definite}: $\|\vec x\| =0$ only if $\vec x=\vec 0$ \\[.3cm] \pause
      \item is \structure{homogeneous}: $\| a\vec x\|= |a|\cdot \|\vec x\|$ where $a\in \real$ \\[.3cm] \pause
      \item fulfills the \structure{triangle inequality}:
      \begin{displaymath}
        \forall \vec x, \vec y:~\|\vec x+\vec y\| \leq \|\vec x\|+ \|\vec y\|
      \end{displaymath}
    \end{enumerate}
  \end{citeblock}
\end{frame}

\begin{frame}
  \frametitle{Norms \cont}
%  \begin{citeblock}{Definition}
    \begin{itemize}
      \item The \structure{$L_0$-norm} of a $d$-dimensional vector denotes the number of \\
        non-zero entries. Despite its name, the $L_0$-norm is not a norm \\
        because it is not homogeneous. \pause
      \item The \structure{$L_p$-norm} ($p \geq 1$) of a $d$-dimensional vector is defined as
        \begin{displaymath}
          \| \vec x\|_p = \left( \sum_{i=1}^d  |x_i|^p \right)^{\frac{1}{p}}
        \end{displaymath}
    \end{itemize}
%  \end{citeblock}
\end{frame}

\begin{frame}
  \frametitle{Norms \cont}

  \begin{itemize}
    \item \structure{$L_1$-norm:} sum of absolute values
      \begin{displaymath}
        \| \vec x\|_1 = \sum_{i=1}^d |x_i|
      \end{displaymath}
      \pause
    \item \structure{$L_2$-norm:} sum of squared values
      \begin{displaymath}
        \| \vec x\|_2 = \left(\sum_{i=1}^d x_i^2\right)^{\frac{1}{2}}
      \end{displaymath}
      \pause
    \item \structure{$L_\infty$-norm:} maximum norm
      \begin{displaymath}
        \| \vec x\|_\infty =
        \lim_{p\rightarrow \infty} \left( \sum_{i=1}^d  |x_i|^p \right)^{\frac{1}{p}} =
        \max_i \{ |x_i| \ ; \ i=1,2, \dots, d \}
      \end{displaymath}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Norms \cont}

  \begin{citeblock}{Definition}

    Let $\mat P$ be a symmetric positive definite matrix. \\
    The \structure{\emph{quadratic $L_{\mat P}$-norm}} is defined by
    \begin{displaymath}
      \| \vec x\|_{\mat P} =
      \sqrt{ \vec x^T \mat P \vec x} \pause =
      \sqrt{ (\mat P^\frac{1}{2} \vec x)^T \mat P^\frac{1}{2} \vec x} \pause =
      \| \mat P^\frac{1}{2} \vec x\|_2
    \end{displaymath}
  \end{citeblock}
\end{frame}

\begin{frame}
  \frametitle{Norms \cont}

  \structure{Note:}

  \begin{itemize}
    \item The $L_2$-norm is the same as the quadratic $L_{\mat 1}$-norm. \\[.5cm] \pause
    \item The \structure{Mahalanobis distance} between two vectors $\vec x$ and $\vec y$ based on the covariance matrix $\mat \Sigma$ is given by the quadratic $L_{\mat \Sigma^{-1}}$-norm:
      \begin{displaymath}
        \| \vec x - \vec y\|_{\mat \Sigma^{-1}} =
        \sqrt{ (\vec x - \vec y)^T \mat\Sigma^{-1} (\vec x - \vec y)}
      \end{displaymath}
      \pause
    \item A norm is a measure for the length of a vector. It can also be used to measure the distance between two vectors $\vec x$ and $\vec y$:
      \begin{displaymath}
        \mbox{dist}(\vec x, \vec y) = \| \vec x - \vec y\|
      \end{displaymath}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Norms \cont}

  \structure{Norms of matrices} can be defined by norms of vectors. \\[.25cm]

  \begin{citeblock}{Definition}

    \small
    Let $\|.\|_p$ and $\|.\|_q$ be norms for vectors in $\real^m$ and $\real^n$. \\
    The \structure{\emph{operator norm}} of a matrix $\mat X\in \real^{m\times n}$  is defined by

    \begin{displaymath}
      \| \mat X\|_{p,q} = \sup\{ \| \mat X \vec u\|_p ; \  \| \vec u\|_q\leq 1 \}
    \end{displaymath}
  \end{citeblock}
  \pspread

  \begin{ovalblock}{Example}
    \small
    If $p=q=2$, i.\,e.\ we use the $L_2$-norm twice, the operator norm of $\mat X$ \\
    results in the maximum singular value:

    \begin{displaymath}
      \| \mat X\|_{2,2}=   \| \mat X\|_{2} = \sigma_{\max}{(\mat X)}= \sqrt{\lambda_{\max}(\mat X^T\mat X)}
    \end{displaymath}
  \end{ovalblock}
\end{frame}

\subsection{Unit Balls}

\begin{frame}
  \frametitle{Unit Balls}

  \begin{citeblock}{Definition}

    The set
    \begin{displaymath}
      {\cal B} = \{ \vec x; \|\vec x\| \leq 1\}
    \end{displaymath}
    of all vectors $\vec x$ of length less or equal to one according to the norm $\|.\|$ \\
    is called the \structure{\emph{unit ball}}.
  \end{citeblock}
\end{frame}

\begin{frame}
  \frametitle{Unit Balls \cont}

  The unit ball for the $L_1$-norm:

  \begin{figure}
    \resizebox{.4\linewidth}{!}{
      \input{\texfigdir/unit_ball_L1.pstex_t}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Unit Balls \cont}

  The unit ball for the $L_2$-norm:

  \begin{figure}
    \resizebox{.4\linewidth}{!}{
      \input{\texfigdir/unit_ball_L2.pstex_t}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Unit Balls \cont}

  The unit ball for the $L_{\infty}$-norm:

  \begin{figure}
    \resizebox{.4\linewidth}{!}{
      \input{\texfigdir/unit_ball_Linfinity.pstex_t}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Unit Balls \cont}

  The unit ball for the $L_{\mat P}$-norm:
  \begin{figure}
    \resizebox{.4\linewidth}{!}{
      \input{\texfigdir/unit_ball_LP.pstex_t}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Unit Balls \cont}

  Summary: unit balls for the $L_{\infty}$-, $L_4$-, $L_2$-, $L_1$-, $L_{0.5}$- and $L_0$-norm

  \begin{figure}
    \resizebox{\linewidth}{!}{
      \input{\texfigdir/unit_ball_Linfinity.pstex_t} \hspace{1cm}
      \input{\texfigdir/unit_ball_L4.pstex_t} \hspace{1cm}
      \input{\texfigdir/unit_ball_L2.pstex_t} \hspace{1cm}
      \input{\texfigdir/unit_ball_L1.pstex_t} \hspace{1cm}
      \input{\texfigdir/unit_ball_L0_5.pstex_t} \hspace{1cm}
      \input{\texfigdir/unit_ball_L0.pstex_t} \hspace{1cm}
    }
  \end{figure}

 The $L_{0.5}$- and the $L_0$-norm are not norms

\end{frame}

\input{nextTime.tex}

\subsection{Norm Dependent Linear Regression}

\begin{frame}
  \frametitle{Norm Dependent Linear Regression}

  In pattern recognition and pattern analysis (as in many other fields) one of the most important norm dependent linear regression problems is:

  \begin{displaymath}
   \mbox{minimize}\quad \| \mat A \vec x -\vec b\|
  \end{displaymath}

  or alternatively

  \begin{displaymath}
   \hat{\vec x} = \argmin_{\vec x} \| \mat A \vec x -\vec b\|
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Norm Dependent Linear Regression \cont}

  \begin{itemize}
    \item Different norms will lead to different results. \\[.5cm]\pause
    \item The estimation error $\epsilon\in \real$ is defined by $\epsilon= \|\vec x^*-\hat{\vec x}\|$, where $\vec x^*$ denotes the correct value. \\[.5cm] \pause
    \item The \structure{residual} $\vec r= (r_1, r_2, \dots, r_m)^T$ is defined by $\vec r= \mat A \vec x -\vec b$. \\[.5cm] \pause
    \item If $\vec b$ is in the range of $\mat A$, the residual will be the zero vector.
  \end{itemize}
\end{frame}

\subsubsection{Least-Squares Linear Regression}

\begin{frame}
  \frametitle{Least-Squares Linear Regression}

  \structure{Minimization of the residual using the $L_2$-norm:}

  \begin{eqnarray*}
    \hat{\vec x} &=& \argmin_{\vec x} \| \mat A \vec x -\vec b\|_2 \\[.15cm] \pause
                 &=& \argmin_{\vec x} \sum_{i=1}^m r_i^2 \\[.15cm] \pause
                 &=& \argmin_{\vec x} \ (\mat A \vec x -\vec b)^T (\mat A \vec x -\vec b) \\[.15cm] \pause
                 &=& \argmin_{\vec x} \ \big( \vec x^T \mat A^T \mat A\vec x - \vec x^T\mat A^T \vec b - \vec b^T \mat A\vec x + \vec b^T \vec b \big) \\[.15cm] \pause
                 &=& \argmin_{\vec x} \ \big( \vec x^T \mat A^T \mat A\vec x - 2 \vec b^T \mat A\vec x + \vec b^T \vec b \big) \\
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Least-Squares Linear Regression \cont}

  The \structure{partial derivatives} are:

  \begin{displaymath}
    \frac{\partial}{\partial \vec x} \ \left( \vec x^T \mat A^T \mat A\vec x - 2 \vec b^T \mat A\vec x + \vec b^T \vec b \right)
    = \pause 2 \mat A^T \mat A \vec x - 2 \mat A^T \vec b = \vec{0}
  \end{displaymath}

  Using the partial derivatives we get a \structure{closed form solution} for the $L_2$-norm: \pause

  \begin{displaymath}
    \hat{\vec x} = (\mat A^T \mat A)^{-1}\mat A^T \vec b
  \end{displaymath}

  if the columns of $\mat A$ are mutually independent.
\end{frame}

\subsubsection{Chebyshev Linear Regression}

\begin{frame}

  \frametitle{Chebyshev Linear Regression}

  \structure{Minimization of the residual using the $L_\infty$-norm:}

  \begin{displaymath}
    \mbox{minimize}\quad \Bigg\{ \| \mat A \vec x -\vec b\|_\infty = \max\ \{ |r_1|, |r_2|, \dots, |r_m|\} \Bigg\}
  \end{displaymath}
  \pause

  This optimization problem can be rewritten in terms of a LP-problem:

  \begin{eqnarray*}
    \begin{array}{lcl}
     \mbox{minimize}& & r\\
     \mbox{subject to} & &   - r\cdot \vec 1 \preceq \mat A \vec x - \vec b \preceq r\cdot \vec 1
    \end{array}
  \end{eqnarray*}

  where $r\in \real$ and $\vec 1 \in \{1\}^m$.
\end{frame}

\subsubsection{Sum of Absolute Residuals}

\begin{frame}
  \frametitle{Sum of Absolute Residuals}

  \structure{Minimization of the residual using the $L_1$-norm:}

  \begin{displaymath}
    \mbox{minimize}\quad \Bigg\{ \| \mat A \vec x -\vec b\|_1 = \sum_{i=1}^m  |r_i| \Bigg\}
  \end{displaymath}
  \pause

  This optimization problem can be rewritten in terms of a LP-problem:

  \begin{eqnarray*}
    \begin{array}{lcl}
      \mbox{minimize}   & & \vec 1^T \vec r \\
      \mbox{subject to} & & - \vec r \preceq  \mat A \vec x -\vec b\preceq \vec  r
    \end{array}
  \end{eqnarray*}

  where $\vec r\in \real^m$ and $\vec 1 \in \{1\}^m$.
\end{frame}

\subsubsection{Ridge Regression and Unit Balls}

\begin{frame}
  \frametitle{Ridge Regression and Unit Balls}

  \structure{Ridge regression} is defined via the optimization problem

  \begin{displaymath}
    \mbox{minimize} \quad \mat \|A\vec x -\vec b\|_2+\lambda \cdot\|\vec x\|_2
  \end{displaymath}

  \begin{center}
    \resizebox{.4\linewidth}{!}{
      \alt<7->{
        \input{\texfigdir/ridge_regression7.pstex_t}
      }{\alt<6>{
        \input{\texfigdir/ridge_regression6.pstex_t}
      }{\alt<5>{
        \input{\texfigdir/ridge_regression5.pstex_t}
      }{\alt<4>{
        \input{\texfigdir/ridge_regression4.pstex_t}
      }{\alt<3>{
        \input{\texfigdir/ridge_regression3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/ridge_regression2.pstex_t}
      }{
        \input{\texfigdir/ridge_regression1.pstex_t}
      }}}}}}
    }
  \end{center}
\end{frame}

\subsubsection{Lasso and Unit Balls}

\begin{frame}
  \frametitle{Lasso and Unit Balls}

  The \structure{lasso} (Tibshirani 1996) is defined via the optimization problem

  \begin{displaymath}
    \mbox{minimize} \quad \mat \|A\vec x -\vec b\|_2+\lambda \cdot\|\vec x\|_1
  \end{displaymath}

  \begin{center}
    \resizebox{.4\linewidth}{!}{
      \alt<6>{
        \input{\texfigdir/lasso6.pstex_t}
      }{\alt<5>{
        \input{\texfigdir/lasso5.pstex_t}
      }{\alt<4>{
        \input{\texfigdir/lasso4.pstex_t}
      }{\alt<3>{
        \input{\texfigdir/lasso3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/lasso2.pstex_t}
      }{
        \input{\texfigdir/lasso1.pstex_t}
      }}}}}
    }
  \end{center}
\end{frame}

\subsubsection{Compressed Sensing}

\begin{frame}
  \frametitle{Compressed Sensing}

  \begin{itemize}
    \item In the previous chapter we motivated regularized linear regression. \\[.25cm]
    \item Assume we have fewer measurements than required to estimate the parameter vector $\vec x$. \\[.25cm]
    \item Solution of the underdetermined case required. \\[.25cm]
    \item We call a vector $S$-sparse if its support, i.\,e.\ the number of non-zero entries, is less or equal to $S$  \\[.25cm] \pause
    \item The vector $\vec x$ can be recovered mostly always by solving the convex optimization problem (quadratic programming):
      \begin{eqnarray*}
        \mbox{minimize}   & &\quad \|\vec x\|_1\\
        \mbox{subject to} & & \quad \mat A\vec x =\vec b.
      \end{eqnarray*}
  \end{itemize}
\end{frame}

\subsection{Convex and Non-Convex Penalty Functions}

\begin{frame}
  \frametitle{Penalty Function}

  Motivated by the discussion of different norms, we now introduce and study \structure{penalty functions}.
  \spread

  \begin{citeblock}{Definition}

    The \structure{\emph{penalty function approximation problem}} is defined as follows:

    \begin{eqnarray*}
      \mbox{minimize}   & &\sum_{i=1}^m \phi(r_i) \\
      \mbox{subject to} & & \vec r= (r_1, r_2, \dots, r_m)^T = \mat A\vec x -\vec b,
    \end{eqnarray*}

    where $\phi:\real \rightarrow \real$ is the penalty function for the components of the \\
    residual vector.
  \end{citeblock}
 \end{frame}

\begin{frame}
  \frametitle{Penalty Function \cont}

  \structure{Note:} \\[.5cm]

  \begin{itemize}
    \item The penalty function $\phi$ assigns \structure{costs} to residuals. \\[.5cm]
    \item If $\phi$ is a convex function, the penalty function approximation problem \\
      is a convex optimization problem.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Penalty Function \cont}

  Penalty functions of the $L_1$-,  $L_2$-norms:

  \begin{displaymath}
    \phi_{L_1}(r) = |r|;  \qquad \qquad \qquad \qquad \qquad \phi_{L_2}(r) = r^2
  \end{displaymath}
  \pause

  \vspace{-0.5cm}
  \begin{figure}
    \resizebox{.4\linewidth}{!}{
      \input{\texfigdir/penalty_L1.pstex_t}
    }
    \hspace{1cm}
    \resizebox{.4\linewidth}{!}{
      \input{\texfigdir/penalty_L2.pstex_t}
    }
  \end{figure}

  \begin{itemize}
    \item In $L_1$ small deviations are weighted higher than using $L_2$.
    \item In $L_1$ large deviations are weighted lower than using $L_2$.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Penalty Function \cont}

  \structure{Log barrier function}

  \begin{displaymath}
    \phi_\mathsf{barrier}(r) =
      \left\{
        \begin{array}{cl}
          -a^2\log \left(1-\left( \frac{r}{a}\right)^2 \right),  & \quad \mbox{if} \quad |r| < a\\
          \infty,                                                & \quad \mbox{otherwise}
        \end{array}
      \right.
  \end{displaymath}

  \begin{figure}
    \resizebox{.6\linewidth}{!}{
      \alt<3->{
        \input{\texfigdir/penalty_LogBarrier3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/penalty_LogBarrier2.pstex_t}
      }{
        \input{\texfigdir/penalty_LogBarrier1.pstex_t}
      }}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Penalty Function \cont}

  \structure{Dead zone linear penalty function}

  \begin{displaymath}
    \phi_\mathsf{dz}(r) =
      \left\{
        \begin{array}{cl}
          0,     & \quad \mbox{if} \quad |r| \leq a\\
          |r|-a, & \quad \mbox{otherwise}
        \end{array}
      \right.
  \end{displaymath}

  \begin{figure}
    \resizebox{.6\linewidth}{!}{
      \alt<3->{
        \input{\texfigdir/penalty_DeadZoneLinear3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/penalty_DeadZoneLinear2.pstex_t}
      }{
        \input{\texfigdir/penalty_DeadZoneLinear1.pstex_t}
      }}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Penalty Function \cont}

  \structure{Large error penalty function}

  \begin{displaymath}
    \phi_\mathsf{e}(r) =
      \left\{
        \begin{array}{cl}
          r^2, & \quad \mbox{if} \quad |r| \leq a \\
          a^2, & \quad \mbox{otherwise}
        \end{array}
      \right.
  \end{displaymath}

  \begin{figure}
    \resizebox{.6\linewidth}{!}{
      \alt<3->{
        \input{\texfigdir/penalty_LargeError3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/penalty_LargeError2.pstex_t}
      }{
        \input{\texfigdir/penalty_LargeError1.pstex_t}
      }}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Penalty Function \cont}

  \structure{Huber function}

  \begin{displaymath}
    \phi_\mathsf{Huber}(r) =
      \left\{
        \begin{array}{cl}
          r^2,            & \quad \mbox{if} \quad |r| \leq a \\
          a\cdot(2|r|-a), & \quad \mbox{otherwise}
        \end{array}
      \right.
  \end{displaymath}

  \begin{figure}
    \resizebox{.6\linewidth}{!}{
      \alt<3->{
        \input{\texfigdir/penalty_Huber3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/penalty_Huber2.pstex_t}
      }{
        \input{\texfigdir/penalty_Huber1.pstex_t}
      }}
    }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Penalty Functions \cont}

  \begin{figure}
    \resizebox{1.0\linewidth}{!}{
      \alt<6->{
        \input{\texfigdir/penalty6.pstex_t}
      }{\alt<5>{
        \input{\texfigdir/penalty5.pstex_t}
      }{\alt<4>{
        \input{\texfigdir/penalty4.pstex_t}
      }{\alt<3>{
        \input{\texfigdir/penalty3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/penalty2.pstex_t}
      }{
        \input{\texfigdir/penalty1.pstex_t}
      }}}}}
    }
  \end{figure}
\end{frame}

\subsection{Lessons Learned}

\begin{frame}
  \frametitle{Lessons Learned}

  \begin{itemize}
    \item We have considered vector and matrix norms in more detail. \\[.3cm]
    \item Important vector norms: $L_1$, $L_2$, $L_\infty$, and $L_{\mat P}$. \\[.3cm]
    \item Unit balls \\[.3cm] \pause
    \item Linear regression for different norms: range from closed form solution to LP-problem. \\[.3cm]
    \item Regularized linear regression: range from closed form solution through QP-problem up to combinatorial optimization. \\[.3cm]
    \item We need to know the basics of algorithms for unconstrained and constrained optimization as well as convex optimization.
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsection{Further Readings}

\begin{frame}
  \frametitle{Further Readings}

  \begin{itemize}
    \item G.~Golub, C.~F.~Van~Loan: \\
      \structure{Matrix Computations}, 3rd Edition, \\
      The Johns Hopkins University Press, Baltimore, 1996. \\[.15cm]
    \item Lloyd N. Trefethen, David Bau III: \\
      \structure{Numerical Linear Algebra}, \\
      SIAM, Philadelphia, 1997. \\[0.15cm]
    \item S.~Boyd, L.~Vandenberghe: \\
      \structure{Convex Optimization}, \\
      Cambridge University Press, 2004. \\
      \point{\small \url{http://www.stanford.edu/~boyd/cvxbook/}} \\
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Further Readings \cont}

  \begin{itemize}
    \item Compressed sensing is one of the most recent hot topics in pattern recognition and image processing. An excellent source is: \\[.3cm]
      \centerline{\structure{\url{http://www.dsp.ece.rice.edu/cs}}}
      \vspace{.3cm}
      or the recent workshop on compressed sensing at Duke University: \\[.3cm]
    \structure{\url{http://people.ee.duke.edu/\%7Elcarin/compressive-sensing-workshop.html}}.
  \end{itemize}
\end{frame}

\subsection{Comprehensive Questions}

\begin{frame}
  \frametitle{Comprehensive Questions}

  \begin{itemize}
    \item What is the difference between the $L_p$- (p $\ge$ 1) and the $L_{\mat P}$-norm? \\[1cm]
    \item How do the unit balls look like for $L_{\infty}$-, $L_4$-, $L_2$-, $L_1$- and $L_0$-norm? \\[1cm]
    \item What is the benefit of using the $L_1$- over the $L_2$-norm for sparse, underdetermined problems? \\[1cm]
    \item What specific property of penalty functions is of special interest and \\
      why do we need different penalty functions at all?
  \end{itemize}
\end{frame}


\section{Overview}
\label{s:overview}

We start with an overview of how \sys works by describing
how it can prove the equivalence of two functions: a
recursive function |sumTo| that adds up the numbers
from |1| to |n|, and a \emph{tail-recursive} variant
|sumToTR| that uses a helper |loop| with an accumulator
|acc| to compute the same result.

\begin{figure*}[t!]
\centering
\begin{minipage}[t]{0.40\textwidth}
% \begin{minted}[linenos, firstnumber=1,breaklines=true]{ocaml}
\begin{ocode}
(* -- Recursive Summation -- *)

let rec sumTo n =
  if n = 0
    then 0
    else n + sumTo (n - 1)
\end{ocode}
\end{minipage}
\hspace{0.14in}
\begin{minipage}[t]{0.49\textwidth}
% \begin{minted}[linenos, firstnumber=1,breaklines=true]{ocaml}
\begin{ocode}
(* -- Tail-Recursive Summation -- *)

let sumToTR n =
  let rec loop n acc =
    if n = 0
      then acc
      else loop (n - 1) (acc + n)
  in
    loop n 0
\end{ocode}
\end{minipage}
\caption{\textbf{(L)} Function that sums the numbers to $n$
         and \textbf{(R)} Tail-recursive accumulator based variant.
}
\label{fig:example}
\end{figure*}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS671: Machine Learning
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 1}

Determine the interior, closure and border of the set $S = \{x \in \mathbb{R}^3 | x_1^2 + x_2^2 + x_3^2 < 1, x_1+x_2+x_3 = a, a > 0\}$.
Note that the result depends on the value of $a$.

\subsection*{Solution}

Set $S$ can be represented as the intersection of two sets $S_{\text{sub}_1} = \{x \in \mathbb{R}^3 | x_1^2 + x_2^2 + x_3^2 < 1\}$ and $S_{\text{sub}_2} = \{x \in \mathbb{R}^3 | x_1 + x_2 + x_3 = a, a > 0\}$; the former consists of the set of points inside a sphere of radius 1 and the latter consists of the set of points on a plane whose x-,y- and z-intercepts are $a$.

For clarification, define Point $P = (x_0, y_0, z_0)$ as the point at which the hyperplane $x + y + z = a$ would be tangent to the sphere of radius 1.
This point is important in that all choices of $a$ larger than or equal to $x_0 + y_0 + z_0$ will make the plane to pass over the interior of the sphere represented by $S_{\text{sub}_1}$, resulting in $S_{\text{sub}_1} \cap S_{\text{sub}_2} = \emptyset$.
On the other hand, in cases where $0 \leq a \leq x_0+y_0+z_0$, the plane would intersect with $S_{\text{sub}_1}$, making an open circle in $\mathbb{R}^3$.

Our objective, is thus to attain coordinates of point $P$.
Since this point is on the sphere and the hyperplane, we'll have

\begin{equation}
x_0^2 + y_0^2 + z_0^2 = 1
\label{eq11}
\end{equation}

\begin{equation}
x_0 + y_0 + z_0 = a
\label{eq12}
\end{equation}

Substituting Eq. \ref{eq11} into Eq. \ref{eq12}, Eq. \ref{eq13} can be obtained.

\begin{equation}
x_0y_0 + x_0z_0 + y_0z_0 = \frac{a^2-1}{2}
\label{eq13}
\end{equation}

However, since $x_0 + y_0 = a - z_0$, Eq. \ref{eq13} will lead to

\begin{equation}
z_0 = \frac{a}{2} \pm \sqrt{x_0y_0 + \frac{1}{2} - \frac{a^2}{4}}
\label{eq14}
\end{equation}

Since there is only one point $P$ with unique coordinates, Eq. \ref{eq15} must hold.

\begin{equation}
x_0y_0 + \frac{1}{2} = \frac{a^2}{4}
\label{eq15}
\end{equation}

And using $z_0 = \frac{1}{2}$,

\begin{equation}
\begin{aligned}
a &= \sqrt{3}\\
P(x_0, y_0, z_0) &= (\frac{1}{\sqrt{3}},\frac{1}{\sqrt{3}},\frac{1}{\sqrt{3}})
\end{aligned}
\label{eq16}
\end{equation}

Therefore, depending on the choice of $a$, two cases exist for the intersection of $S_{\text{sub}_1}$ and $S_{\text{sub}_2}$.
\begin{itemize}\itemsep=0pt
\item[] If $a \geq \sqrt{3}$, the hyperplane would pass over the interior of the sphere and the intersection would be empty, causing $I(S), K(S), \partial(S) = \emptyset$.
Noteworthy that in the case where hyperplane is tangent to the closed sphere of radius 1, $S_{\text{sub}_1} \cap S_{\text{sub}_2} = \emptyset$ would still hold, since $S_{\text{sub}_1}$ only contains interior of the sphere of radius 1.
\item[] If $0 \leq a < \sqrt{3}$, intersection of the hyperplane and interior of the sphere would be a filled but open circle in $\mathbb{R}^3$.
In this case, interior of the set $S$ will be the filled but open circle in $\mathbb{R}^3$ described by Eq. \ref{eq17}.

\begin{equation}
I(S) = S = \{x \in \mathbb{R}^3 | x_1^2 + x_2^2 + x_3^2 < 1, x_1 + x_2 + x_3 = a\}
\label{eq17}
\end{equation}

As $S$ is an open set, boundary of $S$, $\partial(S)$ will be the circle resulted as intersection of the plane and the sphere with radius 1.
Therefore $\partial(S)$ can be described by Eq. \ref{eq18}.

\begin{equation}
\partial(S) = \{x \in \mathbb{R}^3 | x_1^2 + x_2^2 + x_3^2 = 1, x_1 + x_2 + x_3 = a\}
\label{eq18}
\end{equation}

And since closure of any set is the union of its interior and its boundary, $K(S)$ can be expressed as given in Eq. \ref{eq19}.

\begin{equation}
K(S) = I(S) \cap \partial(S) = \{x \in \mathbb{R}^3 | x_1^2 + x_2^2 + x_3^2 \leq 1, x_1 + x_2 + x_3 = a\}
\label{eq19}
\end{equation}

\end{itemize}


If you haven't downloaded and unzipped \href{https://libaoj.in/courses/2021f/MATH3341/zip/Math.3341.zip}{\texttt{Math.3341.zip}}. Download and unzip it under \verb|H:| (H Drive if you are working on the Remote Lab). Change the current working directory by typing \verb|cd H:\Math.3341\Math.3341.Lab.07| in the Command Window, and type \verb|edit lab_07_script| in the Command Window to edit \verb|lab_07_script.m|.

%---------------------------------------------
\section{Debugging}
%---------------------------------------------
In this lab you will learn how to debug code and develop ways to make code more readable by employing good coding practices.

Your goal is to fix the bugs in \verb|lab_07_script.m| and \verb|lab_07_function.m|. These files solve the linear system $A \mathbf{x} = \mathbf{b}$ given by
\begin{align*}
    4 x_1 + 3 x_2       & = 24 \\
    3 x_1 + 4 x_2 - x_3 & = 30 \\
    - x_2 + 4 x_3       & = -24 \\
\end{align*}
using the Gauss-Seidel method with Successive Over Relaxation (SOR). This method gives means to speed up the convergence of our iterative method. The only change from the Gauss-Seidel method is the use of a parameter $\omega$. Depending on the choice of this $\omega$, the Gauss Seidel method can be performed in significantly less iterations than the original method. The iterative step in this method is now
$$
x_i^{(k)} = (1 - \omega) x_i^{(k-1)} + \frac{\omega}{a_{ii}} \left[ b_i - \sum_{j=1}^{i-1} a_{ij} x_j^{(k)} - \sum_{j=i+1}^{n} a_{ij} x_j^{(k - 1)} \right].
$$
These code files are riddled with various errors. Your task is to look through the code and correct all of the issues you encounter. Methods for doing this efficiently will be explained in the lab. Note that the necessary corrections may involve any of the following:

\begin{enumerate}[1.]
    \item changing variable names,
    \item fixing indexing,
    \item suppressing output,
    \item changing code style,
    \item adding proper indentation,
    \item adding comments,
    \item removing redundant code.
\end{enumerate}

Once you finish debugging, call \verb|diary('lab_07_output.txt')|, then run the script \verb|lab_07_script.m|, and call \verb|diary off| to save the output. The outputs and figures should be exactly same as those on the following pages. You will upload files \verb|sor_gauss_seidel.tex|, \verb|lab_07_script.m|, \verb|lab_07_function.m|, \verb|lab_07_output.txt|, \verb|lab_07_plot_1.pdf|, and \verb|lab_07_plot_2.pdf|. Recompile, and submit the generated \verb|.pdf| file on WyoCourses.

\newpage

\section{Results}

\subsection{Output}

\lstinputlisting[style=Plain]{../Math.3341.Lab.07.ans/lab_07_output.txt}

\subsection{Formatted output}

\input{../Math.3341.Lab.07.ans/sor_gauss_seidel.tex}

\newpage

\subsection{Plots}
\begin{figure}[!hbtp]
    \centering
    \includegraphics[width=0.80\textwidth]{../Math.3341.Lab.07.ans/lab_07_plot_1.pdf}
    \includegraphics[width=0.80\textwidth]{../Math.3341.Lab.07.ans/lab_07_plot_2.pdf}
    \caption{Solution and residual}
    \label{fig:sol}
\end{figure}


\chapter{Gaussian Approximations}

\section*{5.1. Equal percentages}
\addcontentsline{toc}{section}{5.1. Equal percentages}
In the last paragraph of Section 2.1, the same percentage
99.999999975\%, appeared twice. Explain why you know that
these two percentages must be the same, even if you don't
know what the common value is.

\vspace{1em}

\begin{proof}
    We've seen that for a high number of flips, the binomial
    distribution reduces to a Gaussian distribution
    with the standard deviation $\sigma = \sqrt{n/4}$. Since we are dealing with a
    Gaussian distribution, the probability that the number of Heads
    obtained differs from the expected number by at most $\Delta x$ Heads is given
    by the number of standard deviations needed to reach $\Delta x$.
    The two experiments in Section 2.1 were:
    \begin{enumerate}[(i)]
        \item $n_1 = 10^5$ flips and the probability that the expected number of
            Heads differs by more than $\Delta p_1 = 1\%$ of the expected value
            is given by our percentage. Hence, the standard deviation is given by
            $\sigma_1 = \sqrt{10^5 / 4} = 50\sqrt{10}$ and the number of flips
            that deviate from the expected value is $\Delta x_1 = n_1 \Delta p_1 = 10^3$.

        \item $n_1 = 10^5$ flips and the probability that the expected number of
            Heads differs by more than $\Delta p_2 = 0.01\%$ of the expected value
            is given by our percentage. Hence, the standard deviation is given by
            $\sigma_2 = \sqrt{10^9 / 4} = 5000\sqrt{10}$ and the number of flips
            that deviate from the expected value is $\Delta x_2 = n_2 \Delta p_2 = 10^5$.
    \end{enumerate}

    Now, since the number of standard deviations to reach the needed $\Delta x$'s are
    equal for both cases,
    \[
        \frac{\Delta x_1}{\sigma_1} = \frac{10^3}{50\sqrt{10}} = 5 \sqrt{10}
        = \frac{10^5}{5000\sqrt{10}}
        = \frac{\Delta x_2}{\sigma_2}
    \]
    we have that the percentage that the number of Heads does not deviate
    from the expected value more than $\Delta x$ Heads is the same for both
    cases, that is 99.999999975.
\end{proof}

\section*{5.2. Rolling sixes}
\addcontentsline{toc}{section}{5.2. Rolling sixes}
In the solution to Problem 2.13 (known as the Newton-Pepys problem), we
noted that the answer to the question, "If $6n$ dice are rolled, what is the
probability of obtaining at least $n$ 6's?", approaches 1/2 in the $n \to \infty$
limit. Explain why this is the case.

\vspace{1em}

\begin{proof}
    We know that for $n \to \infty$ the Binomial distribution reduces to the
    Gaussian distribution. Let $X$ denote the number of 6's that are obtained
    in $6n$ rolls. Assuming that the probability of a dice to be rolled as a 6
    is 1/6, we get that $E[X] = n$, i.e. the expected number of 6's after $6n$
    rolls is $n$. As $n \to \infty$, the probability
    of obtaining exactly $n$ 6's is negligible, so the probability that we
    get at least $n$ sixes is roughly given by the area of the right half of the
    Gaussian approximation. Therefore, for $n \to \infty$ the sought probability approaches
    $1/2$ (half of the area of a normalized Gaussian).
\end{proof}

\section*{5.3. Coin flips}
\addcontentsline{toc}{section}{5.3. Coin flips}
If you flip 10$^4$ coins, how surprised would you be if the observed percentage
of Heads differs from the expected value of 50\% by more than 1\%? Answer
the same question for 10$^6$ coins. (These numbers are large enough so that
the binomial distribution can be approximated by a Gaussian).

\vspace{1em}

\begin{proof}
    Since the number of flips are large enough, the Binomial distributions are reduced
    to Gaussians. We analyze the two cases separately:
    \begin{enumerate}[(i)]
        \item We start by seeing that, the standard deviation is given by $\sigma_1 = \sqrt{10^4 / 4} = 50$, and
            that $\Delta x_1 = 10^4 \cdot 1\% = 100$. Now, we "shift" the Gaussian so that
            $\mu = 0$. The graph will show us the probabilities of the observed
            value relative to the expected value. Therefore, the probability
            that the observed number of Heads differs from the expected value of
            50\% by more than 1\% is given by:
            \[
                p_1
                = 1 - \frac{1}{\sqrt{2 \pi {\sigma_1}^2}}
                    \int_{-\Delta x_1}^{\Delta x_1} e^{-\frac{x^2}{2{\sigma_1}^2}} dx
                = 1 - \frac{1}{50\sqrt{2\pi}}
                    \int_{-100}^{100} e^{-\frac{x^2}{2 \cdot 50^2}} dx
                \approx 1 - 0.9544 = 0.0455 = 4.55\%
            \]

        As a result, I would be surprised, but not very much so. 4.55\% is not a big percentage, but
        certainly not negligible.

    \item Analogously with the other case, we have that $\sigma_2 = \sqrt{10^6/4} = 500$ and
        $\Delta x_2 = 10^6 \cdot 1\% = 10^3$. By "shifting" the Gaussian again such that
        $\mu = 0$, the probability that the observed number of Heads differs from
        the expected value of 50\% by more than 1\% is:
        \[
            p_2
            = 1 - \frac{1}{\sqrt{2 \pi {\sigma_2}^2}}
                \int_{-\Delta x_2}^{\Delta x_2} e^{-\frac{x^2}{2{\sigma_2}^2}} dx
            = 1 - \frac{1}{500\sqrt{2}}
                \int_{-10^3}^{10^3} e^{-\frac{x^2}{2 \cdot 500^2}} dx
            \approx 0 \approx 0\%
        \]

        Therefore, I would be extremely surprised by the result, as this occuring seems
        very close to impossible.
    \end{enumerate}
\end{proof}

\section*{5.4 Identical distributions}
\addcontentsline{toc}{section}{5.4. Identical distributions}
A thousand dice are rolled. Fig. 5.15 shows the probability distribution (given
by Eq. (5.15)) for the number of 6's that appear, relative to the expected number
(which is 167). How many $\emph{coins}$ should you flip if you want the probability
distribution for the number of Heads that appear (relative to the expected number)
to look exactly like the distribution in Fig. 5.15 (at least in the Gaussian
approximation)?

\begin{figure}[H]
    \center{\includegraphics[width=0.6\linewidth]{figure5_15.png}}
    \caption*{\textbf{Figure 5.15}: The probability distribution for the number of 6's in 1000
    dice rolls, relative to the expected number, 167.}
\end{figure}

\vspace{1em}

\begin{proof}
    Both the dice and coin distributions are binomial distributions, so we'll approximate
    them as Gaussian distributions. A Gaussian distribution is uniquely identified by its
    mean and standard deviation, so two distributions are identical if they have these
    two attributes identical. Since we are interested by the distribution
    for the number of Heads relative to the expected number, we don't care about the mean.
    Therefore, the standard deviation of the dice distribution must be the same as the one
    of the coin distribution. Knowing that the standard deviation for a Gaussian approximation
    of a binomial distribution is given by $\sqrt{np(1 - p)}$, and by considering fair coins/dice,
    we notice that the standard deviations of the distributions are equal for:
    \[
        \sqrt{1000 \cdot \frac{1}{6} \cdot \frac{5}{6}} = \sqrt{\frac{n}{4}}
        \iff \frac{50\sqrt{2}}{6} = \frac{\sqrt{n}}{2}
        \iff 50\sqrt{2} = 3\sqrt{n} \iff n = \frac{5000}{9} \approx 556
    \]

    Therefore, we need to flip 556 coins to obtain a very close Gaussian distribution to
    Fig. 5.15.
\end{proof}

\section*{5.5. Gambler's fallacy}
\addcontentsline{toc}{section}{5.5. Gambler's fallacy}
Assume that after 20 coin flips, you have obtained only five Heads. The
probability of this happening is small (about 1.5\%, since $\binom{20}{5} / 2^{20} = 0.0148$),
but not negligible. Since the law of large numbers says that the fraction of Heads approaches
50\% as the number of flips gets large, should you expect to see more Heads and Tails in
future flips?

\vspace{1em}

\begin{proof}
    Since coin flips are independent events, we can't make any assumption about future flips
    based on the already done 20 flips. Also, we can't assume anything based on the law of large
    numbers, since it assumes that the number of trials goes to infinity, so any unusual trends could
    occur at any time, and their effect will be diminished as the number of trials increases.
\end{proof}

\section*{5.6. Finding the Gaussian}
\addcontentsline{toc}{section}{5.6. Finding the Gaussian}
What is the explicit form of the Gaussian function $f(x)$ that matches
up with the fourth histogram in Fig. 5.11? Assume that $n_t = 10$ is large
enough so that the Gaussian approximation does indeed hold.

\begin{figure}[H]
    \center{\includegraphics[width=0.6\linewidth]{figure5_15_histogram.png}}
    \caption*{\textbf{Figure 5.11}: Fourth Histogram}
\end{figure}

\vspace{1em}

\begin{proof}
    We assume that $n_t = 10$ is large enough so the Gaussian approximation holds.
    Let $X$ denote the outcome of a dice roll and let $Y$ represent the sum of 10 such outcomes,
    so $Y = X_1 + X_2 + \ldots X_{10}$, where the $X_i$s are distributed the same as $X$.
    One can easily show that $\sigma_X = 1.71$ and $\mu_X = 3.5$. The mean of $Y$ is easily
    obtained by using the linearity of expectation:
    \[
        \mu_Y = E[Y] = E\bigg[\sum_{i = 1}^{10}X_i\bigg] = 10E[X] = 10\mu_X = 35
    \]

    The variance of $Y$ is computed by also using the fact that the $X_i$ variables
    are independent, so the expectation of their product can be split:
    \begin{align*}
        {\sigma_Y}^2
        = E[Y^2] - {\mu_Y}^2
        &= E\bigg[\bigg(\sum_{i = 1}^{10} X_i\bigg)^2\bigg] - 100\mu_X^2
        = 10E[X^2] + 2\sum_{i = 1}^{10}\sum_{j = i + 1}^{10} E[X_i X_j] - 100\mu_X^2 \\
        &= 10E[X]^2 - 110\mu_X^2 + 10\mu_X^2
        = 10E[X]^2 - 10\mu_X^2 = 10\sigma_X^2 \approx 29.24
    \end{align*}

    From the Central Limit Theorem, we know that $Y$ will be the Gaussian approximation
    of the process described by the histogram. Therefore, since the probabilities
    represent the number of sums with a specific value (height of the columns), divided
    by the number of total sums ($n_s$), the Gaussian function associated to the histogram
    is given by the PDF of $Y$, scaled by $n_s$:
    \[
        f(x) = n_s G(y | \mu_Y, \sigma_Y^2)
        = \frac{10^5}{29.24\sqrt{2\pi}} e^{-\frac{(y - 35)^2}{2\cdot 29.24^2}}
    \]
\end{proof}

\section*{5.7. Standard deviations}
\addcontentsline{toc}{section}{5.7. Standard deviations}
Calculate the theoretically predicted standard deviations of the histograms in
Figs. 5.13 and 5.14, and check that your results are consistent with a visual
inspection of the histograms. You will need the result from Problem 4.3 for
Fig. 5.14.

\vspace{1em}

\begin{proof}
    We analyze the two figures separately:

    \vspace{1em}

    \textbf{Fig. 5.13} The histogram shows the result of taking $n_s = 10^5$ averages of
    $n_t = 100$ numbers chosen from the distribution in Fig. 5.12. Let $X$ be a
    random variable modeled by the distribution in Fig. 5.12. Then,
    \[
        P(X = 2) = 0.6 \hspace{2em} P(X = 3.1) = 0.1 \hspace{2em} P(X = 7) = 0.3
    \]

    Therefore, the mean of $X$ is
     \[
         \mu_X = E[X] = 2 \cdot 0.6 + 3.1 \cdot 0.1 + 7 \cdot 0.3 = 3.61
    \]
    while the variance is given by
    \[
        \sigma_X^2 = E[(X - \mu)^2] = (2 - 3.61)^2 \cdot 0.6 + (3.1 - 3.61)^2 \cdot 0.1 + (7 - 3.61)^2 \cdot 0.3
        \approx 5.26
    \]

    Now, let $Y$ denote the average of $100$ numbers distributed like $X$ (denoted by $X_i$), so
    \[
        Y = \frac{1}{100}\sum_{i = 1}^{100} X_i
    \]

    By using the linearity of expectation, the mean of $Y$ is
    \[
        \mu_Y = E[Y] = \frac{1}{100}E\bigg[\sum_{i = 1}^{100} X_i\bigg]
        = \frac{1}{100}\sum_{i = 1}^{100} E[X_i]
        = E[X] = \mu_X = 3.61
    \]

    and the variance is given by:
    \[
        \text{Var}(Y) = \text{Var}\bigg(\frac{1}{100} \sum_{i = 1}^{100} X_i\bigg)
        = \frac{1}{10^4} \text{Var}\bigg(\sum_{i = 1}^{100} X_i\bigg)
        = \frac{1}{10^4} \sum_{i = 1}^{100} \text{Var}(X_i)
        = \frac{1}{100} \text{Var}(X) = 0.0526
    \]
    which gives the standard deviation:
    \[
        \sigma_Y \approx 0.23
    \]

    \begin{figure}[H]%
        \centering
        \subfloat[\centering Histogram in Fig 5.13]{{\includegraphics[width=0.49\linewidth]{5_7_fig513.png} }}%
        \qquad
        \subfloat[\centering Gaussian with $\mu = 3.61, \sigma = 0.23$, scaled by $10^5$]{{\includegraphics[width=0.44\linewidth]{5_7_gauss2.png} }}%
    \end{figure}

    \vspace{1em}

    \textbf{Fig. 5.14.} The histogram portrays the results of taking $n_s = 10^5$ averages
    of $n_t = 50$ numbers chosen from a uniform distribution (from 0 to 1). Let X be
    distributed by that uniform distribution. The mean of the uniform distribution
    is just $\mu_X = 1/2$ and the variance is given by the result from Problem 4.3, i.e.
    $\sigma_X^2 = 1/12$. Now, let $Y$ denote the average of 50 numbers chosen from this
    distribution, so basically
    \[
        Y = \frac{1}{50} \sum_{i = 1}^{50} X_i
    \]
    where the $X_i$ variables are distributed like $X$.
    If we assume that $n_t = 50$ is large enough for the Central Limit Theorem to apply,
    we find that $Y$ is modeled by a Gaussian distribution. Note that the 1/50 scaler
    does not change this, because a scaled Gaussian is still a Gaussian. Therefore, by
    using the linearity of expectation, we get the mean of $Y$:
    \[
        \mu_Y = E[Y] = \frac{1}{50}E\bigg[\sum_{i = 1}^{50} X_i\bigg]
        = \frac{1}{50}\sum_{i = 1}^{50} E[X_i]
        = E[X] = \mu_X = \frac{1}{2}
    \]

    Similarly, by using the fact that the variance of a sum of random variables is equal
    to the sum of variances (for independent variables), we have that:
    \[
        \text{Var}(Y) = \text{Var}\bigg(\frac{1}{50} \sum_{i = 1}^{50} X_i\bigg)
        = \frac{1}{50^2} \text{Var}\bigg(\sum_{i = 1}^{50} X_i\bigg)
        = \frac{1}{50^2} \sum_{i = 1}^{50} \text{Var}(X_i)
        = \frac{1}{50} \text{Var}(X) = \frac{1}{600}
    \]
    which gives the standard deviation
    \[
        \sigma_Y = \frac{1}{10\sqrt{6}}
    \]

    We can confirm that this result is valid by comparing the plot of the corresponding Gaussian
    function in the figure with the histogram and seeing that they are are very similar:
    \begin{figure}[H]%
        \centering
        \subfloat[\centering Histogram in Fig 5.14]{{\includegraphics[width=0.49\linewidth]{5_7_fig514.png} }}%
        \qquad
        \subfloat[\centering Gaussian with $\mu = 1/2, \sigma = \frac{1}{10\sqrt{6}}$, scaled by $10^5$]{{\includegraphics[width=0.44\linewidth]{5_7_gauss1.png} }}%
    \end{figure}
\end{proof}


%
% Chapter 1.2
%

\section*{1.2 Types of Functions}

A linear function can be written in \textbf{slope-intercept form} (\(y=mx+b\)) where \(m\) is the slope of the line and \(b\) is the y-intercept.
\\\\
A function \(P\) is a \textbf{polynomial} if
$$P(x)=a_n{x}^n+a_{n-1}x^{n-1}+ \cdots +a_2x^2+a_1x+a_0$$
where n is a non-negative integer and the numbers \(a_0, a_1, a_2, \ldots ,a_n\) are constants called the \textbf{coefficients} of the polynomial. The domain of any polynomial is \(\mathbb{R} = (-\infty, \infty)\). If the coefficient \(a_n \neq 0\), then the degree of the polynomial is \(n\).
\\\\
A polynomial of degree 2 is of the form \(P(x)=ax^2+bx+c\) and is called a \textbf{quadratic function}. Its graph is always a parabola that opens upward if \(a > 0\) and downward if \(a < 0\).
\\\\
A polynomial of degree 3 is of the form
\[ P(x)=ax^3+bx^2+cx+d \text{ where } a \neq 0 \]
and is called a \textbf{cubic function}.
\\\\
A function of the form \(f(x) = x^a\), where \(a\) is a constant, is called a \textbf{power function}.
\\\\
A \textbf{rational function} \(f\) is a ratio of two polynomials:
\[ f(x)=\frac{P(x)}{Q(x)} \]
where \(P\) and \(Q\) are polynomials. The domain consists of all \(x\)-values such that \( Q(x) \neq 0 \).

\subsection*{Graphs of Basic Trigonometric Functions}

\begin{center}
\begin{tikzpicture}
\begin{axis}[
    %axis lines = bottom,
    grid=both,
    ymin=-4,
    ymax=4,
    xmax=5,
    xmin=-5
]

% Sine function
\addplot [
    domain=-5:5,
    samples=100
]
{sin(deg(x))};
\addlegendentry{$sin(x)$}

% Cosine function
\addplot [
    domain=-5:5,
    samples=100,
    dashed
]
{cos(deg(x))};
\addlegendentry{$cos(x)$}

% Cosine function
\addplot [
    domain=-5:5,
    samples=100,
    dotted
]
{tan(deg(x))};
\addlegendentry{$tan(x)$}

\end{axis}
\end{tikzpicture}
\end{center}
Notice that for both sine and cosine functions the domain is \((-\infty, \infty)\) and the range is the closed interval \([-1, 1]\). In addition, sine and cosine functions are periodic functions with a period of 2\(\pi\).
$$\sin(x+2\pi)=\sin(x) \quad \quad \cos(x+2\pi)=\cos(x)$$
\textbf{Exponential functions} are functions of the form \(f(x)=b^x\) where the base \(b\) is a positive constant.
\\\\
\textbf{Logarithmic functions} are functions written as \(f(x)=\log_b(x)\), where the base \(b\) is a positive constant, are the inverse functions of exponential functions.