\chapter{Metric spaces}
\label{ch:metric_space}
At the time of writing, I'm convinced that metric topology is
the morally correct way to motivate point-set topology
as well as to generalize normal calculus.\footnote{Also,
``metric'' is a fun word to say.}
So here is my best attempt.

The concept of a metric space is very ``concrete'', and lends itself easily to visualization. Hence throughout this chapter you should draw lots of pictures as you learn about new objects, like convergent sequences, open sets, closed sets, and so on.

\section{Definition and examples of metric spaces}
\prototype{$\RR^2$, with the Euclidean metric.}
\begin{definition}
	A \vocab{metric space} is a pair $(M, d)$ consisting of
	a set of points $M$
	and a \vocab{metric} $d : M \times M \to \mathbb R_{\ge 0}$.
	The distance function must obey:
	\begin{itemize}
		\ii For any $x,y \in M$, we have $d(x,y) = d(y,x)$; i.e.\ $d$ is symmetric.
		\ii The function $d$ must be \vocab{positive definite}
		which means that $d(x,y) \ge 0$ with equality if and only if $x=y$.
		\ii The function $d$ should satisfy the \vocab{triangle inequality}: for all $x,y,z \in M$,
		\[ d(x,z) + d(z,y) \ge d(x,y). \]
	\end{itemize}
\end{definition}
\begin{abuse}
	Just like with groups, we will abbreviate $(M,d)$ as just $M$.
\end{abuse}
\begin{example}[Metric spaces of $\RR$]
	\listhack
	\begin{enumerate}[(a)]
		\ii The real line $\RR$ is a metric space under the metric $d(x,y) = \left\lvert x-y \right\rvert$.
		\ii The interval $[0,1]$ is also a metric space with the same distance function.
		\ii In fact, any subset $S$ of $\RR$ can be made into a metric space in this way.
	\end{enumerate}
\end{example}
\begin{example}[Metric spaces of $\RR^2$]
	\listhack
	\begin{enumerate}[(a)]
		\ii We can make $\RR^2$ into a metric space by imposing the Euclidean distance function
		\[ d\left( (x_1, y_1), (x_2, y_2) \right) = \sqrt{(x_1-x_2)^2 + (y_1-y_2)^2}. \]
		\ii Just like with the first example, any subset of $\RR^2$ also becomes a metric space after we inherit it.
		The unit disk, unit circle, and the unit square $[0,1]^2$
		are special cases.
	\end{enumerate}
\end{example}
\begin{example}[Taxicab on $\RR^2$]
	It is also possible to place the \vocab{taxicab distance} on $\RR^2$:
	\[ d\left( (x_1, y_1), (x_2, y_2) \right) =
		\left\lvert x_1-x_2 \right\rvert + \left\lvert y_1-y_2 \right\rvert.
		\]
	For now, we will use the more natural Euclidean metric.
\end{example}

\begin{example}[Metric spaces of $\RR^n$]
	We can generalize the above examples easily.
	Let $n$ be a positive integer.
	\begin{enumerate}[(a)]
		\ii We let $\RR^n$ be the metric space whose points are points in $n$-dimensional Euclidean space,
		and whose metric is the Euclidean metric
		\[
			d\left(
			\left( a_1, \dots, a_n \right), \left( b_1, \dots, b_n \right)
			\right)
			= \sqrt{(a_1-b_1)^2 + \dots + (a_n-b_n)^2}.
		\]
		This is the $n$-dimensional \vocab{Euclidean space}.
		\ii The open \vocab{unit ball} $B^{n}$ is the subset of $\RR^n$
		consisting of those points $\left( x_1, \dots, x_n \right)$
		such that $x_1^2 + \dots + x_n^2 < 1$.
		\ii The \vocab{unit sphere} $S^{n-1}$ is the subset of $\RR^n$
		consisting of those points $\left( x_1, \dots, x_n \right)$
		such that $x_1^2 + \dots + x_n^2 = 1$, with the inherited metric.
		(The superscript $n-1$ indicates that $S^{n-1}$ is an $n-1$ dimensional space,
		even though it lives in $n$-dimensional space.)
		For example, $S^1 \subseteq \RR^2$ is the unit circle,
		whose distance between two points is the length of the chord joining them.
		You can also think of it as the ``boundary'' of the unit ball $B^n$.
	\end{enumerate}
\end{example}
\begin{example}
	[Function space]
	We can let $M$ be the space of
	continuous functions $f \colon [0,1] \to \RR$ and define the metric
	by $d(f,g) = \int_0^1 \left\lvert f-g \right\rvert \; dx$.
	(It admittedly takes some work to check $d(f,g) = 0$ implies $f=g$,
	but we won't worry about that yet.)
\end{example}

Here is a slightly more pathological example.
\begin{example}
	[Discrete space]
	Let $S$ be any set of points (either finite or infinite).
	We can make $S$ into a \vocab{discrete space} by declaring
	\[
		d(x,y)
		=
		\begin{cases}
			1 & \text{if $x \neq y$} \\
			0 & \text{if $x = y$}.
		\end{cases}
	\]
	If $\left\lvert S \right\rvert = 4$ you might think of this space
	as the vertices of a regular tetrahedron, living in $\RR^3$.
	But for larger $S$ it's not so easy to visualize\dots
\end{example}
\begin{example}[Graphs are metric spaces]
	Any connected simple graph $G$ can be made into a metric space
	by defining the distance between two vertices to be the
	graph-theoretic distance between them.
	(The discrete metric is the special case when $G$ is the complete graph on $S$.)
\end{example}
\begin{ques}
	Check the conditions of a metric space for the metrics on the discrete space
	and for the connected graph.
\end{ques}

\begin{abuse}
	From now on, we will refer to $\RR^n$ with the Euclidean metric
	by just $\RR^n$.
	Moreover, if we wish to take the metric space for a subset $S \subseteq \RR^n$
	with the inherited metric, we will just write $S$.
\end{abuse}

\section{Convergence in metric spaces}
\prototype{The sequence $\frac1n$ (for $n=1,2,\dots$) in $\RR$.}

Since we can talk about the distance between two points, we can talk about what it means for a sequence of points to converge.
This is the same as the typical epsilon-delta definition, with absolute values replaced by the distance function.

\begin{definition}
	Let $(x_n)_{n \ge 1}$ be a sequence of points in a metric space $M$.
	We say that $x_n$ \vocab{converges} to $x$ if the following condition holds:
	for all $\eps > 0$, there is an integer $N$ (depending on $\eps$)
	such that $d(x_n, x) < \eps$ for each $n \ge N$.
	This is written \[ x_n \to x \] or more verbosely as \[ \lim_{n \to \infty} x_n = x. \]
	We say that a sequence converges in $M$ if it converges to a point in $M$.
\end{definition}
You should check that this definition coincides with your intuitive notion of ``converges''.
\begin{abuse}
	If the parent space $M$ is understood, we will allow ourselves
	to abbreviate ``converges in $M$'' to just ``converges''.
	However, keep in mind that convergence is defined relative to the parent space;
	the ``limit'' of the space must actually be a point in $M$ for a sequence to converge.
\end{abuse}

\begin{center}
	\begin{asy}
		size(9cm);
		Drawing("x_1", (-9,0.1), dir(90));
		Drawing("x_2", (-6,0.8), dir(90));
		Drawing("x_3", (-5,-0.3), dir(90));
		Drawing("x_4", (-2, 0.8), dir(90));
		Drawing("x_5", (-1.7, -0.7), dir(-90));
		Drawing("x_6", (-0.6, -0.3), dir(225));
		Drawing("x_7", (-0.4, 0.3), dir(90));
		Drawing("x_8", (-0.25, -0.24), dir(-90));
		Drawing("x_9", (-0.12, 0.1), dir(45));
		dot("$x$", (0,0), dir(-45), blue);
		draw(CR(origin, 1.5), blue+dashed);
	\end{asy}
\end{center}

\begin{example}
	Consider the sequence
	$x_1 = 1$, $x_2 = 1.4$, $x_3 = 1.41$, $x_4 = 1.414$, \dots.
	\begin{enumerate}[(a)]
		\ii If we view this as a sequence in $\RR$, it converges to $\sqrt 2$.
		\ii However, even though each $x_i$ is in $\QQ$,
		this sequence does NOT converge when we view it as a sequence in $\QQ$!
	\end{enumerate}
\end{example}

\begin{ques}
	What are the convergent sequences in a discrete metric space?
\end{ques}

\section{Continuous maps}
In calculus you were also told (or have at least heard)
of what it means for a function to be continuous.
Probably something like
\begin{quote}
	A function $f \colon \RR \to \RR$
	is continuous at a point $p \in \RR$
	if for every $\eps > 0$ there exists a $\delta > 0$ such that
	$\left\lvert x-p \right\rvert < \delta
		\implies
		\left\lvert f(x) - f(p) \right\rvert < \eps$.
\end{quote}
\begin{ques}
	Can you guess what the corresponding definition for metric spaces is?
\end{ques}

All we have do is replace the absolute values with the more general distance functions: this gives us a definition of continuity for any function $M \to N$.

\begin{definition}
	Let $M = (M, d_M)$ and $N = (N, d_N)$ be metric spaces.
	A function $f \colon M \to N$ is \vocab{continuous}
	at a point $p \in M$
	if for every $\eps > 0$ there exists a $\delta > 0$ such that
	\[ d_M(x,p) < \delta \implies d_N(f(x), f(p)) < \eps. \]
	Moreover, the entire function $f$ is continuous
	if it is continuous at every point $p \in M$.
\end{definition}
Notice that, just like in our definition of an isomorphism of a group,
we use the metric of $M$ for one condition
and the metric of $N$ for the other condition.

This generalization is nice because it tells us immediately how we could carry over continuity arguments in $\RR$ to more general spaces like $\CC$.
Nonetheless, this definition is kind of cumbersome to work with,
because it makes extensive use of the real numbers
(epsilons and deltas).
Here is an equivalent condition.
\begin{theorem}[Sequential continuity]
	\label{thm:seq_cont}
	A function $f \colon M \to N$ of metric spaces
	is continuous at a point $p \in M$
	if and only if the following property holds:
	if $x_1$, $x_2$, \dots is a sequence in $M$ converging to $p$,
	then the sequence $f(x_1)$, $f(x_2)$, \dots in $N$ converges to $f(p)$.
\end{theorem}
\begin{proof}
	One direction is not too hard:
	\begin{exercise}
		Show that $\eps$-$\delta$ continuity implies sequential continuity
		at each point.
	\end{exercise}
	Conversely, we will prove if $f$ is not $\eps$-$\delta$ continuous at $p$
	then it does not preserve convergence.

	If $f$ is not continuous at $p$, then there is a ``bad'' $\eps > 0$,
	which we now consider fixed.
	So for each choice of $\delta = 1/n$,
	there should be some point $x_n$ which is within $\delta$ of $p$,
	but which is mapped more than $\eps$ away from $f(p)$.
	But then the sequence $x_n$ converges to $p$,
	and $f(x_n)$ is always at least $\eps$ away from $f(p)$, contradiction.
\end{proof}

Example application showcasing the niceness of sequential continuity:
\begin{proposition}[Composition of continuous functions is continuous]
	Let $f \colon M \to N$ and $g \colon N \to L$ be continuous maps of metric spaces.
	Then their composition $g \circ f$ is continuous.
\end{proposition}
\begin{proof}
	Dead simple with sequences:
	Let $p \in M$ be arbitrary and let $x_n \to p$ in $M$.
	Then $f(x_n) \to f(p)$ in $N$ and $g(f(x_n)) \to g(f(p))$ in $L$, QED.
\end{proof}

\begin{ques}
	Let $M$ be any metric space and $D$ a discrete space.
	When is a map $f \colon D \to M$ continuous?
\end{ques}

\section{Homeomorphisms}
\prototype{The unit circle $S^1$ is homeomorphic to the boundary of the unit square.}

When do we consider two groups to be the same?
Answer: if there's a structure-preserving map
between them which is also a bijection.
For metric spaces, we do exactly the same thing,
but replace ``structure-preserving'' with ``continuous''.

\begin{definition}
	Let $M$ and $N$ be metric spaces.
	A function $f \colon M \to N$ is a
	\vocab{homeomorphism} if it is a bijection,
	and both $f \colon M \to N$
	and its inverse $f\inv \colon N \to M$ are continuous.
	We say $M$ and $N$ are \vocab{homeomorphic}.
\end{definition}
Needless to say, homeomorphism is an equivalence relation.

You might be surprised that we require $f\inv$ to also be continuous.
Here's the reason: you can show that if $\phi$ is
an isomorphism of groups, then $\phi\inv$ also preserves the group operation,
hence $\phi\inv$ is itself an isomorphism.
The same is not true for continuous bijections,
which is why we need the new condition.
\begin{example}
	[Homeomorphism $\neq$ continuous bijection]
	\listhack
	\begin{enumerate}[(a)]
		\ii There is a continuous bijection
		from $[0,1)$ to the circle, % chktex 9
		but it has no continuous inverse.
		\ii Let $M$ be a discrete space with size $|\RR|$.
		Then there is a continuous function $M \to \RR$
		which certainly has no continuous inverse.
	\end{enumerate}
\end{example}

Note that this is the topologist's definition of ``same'' --
homeomorphisms are ``continuous deformations''.
Here are some examples.

\begin{example}[Examples of homeomorphisms]
	\listhack
	\begin{enumerate}[(a)]
		\ii Any space $M$ is homeomorphic to itself
		through the identity map.
		\ii The old saying: a doughnut (torus) is
		homeomorphic to a coffee cup.
		(Look this up if you haven't heard of it.)
		\ii The unit circle $S^1$ is homeomorphic
		to the boundary of the unit square.
		Here's one bijection between them, after an appropriate scaling:
		\begin{center}
			\begin{asy}
				size(1.5cm);
				draw(unitcircle);
				pair A = (1.4, 1.4);
				pair B = rotate(90)*A;
				pair C = rotate(90)*B;
				pair D = rotate(90)*C;
				draw(A--B--C--D--cycle);
				dot(origin);
				pair P = Drawing(dir(70));
				pair Q = Drawing(extension(origin, P, A, B));
				draw(origin--Q, dashed);
			\end{asy}
		\end{center}
	\end{enumerate}
\end{example}
\begin{example}
	[Metrics on the unit circle]
	It may have seemed strange that our metric function on $S^1$
	was the one inherited from $\RR^2$,
	meaning the distance between two points
	on the circle was defined to be the length of the chord.
	Wouldn't it have made more sense to use the circumference of the
	smaller arc joining the two points?

	In fact, it doesn't matter:
	if we consider $S^1$ with the ``chord'' metric
	and the ``arc'' metric, we get two homeomorphic spaces,
	as the map between them is continuous.

	The same goes for $S^{n-1}$ for general $n$.
\end{example}

\begin{example}
	[Homeomorphisms really don't preserve size]
	Surprisingly, the open interval $(-1,1)$
	is homeomorphic to the real line $\RR$!
	One bijection is given by
	\[ x \mapsto \tan(\pi/2 x) \]
	with the inverse being given by $t \mapsto \frac2\pi \arctan(t)$.

	This might come as a surprise,
	since $(-1,1)$ doesn't look that much like $\RR$;
	the former is ``bounded'' while the latter is ``unbounded''.
\end{example}

\section{Extended example/definition: product metric}
\prototype{$\RR \times \RR$ is $\RR^2$.}

Here is an extended example which will occur later on.
Let $M = (M, d_M)$ and $N = (N, d_N)$ be metric spaces (say, $M = N = \RR$).
Our goal is to define a metric space on $M \times N$.

Let $p_i = (x_i,y_i) \in M \times N$ for $i=1,2$.
Consider the following metrics on the set of points $M \times N$:
\begin{align*}
	d_{\text{max}} ( p_1, p_2 )
		&\defeq \max \left\{ d_M(x_1, x_2), d_N(y_1, y_2) \right\}  \\
	d_{\text{Euclid}} ( p_1, p_2 )
		&\defeq \sqrt{d_M(x_1,x_2)^2 + d_N(y_1, y_2)^2} \\
	d_{\text{taxicab}} \left( p_1, p_2 \right)
		&\defeq d_M(x_1, x_2) + d_N(y_1, y_2).
\end{align*}
All of these are good candidates.
We are about to see it doesn't matter which one we use:
\begin{exercise}
Verify that
\[ d_{\text{max}}(p_1,p_2)
	\le d_{\text{Euclid}}(p_1, p_2)
	\le d_{\text{taxicab}}(p_1, p_2)
	\le 2d_{\text{max}}(p_1, p_2). \]
Use this to show that the metric spaces we obtain
by imposing any of the three metrics are homeomorphic,
with the homeomorphism being just the identity map.
\end{exercise}

\begin{definition}
	Hence we will usually simply refer to
	\emph{the} metric on $M \times N$,
	called the \vocab{product metric}.
	It will not be important which of the three metrics we select.
\end{definition}

\begin{example}[$\RR^2$]
	If $M = N = \RR$, we get $\RR^2$, the Euclidean plane.
	The metric $d_{\text{Euclid}}$ is the one we started with,
	but using either of the other two metric works fine as well.
\end{example}

The product metric plays well with convergence of sequences.
\begin{proposition}
	[Convergence in the product metric is by component]
	We have $(x_n, y_n) \to (x,y)$
	if and only if $x_n \to x$ and $y_n \to y$.
\end{proposition}
\begin{proof}
	We have $d_{\text{max}} \left( (x,y), (x_n, y_n) \right)
	= \max\left\{ d_M(x, x_n), d_N(y, y_n) \right\}$
	and the latter approaches zero as $n \to \infty$
	if and only if $d_M(x,x_n) \to 0$ and $d_N(y, y_n) \to 0$.
\end{proof}

Let's see an application of this:
\begin{proposition}
	[Addition and multiplication are continuous]
	\label{prop:arithmetic_continuous}
	The addition and multiplication maps
	are continuous maps $\RR \times \RR \to \RR$.
\end{proposition}
\begin{proof}
	For multiplication: for any $n$ we have
	\begin{align*}
		x_n y_n &= \left( x + (x_n-x) \right)
		\left( y + (y_n-y) \right) \\
		&= xy + y(x_n-x) + x(y_n-y) + (x_n-x)(y_n-y) \\
		\implies \left\lvert x_n y_n - xy \right\rvert
		&\le \left\lvert y \right\rvert
			\left\lvert x_n - x \right\rvert
		+ \left\lvert x \right\rvert
			\left\lvert y_n - y \right\rvert
		+ \left\lvert x_n - x \right\rvert
			\left\lvert y_n - y \right\rvert.
	\end{align*}
	As $n \to \infty$, all three terms on the
	right-hand side tend to zero.
	The proof that $+ \colon \RR \times \RR \to \RR$ is continuous
	is similar (and easier): one notes for any $n$ that
	\[ |(x_n + y_n) - (x+y)| \le |x_n-x| + |y_n-y| \]
	and both terms on the right-hand side
	tend to zero as $n \to \infty$.
\end{proof}
\Cref{prob:subtract_divide} covers the other two operations,
subtraction and division.
The upshot of this is that, since compositions are also continuous,
most of your naturally arising real-valued functions
will automatically be continuous as well.
For example, the function $\frac{3x}{x^2+1}$
will be a continuous function from $\RR \to \RR$,
since it can be obtained by composing $+$, $\times$, $\div$.

\section{Open sets}
\prototype{The open disk $x^2+y^2<r^2$ in $\RR^2$.}

Continuity is really about what happens ``locally'':
how a function behaves ``close to a certain point $p$''.
One way to capture this notion of ``closeness''
is to use metrics as we've done above.
In this way we can define an $r$-neighborhood of a point.

\begin{definition}
	Let $M$ be a metric space.
	For each real number $r > 0$ and point $p \in M$, we define
	\[ M_r(p) \defeq \left\{ x \in M: d(x,p) < r \right\}. \]
	The set $M_r(p)$ is called an \vocab{$r$-neighborhood} of $p$.
\end{definition}
\begin{center}
	\begin{asy}
		size(4cm);
		bigblob("$M$");
		pair p = Drawing("p", (0.3,0.1), dir(-90));
		real r = 1.8;
		draw(CR(p,r), dashed);
		label("$M_r(p)$", p+r*dir(-65), dir(-65));
		draw(p--(p+r*dir(130)));
		label("$r$", midpoint(p--(p+r*dir(130))), dir(40));
	\end{asy}
\end{center}

We can rephrase convergence more succinctly in terms of $r$-neighborhoods.
Specifically, a sequence $(x_n)$ converges to $x$
if for every $r$-neighborhood of $x$,
all terms of $x_n$ eventually stay within that $r$-neighborhood.

Let's try to do the same with functions.
\begin{ques}
	In terms of $r$-neighborhoods,
	what does it mean for a function $f \colon M \to N$
	to be continuous at a point $p \in M$?
\end{ques}

Essentially, we require that the pre-image of every $\eps$-neighborhood has
the property that some $\delta$-neighborhood exists inside it.
This motivates:
\begin{definition}
	A set $U \subseteq M$ is \vocab{open} in $M$
	if for each $p \in U$, some $r$-neighborhood of $p$
	is contained inside $U$.
	In other words, there exists $r>0$ such that $M_r(p) \subseteq U$.
\end{definition}
\begin{abuse}
	Note that a set being open is defined
	\emph{relative to} the parent space $M$.
	However, if $M$ is understood we can abbreviate
	``open in $M$'' to just ``open''.
\end{abuse}

\begin{figure}[ht]
	\centering
	\begin{asy}
		size(5cm);
		draw(unitcircle, dashed);
		pair P = Drawing("p", (0.6,0.2), dir(-90));
		draw(CR(P, 0.3), dotted);
		MP("x^2+y^2<1", dir(45), dir(45));
	\end{asy}
	\caption{The set of points $x^2+y^2<1$ in $\RR^2$ is open in $\RR^2$.}
	\label{fig:example_open}
\end{figure}

\begin{example}[Examples of open sets]
	\listhack
	\begin{enumerate}[(a)]
		\ii Any $r$-neighborhood of a point is open.
		\ii Open intervals of $\RR$ are open in $\RR$, hence the name!
		This is the prototypical example to keep in mind.
		\ii The open unit ball $B^n$ is open in $\RR^n$ for the same reason.
		\ii In particular, the open interval $(0,1)$ is open in $\RR$.
		However, if we embed it in $\RR^2$, it is no longer open!
		\ii The empty set $\varnothing$ and the whole set of points $M$ are open in $M$.
	\end{enumerate}
\end{example}
\begin{example}
	[Non-examples of open sets]
	\listhack
	\begin{enumerate}[(a)]
		\ii The closed interval $[0,1]$ is not open in $\RR$.
		There is no $\eps$-neighborhood of the point $0$
		which is contained in $[0,1]$.
		\ii The unit circle $S^1$ is not open in $\RR^2$.
	\end{enumerate}
\end{example}
\begin{ques}
	What are the open sets of the discrete space?
\end{ques}

Here are two quite important properties of open sets.
\begin{proposition}
	[Intersections and unions of open sets]
	\listhack
	\begin{enumerate}[(a)]
		\ii The intersection of finitely many open sets is open.
		\ii The union of open sets is open, even if there are infinitely many.
	\end{enumerate}
\end{proposition}
\begin{ques}
	Convince yourself this is true.
\end{ques}
\begin{exercise}
	Exhibit an infinite collection of open sets in $\RR$
	whose intersection is the set $\{0\}$.
	This implies that infinite intersections of open sets are not necessarily open.
\end{exercise}

The whole upshot of this is:
\begin{theorem}[Open set condition]
	\label{thm:open_set}
	A function $f \colon M \to N$ of metric spaces is continuous
	if and only if the pre-image of every open set in $N$ is open in $M$.
\end{theorem}
\begin{proof}
	I'll just do one direction\dots
	\begin{exercise}
		Show that $\delta$-$\eps$ continuity follows from
		the open set continuity.
	\end{exercise}
	Now assume $f$ is continuous.
	First, suppose $V$ is an open subset of the metric space $N$;
	let $U = f\pre(V)$. Pick $x \in U$, so $y = f(x) \in V$; we want an
	open neighborhood of $x$ inside $U$.

	\begin{center}
		\begin{asy}
			size(12cm);
			bigblob("$N$");
			pair Y = Drawing("y", origin, dir(75));
			real eps = 1.5;
			draw(CR(Y, eps), dotted);
			label("$\varepsilon$", Drawing(Y--(Y+eps*dir(255))));
			label("$V$",
				Drawing(shift(-0.5,0)*rotate(190)*scale(3.2,2.8)*unitcircle, dashed));
			add(shift( (13,0) ) * CC());
			label("$f$", Drawing( (4.5,0)--(8,0), EndArrow));

			bigblob("$M$");
			real delta = 1.1;
			pair X = Drawing("x", (-1.5,-0.5), dir(-45));
			label("$\delta$", Drawing(X--(X+delta*dir(155))));
			draw(CR(X, delta), dotted);
			label("$U = f^{\text{pre}}(V)$",
				Drawing(shift(-1.5,-0.3)*rotate(235)*scale(2.4,1.8)*unitcircle, dashed));
		\end{asy}
	\end{center}

	As $V$ is open, there is some small $\eps$-neighborhood around $y$
	which is contained inside $V$.
	By continuity of $f$, we can find a $\delta$ such that the $\delta$-neighborhood
	of $x$ gets mapped by $f$ into the $\eps$-neighborhood in $N$,
	which in particular lives inside $V$.
	Thus the $\delta$-neighborhood lives in $U$, as desired.
\end{proof}

%From this we can get a new definition of homeomorphism
%which makes it clear why open sets are good things to consider.
%\begin{theorem}
%	[Open set formulation of homeomorphism]
%	A function $f \colon M \to N$ of metric spaces is a
%	homeomorphism if and only if
%	\begin{enumerate}[(i)]
%		\ii It is a bijection of the underlying points.
%		\ii It induces a bijection of the open sets of $M$ and $N$:
%		for any open set $U \subseteq M$ the set $f\im(U)$ is open,
%		and for any open set $V \subseteq N$ the set $f\pre(V)$ is open.
%	\end{enumerate}
%\end{theorem}

\section{Closed sets}
\prototype{The closed unit disk $x^2+y^2 \le r^2$ in $\RR^2$.}
It would be criminal for me to talk about open sets
without talking about closed sets.
The name ``closed'' comes from the definition in a metric space.
\begin{definition}
	Let $M$ be a metric space.
	A subset $S \subseteq M$ is \vocab{closed} in $M$ if the following property holds:
	let $x_1$, $x_2$, \dots be a sequence of points in $S$
	and suppose that $x_n$ converges to $x$ in $M$.
	Then $x \in S$ as well.
\end{definition}
\begin{abuse}
	Same caveat: we abbreviate ``closed in $M$'' to just ``closed''
	if the parent space $M$ is understood.
\end{abuse}
Here's another way to phrase it.
The \vocab{limit points} of a subset $S \subseteq M$ are defined by
\[ \lim S \defeq \left\{ p \in M :
	\exists (x_n) \in S \text{ such that } x_n \to p \right\}. \]
Thus $S$ is closed if and only if $S = \lim S$.

\begin{exercise}
	Prove that $\lim S$ is closed even if $S$ isn't closed. (Draw a picture.)
\end{exercise}
For this reason, $\lim S$ is also called the
\vocab{closure} of $S$ in $M$, and denoted $\ol S$.
It is simply the smallest closed set which contains $S$.

\begin{example}
	[Examples of closed sets]
	\listhack
	\begin{enumerate}[(a)]
		\ii The empty set $\varnothing$ is closed in $M$ for vacuous reasons:
		there are no sequences of points with elements in $\varnothing$.
		\ii The entire space $M$ is closed in $M$ for tautological reasons.
		(Verify this!)
		\ii The closed interval $[0,1]$ in $\RR$ is closed in $\RR$, hence the name.  Like with open sets, this is the prototypical example of a closed set to keep in mind!
		\ii In fact, the closed interval $[0,1]$ is even closed in $\RR^2$.
	\end{enumerate}
\end{example}
\begin{example}
	[Non-examples of closed sets]
	Let $S=(0,1)$ denote the open interval.
	Then $S$ is not closed in $\RR$
	because the sequence of points
	\[
		\frac12, \;
		\frac14, \;
		\frac18, \;
		\dots
	\]
	converges to $0 \in \RR$, but $0 \notin (0,1)$.
\end{example}

I should now warn you about a confusing part of this terminology.
Firstly, \textbf{``most'' sets are neither open nor closed}.
\begin{example}[A set neither open nor closed]
	The half-open interval $[0,1)$ %chktex 9
	is neither open nor closed in $\RR$.
\end{example}
Secondly, it's \textbf{also possible for a set to be both open and closed};
this will be discussed in \Cref{ch:top_more}.

The reason for the opposing terms is the following theorem:
\begin{theorem}[Closed sets are complements of open sets]
	Let $M$ be a metric space, and $S \subseteq M$ any subset.
	Then the following are equivalent:
	\begin{itemize}
		\ii The set $S$ is closed in $M$.
		\ii The complement $M \setminus S$ is open in $M$.
	\end{itemize}
\end{theorem}
\begin{exercise}
	[Great]
	Prove this theorem!
	You'll want to draw a picture to make it clear what's happening:
	for example, you might take $M = \RR^2$ and $S$ to be the closed unit disk.
\end{exercise}

\section{\problemhead}
\begin{problem}
	Let $M = (M,d)$ be a metric space.
	Show that \[ d \colon M \times M \to \RR \]
	is itself a continuous function
	(where $M \times M$ is equipped with the product metric).
\end{problem}

\begin{problem}
	Are $\QQ$ and $\NN$ homeomorphic subspaces of $\RR$?
	\begin{hint}
		No.
	\end{hint}
	\begin{sol}
		In $\QQ$, no singleton set is open,
		whereas in $\NN$, they all are (in fact $\NN$ is discrete).
	\end{sol}
\end{problem}

\begin{problem}
	[Continuity of arithmetic continued]
	\label{prob:subtract_divide}
	Show that subtraction is a continuous map $- \colon \RR \times \RR \to \RR$,
	and division is a continuous map $\div \colon \RR \times \RR_{>0} \to \RR$.
	\begin{hint}
		You can do this with bare hands.
		You can also use composition.
	\end{hint}
	\begin{sol}
		For subtraction, the map $x \mapsto -x$ is continuous
		so you can view it as a composed map
		\begin{center}
		\begin{tikzcd}
			\RR \times \RR \ar[r, "{(\id,-x)}"]
				& \RR \times \RR \ar[r, "+"] & \RR \\[1ex]
			(a,b) \ar[r, mapsto] & (a,-b) \ar[r, mapsto] &  a-b.
		\end{tikzcd}
		\end{center}
		Similarly, if you are willing to believe $x \mapsto 1/x$
		is a continuous function, then division is composition
		\begin{center}
		\begin{tikzcd}
			\RR \times \RR_{>0} \ar[r, "{(\id, 1/x)}"]
				& \RR \times \RR_{>0} \ar[r, "\times"] & \RR \\
			(a,b) \ar[r, mapsto] & (a,1/b) \ar[r, mapsto] & a/b.
		\end{tikzcd}
		\end{center}
		If for some reason you are suspicious that $x \mapsto 1/x$ is continuous,
		then here is a proof using sequential continuity.
		Suppose $x_n \to x$ with $x_n > 0$ and $x > 0$
		(since $x$ needs to be in $\RR_{>0}$ too).
		Then \[ \left\lvert \frac{1}{x} - \frac 1{x_n} \right\rvert
			= \frac{\left\lvert x_n-x \right\rvert}{\left\lvert x x_n \right\rvert}.
		\]
		If $n$ is large enough, then $\left\lvert x_n \right\rvert > x/2$;
		so the denominator is at least $x^2/2$,
		and hence the whole fraction is at most
		$\frac{2}{x^2} \left\lvert x_n-x \right\rvert$,
		which tends to zero as $n \to \infty$.
	\end{sol}
\end{problem}

\begin{problem}
	Exhibit a function $f \colon \RR \to \RR$ such that
	$f$ is continuous at $x \in \RR$ if and only if $x=0$.
	\begin{hint}
		$\pm x$ for good choices of $\pm$.
	\end{hint}
	\begin{sol}
		Let $f(x) = x$ for $x \in \QQ$ and $f(x) = -x$ for irrational $x$.
	\end{sol}
\end{problem}

\begin{problem}
	\yod
	Prove that a function $f \colon \RR \to \RR$ which is strictly increasing
	must be continuous at some point.
	\begin{hint}
		Project gaps onto the $y$-axis.
		Use the fact that uncountably many positive reals cannot have finite sum.
	\end{hint}
	\begin{sol}
		Assume for contradiction it is completely discontinuous;
		by scaling set $f(0) = 0$, $f(1) = 1$ and focus just on $f : [0,1] \to [0,1]$.
		Since it's discontinuous everywhere,
		for every $x \in [0,1]$ there's an $\eps_x > 0$
		such that the continuity condition fails.
		Since the function is strictly increasing,
		that can only happen if the
		function misses all $y$-values in the interval
		$(fx-\eps_x, fx)$ or $(fx, fx+\eps_x)$ (or both).

		Projecting these missing intervals to the $y$-axis you find uncountably
		many intervals (one for each $x \in [0,1]$) all of which are disjoint.
		In particular, summing the $\eps_x$ you get that a sum of uncountably
		many positive reals is $1$.

		But in general it isn't possible for an uncountable family $\mathcal F$
		of positive reals to have finite sum.
		Indeed, just classify the reals into buckets $\frac1k \le x < \frac1{k-1}$.
		If the sum is actually finite then each bucket is finite,
		so the collection $\mathcal F$ must be countable, contradiction.
	\end{sol}
\end{problem}


\subsection{Axiom of union}

\subsubsection{Motivation}

While we have described various sets, we have not said that they exist. That is, if \(A\) and \(B\) both exist, then currently we cannot ensure \(A\land B\) exists, just that it can be described.

The axiom of union enables us to ensure all sets from unions and intersections exist.

\subsubsection{Axiom of union}

\(\forall a\exists b \forall c [c\in b\leftrightarrow \exists d(c\in d\land d\in a)]\)

That is, for every set \(a\), there exists a set \(b\) where all the elements in \(b\) are the elements of the elements in \(a\).

Here, \(b\) is the union of the sets in \(a\).


\chapter{Logistic Regression and Regularization}

Now we turn away from regression to classification problems. Don't be confused by the name `logistic regression,' it's actually just named for the mathematical function and it's a common approach to classification.

\section{Binary Classification}
In classification problems, instead of our output being continuous, we expect it to fall into discrete classes. We'll start with the simplest case: binary classification. Here, we have our output variable $y \in \{0, 1\}$. Typically, we take $0$ as the negative class and $1$ as the positive class.

Consider the following example: we have a sample of eight tumors, and we want to determine if they're malignant based on the tumor size. These are plotted in Figure \ref{logreg-eg-maltumor-noregline}. One thing we can do is assume a linear relationship with hypothesis $h_\theta\left( \vec{x} \right) = \vec{\theta}^\intercal \vec{x}$. This shown in Figure \ref{logreg_eg1_maltumor_linreg1}.

\begin{figure}[h]
	\centering
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
  		\includegraphics[scale=0.3]{logreg_eg1_maltumor.pdf}
   		\caption[]{Sample tumor data.}
   		\label{logreg-eg-maltumor-noregline}
	\end{subfigure}
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
   		\includegraphics[scale=0.3]{logreg_eg1_maltumor_linreg1.pdf}
   		\caption[]{Plot of tumors by size with linear regression line $y =  \frac{x-1}{6} - 0.25$.}
   		\label{logreg_eg1_maltumor_linreg1}
	\end{subfigure}
	\caption[]{Plots of tumors by size.}
\end{figure}

To try and make predictions, we can threshold the output at $h_\theta\left( x \right) = 0.5$, and then:
\begin{itemize}
\item If $h_\theta\left(x\right) \geq 0.5$, then predict $y = 1$
\item If $h_\theta\left( x \right) < 0.5$, then predict $y = 0$
\end{itemize}
and you can see this in Figure \ref{logreg_eg1_maltumor_linreg1_threshold.pdf}.

\begin{figure}[h]
	\centering
	\graphicspath{{./Figures/}}
	\includegraphics[scale=0.4]{logreg_eg1_maltumor_linreg1_threshold.pdf}
	\caption[]{Linear regression plotted with classification regions. }
	\label{logreg_eg1_maltumor_linreg1_threshold.pdf}
\end{figure}

In this example, it would seem like linear regression is a good classifier. However, what if we add a new data point for a large tumor. Suddenly, our results look like this

\begin{figure}[h] %  figure placement: here, top, bottom, or page
	\centering
	\graphicspath{{./Figures/}} %Use this to import an image from a subfolder.
	\includegraphics[scale=0.4]{logreg_eg1_maltumor_linreg1_newpoint.pdf}
	\caption[]{Linear regression plotted with classification regions after a new data point is added. Notice how one of the malignant tumors is now being misclassified as benign.}
	\label{logreg_eg1_maltumor_linreg1_newpoint.pdf}
\end{figure}

and now, we have a malignant tumor being misclassified as benign. Ergo, maybe linear regression isn't the best way to build a binary classifier.

\section{Hypothesis Representation}
In linear regression, our hypothesis was $h_\theta\left(\vec{x}\right) = \vec{\theta}^\intercal \vec{x}$ For logistic regression, we want our hypothesis to satisfy $0 \leq h_\theta\left(x\right) \leq 1$. To do this, we use the sigmoid function.\footnote{This is also called the logistic function, and is the namesake for logistic regression.} To make this work, we modify our hypothesis to be
\begin{equation}
h_\theta\left(x\right) = g\left(\vec{\theta}^\intercal \vec{x}\right)
\end{equation}
where the function $g\left(z\right)$ is defined as
\begin{equation}
g\left(z\right) = \frac{1}{1 + e^{-z}}
\end{equation}
Thus, to get the hypothesis function using the sigmoid function, just set $z = \vec{\theta}^\intercal \vec{x}$.

The sigmoid function, shown in Figure \ref{logreg_eg2_sigmoid_func_plot.pdf}, maps any real number onto the interval $\left(0, 1\right)$. This makes it immensely useful for transforming an arbitrary function for use with classification.

\begin{figure}[h] %  figure placement: here, top, bottom, or page
	\centering
	\graphicspath{{./Figures/}} %Use this to import an image from a subfolder.
	\includegraphics[scale=0.4]{logreg_eg2_sigmoid_func_plot.pdf}
	\caption[]{Sample plot of the sigmoid function.}
	\label{logreg_eg2_sigmoid_func_plot.pdf}
\end{figure}

\subsection{Interpretation of the Logistic Hypothesis Function}

When examining the hypothesis function output for logistic regression, we interpret $h_\theta\left(x \right)$ is the estimated probability that $y=1$ on an input example $x$. For example, let's revisit the tumor size question from above. We have
$$
\vec{x} = \left[\begin{array}{c}x_0 \\ x_1 \end{array}\right] = \left[\begin{array}{c}1 \\ \text{tumor size} \end{array}\right]
$$
If our hypothesis $h_\theta\left( x \right) = 0.7$, then we can tell the patient that there is a 70\% chance of the tumor being malignant.

Slightly more formally, we interpret $h_\theta\left(x\right)$ as:\footnote{This is read as "the probability that $y = 1$, given $x$, parameterized by $\theta$.}
\begin{equation}
h_\theta\left(x\right) = P\left(y=1 | x; \theta\right)
\end{equation}
Thus, by the rules of probability:
\begin{align}
P\left(y=0 | x; \theta\right) + P\left(y=1 | x; \theta\right) = 1 \\
P\left(y=0 | x; \theta\right) = 1 - P\left(y=1 | x; \theta\right)
\end{align}

\subsection{Fitting Logistic Regression to a Binary Classifier}
\label{chaplogreg-sect-hyporeg-subsectbinclasfit}
Now, we need to fit our hypothesis function into a binary classfier: $0$ or $1$. Using our probabilistic interpretation of the logistic hypothesis function, we can make the following supposition:

\begin{align}
y  = 1 &\text{ given that } h_\theta\left(x\right) \geq 0.5 \\
y = 0 &\text{ given that } h_\theta\left(x\right) < 0.5
\end{align}

Recall the plot of the sigmoid function in Figure \ref{logreg_eg2_sigmoid_func_plot.pdf}. We see that $g\left(z\right) \geq 0.5$ when $z \geq 0$. In our case, if we're setting $z = \vec{\theta}^\intercal \vec{x}$, then we have:

\begin{equation}
h_\theta\left(x\right) = g\left(\vec{\theta}^\intercal\vec{x}\right) \geq 0.5 ~~\mbox{ when }~~ \vec{\theta}^\intercal\vec{x} \geq 0
\end{equation}

\noindent From this, we can now state
\begin{align}
\vec{\theta}^\intercal\vec{x} \geq 0 &\implies y = 1 \\
\vec{\theta}^\intercal\vec{x} < 0 &\implies y= 0
\end{align}

\noindent When utilizing the sigmoid function, keep the following in mind:
\begin{itemize}
\item When $z = 0$, then $e^0 = 1$ so $g\left(x\right) = \frac{1}{2}$
\item When $z$ goes to $\infty$, we have $e^{-\infty} \to 0$, and this implies $g\left(x\right) = 1$
\item As $z \to -\infty$, $e^\infty \to \infty \implies g\left(z\right) = 0$
\end{itemize}

\section{Decision Boundary}
\begin{figure}[h] %  figure placement: here, top, bottom, or page
	\centering
	\graphicspath{{./Figures/}} %Use this to import an image from a subfolder.
	\includegraphics[scale=0.4]{logreg_eg3_decision_bndy_noline.pdf}
	\caption[]{Sample data.}
	\label{logreg_eg3_decision_bndy_noline.pdf}
\end{figure}

Consider the data plotted in Figure \ref{logreg_eg3_decision_bndy_noline.pdf}. Suppose our hypothesis is given by
$$
h_\theta\left(x\right) = g\left(\theta_0 + \theta_1x_1 + \theta_2x_2\right)
$$
We haven't yet discussed how to fit the parameters of this model (that's coming up next), but suppose we choose the following values for the parameters
$$
\vec{\theta} = \left[\begin{array}{c}\theta_0 \\ \theta_1 \\ \theta_2\end{array}\right] = \left[\begin{array}{c} -3 \\ 1 \\ 1 \end{array}\right]
$$

Given this choice of parameters, let's figure out where $y=1$ and where $y=0$. From \S\ref{chaplogreg-sect-hyporeg-subsectbinclasfit}, recall that we predict $y=1$ when $\vec{\theta}^\intercal\vec{x} \geq 0$, so here, we predict $y=1$ if $-3 + x_1 + x_2 \geq 0$. If we solve this for $x_1 + x_2$ we get
$$
x_1 + x_2 \geq 3 \implies y = 1
$$
If we change this to a pure equality, $x_1 + x_2 = 3$, we have the equation of a straight line (shown on Figure \ref{logreg_eg3_decision_bndy_withline.pdf}). The line drawn, is called the \textbf{decision boundary}. The decision boundary is the line created by the hypothesis function that separates the area where we classify $y=0$ and where $y=1$.

\begin{figure}[h] %  figure placement: here, top, bottom, or page
	\centering
	\graphicspath{{./Figures/}} %Use this to import an image from a subfolder.
	\includegraphics[scale=0.4]{logreg_eg3_decision_bndy_withline.pdf}
	\caption[]{Some sample binary data with a plotted decision boundary. Here, the blue circles represent $y=0$, the green triangles $y=1$ and the purple line is the decision boundary.}
	\label{logreg_eg3_decision_bndy_withline.pdf}
\end{figure}

To be clean, the decision boundary is a property of the hypothesis function, and not a property of the dataset. We fit the parameters of the hypothesis based on the training data, but once those parameters are set, the decision boundary is a property solely of the hypothesis function.

Now, suppose we have data as shown below in Figure \ref{logreg_eg3_decision_bndy_nonlinear_nocirc.pdf}. It's fairly obvious that no straight line decision boundary will work for this data. Again, we don't know how to fit the parameters for this model yet, but say our hypothesis function looks like this
$$
h_\theta\left(x\right) = g\left(\theta_0 + \theta_1x_1 + \theta_2x_2 + \theta_3x_1^2 + \theta_4x_2^2\right)
$$
Imagine we fit the parameters appropriately, and we get
$$
\vec{\theta} = \left[\begin{array}{c} \theta_0 \\ \theta_1 \\ \theta_2 \\ \theta_3 \\ \theta_4 \end{array}\right] = \left[\begin{array}{c}-1 \\ 0 \\ 0 \\ 1 \\ 1 \end{array}\right]
$$
Then, our hypothesis predicts that $y=1$ when $x_1^2 + x_2^2 \geq 1$. This is the equation for a circle of radius $1$, centered at the origin (see Figure \ref{logreg_eg3_decision_bndy_nonlinear.pdf}).
\begin{figure}[h]
	\centering
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
  		\includegraphics[scale=0.4]{logreg_eg3_decision_bndy_nonlinear_nocirc.pdf}
   		\caption[]{This is a sample of data with no clear linear decision boundary.}
   		\label{logreg_eg3_decision_bndy_nonlinear_nocirc.pdf}
	\end{subfigure}
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
   		\includegraphics[scale=0.4]{logreg_eg3_decision_bndy_nonlinear.pdf}
   		\caption[]{By altering our hypothesis function to include polynomial terms, we can have a non-linear decision boundary.}
   		\label{logreg_eg3_decision_bndy_nonlinear.pdf}
	\end{subfigure}
	\caption[]{Data that can't be fit with a linear decision line.}
\end{figure}
In this case, we predict $y=1$ everywhere outside the purple circle, and $y=0$ everywhere inside the circle.

With even higher order polynomials, we can get even more complicated decision boundaries.

\section{Cost Function}
Imagine we have a training set of data with $m$ examples
$$
\left\{ \left(x^{\left(1\right)}, y^{\left(1\right)}\right), \left(x^{\left(2\right)}, y^{\left(2\right)}\right), \cdots, \left(x^{\left(m\right)}, y^{\left(m\right)}\right) \right\}
$$
and $n$ features, represented by an $n+1$-dimensional feature vector
$$
x \in \left[\begin{array}{c} x_0 \\ x_1 \\ \vdots \\ x_n \end{array}\right]
$$
where $x_0 = 1$ and our output $y \in \{0, 1\}$. Our hypothesis is given by
\begin{equation}
h_\theta\left(x\right) = \frac{1}{1 + e^{-\theta^{{}^\intercal}x}}
\end{equation}
How do we choose the parameters for this model? For linear regression, we had the following cost function (adjusted slightly, we moved the $\tfrac{1}{2}$ to the inside of the summation)
\begin{equation}
J\left(\theta\right) = \frac{1}{m} \sum_{i=1}^m \frac{1}{2} \left(		h_\theta\left(x^{\left(i\right)}\right) - y^{\left(i\right)}	\right)^2
\end{equation}
but we're going to change how we write this function a little bit. Instead, we'll write
\begin{equation}
J\left(\theta\right) = \frac{1}{m} \sum_{i=1}^m \text{Cost}\left(	h_\theta\left(x^{\left(i\right)}\right), y^{\left(i\right)}\right)
\end{equation}
where we'll define the cost to be
\begin{equation}
\text{Cost}\left(	h_\theta\left(x^{\left(i\right)}\right), y^{\left(i\right)}\right) = \frac{1}{2} \left(h_\theta\left(x^{\left(i\right)}\right) - y^{\left(i\right)}	\right)^2
\end{equation}
This allows us to see more clearly that the cost function is really the sum over the cost term. To simplify even further, we'll remove the superscripts $\left(i\right)$
\begin{equation}
\text{Cost}\left(h_\theta\left(x\right), y\right) = \frac{1}{2} \left(	h_\theta\left(x\right) - y \right)^2
\end{equation}

If we try to minimize this function called Cost, it turns out to be a non-convex function. That means that the may be several local minima, which would prevent our gradient descent algorithm from working well. You can see a sample non-convex function in Figure \ref{logreg_eg4_sample_nonconvex_curve.pdf}. What we want instead, is a convex function (like a parabola) that only has a single minimum that is the global minimum. The sigmoid function is a non-linear signal function, so $J\left(\theta\right)$ ends up being non-convex.
\begin{figure}[h] %  figure placement: here, top, bottom, or page
	\centering
	\graphicspath{{./Figures/}} %Use this to import an image from a subfolder.
	\includegraphics[scale=0.6]{logreg_eg4_sample_nonconvex_curve.pdf}
	\caption[]{A non-convex curve. Notice all of the local minima.}
	\label{logreg_eg4_sample_nonconvex_curve.pdf}
\end{figure}

We need to define a new (convex) function that will allow us to determine the parameters in our hypothesis. For logistic regression, we use the following function
\begin{equation}
\text{Cost}\left(h_\theta\left(x\right), y\right) = \begin{cases} -\log\left(h_\theta\left(x\right)\right) & \text{if } y = 1 \\ -\log\left(1 - h_\theta\left(x\right)\right) &\text{if } y = 0 \end{cases}
\end{equation}
We plot this function below in Figure \ref{chaplogreg-sectcostfunct-plotcostfuncsample}.

\begin{figure}[h]
	\centering
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
  		\includegraphics[scale=0.5]{logreg_eg5_cost_func_y1.pdf}
   		\caption[]{$y=1$.}
   		\label{logreg_eg5_cost_func_y1.pdf}
	\end{subfigure}
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
   		\includegraphics[scale=0.5]{logreg_eg5_cost_func_y0.pdf}
   		\caption[]{$y=0$}
   		\label{logreg_eg5_cost_func_y0.pdf}
	\end{subfigure}
	\caption[]{The piecewise function used for the logistic regression cost function. }
	\label{chaplogreg-sectcostfunct-plotcostfuncsample}
\end{figure}

The shape of the curve comes from standard plot of $\log\left(x\right)$, and we just use a negative to flip it upside-down. This function, has some very desirable properties for us right now.

\noindent \begin{minipage}{\linewidth}
\begin{itemize}
\item If $y = 1$, then $h_\theta\left(x\right) = 1$ and the cost is zero. However, as $h_\theta\left(x\right) \to 0$ then $\text{cost} \to \infty$. This captures the intuition that if $h_\theta\left(x\right) = 0$, but $y=1$, we'll penalize the learning algorithm by a very large cost.
\item For $y=0$, this is reversed. If we have $y=0$ and $h_\theta\left(x\right) = 0$, then the cost is $0$. If $y=0$, the cost grows very large as $h_\theta\left(x\right)$ increases towards $1$.
\end{itemize}
\end{minipage}

\subsection{Simplified Cost Function}
Recall our cost function
\begin{equation}
J\left(\theta\right) = \frac{1}{m} \sum_{i=1}^m \text{Cost}\left(	h_\theta\left(x^{\left(i\right)}\right), y^{\left(i\right)}\right)
\end{equation}
where the cost is
\begin{equation}
\text{Cost}\left(h_\theta\left(x\right), y\right) = \begin{cases} -\log\left(h_\theta\left(x\right)\right) & \text{if } y = 1 \\ -\log\left(1 - h_\theta\left(x\right)\right) &\text{if } y = 0
\end{cases}
\end{equation}
and $y \in \{0, 1\}$.
Since $y$ is always either $0$ or $1$, we can take advantage of this to write a simplified version of our cost function:
\begin{equation}
\text{Cost}\left(h_\theta\left(x\right), y\right) = -y\log\left(h_\theta\left(x\right)\right) - \left( 1-y \right) \log\left(1 - h_\theta\left(x\right)\right)
\end{equation}
For either value of $y$, one of the terms will be multiplied by zero and disappear. So now, our cost function is\footnote{Just as an FYI, this cost function is derived statistically using maximum likelihood estimation}
\begin{align}
J\left(\theta\right) &= \frac{1}{m} \sum_{i=1}^m \text{Cost}\left(	h_\theta\left(x^{\left(i\right)}\right), y^{\left(i\right)}\right) \nonumber \\
\label{chaplogreg-sectsimpcostfunct-simpcostfuncformula}
&= -\frac{1}{m} \left[ \sum_{i=1}^m y^{\left(i\right)} \log h_\theta\left(x^{\left(i\right)}\right) + \left(1 - y^{\left(i\right)}\right) \log \left(1 - h_\theta\left(x^{\left(i\right)}\right)\right)\right]
\end{align}
We want to minimize the cost function$J\left(\theta\right)$ to fit the parameters $\theta$, so we can make our predictions using the hypothesis function. Again, we determine $\theta$ by calculating
$$
\min_\theta J\left(\theta\right)
$$
and predict using
$$
h_\theta\left(x\right) = \frac{1}{1 + e^{\theta^{{}^\intercal}x}}
$$
using the calculated parameters. Now, we just need to determine how to minimize $J\left(\theta\right)$.

\subsection{Gradient Descent for Logistic Regression}

We again return to gradient descent, of the form

\textbf{Repeat until convergence:}
\begin{equation}
\label{chaplogreg-sectcostfunct-subsectgraddscnt-logreggraddscnt}
\theta_j := \theta_j - \alpha \frac{\partial}{\partial \theta_j} J\left( \theta \right)
\end{equation}
where we simultaneously update all $\theta_j$. If we calculate the partial derivative $\frac{\partial}{\partial \theta_j} J\left(\theta\right)$, we find
\begin{equation}
\frac{\partial}{\partial \theta_j} J\left(\theta\right) = \frac{1}{m}\sum_{i=1}^m \left( h_\theta\left(x^{\left(i\right)}\right) - y^{\left(i\right)}\right) x_j^{\left(i\right)}
\end{equation}
Plugging this back into the formula for gradient descent, we get

\textbf{Repeat until convergence:}
\begin{equation}
\theta_j := \theta_j - \frac{\alpha}{m} \sum_{i=1}^m \left( h_\theta\left(x^{\left(i\right)}\right) - y^{\left(i\right)}\right) x_j^{\left(i\right)}
\end{equation}
But wait! This looks exactly like the formula for linear regression! Is this actually a different algorithm? Yes! It is! The difference here is that the hypothesis function $h_\theta\left(x\right)$ is a different function.

Remember that for gradient descent, we often need to apply feature scaling to make the algorithm run faster.

\section{Vectorized Equations}
In equation \ref{chaplogreg-sectsimpcostfunct-simpcostfuncformula}, we derived a simplified version of the cost function. We can do the same thing with a vectorized implementation. First, we state the vectorized hypothesis function as
\begin{equation}
h = g\left( X\vec{\theta}\right)
\end{equation}
and then we can write the simplified cost function as
\begin{equation}
J\left(\theta\right) = \frac{1}{m}\cdot \left( -\vec{y}^\intercal \log\left(h\right) - \left(1 - \vec{y}\right)^\intercal \log \left(1 - h\right)\right)
\end{equation}
Finally, we write out the vectorized form for gradient descent
\begin{equation}
\vec{\theta} := \vec{\theta} - \frac{\alpha}{m}X^\intercal \left( g\left(X\vec{\theta}\right) - \vec{y}\right)
\end{equation}

\section{Advanced Optimization}

There are other, more sophisticated algorithms that are able to more quickly optimize $\theta$. These algorithms are a little more complicated, so you shouldn't try to write them yourself unless you're an expert in numerical computing.

In particular, there are three algorithms that we'll mention:
\begin{itemize}
\item Conjugate gradient algorithm
\item Broyden-Fletcher-Goldfarb-Shanno (BFGS) algorithm
\item L-BFGS-B
\end{itemize}
In Python, these algorithms (and several others) are available in the {\tt scipy.optimize} package. The algorithm is chosen using the {\tt method='X'} flag, where X can be CG, BFGS, or L-BFGS-B, respectively, to use any of the above algorithms.
\begin{minted}{python}
from scipy.optimize import minimize
\end{minted}
We'll go through an example or two in the coding section.

%%%%%%%%%%%%%%%%%%%%%%%%
%								%
%		Add hyperlink to coding			%
%								%
%%%%%%%%%%%%%%%%%%%%%%%%

\section{Multiclass Classification: One-vs-All}

What is a multiclass classification problem? Here are some examples:
\begin{itemize}
\item You want to build an algorithm to automatically tag your email with different categories: work, friends, family, and hobby. Here, you have a classification problem for four classes.
\item For a medical visit, you might want to classify patients into not ill, having a cold, or having the flu.
\item To build an algorithm that classifies the weather into sunny, cloudy, rain, and snow.
\end{itemize}
\begin{figure}[h]
	\centering
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
  		\includegraphics[scale=0.45]{logreg_eg6_binary_eg_data.pdf}
   		\caption[]{Binary Classification}
   		\label{logreg_eg6_binary_eg_data.pdf}
	\end{subfigure}
	\begin{subfigure}[t]{0.45\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
   		\includegraphics[scale=0.45]{logreg_eg6_multiclass_eg_data.pdf}
   		\caption[]{Multiclass Classification}
   		\label{logreg_eg6_multiclass_eg_data.pdf}
	\end{subfigure}
	\caption[]{Examples of binary and multiclass classification.}
%	\label{}
\end{figure}
Previously, for a binary classification problem, our data looked like the data in Figure \ref{logreg_eg6_binary_eg_data.pdf}, with multiclass classification, the data looks more like Figure \ref{logreg_eg6_multiclass_eg_data.pdf}.
\begin{figure}[h] %  figure placement: here, top, bottom, or page
	\centering
	\graphicspath{{./Figures/}} %Use this to import an image from a subfolder.
	\includegraphics[scale=0.45]{logreg_eg6_multiclass_onevall_step1.pdf}
	\caption[]{One-vs-all classification. We're selecting one class to be our positive class, and the rest all become the negative class.}
	\label{logreg_eg6_multiclass_onevall_step1.pdf}
\end{figure}

We know how to perform binary classification, but how to we make this work with more classes? We can use the idea of one-vs-all classification.\footnote{This can also be called one-vs-rest classification.} Here's how it works; let's say we have a training set of three classes (such as in Figure \ref{logreg_eg6_multiclass_eg_data.pdf}). What we can do is turn this into three separate binary classification problems.\footnote{If we have $n$ different classes, our problem splits into $n+1$ different binary classification problems.This is because the vector $\vec{y}$ starts at index $0$, so $\left|\left|\vec{y}\right|\right| = n+1$.} Start by picking a class, say the blue circles, and make it our positive class; then lump all the other data into the negative class (see Figure \ref{logreg_eg6_multiclass_onevall_step1.pdf}).

We fit a classifier to this, called $h_\theta^{\left(1\right)}\left(x\right)$. We then do this for the two other classes, and fit them to logistic regression classifiers $h_\theta^{\left(2\right)}\left(x\right)$ and $h_\theta^{\left(3\right)}\left(x\right)$. Here, we've fit three classifiers
\begin{equation}
h_\theta^{\left(i\right)}\left(x\right) = P\left(y=i | x; 0\right)	~~\mbox{\;\;\;\;\;\;\;\;\;\; for } i = \{1, 2, 3\}
\end{equation}
that are trying to estimate the probability that $y$ is equal to class $i$, given $x$ and parameterized by $\theta$. So  $h_\theta^{\left(i\right)}\left(x\right)$ is trying to estimate the probability that a data point is of class $i$.

For a new input $x$, to make a prediction, we pick the class $i$ that maximizes
\begin{equation}
\max_i  h_\theta^{\left(i\right)}\left(x\right)
\end{equation}
and this tells us which class to assign the new input to.

\section{Regularization}
For the two machine learning algorithms we've seen so far, they tend to work pretty well. But when applied to specific datasets, they can run into a problem called overfitting, and this can cause them to perform very poorly. We're going to discuss a little more detail about this problem, and then go into ways to ameliorate it and increase our algorithm performance. Let's plot some housing data, and then take a look at three potential regressions.

\begin{figure}[h]
	\centering
	\begin{subfigure}[t]{0.3\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
  		\includegraphics[scale=0.35]{logreg_eg7_housing_data_linreg.pdf}
   		\caption[]{Linear Regression}
   		\label{logreg_eg7_housing_data_linreg.pdf}
	\end{subfigure}
	\begin{subfigure}[t]{0.3\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
   		\includegraphics[scale=0.35]{logreg_eg7_housing_data_quadreg.pdf}
   		\caption[]{Quadratic Regression}
   		\label{logreg_eg7_housing_data_quadreg.pdf}
	\end{subfigure}
	\begin{subfigure}[t]{0.3\textwidth}
   		\centering
    		\graphicspath{{./Figures/}}
   		\includegraphics[scale=0.35]{logreg_eg7_housing_data_quadreg_overfit.pdf}
   		\caption[]{Quadratic Regression with Additional Terms}
   		\label{logreg_eg7_housing_data_quadreg_overfit.pdf}
	\end{subfigure}
	\caption[]{Plots of linear and quadratic regressions with varying amounts of terms (features). }
%	\label{}
\end{figure}

Figure \ref{logreg_eg7_housing_data_linreg.pdf} is a linear regression on this data. This is the simplest regression, but looking at the data, it seems pretty clear that this isn't really a good fit. As the size of the house increases, the housing prices seem to plateau after a certain point, whereas the linear regression line keeps increasing. This is known as \textbf{underfitting}, or \textbf{high bias}.\footnote{The term \textit{bias} is somewhat or a historical or technical term. It carries with it the idea that if fitting a straight line to the data, it's as if the algorithm has a preconception (or bias) that the housing prices should vary linearly with their size, despite evidence in the data that this is not the case. Irrespective of the evidence to the contrary, the algorithm fits the data to a straight line, and it ends up not being a very good fit.} Both of these roughly mean that the regression just isn't fitting to the training data very well. In Figure \ref{logreg_eg7_housing_data_quadreg.pdf}, we can fit a quadratic function to the data. This seems to work pretty well, and looks like a better fit than the linear regression. Figure \ref{logreg_eg7_housing_data_quadreg_overfit.pdf} shows what happens when we \textbf{overfit} the line. This is also known as \textbf{high variance}. The curve fits the training set very well, but it particular to the specific training set, so it doesn't work well for any other data set.

Let's formally define these terms now:
\begin{description}
\item[Overfitting] If we have too many features, the learned hypothesis may fit the training set very well ($J\left(\theta\right) \approx 0$), but fails to generalize to new examples. This is also called high variance.
\item[Underfitting] This occurs when the form of our hypothesis function maps poorly to the trend of the data. It is usually caused by a function that is too simple or uses too few features. This is also called high bias.
\end{description}
This apply to both linear and logistic regression.

So how do we prevent our models from overfitting to the data? For the simple examples where we have one or two features, we can plot the data and determine it that way; but when we start working with data sets that have hundreds of features, this no longer works. We have two main options to address overfitting:
\begin{enumerate}
\item Reduce the number of features
	\begin{itemize}
	\item Manually select which features to keep, or
	\item Use a model selection algorithm (which we'll see later)
	\end{itemize}
\item Regularization
	\begin{itemize}
	\item Keep all the features, but reduce the parameters $\theta_j$
	\end{itemize}
\end{enumerate}
Regularization works well when we have a lot of slightly-useful features.

\subsection{Cost Function}
If we suspect overfitting in our hypothesis function, we can reduce the weight of some of the terms in our function by increasing their cost. Say we have a hypothesis of the form
$$
h_\theta\left(x\right) = \theta_0 + \theta_1x + \theta_2x^2 + \theta_3x^3 + \theta_4x^4
$$
To reduce the effect of $\theta_3x^3$ and $\theta_4x^4$ without actually getting rid of those features, we instead modify our cost function to something like this
$$
\min_\theta \frac{1}{2m} \sum_{i=1}^m \left(h_\theta \left(x ^{\left(i\right)}\right) - y^{\left(i\right)}\right)^2 + 1000\left(\theta_3\right)^2 + 1000 \left(\theta_4\right)^2
$$
where $1000$ is an arbitrary large number that we chose. This inflates the cost of both $\theta_3$ and $\theta_4$, so to reduce our cost function, we end up reducing the values of $\theta_3$ and $\theta_4$ to near zero. This will greatly reduce the values of $\theta_3x^3$ and $\theta_4x^4$ in our hypothesis function. This usually yields a simpler hypothesis that is less prone to overfitting.

In a more general sense, having small values for \textit{all} parameters typically decreases the likelihood that we'll overfit our regression.

To implement regularization, we modify the cost function by adding a regularization term at the end to shrink every parameter, where $\lambda$ is called the regularization parameter.
\begin{equation}
J_\theta = \frac{1}{2m}\left[ \sum_{i=1}^m \left(h_\theta\left( x^{(i)}\right) - y^{(i)}\right)^2 + \lambda\sum_{j=1}^n \theta_j^2\right]
\end{equation}
By convention, these summations start at $1$, so we don't regularize $\theta_0$. In this equation, the regularization parameter $\lambda$ controls the trade-off between two different goals: fitting the training data well (the first term of the equation), and keeping the parameters small (the regularization term).

Conceptually, it can be somewhat difficult to see why keeping the parameters small reduces overfitting, but if you program this yourself, you'll see this effect firsthand. What actually happens is that by using the above cost function with regularization, the resulting output of the hypothesis function is smoothed out to reduce overfitting. However, if $\lambda$ is chosen to be too large, it may smooth out the function too much and underfit. For example, if $\lambda \sim 10^{10}$ for our housing data problem, then $\theta_j \to 0$ for $j \neq 0$.This leaves us with a regression that looks like $h_\theta\left(x\right) = \theta_0$, meaning our hypothesis is just a horizontal line.

\subsection{Regularized Linear Regression}
We've previously looked at two algorithms for linear regression: one based on gradient descent, and one based on the normal equation. Now, we'll take these two algorithms and generalize them to regularized linear regression. Here is our cost function
\begin{equation}
J_\theta = \frac{1}{2m}\left[ \sum_{i=1}^m \left(h_\theta\left( x^{(i)}\right) - y^{(i)}\right)^2 + \lambda\sum_{j=1}^n \theta_j^2\right]
\end{equation}
and our goal is to get
\begin{equation}
\min_\theta J\left(\theta\right)
\end{equation}

\subsubsection{Gradient Descent}
Our standard gradient descent algorithms looks something like this:

\textbf{Repeat \{}
$$
\theta_j := \theta_j - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_j^{(i)} ~~\mbox{\;\;\;\;\;\;\;\;\;\; for } j = 0, 1, 2, \dots, n
$$
\textbf{\}} \\

\noindent To make this easier, we're just going to write the update for $\theta_0$ separately.

\textbf{Repeat \{}
\begin{subequations}
\begin{align}
\theta_0 &:= \theta_0 - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_0^{(i)} \\
\theta_j &:= \theta_j - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_j^{(i)} ~~\mbox{\;\;\;\;\;\;\;\;\;\; for } j = 1, 2, \dots, n \label{chaplogreg-sectregular-subsect-linreg-graddesc-oldeqtn}
\end{align}
\end{subequations}
\textbf{\}} \\
Now, we want to modify these equations to include our new regularization parameter. The $\theta_0$ equation stays the same since we don't regularize $\theta_0$, but equation \ref{chaplogreg-sectregular-subsect-linreg-graddesc-oldeqtn} becomes
\begin{subequations}
\begin{align}
\theta_0 &:= \theta_0 - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_0^{(i)} \\
\theta_j &:= \theta_j - \alpha \left[\frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_j^{(i)} + \frac{\lambda}{m}\theta_j \right] ~~\mbox{\;\;\;\;\;\;\;\;\;\; for } j = 1, 2, \dots, n \label{chaplogreg-sectregular-subsect-linreg-graddesc-neweqtn}
\end{align}
\end{subequations}
If you do the calculus out, you can prove that the large term in brackets in equation \ref{chaplogreg-sectregular-subsect-linreg-graddesc-neweqtn} is the partial derivative of the new $J\left(\theta\right)$ with the regularization term.
$$
\frac{\partial}{\partial \theta_j} J\left(\theta\right)
$$
With some manipulation, we can rewrite our update rule as
\begin{equation}
\theta_j := \theta_j\left(1 - \alpha\frac{\lambda}{m}\right) - \alpha\frac{1}{m}\sum_{i=1}^m \left(h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_j^{(i)}
\end{equation}
where in the first term in the equation, the value $1 - \alpha\frac{\lambda}{m}$ will always be less than one. Intuitively, the term that's always less than one will serve to reduce $\theta_j$ by a small amount, and the other term is actually exactly the same as our previous gradient descent algorithm before we introduced regularization.

\subsubsection{The Normal Equation}
Gradient descent was just one of the two algorithms we've explored as a solution to linear regression; the second algorithm was based on the normal equation. With the normal equation, we crafted the design matrix $X$ where each row corresponds to a separate training example, and we have an $m$-dimensional vector $\vec{y}$ that contains our labels.
$$
X = \left[\begin{array}{c}
\left(x^{(1)}\right)^\intercal \\ \left(x^{(2)}\right)^\intercal \\ \vdots \\ \left(x^{(m)}\right)^\intercal
\end{array}\right] ~~\mbox{ \;\;\;\;\;\;\;\;\;\; }~~ \vec{y} = \left[\begin{array}{c} y^{(1)} \\ y^{(2)} \\ \vdots \\ y^{(m)} \end{array}\right]
$$
where we calculated the vector $\vec{\theta}$ as
$$
\vec{\theta} = \left( X^\intercal X\right)^{-1} X^\intercal \vec{y}
$$
This minimized the unregularized cost function $J\left(\theta\right)$. To add in regularization, we add another term inside the parentheses
\begin{equation}
\vec{\theta} = \left( X^\intercal X + \lambda L \right)^{-1} X^\intercal \vec{y}
\end{equation}
where the matrix $L$ is a diagonal matrix\footnote{In linear algebra, a diagonal matrix is a matrix (usually a square matrix) in which the off-diagonal elements are all zero. The main diagonal entries themselves may or may not be zero.} given by:
\begin{equation}
L = \left[\begin{array}{cccccc}
0 &  &  &  &  &  \\
 & 1 & & & & \\
& & 1 & & & \\
& & & 1 & & \\
& & & & \ddots & \\
& & & & & 1
\end{array}\right]
\end{equation}
The matrix $L$ has $0$ in it's top left spot, then $1$'s down the diagonal and zeros everywhere else. It has dimensions $\left(n+1\right) \times \left(n+1\right)$. Intuitively, this is the identity matrix (sans $x_0$), multiplied by a single number $\lambda \in \mathbb{R}$.

Recall that if the number of examples $m \leq$ the number of features $n$, then $X^\intercal X$ is non-invertible.\footnote{A non-invertible square matrix is also called singular or degenerate.} We got around this programatically by using the pseudoinverse. Fortunately, regularization also takes care of this for us as well. As long as $\lambda \in \mathbb{R}_{>0}$ is strictly greater than zero, then $X^\intercal X + \lambda L$ will be strictly invertible.

\subsection{Regularized Logistic Regression}
Logistic regression, just like linear regression, is prone to overfitting if you overfit with higher order polynomials. As we saw in equation \ref{chaplogreg-sectsimpcostfunct-simpcostfuncformula}, our logistic regression cost function is given by
$$
J\left(\theta\right) = -\frac{1}{m} \left[ \sum_{i=1}^m y^{\left(i\right)} \log h_\theta\left(x^{\left(i\right)}\right) + \left(1 - y^{\left(i\right)}\right) \log \left(1 - h_\theta\left(x^{\left(i\right)}\right)\right)\right]
$$
To morph this into a regularized equation, we need to add a regularization term at the end, resulting in
\begin{equation}
J\left(\theta\right) = -\frac{1}{m} \left[ \sum_{i=1}^m y^{\left(i\right)} \log h_\theta\left(x^{\left(i\right)}\right) + \left(1 - y^{\left(i\right)}\right) \log \left(1 - h_\theta\left(x^{\left(i\right)}\right)\right)\right] + \frac{\lambda}{2m}\sum_{j=1}^n \theta_j^2
\end{equation}
Note that in the second summation, $j \in \left[1, n\right]$, so this excludes $\theta_0$.

In a similar fashion to what we did with linear regression, we're going to make the same changes to gradient descent for logistic regression. We start with our standard gradient descent equation (equation \ref{chaplogreg-sectcostfunct-subsectgraddscnt-logreggraddscnt}), and separate out the $\theta_0$ term into its own equation. This gives us the same initial result as linear regression.
\begin{align}
\theta_0 &:= \theta_0 - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_0^{(i)} \\
\theta_j &:= \theta_j - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_j^{(i)} ~~\mbox{\;\;\;\;\;\;\;\;\;\; for } j = 1, 2, \dots, n
\end{align}
Then, we again modify the second update rule
\begin{subequations}
\begin{align}
\theta_0 &:= \theta_0 - \alpha \frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_0^{(i)} \\
\theta_j &:= \theta_j - \alpha \left[\frac{1}{m} \sum_{i=1}^m \left( h_\theta\left(x^{(i)}\right) - y^{(i)}\right) x_j^{(i)} + \frac{\lambda}{m}\theta_j \right] ~~\mbox{\;\;\;\;\;\;\;\;\;\; for } j = 1, 2, \dots, n
\end{align}
\end{subequations}
Cosmetically, this is the same formula as for linear regression, but the hypothesis is different, so the equations end up being different. Again, the bracketed term is the new partial derivative of $J\left(\theta\right)$ with the regularized cost function.

\section{Python Labs: Coding Logistic Regression in Python}
\section{Homework}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{The Lebesgue function spaces $L^p$}
\label{sec:lp}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Integral inequalities}

\begin{lemma}[Young's inequality]
  \defindex{Young's inequality}
  For two arbitrary numbers $a$ and $b$, and a positive number
  $\gamma$ holds
  \begin{gather}
    \label{eq:lp:1}
    2\left|ab\right| \le \gamma a^2 + \frac1\gamma b^2.
  \end{gather}
\end{lemma}
\begin{proof}
  From the binomial formulas we have
  \begin{gather*}
    0 \le
    \begin{cases}
      (a+b)^2 &= a^2+b^2+2ab,
      \\
      (a-b)^2 &= a^2+b^2-2ab,
    \end{cases}
  \end{gather*}
  and thus by bringing $2ab$ to the left,
  \begin{gather}
    \label{eq:lp:2}
    \pm 2ab \le a^2+b^2.
  \end{gather}
  For $\gamma \neq 1$, the inequality is obtained by replacing $a$
  with $\gamma a$ and $b$ with $b/\gamma$ in~\eqref{eq:lp:2} and
  observing that this does not change the left hand side.
\end{proof}

\begin{lemma}[Cauchy-Bunyakovsky-Schwarz inequality\footnote{Also known as
    Cauchy-Schwarz or Schwarz's inequality}]
  \defindex{Cauchy-Bunyakovsky-Schwarz inequality}
  If functions $f$ and $g$ and their squares are integrable on a
  subset $\Omega$ of $\R^d$, so is their pointwise product $fg$, and
  there holds
  \begin{gather}
    \label{eq:lp:3}
    \int_\Omega f(x)g(x)\dx \le \sqrt{\int_\Omega f^2(x)\dx} \sqrt{\int_\Omega g^2(x)\dx}.
  \end{gather}
\end{lemma}

\begin{proof}
  First, by Young's inequality we obtain for arbitrary
  $\gamma$ that
  \begin{gather*}
    2 \int_\Omega f(x)g(x)\dx
    \le \gamma \int_\Omega f^2(x)\dx
    + \frac1\gamma \int_\Omega g^2(x)\dx.
  \end{gather*}
  Choosing $\gamma$ such that both terms on the right are equal,
  namely
  \begin{gather*}
    \gamma = \left. \sqrt{\int_\Omega g^2(x)\dx} \right/ \sqrt{\int_\Omega f^2(x)\dx},
  \end{gather*}
  the inequality is obtained.
\end{proof}

\begin{note}
  The Cauchy-Bunyakovsky-Schwarz inequality is an immediate extension
  of the Cauchy inequality for discrete sums
  \begin{gather}
    \sum_k a_k b_k \le \sqrt{\sum_ka_k^2} \sqrt{\sum_kb_k^2}
  \end{gather}
\end{note}

\begin{lemma}[Hölder's inequality]
  Assume that the functions $|f|^p$ and $|g|^q$ are integrable with
  $1<p,q<\infty$ and
  \begin{gather}
    \label{eq:lp:5}
    \frac1p+\frac1q=1,
  \end{gather}
  then the function $fg$ is integrable on $\Omega$ and
  \begin{gather}
    \label{eq:lp:6}
    \int_\Omega |fg| \dx \le \sqrt[p] {\int_\Omega |f|^p \dx} \sqrt[q] {\int_\Omega |g|^q \dx}.
  \end{gather}
\end{lemma}

\begin{lemma}[Minkowski's inequality]
  Let the functions $|f|^p$ and $|g|^p$ be integrable on $\Omega$ for
  $1\le p < \infty$. Then, the function $|f+g|^p$ is integrable on
  $\Omega$ and
  \begin{gather}
    \label{eq:lp:7}
    \sqrt[p] {\int_\Omega |f+g|^p \dx}
    \le \sqrt[p] {\int_\Omega |f|^p \dx}
    +\sqrt[p] {\int_\Omega |g|^p \dx}
  \end{gather}
\end{lemma}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{The real Hilbert spaces $L^2(\Omega)$}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{intro}
  For a bounded or unbounded subset $\Omega\subseteq \R^d$, the set of
  functions with squares, which are integrable with finite integrals
  forms a vector space. This is assured by the Minkowski's
  inequality~\eqref{eq:lp:7}. Our goal in this section is equipping this
  space with an inner product and a norm. Once a norm has been
  defined, we will show that the space obtained is complete, thus a
  \putindex{Hilbert space}.
  We start this section by the attempt to introduce an inner
  product.
\end{intro}

\begin{lemma}
\label{lemma:lp:bilinear}
  Let $\Omega \subseteq \R^d$ be bounded or unbounded.
  The form
  \begin{gather}
    \label{eq:lp:4}
    \scal(f,g) = \int_\Omega f(x)g(x)\dx
  \end{gather}
  is defined and bounded for functions $f$ and $g$ with integrable
  squares. It is bilinear, positive semidefinite, and symmetric.
\end{lemma}

\begin{proof}
  According to the Cauchy-Bunyakovsky-Schwarz
  inequality~\eqref{eq:lp:3}, boundedness of $\scal(f,g)$ follows from
  boundedness of the integrals of $f^2$ and $g^2$.
  Linearity in $f$ and $g$ are an immediate consequence of the
  linearity of the integral in
  equation~\eqref{eq:integral:3}. Symmetry follows from the fact that
  we can change the order of $f$ and $g$ in the product under the
  integral. Finally, we have
  \begin{gather*}
    0 \le \int_\Omega f^2(x)\dx = \scal(f,f).
  \end{gather*}
\end{proof}

\begin{intro}
  We have obtained $\scal(f,f) \ge 0$, but the definiteness of an
  inner product requires that $\scal(f,f) = 0$ implies $f=0$. On the
  other hand, $\scal(f,f) = 0$ holds for any function, which is zero
  almost everywhere in $\Omega$, but can have arbistrary values on a
  set of measure zero. Thus, we have no hope to define a definite
  inner product by integration on a function space. The solution to
  this dilemma is a modification of the function space by a little
  trick. Essentially, we turn around the definitions and define the
  relation $f=0$ through the condition $\scal(f,f) = 0$. We will now
  do this in a mathematically sound way.
\end{intro}

\begin{lemma}
  Let $F$ be a set of functions on $\R^d$. We introduce the relation
  $f\simeq g$ if $f$ and $g$ differ at most on a set of measure zero,
  or
  \begin{gather}
    f\simeq g
    \quad\Leftrightarrow f(x)-g(x)=0 \text{a.e.}
  \end{gather}
 This relation is an \putindex{equivalence relation}.
\end{lemma}

\begin{proof}
  We have to show reflexivity, symmetry, and transitivity of
  ``$\simeq$''. Obviously, since $f(x)=f(x)$ for all $x$, we have
  $f\simeq f$. Similarly obvious from the definition is that $f\simeq
  g$ implies $g\simeq f$. Finally, let $f\simeq g$ and $g\simeq
  h$. Then,
  \begin{gather*}
    f(x)-h(x) = \bigl(f(x)-g(x)\bigr) + \bigl(g(x)-h(x)\bigr).
  \end{gather*}
  Accordingly, the set on which $f$ and $h$ differ is at most the union of
  two set of measure zero, thus $f\simeq h$.
\end{proof}

\begin{lemma}
  The set of functions
  \begin{gather*}
    Z = \bigl\{ f \big| f\equiv 0 \bigr\},
  \end{gather*}
  where ``0'' is the function which is zero everywhere, is a vector
  space. Thus, for any vector space $V$ of functions, the quotient set
  $V/Z$ is a vector space.
\end{lemma}

\begin{proof}
  The first part of the proof is obvious. Thus, for any element of $V$
  vector addition and scalar multiplication can be split into their
  components in $Z$ and the remainder, which makes them well defined
  on the quotient set.
\end{proof}

\begin{definition}
  \defindex{L2@$L^2$}

  Let $V$ be the space of functions with bounded square integrals on a
  set $\Omega\subseteq \R^d$. Let $Z$ be the equivalence class of the
  zero function according to the preceding lemmas. We define the
  vector space $L^2(\Omega)$ as the quotient space $V/Z$. The space
  $L^2(\Omega)$ is equipped with the inner product $\scal(.,.)$
  according to equation~\eqref{eq:lp:4} and the norm
  \begin{gather}
    \norm{f} = \norm{f}_2 := \sqrt{\scal(f,f)}.
  \end{gather}
\end{definition}

\begin{note}
  We say for two functions $f$ and $g$ in $L^2(\Omega)$ that ``$f=g$''
  if the functions coincide almost everywhere.
  Thus, the missing definiteness of the inner product in
  Lemma~\ref{lemma:lp:bilinear} is obviously cured by considering
  equivalence classes. Thus, the norm is actually a norm, and
  $L^2(\Omega)$ for the moment is a pre-Hilbert space, that is, an
  inner product space, which is not necessarily complete.
\end{note}

\begin{remark}
  Consistent with other authors, we usually refer to an element of
  $L^2(\Omega)$ as a \define{square integrable function} or
  $L^2$-function. Nevertheless, it is important to keep in mind that
  these elements are not functions at all. In particular, they do not
  assign a value $f(x)$ to a given point $x\in \R^d$, since $\{x\}$ is
  a set of measure zero, and thus we would be allowed to change this
  value any time. This observation is important whenever we deal with
  function spaces $L^2(\Omega)$ and similar objects. Later we will
  have to answer the question what kind of evaluation of an
  $L^2$-function is actually permitted.
\end{remark}

\begin{definition}
  We say that a sequence $f_n$ in $L^2(\Omega)$ converges to an
  element $f\in L^2(\Omega)$ if $\norm{f_n-f}\to 0$ as $n\to\infty$. A
  sequence $f_n$ in $L^2(\Omega)$ is called a \define{Cauchy
    sequence}, if the \define{Cauchy criterion}
  \begin{gather}
    \label{eq:lp:cauchy}
    \forall \epsilon>0
    \quad \exists n_\epsilon
    \quad\forall m,n>n_\epsilon :
    \norm{f_m-f_n} < \epsilon.
  \end{gather}
  For the latter, we also say $\norm{f_m-f_n}\to 0$ as
  $n,m\to\infty$.
\end{definition}

\begin{theorem}[Fischer-Riesz]
  Let $\Omega\subseteq \R^d$. Then,
  $L^2(\Omega)$ is a Hilbert space, that is, it is complete, that is,
  every Cauchy sequence $f_n$ in $L^2(\Omega)$ converges to an
  element $f\in L^2(\Omega)$.
\end{theorem}

\begin{proof}
  First, we note that the Cauchy criterion is necessary for
  convergence, since by the triangle inequality, we have for
  $m,n\to\infty$:
  \begin{gather*}
    \norm{f_m-f_n} \le \norm{f_m-f} + \norm{f-f_n} \to 0.
  \end{gather*}
  Now we assume that the Cauchy criterion holds. Then, there exists
  a sequence $n_1,n_2,\dots$ such that $\norm{f_{n_{k+1}}- f_{n_k}}
  <2^{-k}$. Now we have to distinguish between bounded domains
  $\Omega$ and unbounded domains. First, for bounded domains, the
  Cauchy-Bunyakovsky-Schwarz inequality implies that
  \begin{gather*}
    \int_\Omega 1 \left|f_{n_{k+1}}(x)- f_{n_k}(x)\right|\dx \le \sqrt{m(\Omega)}
    \norm{f_{n_{k+1}}- f_{n_k}} \le \sqrt{m(\Omega)}2^{-k},
  \end{gather*}
  hence the series
  \begin{gather*}
    \sum_{k=1}^\infty \int_\Omega |f_{n_{k+1}}(x)- f_{n_k}(x)|\dx
  \end{gather*}
  converges to a finite value. The sequence of functions defined by
  \begin{gather*}
    s_m(x) = \sum_{k=1}^m |f_{n_{k+1}}(x)- f_{n_k}(x)|,
  \end{gather*}
  is monotonically increasing. Furthermore, we have seen above that
  their integrals are uniformly bounded. Thus, by the Beppo-Levi
  theorem, $s_m(x)$ converges almost everywhere to an integrable
  function. This on the other hand implies that for almost every $x\in
  \Omega$ the sequence $f_{n_k}(x)$ is a Cauchy sequence and thus
  converges to a limit value $f(x)$.

  If the domain $\Omega$ is unbounded and its measure is not finite,
  the above argument can be applied to any finite subdomain. Covering
  $\Omega$ with a countable sequence of such subdomains, we can
  likewise conclude convergence to a limit function $f$ for
  almost every $x\in\Omega$.

  Now we prove that $f\in L^2(\Omega)$. We observe that
  \begin{gather*}
    \norm{f_{n_k}} \le \norm{f_{n_1}} + \norm{f_{n_k}-f_{n_1}} \le \norm{f_{n_1}}+\frac12.
  \end{gather*}
  Thus, $\norm{f_{n_k}}^2$ is uniformly bounded and as $|f_{n_k}(x)|^2
  \to |f(x)|^2$ almost everywhere, Fatou's lemma asserts that
  $\norm{f}$ is finite and accordingly $f\in L^2(\Omega)$.

  It remains to show that $\norm{f_n-f}\to 0$.
  \begin{todo}
    Show that norms converge
  \end{todo}
\end{proof}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{The real Banach spaces $L^p(\Omega)$}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{intro}
  We saw that the Cauchy-Bunyakovsky-Schwarz inequality has a
  generalization for exponents different from 2 in Hölder's
  inequality, further that Minkowski's inequality holds for arbitrary
  $p$ with $1\le p<\infty$. This suggests to define a norm through
  \begin{gather}
    \label{eq:lp:8}
    \norm{f}_p = \sqrt[p] {\int_\Omega |f|^p \dx},
  \end{gather}
  and it is indeed an easy exercise to prove the norm properties
  analogue to the previous subsection.
\end{intro}

\begin{definition}
  A function $f:\Omega \to \R$ is called \define{essentially bounded}
  from above, if there is a number $a$ such that the set
  \begin{gather*}
    (f>a) := \bigl\{x\in\Omega \big| f(x)>a \bigr\},
  \end{gather*}
  has measure zero. The number $a$ is called an \define{essential
    upper bound}. For an essentially bounded function, we define the
  \define{essential supremum} $\esssup f$ as the infimum of all
  essential upper bounds:
  \begin{gather*}
    \esssup f = \inf_{a\in \R} (f>a) \text{ has measure zero.}
  \end{gather*}
\end{definition}

\begin{definition}
  For $1\le p < \infty$, the space $L^p(\Omega)$ is the space of all
  functions such that $|f|^p$ is integrable with its norm defined
  in~\eqref{eq:lp:8}. The space $L^\infty(\Omega)$ is the space of all
  essentially bounded functions on $\Omega$, and its norm is
  \begin{gather*}
    \norm{f}_\infty = \esssup_{x\in\Omega} f
  \end{gather*}
\end{definition}

\begin{note}
  For $p\neq2$, the $L^p$-norm is not defined by an inner product,
  thus $L^p(\Omega)$ cannot be a Hilbert space.
\end{note}

\begin{theorem}
  The spaces $L^p(\Omega)$ are complete, thus, they are Banach spaces.
\end{theorem}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\chapter{Likelihood and estimators}
\label{chap:like}
In this chapter likelihood and its estimators are introduced.
\\
\\ % Casella berger, s290, s315
Another way to summarize data is the likelihood function. The theory presented here is from \cite{casella2002statistical}. The likelihood function is defined as,
\begin{defn}
Let $f(\boldsymbol{x}|\theta)$ denote the joint pdf or pmf of the sample $\boldsymbol{X} = (X_1,...,X_n)$. Then, given that $\boldsymbol{X} = \boldsymbol{x}$ is observed, the function $\theta$ is defined by
\begin{equation}
L(\theta | \boldsymbol{x}) = f(\boldsymbol{x}|\theta)
\label{eq:like}
\end{equation}
is called the likelihood function.
\end{defn}
By using this function one can find estimators for $\theta$. These estimators are the ones that maximizes the likelihood function. Furthermore these estimators are most likely. Hence the they are given the name maximum likelihood estimators. The formal definition is as follows
\begin{defn} % side 316
For each sample point $\boldsymbol{x}$, let $\hat{\theta}(\boldsymbol{x})$ be parameter value at which $L(\theta | \boldsymbol{x})$ attains its maximum as a function of $\theta$, with $\boldsymbol{x}$ held fixed. A maximum likelihood estimator(MLE) of the parameter $\theta$ based on a sample $\boldsymbol{X}$ is $\hat{\theta}(\boldsymbol{x})$.
\end{defn}
To maximum of the likelihood can be found analytically or numerically. Analytically it is recommended to take the log of the likelihood function. This for easier derivation.


\section{Uniform Continuity}
Let $U\in\mathbb C$ and $f$ a scalar function on $U$.
We know what continuity means in the sense of metric space.
\begin{definition}
    We say $f$ is uniformly continuous on $U$ if for any $\epsilon>0,\exists\delta>0,\forall x,y\in U$, $|x-y|<\delta\implies |f(x)-f(y)|<\delta$.
\end{definition}
Note that the difference between uniform convergence and our initial form of convergence is that the value of $\delta$ does not depend on the point $x$.
\begin{example}
    The standard example that a function is continuous but not uniformly continuous is that $f(x)=x^2$ on $\mathbb R$.
    To see why, observe that we take $\epsilon=1$, then choose any $\delta>0$,
    $$(x+\delta/2)^2-x^2=\delta x+\delta^2/4$$
    We can just choose $x>1/\delta$ and the value would exceed $1$.
\end{example}
\begin{theorem}
    Let $f$ be a scalar function defined on a closed interval $[a,b]$, then if $f$ is continuous then it is uniformly continuous.
\end{theorem}
\begin{proof}
    (Heine-Borel Theorem gives compactness of the interval which provides a direct proof.)\\
    Assume that it is not the case, so
    $$\exists\epsilon>0,\forall\delta>0,\exists x,y\in[a,b], |x-y|<\delta\land |f(x)-f(y)|>\epsilon$$
    Choose such a `bad' $\epsilon$, consider $\delta_n=1/n$ and choose $x_n,y_n$ accordingly.\\
    By Bolzano-Wiestrass, there is a subsequence $x_{k_m}$ of $x_n$ that converges to some $x$ as $x\to\infty$.
    Note that since the interval is closed $x\in[a,b]$.
    Then, $|y_{k_m}-x|\le |x_{k_m}-x|+|x_{k_m}-y_{k_m}|<\frac{1}{k_m}+\epsilon$
    for any $\epsilon>0$. so $y_{k_m}\to x$.\\
    Since $f$ is continuous at $x$, there is some $\delta$ such that for every $y\in[a,b]$, $|x-y|<\delta\implies |f(x)-f(y)|<\epsilon/2$.
    There is some $N$ such that $m>N\implies |x_{k_m}-x|<\delta,|y_{k_m}-x|<\delta$.
    So $\epsilon<|f(x_{k_m})-f(y_{k_n})|\le |f(x_{k_m})-f(x)|+|f(x)-f(y_{k_m})|<2\epsilon/2=\epsilon$
    This is a contradiction.
\end{proof}
\begin{corollary}
    A continuous function on a closed interval is integrable.
\end{corollary}
It follows that a continuous function on a closed interval is integrable.
%So given $\epsilon$, by the above theorem, there is some $\delta>0,\forall x,y\in[a,b]$, $|x-y|<\delta\implies |f(x)-f(y)|<\epsilon$, so