\documentclass[landscape, letterpaper]{article}

\usepackage[margin=.25in]{geometry}
\usepackage{fontspec}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{multicol}

\renewcommand{\complement}[1]{{#1}^\mathsf{c}}

\begin{document}
\begin{multicols}{2}
    \section*{Induction}
    \textbf{Let} $P(n)$ be the property to be proven, let $a$ be fixed integer.

    Suppose the following:
    \begin{enumerate}
        \item \textbf{Basis:} $P(a)$ is true.
        \item \textbf{Induction:} For all integers $k\geq n$, if $P(k)$ is true then $P(k+1)$ is true.
        \item Hence for all integers $n\geq a, P(n)$ is true.
    \end{enumerate}
    \subsection*{Strong Induction}
    \textbf{Let} $P(n)$ be the property to be proven, let $a$ and $b$ be fixed integers where $a\leq b$.

    Suppose the following:
    \begin{enumerate}
        \item \textbf{Basis:} $P(a), P(a+1), \ldots P(b)$ are true.
        \item \textbf{Induction:} For all integers $k\geq b$, if $P(i)$ is true for all integers from $a$ to $k$ then $P(k+1)$ is true.
        \item Hence for all integers $n\geq a, P(n)$ is true.
    \end{enumerate}
    \section*{Number Theory}
    \begin{itemize}
        \item \textbf{Even} $n$ is even $\iff \exists k, n = 2k$.
        \item \textbf{Odd} $n$ is odd $\iff \exists k, n = 2k+1$.
        \item \textbf{Prime} $n$ is prime $\iff \forall r, s$, if $n = rs$ then $r=1$ and $s = n$ or vice versa.
        \item \textbf{Composite} $n$ is composite $\iff \exists r, s$, if $n = rs$ and $1 < r < s$ and $1 < s < n$.
        \item \textbf{Rational} $r$ is rational $\iff \exists a, b$ such that $r = \frac{a}{b}$ and $b \neq 0$.
        \item \textbf{Divisible} $d|n \iff \exists k$ such that $n=dk$.
    \end{itemize}
    \section*{Sets}
    Can be proven by using Induction, Algebra or Element method.

    Element Method: Usually with equality or subset relations: prove that an element in LHS is an element in RHS.
    \begin{itemize}
        \item Proper Subset: $A \subseteq B$ \textbf{AND} there is one element in B that is \textbf{not} in A
        \item Power Set: $\wp(\{x, y, z\}) = \{\emptyset, \{x\}, \{y\}, \{z\},\\
                  \{x, y\},\{x, z\}, \{y, z\}, \{x, y, z\}$
    \end{itemize}
    \subsection*{Identities}
    \begin{itemize}
        \item Commutative Law
              \begin{itemize}
                  \item $A \cup B = B \cup A $ and $A \cap B = B \cap A$
              \end{itemize}
        \item Associative Law
              \begin{itemize}
                  \item $(A \cup B) \cup C = A \cup (B \cup C)$ and $(A \cap B) \cap C = A \cap (B \cap C)$
              \end{itemize}
        \item Distributive Law
              \begin{itemize}
                  \item $A \cup (B \cap C) = (A \cup B) \cap (A \cup C)$
                  \item $A \cap (B \cup C) = (A \cap B) \cup (A \cap C)$
              \end{itemize}
        \item Identity Law
              \begin{itemize}
                  \item $A \cup \varnothing = A$ and $A \cap U$
              \end{itemize}
        \item Idempotent Law
              \begin{itemize}
                  \item $A \cup A = A$ and $A\cap A = A$
              \end{itemize}
        \item Universal Bound Law
              \begin{itemize}
                  \item $A \cup U = U$ and $A  \cap \varnothing = \varnothing$
              \end{itemize}
        \item DeMorgan's Law
              \begin{itemize}
                  \item $\complement{(A \cup B)} = \complement{A} \cap \complement{B}$ and $\complement{(A \cap B)} = \complement{A} \cup \complement{B}$
              \end{itemize}
        \item Absorption Law
              \begin{itemize}
                  \item $A \cup (A \cap B) = A$ and $A \cap (A \cup B) = A$
              \end{itemize}
        \item Set Difference Law
              \begin{itemize}
                  \item $A-B = A \cap \complement{B}$
              \end{itemize}
    \end{itemize}
    \section*{Functions}
    A function $f$ from a set $X$ to a set $Y$ denoted by $f: X \to Y$ where $X$ is the domain and $Y$ is the co-domain.
    Every element in $X$ relates to some element in $Y$ and each element in $X$ must map to at most one element in $Y$.
    \subsection*{One-to-One functions}
    A function is called one-to-one or injective when no two elements in $X$ point to the same element in $Y$.
    \subsection*{Onto functions}
    A function is called onto or surjective iff every element in $Y$ has at least one corresponding element in $X$.
    \subsection*{Inverse functions}
    $F^{-1}(y) = x\text{ where }F(x) = y$.
    $F(x)$ has to be both one-to-one and onto (bijective).
    \section*{Relations}
    A partition of set $A$ is a collection of nonempty, mutually disjoint subsets whose union is $A$, i.e. $A_1 \cup A_2 \cup A_3 \cup \ldots \cup A_i = A$
    \begin{itemize}
        \item Reflexive $ \iff \forall x \in A, x R x$
              \begin{itemize}
                  \item $x=x$
                  \item Each element is related to itself
              \end{itemize}
        \item Symmetric $ \iff \forall x, y \in A, x R y \implies y R x$
              \begin{itemize}
                  \item $|x-y| = |y-x|$
                  \item Each element related to another is related back to that element.
              \end{itemize}
        \item Transitive $\iff \forall x, y, z \in A, (x R y \land y R z)\implies x R z$
              \begin{itemize}
                  \item $A = B = C \therefore A=C$
              \end{itemize}
        \item Partition-induced relation $\iff \forall x, y \in A, x R y \iff (x, y) \in \text{single partition }A_i$
        \item Equivalence Relation: When a relation is relation is reflexive, symmetric and transitive.
        \item Identity Relation: $I_A = \{(x, y) \in A \times A~|~x=y\}$
    \end{itemize}
    \subsection*{Equivalence Classes}
    The subset of all elements that are related to $a$.

    Suppose $A$ is a set and $R$ is an equivalence relation on $A$. For each element $a$ in $A$, the \textbf{equivalence class of $a$}, $[a]$ or \textbf{class of $a$} is the set of all elements $x$ in $A$ such that $x$ is related to $a$ by $R$. $$[a] = \{x \in A~|~x~R~a\}$$
    \section*{Partial Order Relations}
    A relation that is reflexive, antisymmetric and transitive.
    \subsection*{Antisymmetry}
    A relation is antisymmetric if $$x R y \land y R x,~x = y$$
    A relation is \textbf{not} antisymmetric if $$x R y \land y R x\text{ but }x \neq y$$
    \section*{Examples}
    \newtheorem{example}{Example}
    \subsection*{Strong Induction}
    \begin{example}
        \begin{equation*}
            \begin{aligned}
                f(n) = \begin{cases}
                    1                   & \text{if }n=1      \\
                    3                   & \text{if }n=2      \\
                    f(n-1) + 2f(n-2) +3 & \text{if } n\geq 3
                \end{cases}
            \end{aligned}
        \end{equation*}
        Using mathematical induction, prove that for all $n \geq 3: f(n) \geq 2^n$

        Answer:
        \begin{equation*}
            \begin{aligned}
                f(k+1) =f(k)+ 2f(k-1)+3              \\
                \text{Use fact that $f(n) \geq 2^n$} \\
                f(k+1) \geq 2^k+2\cdot 2^{k-1} +3    \\
                \text{and } f(k+1) \geq 2^k + 2^k +3 = 2^{k+1} + 3 \geq 2^{k+1}
            \end{aligned}
        \end{equation*}
    \end{example}
    \begin{example}
        What is the smallest value of k such that any integer postage of k, or
        more, cents can be formed by using only 3-cent and 8-cent stamps?

        Use mathematical induction to prove that all integer postages of k, or more, cents can be
        obtained.
        \begin{equation*}
            \begin{aligned}
                \textbf{Basis: } P(n):                             & n = 3a + 8b       \\
                \textbf{Induction: } P(n+1):                       & n+1 = 3a' + 8b'   \\
                \textbf{Case 1: } \text{if } b \geq 1, n = 3a + 8b & \text{, so}       \\
                n+1 = 3a+1+8b                                      & = 8(b-1) + 9 + 3a \\
                n+1                                                & = 8(b-1) +3(a+3)  \\
                \textbf{Hence }a' = a+3      \text{ and }          & b' = b - 1        \\
                \textbf{Case 2: } \text{if } b = 0, n = 3a         &                   \\
                \text{Since }n \geq 14,~a \geq 5                                       \\
                n+1 = 1 + 15 + 3(l-5)                              & = 16+ 3(l-5)      \\
                \textbf{Hence }a' = l-5      \text{ and }          & b' = 2            \\
            \end{aligned}
        \end{equation*}
    \end{example}

\end{multicols}
\end{document}


\subsubsection{Bet sizing}
\label{sec:methods_pipeline_bet_sizing}

Labels and metalabels will be used together with other features to train a secondary
model whose output estimated probability for each new label will be used to size
the bet for that event. One way to do this is to scale the predicted probability,
by the budget total and get value in currency for the bet. However, if we did so,
there would not be any awareness about other concurrent bets, so we might misuse
the available budget to run other concurrent bets. Lopez de Prado in chapter 10
of \cite{lopez_de_prado} proposes the following:

\begin{enumerate}
  \item Let $p_{(x)}$ be the probability of label $x$ that takes the one of the
        values in $[-1, 1]$.
  \item Run a statistical test where $H_{0}: p_{(x=1)} = 0.5$ with the statistic
        $z = \frac{p_{(x=1)} - 0.5}{\sqrt{0.5(1-0.5)}} = \frac{p_{(x=1)} - 0.5}{0.5}$
  \item Let the bet size $m$ be: $m = 2 \Phi_{(z)} - 1$ where $\Phi_{(z)}$ is
        the cumulative distribution function of the standard normal
        distribution. $m \in [-1; 1]$
\end{enumerate}

Once we have a vector of $m$ values, which has a one to one relationship with
each label $x$, we can average the bet with those concurrent triple barrier
windows as the come in the pipeline. Averaging does not change the bet size
for open windows, but changes the size for new bets that overlap with open
windows.

Finally, a portfolio manager might also consider bet size discretization by
means of setting an integer number of equally sized budget partitions. The
discretized value of the bet after the average is $m^* = round\Bigg( \frac{m}{d} \Bigg) d$
where $d = \frac{1}{Number of partitions}$. And the only remaining step when
running the strategy is to scale the budget by $m^*$ to have the final bet size
in currency. The goal of this step is to reduce jitter which causes overtrading.
Typical number of partitions are below ten.


\documentclass[10pt]{article}

\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amsthm,amssymb, graphicx, multicol, array, enumerate, gensymb}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}

\newenvironment{problem}[2][Problem]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}

\begin{document}

\title{Mathematics problems}
\date{}
\maketitle

 \section{Elementary algebra}

\begin{problem}{1.1}
Simplify $$\frac{z^{17}}{z^3 \cdot z^5}$$
\end{problem}

\begin{problem}{1.2}
Solve for $x$:
$$6^2 \cdot 6^x = 6^6$$
\end{problem}

\begin{problem}{1.3}
Calculate the missing value. If $x \cdot y$ is 5, then $x^3y^3=\dots$
\end{problem}

\begin{problem}{1.4}
Calculate
$$\frac{\sqrt{2^{10}}}{\sqrt{4^3}}$$
\end{problem}

\begin{problem}{1.5}
True or False ($x$ and $y$ and $z$ are real numbers):
\begin{enumerate}[(a)]
    \item $x+y=y+x$
    \item $x(y+z)=xy+xz$
    \item $x^{y+z}=x^y+x^z$
    \item $\frac{x^y}{x^z}=x^{y-z}$
\end{enumerate}
\end{problem}

\begin{problem}{1.6}
Find the solution set for the inequality below:
$$\frac{2x-5}{2}\ge4$$
\end{problem}

\section{Functions of one variable}

\begin{problem}{2.1 (Based on SYD 2.5.6)}
The relationship between temperatures measured in Celsius and Fahrenheit is linear. 0\degree C is equivalent to 32\degree F and 100\degree C is the same as 212\degree F.
 Which temperature is measured by the same number on both scales?
\end{problem}

\begin{problem}{2.2}
Take the following function $f(x)=5x+4$. Find y if $f(y)=24$.
\end{problem}

\begin{problem}{2.3}
Find all values of x that satisfy:
$$10^{x^2-2x+2}=100$$
\end{problem}

\begin{problem}{2.4}
Solve the following problem. If the annual GDP growth of a country is 3\%, how long does it take the economy to double its GDP?
\end{problem}

\begin{problem}{2.5}
Calculate the following value
$$\ln(1/e)$$
\end{problem}

\section{Calculus}

\begin{problem}{3.1}
Calculate the following sum
$$\sum\limits_{i=0}^{\infty} \left( \frac{1}{8^i}+0.5^i\right)$$
\end{problem}

\begin{problem}{3.2}
Find the following limit
$$\lim\limits_{x \rightarrow 3}\frac{x-3}{2}$$
\end{problem}

\begin{problem}{3.3}
Find the slope of the function $f(x)=x^2-4$ at $(-1,-3)$.
\end{problem}

\begin{problem}{3.4}
Find the following derivative
$$\frac{\mathrm{d}}{\mathrm{d}\, x} \frac{x^2+3}{x+2}$$
\end{problem}

\begin{problem}{3.5}
Find the following second derivative
 $$\frac{\mathrm{d^2}}{\mathrm{d}\, x^2} 4x^3+4$$
\end{problem}

\begin{problem}{3.6}
Is the function  $f(x)=\frac{1}{x}$ continuous at $0$? Why?
\end{problem}

\begin{problem}{3.7}
Consider the following function. Find all of its stationary points and classify them as local minima, local maxima or inflection points. Also decide whether it is convex or concave. If it has one or more inflection points then define where it is locally concave or locally convex.
$$f(x)=3x^3-9x$$
\end{problem}

\begin{problem}{3.8}
Let $f(x,y)=x^2y^3$. Calculate $f(2,3)$
\end{problem}

\begin{problem}{3.9}
Consider the following function: $f(x,y)=\ln(x-y)$. For what combinations of $x$ and $y$ is this function defined?
\end{problem}

\begin{problem}{3.10}
Find the following partial derivative:
$$\frac{\partial^2}{\partial \, x^2} x^5+xy^3$$
\end{problem}

\begin{problem}{3.11}
Find the local maxima or minima of the following function:
$$f(x,y)=\sqrt{xy}-0.5x-0.5y$$
\end{problem}

\begin{problem}{3.12}
Solve the following constrained optimization problem using Lagrange's method:
$\max x^2y^2$ s.t. $x+y=5$
\end{problem}

\section{Linear algebra}

\begin{problem}{4.1}
Take the following matrices:
$$A=\begin{bmatrix} 2 & 3\\ 4 & 1 \\ 1 & 2\end{bmatrix}$$
$$B=\begin{bmatrix} 1 & 4 & 1\\2 & 1 & 2\end{bmatrix}$$
What is $A \cdot B$?
\end{problem}

\begin{problem}{4.2}
Take the following matrices:
$$A=\begin{bmatrix} 2 & 3\\ 4 & 1 \\ 1 & 2\end{bmatrix}$$
$$B=\begin{bmatrix} 1 & 4 & 1\\2 & 1 & 2\end{bmatrix}$$
What is $B \cdot A$?
\end{problem}

\begin{problem}{4.3}
What is the transpose of the following matrix?
$$\begin{bmatrix}3.3 & 5.1 & 4.7\\ 2 & 6.1 & 1.23 \\ 4 & 5.76 & 0\end{bmatrix}$$
\end{problem}

\begin{problem}{4.4}
Calculate the determinant of
$$\begin{bmatrix}2 & 3 \\ 4 & 5 \end{bmatrix} $$
\end{problem}

\section{Probability theory}

\begin{problem}{5.1}
You run an experiment where you flip a coin four times. Each time you get either heads (H) or tails (T). What is the sample space of your experiment?
\end{problem}

\begin{problem}{5.2}
Assume that in a certain country 1\% of the population uses a certain drug. You have a way to test drug use, which will give you a positive result in 99\% of the cases where the individual is indeed a drug user and a negative result in 99.5\% of the cases where the individual doesn't use the drug. What is the probability that someone with a positive drug test is indeed a drug user?
\end{problem}

\begin{problem}{5.3}
You run an experiment in which you toss a dice twice and sum up the results. What is the expected value of this sum?
\end{problem}
\end{document}


\section{Implementation}

\subsection{Dataset Generation}
The first step of the experiment is to generate an artificial dataset with $P$ random N-dimensional feature vectors and binary labels $\mathcal{D} = \{ \xi^\mu, S^\mu \,|\, \xi^\mu \in \mathds{R}^N, S^\mu \in \{+1, -1\} \}_{\mu=1}^P$.
The feature vectors $\xi^\mu$ have independent random components $\xi_j^\mu \sim \mathcal{N}(0, 1)$ and are generated using the \texttt{randn} command in Matlab.
The labels $S^\mu$ are assuming the values $\{+1, -1\}$ with equal probability of $1/2$.

\begin{lstlisting}[language=Matlab]
  function [X, y] = generate_dataset(P, N)
      X = randn(P, N);
      y = iff(rand(P, 1) < 0.5, -1, 1);
  end
\end{lstlisting}

\subsection{Perceptron Training}
The next step is to use the Rosenblatt algorithm to implement and train a perceptron on the generated dataset.
The Rosenblatt algorithm is a sequential procedure that learns the weights of the perceptron from the examples.
The weights are initialized to zero, then the training examples are presented one by one to the perceptron for a given number of iterations.
If the example is correctly classified according to the current weights, no update is performed;
if the example is wrongly classified, the weights are updated according to the following rule:
\begin{gather}
    \mathsf{\bm{w}}(0) = 0, \\
    \mathsf{\bm{w}}(t + 1) =
        \begin{cases}
            \mathsf{\bm{w}}(t) + \frac{1}{N} \xi^{\mu(t)} S^{\mu(t)} &\text{if $E^{\mu(t)} \leq 0$}\\
            \mathsf{\bm{w}}(t) &\text{else}
        \end{cases},
    \label{eq:update-rule}
\end{gather}
where $E^{\mu(t)} = \mathsf{\bm{w}}(t) \cdot \xi^{\mu(t)} S^{\mu(t)}$, $t = 1, 2, ...$ represents the current time step and $\mu(t) = 1, 2, ..., P, 1, 2, ...$ identifies the current example.
The algorithm is stopped either when $E^\mu > 0$ for all examples in the dataset or the maximum number of iterations $n_{max}$ is reached.

\subsection{Experiments}
For a fixed value of $P$ and $N$, $n_D$ independent datasets are generated.
A new perceptron is trained on each dataset for at most $n_{max}$ iterations.
For each dataset, we compute the rate fraction $Q_{l.s.}$ of successful runs, where a run is successful if the perceptron correctly classifies each example in the dataset at the end of the training phase.
$Q_{l.s.}$ gives an estimate of the probability that the perceptron finds a linear separation in a random dataset of $P$ independent points in $N$ dimension.

Multiple experiments are run for different values of $P$ and $N$ in order to compute $Q_{l.s.}$ as a function of $\alpha = P / N$.
In other words, we study the probability of the perceptron to find a linear separation as a function of the rate between the number of examples and the dimension of the input.

\subsection{Bonus}
\subsubsection{Weight Update Criterion}
We modified the function to train the perceptron to update the weights vector $\mathsf{\bm{w}}$ if $E^\mu < c$ instead of $E^\mu < 0$ (see \cref{eq:update-rule}).
Also, we changed the algorithm to stop the training only if $E^{\mu} > c$ for all examples in the dataset or the maximum number of iterations $n_{max}$ is reached.

\subsubsection{Inhomogeneous Perceptrons}
Rosenblatt perceptrons only learns separation hyperplanes that goes through the origin (homogeneous).
In general, a solution to the classification problem may be a hyperplane that does not go through the origin (inhomogeneous), but be in the form:
\begin{equation}
    S = sign(\mathsf{\bm{w}} \cdot \xi + \theta) = \pm 1
\end{equation}

It is possible to generalize the Rosenblatt perceptron to learn also inhomogeneous separation hyperplanes by adding an artificial dimension to the dataset and forcing it to a non-zero constant (e.g. $-1$) and increasing the size of the weights vector $\mathsf{\bf{w}}$ by $1$ (the last element of $\mathsf{\bf{w}}$ corresponds to the intercept $\theta$).


\section{ML}

\subsection{Principal Component Analysis}

\marginnote{
\begin{theorem}
Let $L$ be a $k$-dimensional subspace of $\mathbb{R}^n$.
Then each vector $x$ in $\mathbb{R}^n$ can be written uniqely in the form
\[
x = h + p
\]
where $p$ is in $L$ and $h$ is in $L^{\perp}$.
\end{theorem}
\begin{proof}
Let ${e_1, \ldots, e_k}$ be an orthogonal basis for $L$, and define $p$ as
\[
p = \frac{x \cdot e_1}{e_1 \cdot e_1} + \ldots + \frac{x \cdot e_k}{e_k \cdot e_k}.
\]
Being a linear combination of $e_1, \ldots, e_k$, $p$ is in $L$.
Let $h = x - p$.
Since each $e_{i}$, $i = 1, \ldots, k$ is orthogonal to all the rest basis vectors,
it follows that for all $i$
\begin{align*}
h \cdot e_i &= (x - p) \cdot e_i \\
&= x \cdot e_i - \left(\frac{x \cdot e_i}{e_i \cdot e_i}\right) \cdot e_i \underbrace{- 0 - 0 \ldots - 0}_{(k-1) \text{ times}} \\
&= x \cdot e_i - x \cdot e_i = 0.
\end{align*}
Thus, we can conclude that $h$ is in $L^{\perp}$.

To show that the decomposition stated in the theorem is unique,
suppose there is another one: $x = h_1 + p_1$ where $h_1 \in L$ and $p_1 \in L$.
Then $h + p = h_1 + p_1$ as both sides equal $x$ and
\[
p - p_1 = h - h_1.
\]
It follow that the vector $v = p - p_1$ is in $L$ and in $L^{\perp}$
which is possible if and only if $v = 0$.
\end{proof}

From this theorem it directly follows that
\[
\lVert x \rVert^2 = \lVert h \rVert^2 + \lVert p \rVert^2.
\]
Thus, holding the observation vector $x$ fixed
maximizing $\lVert h \rVert$ and minimizing $\lVert p \rVert$
are equivalent in order to get the longest projection of $x$ onto $L$.
}

PCA is a tool to preserve as much variability in $n$-dimensional data as possible
by projecting it onto $q$-dimensional subspace spanned by the principal components.
There are two ways to find these principal components.
One can either maximize the length of the projection of some vector $x$
onto the $q$-dimensional subspace,
or minimize the length of its orthogonal component.
Both ways are equivalent which can be shown by the orthogonal decomposition theorem.
However, there is also a geometrical interpretaion.

\begin{theorem}
In PCA the best-fit $q$-dinesional subspace for a set of observations $x_1, \ldots, x_n$
is defined by solving either
\[
\sum_{n=1}^N \lVert h \rVert^2  \to \min_{v_1, \ldots, v_k}
\]
which minimizes the distance between the observation and the subspace, or
\[
\sum_{n=1}^N \lVert p \rVert^2  \to \max_{w_1, \ldots, w_k}.
\]
which maximizes the varince of projected data.
\end{theorem}

\begin{proof}
Consider a space of $n$ observations and two factors.
Additionaly assume that all the observations are centred and
the best-fit line is already found.

Let us drop the perpendiculars from all the observation points onto the best-fit line
and denote them as $a_i$, $i = 1, \ldots, n$.
Then there is a right triangle for each observation.
We denote another leg as $b_i$ and the distance of observation point from the origin as $c_i$.
Thus, for every point we can apply the Pythagorean theorem: $c_i^2 = a_i^2 + b_i^2$.
Summing through all the observations, we get
%\begin{marginfigure}
%  \includegraphics[scale=0.7]{figures/05_pca.pdf}
%  \caption{Equivalence of minimizing the perpendicular lengths $a_i$ and
%  maximazing the projection length $b_i$}
%  \label{fig:pca}
%end{marginfigure}
\[
\sum\limits_{i=1}^n c_i^2 = \sum\limits_{i=1}^n a_i^2 + \sum\limits_{i=1}^n b_i^2
\]
Being unable to change the distance of the observation from the origin,
we can either minimize $\sum_{i=1}^n a_i^2$, or maximize $\sum_{i=1}^n b_i^2$
in order to get the principal component.

\end{proof}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 9}

Prove that if a binary tree has depth $n$, then it has at most $2^n$ leaves.

\subsection*{Solution}

Proof is given by induction on depth $n$ of binary tree.
By definition of \textit{binary} trees, it is assumed each node has at most two children.
\begin{proof}
Induction on depth $n$ of the binary tree.
\begin{itemize}
\item[] Argument is trivially true for \emph{base step} $n=0$ where there is only one node (if any) with no children.
In this case, the node (if exists) will be the only leaf and number of leaves is either zero or one.
Therefore, a binary tree with depth $0$ has at most $2^0=1$ leaf.
\item[] When $n=1$, there is only one parent node and by definition it has at most two children.
It follows that any additional node should be child of one of those children which will contradict our earlier assumption that depth of tree is $n=1$.

\item[] \emph{Inductive hypothesis} is formed as follows.
A binary tree with depth $n$ has at most $2^n$ leaves.
We show that a binary tree with depth $n+1$ has at most $2^{n+1}$ leaves.

\item[] The only way to obtain a binary tree with depth $n+1$ is to add at least one child to at least one of the leaves of a binary tree with depth $n$.
By inductive hypothesis, as there are at most $2^n$ leaves in a binary tree with depth $n$, we can at most promote $2^n$ leaves to parents.
As each parent has at most two children, maximum number of children in a binary tree with depth $n+1$ will be $2 \times 2^n = 2^{n+1}$.
As all former leaves will be parents with 2 children, any additional node should be child of one of the new leaves which contradicts our earlier assumption about depth of the tree.

\end{itemize}
\end{proof}


\section{Axioms}

\begin{axiom}[\cindex{Axiom of Extensionality}]\label{axiomofextensionality}
    If $X$ and $Y$ have the same elements, then $X=Y$.
    \begin{equation}
        \forall u (u \in X \leftrightarrow u \in Y ) \rightarrow X = Y
    \end{equation}
\end{axiom}

\begin{axiom}[\cindex{Axiom of Pairing}]
    For any $a$ and $b$ there exists a set $\set{a,b}$ that contains exactly $a$ and $b$.
    \begin{equation}
        \forall a \forall b \exists c \forall x (x \in c \leftrightarrow x = a \vee x = b )
    \end{equation}
    A \cindex{singleton} $\set{a}$ is the set $\set{a} = \set{a, a}$. An \cindex{ordered pair} $(a,b)$ is the set $(a,b) = \set{\set{a}, \set{a,b}}$.
\end{axiom}

\begin{axiom}[\cindex{Axiom Schema of Seperation}]
    If $P$ is a property with parameter $p$, then for any $X$ and $p$ there exists a set $Y = \set{u \in X: P(u,p)}$ that contains all those $u \in X$ that have property $P$.
    \begin{equation}
        \forall X \forall p \exists Y \forall u \left(u \in Y \leftrightarrow u \in X \wedge \varphi(u, p) \right)
    \end{equation}
    If define class $C = \varphi(u, p)$, then $\forall X \exists Y (C \wedge X )= Y$, so a subclass of a set is a set. The empty class $\emptyset = \set{u: u \neq u} $ is a \cindex{empty set}.
\end{axiom}

\begin{axiom}[\cindex{Axiom of Union}]
    For any $X$ there exists a set $Y = \cup X$, the union of all elements of $X$.
    \begin{equation}
        \forall X \exists Y \forall u \left(u \in Y \leftrightarrow \exists z (z \in X \wedge u \in z ) \right)
    \end{equation}
\end{axiom}

\begin{axiom}[\cindex{Axiom of Power Set}]
    For any $X$ there exists a set $Y = P(X)$, the set of all subset of $X$.
    \begin{equation}
        \forall X \exists Y \forall u (u \in Y \leftrightarrow u \subset X )
    \end{equation}
\end{axiom}

\begin{axiom}[\cindex{Axiom of Infinity}]
    There exists an infinite set.
    \begin{equation}
        \exists S \left( \emptyset \in S \wedge (\forall x \in S ) x \cup \set{x} \in S \right)
    \end{equation}
    A set $S$ with above property is called \cindex{inductive}.
\end{axiom}

\begin{axiom}[\cindex{Axiom Schema of Replacement}]
    If a class $F$ is a function, then for any $X$ there exists a set $Y = F(X) = \set{F(x): x \in X}$.
    \begin{equation}
        \forall x \forall y \forall z \left( \varphi(x,y,p) \wedge \varphi(x,z,p) \rightarrow y = z \right) \rightarrow \forall X \exists Y \forall y \left( y \in Y \rightarrow (\exists x \in X ) \varphi(x,y,p) \right)
    \end{equation}
    So if a class $F$ is a function and $\domain{f}$ is a set, then $\range{f}$ is a set.
\end{axiom}

\begin{axiom}[\cindex{Axiom of Regularity}]\label{axiomofregularation}
    Every nonempty set has an $\in$-minimal element.
\end{axiom}

\begin{axiom}[\cindex{Axiom of Choice}]\label{axiomofchoice}
    Every family of nonempty set has a choice function.
\end{axiom}

The \thmref{axiomofextensionality} to \thmref{axiomofregularation} is the \cindex{Zermelo-Fraenkel} axiomatic set theory \cindex{ZF}. \cindex{ZFC} denote the ZF + \cindex{AC}, the axiom of choice.

\begin{theorem}[\cindex{Russell's Paradox}]
    There is no set whose elements are all those sets that are not member of themselves: $S = \set{X : X \notin X}$. So the set of all set does not exist.
\end{theorem}

\begin{definition}
    A binary relation $f$ is a \cindex{function} if $(x,y) \in f$ and $(x, z) \in f$ implies $y = z$. For a function $f$ from $X$ to $Y$ $f : X \rightarrow Y$, if $Y = \range{f}$, $f$ is \cindex{onto}. If $f(x) = f(y)$ implies $x=y$, $f$ is \cindex{one-to-one}. The \cindex{inverse image} $f_{-1} (Y) = \set{x: f(x) \in Y}$. If $f$ is one-to-one, then the \cindex{inverse} is $f^{-1} (y) = x$.
\end{definition}

\begin{definition}
    The \cindex{restriction} of a function $f$ to a set $X$ is:
    \begin{equation}
        f\restriction_X = \set{(x,y) \in f : x \in X}
    \end{equation}
\end{definition}

\begin{definition}[class]
    if $\varphi(x, p_1, \dots, p_n)$ is a formula, then $C = \set{x: \varphi(x, p_1, \dots, p_n)}$ is a \cindex{class}. So a formula defines a class.
\end{definition}

\begin{definition}[universe]
    The \cindex{universe} is the class of all sets: $V = \set{x: x = x}$.
\end{definition}

\begin{definition}
    A class that is not a set is a \cindex{proper class}.
\end{definition}


\section*{Exercises}

\begin{ex}
  Find the roots of the polynomial $p(z) = z^2-6z+13$.
  \begin{sol}
    By the quadratic formula, $\displaystyle z=\frac{6\pm\sqrt{16}}{2}
      = 3\pm 2i$.
  \end{sol}
\end{ex}

\begin{ex}
  Find the roots of $p(z) = z^3 + 5z^2 + 4z - 10$. Hint: one of the
  roots is $z=1$.
  \begin{sol}
    $p(z)$ factors as $(z-1)(z^2+6z+10)$. The roots are $z=1$,
    $z=-3+i$, and $z=-3-i$.
  \end{sol}
\end{ex}

\begin{ex}
  Factor $z^3 - 3z + 2$ into a product of linear factors.
  \begin{sol}
    By trial and error, we find the roots $z=1$ and $z=-2$. Moreover,
    $z=1$ is a double root. We have $z^3 - 3z + 2 = (z-1)(z-1)(z+2)$.
  \end{sol}
\end{ex}

\begin{ex}
  Let $p(x) =a_nx^n+a_{n-1}x^{n-1}+\ldots+a_1x+a_0$ be a
  polynomial with real coefficients, i.e., such that all the $a_k$
  are real numbers. Suppose that $z$ is a root of $p$. Show that
  $\conjugate{z}$ is also a root of $p$.
  \begin{sol}
    If $p(z) =0$ and $a_0,\ldots,a_k$ are real, we have
    \begin{eqnarray*}
      p(\conjugate{z})
      &=& a_n\conjugate{z}^n+a_{n-1}\conjugate{z}^{n-1}+\ldots+a_1\conjugate{z}+a_0 \\
      &=&\conjugate{a_n}\ \conjugate{z}^n+\conjugate{a_{n-1}}
          \ \conjugate{z}^{n-1}+\ldots+\conjugate{a_1}
          \ \conjugate{z}+\conjugate{a_0}
      \\
      &=& \conjugate{a_nz^n}+\conjugate{a_{n-1}z^{n-1}}+\ldots+\conjugate{a_1z}+
          \conjugate{a_0} \\
      &=& \conjugate{a_nz^n+a_{n-1}z^{n-1}+\ldots+a_1z+a_0} \\
      &=& \conjugate{p(z)} \\
      &=& 0.
    \end{eqnarray*}
  \end{sol}
\end{ex}


\section{Linear transformations}

\begin{outcome}
  \begin{enumerate}
  \item Determine whether a vector function $T:\R^n\to\R^m$ is a
    linear transformation.
  \end{enumerate}
\end{outcome}

In calculus, a \textbf{function}%
\index{function} (or \textbf{map}%
\index{map|see{function}}) $f:\R\to\R$ is a rule that maps a real
number $x\in\R$ to a real number $f(x)\in\R$. In linear algebra, we
can generalize this concept to vectors. A \textbf{vector function}%
\index{vector function}%
\index{function!vector function} $T:\R^n\to\R^m$ is a rule that inputs
an $n$-dimensional vector $\vect{v}\in\R^n$ and outputs an
$m$-dimensional vector $T(\vect{v})\in\R^m$. The following are some
examples of vector functions:
\begin{equation}\label{eqn:vector-functions}
  T_1\paren{\begin{mymatrix}{c} x \\ y \end{mymatrix}}
  = \begin{mymatrix}{c} x^2 \\ x+y \\ y^2 \end{mymatrix},\quad
  T_2\paren{\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}}
  = \begin{mymatrix}{c} x+y \\ x+y+z \\ 0 \end{mymatrix},\quad
  T_3\paren{\begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}}
  = \begin{mymatrix}{c} e^{x+z} \\ \sqrt{y} \end{mymatrix}.
\end{equation}
Of these, the first is a function $T_1:\R^2\to\R^3$, the second is a
function $T_2:\R^3\to\R^3$, and the third is a function
$T_3:\R^3\to\R^2$.  We can evaluate a vector function by applying it
to a vector, for example,
\begin{equation*}
  T_1\paren{\begin{mymatrix}{c} 1 \\ 2 \end{mymatrix}}
  = \begin{mymatrix}{c} 1^2 \\ 1+2 \\ 2^2 \end{mymatrix}
  = \begin{mymatrix}{c} 1 \\ 3 \\ 4 \end{mymatrix},\quad
  T_1\paren{\begin{mymatrix}{c} 0 \\ 1 \end{mymatrix}}
  = \begin{mymatrix}{c} 0^2 \\ 1+1 \\ 1^2 \end{mymatrix}
  = \begin{mymatrix}{c} 0 \\ 1 \\ 1 \end{mymatrix},
\end{equation*}
and so on. The study of arbitrary vector functions and their
derivatives and integrals is the subject of {\em multivariable
  calculus}%
\index{calculus!multivariable}%
\index{multivariable calculus}. In linear algebra, we will only be
concerned with \textbf{linear vector functions}%
\index{vector function!linear|see{linear transformation}}%
\index{linear function|see{linear transformation}}%
\index{function!linear|see{linear transformation}}, which are
also called \textbf{linear transformations}%
\index{linear transformation} or \textbf{linear maps}%
\index{linear map|see{linear transformation}}. They are defined as follows.

\begin{definition}{Linear transformation}{linear-transformation}
  A vector function $T:\R^n\to \R^m$ is called a \textbf{linear
    transformation}%
  \index{linear transformation!on $\R^n$}, or simply \textbf{linear}, if it
  satisfies the following two conditions:
  \begin{enumerate}
  \item $T$ preserves addition, i.e., for all\/
    $\vect{v},\vect{w}\in\R^n$, we have
    $T(\vect{v}+\vect{w}) = T(\vect{v}) + T(\vect{w})$;
  \item $T$ preserves scalar multiplication, i.e, for all\/
    $\vect{v}\in\R^n$ and scalars $k$, we have
    $T(k\vect{v}) = kT(\vect{v})$.
  \end{enumerate}
\end{definition}

\begin{example}{Linear and non-linear transformations}{linear-transformation}
  Which of the vector functions in {\eqref{eqn:vector-functions}} are
  linear transformations?
\end{example}

\begin{solution}
  \begin{enumerate}
  \item[(a)] The function $T_1$ is not a linear transformation. For
    example, let $\vect{v}=\begin{mymatrix}{r} 1 \\
      0 \end{mymatrix}$. Then
    \begin{equation*}
      T_1(\vect{v})
      ~=~ T_1\paren{\begin{mymatrix}{c} 1 \\ 0 \end{mymatrix}}
      ~=~ \begin{mymatrix}{c} 1 \\ 1 \\ 0 \end{mymatrix}
      \quad\mbox{and}\quad
      T_1(2\vect{v})
      ~=~ \paren{\begin{mymatrix}{c} 2 \\ 0 \end{mymatrix}}
      ~=~ \begin{mymatrix}{c} 4 \\ 2 \\ 0 \end{mymatrix}
      ~\neq~ 2\begin{mymatrix}{c} 1 \\ 1 \\ 0 \end{mymatrix}.
    \end{equation*}
    Since $T_1(2\vect{v}) \neq 2T_1(\vect{v})$, the vector function
    $T_1$ does not preserve scalar multiplication, and therefore it is
    not a linear transformation.
  \item[(b)] The function $T_2$ is a linear transformation. For
    example, to prove that $T_2$ preserves addition, consider two
    arbitrary vectors
    \begin{equation*}
      \vect{v} =
      \begin{mymatrix}{c}
        x_1 \\
        y_1 \\
        z_1 \\
      \end{mymatrix}
      \quad\mbox{and}\quad
      \vect{w} =
      \begin{mymatrix}{c}
        x_2 \\
        y_2 \\
        z_2 \\
      \end{mymatrix}.
    \end{equation*}
    We have
    \begin{equation*}
      T_2(\vect{v}+\vect{w})
      ~=~ T_2\paren{
        \begin{mymatrix}{c}
          x_1+x_2 \\
          y_1+y_2 \\
          z_1+z_2 \\
        \end{mymatrix}}
      ~=~ \begin{mymatrix}{c}
        (x_1+x_2)+(y_1+y_2) \\
        (x_1+x_2)+(y_1+y_2)+(z_1+z_2) \\
        0
      \end{mymatrix}
    \end{equation*}
    and
    \begin{equation*}
      T_2(\vect{v})+T_2(\vect{w})
      ~=~
      \begin{mymatrix}{c}
        x_1+y_1 \\
        x_1+y_1+z_1 \\
        0 \\
      \end{mymatrix}
      + \begin{mymatrix}{c}
        x_2+y_2 \\
        x_2+y_2+z_2 \\
        0 \\
      \end{mymatrix}
      ~=~ \begin{mymatrix}{c}
        (x_1+y_1)+(x_2+y_2) \\
        (x_1+y_1+z_1)+(x_2+y_2+z_2) \\
        0 \\
      \end{mymatrix}.
    \end{equation*}
    Since the two sides are evidently equal, $T_2$ preserves
    addition. The fact that it preserves scalar multiplication can be
    shown by a similar calculation.
  \item[(c)] The function $T_3$ is not a linear transformation. For
    example, consider
    $\vect{v}=\begin{mymatrix}{c}0\\1\\0\end{mymatrix}$ and
    $\vect{w}=\begin{mymatrix}{c}1\\1\\0\end{mymatrix}$.
    Then
    \begin{equation*}
      T_3(\vect{v}+\vect{w})
      ~=~ T_3\paren{\begin{mymatrix}{c} 1 \\ 2 \\ 0\end{mymatrix}}
      ~=~ \begin{mymatrix}{c} e \\ \sqrt{2} \end{mymatrix},
    \end{equation*}
    and
    \begin{equation*}
      T_3(\vect{v})+T_3(\vect{w})
      ~=~ \begin{mymatrix}{c} 1 \\ 1 \end{mymatrix}
      + \begin{mymatrix}{c} e \\ 1 \end{mymatrix}
      ~=~ \begin{mymatrix}{c} e+1 \\ 2 \end{mymatrix}.
    \end{equation*}
    Since $T_3(\vect{v}+\vect{w})\neq T_3(\vect{v})+T_3(\vect{w})$,
    the vector function $T_3$ does not preserve addition, and therefore
    it is not linear.
  \end{enumerate}
\end{solution}

An easy fact about linear transformation is that they preserve the
origin, i.e., they satisfy $T(\vect{0}) = \vect{0}$. This can be seen,
for example, by considering
$T(\vect{0}) = T(\vect{0}+\vect{0}) = T(\vect{0}) + T(\vect{0})$ and
then subtracting $T(\vect{0})$ from both sides of the equation.  This
gives an easier way to see that $T_3$ in the above example is not a
linear transformation, since $T_3(\vect{0}) \neq \vect{0}$. On the
other hand, of course not every function that preserves the origin is
linear. For example, $T_1$ is not linear although it satisfies
$T_1(\vect{0})=\vect{0}$.

The following characterization of linearity is often useful, as it
permits us to check just one property instead of two.

\begin{proposition}{Alternative characterization of linear transformations}{linear-transformation-alternative}
  A vector function $T:\R^n\to \R^m$ is linear if and only if it
  satisfies the following condition, for all
  $\vect{v},\vect{w}\in\R^n$ and scalars $a,b$:
  \begin{equation*}
    T(a\vect{v}+b\vect{w}) = aT(\vect{v}) + bT(\vect{w}).
  \end{equation*}
\end{proposition}

\begin{proof}
  First, assume that $T$ is linear. Then from preservation of addition
  and scalar multiplication, we have
  $T(a\vect{v}+b\vect{w}) = T(a\vect{v})+T(b\vect{w}) = aT(\vect{v}) +
  bT(\vect{w})$. Conversely, assume that $T$ satisfies
  $T(a\vect{v}+b\vect{w}) = aT(\vect{v}) + bT(\vect{w})$ for all
  vectors $\vect{v},\vect{w}$ and scalars $a,b$. Then we get
  preservation of addition by setting $a=b=1$, and preservation of
  scalar multiplication by setting $b=0$.
\end{proof}


\color{black}
\subsection*{action primitives from in-game data}

Given the sequence
\begin{equation*}
\vec{x}[1]:\vec{x}[2]:\vec{x}[3]:\vec{x}[3]:\cdots:\vec{x}[n]
\end{equation*}
of subsequent 3D locations contained in file \texttt{q3dm1-path2.csv}, compute a sequence
\begin{equation*}
\vec{v}[1]:\vec{v}[2]:\vec{v}[3]:\vec{x}[3]:\cdots:\vec{v}[n-1]
\end{equation*}
of velocity vectors where
\begin{equation*}
\vec{v}[t] = \vec{x}[t+1] - \vec{x}[t]
\end{equation*}

\vspace{2cm}
Given your sequence of velocity vectors, compute their average
\begin{equation*}
\mathbb{E} \bigl[ \vec{v} \bigr] = \frac{1}{n-1} \sum_{t=1}^{n-1} \vec{v}[t]
\end{equation*}
Enter your result here. That is, replace the dots in the following expression by the appropriate numbers rounded to \emph{four} decimals. \color{blue}
%%%%%
%%%%% enter your answer here, i.e. replace the \ldots with the numbers you computed
%%%%%
\begin{equation*}
\mathbb{E} \bigl[ \vec{v} \bigr] = \begin{bmatrix} -0.338 \\  0.1112 \\ 0.     \end{bmatrix}
\end{equation*}
%%%%%
%%%%%
%%%%%
\color{black}

\vspace{2cm}
Run $k$-means clustering on the velocity vectors $\vec{v}[t]$ to estimate representative velocities or \emph{action primitives} $\vec{a}_1, \vec{a}_2, \ldots, \vec{a}_k$. For $k=24$, compute the average
\begin{equation*}
\mathbb{E} \bigl[ \vec{a} \bigr] = \frac{1}{k} \sum_{i=1}^{k} \vec{a}_i
\end{equation*}
Enter your result here. That is, replace the dots in the following expression by the appropriate numbers rounded to \emph{four} decimals. \color{blue}
%%%%%
%%%%% enter your answer here, i.e. replace the \ldots with the numbers you computed
%%%%%
\begin{equation*}
\mathbb{E} \bigl[ \vec{a} \bigr] = \begin{bmatrix} -7.8100e-02 \\ -5.9163e+00 \\ -4.0000e-04 \end{bmatrix}
\end{equation*}
%%%%%
%%%%%
%%%%%
\color{black}


\subsection{Polynomials}
\noindent
We'll use our derivative formula and induction to show that
\begin{equation*}
	\Laplace{t^n} = \frac{n!}{s^{n+1}} \text{, } n \geq 0.
\end{equation*}
Although the formula for $n = 0$ is clearly the same as $\Laplace{1}$, we'll use $n = 1$ as a base case to get a little more insight into where the formula comes from.\\
\begin{equation*}
	\Laplace{1} = \Laplace{t'} = \frac{1}{s}.
\end{equation*}
Using our derivative formula,
\begin{align*}
	\frac{1}{s} &= s\Laplace{t} - 0^{1} \\
	&\implies \Laplace{t} = \frac{1}{s^2}.
\end{align*}
Assuming the following is true,
\begin{equation*}
	\Laplace{t^n} = \frac{n!}{s^{n+1}} \text{, } n \geq 0.
\end{equation*}
We'll show that the $n+1$ case follows.
\begin{equation*}
	\Laplace{t^n} = \Laplace{\left(\frac{t^{n+1}}{n+1}\right)'} = \frac{n!}{s^{n+1}}.
\end{equation*}
Using our derivative formula and the linearity of the Laplace transform,
\begin{align*}
	\frac{n!}{s^{n+1}} &= \frac{s}{n+1}\Laplace{t^{n+1}} - 0^{n+1} \\
	&\implies \Laplace{t^{n+1}} = \frac{(n+1)!}{s^{n+2}},
\end{align*}
which is the $n+1$ case, meaning we have proven the general formula as correct.


\subsection{Defining circles}

\(x^2 + y^2 = r^2\)

\subsection{Area of a circle}

\(A=\pi r^2\)

\subsection{Circumference of a circle}

\(C=2\pi r\)


\lab{Complex Numbers}{Complex Numbers}
\label{Lab:complex_intro}

\objective{Create visualizations of complex functions. Visually estimate their zeros and poles, and gain intuition about their behavior in the complex plane.}

\section*{Representations of Complex Numbers}

A complex number $z = x+iy$ can be written in \emph{polar coordinates} as $re^{i\theta}$ where
\begin{itemize}
\item $r=\sqrt{x^2+y^2}$ is the magnitude of $z$, and
\item $\theta = \arctan(y/x)$ is the angle between $z$ and 0, as in Figure \ref{fig:polar_coords}.
\end{itemize}
Conversely, Euler's formula implies $re^{i\theta} = r\cos(\theta) + ir\sin(\theta)$. Then if we set $re^{i\theta}=x+iy$ and equate real and imaginary parts, we find $x=r\cos(\theta)$ and $y=r\sin(\theta)$.

\begin{figure}
\begin{tikzpicture}[dot/.style={circle,fill=black,minimum size=3pt,inner sep=0pt,
            outer sep=-1pt}, >=stealth', thick, xscale=1.4]

\draw[-](-.5,0)--(3,0);
\draw[-](0,-.5)--(0,3);

\draw[-, dashed, gray, anchor=east](2.6,2.3)--(0,2.3);
\node[draw=none]()at(-.3,2.3){$iy$};
\draw[-, dashed, gray, anchor=north ](2.6,2.3)--(2.6,0);
\node[draw=none]()at(2.6,-.3){$x$};

\draw[gray](.75,0) arc (0:45:.7 and .7);

\draw[-](0,0)--(2.6,2.3);
\node[dot,draw](point)at(2.6,2.3){};
\node[draw=none]()at(.9,.35){$\theta$};

\draw [decorate,decoration={brace,amplitude=10pt},rotate=-45, gray] (0,0) --(.2,3.45);
\node[draw=none]()at(.95,1.55){$r$};

\end{tikzpicture}
\caption{The complex number represented by the black dot equals both $x+iy$ and $re^{i\theta}$, when $\theta$ is written in radians.}
\label{fig:polar_coords}
\end{figure}

NumPy makes it easy to work with complex numbers and convert between coordinate systems.
The function \li{np.angle()} returns the angle of a complex number (between $-\pi$ and $\pi$) and the function \li{np.absolute()} returns the magnitude.
Use these to compute $\theta$ and $r$, respectively.
These functions also operate elementwise on NumPy arrays.

Note that in Python, \li{1j} is used for the complex number $i=\sqrt{-1}$.
See the code below for an example.

\begin{lstlisting}
>>> import numpy as np
>>> from matplotlib import pyplot as plt
# Set z = 2 - 2i
>>> z = 2 - 2*1j
>>> theta = np.angle(z)
>>> r = np.absolute(z)
# np.angle() returns a value between -pi and pi.
>>> print r, theta
(2.8284271247461903, -0.78539816339744828)
# Check that z=re^(i*theta)
>>> np.allclose(z, r*np.exp(1j*theta))
True
\end{lstlisting}

\section*{Visualizing complex functions}
Suppose we wish to graph a function $f(z): \mathbb{C} \rightarrow \mathbb{C}$.
The difficulty is that $\mathbb{C}$ has 2 real dimensions, so the graph of $f$ should use 4 real dimensions.
 Since we already have ways to visualize 3 dimensions, we should choose one dimension to ignore.
 We will ignore the magnitude $r = |f(z)|$ of the output.

To visualize $f$, we will assign a color to each point $z \in \mathbb{C}$.
The color will correspond to the angle $\theta$ of the output $f(z)$.
As an example, we have plotted the identity function $f(z)=z$ in Figure \ref{fig:identity}.
As $\theta$ goes from 0 to $2\pi$, the colors cycle smoothly counterclockwise from red to green to purple and back to red.

This kind of plot uses rectangular coordinates in the domain and polar coordinates (or rather, just the $\theta$-coordinate) in the codomain.
Note that this kind of plot tells us nothing about $|f(z)|$.

You can create the plot in Figure \ref{fig:identity} as follows.
Begin by creating a grid of complex numbers.
We create the real and imaginary parts separately, and then use \li{np.meshgrid()} to turn them into a single array of complex numbers.
\begin{lstlisting}
>>> x = np.linspace(-1, 1, 401)
>>> y = np.linspace(-1, 1, 401)
>>> X, Y = np.meshgrid(x, y)
>>> Z = X + 1j*Y
\end{lstlisting}

Now we compute the angles of the points in \li{Z} and plot them using \li{plt.pcolormesh()}.
We use the colormap \li{'hsv'}, which is red at both ends, so that $0$ and $2 \pi$ will map to the same color.

\begin{lstlisting}
>>> plt.pcolormesh(X, Y, np.angle(Z), cmap='hsv')
>>> plt.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=.7\textwidth]{Identity.png}
\caption{Plot of $f: \mathbb{C} \rightarrow \mathbb{C}$ defined by $f(z)=z$.
The color at each point $z$ represents the argument of $f(z)$.}
\label{fig:identity}
\end{figure}

\begin{problem}
Write the following function to plot any function from $\mathbb{C}$ to $\mathbb{C}$.
Plot the angle only, as above, ignoring the magnitude.
\begin{lstlisting}
def plot_complex(f, xbounds, ybounds, res=401):
    '''Plot the complex function f.

    INPUTS:
    f        - A function handle. Should represent a function
    			from C to C.
    xbounds  - A tuple (xmin, xmax) describing the bounds on the real part
    			of the domain.
    ybounds  - A tuple (ymin, ymax) describing the bounds on the imaginary
    			part of the domain.
    res      - A scalar that determines the resolution of the plot.
    			Defaults to 401.
    '''
\end{lstlisting}
Check your function on $f(z) = z$ (graphed in Figure \ref{fig:identity}) and on the function $f(z) = \sqrt{z^2+1}$, which is graphed in Figure \ref{fig:check_plot}.

Hint: When you call \li{plt.pcolormesh()}, specify the keyword arguments \li{vmin} and \li{vmax}.
These define which values should map to each end of the color scale.
We want $-\pi$ to map to the low end of the color scale, and $\pi$ to map to the high end.
If not specified, matplotlib will scale the colormap to fit your data exactly.

\label{prob:plot_complex}
\end{problem}

\begin{figure}[H]
\includegraphics[width=.6\textwidth]{check_plot.png}
\caption{Plot of the angle of $\sqrt{z^2+1}$ on the domain $\{x+iy \mid x \in [-3,3] , \; y \in [-3,3]\}$ created by \li{plot_complex()}.}
\label{fig:check_plot}
\end{figure}

The choice to ignore the magnitude may seem arbitrary.
We can also write a complex plotting function to ignore the angle and only plot the magnitude.
This will give us some different intuition about the function, while losing some information that we would get from the angle plot.

\begin{problem}
Write a new complex plotting function called \li{plot_complex_magnitude} which ignores the angle and plots only the magnitude.
This should resemble your answer to Problem \ref{prob:plot_complex}, with small modifications.
Leave \li{vmin} and \li{vmax} unspecified when plotting.

Check your function on $f(z) = \sqrt{z^2+1}$.
Your plot should look like the right subplot in Figure \ref{fig:check_plot_2}.
Note the difference between this plot and the one from the previous problem.

Hint: A wraparound colormap like \li{'hsv'} doesn't work well here.
Use any sequential colormap that makes it easy to distinguish between high and low values. See the matplotlib documentation for a list of colormaps.

\end{problem}

\begin{figure}[H]
\includegraphics[width=\textwidth]{check_plot_magnitude.png}
\caption{Plots of $\sqrt{z^2+1}$ on $\{x+iy \mid x \in [-3,3] , \; y \in [-3,3]\}$, visualizing the angle and the magnitude of the function.
Notice how a discontinuity is clearly visible on the left, but disappears from the plot on the right.}
\label{fig:check_plot_2}
\end{figure}

\section*{Analyzing Complex Plots}

The angle plot is generally more useful than the magnitude plot for visualizing function behavior, zeros, and poles.
Throughout the rest of the lab, use \li{plot_complex} to plot only the angle, and ignore the magnitude.

\subsection*{Zeros}

Complex plots can be surprisingly informative.
From an angle plot we can estimate not only a function's zeros, but also their multiplicities.

\begin{problem}\label{prob:zeros}
\leavevmode
\begin{enumerate}
\item Use \li{plot_complex()} to plot the functions $z^2$, $z^3$, and $z^4$.
\item Plot $z^3 - iz^4 - 3z^6$ on the domain $\{x+iy \mid x \in [-1,1] , \; y \in [-1,1]\}$ (this plot is Figure \ref{fig:zeros}).
Compare it to your plot of $z^3$, especially near the origin.
Based on these plots, what can you learn about the zeros of a function from its graph?
\end{enumerate}
\end{problem}

In Problem \ref{prob:zeros} you should have noticed that in a plot $z^n$, the colors cycle $n$ times counterclockwise around 0.
(Note: For the remainder of this lab we will define red $\rightarrow$ yellow $\rightarrow$ green $\rightarrow$ blue $\rightarrow$ red to be the ``forward" direction, such that the colors are circling counterclockwise in Figure \ref{fig:identity}.)

This is explained by looking at $z^n$ in polar coordinates:
\[
z^n = (re^{i \theta})^n = r^n e^{i(n\theta)}.
\]
Multiplying $\theta$ by a number greater than $1$ compresses the graph along the ``$\theta$-axis'' by a factor of $n$.
In other words, the output angle repeats itself $n$ times in one cycle of $\theta$.
Compare this to replacing $f(x)$ with $f(nx)$ when $f$ is a function from $\mathbb{R}$ to $\mathbb{R}$.

From Problem \ref{prob:zeros} you should also have noticed that the plot of $z^3 - iz^4 - 3z^6$ looks a lot like the plot of $z^3$ near the origin.
This is because when $z$ is very small, $z^4$ and $z^6$ are much smaller than $z^3$, and so the behavior of $z^3$ dominates the function.

In general, $f(z)$ has a \emph{zero of order $n$ at $z_0$} if the Taylor series of $f(z)$ centered at $z_0$ can be written as
\[
f(z) = \sum_{k=n}^{\infty} a_k(z-z_0)^k \qquad \text{with} \; a_n \neq 0.
\]
In other words, $f(z) = a_n(z-z_0)^n + a_{n+1}(z-z_0)^{n+1} + \ldots$.
In a small neighborhood of $z_0$, the quantity $|z-z_0|^{n+k}$ is much smaller than $|z-z_0|^n$, and so the function behaves like $a_n(z-z_0)^n$.
This explains why we can estimate the order of a zero by counting the number of times the colors circle a point (see Figure \ref{fig:zeros}).

\begin{figure}
\includegraphics[width=.6\textwidth]{zeros.png}
\caption{Plot of $f(z)=z^3 - iz^4 - 3z^6$ on the domain $\{x+iy \mid x \in [-1,1] , \; y \in [-1,1]\}$.
From this plot we see that $f(z)$ has a zero of order 3 at the origin, and 3 zeros of order 1 scattered around it.
This accounts for the 6 roots of $f(z)$ that are guaranteed to exist by the Fundamental Theorem of Algebra.}
\label{fig:zeros}
\end{figure}

\subsection*{Poles}

The plots created by \li{plot_complex()} also contain information about the poles of the function plotted.

\begin{problem}\label{prob:poles}
\leavevmode
\begin{enumerate}
\item Use \li{plot_complex()} to plot the function $f(z) = 1/z$.
Compare this to the plot of $f(z)=z$ in Figure \ref{fig:identity}.
\item Plot $z^{-2}$, $z^{-3}$, and $z^2+iz^{-1}+z^{-3}$ on the domain $\{x+iy \mid x \in [-1,1] , \; y \in [-1,1]\}$.
Compare the plots of the last two functions near the origin.
Based on these plots, what can you learn about the poles of a function from its graph?
\end{enumerate}
\end{problem}

In Problem \ref{prob:poles} you should have noticed that in the graph of $1/z^{n}$, the colors cycle $n$ times \emph{clockwise} around 0.
Again this can be explained by looking at the polar representation:
\[
z^{-n} = (re^{i \theta})^{-n} = r^{-n} e^{i(-n\theta)}.
\]
The minus-sign on the $\theta$ reverses the direction of the colors, and the $n$ makes them cycle $n$ times.

In general, a function has a \emph{pole of order n} at $z_0$ if its Laurent series on a punctured neighborhood of $z_0$ is
\[
f(z) = \sum_{k=-n}^\infty a_k(z-z_0)^k  \qquad \text{with} \; a_{-n} \neq 0.
\]
In other words, $f(z) = a_{-n}(z-z_0)^{-n}+a_{-n+1}(z-z_0)^{-n+1} + \ldots$.
Since $|z-z_0|^{-n+k}$ is much smaller than $|z-z_0|^{-n}$ when $|z-z_0|$ is small, near $z_0$ the function behaves like $a_{-n}(z-z_0)^{-n}$.
This explains why we can estimate the order of a pole by counting the number of times the colors circle a point in the clockwise direction.

Finally, a function has an \emph{essential pole} at $z_0$ if its Laurent series in a punctured neighborhood of $z_0$ requires infinitely many terms with negative exponents.
For example,
\[
e^{1/z} = \sum_{k=0}^{\infty}\frac{1}{n! z^n} = 1+\frac{1}{z}+\frac{1}{2}\frac{1}{z^2}+\frac{1}{6}\frac{1}{z^3}+\ldots.
\]
The plot of $f(z) = e^{1/z}$ is in Figure \ref{fig:essential_singularity}.
The colors cycle infinitely many times around an essential singularity.

\begin{figure}
\includegraphics[width=.6\textwidth]{essential_pole.png}
\caption{Plot of $e^{1/z}$ on the domain $\{x+iy \mid x \in [-1,1] , \; y \in [-1,1]\}$.
The colors circle clockwise around the origin because it is a singularity, not a zero.
Because the singularity is essential, the colors repeat infinitely many times.}
\label{fig:essential_singularity}
\end{figure}

\subsection*{Using Plots to Estimate Poles and Zeros}
To summarize, poles and zeros can be estimated from a complex plot with the following rules.
\begin{itemize}
\item Colors circle counterclockwise around zeros.
\item Colors circle clockwise around poles.
\item The number of times the colors cycle equals the order of the zero or pole.
\end{itemize}

\begin{problem}\label{prob:findpz}
Plot these functions on the domains given.
Estimate the number and order of their poles and zeros.
\begin{itemize}
\item $f(z) = e^z$ on $\{ x+iy \mid x \in [-8,8], \; y \in [-8,8]\}$
%\item $z^2-2z^7+2z^6-4z^5+2z^4-2z^3-5z^2+4z-4$ from $x \in [-2.5,2.5]$ and $y \in [-2.5,2.5]$
\item $f(z) = \tan(z)$ on $\{x+iy \mid x \in [-8,8], \; y \in [-8,8]\}$
\item $f(z) = \frac{16z^4+32z^3+32z^2+16z+4}{16z^4-16z^3+5z^2}$ on $\{x+iy \mid x \in [-1,1], \; y \in [-1,1]\}$
%\item $f(z) = \sin{\frac{1}{z}}$ on ${x+iy \mid x \in [-.8,.8], y \in [-.8,.8]\}$
\end{itemize}
\end{problem}

One useful application of complex plots is to estimate the zeros of polynomials and their multiplicity.

\begin{problem}\label{prob:find_roots}
Use complex plots to determine the multiplicity of the zeros of each of the following polynomials.
Use the Fundamental Theorem of Algebra to ensure that you have found them all.
\begin{enumerate}
\item $-4z^5+2z^4-2z^3-4z^2+4z-4$
\item $z^7 + 6z^6 - 131z^5 - 419z^4 + 4906z^3 - 131z^2 - 420z + 4900$
\end{enumerate}
\end{problem}

Plotting functions is not a substitute for rigorous mathematics.
Often, plots can be deceptive.

\begin{problem}\label{prob:caution}
\leavevmode
\begin{enumerate}
\item This example shows that sometimes you have to ``zoom in'' to see all the information about a pole.
\begin{enumerate}
\item Plot the function $f(z) =\sin( \frac{1}{100z})$ on the domain $\{x+iy \mid x \in [-1,1],\; y \in [-1,1]\}$.
What might you conclude about this function?
\item Now plot $f(z)$ on $\{x+iy \mid x \in [-.01,.01], \; y \in [-.01,.01]\}$.
Now what do you conclude about the function?
\end{enumerate}
\item This example shows that from far away, two distinct zeros (or poles) can appear to be a single zero (or pole) of higher order.
\begin{enumerate}
\item Plot the function $f(z) = z+1000z^2$ on the domain $\{x+iy \mid x \in [-1,1], \; y \in [-1,1]\}$.
What does this plot imply about the zeros of this function?
\item Calculate the true zeros of $f(z)$.
\item Plot $f(z)$ on a domain that allows you to see the true nature of its zeros.
\end{enumerate}
\end{enumerate}
\end{problem}

%%% Long Comment: Old problems having to do with branch cuts and contour integration%%%
\begin{comment}

From these plots you can see the poles and zeros. The zeros are the black dots and the poles are the white dots on the plots. the order of the pole or zero can be read from the number of times the full set of colors show up around the pole or zero. As you can see from Figure \ref{fig:funcplot} $x^2-1$ has two zeros of order one at $1$ and $-1$. $x^4-\frac{1}{x^4}$ has eight zeros of order one and a pole of order four.
For essential singularities the colors circle outward like in \ref{fig:e}, the plot of $e^\frac{1}{z}$.

\begin{figure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{function.png}
\end{subfigure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{function2.png}
\end{subfigure}
\caption{Colorplot of the  function $x^2 - 1$ and $x^4-\frac{1}{x^4}$.}
\label{fig:funcplot}
\end{figure}

\begin{figure}
\includegraphics[width=\textwidth]{function1.png}
\caption{Color plot of the $e^\frac{1}{z}$.}
\label{fig:e}
\end{figure}

\begin{problem}

Plot the following functions and look at the plots. Write a function that prints out an estimate of the roots and/or poles and their order.
\begin{itemize}
\item Plot $e^z$ from $x \in [-8,8]$ and $y \in [-8,8]$, \item $z^2-2z^7+2z^6-4z^5+2z^4-2z^3-5z^2+4z-4$ from $x \in [-2.5,2.5]$ and $y \in [-2.5,2.5]$
\item Plot $\frac{16z^4+32z^3+32z^2+16z+4}{16z^4-16z^3+5z^2}$ from $x \in [-1,1]$ and $y \in [-1,1]$,
\item Plot $\sin{\frac{1}{z}}$ from $x \in [-.8,.8]$ and $y \in [-.8,.8]$,
\end{itemize}

\end{problem}

\section*{Multi-Valued Functions}

Another important topic in Complex Analysis is the study of multiple valued functions.
These functions arise as we consider the inverses of functions that are not strictly one to one on the complex plane.
A classic example is $\sqrt{x}$, which may take two values for every nonzero point of the complex plane.

In the Real numbers we worked with functions like this by simply restricting their output on a certain domain.
We can do a similar thing in the Complex plane.
Loosely speaking, such a restriction is called a branch.
Computationally we restrict the output to a single portion of the actual possible values of the multifunction.
We call inverse functions that have multiple values like this ``multi-valued functions" or ``multifunctions."
NumPy automatically restricts the output of multi-valued functions. So to get other cuts you have to modify the function to get the cut you want, like multiplying the output of the $\sqrt{x}$ by $-1$.
Figure \ref{fig:sqrt} shows the two cuts surfaces for $\sqrt{z}$ in the complex plane.

\begin{figure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{possqrt.png}
\end{subfigure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{negsqrt.png}
\end{subfigure}
\caption{The positive and negative cut respectively of the function $\sqrt{z}$.}
\label{fig:sqrt}
\end{figure}

These are some very basic examples.
Another simple example is $\ln\left(z\right)$, which has a single value for the real part and infinitely many possible values for its imaginary part.
This is because for any complex $z\neq 0$, we have $e^z=e^{z+2n\pi}$ where $n$ is any integer.

\begin{problem}
Plot three cuts of $\ln\left(z\right)$ and$\arctan{x}$ on $x \in [-2,2]$ and $y \in [-2,2]$
\end{problem}

All three of these functions can be made analytic at almost any point, except at their singularities, but that depends on how we cut the domain to give it a single value.
When Integrating such functions, be careful about integrating across such cuts in the domain.

\begin{problem}
Write two functions, both accepting a natural number $n$.
Have one function plot the Riemann surface for the real part $f(z)=\sqrt[n]{z}$ and the other plot the imaginary part.

Hint: Convert $z$ in $f(z)=\sqrt[n]{z}$ to polar form as $z=re^{\theta + 2k\pi}$
If you then plug this into $\sqrt[n]{z}$ the function takes the form
\[f(z)=\sqrt[n]{r} e^{i \frac{\theta + 2 \pi k}{n}}\]
Notice that here $f(z)$ has distinct values for $k = 0, 1, \dots, n-1$ (a total of $n$ different values). Each value of $k$ corresponds to a different branch, which you can plot as $n$ separate surfaces.

If you use just one surface to plot each branch you will get erroneous vertical lines from jump discontinuities. Split each branch into two surfaces to get rid of these lines. You can investigate where the discontinuities occur by first plotting it as just one surface. The discontinuities happen at the same place for all $n$;
\end{problem}

\section*{Contour Integrals in the Complex Plane}

From multivariable calculus, you may recall that an integral may be taken along a path.
This is very similar to what can be done in the complex plane.
Consider the function $f(z)$ on the complex plane.
Let $z=x+iy$.
Let $u$ and $v$ be the real and imaginary parts of $f$ respectively.
We integrate $f$ along some contour $C$ in the complex plane, beginning at $z=a$ and ending at $z=b$.
This integral may be written
\[\int_c f(z)dz\]
Parameterizing $z$, we have
\[\int_a^b f\left( c\left(t\right)\right) c'\left(t\right) dt\]
Expanding into real and imaginary parts (where $c\left(t\right) = x\left(t\right) + i y\left(t\right)$), we have
\[\int_a^b \left(u \left(c \left(t\right)\right) x'\left(t\right)-v\left(c \left(t\right)\right) y'\left(t\right)\right) dt + i \int_a^b\left(v \left(c \left(t\right)\right)x'\left(t\right)+u\left(c \left(t\right)\right) y'\left(t\right)\right) dt\]
We have now written this complex integral as the sum of two real valued integrals in $\mathbb{R}$.
Note that this implies that $\int_C f(z) dz$ may depend on the contour we choose and not just on the endpoints $a$ and $b$.

\begin{problem}
Write a function which takes a complex function $f(z)$, a contour parameterization $c(t)$ of a contour $c$, and the integration bounds on $t$ and returns the integral of $f$ along the contour $c$.
Use the numerical integration function \li{sympy.mpmath.quad} and the numerical derivative function \li{sympy.mpmath.diff} included in mpmath (which is, in turn, included as a submodule of sympy).
These functions already work for complex numbers.
To do something similar with the integration routines in SciPy, we would have to separate the function into real and imaginary parts, as is shown above.

Using the function you just defined, integrate the following functions along the following contours
\begin{itemize}
\item $\bar{z}$ counterclockwise along the unit ball starting and ending at $1$
\item $\bar{z}$ along a straight line from $0$ to $1+i$
\item $\bar{z}$ along the real axis from $0$ to $1$, then along the line from $1$ to $1+i$
\item $\bar{z}$ along the unit ball centered at $i$ from $0$ to $1+i$
\item $e^z$ counterclockwise along the unit ball starting and ending at $1$
\item $e^z$ along a straight line from $0$ to $1+i$
\item $e^z$ along the real axis from $0$ to $1$, then along the line from $1$ to $1+i$
\item $e^z$ along the unit ball centered at $i$ from $0$ to $1+i$
\end{itemize}
\end{problem}

Notice that, for a holomorphic function on a simply connected domain, the integrals from one point to another are not path dependent for any contours that lie within the domain.
An immediate consequence of the theorem is that for a complex function $f$, holomorphic on a simply connected domain $D$, and a contour $C$ lying entirely within $D$ which begins and ends at some point $a\in D$,
\[\int_C f(z)dz=0\]

The quadrature algorithms used in many of the integration algorithms work along a straight line between the integration bounds in the complex plane, so for holomorphic functions we should be able to use the integration function we wrote earlier.
For example, integrating $e^z$ from $-1-i$ to $1+i$ can be done numerically like this:
\begin{lstlisting}
from sympy import mpmath as mp
mp.quad(lambda z: mp.exp(z), (complex(-1, -1), complex(1, 1)))
\end{lstlisting}

\section*{The Cauchy Integral Formula}

Another major theorem in complex analysis is called Cauchy's Integral Formula (not to be confused with Cauchy's Integral Theorem).
It states that for a domain $D$ in the complex plane, containing some contour $C$ and the interior of $C$, for any $z_0$ in the interior of $C$,
\[f(z_0)=\frac{1}{2\pi i} \int_C \frac{f(z)}{z-z_0} dz\]

With more work, this theorem can be used to show that any function $f$ holomorphic on some domain $D$ is also infinitely differentiable on that domain.
In fact, the $n$th derivative of $f$ is given by the formula
\[f^{(n)}(z_0) = \frac{n!}{2\pi i} \int_C \frac{f(z)}{(z-z_0)^{n+1}} dz\]
This result is also important because it allows us to relate the value of $f$ on the inside of a contour to the value of $f$ on the contour itself.
In other words, the values of $f$ inside the contour depend only on the values of $f$ along the contour itself.
A related theorem (the Morera theorem) states that if some function $f$ is continuous on a domain $D$ and for every contour beginning and ending at the same point, the formula $\int_C f(z) dz = 0$ holds, then $f$ is holomorphic on $D$.

\begin{problem}
Using Cauchy's Integral Formula, write a python function which returns a callable function which evaluates a complex function $f$ along the interior of a contour $C$.
It should accept a callable function for the parameterization of $C$, a callable function for the values of $f$ along $C$, and the bounds on the parameter used.
Assume in your function that $C$ begins and ends at the same point and that $f$ also begins and ends at the same value (so that $f$ is continuous along $C$)
Try it out on simple functions like $e^x$ with complex values and compare what you get with what calling the functions normally gives you.
\end{problem}

Notice that in Cauchy's Integral Formula, we are integrating along a contour that begins and ends at the same point.
The function is also holomorphic at every point except $z_0$. At $z_0$ the integrand is undefined and has a singularity.
This integral around a singularity has some useful properties.
We will discuss these properties later on.

\end{comment}
%%%End of long comment%%%

\subsection*{Multi-Valued Functions}
Every complex number has two complex square roots, since if $w^2=z$, then also $(-w)^2=z$.
If $z$ is not zero, these roots are distinct.

Over the nonnegative real numbers, it is possible to define a continuous square root function.
However, it is not possible to define a continuous square root function over any open set of the complex numbers that contains 0.
This is intuitive after graphing $\sqrt{z}$ on the complex plane.

\begin{problem}
\begin{enumerate}
\item Use \li{plot_complex} to graph $f(z) = \sqrt{z}$.
Use \li{np.sqrt()} to take the square root.
\item Now plot $f(z) = -\sqrt{z}$ to see the ``other square root'' of $z$.
Describe why these two plots look the way they do.
\end{enumerate}
\end{problem}

Just as raising $z$ to a positive integer ``compresses the $\theta$-axis'', making the color wheel repeat itself $n$ times around 0, raising $z$ to a negative power \emph{stretches} the $\theta$-axis, so that only one $n^{th}$ of the color wheel appears around 0.
The colors at the ends of this $n^{th}$-slice are not the same, but they appear next to each other in the plot of $z^{-n}$.
This discontinuity will appear in every neighborhood of the origin.

If your domain does not contain the origin, it is possible to define a continuous root function by picking one of the roots.

\section*{Appendix}
It is possible to visualize the argument and the modulus of the output of a complex function $f(z)$.
One way to do so is to assign the modulus to a \emph{lightness} of color.
For example, suppose we have a complex number with argument 0, so it will map to red in the color plots described above.
If its modulus is very small, then we can map it to a blackish red, and if its modulus is large, we can map it to a whitish red.
With this extra rule, our complex plots will still be very much the same, except that zeros will look like black dots and poles will look like white dots (see Figure \ref{fig:example} for an example).

The code below implements the map we just described.
Be warned that this implementation does not scale well.
For example, if you try to plot a complex function whose outputs are all very small in modulus, the entire plot will appear black.

\begin{lstlisting}
import numpy as np
import matplotlib.pyplot as plt
from colorsys import hls_to_rgb

def colorize(z):
    '''
    Map a complex number to a color (or hue) and lightness.

    INPUT:
    z - an array of complex numbers in rectangular coordinates

    OUTPUT:
    If z is an n x m array, return an n x m x 3 array whose third axis encodes
    (hue, lightness, saturation) tuples for each entry in z. This new array can
    be plotted by plt.imshow().
    '''

    zy=np.flipud(z)
    r = np.abs(zy)
    arg = np.angle(zy)

    # Define hue (h), lightness (l), and saturation (s)
    # Saturation is constant in our visualizations
    h = (arg + np.pi)  / (2 * np.pi) + 0.5
    l = 1.0 - 1.0/(1.0 + r**0.3)
    s = 0.8

    # Convert the HLS values to RGB values.
    # This operation returns a tuple of shape (3,n,m).
    c = np.vectorize(hls_to_rgb) (h,l,s)

    # Convert c to an array and change the shape to (n,m,3)
    c = np.array(c)
    c = c.swapaxes(0,2)
    c = c.swapaxes(0,1)
    return c
\end{lstlisting}

The following code uses the \li{colorize()} function to plot  $\frac{z^2 - 1}{z}$. The output is Figure \ref{fig:example}.

\begin{lstlisting}
>>> f = lambda z :  (z**2-1)/z
>>> x = np.linspace(-.5, 1.5, 401)
>>> y = np.linspace(-1, 1, 401)
>>> X,Y = np.meshgrid(x,y)
>>> Z=f(X+Y*1j)
>>> Zc=colorize(Z)
>>> plt.imshow(Zc, extent=(-.5, 1.5, -1, 1))
>>> plt.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=\textwidth]{example.png}
\caption{Plot of the function $\frac{z^2 - 1}{z}$ created with \li{colorize()}.
Notice that the zero at 1 is a black dot and the pole at 0 is a white dot.}
\label{fig:example}
\end{figure}

%The only thing I can think of is to specify more clearly is to set vmin and vmax to -pi to pi.  Otherwise I thought it was all fine!


\noindent
\subsubsection{Sine}
Let $a$ be a constant.\\
By definitions of a Laplace transform and an improper integral,
\begin{align*}
	\Laplace{\sin{(at)}} &= \lim\limits_{n\to\infty}{\int_{0}^{n}{\sin{(at)}e^{-st}\mathrm{d}t}} \\
	&= \frac{-1}{s^2 + a^2}\lim\limits_{n\to\infty}{\left[e^{-st}\left(s\sin{(at)} + a\cos{(at)}\right)\right]_{0}^{n}} \\
	&= \frac{-1}{s^2 + a^2}\left(\lim\limits_{n\to\infty}\left(e^{-sn}\left(s\sin{(an)} + a\cos{(an)}\right)\right) - \left(e^{-s\cdot 0}\left(s\sin{(a\cdot 0)} + a\cos{(a\cdot 0)}\right)\right)\right).
\end{align*}
Both $\sin$ and $\cos$ have maximum values of 1, so we can say that the left part of them expression has a maximum value at most $s + a$.
For positive $s$, the exponential dominates and the expression goes to 0 in the limit.
\begin{align*}
	&= \frac{-1}{s^2 + a^2}\left(0 - a\right) \text{, } s > 0 \\
	&= \frac{a}{s^2 + a^2} \text{, } s > 0.
\end{align*}
So,
\begin{equation*}
	\Laplace{\sin{(at)}} = \frac{a}{s^2 + a^2} \text{, } s > 0.
\end{equation*}


\lab{Applications}{Least-squares fitting I}{Least-squares fitting I}
\label{LeastSquaresCircle}

\objective{This section will introduce Least Squares and teach a more advanced application of Least Squares: fitting data to an circle.}
\section*{Least Squares}

It is well known that the displacement of a spring is proportional to the force acting upon it, that is, $F = k x$.  The proportionality constant $k$ is called Hooke's spring constant.  Consider a laboratory experiment where different loads are placed on a spring and the displacement is measured and recorded in the table below:
\vspace{5mm}\\
\begin{center}
\begin{tabular}{|c|c|}
	\hline
x & F \\
(cm) & (dyne)\\
\hline
1.04  & 3.11 \\
2.03  &  6.01\\
2.95  &  9.07\\
3.92  &  11.99\\
5.06  &  15.02\\
6.00  &  17.91\\
7.07  &  21.12\\
\hline
\end{tabular}
\end{center}
\vspace{5mm}
To find the spring constant $k$, we simply need to solve the following linear system
\[
\begin{pmatrix}
1.04\\
2.03\\
2.95\\
3.92\\
5.06\\
6.00\\
7.07\\
\end{pmatrix}
\begin{pmatrix}k\end{pmatrix} =
\begin{pmatrix}
3.11 \\
6.01\\
9.07\\
11.99\\
15.02\\
17.91\\
21.12\\
\end{pmatrix}.
\]
However, there is no solution to this system because it is overdetermined.  Instead, we seek the ``best'' $k$ that fits the data.  Least squares (which we mentioned in a previous exercise) allows us to find that ``best'' solution. We can find the least squares solution by computing the following in Python:
\begin{lstlisting}[style=python]
: A = sp.vstack([1.04,2.03,2.95,3.92,5.06,6.00,7.07])
: b = sp.vstack([3.11,6.01,9.07,11.99,15.02,17.91,21.12])
: k = sp.dot(sp.dot(la.inv(sp.dot(A.T,A)),A.T),b);k
: # Using the built in least squares function this is just: k=la.lstsq(A,b);k
array([[ 2.99568294]])
\end{lstlisting}
Hence, we find the spring constant to be $k = 2.9957$.  We plot the data against the best fit as follows:
\begin{figure}[h!]
\label{fig1}
\begin{center}
\includegraphics[width=\textwidth]{line_lstsq}
\caption{The graph of the spring data together with its linear fit}
\label{Fig:SpringFit}
\end{center}
\end{figure}

\begin{lstlisting}[style=python]
: x0 = sp.linspace(0,8,100)
: y0 = k[0]*x0
: from matplotlib import pyplot as plt
: plt.plot(A,b,'*',x0,y0)
: plt.show()
\end{lstlisting}
See Figure \ref{Fig:SpringFit} to see how well the line fits the data.

\section*{General Line Fitting}

Suppose that we wish to fit a general line, that is $y=m x+b$, to the data set $\{(x_k,y_k)\}^n_{k=1}$.  Assume that the line does not cross through the origin, as in the previous example.  Then we seek both a slope and a $y$-intercept.  In this case, we set up the following linear system $A x = b$, or more precisely
\[
\begin{pmatrix}
x_1 & 1\\
x_2 & 1\\
x_3 & 1\\
\vdots & \vdots\\
x_n & 1
\end{pmatrix}
\begin{pmatrix}
m\\
b
\end{pmatrix}=
\begin{pmatrix}
y_1\\
y_2\\
y_3\\
\vdots\\
y_n
\end{pmatrix}.
\]
Note that $A$ has rank $2$ as long as not all of the $x_k$ values are the same.  Hence, the least squares solution will fit the best line for this data.

\begin{problem}
Load the \texttt{linepts} array from \texttt{data.npz}.
This consists of two columns corresponding to the $x$ and $y$ values of a given data set.  Use least squares to find the slope and $y$-intercept that best fits the data.  Then plot the data points and the line on the same graph.  Finish off the problem with a discussion of what you've learned.
\end{problem}

\section*{Fitting data to a circle}

Recall that the equation of a circle, with radius $r$ centered at $(c_1,c_2)$, is given by
\begin{equation}
\label{circle}
(x-c_1)^2 + (y-c_2)^2 = r^2.
\end{equation}
Suppose we are given a set of data points closely forming a circle $\{(x_i,y_i)\}^n_{i=1}$.  The ``best'' fit is found via least squares by expanding \eqref{circle} to get
\[
2 c_1 x + 2 c_2 y + c_3 = x^2 + y^2,
\]
where $c_3 = r^2 - c_1^2 - c_2^2$.  Then we can write the linear system $A x = b$ as
\[
\begin{pmatrix}
2 x_1 & 2 y_1 & 1\\
2 x_2 & 2 y_2 & 1\\
\vdots & \vdots & \vdots \\
2 x_n & 2 y_n & 1
\end{pmatrix}
\begin{pmatrix}
c_1\\
c_2\\
c_3
\end{pmatrix}=
\begin{pmatrix}
x_1^2 + y_1^2\\
x_2^2 + y_2^2\\
\vdots\\
x_n^2 + y_n^2
\end{pmatrix},
\]
where the matrix $A$ and the vector $b$ are obtained by the given data and the unknown $x$ contains the information about the center and radius of the circle and is obtained by finding the least squares solution.

\section*{Example}

In this section, we fit the following points to a circle:
\begin{align*}
&(134,76),(104,146),(34,176),(-36,146),\\
&(-66,76),(-36,5),(34,-24),(104,5),(134,76)
\end{align*}

We enter them into Python as a $9\times 2$ array:
\begin{lstlisting}[style=python]
: P = sp.array([[134,76],[ 104,146],[ 34,176],[ -36,146],[ -66,76],[ -36,5],[ 34,-24],[ 104,5],[ 134,76]])
\end{lstlisting}
Then we can separate the $x$ and $y$ coordinates by the commands \li{P[:,0]} and \li{P[:,1]}, respectively.  Hence, we compute $A$ and $b$ by entering the following:
\begin{lstlisting}[style=python]
: A = sp.column_stack((2*P,sp.ones((9,1),dtype=sp.int_)))
: b = P[:,0]**2 + P[:,1]**2
\end{lstlisting}
Hence, we get the least squares solution
\begin{lstlisting}[style=python]
: x = sp.dot(sp.dot(la.inv(sp.dot(A.T,A)),A.T),b)
\end{lstlisting}
Then we find $c_1$, $c_2$, and $r$ by:
\begin{lstlisting}[style=python]
: c1 = x[0]
: c2 = x[1]
: c3 = x[2]
: r = sp.sqrt(c1**2 + c2**2 + c3)
\end{lstlisting}
We plot this by executing
\begin{lstlisting}[style=python]
: theta = sp.linspace(0,2*sp.pi,200)
: plt.plot(r*sp.cos(theta)+c1,r*sp.sin(theta)+c2,'-',P[:,0],P[:,1],'*')
: plt.show()
\end{lstlisting}

\begin{problem}
Load the \texttt{circlepts} array from \texttt{data.npz}.
This consists of two columns corresponding to the $x$ and $y$ values of a given data set.  Use least squares to find the center and radius of the circle that best fits the data.  Then plot the data points and the circle on the same graph.  Finish off the problem with a discussion of what you've learned.
\end{problem}

\begin{problem}
The general equation for an ellipse is:
\[
A(x-x_0)^2 + B(x-x_0)(y-y_0) + C(y-y_0)^2 = 1
\]

Write a program that uses least squares to fit data to an ellipse. One option to finding the center point $(x_0,y_0)$ is to use the mean function. Test the program on \texttt{circlepts}. Also test it against sp.dot(circlepts, sp.array([[2,0],[0,1]]) ) . Plot the result. How well does your function work?
\end{problem}


\section{Indicator Variables}

Indicator variables are useful devices that simplify probability calculations involving binary outcomes, namely wether the outcome lies in a certain region of event space or not. Specifically, they allow one to translate set expressions into algebraic expressions.

Let $A\subseteq S$ be a subset of event space $S$ (ex. $A\equiv$ "it rains tomorrow"). Then the indicator variable $I_A$:

\begin{equation}
I_A = \left\{\begin{array}{l} 1 \mathrm{\ if\ outcome\ in\ A}\\0 \mathrm{\ if\ outcome\ in\ A^c} \end{array}\right.
\end{equation}

Which means that:
\begin{equation}
\mathbb{E}(I_A) = 1\times p(A) + 0\times p(A^c)= p(A)
\end{equation}

To indicator variables for different events $A,B,C...$ can also be combined:
\begin{equation}
I_{A\cap B\cap C\cap ...} = I_A I_B I_C ...
\end{equation}

And indicator variables for the complement can be constructed trivially:
\begin{equation}
I_{A^c} = 1 - I_A
\end{equation}

Whenever accounting for complicated combinations of events becomes overwhelming, indicator variables are often a good approach.

\subsection{Example: The Party Problem}

The party problem is a typical interview question, so I will include the two easier problems that do not make use of indicator variables. (This version of my notes also still has bitterness included.)

There are $n$ drunk kids at a party that is presumably getting shut down by the fun police in Cambridge, MA. They (meaning the kids, presumably) grab their coats at random, and the problem is built around thinking about how many people wind up with the correct coat.

\subsubsection{Lame Interview Question 1: Every one finds their coat}
The common version of this problem asks "what is the probability that all of the kids wind up with the right coat". This is much easier than the general case. Imagine the list of party guests as a sequence $(1,2,3,4,...,n-1,n)$ and the coats they grab as a random permutation of that sequence, such as $\alpha = (6,1,40,21,9,...)_n$. There are $n!$ such permutations, and they are all equally likely. The probability of all guests getting the correct coat is the probability that the permutation happens to be the single correct one, i.e. $p(\alpha = (1,2,3,4,...,n-1,n)_n)$, which is the probality of one particular permutation, i.e. $p(\alpha = (1,2,3,4,...,n-1,n)_n) = \frac{1}{n!}$.

\subsubsection{Lame Interview Question 2: At least r people find their coat}
This is still pretty easy, because the permutations that are correct are easily counted. If at least $r$ coats are correctly assigned, then there are $\left(\begin{array}{l}n\\r\end{array}\right)$ ways of choosing wich of the $r$ people wind up with the right coat. Then, while the location of $r$ indices in the sequence is fixed, $n-r$ indices can be assigned arbitrarily.

The number of permutations where at least $r$ are assigned correctly are then:

\begin{equation}
\left(
\begin{array}{l}
n\\
r
\end{array}
\right)(n-r)!
\end{equation}

And the probability of at least $r$ people finding their coat is the number of permutations multiplied with the probability of an individual permutation (that is, $\frac{1}{n!}$):

\begin{equation}
p(\mathrm{\#\ correct} \geq r)=\left(
\begin{array}{l}
n\\
r
\end{array}
\right)\frac{(n-r)!}{n!} = \frac{1}{r!}
\end{equation}

Where $r\leq n$.

\subsubsection{Not lame: Exactly r people find their coat}
So, then, what's the probability that exactly nobody finds their coat? What's the probability that 2 people find their coat but nobody else does? What's the probability that $r$ out of $n$ people find their coat? After thinking quickly on your feet for two seconds, you realize that the answer is obviously:

\begin{equation}
p(\mathrm{\#\ correct} = r) = \frac{1}{r!}\sum_{s=0}^{n-r} \frac{(-1)^s}{s!}
\end{equation}

The interviewer grunts ambiguously. They never call you back. You never find out why. You can't sleep. You can't eat. You become an anarchist and you declare war on the system.

The first two versions here are what I've come across in interview prep-type materials. I find them a bit annoying, because they represent particular cases that are much simpler than the general case. Applicants in the habit of studying stupid interview questions are rewarded because those answers are easily memorized (false positive). Applicants who intuit the complexity of the general problem, and who don't know the answer beforehand, might become overwhelmed during an interview and fail (false-ish negative).

/ rant

My approach here follows the extraordinary lecture notes https://mast.queensu.ca/~stat455/ by Glen Takahara at the University of Queensland. The difficulty of the problem is that events of a coat being picked up are interrelated: whether one person picks up their correct coat alters the probability of another person also picking up their correct coat. The beauty of this solution is that, rather than messing about with conditional probabilities, it looks at subsets of the event space, and then uses indicator variables to translate set expressions into algebraic expressions.

It uses the properties:

\begin{equation}
I_A = \left\{\begin{array}{l} 1 \mathrm{\ if\ outcome\ in\ A}\\0 \mathrm{\ if\ outcome\ in\ A^c} \end{array}\right.
\end{equation}

Which means that:
\begin{equation}
\mathbb{E}(I_A) = 1\times p(A) + 0\times p(A^c)= p(A)
\end{equation}

To indicator variables for different events can also be combined:
\begin{equation}
I_{A\cap B\cap C\cap ...} = I_A I_B I_C ...
\end{equation}

And indicator variables for the complement can be constructed trivially:
\begin{equation}
I_{A^c} = 1 - I_A
\end{equation}

Let $A_i$ be the region of state space in which guest $i$ grabbed the right coat. Let's say a *particular* subset $\{i\}_r$ of $r$ guests grabs their correct coats (for example, $\{i\}_r = \{5,9,11,24,...\}_r)$, and that the set of remaining $n-r$ guests $\{j\}_{n-r}= \{1,2,3,...\}_n\setminus\{i\}_r$ grab the wrong coat. The area of state space that corresponds to this outcome is:

\begin{equation}
A_{\{i\}_r,\{j\}_{n-r}}=\bigcap_{i\in\{i\}_r}A_i\bigcap_{j\in\{j\}_{n-r}}A_j^c
\end{equation}

The event that the outcome lies within that region of configuration space can be described with an indicator function:

\begin{equation}
I_{A_{\{i\}_r,\{j\}_{n-r}}} = \prod_{\{i\}_r}I_{A_i}\prod_{\{j\}_{n-r}}(1-I_{A_j})
\end{equation}

And the probability of those *particular* $r$ people finding their coat is it's expectation value, $p(A_{\{i\}_r,\{j\}_{n-r}}) = \mathbb{E}(I_{A_{\{i\}_r,\{j\}_{n-r}}})$. Good stuff.

The expression above will consist of a bunch of products of indicator variables that describe whether a particular guest wound up with the correct coat. We know that the product of indicator variables corresponds to an indicator variable for the *intersection* of the corresponding subsets of the state space. Explicitly, if it is a product of $s$ indicator variables for some particular set of coats coat $\{k\}_s$:

\begin{equation}
\prod_{k} I_{A_k} = I_{\bigcap_{k}A_k}
\end{equation}

And $\mathbb{E}(I_{\bigcap_{k} A_k}) = p(\bigcap_{k} A_k)$ is the probability of a particular $s$ coats being picked up correctly, which is $(n-s)!/n!$. (There is no binomial factor, because it's one *specific* set of $s$ coats).

Products of the sort $\prod^n (1-x_i)$ can be expanded:

\begin{equation}
\prod^n (1-x_i) = \sum_{s=0}^n (-1)^s \sum_{1\leq i_1,...,i_s\leq n} x_{i_1}x_{i_2}...x_{i_s}
\end{equation}

The sum $\sum_{1\leq i_1,...,i_s\leq n}$ is over all possible sets of up to $s$ indices that can be drawn from $\{1,2,3,...,n\}$. A simple example with $n=3$:

\begin{equation}
(1-x_1)(1-x_2)(1-x_3) = \underbrace{1}_{s=0} - \underbrace{(x_1 + x_2 + x_3)}_{s=1} + \underbrace{(x_1x_2 + x_2x_3 + x_1x_3)}_{s=2} - \underbrace{(x_1x_2x_3)}_{s=3}
\end{equation}

The number of terms of order $s$ is the amount of ways that $s$ indices can be sampled from $n$ indices, $\left(\begin{array}{l}n\\s\end{array}\right)$.

Returning to the original problem, then:

\begin{equation}
\begin{array}{ll}
I_{A_{\{i\}_r,\{j\}_{n-r}}} &= \prod_{\{i\}_r}I_{A_i}\prod_{\{j\}_{n-r}}(1-I_{A_j})\\
&=\sum_{s=0}^{n-r} (-1)^s \underbrace{\sum_{n-r\leq j_1,...,j_s\leq n}}_{\mathrm{sum\ of\ }\left(\begin{array}{l}n-r\\s\end{array}\right)\mathrm{\ terms\ \ }} \underbrace{\prod_{\{i\}_r} I_{A_i}\prod_{\{j\}_s}I_{A_j}}_{\mathrm{product\ of\ r+s\ terms\ \ }}
\end{array}
\end{equation}

By linearity of expected value, and using the relationship of the expected value of indicator variables to their probabilities:

\begin{equation}
\begin{array}{rl}
\mathbb{E}(I_{A_{\{i\}_r,\{j\}_{n-r}}}) &= \sum_{s=0}^{n-r} (-1)^s \underbrace{\sum_{n-r\leq j_1,...,j_s\leq n}}_{\mathrm{sum\ of\ }\left(\begin{array}{l}n-r\\s\end{array}\right)\mathrm{\ terms\ \ }} p\left(\underbrace{I_{\bigcap_{\{i\}_r}A_i\bigcap_{\{j\}_s}A_j}}_{\mathrm{r+s\ coats\ picked\ up\ correctly}}\right)\\
&=\sum_{s=0}^{n-r} (-1)^s \left(\begin{array}{c}n-r\\s\end{array}\right)\frac{(n-r-s)!}{n!}
\end{array}
\end{equation}

Now, this is already a pretty neat expression for the probability of a *particular* $r$ coats being picked up correctly (i.e. Joe, Mary, and "Hans" picked up the right coat). We don't really care which of the $r$ guests got lucky, though, so since there are $\left(\begin{array}{c}n\\r\end{array}\right)$ ways of $r$ coats having been picked out correctly, you sum over all of them:

\begin{equation}
\begin{array}{ll}
p(\mathrm{\#\ correct} = r) &= \left(\begin{array}{c}n\\r\end{array}\right) \sum_{s=0}^{n-r} (-1)^s \left(\begin{array}{c}n-r\\s\end{array}\right)\frac{(n-r-s)!}{n!}\\
&= \sum_{s=0}^{n-r} (-1)^s \frac{n!}{r!(n-r)!}\frac{(n-r)!}{s!(n-r-s)!}\frac{(n-r-s)!}{n!}\\
&= \frac{1}{r!}\sum_{s=0}^{n-r} \frac{(-1)^s}{s!}
\end{array}
\end{equation}

\begin{figure}
\centering
    \includegraphics[width=0.7\textwidth]{partyproblem.png}
    \caption{Simulated and analytically calculated probabilities that exactly $r$ people at a $n=100$ people party randomly pick up their coat.}
    \label{fig:proba_partyproblem}
\end{figure}


\chapter{Linear Transformations}
Let's pause the discussion of tensor product spaces and dual spaces, and consider again the underlying vector space $V$ with a basis $\{\vec{e}_\mu\}$.
The only thing to recall about the dual space is that $V^*$ has a basis $\{\vec{e}^\nu\}$ chosen such that $\langle \vec{e}^\nu, \vec{e}_\mu \rangle = \delta^\nu_\mu$.
As with any vector space, in order for $\{\vec{e}_\mu\}$ to be a basis, it must
\begin{enumerate}
    \item span $V$, and
    \item be a linearly independent set.
\end{enumerate}
There are infinitely many such basis which meet the requirements, so our choice of basis for $V$ is rather arbitrary\footnote{As an example, take any basis of $V$, and scale one component by a scalar factor from $\mathbb{F}$ to create a new, valid basis.}.
As long as $\{\vec{e}_\mu\}$ is a basis for $V$, any vector $\vec{v} \in V$ can be written as a linear combination of $\{\vec{e}_\mu\}$, so
\[ \vec{v} = A^\mu \vec{e}_\mu,\;A \in \mathbb{F}. \]
But suppose that we have another basis for $V$, called $\{\unit{e}_\nu\}$.
We can also write $\vec{v}$ as a linear combination of these basis vectors,
so
\[ \vec{v} = B^\nu \unit{e}_\nu,\;B \in \mathbb{F}. \]
The vector itself hasn't changed when we represent it in different ways, so we know that these two linear expressions must be equal to one another.
If we want, we can change from one basis to the other, and we do so via \emph{linear transformations}.

\subsection{Change of Basis}
Let's begin by writing one set of basis vectors in terms of the other.
Our goal will be to write each member of $\{\unit{e}_\nu\}$ in terms of linear combinations of $\{\vec{e}_\mu\}$.
To do so, we say that each element of $\{\unit{e}_\nu\}$ can be represented by the sum of scalar products of each element of $\{\vec{e}_\mu\}$, so
\begin{align*}
    \unit{e}_0 &= \tensor{\Lambda}{_0^0}\vec{e}_0 + \tensor{\Lambda}{_0^1}\vec{e}_1 + \cdots \\
    \unit{e}_1 &= \tensor{\Lambda}{_1^0}\vec{e}_0 + \tensor{\Lambda}{_1^1}\vec{e}_1 + \cdots \\
    &\vdots \\
    \unit{e}_\nu &= \tensor{\Lambda}{_\nu^\mu}\vec{e}_\mu.
\end{align*}
Here, the lower index $\nu$ on $\Lambda$ tells us which of $\{\unit{e}_\nu\}$ we are writing, and the upper index $\mu$ tells us which of our original basis vectors $\vec{e}_\mu$ we are scaling.
If you treat this as a linear algebra problem, it becomes apparent that $\tensor{\Lambda}{_\nu^\mu}$ is a matrix representing the linear transformation between two vectors.
For the sake of brevity, if we were to hold that $V$ is a four dimensional vector space, then we could write the entire transformation as
\[
    \begin{bmatrix}
        \unit{e}_0 \\
        \unit{e}_1 \\
        \unit{e}_2 \\
        \unit{e}_3
    \end{bmatrix} =
    \begin{bmatrix}
        \tensor{\Lambda}{_0^0} & \tensor{\Lambda}{_0^1} & \tensor{\Lambda}{_0^2} & \tensor{\Lambda}{_0^3} \\
        \tensor{\Lambda}{_1^0} & \tensor{\Lambda}{_1^1} & \tensor{\Lambda}{_1^2} & \tensor{\Lambda}{_1^3} \\
        \tensor{\Lambda}{_2^0} & \tensor{\Lambda}{_2^1} & \tensor{\Lambda}{_2^2} & \tensor{\Lambda}{_2^3} \\
        \tensor{\Lambda}{_3^0} & \tensor{\Lambda}{_3^1} & \tensor{\Lambda}{_3^2} & \tensor{\Lambda}{_3^3}
    \end{bmatrix}
    \begin{bmatrix}
        \vec{e}_0 \\
        \vec{e}_1 \\
        \vec{e}_2 \\
        \vec{e}_3
    \end{bmatrix},
\]
where the lower index $\nu$ represents the row and the upper index $\mu$, the column.

However, it's very important to note that this transformation matrix is \emph{not} a tensor, even though it kind of looks like one.
One way we can tell that this is not a tensor is that the indices are in the wrong order; all tensors have upper indices first, and then lower indices.
Another key difference is the lack of a basis; while $\{\unit{e}_\nu\}$ and $\{\vec{e}_\mu\}$ are basis vectors of $V$, $\tensor{\Lambda}{_\nu^\mu}$ is not a vector in this space, and we haven't defined any basis for $\tensor{\Lambda}{_\nu^\mu}$, or even a tensor product space for $\tensor{\Lambda}{_\nu^\mu}$ to live in.
Finally, recall that tensors are maps between the Cartesian products of covectors and vectors to a scalar field,
\[ \tensor{T}{^\mu_\nu} : V^* \times V \to \mathbb{F}, \]
while $\tensor{\Lambda}{_\nu^\mu}$ is mapping between vectors and other vectors in the same space, so
\[ \tensor{\Lambda}{_\nu^\mu} : V \to V. \]

\subsection{Transforming Vectors}
Suppose we have a vector $\vec{w}$, written equivalently in our two different bases as $A^\mu \vec{e}_\mu$ and $B^\nu \unit{e}_\nu$.
Since we defined a linear transformation $\unit{e}_\nu = \tensor{\Lambda}{_\nu^\mu} \vec{e}_\mu$ to change between basis vectors, we can rewrite this as
\[ \vec{w} = B^\nu \unit{e}_\nu = B^\nu \tensor{\Lambda}{_\nu^\mu} \vec{e}^\mu. \]
Now $\vec{w}$ is written in terms of the $\vec{e}_\mu$ basis, and since we already know that $\vec{w} = A^\mu \vec{e}_\mu$, we know that
\[ A^\mu = B^\nu\tensor{\Lambda}{_\nu^\mu}. \]
This allows us to transform the coefficients between bases, along with basis vectors, which means that we can transform \emph{any} vector now using this method.

\subsection{Reverse Transformations}
We've just demonstrated how to go from a $\vec{e}_\mu$ to $\unit{e}_\nu$, and from $B^\nu$ to $A^\mu$, but what if we want to go the other direction?
Well, since all these transformations are \emph{linear}, and therefore \emph{invertable}\footnote{There's actually an additional requirement placed on us here. Not all linear transformations are invertible, since some destroy information. Consider the transformation $T : x \mapsto 0$, which takes any input and spits out $0$. You can check that this satisfies every property of linearity, but its clear that it's not an invertible transformation: if I give you an $x$, you can tell me that $T(x)$ is zero, but if I ask ``what number did I plug into $T$ to get $0$?'', you'll have infinitely many possibilities. In order for $\Lambda$ to be invertible, it must have a non-zero determinant. You can think of the determinant as the ``signed volume'' of the unit $n$-cube after transformation by $\Lambda$, where $n = \dim(\Lambda)$. It's a good excercise to think about why we know for certain that transformations between bases are always invertible.}, we can simply use the inverse matrix $\tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu}$ to do just that.
This gives us a nice table to show how various structures transform using $\Lambda$:
\[
    \begin{matrix}
        \unit{e}_\nu = \tensor{\Lambda}{_\nu^\mu} \vec{e}_\mu &  & \vec{e}_\mu = \tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu}\unit{e}_\nu \\
        & & \\
         A^\mu = B^\nu\tensor{\Lambda}{_\nu^\mu} & & B^\nu = A^\mu \tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu}
    \end{matrix}
\]
Entries in the first column transform using $\Lambda$, while entries in the second transform using $\Lambda^{-1}$.
The first row shows how the basis vectors transform, and the second row shows how the coefficients transform.
Its important to notice a few things here.
\begin{itemize}
    \item When we invert the transformation $\Lambda$, the indices switch places; in $\tensor{\Lambda}{_\nu^\mu}$, $\nu$ is the lower index while $\mu$ is the upper, while in $\tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu}$, $\mu$ is the lower index while $\nu$ is the upper.
    \item Basis vectors transform with $\Lambda$ or $\Lambda^{-1}$ on the left, while coefficients transform with it on the right; this is because the coefficients are scalars, while the basis vectors are vectors, and so $\Lambda$ acts as an operator.
    \item In each change of basis, we need to change both the basis vector and the coefficient. However, basis vectors and coefficients in the same transformation use \emph{different} matrices; we don't just use $\Lambda$ to go from the $\nu$ vectors to the $\mu$ vectors, we need both $\Lambda$ and $\Lambda^{-1}$.
    Consider that if we want to change $B^\nu \unit{e}_\nu$ into a $\mu$ basis, we use the following two transformations to do so:
    \[
        \begin{matrix}
            \unit{e}_\nu = \tensor{\Lambda}{_\nu^\mu} \vec{e}_\mu &  & \mathcolor{gray}{\vec{e}_\mu = \tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu}\unit{e}_\nu} \\
            & & \\
             \mathcolor{gray}{A^\mu = B^\nu\tensor{\Lambda}{_\nu^\mu}} & & B^\nu = A^\mu \tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu}
        \end{matrix}
    \]
    To see why we need both $\Lambda$ and $\Lambda^{-1}$ in a single transformation, let's go back to our vector $\vec{w}$, and transform it entirely from the $\nu$ basis back to the $\mu$ basis.
    \[
        \vec{w} = B^\nu \unit{e}_\nu = A^\mu \tensor{\left(\Lambda^{-1}\right)}{_\mu^\nu} \tensor{\Lambda}{_\nu^\mu} \vec{e}_\mu = A^\mu \vec{e}_\mu.
    \]
    Notice how the $\Lambda$ and $\Lambda^{-1}$ end up right next to one another, and because they are inverses of one another, they cancel out, leaving us with just the resultant components and basis vectors.
    If we want to end up with no matrices in our final vector, we have to use both $\Lambda$ and $\Lambda^{-1}$ to transform between bases.
    \item $\Lambda$ and $\Lambda^{-1}$ are arbitrary choices based on the direction of transformation we define.
    The above table is based on going from $\unit{e}_\nu$ basis vectors to $\vec{e}_\mu$ basis vectors, but we could just as easily have defined $\Lambda$ in a way that we go from $\vec{e}_\mu$ back to $\unit{e}_\nu$.
    However, regardless of which way we choose, notice that the components of vectors always transform using the inverse transformation as the basis vectors do; if $\vec{e}_\mu$ uses $\Lambda$, then $A^\mu$ will use $\Lambda^{-1}$, and if $\unit{e}_\nu$ uses $\Lambda^{-1}$, then $B^\nu$ will use $\qty(\Lambda^{-1})^{-1} = \Lambda$.
\end{itemize}

\subsection{Covariance and Contravariance}
The fact that components of basis vectors transform using the inverse transformation is where we get the notion of \emph{covariance} and \emph{contravariance}; a \emph{covariant} object is anything that transforms in the same way as the basis vectors, while a \emph{contravariant} object is anything that transforms in the inverse way.
This distinction is one of the major benefits of using the Einstein summation convention, since it let's us know at a glance how a given object will transform: anything with lower indices, like basis vectors, will transform covariantly, while anything with upper indices, like scalar components, will transform contravariantly.

Note that contravariant $\not=$ `uses $\Lambda^{-1}$'.
Contravariance is always defined with respect to how the basis transforms.
If the basis transforms with $\Lambda$, then a contravariant object \emph{will} use $\Lambda^{-1}$, but if the basis transforms using $\Lambda^{-1}$, then a contravariant object will transform with $\Lambda$.