\section*{Exercises}

\begin{ex}
  Consider the following augmented matrix in which $\ast$ denotes an
  arbitrary number and $\blacksquare$ denotes a non-zero number. Determine
  whether the given augmented matrix is consistent. If consistent, is the
  solution unique?
  \begin{equation*}
    \begin{mymatrix}{ccccc|c}
      \blacksquare & \ast & \ast & \ast & \ast & \ast \\
      0 & \blacksquare & \ast & \ast & 0 & \ast \\
      0 & 0 & \blacksquare & \ast & \ast & \ast \\
      0 & 0 & 0 & 0 & \blacksquare & \ast
    \end{mymatrix}
  \end{equation*}
  \begin{sol}
    The solution exists but is not unique.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the following augmented matrix in which $\ast$ denotes an arbitrary
  number and $\blacksquare$ denotes a non-zero number. Determine whether the
  given augmented matrix is consistent. If consistent, is the solution unique?
  \begin{equation*}
    \begin{mymatrix}{ccc|c}
      \blacksquare & \ast & \ast & \ast \\
      0 & \blacksquare & \ast & \ast \\
      0 & 0 & \blacksquare & \ast
    \end{mymatrix}
  \end{equation*}
  \begin{sol}
    A solution exists and is unique.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the following augmented matrix in which $\ast$ denotes an arbitrary
  number and $\blacksquare$ denotes a non-zero number. Determine whether the
  given augmented matrix is consistent. If consistent, is the solution unique?
  \begin{equation*}
    \begin{mymatrix}{ccccc|c}
      \blacksquare & \ast & \ast & \ast & \ast & \ast \\
      0 & \blacksquare & 0 & \ast & 0 & \ast \\
      0 & 0 & 0 & \blacksquare & \ast & \ast \\
      0 & 0 & 0 & 0 & \blacksquare & \ast
    \end{mymatrix}
  \end{equation*}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Consider the following augmented matrix in which $\ast$ denotes an arbitrary
  number and $\blacksquare$ denotes a non-zero number. Determine whether the
  given augmented matrix is consistent. If consistent, is the solution unique?
  \begin{equation*}
    \begin{mymatrix}{ccccc|c}
      \blacksquare & \ast & \ast & \ast & \ast & \ast \\
      0 & \blacksquare & \ast & \ast & 0 & \ast \\
      0 & 0 & 0 & 0 & \blacksquare & 0 \\
      0 & 0 & 0 & 0 & \ast & \blacksquare
    \end{mymatrix}
  \end{equation*}
  \begin{sol}
    There might be a solution. If so, there are infinitely many.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose a system of equations has fewer equations than
  variables. Will such a system necessarily be consistent? If so,
  explain why and if not, give an example which is not consistent.
  \begin{sol}
    No. Consider $x+y+z=2$ and $x+y+z=1$.
  \end{sol}
\end{ex}

\begin{ex}
  If a system of equations has more equations than variables, can it
  have a solution? If so, give an example and if not, explain why not.
  \begin{sol}
    These can have a solution. For example, $x+y=1$, $2x+2y=2$,
    $3x+3y=3$ even has an infinite set of solutions.
  \end{sol}
\end{ex}

\begin{ex}
  Find $h$ such that
  \begin{equation*}
    \begin{mymatrix}{rr|r}
      2 & h & 4 \\
      3 & 6 & 7
    \end{mymatrix}
  \end{equation*}
  is the augmented matrix of an \textit{inconsistent} system.
  \begin{sol}
    $h=4$.
  \end{sol}
\end{ex}

\begin{ex}
  Find $h$ such that
  \begin{equation*}
    \begin{mymatrix}{rr|r}
      1 & h & 3 \\
      2 & 4 & 6
    \end{mymatrix}
  \end{equation*}
  is the augmented matrix of a \textit{consistent} system.
  \begin{sol}
    Any $h$ will work.
  \end{sol}
\end{ex}

\begin{ex}
  Find $h$ such that
  \begin{equation*}
    \begin{mymatrix}{rr|r}
      1 & 1 & 4 \\
      3 & h & 12
    \end{mymatrix}
  \end{equation*}
  is the augmented matrix of a \textit{consistent} system.
  \begin{sol}
    Any $h$ will work.
  \end{sol}
\end{ex}

\begin{ex}
  Choose $h$ and $k$ such that the augmented matrix shown has each of the following:
  \begin{enumerate}
  \item one solution
  \item no solution
  \item infinitely many solutions
  \end{enumerate}
  \begin{equation*}
    \begin{mymatrix}{rr|r}
      1 & h & 2 \\
      2 & 4 & k
    \end{mymatrix}
  \end{equation*}
  \begin{sol}
    If $h\neq 2$ there will be a unique solution for any $k$. If $h=2$ and
    $k\neq 4$, there are no solutions. If $h=2$ and $k=4$, then there are
    infinitely many solutions.
  \end{sol}
\end{ex}

\begin{ex}
  Choose $h$ and $k$ such that the augmented matrix shown has each of the following:
  \begin{enumerate}
  \item one solution
  \item no solution
  \item infinitely many solutions
  \end{enumerate}
  \begin{equation*}
    \begin{mymatrix}{rr|r}
      1 & 2 & 2 \\
      2 & h & k
    \end{mymatrix}
  \end{equation*}
  \begin{sol}
    If $h\neq 4$, then there is exactly one solution. If $h=4$ and $k\neq 4$,
    then there are no solutions. If $h=4$ and $k=4$, then there are infinitely
    many solutions.
  \end{sol}
\end{ex}

\begin{ex}
  Determine if the system is consistent. If so, is the solution unique?
  \begin{equation*}
    \begin{array}{c}
      x+2y+z-w=2 \\
      x-y+z+w=1 \\
      2x+y-z=1 \\
      4x+2y+z=5
    \end{array}
  \end{equation*}
  \begin{sol}
    There is no solution. The system is inconsistent. You can see this from the
    augmented matrix. $\begin{mymatrix}{rrrr|r}
      1 & 2 & 1 & -1 & 2 \\
      1 & -1 & 1 & 1 & 1 \\
      2 & 1 & -1 & 0 & 1 \\
      4 & 2 & 1 & 0 & 5
    \end{mymatrix}$, {\ef}: $\begin{mymatrix}{rrrr|r}
      1 & 2 & 1 & -1 & 2 \\
      0 & -3 & 0 & 2 & -1 \\
      0 & 0 & -3 & 0 & -2 \\
      0 & 0 & 0 & 0 & 1
    \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}
  Determine if the system is consistent. If so, is the solution unique?
  \begin{equation*}
    \begin{array}{c}
      x+2y+z-w=2 \\
      x-y+z+w=0 \\
      2x+y-z=1 \\
      4x+2y+z=3
    \end{array}
  \end{equation*}
  \begin{sol}
    Solution is: $w=\frac{3}{2}y-1$, $x=\frac{2}{3}-\frac{1}{2}y$,
    $z=\frac{1}{3}$.
  \end{sol}
\end{ex}

\begin{ex}
  Determine which matrices are in {\ef}.
  \begin{equation*}
    (a)~
    \begin{mymatrix}{rrrr}
      1 & 1 & 2 & 0 \\
      0 & 0 & 3 & 2 \\
      0 & 0 & 0 & 0
    \end{mymatrix}
    \quad
    (b)~
    \begin{mymatrix}{rrr}
      1 & 0 & 0 \\
      2 & 1 & 7
    \end{mymatrix}
    \quad
    (c)~
    \begin{mymatrix}{rrrrrr}
      0 & 1 & 0 & 0 & 5 \\
      0 & 0 & 1 & 0 & 4 \\
      0 & 0 & 0 & 1 & 3
    \end{mymatrix}
  \end{equation*}
  \begin{sol}
    (a) Yes. (b) No. (c) Yes.
  \end{sol}
\end{ex}

\begin{ex}\label{ex:rr-ef}
  Row reduce each of the following matrices to {\ef}.
  \begin{equation*}
    (a)~
    \begin{mymatrix}{rrrr}
      2 & -1 & 3 & -1 \\
      1 & 0 & 2 & 1 \\
      1 & -1 & 1 & -2
    \end{mymatrix}
    \quad
    (b)~
    \begin{mymatrix}{rrrr}
      0 & 0 & -1 & -1 \\
      1 & 1 & 1 & 0 \\
      1 & 1 & 0 & -1
    \end{mymatrix}
    \quad
    (c)~
    \begin{mymatrix}{rrrr}
      3 & -6 & -7 & -8 \\
      1 & -2 & -2 & -2 \\
      1 & -2 & -3 & -4
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    (d)~
    \begin{mymatrix}{rrrr}
      2 & 4 & 5 & 15 \\
      1 & 2 & 3 & 9 \\
      1 & 2 & 2 & 6
    \end{mymatrix}
    \quad
    (e)~
    \begin{mymatrix}{rrrr}
      4 & -1 & 7 & 10 \\
      1 & 0 & 3 & 3 \\
      1 & -1 & -2 & 1
    \end{mymatrix}
    \quad
    (f)~
    \begin{mymatrix}{rrrr}
      3 & 5 & -4 & 2 \\
      1 & 2 & -1 & 1 \\
      1 & 1 & -2 & 0
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    (g)~
    \begin{mymatrix}{rrrr}
      -2 & 3 & -8 & 7 \\
      1 & -2 & 5 & -5 \\
      1 & -3 & 7 & -8
    \end{mymatrix}
  \end{equation*}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Find the general solution of the system whose augmented matrix is
  \begin{equation*}
    (a)~
    \begin{mymatrix}{rrr|r}
      1 & 2 & 0 & 2 \\
      1 & 3 & 4 & 2 \\
      1 & 0 & 2 & 1
    \end{mymatrix}
    \quad
    (b)~
    \begin{mymatrix}{rrr|r}
      1 & 2 & 0 & 2 \\
      2 & 0 & 1 & 1 \\
      3 & 2 & 1 & 3
    \end{mymatrix}
    \quad
    (c)~
    \begin{mymatrix}{rrr|r}
      1 & 1 & 0 & 1 \\
      1 & 0 & 4 & 2
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    (d)~
    \begin{mymatrix}{rrrrr|r}
      1 & 0 & 2 & 1 & 1 & 2 \\
      0 & 1 & 0 & 1 & 2 & 1 \\
      1 & 2 & 0 & 0 & 1 & 3 \\
      1 & 0 & 1 & 0 & 2 & 2
    \end{mymatrix}
    \quad
    (e)~
    \begin{mymatrix}{rrrrr|r}
      1 & 0 & 2 & 1 & 1 & 2 \\
      0 & 1 & 0 & 1 & 2 & 1 \\
      0 & 2 & 0 & 0 & 1 & 3 \\
      1 & -1 & 2 & 2 & 2 & 0
    \end{mymatrix}
  \end{equation*}

  \begin{sol}
    (b) The {\ef} is
    $\def\arraystretch{1.3}
    \begin{mymatrix}{rrr|r}
      1 & 0 & \frac{1}{2} & \frac{1}{2} \\
      0 & 1 & -\frac{1}{4} & \frac{3}{4} \\
      0 & 0 & 0 & 0
    \end{mymatrix}$. Therefore, the solution is of the form $z=t$,
    $y=\frac{3}{4}+t\paren{\frac{1}{4}}$, $x=\frac{1}{2}-\frac{1}{2}t$,
    where $t\in \R$.

    (c) The {\ef} is $\begin{mymatrix}{rrr|r}
      1 & 0 & 4 & 2 \\
      0 & 1 & -4 & -1
    \end{mymatrix}$ and so the solution is $z=t$, $y=-1+4t$, $x=2-4t$.

    (d) The {\ef} is $\begin{mymatrix}{rrrrr|r}
      1 & 0 & 0 & 0 & 9 & 3 \\
      0 & 1 & 0 & 0 & -4 & 0 \\
      0 & 0 & 1 & 0 & -7 & -1 \\
      0 & 0 & 0 & 1 & 6 & 1
    \end{mymatrix}$ and so $x_5=t$, $x_4=1-6t$, $x_3=-1+7t$,
    $x_2=4t$, $x_1=3-9t$.

    (e) The {\ef} is
    $\def\arraystretch{1.3}
    \begin{mymatrix}{rrrrr|r}
      1 & 0 & 2 & 0 & -\frac{1}{2} & \frac{5}{2} \\
      0 & 1 & 0 & 0 & \frac{1}{2} & \frac{3}{2} \\
      0 & 0 & 0 & 1 & \frac{3}{2} & -\frac{1}{2} \\
      0 & 0 & 0 & 0 & 0 & 0
    \end{mymatrix}$. Therefore, let $x_5=t$, $x_3=s$. Then the other
    variables are given by $x_4=-\frac{1}{2}-\frac{3}{2}t$,
    $x_2=\frac{3}{2}-t\frac{1}{2}$, $x_1=\frac{5}{2}+\frac{1}{2}t-2s$.
  \end{sol}
\end{ex}

\begin{ex}
  Solve the system of equations $7x+14y+15z=22$, $2x+4y+3z=5$, and $3x+6y+10z=13$.
  \begin{sol}
    Solution is: $\mat{x=1-2t,z=1,y=t}$.
  \end{sol}
\end{ex}

\begin{ex}
  Solve the system of equations $3x-y+4z=6$,
  $y+8z=0$, and $-2x+y=-4$.
  \begin{sol}
    Solution is: $\mat{x=2-4t,y=-8t,z=t}$.
  \end{sol}
\end{ex}

\begin{ex}
  Solve the system of equations $9x-2y+4z=-17$, $13x-3y+6z=-25$, and
  $-2x-z=3$.
  \begin{sol}
    Solution is: $\mat{x=-1,y=2,z=-1}$.
  \end{sol}
\end{ex}

\begin{ex}
  Solve the system of equations
  $65x+84y+16z=546$, $81x+105y+20z=682$, and $84x+110y+21z=713$.
  \begin{sol}
    Solution is:
    $\mat{x=2,y=4,z=5}$.
  \end{sol}
\end{ex}

\begin{ex}
  Solve the system of equations
  $8x+2y+3z=-3$, $8x+3y+3z=-1$, and $4x+y+3z=-9$.
  \begin{sol}
    Solution is: $\mat{x=1,y=2,z=-5}$.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose a system of equations has fewer equations than variables and
  you have found a solution to this system of equations. Is it possible that
  your solution is the only one?\ Explain.
  \begin{sol}
    No. Consider $x+y+z=2$ and $x+y+z=1$.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose a system of linear equations has an augmented
  matrix with 2 rows and 4 columns and the last column is a pivot
  column. Could the system of linear equations be consistent? Explain.
  \begin{sol}
    No. This would lead to $0=1$.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose the coefficient matrix of a system of $n$ equations with $n$
  variables has the property that every column is a pivot column. Does it
  follow that the system of equations must have a solution? If so, must the
  solution be unique? Explain.
  \begin{sol}
    Yes. It has a unique solution.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose there is a unique solution to a system of linear equations.
  What must be true of the pivot columns in the augmented matrix?
  \begin{sol}
    The last column must not be a pivot column. The remaining columns must each be pivot
    columns.
  \end{sol}
\end{ex}

\begin{ex}
  The steady state temperature, $u$, of a plate solves Laplace's
  equation, $\Delta u=0$. One way to approximate the solution is to
  divide the plate into a square mesh and require the temperature at
  each node to equal the average of the temperature at the four
  adjacent nodes. In the following picture, the numbers represent the
  observed temperature at the indicated nodes. Find the temperature at
  the interior nodes, indicated by $x$, $y$, $z$, and $w$. One of the
  equations is $z=\frac{1}{4}(10+0+w+x)$.

  \begin{center}
    \begin{tikzpicture}[scale=0.8]
      \draw (1,0) -- (1,3);
      \draw (2,0) -- (2,3);
      \draw (0,1) -- (3,1);
      \draw (0,2) -- (3,2);
      \fill (0,1) circle (3pt) node[left]{$10$};
      \fill (0,2) circle (3pt) node[left]{$10$};
      \fill (1,0) circle (3pt) node[below]{$0$};
      \fill (1,1) circle (3pt) node[above right]{$z$};
      \fill (1,2) circle (3pt) node[above right]{$x$};
      \fill (1,3) circle (3pt) node[above]{$20$};
      \fill (2,0) circle (3pt) node[below]{$0$};
      \fill (2,1) circle (3pt) node[above right]{$w$};
      \fill (2,2) circle (3pt) node[above right]{$y$};
      \fill (2,3) circle (3pt) node[above]{$20$};
      \fill (3,1) circle (3pt) node[right]{$30$};
      \fill (3,2) circle (3pt) node[right]{$30$};
    \end{tikzpicture}
  \end{center}
  \begin{sol}
    You need $\def\arraystretch{1.2}
    \begin{array}{r}
      \frac{1}{4}(20+30+w+x) -y=0 \\
      \frac{1}{4}(y+30+0+z) -w=0 \\
      \frac{1}{4}(20+y+z+10) -x=0 \\
      \frac{1}{4}(x+w+0+10) -z=0
    \end{array}$.
    Solution is: $\mat{x=15,y=20,z=10,w=15}$.
  \end{sol}
\end{ex}

\begin{ex}
  Find the rank of the following matrices.
  \begin{equation*}
    (a)~
    \begin{mymatrix}{rrrr}
      4 & -16 & -1 & -5 \\
      1 & -4 & 0 & -1 \\
      1 & -4 & -1 & -2
    \end{mymatrix}
    \quad
    (b)~
    \begin{mymatrix}{rrrr}
      3 & 6 & 5 & 12 \\
      1 & 2 & 2 & 5 \\
      1 & 2 & 1 & 2
    \end{mymatrix}
    \quad
    (c)~
    \begin{mymatrix}{rrrrr}
      0 & 0 & -1 & 0 & 3 \\
      1 & 4 & 1 & 0 & -8 \\
      1 & 4 & 0 & 1 & 2 \\
      -1 & -4 & 0 & -1 & -2
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    (d)~
    \begin{mymatrix}{rrrrr}
      1 & -2 & 0 & 3 & 11 \\
      1 & -2 & 0 & 4 & 15 \\
      1 & -2 & 0 & 3 & 11 \\
      0 & 0 & 0 & 0 & 0
    \end{mymatrix}
    \quad
    (e)~
    \begin{mymatrix}{rrr}
      -2 & -3 & -2 \\
      1 & 1 & 1 \\
      1 & 0 & 1 \\
      -3 & 0 & -3
    \end{mymatrix}
  \end{equation*}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Suppose $A$ is an $m\times n$-matrix. Explain why the rank of $A$ is
  always no larger than $\min (m,n)$.
  \begin{sol}
    The rank is the number of pivot entries in the {\ef}. There is at
    most one pivot entry in each row and column. Therefore, the rank
    cannot be larger than the number of rows or the number of columns;
    in other words, the rank is at most $\min(m,n)$.
  \end{sol}
\end{ex}

\begin{ex}
  State whether each of the following sets of data is
  possible for a system of equations. If possible, describe the
  solution set.  That is, indicate whether there exists a unique
  solution, no solution or infinitely many solutions. Here, $A$ is
  the coefficient matrix, and $\mat{A\mid B}$ denotes the
  augmented matrix of the system.

  \begin{enumerate}
  \item $A$ is a $5\times 6$-matrix, $\rank(A) =4$ and
    $\rank\mat{A\mid B} =4$.

  \item $A$ is a $3\times 4$-matrix, $\rank(A) =3$ and
    $\rank\mat{A\mid B} =2$.

  \item $A$ is a $4\times 2$-matrix, $\rank(A) =4$ and
    $\rank\mat{A\mid B} =4$.

  \item $A$ is a $5\times 5$-matrix, $\rank(A) =4$ and
    $\rank\mat{A\mid B} =5$.

  \item $A$ is a $4\times 2$-matrix, $\rank(A) =2$ and
    $\rank\mat{A\mid B} =2$.
  \end{enumerate}

  \begin{sol}
    \begin{enumerate}
    \item The {\ef} has 4 non-zero rows and 6 columns, so there are 2
      free variables, and the system has infinitely many solutions.
    \item Such a system of equations of equations does not exist. If
      you add in another column, the rank does not get smaller.
    \item Such a system of equations does not exist,
      because the rank cannot equal 4 if there are only two columns.
    \item The {\ef} has 4 non-zero rows on the left-hand side, but 5
      non-zero rows if we also include the right-hand side. Therefore
      the system is inconsistent, i.e., it has no solutions.
    \item The {\ef} has 2 non-zero rows, so there are 2
      pivot variables and no free variables. Therefore, the system has
      a unique solution.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the system $-5x+2y-z=0$ and $-5x-2y-z=0$. Both equations
  equal zero and so $-5x+2y-z=-5x-2y-z$ which is equivalent to $y=0$. Does it follow that $x$
  and $z$ can equal anything?  Notice that when $x=1$, $z=-4$, and $y=0$ are plugged in
  to the equations, the equations do not equal $0$. Why?
  \begin{sol}
    These are not legitimate row
    operations. They do not preserve the solution set of the system.
  \end{sol}
\end{ex}


\subsubsection{Incremental Numerical Solutions of Maxwell's Equations}

The following calculations are done in one dimension only for simplicity but can be easily expanded to more dimensions (ref.~\cite{numel} for detailed calculations in multiple dimensions).

Given a function $f$ we can calculate its value at a point $x_0\pm\frac{\delta}{2}$ as a Tailor expansion at $x_0$ with

\begin{align}
  f\left(x_0+\frac{\delta}{2}\right)&=f(x_0)+\frac{\delta}{2}f'(x_0)+\frac{1}{2!}\left(\frac{\delta}{2}\right)^2f''(x_0)+\frac{1}{3!}\left(\frac{\delta}{2}\right)^3f'''(x_0)+\dots,\\
  f\left(x_0-\frac{\delta}{2}\right)&=f(x_0)-\frac{\delta}{2}f'(x_0)+\frac{1}{2!}\left(\frac{\delta}{2}\right)^2f''(x_0)-\frac{1}{3!}\left(\frac{\delta}{2}\right)^3f'''(x_0)+\dots.
\end{align}

Substracting these equations leads to

\begin{equation}
  f\left(x_0+\frac{\delta}{2}\right)-f\left(x_0-\frac{\delta}{2}\right)=\delta f'(x_0)+\frac{2}{3!}\left(\frac{\delta}{2}\right)^3f'''(x_0)+\dots.
\end{equation}

Dividing by $\delta$ and rearranging gives

\begin{equation}
  \left.\frac{df(x)}{dx}\right|_{x=x_0} = \frac{f(x_0+\frac{\delta}{2})-f(x_0-\frac{\delta}{2})}{\delta} + O(\delta^2),
\end{equation}

where $O(\delta^2)$ stands for higher order $\delta$ terms which can be neglected for small $\delta$. Assuming the electric field only has a $z$-component, Maxwell's equations give

\begin{align}
  -\mu\frac{\partial\mathbf{H}}{\partial t}&=\nabla\times\mathbf{E} \quad\Rightarrow\quad \mu\frac{\partial H_y}{\partial t} = \frac{\partial E_z}{\partial x},\\ \label{eq:maxwell}
  \epsilon\frac{\partial\mathbf{E}}{\partial t}&=\nabla\times\mathbf{H} \quad\Rightarrow\quad \epsilon\frac{\partial E_z}{\partial t} = \frac{\partial H_y}{\partial x},
\end{align}

with the electric field $\mathbf{E}$, the magnetic field $\mathbf{H}$, permittivity $\epsilon$ and permeability $\mu$.


After data structures, which are a complex part of programming knowledge, we can move on to an even more ambitious goal. We will now try and build a full-blown programming language with our logic language. We will begin with simple arithmetic expressions, which we will augment with memory and flow-control statements.

\subsection{Evaluating simple arithmetic expressions}
Simple arithmetic expressions are defined as trees\footnote{These are, surprisingly, commonly referred to as \textit{expression trees}.}. Each node of the tree is either a \texttt{+} or a \texttt{*}, and the two children of the nodes are the sub expressions that must be, respectively, added or multiplied together. The recursion ends when we find a leaf (\texttt{\$}), which contains a single integer value. Examples of expressions that we can represent are \texttt{3 + 5}, \texttt{5 + 2 * 3}, etc.

The reason why such expressions can be represented with a tree can be visualized graphically quite intuitively. Consider expression \texttt{5 + 2 * 3}; we can render it into a tree as:

\begin{lstlisting}
  +
 / \
5  *
  / \
 2   3
\end{lstlisting}

The above definition of expressions can be summed up into three new data declarations:

\begin{lstlisting}
Data [Expr] "+" [Expr] Priority 10 Type Expr
Data [Expr] "*" [Expr] Priority 20 Type Expr
Data [] "$" [<<int>>] Priority 10000 Type Value
\end{lstlisting}

Of course a \texttt{Value} sometimes needs to be used where an \texttt{Expr} is expected, therefore we add\footnote{At this point we might even say ``the usual''.}:

\begin{lstlisting}
Value is Expr
\end{lstlisting}

Evaluating an expression is done with the \texttt{eval} function, which takes an \texttt{Expr} as input and then returns an \texttt{<<int>>}:

\begin{lstlisting}
Func "eval" [Expr] Priority 1 Type Expr => <<int>>
\end{lstlisting}

When the expression to evaluate is an integer constant (the \texttt{\$} operator is just a wrapper to make an \texttt{<<int>>} appear as an \texttt{Expr}), then the result of the evaluation is just the integer constant itself:

\begin{lstlisting}
---------------
eval ($i) => i
\end{lstlisting}

When the expression to evaluate is a compound expression of sub-expressions \texttt{a} and \texttt{b}, then: \begin{inparaenum}[\itshape i\upshape)]
\item we evaluate the sub-expressions into \texttt{x} and \texttt{y} respectively;
\item we combine \texttt{x} and \texttt{y} into the final result \texttt{res} by invoking native operations such as machine integer addition between \texttt{<<} and \texttt{>>} brackets.
\end{inparaenum}

\begin{lstlisting}
eval a => x
eval b => y
<<x+y>> => res
------------------
eval (a+b) => res

eval a => x
eval b => y
<<x*y>> => res
------------------
eval (a*b) => res
\end{lstlisting}

It is trivial to extend the above to yet more arithmetic operations such as subtraction, division, etc.

Consider now the evaluation of the expression seen before: \texttt{\$5 + (\$2 * \$3)}:

\begin{lstlisting}
-----------------------
eval ($5+($2*$3)) => ?
\end{lstlisting}

The first rule that is instanced is the rule for addition:

\begin{lstlisting}
eval $5 => x
eval ($2*$3) => y
<<x+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

The premise above (\texttt{\$5}) is very easily solved by applying the rule for constants:

\begin{lstlisting}
x := 5
--------------
eval $5 => x
eval ($2*$3) => y
<<x+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

At this point we cannot proceed before unwinding the stack by one step, thus we obtain:

\begin{lstlisting}
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

We now have to resolve the second premise, thus we apply the rule for multiplication and we obtain:

\begin{lstlisting}
eval $2 => x'
eval $3 => y'
<<x'*y'>> => res'
y := res'
------------------
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

We can now use a rule to solve the evaluation of the first constant:

\begin{lstlisting}
x' := 2
--------------
eval $2 => x'
eval $3 => y'
<<x'+y'>> => res'
y := res'
------------------
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

We unwind the stack by one step:

\begin{lstlisting}
eval $3 => y'
<<2*y'>> => res'
y := res'
------------------
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

Again we apply the rule for constants:

\begin{lstlisting}
y' := 3
--------------
eval $3 => y'
<<2+y'>> => res'
y := res'
------------------
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

We unwind the stack yet again:

\begin{lstlisting}
<<2*3>> => res'
y := res'
------------------
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

At this point we can just unwind the stack multiple times, from:

\begin{lstlisting}
y := 6
------------------
eval ($2*$3) => y
<<5+y>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

to:

\begin{lstlisting}
<<5+6>> => res
-------------------------
eval ($5+($2*$3)) => res
\end{lstlisting}

And finally we obtain the desired result of:

\begin{lstlisting}
------------------------
eval ($5+($2*$3)) => 11
\end{lstlisting}

\subsection{Evaluating expressions with memory}

Suppose that we now want to be able to define expressions with symbols such as \texttt{``x''} or \texttt{``y''} in the middle, for example: \texttt{5 + 2 * x}.

The immediate result is that to represent such an expression we need a new data term for variables\footnote{This term, much like \texttt{\$} for integer constants, is essentially just a wrapper around \texttt{<<string>>} that allows us to treat a string as an \texttt{Expr}.}

\begin{lstlisting}
Data [] "!" [<<string>>] Priority 10000 Type Variable
\end{lstlisting}

Of course since we want variables to be useable where expressions are expected, we add:

\begin{lstlisting}
Variable is Expr
\end{lstlisting}

Evaluation of an expression cannot be done directly, because we encounter variables we do not know what value to assign them. For this reason we shall extend the \texttt{eval} function so that it also takes as input parameter a \texttt{Memory}:

\begin{lstlisting}
Func [] "eval" [Expr Memory] Priority 1 Type Expr => <<int>>
\end{lstlisting}

In the following we assume that \texttt{Memory} is any container of data. The concrete implementation of \texttt{Memory} is actually not that important for the purpose of our scneario: it could be a list of \texttt{<<string>>}, \texttt{<<int>>} pairs, a binary search tree, a balanced binary search tree, or some other data structure such as those defined in Chapter \ref{chap:dataStructures}. The only requirement that we impose is that we have an operator to \texttt{lookup} the value of a variable given its name:

\begin{lstlisting}
Func [Memory] "lookup" [<<string>>] Priority 10 Type Expr => <<int>>
\end{lstlisting}

The rules that we have seen so far are substantially unchanged, that is they do not use the memory but rather just pass it around:

\begin{lstlisting}
-----------------
eval ($i) m => i

eval a m => x
eval b m => y
<<x+y>> => res
------------------
eval (a+b) m => res

...
\end{lstlisting}

When we encounter a variable, which is prefixed by \texttt{!}, we simply look up its value from memory and return it:

\begin{lstlisting}
m lookup v => res
------------------
eval !v m => res
\end{lstlisting}

Consider now a very simple evaluation: expression \texttt{!v + \$3}, where memory is \texttt{[v -> 10]}. We start at:

\begin{lstlisting}
------------------------------
eval (!v + $3) [v -> 10] => ?
\end{lstlisting}

We begin by applying the rule for addition, which yields:

\begin{lstlisting}
eval !v [v -> 10] => x
eval $3 [v -> 10] => y
<<x + y>> => res
--------------------------------
eval (!v + $3) [v -> 10] => res
\end{lstlisting}

We now use the rule for variable lookup, which results in:

\begin{lstlisting}
[v -> 10] lookup v => res'
x := res'
--------------------------
eval !v [v -> 10] => x
eval $3 [v -> 10] => y
<<x + y>> => res
--------------------------------
eval (!v + $3) [v -> 10] => res
\end{lstlisting}

The result of looking up is that of obtaining the appropriate value from memory, therefore leading to:

\begin{lstlisting}
x := 10
--------------------------
eval !v [v -> 10] => x
eval $3 [v -> 10] => y
<<x + y>> => res
--------------------------------
eval (!v + $3) [v -> 10] => res
\end{lstlisting}

We must now unwind the stack once for \texttt{x}:

\begin{lstlisting}
eval $3 [v -> 10] => y
<<10 + y>> => res
--------------------------------
eval (!v + $3) [v -> 10] => res
\end{lstlisting}

We have to evaluate constant \texttt{\$3}, which immediately yields \texttt{3} itself as the result:

\begin{lstlisting}
<<10 + 3>> => res
--------------------------------
eval (!v + $3) [v -> 10] => res
\end{lstlisting}

The final unwinding step gives us the expected result of:

\begin{lstlisting}
-------------------------------
eval (!v + $3) [v -> 10] => 13
\end{lstlisting}

\subsection{Evaluating statements}

Let us now consider yet another extension: we want to be able to assign variables. This means that we have a new possible expression data term, which we will use for assigning values to variables:

\begin{lstlisting}
Data [Variable] ":=" [Expr] Priority 1 Type Expr
\end{lstlisting}

Evaluation of expressions such as assignment can be considered to return nothing, and just affect memory. We therefore define the \texttt{nothing} value:

\begin{lstlisting}
Data [] "nothing" [] Priority 1 Type Value
\end{lstlisting}

A natural extension of expression that comes from assignment is the fact that expressions can also be sequenced together, much like lists:

\begin{lstlisting}
Data [] "nil" [] Priority 1 Type Expr
Data [Expr] ";" [Expr] Priority 1 Type Expr
\end{lstlisting}

After evaluating an expression, we do not just get the resulting value; this happens because if one expression contains an assignment then evaluation will return the resulting value, but also the changed memory after the assignment. We therefore define a new data type that contains both a value and memory, and extend the return type of \texttt{eval} so that it now returns such a pair:

\begin{lstlisting}
Data [Value] "," [Memory] Priority 1 Type ValueMemory
Func [] "eval" [Expr Memory] Priority 1 Type Expr => ValueMemory
\end{lstlisting}

Evaluating a null expression is quite simple: we just return nothing and pass the memory through:

\begin{lstlisting}
------------------------
eval nil m => nothing,m
\end{lstlisting}

When evaluating a sequence of expressions, we:
\begin{inparaenum}[\itshape i\upshape)]
\item evaluate the first sub-expression with the input memory \texttt{m}, therefore obtaining the new memory \texttt{m1};
\item evaluate the second sub-expression with the new memory \texttt{m1}, therefore obtaining a result \texttt{res} and new memory \texttt{m2};
\item return \texttt{res} and \texttt{m2} as the final result of the whole expression:
\end{inparaenum}

\begin{lstlisting}
eval a m0 => nothing,m1
eval b m1 => res,m2
------------------------
eval (a;b) m0 => res,m2
\end{lstlisting}

It is important to realize that the \texttt{;} operator role is simply that of letting the changes done to memory ``fall through'' from one expression to the next.

Assume now that we have some implementation of a function that adds (or updates) a key-value parir into memory:

\begin{lstlisting}
Func [Memory] "add" [<<string>> <<int>>] Priority 1 Type Expr => Memory
\end{lstlisting}

We can use this function to define how the assignment operator works:
\begin{inparaenum}[\itshape i\upshape)]
\item we evaluate the right-hand side of the assignment, obtaining a result \texttt{res} and a potentially changed memory \texttt{m1};
\item we bind \texttt{res} to \texttt{v}, therefore obtaining a new memory \texttt{m2};
\item we return \texttt{nothing} and the final memory \texttt{m2} as the final result.
\end{inparaenum}

\begin{lstlisting}
eval e m => res,m1
m1 add v res => m2
-------------------------------
eval (!v := e) m => nothing,m2
\end{lstlisting}

This actually gives us a complete memory model which can be extended to operators of all kinds. As strange as it may sound to think of memory as just another value that we freely pass around, consider the possibilities that this manner of reasoning offers us:
\begin{inparaenum}[\itshape i\upshape)]
\item we might decide to build a system where memory can be split so that one thread gets some variables and another thread gets some other variables, so that no risky interaction is possible;
\item we might define an operator that saves memory in order roll-back the program state to an earlier safe state in case of error;
\item we might define a transactional operator that creates a new copy of memory which is only committed after some kind of confirmation;
\item etc.
\end{inparaenum}

The possibilities of such a model are endless, and it is important not to be fooled by the fact that we are limiting ourselves \textit{on purpose} to implement only well-known models of computation. \footnote{Indeed, new models of computation could just as easily be built, even though they would be quite poor examples for illustration!}

\subsection{Evaluating control flow statements}

Most of the work has been done so far, but we can now add some finishing touches to turn our tiny language into something more complete, albeit still quite primitive. The final extensions to our language will therefore be a series of control flow operators such as \texttt{if}, \texttt{while}, etc., plus a few ``decorative'' symbols such as \texttt{then}, \texttt{else}, and \texttt{do} which, although not strictly necessary, make code much more readable:

\begin{lstlisting}
Data [Expr] "gt" [Expr] Priority 10 Type Expr
Data [] "if" [Expr Then Expr Else Expr] Priority 5 Type Expr
Data [] "then" [] Priority 10000 Type Then
Data [] "else" [] Priority 10000 Type Else
Data [] "while" [Expr Do Expr] Priority 5 Type Expr
Data [] "do" [] Priority 10000 Type Do
\end{lstlisting}

Since we will now also need to manipulate boolean expressions, we will create a new data type to wrap boolean values:

\begin{lstlisting}
Keyword [] "?" [<<bool>>] Priority 10000 Type Value
\end{lstlisting}

Since evaluation may now return values as well, we update the definition of \texttt{eval} so that its return type is now \texttt{Value} instead of just \texttt{<<int>>}. Evaluating a \texttt{gt} (greater than) comparison now looks very much like the evaluation of a sum or a product:

\begin{lstlisting}
eval a m0 => $x,m1
eval b m1 => $y,m2
x > y
-----------------------------
eval (a gt b) m0 => ?true,m2

eval a m0 => $x,m1
eval b m1 => $y,m2
x <= y
------------------------------
eval (a gt b) m0 => ?false,m2
\end{lstlisting}

When evaluating a conditional statement, then we first evaluate the condition. If the condition evaluates to a value of \texttt{true}, then we evaluate the \texttt{then} branch and return the result of its evaluation:

\begin{lstlisting}
eval c m0 => ?true, m1
eval a m1 => res,m2
---------------------------------------
eval (if c then a else b) m0 => res,m2
\end{lstlisting}

If the condition evaluates to a value of \texttt{false}, then we evaluate the \texttt{else} branch and return the result of its evaluation:

\begin{lstlisting}
eval c m0 => ?false, m1
eval b m1 => res,m2
---------------------------------------
eval (if c then a else b) m0 => res,m2
\end{lstlisting}

The evaluation of a \texttt{while} loop is just as simple. We evaluate the condition of the \texttt{while} loop, and if we obtain a result of \texttt{false} then we are done and we can return a result of \texttt{nothing}:

\begin{lstlisting}
eval c m0 => ?false,m1
-------------------------------------
eval (while c do b) m0 => nothing,m1
\end{lstlisting}

If the condition of the loop evaluates to \texttt{true}, then we evaluate the body of the loop once, obtaining a new memory \texttt{m2}. This new memory will be used to re-evaluate the \texttt{while} loop:

\begin{lstlisting}
eval c m0 => ?true, m1
eval b m1 => nothing,m2
eval (while c do b) m2 => nothing,m3
-------------------------------------
eval (while c do b) m0 => nothing,m3
\end{lstlisting}

In this case we will skip a practical example as it would be quite long, and it does not really add much to the examples seen previously in the chapter.

\paragraph{Conclusions}
It is quite possible to further extend our tiny language to all sorts of additional niceness. For example, we could define functions, we could distinguish between \textit{heap} and \textit{stack} memory, we could define data structures, etc. Of course this would quite complicate our definitions, but surprisingly less than one would expect. For example, a much richer implementation that also contains declaration and even scoping of variables (the fundamental construct that makes function definition possible) requires as little as 360 lines of code. The implementation of a real-time oriented programming language for game development (Casanova) took less than 600 lines of code. Compared with the equivalent definition of a compiler written by hand, we are talking about a significant difference; the Casanova compiler written in F\# is almost two orders of magnitude larger than the new implementation, at a whopping 10,000 lines of code.

Moreover, primitive or not, our tiny language now supports integer variables and some looping constructs. This means that the sample language we have defined in this chapter enjoys a very important property known as \textit{Turing completeness}, which guarantees that it is powerful enough to compute anything that is computable. This important property and its consequences will be further elaborated in Chapter \ref{chap:closingRemarks}.


\section{Integral of $\frac{1}{x^4+1}$}
When we first see function $\frac{1}{x^4+1}$, it always reminds us of the
derivative of arc tangent (which is $\frac{1}{x^2+1}$). Therefore we can solve
this using some intuitions of arc tangent. First step, we try to make it into
multiplication of polynomials in order to facilitate the integration:

$$
\begin{aligned}
	x^4+1
	&=(x^4+2x^2+1)-2x^2 \\
	&=(x^2+1)^2-(\sqrt2x)^2 \\
	&=(x^2-\sqrt2x+1)(x^2+\sqrt{2x}+1) \\
\end{aligned}
$$

It seems that we are now able to deal with it using partial fractions:

$$\frac{1}{x^4+1}=\frac{Ax+B}{x^2-\sqrt2x+1}+\frac{Cx+D}{x^2+\sqrt2x+1}$$

Since we know the numerator of the fraction in the left part is one, we can
make the addition of numerator equal to $1$:

$$
\begin{aligned}
	1=&(Ax+B)(x^2+\sqrt2x+1)+(Cx+D)(x^2-\sqrt2x+1) \\
	1=&(Ax^3+A\sqrt2x^2+Ax)+(Bx^2+B\sqrt2x+B)+ \\
	&(Cx^3-C\sqrt2x^2+Cx)+(Dx^2-D\sqrt2x+D) \\
	1=&(A+C)x^3+[\sqrt2(A-C)+B+D]x^2+[A+C+\sqrt2(B-D)]x+B+D
\end{aligned}
$$

We can turn this nasty polynomial equation into linear equations:

$$
\begin{aligned}
	\begin{cases}
		A+C=0 \\
		\sqrt2(A-C)+B+D=0 \\
		A+C+\sqrt2(B-D)=0 \\
		B+D=1 \label{e_d}
	\end{cases} \\
	\because B+D=1 \\
	\therefore \sqrt2(A-C)+1=0 \\
	\therefore A-C=-\frac{1}{\sqrt2} \\
	\because A+C=0 \\
	\therefore 2A=-\frac{1}{\sqrt2} \\
	\therefore A=-\frac{1}{2\sqrt2} \\
	\therefore C=\frac{1}{2\sqrt2} \\
	\because B+D=1,B-D=0 \\
	\therefore B=D=\frac{1}2 \\
	\begin{cases}
		A=-\frac{1}{\sqrt2} \\
		B=\frac{1}2 \\
		C=\frac{1}{\sqrt2} \\
		D=\frac{1}2 \\
	\end{cases} \\
	\therefore \frac{1}{x^4+1}=\frac{-\frac{1}{2\sqrt2}x+
	\frac{1}2}{x^2-\sqrt2x+1}+\frac{\frac{1}{2\sqrt2}x+
	\frac{1}2}{x^2+\sqrt2x+1}
\end{aligned}
$$

Since we split the one fraction into two fractions in which each denominator is
made of a second-power trinomial, so we can complete squares for integrating
for arc tangent.

\begin{eqnarray}
	\begin{aligned}
		x^2-\sqrt2x+1
		&=(x^2-\sqrt2x+\frac{1}2)+\frac{1}2 \\
		&=(x-\frac{1}{\sqrt2})^2+\frac{1}2
	\end{aligned} \label{eqn1}\\
	x^2+\sqrt2x+1=(x+\frac{1}{\sqrt2})^2+\frac{1}2 \label{eqn2}
\end{eqnarray}

Now we can integrate since everything is ready. The integration starts with
partial fractions and then plugging in \ref{eqn1} and \ref{eqn2}

$$
\begin{aligned}
	\int\frac{dx}{x^4+1}&=\int\frac{-\frac{1}{2\sqrt2}x+
	\frac{1}2}{x^2-\sqrt2x+1}dx+\int\frac{\frac{1}{2\sqrt2}x+
	\frac{1}2}{x^2+\sqrt2x+1}dx \\
	&=\frac{1}{4\sqrt2}\left(\int\frac{-2x+2\sqrt2}{(x-\frac{1}{\sqrt2})^2
	+\frac{1}2}dx+\int\frac{2x+2\sqrt2}{(x+\frac{1}{\sqrt2})^2+\frac{1}2}dx
	\right)
\end{aligned}
$$

Now we start dealing with them with the first part:

$$
\begin{aligned}
	\int\frac{-2x+2\sqrt2}{(x-\frac{1}{\sqrt2})^2+\frac{1}2}dx
	&=-\int\frac{2x}{(x-\sqrt2)^2+\frac{1}2}dx+
	\int\frac{2\sqrt2}{(x-\sqrt2)^2+\frac{1}2}dx
\end{aligned}
$$

It seems like a \textbf{$u$-substitution} is applicable in the first integral,
so we let $u=(x-\sqrt2)^2+\frac{1}2$, so $du=2xdx$. Then the whole thing is
transformed into this:

$$
\begin{aligned}
	-\int\frac{2x}{(x-\sqrt2)^2+\frac{1}2}dx+
	\int\frac{2\sqrt2}{(x-\sqrt2)^2+\frac{1}2}dx
	&=-\int\frac{du}u+
	\int\frac{2\sqrt2}{(x-\sqrt2)^2+\frac{1}2}dx \\
	&=-\ln(u)+
	\int\frac{2\sqrt2}{(x-\sqrt2)^2+\frac{1}2}dx \\
	&=-\ln(x^2-\sqrt2x+1)+
	\int\frac{2\sqrt2}{(x-\sqrt2)^2+\frac{1}2}dx \\
	&=-\ln(x^2-\sqrt2x+1)+
	\int\frac{2\sqrt2}{(x-\frac{1}{\sqrt2})^2
	+(\frac{1}{\sqrt2})^2}dx \\
	&=-\ln(x^2-\sqrt2x+1)+\sqrt2\arctan(\sqrt2x-1)
\end{aligned}
$$

The integration of the second integral is very similar, at last the indefinite
integral of $\frac1{x^4+1}$ is:

$$
\frac1{4\sqrt2}\left(\ln({x^2+\sqrt2x+1\over x^2-\sqrt2x+1})+
2\arctan(\sqrt2x+1)+2\arctan(\sqrt2x-1)\right)+C
$$


\section{Differentiation}

\subsection{Basic Definitions}

\subsubsection{Exercise 1}
Let the derivative of $f$ at $a$ be the linear operator $\lambda$. Then, by problem $1-10$, we have that
$\abs{\lambda(h)} \leq M \abs{h}$ for some $M \in \mathbb{R}$. Thus,
\begin{align*}
        \abs{(f(a + h) - f(a)} &= \frac{\abs{f(a + h) - f(a)}}{\abs{h}} * \abs{h} \\
                               &\leq \frac{\abs{f(a + h) - f(a) - \lambda(h)} + \abs{\lambda(h)}}{\abs{h}} * \abs{h} \\
        \implies \lim_{h \to 0} \abs{f(a + h) - f(a)} &\leq \lim_{h \to 0} \frac{\abs{f(a + h) - f(a) - \lambda(h)} + \abs{\lambda(h)}}{\abs{h}} \lim_{h \to 0} \abs{h} \\
                                                      &\leq M * 0 = 0
\end{align*}

\subsubsection{Exercise 2}
If $f$ is independent, we can define $g = f(x, y_0)$ for some arbitrary $y_0$. If there exists $g$ such that
$f(x, y) = g(x)$ for all $x, y$, then we have $f(x, y_1) = g(x) = f(x, y_2)$ so $f$ is independent of the
second variable. In this case, $f'(a, b) = g'(a)$.

\subsubsection{Exercise 3}
A function that is independent of both variables must, by construction, be constant.

\subsubsection{Exercise 4}
(a) When $t < 0$, we have
\begin{align*}
        h(t) = -t\abs{x} g\bigg(\frac{tx}{-t \abs{x}}\bigg) = t\abs{x}g\bigg(\frac{x}{\abs{x}}\bigg)
\end{align*}
which is the same as $h(t)$ when $t > 0$, so $h(t)$ is differentiable with derivative $f(x)$.

(b) Since $g(0, 1) = g(1, 0) = 0$, we have that $f(h, 0) = f(0, k) = 0$. Letting $\lambda = Df(0, 0)$, we have
that
\begin{align*}
        \lim_{(h, 0) \to 0} \frac{\abs{\lambda(h, 0)}}{\abs{h}} &= \lim_{(h, 0) \to 0} \frac{\abs{h} \lambda(1, 0)}{\abs{h}} \\
                                                                &= \lambda(1, 0) = 0
\end{align*}
if $\lambda$ exists. Similarly, considering $(0, k) \to 0$ gives $\lambda(0, 1) = 0$, so $\lambda = 0$.
As a result, we have that $f$ is differentiable at $(0, 0)$ only if
\begin{align*}
        \lim_{(h, k) \to (0, 0)} \frac{\abs{f(h, k)}}{\abs{(h, k)}} = \lim_{(h, k) \to (0, 0)} \abs{g\bigg(\frac{(h, k)}{\abs{(h, k)}}\bigg)} = 0
\end{align*}
which implies that $g = 0$ (we can consider $t(x, y)$ as $t \to 0$ for every $(x, y)$ on the unit circle).

\subsubsection{Exercise 5}
Apply the previous exercise with $g(x, y) = x\abs{y}$.

\subsubsection{Exercise 8}
If $f$ is differentiable, then
\begin{align*}
        \lim_{h \to 0} \frac{\abs{f(a + h) - f(a) - \lambda(h)}}{\abs{h}} &= 0 \\
                                                                          &= \lim_{h \to 0} \frac{\abs{(f_1(a + h) - f_1(a) - \lambda_1(h), f_2(a + h) - f_2(a) - \lambda_2(h))}}{\abs{h}} \\
                                                                          &\geq \lim_{h \to 0} \frac{\abs{f_1(a + h) - f_1(a) - \lambda_1(h)}}{\abs{h}}
\end{align*}
so $f_1$ and $f_2$ are both differentiable. The reverse direction is a straightforward application of the triangle
inequality.

\subsubsection{Exercise 9}
(a) If $f$ is differentiable at $a$, then we can let $g(x) = f(a) + f'(a) (x - a)$. For the other direction,
\begin{align*}
        \lim_{h \to 0} \frac{f(a + h) - a_0 - a_1h}{h} &= 0 \\
        \implies \lim_{h \to 0} \frac{f(a + h) - a_0}{h} &= a_1 \\
        \implies a_0 = f(a)
\end{align*}
so $f$ is differentiable at $a$.

(b) We can break up the $n^{\text{th}}$ order limit into
\begin{align*}
        \lim_{x \to  a} \frac{f(x) - g(x)}{(x - a)^n} &=
        \lim_{x \to a} \frac{f(x) - \sum_{i = 0}^{n - 1} \frac{f^{(i)}(a)}{i!} (x - a)^i}{(x - a)^n} +
        \frac{f^{(n)}(a)}{n!} \\
                                                      &= \frac{f^{(n)}(a)}{n!} - \frac{f^{(n)}(a)}{n!} = 0
\end{align*}
where the last step follows from repeated application of L'Hopital's.

\subsection{Basic Theorems}

\subsubsection{Exercise 11}
(a) Let $G$ be a function such that $G' = g$. Then
\begin{align*}
        f'(x, y) &= g(x + y) (\pi_1'(x, y) + \pi_2'(x, y)) \\
                 &= g(x + y) ((1, 0) + (0, 1)) = (g(x + y), g(x + y))
\end{align*}

(b) Following the setup of (a), we have
\begin{align*}
        f'(x, y) &= g(xy) (y \pi_1'(x, y) + x \pi_2'(x, y)) \\
                 &= g(xy) (y, x)
\end{align*}

\subsubsection{Exercise 12}
(a) Since $f(h, k) = hk f(1, 1)$, the desired result follows immediately from $\lim_{(h, k) \to 0} \frac{\abs{hk}}{\abs{(h, k)}} = 0$.

(b) We have that $f(a + h, b + k) - f(a, b) = f(a, k) + f(h, b) + f(h, k)$ from bilinearity, so $Df(a, b)(x, y) = f(a, y) + f(x, b)$ as
desired.

(c) The function $p: \mathbb{R} \times \mathbb{R} \to \mathbb{R}$ from Theorem 2-3 is bilinear, so it is just a special case
of (b).

\subsubsection{Exercise 14}
(a) We can bound the multilinear case with the bilinear case:
\begin{align*}
        \lim_{h \to 0} \frac{\abs{f(a_1, ..., h_i, ..., h_j, ..., a_k)}}{\abs{h}} &\leq
        \lim_{h \to 0} \frac{\abs{f(a_1, ..., h_i, ..., h_j, ..., a_k)}}{\abs{(h_i, h_j)}} \\ &\leq
        \lim_{(h_i, h_j) \to 0} \frac{\abs{f(a_1, ..., h_i, ..., h_j, ..., a_k)}}{\abs{(h_i, h_j)}} \\
                                                                                &= 0
\end{align*}

(b) Expanding $f(a_1 + h_1, ..., a_n + h_n) - f(a_1, ..., a_n) - Df(a_1, ..., a_n)(h_1, ..., h_n)$
leaves only terms that are at least bilinear, so by part (a) we are done.

\subsubsection{Exercise 16}
Using the fact that $Dx = x$ (from Theorem 2-3), we have that
\begin{align*}
        f'(f^{-1}) \circ f^{-1\prime} (x) &= x \\
        \implies f^{-1\prime} (x) &= [f'(f^{-1} (x))]^{-1}
\end{align*}

\subsection{Partial Derivatives}

\subsubsection{Exercise 19}
Since $f(1, y) = 1$, $D_2 f(1, y) = 0$.

\subsubsection{Exercise 21}
(a) The first term of $f$ is independent of $y$, so we need not consider it. Letting $G_2$ be the antiderivative of
$g_2$, we have that the second term is equivalent to $G_2(x, y) - G_2(x, 0)$. Differentiating this with respect to $y$
yields $g_2(x, y)$ as desired.

(b) Replacing the first integrand with $g_1(t, y)$ and the second integrand with $g_2(0, t)$ will give
$D_2 f(x, y) = g_1(x, y)$.

(c) The functions $f(x, y) = \frac{x^2}{2} + \frac{y^2}{2}$ and $f(x, y) = xy$ will work (obtained by integrating the
desired partial derivatives with respect to $x$ and $y$).

\subsubsection{Exercise 22}
Applying the mean value theorem to each variable gives the desired results.

\subsubsection{Exercise 23}
(a) Consider $(x_1, y_1)$ and $(x_2, y_2)$. Since $D_1 f = D_2 f = 0$, we have that $f$ is constant along the lines from
$(x_1, y_1)$ to $(-\epsilon, y_1)$, $(-\epsilon, y_1)$ to $(-\epsilon, y_2)$, and $(-\epsilon, y_2)$ to $(x_2, y_2)$ by
the logic of the previous exercise (where $\epsilon > 0$). Thus, $f(x_1, y_1) = f(x_2, y_2)$ for all $(x_1, y_1)$ and
$(x_2, y_2)$.

(b) Take the following function:
\begin{align*}
        f(x, y) =
        \begin{cases}
                0 & x < 0 \\
                1 & x \geq 0, y > 0 \\
                -1 & x \geq 0, y < 0 \\
        \end{cases}
\end{align*}
This function is not independent of $y$, since $f(0, 1) \neq f(0, -1)$. However,
$D_2 f = 0$ (due to $x \geq 0, y = 0$ not being in $A$).

\subsubsection{Exercise 25}
Since $\exp(-x^{-2})$ is $C^\infty$ for all $x \neq 0$, $f$ is also $C^\infty$ for all such $x$. For
$x = 0$, we can proceed via L'Hopital's as suggested:
\begin{align*}
        f'(0) &= \lim_{h \to 0} \frac{f(h) - f(0)}{h} \\
              &= \lim_{h \to 0} \frac{\exp(-h^{-2})}{h} \\
              &= \lim_{h \to 0} \frac{\frac{1}{h}}{\exp(h^{-2})} \\
              &= \lim_{h \to 0} \frac{h}{2\exp(h^{-2})} \\
              &= 0
\end{align*}
Repeating this procedure indefinitely shows that $f^{(i)}(0) = 0$ and that $f$ is $C^\infty$.

\subsubsection{Exercise 26}
(a) The strategy from the previous exercise should work.

(b) Consider the following function:
\begin{align*}
        f(x) =
        \begin{cases}
                \exp(-x^{-2}) \exp(-(x - \epsilon)^{-2}) & x \in (0, \epsilon) \\
                0 & x \notin (0, \epsilon)
        \end{cases}
\end{align*}
Now we can define $g(x) = \int_{0}^{x} f / \int_{0}^{\epsilon} f$ as suggested. Since
$f(x) = 0$ for all $x \geq \epsilon$, we have that $\int_{0}^{x} f = \int_{0}^{\epsilon} f$.
Similarly, $\int_{0}^{x} f = 0$ for $x \leq 0$.

(c) The fact that $g$ is $C^\infty$ follows from $f$ being $C^\infty$. Similarly,
plugging $\frac{x_i - a_i}{\epsilon}$ into $f$ gives
$\exp(-\big(\frac{x_i - (a_i + \epsilon)}{\epsilon}\big)^{-2}) \exp(-\big(\frac{x_i - (a_i - \epsilon)}{\epsilon}\big)^{-2})$, from
which it can be seen that $f$ is positive only on
$(a_1 - \epsilon, a_1 + \epsilon) \times ... \times (a_n - \epsilon, a_n + \epsilon)$
and 0 everywhere else.

(d) Since $C$ is compact and in $\mathbb{R}^n$, we can choose finitely many
closed rectangles around points in $C$ such that these rectangles are contained
in $A$ and their union covers $C$. For each such rectangle, we can then mimic
the style of the function $g$ in part (c) to produce a $C^\infty$ function that is
non-zero only in that rectangle. Finally, we can take the sum of these functions
to get a function that is $C^\infty$ and positive on $C$ while being 0 outside of a
closed set contained in $A$.

(e) The function described in (d) satisfies $f(x) \geq \epsilon$ (for some $\epsilon > 0$)
for all $x \in C$ since $C$ is compact. Thus, we can consider $g \circ f$ with
the aforementioned $\epsilon$ ($g$ is described in (b)) to get the desired
function.


\subsection{Linear first-order Ordinary Differential Equations}

\subsubsection{Linear ODEs}

For some we can write:

\(\dfrac{dy}{dt}=f(t,y)\)

\(\dfrac{dy}{dt}=q(t)-p(t)y\)

This can be solved by multiplying by an unknown function \(\mu (t)\):

\(\dfrac{dy}{dt}+p(t)y=q(t)\)

\(\mu (t)[\dfrac{dy}{dt}+p(t)y]=\mu (t)q(t)\)

We can then set \(\mu(t)=e^{\int p(t)dt}\). This means that \(\dfrac{d\mu }{dt}=p(t)u(t)\)

\(\dfrac{d}{dt}[\mu(t)y]=\mu (t)q(t)\)

\(\mu(t)y=\int \mu (t)q(t)dt + C\)

In some cases, this can then be solved.

\subsubsection{Example}

\(\dfrac{\delta y}{\delta x}=cy\)

\(y=Ae^{c(y+a)}\)

\(\dfrac{\delta^2 y}{\delta x^2}=cy\)

\(y=Ae^{\sqrt c (y+a)}\)


% Copyright 2018 Melvin Eloy Irizarry-Gelpí
\chapter{Cayley-Dickson Constructs}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The Cayley-Dickson construction is a method to generate a larger algebra $\mathcal{C}(\mathcal{A})$ given an initial algebra $\mathcal{A}$. Here ``algebra'' is used very loosely.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Pairs}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Let $a$ and $b$ be members of $\mathcal{A}$. A member $z$ of the Cayley Dickson construct $\mathcal{C}(\mathcal{A})$ is an \textbf{ordered-pair}
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
A Cayley-Dickson construct is not quite a tensor product of $\mathcal{A}$ with itself. The operations involving members of $\mathcal{C}(\mathcal{A})$ can be defined in terms of the analogous operations involving members of $\mathcal{A}$.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Addition}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The \textbf{addition} operation works as expected: the sum of two pairs $z_{1}$ and $z_{2}$ is just the pair of additions. That is, if
\begin{align*}
    z_{1} &= \begin{pmatrix}
        a_{1} & b_{1}
    \end{pmatrix} &
    z_{2} &= \begin{pmatrix}
        a_{2} & b_{2}
    \end{pmatrix}
\end{align*}
then
\begin{equation}
    z_{1} + z_{2} = \begin{pmatrix}
        a_{1} + a_{2} & b_{1} + b_{2}
    \end{pmatrix}
\end{equation}
Note that addition is a symmetric operation.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Scalar Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The \textbf{scalar multiplication} operation is also as expected: scaling $z$ by a scalar $c$ is just the pair of scalings. That is, if
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then
\begin{equation}
    cz = \begin{pmatrix}
        ca & cb
    \end{pmatrix}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Negation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The \textbf{negation} operation is a special case of scalar multiplication. If
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then
\begin{equation}
    {-z} = \begin{pmatrix}
        -a & -b
    \end{pmatrix}
\end{equation}
Note that negation is an involutive operation.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Subtraction}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
You can always define the \textbf{subtraction} operation in terms of addition and negation. That is, if
\begin{align*}
    z_{1} &= \begin{pmatrix}
        a_{1} & b_{1}
    \end{pmatrix} &
    z_{2} &= \begin{pmatrix}
        a_{2} & b_{2}
    \end{pmatrix}
\end{align*}
then
\begin{equation}
    z_{1} - z_{2} = \begin{pmatrix}
        a_{1} - a_{2} & b_{1} - b_{2}
    \end{pmatrix}
\end{equation}
Note that addition is an anti-symmetric operation.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Asterisk Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The \textbf{asterisk conjugate} operation on a member of the Cayley-Dickson construct on $\mathcal{A}$ can be stated in terms of negation and asterisk conjugate operations on members of $\mathcal{A}$. Given
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then
\begin{equation}
    {\ast z} = \begin{pmatrix}
        {\ast a} & -b
    \end{pmatrix}
\end{equation}
Note that asterisk conjugation is an involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Parabolic Construct}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
A parabolic Cayley-Dickson construct will introduce elements that square to zero. This will lead to zero-divisors.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Given two members
\begin{align*}
    z_{1} &= \begin{pmatrix}
        a_{1} & b_{1}
    \end{pmatrix} &
    z_{2} &= \begin{pmatrix}
        a_{2} & b_{2}
    \end{pmatrix}
\end{align*}
then the \textbf{parabolic multiplication} operation is
\begin{equation}
    z_{1} \wedge z_{2} = \begin{pmatrix}
        a_{1} \wedge a_{2} & b_{2} \wedge a_{1} + a_{2} \wedge {\ast b_{1}}
    \end{pmatrix}
\end{equation}
Note that the order here is very important. In some cases, this product will be commutative and/or associative, but not always.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quadrance}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Given
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then the \textbf{parabolic quadrance} of $z$ is
\begin{equation}
    \Vert z \Vert^{2} \equiv \Vert a \Vert^{2} = z \wedge {\ast z} = {\ast z} \wedge z
\end{equation}
That is, the quadrance of $z$ is the same as the quadrance of the first element of $z$. In this way it is clear that if $a$ has zero quadrance, then $z$ will have zero quadrance and no inverse.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cloak and Dagger Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The \textbf{cloak and dagger conjugate} operations on a member of the parabolic Cayley-Dickson construct on $\mathcal{A}$ can be stated in terms of cloak and dagger conjugate operations on members of $\mathcal{A}$. Given
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then
\begin{align}
    {\diamond z} &= \begin{pmatrix}
        {\diamond a} & {\dagger b}
    \end{pmatrix} &
    {\dagger z} &= \begin{pmatrix}
        {\dagger a} & {\diamond b}
    \end{pmatrix}
\end{align}
The cloak and dagger conjugates are involutions.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hodge Star}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The \textbf{Hodge star} operation on a member of the parabolic Cayley-Dickson construct on $\mathcal{A}$ can be stated in terms of dagger conjugate and Hodge star operations on members of $\mathcal{A}$. Given
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then
\begin{equation}
     {\star z} = \begin{pmatrix}
         {\dagger {\star b}} &  {\star a}
    \end{pmatrix}
\end{equation}
The Hodge star operation is not always an involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Elliptic Construct}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
An elliptic Cayley-Dickson construct will introduce elements that square to minus one. This might lead to zero-divisors, eventually.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Given two members
\begin{align*}
    z_{1} &= \begin{pmatrix}
        a_{1} & b_{1}
    \end{pmatrix} &
    z_{2} &= \begin{pmatrix}
        a_{2} & b_{2}
    \end{pmatrix}
\end{align*}
then the \textbf{elliptic multiplication} operation is
\begin{equation}
    z_{1} \wedge z_{2} = \begin{pmatrix}
        a_{1} \wedge a_{2} - {\ast b_{2}} \wedge b_{1} & b_{2} \wedge a_{1} + a_{2} \wedge {\ast b_{1}}
    \end{pmatrix}
\end{equation}
Note that the order here is very important. In some cases, this product will be commutative and/or associative, but not always.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quadrance}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Given
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then the \textbf{elliptic quadrance} of $z$ is
\begin{equation}
    \Vert z \Vert^{2} \equiv \Vert a \Vert^{2} + \Vert b \Vert^{2} = z \wedge {\ast z} = {\ast z} \wedge z
\end{equation}
That is, the quadrance of $z$ is the sum of the quadrance of the elements of $z$.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Hyperbolic Construct}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
A hyperbolic Cayley-Dickson construct will introduce elements that square to one. This will lead to zero-divisors.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Given two members
\begin{align*}
    z_{1} &= \begin{pmatrix}
        a_{1} & b_{1}
    \end{pmatrix} &
    z_{2} &= \begin{pmatrix}
        a_{2} & b_{2}
    \end{pmatrix}
\end{align*}
then the \textbf{hyperbolic multiplication} operation is
\begin{equation}
    z_{1} \wedge z_{2} = \begin{pmatrix}
        a_{1} \wedge a_{2} + {\ast b_{2}} \wedge b_{1} & b_{2} \wedge a_{1} + a_{2} \wedge {\ast b_{1}}
    \end{pmatrix}
\end{equation}
Note that the order here is very important. In some cases, this product will be commutative and/or associative, but not always.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quadrance}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Given
\begin{equation}
    z = \begin{pmatrix}
        a & b
    \end{pmatrix}
\end{equation}
then the parabolic quadrance of $z$ is
\begin{equation}
    \Vert z \Vert^{2} \equiv \Vert a \Vert^{2} - \Vert b \Vert^{2} = z \wedge {\ast z} = {\ast z} \wedge z
\end{equation}
That is, the quadrance of $z$ is the difference of the quadrance of the elements of $z$.


\section{Application: The matrix exponential}

\begin{outcome}
  \begin{enumerate}
  \item Compute $e^A$, $\sin A$, and $\cos A$, for a square matrix $A$.
  \item Apply any analytic function to a square matrix.
  \end{enumerate}
\end{outcome}

From calculus, recall that a function is called \textbf{analytic}%
\index{analytic function} if it can be defined by a power series. For example:
\begin{eqnarray*}
  e^{x} &=& 1 + x + \frac{1}{2}x^2 + \frac{1}{3!}x^3 + \frac{1}{4!}x^4 + \ldots \\
  \sin x &=& x - \frac{1}{3!}x^3 + \frac{1}{5!}x^5 - \frac{1}{7!}x^7 \pm \ldots \\
  \cos x &=& 1 - \frac{1}{2}x^2 + \frac{1}{4!}x^4 - \frac{1}{6!}x^6 \pm \ldots
\end{eqnarray*}
We know from calculus that the above power series converge for all
real numbers $x$. Since it makes sense to compute the $n\th$ power of
a square matrix, in principle it also makes sense to plug a matrix
into a power series. For a square matrix $A$, we can define
\begin{eqnarray*}
  e^{A} &=& I + A + \frac{1}{2}A^2 + \frac{1}{3!}A^3 + \frac{1}{4!}A^4 + \ldots \\
  \sin A &=& A - \frac{1}{3!}A^3 + \frac{1}{5!}A^5 - \frac{1}{7!}A^7 \pm \ldots \\
  \cos A &=& I - \frac{1}{2}A^2 + \frac{1}{4!}A^4 - \frac{1}{6!}A^6 \pm \ldots
\end{eqnarray*}
The goal of this section is to investigate whether these power series
converge, and if yes, how to compute the sum of the series. We begin
with the case of a diagonal matrix.

\begin{example}{Exponential of a diagonal matrix}{exponential-diagonal}
  Compute $e^{D}$, where
  \begin{equation*}
    D = \begin{mymatrix}{rr}
      x & 0 \\
      0 & y \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  By definition,
  \begin{eqnarray*}
    e^{D}
    &=& I + D + \frac{1}{2}D^2 + \frac{1}{3!}D^3 + \ldots \\
    &=& \begin{mymatrix}{rr} 1 & 0 \\ 0 & 1 \end{mymatrix}
        + \begin{mymatrix}{cc} x & 0 \\ 0 & y \end{mymatrix}
        + \frac{1}{2}\begin{mymatrix}{cc} x^2 & 0 \\ 0 & y^2 \end{mymatrix}
        + \frac{1}{3!}\begin{mymatrix}{cc} x^3 & 0 \\ 0 & y^3 \end{mymatrix}
        + \ldots \\
    &=& \begin{mymatrix}{cc}
      1 + x + \frac{1}{2}x^2 + \frac{1}{3!}x^3 + \ldots & 0 \\
      0 & 1 + y + \frac{1}{2}y^2 + \frac{1}{3!}y^3 + \ldots \\
    \end{mymatrix} \\
    &=& \begin{mymatrix}{cc}
      e^{x} & 0 \\
      0 & e^{y} \\
    \end{mymatrix}.
  \end{eqnarray*}
  Therefore, the exponential of a diagonal matrix is computed by
  taking the exponential of each diagonal entry. Note that this
  proves, in particular, that the sum converges.
\end{solution}

The same argument also works for applying other analytic functions to
diagonal matrices, for example:
\begin{eqnarray*}
  \sin\begin{mymatrix}{cc} x & 0 \\ 0 & y \end{mymatrix}
  &=& \begin{mymatrix}{cc} \sin x & 0 \\ 0 & \sin y \end{mymatrix}, \\
  \cos\begin{mymatrix}{cc} x & 0 \\ 0 & y \end{mymatrix}
  &=& \begin{mymatrix}{cc} \cos x & 0 \\ 0 & \cos y \end{mymatrix}.
\end{eqnarray*}
But how can we compute the matrix exponential of a non-diagonal
matrix?  This can be done by diagonalization. The following theorem
shows how:

\begin{theorem}{Matrix functions by diagonalization}{matrix-exponential}
  Suppose $A$ is a diagonalizable square matrix, with
  $A=PDP^{-1}$. Then%
  \index{matrix!exponential of}%
  \index{matrix!sine and cosine of}%
  \index{exponential function!of a matrix}%
  \index{sine!of a matrix}%
  \index{cosine!of a matrix}
  \begin{eqnarray*}
    e^{A} &=& Pe^{D}P^{-1}, \\
    \sin A &=& P(\sin D)P^{-1}, \\
    \cos A &=& P(\cos D)P^{-1}.
  \end{eqnarray*}
\end{theorem}

\begin{proof}
  We have
  \begin{eqnarray*}
    e^{A}
    &=& I + A + \frac{1}{2}A^2 + \frac{1}{3!}A^3 + \ldots \\
    &=& I + PDP^{-1} + \frac{1}{2}(PDP^{-1})^2 + \frac{1}{3!}(PDP^{-1})^3 + \ldots \\
    &=& PIP^{-1} + PDP^{-1} + \frac{1}{2}PD^2P^{-1} + \frac{1}{3!}PD^3P^{-1} + \ldots \\
    &=& P(I + D + \frac{1}{2}D^2 + \frac{1}{3!}D^3 + \ldots)P^{-1} \\
    &=& Pe^{D}P^{-1}.
  \end{eqnarray*}
  The proof for $\sin$ and $\cos$ is similar. Indeed, the same method
  works for any analytic function.
\end{proof}

\begin{example}{A matrix exponential}{matrix-exponential}
  Compute $e^{A}$, where
  \begin{equation*}
    A = \begin{mymatrix}{rr}
      -8 & 10 \\
      -5 & 7 \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We first diagonalize $A$. Following the usual method, we find that
  the eigenvalues are $\eigenvar=-3$ and $\eigenvar=2$, with
  corresponding eigenvectors
  \begin{equation*}
    \begin{mymatrix}{r} 2 \\ 1 \end{mymatrix}
    \quad\mbox{and}\quad
    \begin{mymatrix}{r} 1 \\ 1 \end{mymatrix}.
  \end{equation*}
  Therefore, we can diagonalize $A$ as $A=PDP^{-1}$, where
  \begin{equation*}
    P = \begin{mymatrix}{rr} 2 & 1 \\ 1 & 1 \end{mymatrix},
    \quad
    D = \begin{mymatrix}{rr} -3 & 0 \\ 0 & 2 \end{mymatrix},
    \quad\mbox{and}\quad
    P^{-1} = \begin{mymatrix}{rr} 1 & -1 \\ -1 & 2 \end{mymatrix}.
  \end{equation*}
  By Theorem~\ref{thm:matrix-exponential}, we have
  \begin{equation*}
    e^{A}
    = Pe^{D}P^{-1}
    = \begin{mymatrix}{rr} 2 & 1 \\ 1 & 1 \end{mymatrix}
    \begin{mymatrix}{cc} e^{-3} & 0 \\ 0 & e^2 \end{mymatrix}
    \begin{mymatrix}{rr} 1 & -1 \\ -1 & 2 \end{mymatrix}
    = \begin{mymatrix}{cc}
      2e^{-3}-e^2 & -2e^{-3}+2e^2 \\
      e^{-3}-e^2  & -e^{-3}+2e^2 \\
    \end{mymatrix}.
  \end{equation*}
\end{solution}


\section{Factor Modeling}

At least in the context of mathematical finance, factor modeling is a technique to model a covariance-stationary multivariate timeseries $\{\mathbf{x_t} | \mathbf{x_t} \in \mathbb{R}^m, t \in 0,1,...,\tau\}$ in terms of a lower dimensional vector of  time-varying common factors $\mathbf{f} \in \mathbb{R}^k$, and noise $\epsilon \in \mathbb{R}^m$. A linear factor model takes the familiar form of a cross-sectional regression:

\begin{equation}
\mathbf{x_t} = \mathbf{\alpha} + \mathbf{Bf_t}+ \mathbf{\epsilon_t}
\end{equation}

Where $\mathbf{\alpha} \in \mathbb{R}^{m \times 1}$, $\mathbf{B} \in \mathbb{R}^{m \times k}$ and $\mathbf{\epsilon_t} \in \mathbb{R}^{m x\times 1}$. So long as $\mathbf{x}_t$ is covariance stationary, $\mathbf{\alpha}$ and $\mathbf{B}$ can stay the same for all $t$, and the $k$-variate factors $\mathbf{f_t}$ are also covariance stationary, with an, in general, non-diagonal covariance matrix.

The residuals $\epsilon_t$ are $m$-variate white noise, which has zero mean and is uncorrelated across time and asset classes.

\begin{equation}
\begin{array}{l}
\mathbf{\Omega_f} = \mathrm{Cov}\left[ \mathbf{f_t} \right]\\
\mathbf{\Psi} = \mathrm{Cov} \left[ \mathbf{\epsilon_t} \right] = \mathrm{diag} \left( \sigma_1, ..., \sigma_m \right)
\end{array}
\end{equation}

Instead of fitting the factor model cross-sectionally, i.e. across $m$ timeseries, it may also be fit as a time series regression, in which case the covariances are across different time steps of a single time series. The time series regressions for time series can also be grouped in a fashion that results in a factor model that includes both time delays and cross-asset correlations. The simplest canonical factor model is due to Sharpe (1970), which has a single factor, the "market".

\subsection{Comparison to Principal Components Regression}


\subsection{Pooling layers}

The input is a matrix. We place a number of windows on the input matrix. The max of each window is an input to the next layer.

Means fewer parameters, easier to compute, less chance of overfitting

Parameters: height, width of window, stride (amount shifts by each window)

We can also add padding to the edge of the image so we don't lose data.

Same padding (use 0), valid padding (no padding)

Pooling layer compresses, takes 2x2. Max pooling returns highest activiation


\subsection{Bonus: Mobius Band and Stokes'}
We all know what a Mobius Band (or Mobius Strip) is.
In particular, it is not orientable since it only have one side.
Consider $\underline{F}=(-y,x,0)^\top/(x^2+y^2)$, so $\nabla\times\underline{F}=\underline{0}$ whenever $x^2+y^2>0$.
If we are given the parameterization
$$S=\left\{\underline{x}(u,v)=\begin{pmatrix}
    (1+\frac{v}{2}\cos\frac{u}{2})\cos u\\
    (1+\frac{v}{2}\cos\frac{u}{2})\sin u\\
    \frac{v}{2}\sin\frac{u}{2}
\end{pmatrix}:u\in[0,2\pi],v\in[-1,1]\right\}$$
Then if we apply Stokes' Theorem (which we should not), then
$$0=\int_S\nabla\times\underline{F}\cdot\mathrm d\underline{S}=\oint_{\partial S}\underline{F}\cdot\mathrm d\underline{x}$$
But the boundary, which is parameterized as
$$[0,4\pi]\ni t\mapsto \begin{pmatrix}
    (1+\frac{1}{2}\cos\frac{t}{2})\cos t\\
    (1+\frac{1}{2}\cos\frac{t}{2})\sin t\\
    \frac{1}{2}\sin\frac{t}{2}
\end{pmatrix}$$
Then we have
$$\oint_{\partial S}\underline{F}\cdot\mathrm d\underline{x}=4\pi\neq 0$$
Contradiction.


\newcommand{\quotes}[1]{``#1''}
\section{Recommender Systems}

\subsection{K nearest neighbors}

The  $K$-Nearest Neighbor method makes use of ratings by  $K$  other \quotes{similar} users when predicting  $Y_{ai}$ .

Let  $KNN(a)$  be the set of  K  users \quotes{similar} to user  $a$ , and let $sim(a,b)$ be a similarity measure between users  $a$  and  $b \in KNN(a)$. The  K -Nearest Neighbor method predicts a ranking  $Y_{ai}$  to be :

\begin{align*}
\widehat{Y}_{ai} = \displaystyle \frac{\displaystyle \sum _{b \in \text {KNN}(a)} \text {sim}(a,b) Y_{bi}}{\displaystyle \sum _{b \in \text {KNN}(a)} \text {sim}(a,b)}.
\end{align*}

The similarity measure  $sim(a,b)$  could be any distance function between the feature vectors  $xa$ and  $x_b$  of users  $a$  and  $b$ , e.g. the euclidean distance  $\left\|  x_ a-x_ b \right\|$  and the cosine similarity  $c\displaystyle \cos \theta = \frac{x_ a\cdot x_ b}{\left\|  x_ a \right\| \left\|  x_ b \right\| }$ .

\subsection{Collaborative Filtering}

Matrix $Y$ with $n$ rows (users) and $m$ columns (Movies) is sparse (entries missing), $(a,i)$th  entry  $Y_{ai}$  is the rating by user  $a$  of movie  $i$ if this rating has already been given, and blank if not. Goal is to predict matrix $X$ with no missing entries.

Let $D$  be the set of all $(a,i)$ 's for which a user rating  $Y_{ai}$  exists, i.e. $(a,i) \in D$ if and only if the rating of user  $a$  to movie  $i$  exists.

\begin{align*}
\begin{split}
\displaystyle J &= \sum _{(a,i) \in D} \frac{(Y_{ai} - \big [UV^ T\big ]_{ai})^2}{2} + \\
& \frac{\lambda }{2} \left(\sum _{a,k} U_{ak}^2 + \sum _{i,k} V_{ik}^2\right)
\end{split} \\
u&=
\begin{bmatrix}
u_1\\
u_2
\end{bmatrix};
v=
\begin{bmatrix}
v_1\\
v_2\\
v_3
\end{bmatrix}\\
\textbf{Example:}\\
v&=
\begin{bmatrix}
2\\
7\\
8
\end{bmatrix}\\
uv^T& = \begin{bmatrix}
2u_1 & 7u_1 & 8u_1\\
2u_2 & 7u_2 & 8u_2
\end{bmatrix}\\
\end{align*}

Take derivative of Objective function $J$ with respect to every user, set it to zero and find respective $u_i$ value:

\begin{align*}
\frac{d}{du_1}(J)=\frac{d}{du_1}( \frac{(7-8u_1)^2}{2} + \frac{\lambda}{2} u_1^2) = 0\\
\frac{d}{du_2}(J) = 0\\
u_1 = \frac{66}{\lambda + 68};
u_2 = \frac{16}{\lambda +53}
\end{align*}

Use resulting values for $u$ to compute $uv^T$ compare resulting matrix $X$ with matrix $Y$ and start again. Continue until convergence.


\subsection{Bayesian optimisation}

\subsubsection{Introduction}

If we have sampled from the hyperparameter space we know something about the shape.

Can we use this to inform where we should next look?

The shape of the function is \(y=f(\mathbf x)\)

We have observations \(\mathbf X\) and \(\mathbf y\).

So what's our posterior, \(P(y|\mathbf X, \mathbf y)\)?

\subsubsection{Exploration and exploitation}

The can be a tradeoff between:

\begin{itemize}
\item Exploring - which gives us a better shape for \(y=f(x)\); and
\item Exploiting - which gives us a better estimate for the global optimum.
\end{itemize}

\subsubsection{The surrogate function}

We do not know \(y=f(x)\), but we model it as:

\(z(x)=y(x)+\epsilon\)

We can then maximise \(z\)

\subsubsection{Proposing new candidates}

We want an algorithm which maps from our history of observations to a new candidate.

There are different approaches:

\begin{itemize}
\item Probability of improvement - Choosing one with the highest chance of a more optimal value
\item Expected improvement - Choosing one with the biggest expected increase in the optimal value
\item Entropy search - choosing one which reduces uncertainty about the global maximum.
\end{itemize}


\subsection{Trig Functions \& The Unit Circle}
\noindent
Imagine aa circle of radius 1 centered at the origin that we'll call the unit circle. The x and y coordinates of a point on the unit circle are completely determined by the angle $\theta$ in radians between the x-axis and a line from the origin to the point.\\

\noindent
The function $\cos{\theta}$ tells us x-coordinate of the point, while $\sin{\theta}$ tells us the y-coordinate of the point. The function $\tan{\theta} = \frac{\sin{\theta}}{\cos{\theta}}$ tells us the slope of the line from the origin to the point. Most of the trig functions have geometric interpretations as shown below. The most used ones are $\sin$, $\cos$, $\tan=\frac{\sin}{\cos}$, $\cot = \frac{\cos}{\sin}$, $\csc=\frac{1}{\sin}$, and $\sec=\frac{1}{\cos}$.

\begin{figure}[H]
	\label{unitCircle}
	\centering
	\includegraphics[width = 0.75\textwidth]{./backgroundReview/algebraPreCalc/unitCircle2.png}
	\caption{\hyperref{https://en.wikipedia.org/wiki/Unit_circle}{}{}{Wikipedia - Unit circle}}
\end{figure}

\noindent
We can also think about the inverses of these trig functions. These are either notated with a -1 exponent on the function, or the prefix arc in front of the function name. Many of these functions are only defined on a part of the domain $\left[0, 2\pi\right]$. Below is a table of the inverse trig functions and their domains.

\begin{table}[H]
	\centering
	\begin{tabular}{l|l}
		Function  & Domain                                                 \\ \hline
		$\arcsin$ & $\left[-1, 1\right]$           						   \\
		$\arccos$ & $\left[-1, 1\right]$                                   \\
		$\arctan$ & $\left(-\infty, \infty\right)$                         \\
		$\arccot$ & $\left(-\infty, \infty\right)$                         \\
		$\arccsc$ & $\left(-\infty, -1\right] \cup \left[1, \infty\right)$ \\
		$\arcsec$ & $\left(-\infty, -1\right] \cup \left[1, \infty\right)$
	\end{tabular}
\end{table}


\section{Missing lemmas, $\mathbf{RP}^n$ again, and even CW-complexes}
\begin{lemma}
We want to show that $ H_\ast(X_n,X_{n-1})\cong H_\ast(X_n/X_{n-1},\ast)$. We have the characteristic map $\left(\coprod_\alpha D^n,\coprod_\alpha S^{n-1}\right)\to (X_n,X_{n-1})$, where the map $\coprod_\alpha S^{n-1}\to X_{n-1}$ is the attaching map.
\begin{equation*}
\xymatrix{ H_\ast(X_n,X_{n-1})\ar[d]^\cong & H_\ast\left(\coprod_\alpha D^n,\coprod_\alpha S^{n-1}\right)\ar[l]\ar[d]^{\cong,homework}\\
 H_\ast(X_n/X_{n-1},\ast) & H_\ast\left(\bigvee_{\alpha}S^n_\alpha,\ast\right)\ar[l]^\cong}
\end{equation*}
\end{lemma}
For preparation, we will talk about ``strong deformation retracts''. For example, $S^{n-1}\hookrightarrow D^n-\{0\}$. You just deform everything back radially.
\begin{definition}
A subspace of a space $A$ inside $X$ is a \emph{strong deformation retract} if there is a homotopy $h:X\times I\to X$ such that $h(x,0)=x$, $h(x,1)\in A$, and $h(a,t)=a$ if $a\in A$.
\end{definition}
\begin{example}
For example, for the map $S^{n-1}\hookrightarrow D^n-\{0\}$ can be defined as $h(x,t)=(1-t)x+t\frac{x}{||x||}$.
\end{example}
A strong deformation retract is a homotopy equivalence, because we can just define the homotopy inverse to be $h(-,1)$. Then $A\hookrightarrow X\xrightarrow{h(-,1)}A$ is the identity, and $X\xrightarrow{h(-,1)}A\hookrightarrow X$ is homotopic to the identity.
\begin{example}
The map $\coprod_\alpha S^{n-1}_\alpha\xrightarrow\coprod(D^{n-1}_\alpha-\{0\})$.
\end{example}
Terminology: if $X$ is a CW-complex with filtration $X_0\subseteq X_1\subseteq\cdots\subseteq X$. A choice of characteristic maps is a ``cell structure'' for $X$. Note that this isn't specified in the CW-structure.

\begin{proof}[Proof of the lemma]
Let $X$ be a CW-complex, with a choice of a cell structure, say with characteristic maps $g_\alpha:D^n_\alpha\to X_n$. Let $C_n=\{g_\alpha(0)|\alpha\in A_n\}$. We know that $X_{n-1}\hookrightarrow X_n-C_n$. We claim that this is a strong deformation retract. This follows from our observation that $\coprod_\alpha S^{n-1}_\alpha\xrightarrow\coprod(D^{n-1}_\alpha-\{0\})$ is a strong deformation retract. In particular, $X_{n-1}\hookrightarrow X_n-C_n$ is a homotopy equivalence.

For example, consider the torus. If you look at the fundamental polygon, and remove a hole, you can retract everything back to the boundary.

Now, we have:
\begin{equation*}
\xymatrix{ H_\ast\left(\coprod_\alpha D^n_\alpha,\coprod_\alpha S^{n-1}_\alpha\right)\ar[r]\ar[d] & H_\ast(X_n,X_{n-1})\ar[d]\\
 H_\ast\left(\coprod_\alpha D^n_\alpha,\coprod_\alpha (D^n_\alpha-\{0\})\right) & H_\ast(X_n,X_n-C_n)}
\end{equation*}
The downwards arrows are isomorphisms because of strong deformation retractions, homotopy invariance, lexseq, and the 5-lemma. Recall that if $U\subseteq A\subseteq X$, then $ H_\ast(X-U)\cong H_\ast(X,A)$ if $\overline{U}\subseteq \mathrm{int}(A)$. Suppose we consider $X_{n-1}\subseteq X_n-C_n\subseteq X_n$. This is an excision because $X_{n-1}$ is already closed, and $X_n-C_n$ is already open. Then excision tells us that $ H_\ast(X_n-X_{n-1},X_n-X_{n-1}-C_n)$. This means we can extend the diagram as follows.
\begin{equation*}
\xymatrix{ H_\ast\left(\coprod_\alpha D^n_\alpha,\coprod_\alpha S^{n-1}_\alpha\right)\ar[r]\ar[d] & H_\ast(X_n,X_{n-1})\ar[d]\\
 H_\ast\left(\coprod_\alpha D^n_\alpha,\coprod_\alpha (D^n_\alpha-\{0\})\right) & H_\ast(X_n,X_n-C_n)\\
 H_\ast(\coprod_\alpha(D^n_\alpha-S^{n-1}_\alpha),\coprod_\alpha(D^n_\alpha-S^{n-1}_\alpha-\{0\}))\ar[r]\ar[u]^\cong & H_\ast(X_n-X_{n-1},X_n-X_{n-1}-C_n)\ar[u]^\cong}
\end{equation*}
The left arrow on the second row is the excision from $\coprod_\alpha S^{n-1}_\alpha\subseteq \coprod_\alpha D^n_\alpha-\{0\}\subseteq \coprod_\alpha D^n_\alpha$. The bottom right arrow is an isomorphism because $\coprod_\alpha(D^n_\alpha-S^{n-1}_\alpha),\coprod_\alpha(D^n_\alpha-S^{n-1}_\alpha-\{0\})\to (X_n-X_{n-1},X_n-X_{n-1}-C_n)$ is a homeomorphism, and hence an isomorphism. This concludes the proof of the lemma.
\end{proof}
Now for the second lemma
\begin{lemma}
We have:
\begin{equation*}
\xymatrix{\ar[r]\cdots & H_q(X_{q-1})\ar[r]\ar@{=}[d] & H_q(X_q)\ar@{->>}[r]\ar[d]^\cong\ar[drr] & H_q(X_{q+1})\ar[r]^\cong\ar[dr] & H_q(X_{q+2})\ar[d]\ar[r]^\cong & \cdots\ar[dl]\\
& 0 & H_n(C_\ast(X_n))\ar@{=}[d] & & H_n(X) \\
& & \ker(C_n(X)\xrightarrow{d}C_{n-1}(X))\ar@{^(->}[d] & &\\
& & C_n(X_n) & &}
\end{equation*}
So $ H_q(X_q)$ is free abelian. The lemma is that $ H_n(X_{n+1})\to H_n(X)$ is an isomorphism.
\end{lemma}
For preparation, we'll talk about subcomplexes.
\begin{definition}
Let $X$ be a CW-complex with a cell structure $\{g_\alpha:D^n_\alpha\to X_n|\alpha\in A_n\}$. A subcomplex is a subspace $Y\subseteq X$ such that for all $n$, there are $B_n\subseteq A_n$ such that $Y_n=Y\cap X_n$ is a CW-filtration for $Y$ with characteristic maps $\{g_\beta|\beta\in B_n\}$.
\end{definition}
\begin{example}
$X_n\subseteq X$ is a subcomplex.
\end{example}
\begin{prop}[Bredon, p. 196]
Let $X$ be a CW-complex with a chosen cell structure. Let $K\subseteq X$ be compact. Then $K$ sits inside some finite subcomplex.
\end{prop}
\begin{remark}
For fixed cell structures, unions and intersections of subcomplexes are subcomplexes.
\end{remark}
\begin{proof}[Proof of lemma 2]
Let's do surjectivity. Pick $c\in Z_n(C_\bullet)(X)$. Well, $c=\sum c_i\sigma_i$ where $\sigma_i:\Delta^n\to X$. Since $\Delta^n$ is compact, $\sigma_i(\Delta^n)$ is compact, and thus $\bigcup\sigma_i(\Delta^n)$ is compact, and hence it lies in a finite subcomplex. Hence it sits in some $X_N$ for some $N$, possibly very large. Thus $c\in S_n(X_N)\subseteq S_n(X)$. It's still a cycle because it was a cycle before. (This is a stronger result, we've proved that cycles come from cycles). This is more than enough.

Let's do injectivity. Let $c\in Z_n(C_\bullet)(X_{n+1})$. If $i_\ast$ denotes the maps $ H_n(X_q)\to H_n(X)$, then $i_\ast c\in Z_n(C_\bullet)(X)$. Suppose there was $b$ such that $db=i_\ast(c)$, so that $i_\ast(c)=0$ in $ H_n(X)$. Well, $b=\sum b_i \tau_i$ where $\tau_i:\Delta^{n+1}\to X$. Then $\bigcup \tau_i(\Delta^{n+1})$ is compact, and thus sits inside $X_M$. So $b\in S_{n+1}(X_M)$, so the equation $db=i_\ast(c)$ is still true in $X_M$. So $[c]=0$ in $ H_n(X_M)$. It's not quite what I wanted.

This is good enough, because the maps $ H_n(X_{n+1})\to H_n(X_{n+2})\to \cdots$ are all isomorphisms.
\end{proof}
We'll talk about real projective space next week.
\begin{remark}
Suppose $X$ has only even cells. For example, $\mathbf{CP}^n$, namely complex lines in $\mathbf{C}^{n+1}$ through the origin, or $S^{2n+1}/v\sim \zeta z$ for any $\zeta\in \CC$ such that $|\zeta|=1$. I have a map $S^{2n-1}\to \mathbf{CP}^{n-1}$. We have:
\begin{equation*}
\xymatrix{S^{2n-1}\ar@{^(->}[r]\ar[d] & D^{2n}\ar[d]\\
\mathbf{CP}^{n-1}\ar@{^(->}[r] & \mathbf{CP}^n}
\end{equation*}
The same argument that we had before for $\mathbf{RP}^n$ show that the CW structure on $\mathbf{CP}^n$ is $\CP^0\subseteq\CP^1\subseteq\cdots\subseteq\CP^n$. So $\CP^n=D^0\cup D^2\cup\cdots\cup D^{2n}$.

Anyway, if you had $X$ only with even cells, then $C_{\text{odd}}(X)=0$, so $ H_n(X)=\begin{cases}C_n(X) & n=2k \\ 0 & n=2k+1\end{cases}$. We've shown that:
\begin{equation*}
 H_k(\mathbf{CP}^n)=\begin{cases}
\Z & k=2n\\
0 & k=2n+1
\end{cases}
\end{equation*}
\end{remark}


\lab{Filtering and Convolution}{Filtering and Convolution}
\objective{The Fourier transform reveals things about an audio signal that are not immediately apparent from the soundwave. In this lab we learn to filter noise out of a signal using the discrete Fourier transform, and explore the effect of convolution on sound files.}

% TODO:
%       - Address stereo vs. mono signals, especially for the convolution.

\section*{Cleaning up a Noisy Signal}

Listen to \texttt{Noisysignal1.wav}.
This is a mono recording of a (probably familiar) voice with some annoying noise over it.
The plot of the soundwave isn't very descriptive; in fact, it looks like static.
See Figure \ref{fig:noisysignal}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{figures/noisy}
\caption{The plot of \texttt{Noisysignal1.wav}.}
\label{fig:noisysignal}
\end{figure}

However, if we take the Fourier transform of the signal, we see that the static in Figure \ref{fig:noisysignal} is the result of some concentrated high frequency noise.
(In this case, artificially added).
See Figure \ref{fig:noisyspec}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{figures/noisyspec}
\caption{Spectrum of \li{Noisysignal1.wav}}
\label{fig:noisyspec}
\end{figure}

The noise can be removed by setting the coefficients of the high frequencies to zero.
Since the discrete Fourier transform is symmetric, if we set coeffecient $j$ to $0$, then we must set coefficient $N - j$ to $0$ as well, where $N$ is the number of coefficients.
Then we calculate the inverse Fourier transform to get a new, clean signal.

\begin{lstlisting}
>>> rate,data = wavfile.read('Noisysignal1.wav')

# Calculate the Fourier transform
>>> fsig = sp.fft(data, axis = 0)

# Coefficients 10000 to 20000 were chosen by inspecting the
# plot of the Fourier transform.
>>> for j in xrange(10000, 20000):
...     # Set the chosen coefficients to 0
...     fsig[j] = 0
...     fsig[-j] = 0

# Calculate the inverse Fourier transform, cast it as real,
# and scale it to be compatible with the wavfile format.
>>> newsig = sp.ifft(fsig)
>>> newsig = sp.real(newsig)
>>> newsig = sp.int16(newsig / sp.absolute(newsig).max() * 32767)
\end{lstlisting}

Now we can save the resulting cleaned-up signal \texttt{newsig} to a \li{.wav} file.
The plot of the wave now reveals individual syllables as they are spoken.
See Figure \ref{fig:cleansignal}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{figures/Cleanedsignal}
\caption{The plot of \texttt{Noisysignal1.wav} after being cleaned.}
\label{fig:cleansignal}
\end{figure}

% Problem 1: Clean up a noisy signal (The only thing we have to fear...)
\begin{problem}
Listen to \texttt{Noisysignal2.wav}.
You will probably just hear noise.
Inspect the discrete Fourier transform to see where there is noise.
Remove the noise using the technique described above in order to make the cleaned-up signal audible.
What does the voice say?
Who is the speaker?
(If you don't know the answer to this last question, try a quick Google search.)
\end{problem}

The DFT is commonly used in sound filtering, though identifying the particular frequencies to zero out can be difficult.

\section*{Filtering and Convolution}

The DFT is useful for more than filtering noise out of a signal.
Suppose we have a recording of musical piece played in a small carpeted room with essentially no acoustics (little or no echo), and suppose we would like to apply an effect to make it sound as if the piece were played in a large concert hall or some other room.
The DFT makes this possible when used together with the idea of \emph{convolution}.

When a balloon is popped in large room, although the sound of the actual pop only lasts a few milliseconds, the sound echoes about the room for up to several seconds.
This echoing sound is called an \emph{impulse response} of the room, and is a way of approximating the acoustics of a room.

So first, we need a recording of how the room responds to a short pulse of sound.
Effective ways of producing a loud sound approximating a pulse include firing a (blank) gunshot, popping a balloon, or, if neither of those are available, clapping the hands one time.

Recall that we model sound with discrete samples of a soundwave in rapid succession.
When these sounds are played back, the ear percieves them as a continuous soundwave.
In other words, sound playback is a series of pulses of varying intensities, similar to the pulse in an impulse response.
If we ``mix'' the individual sounds of an instrument in a carpeted room with the impulse response from a concert hall, then the new soundwave will sound as if the instrument is being played in the concert hall.

Since audio needs to be samples frequently (44100 samples per second is standard) to create smooth playback, a recording of a song can be millions of samples.
Each of these samples needs to be combined with the impulse response, which may be several seconds long.
This may be starting to seem computationally infeasible or at least very difficult.
The key is to recognize that this process can be described as a convolution: namely, the final sound is simply the convolution of the our original sound with the impulse response.
We can calculate convolutions quickly using the convolution theorem:

\[\mathcal{F}(f \ast g) = \mathcal{F}(f)\cdot\mathcal{F}(g)\]

where $\mathcal{F}$ is the Fourier Transform, $\ast$ is convolution, and $\cdot$ is component-wise multiplication.
Thus we calculate the convolution of two arrays by simply taking the Fourier transform of each, multiplying them pointwise, and then taking the inverse transform.

% Problem 2 (Optional): Get a good ballon pop to convolve stuff with.
\begin{problem}
(Optional)\footnote{If the instructor does not require this problem then students may use the provided \texttt{balloon.wav} file which contains the sound of a balloon pop in a large room.} Find a large room or area with good acoustics, and record (an approximation to) its impulse response using a balloon pop.
To record the sound, you will want to use at least a decent microphone.
You may want to record it using the program Audacity\footnote{Audacity is free sound manipulation software and may be downloaded at http://audacity.sourceforge.net} and a laptop.
If you use a unidirectional microphone, be sure the microphone is pointing at the balloon when you pop it, so that the direct sound from the pop is picked up.
(If you don't, the result will still be okay.
However, after the convolution it will probably sound somewhat distant, as if we were standing somewhere where we couldn't hear the music directly.)
If you've chosen a good room, the response should be audible for at least a full second.

Include a plot of both the waveform and spectrum of the impulse response you recorded.
\end{problem}

% Problem 3: Convolve the balloon pop with Chopin.
\begin{problem}\label{convolution_problem}
Download and listen to the file \texttt{chopin.wav}.
You will hear a piano being played in a dead room with little or no acoustics.
Using the Convolution Theorem, take the convolution of this signal with the impulse response recorded in the previous problem.
The convolution given in the theorem is \emph{circular}, meaning that sounds at the end of the signal will tend to mix with sounds at the beginning of the signal.
To avoid this effect, add several seconds of silence to the end of \li{chopin.wav} by appending zeroes to the end of the signal.
Also, keep in mind that the Convolution Theorem requires both signals to have the same length; therefore you will need to pad the smaller of your two signals (namely, the impulse response signal) with zeros in order to make it the same size as the other signal.
These zeros should be added to the middles of signals, as we need to maintain its symmetric structure.
Describe the resulting sound.

To summarize:
\begin{enumerate}

\item Read in \texttt{chopin.wav} and the impulse response with \li{wavfile},
\item Add several seconds of silence to the signal from \texttt{chopin.wav},
\item Insert zeros into the middle of the impulse response transform so that it is the same length as,
\item Calculate the convolution of the signals,
\item And finally, calculate the inverse Fourier transform.
\end{enumerate}

\end{problem}

In some instances, a circular convolution is actually desirable.
For instance, an interesting effect is achieved by taking the circular convolution of a long segment of white noise with some other (shorter) sound.
We can create white noise using SciPy's \li{random} module:
\begin{lstlisting}
# Create 10 seconds of mono white noise.
samplerate = 22050
noise = sp.int16(sp.random.randint(-32767, 32767, samplerate * 10))
\end{lstlisting}

% Problem 4: White noise.
\begin{problem}
Create white noise and listen to the resulting sound (\textbf{CAUTION:} Turn your volume way down. It may be very very loud).
This kind of noise is called ``white" because it contains all frequencies with the same strength, or rather, with the same expected strength (since the amplitude of a specific frequency is a matter of chance).
In order to see this, plot the spectrum of the noise.
\end{problem}

Now can take the circular convolution of this noise with some other sound.
For instance, let's use \texttt{tada.wav}.
The result is in \texttt{tada-conv.wav}.
We notice that the original short sound has been sustained to an indefinite length.
The result is not a set of static tones, but rather a rich sound which preserves not only the tones, but the texture, of the original sound; you can hear different tones fluctuating randomly in amplitude over time.
If you were to play this \texttt{tada-conv.wav} on repeat, you would find that, because we used a circular convolution, the sound loops seamlessly from the end back to the beginning; however, most sound players are not capable of doing this properly, so you will probably hear a break in the sound. To demonstrate the ``seamlessness", we can paste together three copies of the sound consecutively:

\begin{lstlisting}
rate, sig = wavfile.read('tada-conv.wav')
sig = sp.append(sig, sig)
sig = sp.append(sig, sig)
\end{lstlisting}

Listen to the resulting sound, and notice that we are not able to identify where the sound loops back to the beginning, because there is no break or click.

% Useless old problem.
\begin{comment}
\begin{problem}
Record yourself singing a few notes (or, feel free to produce some other sound another way).
Take the circular convolution of white noise with this recording.
Now do it again using stereo white noise.
This is just like the mono white noise problem, but make the SciPy array in two dimensions.
It's no problem that your original recording will probably be mono; just make the left and right channels duplicate in the recording (but: be sure to use different left and right channels for the white noise).
Can you hear any difference between the mono and stereo versions of the result?
\end{problem}

Feel free to play around with this. The file \texttt{guitar-conv.mp3} is a collage of sounds created using this technique (mostly using guitar samples). You could probably think of other lots of other things you can do with this.
\end{comment}


\section*{Probability Modeling}
Assumption: Data set is generated iid\\
Find $h:X\rightarrow Y$ that minimizes pred. error $R(h) = \mathbb{E}_{x,y}[l(y;h(x))]$
$h^*(x) = \mathbb{E}[Y|X=x]$ for $R(h) = \mathbb{E}_{x,y}[(y-h(x))^2]$
Pred: $\hat{y} = \hat{\mathbb{E}}[Y|X=x]$

\subsection*{Maximum Likelihood Estimation (MLE)}
Choose a particular parametric form $\hat{P}(Y|X,\theta)$, then optimize the parameters using MLE.

$\theta^* = \underset{\theta}{\operatorname{argmax}} \hat{P}(y_1,...,y_n|x_1,...,x_n,\theta) $\\
$= \underset{\theta}{\operatorname{argmax}} \prod_{i=1}^n \hat{P}(y_i|x_i, \theta) \text{\quad (iid)}$\\
$= \underset{\theta}{\operatorname{argmin}} - \sum_{i=1}^n log \hat{P}(y_i|x_i,\theta)$\\

\subsection*{Example: MLE for Linear Gaussian}
$y_i \sim \mathcal{N} (w^T x_i, \sigma^2):$\\
$y_i = w^T x_i + \epsilon_i, \epsilon_i \sim \mathcal{N}(0, \sigma^2)$\\
Maximizing the log likelihood:\\
$\underset{w}{\operatorname{argmax}} P(y_1,...,y_n|x_1,...,x_n,w)$\\
%$= \underset{w}{\operatorname{argmax}} \prod \limits_i \frac{1}{\sqrt{2\pi\sigma^2}} e^{-\frac{1}{2} \frac{(y_i-w^T x_i)^2}{\sigma^2}}$\\
$= \underset{w}{\operatorname{argmin}}  \sum_i^n (y_i-w^Tx_i)^2$

\subsection*{Bias/Variance/Noise}
Prediction Error = $Bias^2 + Variance + Noise$

\subsection*{Maximum a posteriori estimate (MAP)}
Introduce bias by expressing assumption through a Bayesian prior $w_i \in \mathcal{N}(0, \beta^2)$\\
Bayes rule: $P(w|x,y) = \frac{P(w|x) P(y|x,w)}{P(y|x)}$\\
$ = \frac{P(w) P(y|x,w)}{P(y|x)}$, we assume w is indep. of x.
$ \underset{w}{\operatorname{argmax}} P(w|x,y)$\\
$= \underset{w}{\operatorname{argmin}} - log P(w) - log P(y|x,w) + const.$ \\
$= \underset{w}{\operatorname{argmin}} \frac{1}{2\beta^2} ||w||_2^2 + \frac{1}{2\sigma^2} \sum_{i=1}^n (y_i - w^Tx_i)^2$ \\
$= \underset{w}{\operatorname{argmin}} \lambda ||w||_2^2 + \sum_{i=1}^n (y_i - w^Tx_i)^2$ , $\lambda = \frac{\sigma^2}{\beta^2}$\\
($=\underset{w}{\operatorname{argmax}} P(w) \prod_i P(y_i|x_i,w)$, assuming noise $P(y|x,w)$ iid Gaussian, prior $P(w)$ Gaussian)

\subsection*{Logistic Regression}
Link function: $\sigma(w^Tx) = \frac{1}{1+exp(-w^Tx)}$ (Sigmoid)\\
Logistic regression replaces the assumption of Gaussian noise by iid Bernoulli noise. Can naturally output probabilities.\\
$P(y|x,w) = Ber(y; \sigma(w^Tx)) = \frac{1}{1+exp(-y w^T x)}$\\
%$=\begin{cases}
%1/(1+exp(-w^Tx)) = \sigma(w^T x)\\
%		1 - 1/(1+exp(-w^Tx)) = \sigma (-w^T x)\\
%\end{cases}$
%Learning: $w = \underset{w}{\operatorname{argmax}} P(w|x,y)$\\
%Classification: Use $P(y|x,w) = \frac{1}{1+exp(-yw^Tx)}$ and predict most likely class label.

\subsection*{Example: MLE for logistic regression}
$\underset{w}{\operatorname{argmax}} P(y_{1:n}|w,x_{1:n})\\
= \underset{w}{\operatorname{argmin}} - \sum_{i=1}^n log P(y_i|w,x_i)\\
= \underset{w}{\operatorname{argmin}} \sum_{i=1}^n log(1+exp(-y_i w^T x_i))\\
\hat{R}(w) = \sum_{i=1}^n log(1+exp(-y_i w^T x_i))$ (neg log l. f.)
%negative log likelihood function

%\subsection*{Gradient for logistic regression}
%Loss function $l(w) = log(1+exp(-yw^Tx))$\\
%$\nabla_w l(w) = \frac{1}{1+exp(-yw^Tx)} exp(-yw^Tx) (-yx)$\\
%$=\frac{1}{1+exp(yw^Tx)} (-yx)$\\
%$=P(Y = -y|w, x) (-yx)$

\subsection*{SGD for logistic regression}
1. Initialize w; 2. For t=1,2,...\\
Pick data point $(x,y) \in_{u.a.r} D$\\
Compute probability of misclassification $\hat{P}(Y = -y|w,x) = \frac{1}{1+exp(yw^Tx)}$\\
Update $w \leftarrow w + \eta_t y x \hat{P}(Y = -y|w,x)$

\subsection*{Logistic regression and regularization}
$s = ||w||_2^2$ L2 (Gaussian prior)/$|w||_1$ L1 (Laplace)\\
$\underset{w}{\operatorname{min}} \sum_{i=1}^n log(1+exp(-y_i w^T x_i)) + \lambda s$

\subsection*{SGD for L2-regularized logistic regression}
Update $w \leftarrow w (1-2\lambda \eta_t) + \eta_t y x \hat{P}(Y = -y|w,x)$