\documentclass[main.tex]{subfiles}
\begin{document}

\section*{Thu Nov 28 2019}

We want to generate random numbers in order to draw samples from known distributions.
We can only really generate \emph{pseudo}-random numbers: for example, we vary the integer\(x\) in the formula
%
\begin{align}
  x' = \qty(ax+c) \mod m
\,,
\end{align}
%
with constant integer \(a\), \(c\) and \(m\).

Importantly, the values of the constants and the starting value of \(x\) constitute a \emph{seed} which can be used to reproduce our results.

If we want to produce numbers distributed according to an arbitrary pdf, we first produce uniformly distributed numbers, and then use the laws of probability.
Say we have a desired pdf, \(p(x)\), and numbers uniformly distributed from 0 to 1, then
%
\begin{align}
  \int_{ - \infty }^{y(x)} p(\widetilde{y}) \dd{\widetilde{y}} = \int_{0}^{x} \dd{\widetilde{x}} = x
\,,
\end{align}
%
so if we are able to solve this and make \(y(x)\) explicit as a function of \(x\) we are done! The final expression is given in terms of the cumulative pdf:
%
\begin{align}
  y = P^{-1} (x)
\,.
\end{align}
%
We want to generate numbers \(m\) between 0.1 and 150 distributed according to \(p(m) = m^{-\alpha }\).

The cdf is
%
\begin{align}
\frac{1}{N}\int_{0.1}^{m} m^{-\alpha } \dd{m} = \frac{1}{N}\qty(-\frac{0.1^{-\alpha }}{1-\alpha } +  \frac{1}{1-\alpha } m^{1-\alpha }  )
\,,
\end{align}
%
where the normalization is
%
\begin{align}
  N = \int_{0.1}^{150} p(m) \dd{m} = \frac{1}{1-\alpha } \qty(150^{1-\alpha } - 0.1^{1-\alpha })
\,,
\end{align}
%
therefore we have
%
\begin{align}
  Nx = \qty(-\frac{0.1^{-\alpha }}{1-\alpha } +  \frac{1}{1-\alpha } m^{1-\alpha }  )
\,,
\end{align}
%
so
%
\begin{align}
  m = \qty(\qty(0.1^{1-\alpha } +  (1-\alpha ) Nx))^{1/(1-\alpha )}
\,.
\end{align}

This works; a note of caution: when using a linear axis the histogram (plot of the number ``\(\dd{N}\)'' of counts in a bin of constant width ``\(\dd{m}\)'') looks (up to a proportionality constant) like the probability density function, however if we plot the logarithm of our random variable we get
%
\begin{align}
  \dv{N}{ \log m}  = \dv{N}{m} \dv{m}{\log m} = \dv{N}{m} m
\,,
\end{align}
%
so if we want somthing which is proportional to the pdf we need to normalize the bin counts by multiplying them by \(1/m\).

\end{document}


\section{09/21}
Recall our definition of conditional expectation
\[\expectation{X \given Y = y} = \sum_{x} x \prob{X = x \given Y = y}\]
This is a random variable in terms of the dummy variable $y$. Conditional
expectation is a powerful tool --- for example, recall the Geometric Random
Variable given in~\cref{def:geometricrv}. We previously showed that it has
expected value $\sfrac{1}{p}$. This can similarly be shown using conditional
expectation.

Let $X$ denote the number of trials until a success, and define
\[Y = \begin{cases}1 & \hbox{first trial is success}\\
                   0 & \hbox{otherwise}\end{cases}\]
Then
\begin{align*}\expectation{X}
    &= \expectation{\expectation{X \given Y}}\\
    &= \expectation{X \given Y = 0}\prob{Y = 0} + \expectation{X \given Y = 1}\prob{Y = 1}\\
    &= \expectation{X + 1}(1 - p) + 1 \cdot p\\
    &= (1 - p)\expectation{X} + 1\\
    \shortintertext{solving for $\expectation{X}$}\\
    &= \frac{1}{p}
\end{align*}

A probability distribution is \emph{memoryless} if a future event is not
affected by previous events. This is clearly true for the geometric
distribution.

On the other hand, suppose we toss a coin until we see $k$ \emph{consecutive}
heads. This is clearly \emph{not} memoryless --- once we flip a coin heads, we
only need $k - 1$ more consecutive heads, and similarly, once we flip a tail,
our count resets and we need $k$ more consecutive heads.

Let $N_k$ denote the number of tosses to get $k$ heads in a row, and consider
$\expectation{N_k \given N_{k - 1}}$. We are conditioning on $N_{k - 1}$, hence
we have already flipped $k - 1$ consecutive heads. If the next flip is heads, we
are done. Otherwise, we must flip another $k$ consecutive heads. Thus, we have
\begin{align*}\expectation{N_k \given N_{k - 1}}
    &= \frac{1}{2}\left(N_{k - 1} + 1\right) + \frac{1}{2}\left(N_{k - 1} + 1 + N_k\right)\\
    &= N_{k - 1} + 1 + \frac{1}{2}N_k
\end{align*}
Now, linearity of expectation finishes this exercise:
\begin{align*}\expectation{N_k}
    &= \expectation{\expectation{N_k \given N_{k - 1}}}\\
    &= \expectation{N_{k - 1}} + 1 + \frac{1}{2}\expectation{N_k}
\end{align*}
which yields the recurrence
\begin{align*}
    \expectation{N_1} &= 2\\
    \expectation{N_k} &= 2\expectation{N_{k - 1}} + 2
\end{align*}
The solution to this is $\expectation{N_k} = 2^{k + 1} - 2$.

\subsection{Deviation from Expectation}
Suppose we have a random variable $X$ with mean $\mu$. What is the probability
that $X$ is ``far away'' from $\mu$? For example, consider a game where a coin
is tossed and, should it land heads, the player wins \$1, otherwise the player
loses \$1. Call the player's winnings $X$ and observe that
\[\expectation{X} = \frac{1}{2}\cdot 1 + \frac{1}{2} \cdot -1 = 0\]

On the other hand, suppose the player tosses 10 coins and wins $2^{10} - 1 =
1023$ dollars when \emph{all 10} coins come up heads, otherwise the player loses
\$1. Then
\[\expectation{X} = \frac{1}{2^{10}}\cdot \left(2^{10} - 1\right) + \left(1 - \frac{1}{2}\right) \cdot -1 = 0\]
While these games have the same expectation, we will later see that they have
very different deviation.

\begin{theorem}{Markov's Inequality}{markovinequality}
    Let $X$ be any positive-valued random variable. Then
    \[\prob{X \geq a} \leq \frac{\expectation{X}}{a}\]
\end{theorem}

\begin{proof}
    Define the random variable
    \[I = \begin{cases}
        1 & X \geq a\\
        0 & \hbox{otherwise}
    \end{cases}\]
    and notice that $I \leq \sfrac{X}{a}$ and that $\prob{I = 1} = \prob{X \geq
    a}$. Then
    \[\expectation{I} = \prob{X \geq a} \leq \frac{\expectation{X}}{a}\qedhere\]
\end{proof}

\begin{corollary}{}{}
    Let $X$ be any positive-valued random variable. Then
    \[\prob{X \geq k\expectation{X}} \leq \frac{1}{k}\]
\end{corollary}

For example, suppose we toss $n$ fair coins and let $X$ count the number of
heads. We can determine an upper bound on $\prob{X \geq \sfrac{2n}{3}}$ by
noting that $\expectation{X} = \sfrac{n}{2}$, hence
\[\prob{X \geq \frac{2n}{3}} \leq \frac{\sfrac{n}{2}}{\sfrac{2n}{3}} = \frac{3}{4}\]
This is a fairly coarse bound, and is not very precise.
% For example, the above
% can be computed directly as
% \begin{align*}\prob{X \geq \frac{2n}{3}}
%     &= \sum_{k = \sfrac{2n}{3}}^n \binom{n}{k}\left(\frac{1}{2}\right)^n\\
%     &< \left(\frac{1}{2}\right)^n\sum_{k = \sfrac{2n}{3}}^n \left(\frac{ne}{k}\right)^k
% \end{align*}

\subsection{Variance (Second Moment)}
\begin{definition}{Variance}{variance}
    The \emph{variance} of a random variable $X$, also called its \emph{second
    moment}, is defined
    \[\variance{X} = \expectation{(x - \expectation{X})^2}\]
\end{definition}
\begin{theorem}{}{newvardef}
    \[\variance{X} = \expectation{X^2} - \expectation{X}^2\]
\end{theorem}
\begin{proof}
    By definition,
    \begin{align*}\variance{X}
        &= \expectation{(X - \expectation{X})^2}\\
        &= \expectation{X^2 - 2X\expectation{X} + \expectation{X}^2}\\
        &= \expectation{X^2} - 2\expectation{X}\expectation{X} + \expectation{X}^2\\
        &= \expectation{X^2} - \expectation{X}^2\qedhere
    \end{align*}
\end{proof}

Recall the definition of a Bernoulli Random Variable --- a single trial with
probability of success $p$. Let us write $X = 1$ if our trial is a success, 0
otherwise. Then
\begin{align*}
    \expectation{X} &= p\\
    \variance{X} &= \expectation{X^2} - \expectation{X}^2\\
                 &= 1^2\prob{X = 1} - p^2\\
                 &= p - p^2\\
                 &= p(1 - p)
\end{align*}

Now, let us consider the variance of a sum of random variables.
\begin{theorem}{}{}
    Let $X$ and $Y$ be random variables. Then
    \[\variance{X + Y} = \variance{X} + \variance{Y} + 2\left(\expectation{XY} - \expectation{X}\expectation{Y}\right)\]
\end{theorem}
\begin{proof}
    By~\cref{thm:newvardef},
    \begin{align*}\variance{X + Y}
        &= \expectation{(X + Y)^2} - \expectation{X + Y}^2\\
        &= \expectation{X^2 + 2XY + Y^2} - \left(\expectation{X} + \expectation{Y}\right)^2\\
        &= \expectation{X^2} + 2\expectation{XY} + \expectation{Y^2} - \expectation{X}^2 - 2\expectation{X}\expectation{Y} - \expectation{Y}^2\\
        &= \expectation{X^2} - \expectation{X}^2 + \expectation{Y^2} - \expectation{Y}^2 + 2\left(\expectation{XY} - \expectation{X}\expectation{Y}\right)\\
        &= \variance{X} + \variance{Y} + 2\left(\expectation{XY} - \expectation{X}\expectation{Y}\right)
    \end{align*}
\end{proof}

Before proceeding, we need to define \emph{covariance}.
\begin{definition}{Covariance}{covariance}
    Given random variables $X$ and $Y$, the \emph{covariance} of $X$ and $Y$ is
    defined
    \[\covariance{X}{Y} = \expectation{\left(X - \expectation{X}\right)\left(Y - \expectation{Y}\right)}\]
\end{definition}

\begin{theorem}{}{}
    Given random variables $X$ and $Y$,
    \[\covariance{X}{Y} = \expectation{XY} - \expectation{X}\expectation{Y}\]
\end{theorem}

\begin{proof}
    By definition
    \begin{align*}\covariance{X}{Y}
        &= \expectation{\left(X - \expectation{X}\right)\left(Y - \expectation{Y}\right)}\\
        &= \expectation{XY - \expectation{X}Y - X\expectation{Y} + \expectation{X}\expectation{Y}}\\
        &= \expectation{XY} - \expectation{X}\expectation{Y} - \expectation{X}\expectation{Y} + \expectation{X}\expectation{Y}\\
        &= \expectation{XY} - \expectation{X}\expectation{Y}\qedhere
    \end{align*}
\end{proof}

\begin{corollary}{}{}
    Let $X$ and $Y$ be random variables. Then
    \[\variance{X + Y} = \variance{X} + \variance{Y} + 2\covariance{X}{Y}\]
\end{corollary}

\begin{theorem}{}{}
    Let $X$ and $Y$ be \emph{independent} random variables. Then
    \[\expectation{XY} = \expectation{X}\expectation{Y}\]
\end{theorem}

\begin{proof}
    By definition
    \begin{align*}\expectation{XY}
        &= \sum_{x}\sum_{y} xy\prob{X = x \cap Y = y}\\
        &= \sum_{x}\sum_{y} xy\prob{X = x} \prob{Y = y}\\
        &= \sum_{x}x\prob{X = x}\sum_{y}y\prob{Y = y}\\
        &= \expectation{X}\expectation{Y}\qedhere
    \end{align*}
\end{proof}

\begin{corollary}{}{}
    If $X$ and $Y$ are independent, then $\covariance{X}{Y} = 0$ and
    \[\variance{X + Y} = \variance{X} + \variance{Y}\]
\end{corollary}

Now, suppose we toss $n$ independent fair coins and wish to determine the
variance on the number of heads. Let $X$ denote the number of heads and let
$X_i$ indicate if the $i$-th coin is heads. Clearly, $\variance{X_i} = p(1 -
p)$, and by the previous corollary,
\[\variance{X} = \sum_{i = 1}^n \variance{X_i} = np(1 - p)\]


\section{Tabular and Real-time Forecasts without Retraining}
\label{tabular_ml_models}

Regarding the structure of the feature matrix for the ML models in Sub-section
    \ref{ml_models}, we provide an alternative approach that works without
    the STL method.
Instead of decomposing a time series and arranging the resulting
    seasonally-adjusted time series $a_t$ into a matrix $\mat{X}$, one can
    create a matrix with two types of feature columns mapped to the raw
    observations in $\vec{y}$:
While the first group of columns takes all observations of the same time of
    day over a horizon of, for example, one week ($n_h=7$), the second group
    takes all observations covering a pre-defined time horizon, for example
    $3$ hours ($n_r=3$ for 60-minute time steps), preceding the time step to
    be fitted.
Thus, we exploit the two-dimensional structure of time tables as well, and
    conceptually model historical and recent demand.
The alternative feature matrix appears as follows where the first three
    columns are the historical and the last three the recent demand features:

$$
\vec{y}
=
\begin{pmatrix}
    y_T \\
    y_{T-1} \\
    \dots \\
    y_{1+n_hH}
\end{pmatrix}
~~~~~
\mat{X}
=
\begin{bmatrix}
    y_{T-H}              & y_{T-2H}       & \dots & y_{T-n_hH}
        & y_{T-1}        & y_{T-2}        & \dots & y_{T-n_r} \\
    y_{T-1-H}            & y_{T-1-2H}     & \dots & y_{T-1-n_hH}
        & y_{T-2}        & y_{T-3}        & \dots & y_{T-n_r-1} \\
    \dots                & \dots          & \dots & \dots
        & \dots          & \dots          & \dots & \dots \\
    y_{1+(n_h-1)H}       & y_{1+(n_h-2)H} & \dots & y_1
        & y^*_{1+n_hH-1} & y^*_{1+n_hH-2} & \dots & y^*_{1+n_hH-n_r}
\end{bmatrix}
$$
\

Being a detail, we note that the recent demand features lying on the end of
    the previous day are set to $0$, which is shown with the $^*$ notation
    above.
This alignment of the undecomposed order data $y_t$ ensures that the ML
    models learn the two seasonal patterns independently.
The parameters $n_h$ and $n_r$ must be adapted to the data, but we found the
    above values to work well.

As such matrices resemble time tables, we refer to them as tabular.
However, we found the ML models with vertical time series to outperform the
    tabular ML models, which is why we disregarded them in the study.
This tabular form could be beneficial for UDPs with a demand that exhibits
    a weaker seasonality such as a meal delivery platform.


\lab{Algorithms}{Optimization}{Simplex}
\label{lab:Simplex}
\objective{In this lab, you will write your own implementation of the Simplex method for solving linear optimization problems.}

\section*{Simplex}
In this lab you will write your own implementation of the Simplex method for solving linear optimization problems. Your function should have the following signature

\begin{lstlisting}
>>> x,fval,exit = linprog(c, A, b)
\end{lstlisting}

The function should solve the linear program
\begin{align*}
	\text{maximize } & c^T x \\
	\text{subject to }  & Ax \leq b \\
			&x \geq 0\\
\end{align*}

where $c \in \mathcal{R}^n$ is a column vector, $b \in \mathcal{R}^m$ is a row vector, and $A \in \mathcal{R}^{m \times n}$. The output $x$ is the optimal point and the optimal value is {\tt fval}. The output {\tt exit} is one of the following values:
\begin{itemize}
	\item 1: Successful execution--you found a solution.
	\item 2: The problem is infeasible.
	\item 3: The problem is unbounded.
\end{itemize}
If either of the two error events occur, you should return {\tt None} for the values of {\tt x} and {\tt fval}.

You are free to implement {\tt linprog} any way you see fit, provided it satisfies the above requirements (you cannot just call an existing lp solver however, and you must use the Simplex method!).

\section*{Code Organization}
This is a relatively large program, so you'll want to break it up into function. Below is a list of possible functions.
Here are some of the functions you might want to use. \emph{Tableau} will refer to the Simplex tableau in this lab to avoid confusion with a Python dictionary and will be represented by {\tt T}  in the code.

\begin{lstlisting}
: T = createtableau(c,A,b)
: exit = isfeasible(T)
: exit = simplex(T)
: i,j,exit = choosepivot(T)
: pivot(T,i,j)
\end{lstlisting}

By breaking the program into so many functions, the entire program is only about 150 lines, including comments.  The {\tt simplex} function is just a few lines:

\begin{lstlisting}
: def simplex(T):
:    # choose the first pivot: this sets exit to 1 if finished or 3 if unbounded
:    i, j, exit = choosepivot(T)
:    while exit == 0:
:        pivot(T,i,j)
:        i, j, exit = choosepivot(T)
\end{lstlisting}
The {\tt linprog} function is also pretty short, since it just calls other functions and then gets the return value.

\section*{Tableau Representation}

The first problem you need to address is how to represent the tableau. We're going to use an approach that might be a little confusing at first but greatly simplifies the pivot operations. We recommend creating an $m + n + 1$ square NumPy matrix, where $m$ is the number of constraints and $n$ is the number of decision variables.  The first row represents the objective function and the first column is for the constants. The remaining columns are the coefficients of the variables. The remaining rows express the value of the variables in terms of a constant and the nonbasic variables. To illustrate this approach, let's consider the following example problem:

\begin{align*}
	\mbox{maximize}\qquad
        &    3x_1 + 2x_2 \\
	\mbox{subject to}\qquad
        &     x_1 - x_2 \leq 2 \\
		&	 3x_1 + x_2 \leq 5 \\
		&	 4x_1 + 3x_2 \leq 7 \\
		&     x_1, x_2 \geq 0.
\end{align*}

There is no need to distinguish the slack variables from the decision variables, so we will call our slack variables $x_3, x_4,$ and $x_5$. Then our problem becomes
\begin{align*}
	\mbox{maximize}\qquad
        &    3x_1 + 2x_2 \\
	\mbox{subject to}\qquad
        &     x_1 - x_2 + x_3 = 2 \\
		&	 3x_1 + x_2 + x_4 = 5 \\
		&	 4x_1 + 3x_2 + x_5 = 7 \\
		&     x_1, x_2, x_3, x_4, x_5 \geq 0.
\end{align*}

We initialize simplex by making the decision variables $x_1, x_2$ nonbasic and the slack variables basic (assuming the origin is feasible, but we will deal with initially infeasible problems shortly). The objective function is already written in terms of the nonbasic variables: $\zeta = 0 + 3x_1 + 2x_2$. We want to write all the variables in terms of the nonbasic variables. This information we get from the constraints:
\[ x_3 = 2 - x_1 + x_2, \qquad x_4 = 5 - 3x_1 - x_2, \qquad x_5 = 7 - 4x_1 - 3x_2. \]
For consistency, and simplicity that will be demonstrated later on, we want to express ALL the variables in terms of the nonbasic variables, not just the basic variables, so we add the expressions
\[ x_1 = x_1, \qquad x_2 = x_2 \]
to the tableau. This may seem silly, but it makes pivot operations very simple, as we will see. In matrix form, we then have the following initial tableau.

\[
	T = \begin{bmatrix}
		0 & 3 & 2 & 0 & 0 & 0 \\
		0 & 1 & 0 & 0 & 0 & 0 \\
		0 & 0 & 1 & 0 & 0 & 0 \\
		2 &-1 & 1 & 0 & 0 & 0 \\
		5 &-3 &-1 & 0 & 0 & 0 \\
		7 &-4 &-3 & 0 & 0 & 0
	\end{bmatrix}.
\]
The first row is the objective function written as a constant plus a linear combination of the nonbasic variables. The remaining rows are all the variables written as a constant plus a linear combination of the nonbasic variables. The general form of the $T$ matrix is
\[ T = \begin{bmatrix}
    0 & c^T   & 0 \\
    0 & I_n & 0 \\
    b & -A  & 0
\end{bmatrix}, \]
where $I_n$ is an $n \times n$ identity matrix. Therefore, your {\tt createtableau} function might include code like the following

\begin{lstlisting}
: m,n = A.shape
: s = m + n + 1;    % number of variables + number of constraints + 1
: T = numpy.zeros((s,s))
: T[0, 1:n+1] = c.T
: T[s-m:s, 0] = b.squeeze()
...
\end{lstlisting}

Note that you need the squeeze function, built into numpy, to remove single-dimensional entries from the shape of a numpy array.

\section*{Pivot Operations}

One of the nice features of the proposed tableau representation is the simplicity of pivot operations.
We want to pivot on the column $j$ that has the largest nonzero coefficient in the objective row, $c_j$, or the top row. This variable will become basic after the pivot. The pivot row,  $i$, will be the maximum of
\begin{equation}
\{\frac{b_i}{a_{ik}}| i \text{ is basic }, a_{ik} < 0\}.
\end{equation}

In our example problem, the first pivot should occur with $x_1$ becoming basic since $c_0 = 3$ and $c_1 = 2$.
The table of ratios for our example is

\begin{table}[h]
\centering
\begin{tabular}{c|c|c|c}
basic index i & x3 & x4 & x5\\
\hline
ratio $\frac{b_i}{a_{ik}}$ & $\frac{2}{-1}$&$ \frac{5}{-3}$ &$ \frac{7}{-4}$\\
\end{tabular}
\end{table}
So $x_4$ becomes nonbasic. In other words, we want to pivot at row $i = 4$ and column $j = 1$ in the tableau (the indices are offset by one because of the objective function and the row of constants). The row corresponding to $x_4$ is
\[
\begin{bmatrix} 5 &-3 &-1 & 0 & 0 & 0\end{bmatrix}.
\]
This represents the equation
\[
	x_4 = 5 - 3x_1 - x_2.
\]
Our eventual goal is to solve for $x_1$ and substitute into the remaining rows of the tableau. A simple method to accomplish this is to rewrite the equation so that we have zero on the left-hand side:
\[
	0 = 5 - 3x_1 - x_2 - x_4.
\]
Now, we can normalize this equation so that the coefficient of $x_1$ is -1. This is always accomplished by dividing the equation by the negative of the coefficient of $x_1$:
\begin{equation}\label{eq:zero-equation}
	0 = \frac{5}{3} - x_1 - \frac{1}{3}x_2 - \frac{1}{3}x_4.
\end{equation}
This is represented by the vector
\[
	\begin{bmatrix} 5/3 & -1 & -1/3 & 0 & -1/3 & 0\end{bmatrix}.
\]
Since this left-hand side is zero, I can add any scalar multiple of this equation to any of the equations for $x_i$ and still have an equation for $x_i$. For example, the equation for $x_5$ is
\[
	x_5 = 7 - 4x_1 - 3x_2.
\]
Thus, I can add $-4$ times \eqref{eq:zero-equation} to this equation without changing the left-hand side:
\[ x_5 = 7 - 4x_1 - 3x_2 = 7 - 4x_1 - 3x_2 + -4\left(\frac{5}{3} - x_1 - \frac{1}{3}x_2 - \frac{1}{3}x_4\right) = \frac{1}{3} - \frac{5}{3}x_2 + \frac{4}{3} x_4.
\]
Notice that we end up with an equation that does not include $x_1$ and now has $x_4$, just like we wanted. In fact, this works in all of our equations, including those for the objective function and even for $x_1$! Since the coefficient of $x_1$ in \eqref{eq:zero-equation} is -1, when we scale it by the coefficient of $x_1$ in any particular row, the $x_1$ cancels out. If we want to do this in a single operation, we use the outer product. We set our new tableau to
\[
	T = T + \begin{bmatrix}3 \\ 1 \\ 0 \\ -1 \\ -3 \\ -4\end{bmatrix}\begin{bmatrix} 5/3 & -1 & -1/3 & 0 & -1/3 & 0\end{bmatrix}.
\]
The column vector is just the second column of $T$, which is the column containing the coefficients of $x_1$ in each row. When we compute this sum, we obtain the tableau.
\[
	T = \begin{bmatrix}
		5 &  0 & 1 & 0 & -1 & 0 \\
		5/3 & 0 &-1/3 & 0 &-1/3 & 0 \\
		0 & 0 & 1 & 0 & 0 & 0 \\
		1/3 & 0 & 4/3 & 0 & 1/3 & 0 \\
		0 & 0 & 0 & 0 & 1 & 0 \\
		1/3 & 0 & -5/3 & 0 & 4/3 & 0
	\end{bmatrix}.
\]
You can check that these are all the correct pivots. Most importantly, $x_2$ and $x_4$ are now our nonbasic variables.

\section*{Auxiliary Problems}

When one of the entries of $b$ is negative, we need to run an auxiliary problem to find a feasible point. Try adding $x_0$ as the last variable. That way you just need to add a single row and column to your current tableau $T$. You can check that subtracting $x_0$ from each of the constraints is the same as adding $x_0$ to each of the expressions for the slack variables. Since we also need write $x_0$ in terms of itself ($x_0 = x_0$), this is equivalent to setting the last column equal to 1 for all the rows corresponding to slack variables, plus one new row. We also need to change the objective function to $-x_0$. Let's let $N$ be the matrix for the auxiliary problem. Then we can construct it from $T$ using the code
\begin{lstlisting}
: N = zeros((s+1,s+1))
: N[0:s,0:s] = T
: N[n:s+1,s] = 1
: N[0,s] = -1
\end{lstlisting}

Before you run simplex on this auxiliary tableau, make sure you do a pivot on the last column and the row corresponding to the smallest entry of $b$. Check that the objective function value is 0 when simplex finishes running. If not, your initial problem is infeasible. If the problem is feasible, get the system of equations from $N$ and put them back into $T$, leaving off the last row and column of $N$ (for $x_0$) and the first row (corresponding to the objective function).
\begin{lstlisting}
T[1:s,0:s] = N[1:s,0:s]
\end{lstlisting}
The last thing you need to do is insert your previous objective function. However, you need to re-write it in terms of the current nonbasic variables. Fortunately, $T$ currently contains all of your variables written in terms of the nonbasic variables. You can check (mathematically, or however you want to satisfy yourself) that
\begin{lstlisting}
T[0,0:s] = T[0,0:s]*T
\end{lstlisting}
will put the objective function into the first row, now written in terms of the new nonbasic variables.

\section*{Getting Return Values}

When you reach a stopping point (all the coefficients in the objective function are nonpositive), all you have to do to get the return values is to set $x = T[0,1:n+1]$, since these contain the current values of each $x_i$ and set $\mathtt{fval} = T[0,0]$. This is of course assuming you didn't have any error situations (infeasibility or unboundedness), which you should handle separately.


\section{Convergence in Law}%
\label{sec:convergence_in_law}

\begin{definition}
    $Y_n$ is \textit{bounded in probability} if for any $\epsilon>0\ \exists K$
    and $n_0(\epsilon)$ such that
    \begin{equation*}
        P(|Y_n| > K) < \epsilon \text{ for all } n > n_0(\epsilon).
    \end{equation*}
\end{definition}

\begin{definition}[Convergence in Law/Distribution]
    $Y_n \convl Y$ if $H_n(x) \to H(x)$ for all continuity points $x$.
\end{definition}

\begin{thm}
    If $Y_n \convl H$ then $Y_n$ is bounded in probability.
\end{thm}

\begin{thm}[Slutsky's Theorem]
    If $Y_n \convl Y$, $A_n \convp a$, and $B_n \convp b$, then
    \begin{equation*}
        A_n + B_n Y_n \convl a + b Y.
    \end{equation*}
\end{thm}

\begin{corollary}
    If $Y_n \convl Y$, $R_n \convp 0$, and $B_n \convp 1$, then
    \begin{equation*}
        Y_n + R_n \convl Y \text{ and } Y_n / B_n \convl Y.
    \end{equation*}
\end{corollary}

\begin{thm}
    If $k_n (Y_n - c) \convl H$ and $k_n \to \infty$ then $Y_n \convp c$.
\end{thm}

\begin{definition}
    $Y_n \convp Y$ if $Y_n - Y \convp 0$.
\end{definition}

\begin{thm}
    $Y_n \convp Y \implies Y_n \convl Y$
\end{thm}


\section{Metric Spaces and Non-Traditional Databases}
\subsection{Overview}
Chávez and Navarro \cite{chavez2005metric} gives an accurate description of what a non-traditional database
is, and which problems it intends to solve. To summarise the main idea of the former paper, traditional databases
can be seen as engines that answer queries based on the premise that for a certain key there is a certain element
with a (possibly unique) relationship. In other words, for a certain query $Q$ with constraints $C$ there is an answer
$A$ which satisfies $Q \times C$. In truth, there is nothing wrong with this approach, but it is really complex and
expensive to execute queries to which we do not desire a exact answer, because it does not exists or because it is
resourse-expensive to compute. For instance, if we have a large dataset of points in space and we want to query
the ``10 points closest to a point $Q$'' (also known as a $k$-nearest-neighbour query) we need to
execute this query for each point in the database computing their individual distances as well, an operation with a cost
that is not known beforehand.

This is an interesting property of multimedia objects which makes the use of exact queries pointless as there are
no two objects with the same parameters -unless they are exact copies-. A ``real'' world example of this situation is plagiarism
detection on multimedia files. Plagiarism\cite{dictionary_plagiarism}
can be hard to detect because in most cases, only a part of the multimedia resembles the original work \cite{citation_needed}.
There are many approaches to this problem, but in music one of the easiest representations is the ``spectrogram''\cite{citation_needed}
which is a visual representation of the different waveforms involved on a certain audio file. Spectrograms allows the extraction of
multiple features from a single audio picece such as scale, BPM (beats per minute), movements, entropy and so on.

The values of these features can be allocated as \emph{vectors} $\mathbb{X}$ on a \emph{metric space} $\mathbb{U}$ formally defined as follows:

For a given cartesian pair $\mathbb{U} = (\mathbb{X}, d)$ where $\mathbb{X}$ is a non-empty set of elements and $d$ is a distance function which
operates over $\mathbb{X}$ in the following manner $d: \mathbb{X}\times\mathbb{X}\rightarrow[0,\infty)$, then it is considered a \emph{metric space}
if it satisfies all the following conditions:

\begin{enumerate}
\item{\textbf{Reflexibility:} $\forall x_1,x_2 \in \mathbb{X}, x_1=x_2 : d(x_1,x_2) = 0$}
\item{\textbf{Positiviness:} $\forall x_1,x_2 \in \mathbb{X} : d(x_1,x_2) \geq 0$}
\item{\textbf{Simmetry:} $\forall x_1,x_2 \in \mathbb{X} :d(x_1,x_2) = d(x_2,x_1)$}
\item{\textbf{Triangular inequality:} $\forall x_1,x_2,x_3 \in \mathbb{X} :d(x_1,x_2) \leq d(x_1,x_3) + d(x_3,x_1)$}
\end{enumerate}

\subsection{Metric Range and Nearest Neighbour Queries}
A typical request is to find all the songs from $\mathbb{X}$ which have at most $15\%$ of similarity with our query song $q$. This
kind of queries are also known as \emph{Metric Range Queries} on which the result is a subset $\mathbb{X}'$of $\mathbb{X}$ which satisfies
$\forall x \in \mathbb{X}'; q, x_1, x_2 \in \mathbb{X} : d(x,d) \leq  0.15 \times max(d(x_1,x_2))$, where $0.15 \times max(d(x_1,x_2))$ represents
the 15\% of the maximum distance between elements in the space.

There is a good reason behind to choose the maximum distance between elements instead of the maximum distance allowed by our space,
because the maximum distance is infinity. Thus, the relative nature of the space does not allow such queries to be performed. In a strict
sense, we can generalise metric range queries as follows:

$\forall x \in \mathbb{X}'; q, x_1, x_2 \in \mathbb{X} : d(x,d) \leq d$

Now, for our subset of waveforms if we want to obtain the 10 songs more similar to a certain one we are talking about
a \emph{Nearest Neighbour query}, on which the dataset is filtered and the result is a subset $\mathbb{X}'$ of $\mathbb{X}$ of fixed size $k$ (in
this case 10) satisfying $\forall x_1, x_2... x_n \in \mathbb{X}, x_1 < x_2 < .. < x_n ; q \in \mathbb{X} : \mathbb{X}' = {x_1, x_2, .., x_{10} }$.
Formally this can be defined as $\forall x_1, x_2... x_n \in \mathbb{X}, x_1 < x_2 < .. < x_n; q \in \mathbb{X} : \mathbb{X}' = {x_1, x_2, .., x_k }$.

\subsection{Dimensionality crux}
All the queries previously explained require an evaluation of every element on the index. Thus any improvement on
the execution time can only achieved by reducing the dataset to be compared, but this ignores the fact that we do not know
how expensive it is to compute the distance between two elements. On high-dimensional spaces, distance computation can be a pain,
depending on the function $d$ involved. This dependency on the space dimension is known as the \emph{dimensionality crux}. There are
several approaches to this problem, such as dimensionaly reduction using techniques like \emph{Self Organizing Maps}, which work
under the premise that the real dimensions of the problem are less than the ones given by the dataset, that it depends on a
prior treatment of the data, and that it requires the asumption of data loss.

%TODO continue checking
So, dimensionality crux can be defined as the tradeoff between the amount of data available for computation (thus, the quality of our results) and
the efficiency of the operations performed on this space.

\subsection{Pivot and Permutant based indices}
Given the dimensionality crux, we know that we need to reduce the dataset to compare in order speed up out queries, so, let's change the problem from
searching similar songs to a geolocation problem, on which we need to find a point closer to another in the world map. If we were to
find Curicó (Chile) without any prior knowledge of where it is, then the only option is to search on every label on the map
in the hopes of finding the one that we desire to mark. This is known as \emph{exhaustive search}, which is pretty much like performing a
linear scan over the elements of $\mathbb{X}$.

Everything would be easier if we have an \emph{index} to search on, a small database that can give us some insight on where
the location belongs in order to reduce the problem size even if doesn't give us the exact position. We do not worry about dimensionality crux
implcations at this point, since we are not quering the exact solution, rather a good aproximation of it. Now, assuming that we ask
our index about the location of Curicó it can suggest how far it is from the major cities on the world (\emph{pivot} cities), this kind of indices
is known as \emph{Pivot-based indices}, on which we don't store all the dimensions of the problem, rather than the distance of the element to certain
\emph{key elements} of $\mathbb{X}$. Then by applying the triangular inequality property of metric spaces we can triangulate a reduced
area on the map in which we know that Curicó is situated.

So, assuming that we have a set $\mathbb{P} \subset \mathbb{X}$ of randomly chosen pivots, for a range query $(q,r)$ on $\mathbb{U}$ we do not need to
compute the distance of every element of $\mathbb{X}$ to our query object $q$ since the triangle inequality follows that
$\forall x \in \mathbb{X} : d(p_i,x) \leq d(p_i, q) + d(q,x)$ and also that $\forall x \in \mathbb{X} : d(p_i,q) \leq d(p_i, x) + d(p_i,q)$ thus
we can approximate $d(q,x)$ for all elements on $\mathbb{X}$ only using the reference data given by the closeness of each element to their pivots.

This approach implies that $|\mathbb{P}|$ indices must be created in order to store the information needed for a pivot approach but since we can store them
as sorted arrays of data we perform an approximate binary search on each one of the indices (which can be later retrieved using incremental sorting
techniques \cite{7416566}), thus the search complexity yields $|\mathbb{P}| \times log_2(|\mathbb{X}|)$ for the pivot discard stage plus linear time to retrieve
all of the results from the resulting sets at the cost of  of extra $|\mathbb{P}|\times|\mathbb{X}|$ space.

Taking into account how space expensive is to generate such indices, another approach is to search for Curicó not using the distances to the major cities
at all and only using the information given by the closeness order to each one of the pivots generated in the earlier stage. In this case, such cities which
acts as pivots are know as \emph{permutants}. Then our approach now is to not store all the indices generated by the pivots, rather to map the entire dataset to another index which condenses this queries
creating a single database in which the \emph{real distance calculations} are replaced by computing the permutation distance between each one of the permutants.

This kind of indices are known as \emph{permutation indices} and work under the premise that computing the permutation distance is much simplier than
to compute the actual distance for every object in $\mathbb{X}$. Such distances can be computed by using \emph{vector absolute distance} forumlas such as
\emph{Spearman's footrule} and \emph{Kendall 's tau}.

\subsection{Correlation coefficients and permutation distance metrics}
One of the most well-known formulas for computing the correlation on two datasets is \emph{Spearman's rho}, which gives an interdependence metric
between two continous random variables. For our case, we expect to have two permutations which represents a ranking of each object (namely, our
query object and a test object in the dataset) so, to measure the similarity between those two permutations is a feasible solution at first glance.
But there are two problems given by both the nature of the data and it is implementation. First \emph{Spearman's rho} formula is intended for continous
random variables, as such we always have intermediate objects between each data in the dataset, which is not our case. Second, it is hard to compute
continous variable on a computer, since we do not enjoy o abritrary precision on our current compute machines.

As such, we use the discrete version of correlation coefficients, which are also known as \emph{absolute distance} between vectors. In our case
we have the following formulas for each one of the before mentioned metrics:

Spearman's Footrule, which measures the total element displacement between the identity permutation:
$$ F(\sigma) = \sum_{i} (i-\sigma(i))$$

Kendall's tau, which measures the total number of pairwise inversions:
$$ K(\sigma) = \sum_{i,j : i>j} (\sigma(i)<\sigma(j))$$

%put formal definition


\section{Analytical solution to~\eqref{equ:noisy_min}}
\label{sec:opt}
The problem in~\eqref{equ:noisy_min} does not need iterative solver but has an analytical solution. Substituting only the second constraint, the objective becomes $p(x)=x^2D^2 + (1-x)^4/h_{\min}^2C$ with $x=\sqrt{\mu} \in [0, 1)$. By setting the gradient of $p(x)$ to 0, we can get a cubic equation whose root $x=\sqrt{\mu_p}$ can be computed in closed form using Vieta's substitution. As $p(x)$ is uni-modal in $[0, 1)$, the optimizer for \eqref{equ:noisy_min} is exactly the maximum of $\mu_p$ and $(\sqrt{h_{\max}/h_{\min} }-1 )^2 / (\sqrt{h_{\max}/h_{\min}}+1)^2$, the right hand-side of the first constraint in~\eqref{equ:noisy_min}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 2}

\begin{enumerate}[label=(\alph*)]
\item Show how to sort a set of $n$ numbers in $\mathcal{O}(n\log n)$ time using only the following binary search tree operations: \textsc{Minimum}, \textsc{Insert} and \textsc{Sucessor}.
You may assume each operation takes $\mathcal{O}(\log n)$ time.

\item Let $b_n$ be the number of all possible binary search trees with $n$ nodes.
Trivially $b_0$ is $1$.
Show that $b_n = \sum_{k=0}^{n-1} b_k b_{n-1-k}$.
\end{enumerate}

\subsection*{Solution}

\begin{enumerate}[label=(\alph*)]
\item The algorithm to sort the set of $n$ numbers is shown in Algorithm \ref{alg1}.

First we can use the insert operation $n$ times to build a binary search tree out of $n$ elements.
This will have $n\mathcal{O}(\log n)$ runtime cost.
Then we once use the minimum operation to obtain and print the element with minimum value.
Algorithm will continue by calling the successor operation $n-1$ times such that all elements are printed.
As one call to minimum and $n-1$ calls to successor operation has been made, the runtime will be $\mathcal{O}(n\log n)$.
Therefore, the entire sorting cost is $2n\mathcal{O}(\log n) = \mathcal{O}(n\log n)$.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\For {$i \leftarrow 1 : n$}
\State \textsc{Insert}($A[i]$)
\EndFor
\State $s \leftarrow$ \textsc{Minimum}($A$)
\State print $s$
\For {$i \leftarrow 2 : n$}
\State $s \leftarrow$ \textsc{Successor}($A, s$)
\State print $s$
\EndFor
\end{algorithmic}
\caption{Sort($A$)}\label{alg1}
\end{algorithm}

\item With $n$ nodes, we can convert the problem of finding $b_n$ possible binary search trees to finding a root for the tree, a left subtree and a right subtree.
Suppose there are $k$ elements in the left subtree where $0 \leq k \leq n-1$.

Let $b_k$ be defined as number of binary search trees that can be constructed with $k$ elements.
As we still have $n-k-1$ elements with which to construct a right subtree, each choice of $k$ will give $b_k b_{n-1-k}$ possible binary search trees.
Thus, total number of binary search trees that can be constructed with $n$ elements will be as shown in Eq. \ref{eq21}.

\begin{equation}
b_n = \sum_{k=0}^{n-1} b_k b_{n-1-k}
\label{eq21}
\end{equation}

\end{enumerate}


\section{Brief primer on tensors}\label{sec:tensor}
The material here is based on \cite{lee-book-2000,dullemond-1991-tensor}.

Let \(V\) be an \(n\)-dimensional vector space over the reals. A {\em covector} on \(V\) is a real-valued linear functional \(\omega:V\rightarrow \Re\). The space of all covectors is itself a real vector space  under pointwise addition and multiplication. It is written as \(V^{\star}\) and called the {\em dual space} to \(V\).

\begin{proposition}[\cite{lee-book-2000} Proposition 4.1] For \(V\) an \(n\)-dimensional vector space and \(E_1, \ldots, E_n\) a basis for \(V\), the covectors \(\epsilon^1, \ldots, \epsilon^n\), defined by:
  \[
  \epsilon^i(E_j) = \delta^i_j = \left\{
  \begin{array}{ll}
    1 & \mbox{if \(i =j\)} \\
    0 & \mbox{if \(i\not=j\)}
    \end{array}\right.
    \]
\noindent form a basis for \(V^{\star}\), called the {\em dual basis} to \((E_i)\).
\end{proposition}
%% TODO: Define canonical isomorphism.
Note that it follows that the dimensionality of \(V^{\star}\) is the same as that of \(V\). It is also the case that \(V^{\star\star}\) is canonically isomorphic to \(V\).

\begin{definition}[(Covariant) Tensor] For $V$ a finite-dimensional real vector space, a {\em covariant tensor} of rank $k$ over $V$ is a real-valued multilinear function of $k$ elements of $V$.
\end{definition}

By convention, a $0$-tensor is just a real number. Define $T^k(V)$ to be the set of all tensors of rank $k$ over $V$. $T^k(V)$ is a vector space under point-wise addition and scalar multiplication. Specifically:
\[
\begin{array}{l}
  (\alpha T)(X_1, \ldots, X_k) = \alpha(T(X_1, \ldots, X_k))\\
  (S+T)(X_1, \ldots, X_k)=S(X_1, \ldots, X_K) + T(X_1, \ldots, X_k)
\end{array}
\]

Given two tensors \(S \in T^k(V), T \in T^l(V)\) (for \(V\) a finite-dimensional real vector space), the map:
\[ S \otimes T \defeq \lambda X_1, \ldots, X_{k+l}.S(X_1,\ldots, X_k)T(X_{k+1}, \ldots, X_{k+l})\]
\noindent is a covariant \((k+l)\)-tensor, called the {\em tensor product} of \(S\) and \(T\). More generally, given any two vector spaces \(A,B\), a canonical tensor product \(A\otimes B\) can be defined on them \cite[P.~175]{lee-book-2000}. it consists of linear combinations of objects of the form \(a\otimes b\) for \(a\in A, b\in B\) which is defined in such a way that \(a\otimes b\) depends bilinearly on \(a\) and \(b\).

\begin{proposition}[\cite{lee-book-2000}, Proposition 8.2] Let \(V\) be an \(n\)-dimensional vector space with basis \((E_i)\), and let \(V^{\star}\) have basis \(\epsilon^i\). The set of all \(k\)-tensors of the form \(\epsilon^{i_1}\otimes \ldots \otimes\epsilon^{i_k}\) for \(1 \leq i_1, \ldots, i_k \leq n\) is a basis for \(T^k(V)\).
\end{proposition}

Thus the dimension of \(T^k(V)\) is \(n^k\). We can represent such a tensor extensionally, with a value $r\in \Re$ for each of the \(n^k\) basis vectors \(\epsilon_{i_1} \otimes \ldots \otimes \epsilon_{i_k}\), for \(1 \leq i_1, \ldots, i_k \leq n\). Note while it is tempting to consider such a collection of numbers as a \(k\)-dimensional array of numbers, care must be taken because the operation to be performed on this array are described by tensor contraction rather than matrix multiplication, as we shall see below.

%The output of the tensor for any value \(v_1\otimes \ldots \otimes v_k\) is obtained by an appropriate linear combination of the values on the basis vectors.

More abstractly:
\begin{proposition}[\cite{lee-book-2000}, Corollary 8.5]\label{prop:tensors-are-tensor-products} For \(V\) an \(n\)-dimensional real vector space, the space \(T^k(V)\) of covariant \(k\)-tensors on \(V\) is canonically isomorphic to the \(k\)-fold tensor product \(V^{\star}\otimes \ldots \otimes V^{\star}\).
\end{proposition}

%%If the transformation matrix of an index is the inverse matrix of the basis transformation, then the index is called contravariant and is traditionally denoted with an upper index (superscript). If the transformation matrix of an index is the basis transformation itself, then the index is called covariant and is denoted with a lower index (subscript).

This permits us to generalize the notion of tensors to {\em mixed rank} tensors. For \(V\) an \(n\)-dimensional vector space, the space of {\em contravariant tensors} of rank \(k\) is defined to be:
\[
T_k(V) = V \otimes \ldots \otimes V \]
(with \(k\) copies). Because of the isomorphism between \(V\) and \(V^{\star\star}\), and Proposition~\ref{prop:tensors-are-tensor-products}, an element of \(T_k(V)\) can be identified with a multilinear function in \(V^{\star} \times \ldots \times V^{\star} \rightarrow \Re$. This allows us to define, for any \(k, l \in \mathbb{N}\) the space of {\em mixed tensors} as:
\[T^k_l(V) = V^{\star}\otimes \ldots V^{\star}\otimes V\otimes \ldots \otimes V\]
\noindent where the first product is taken \(k\) times and the second \(l\) times.  Thus \(T^k_l(V)\), the space of \((k,l)\)-tensors, is the space of real-valued multilinear functions of \(k\) vectors and \(l\) covectors.

Mixed-mode tensors  are written with upper indices for its contra-variant dimensions, and lower indices for its covariant dimensions. For instance, a matrix \(M\) is a \((1,1)\)-tensor, and we can write \(M_i^j\) to make its indices explicit. This is the same as \(M_j^k\) -- the identity of indices does not matter. Also, basis vectors will be written with a lower index, and components of a vector with respect to this basis with an upper index.
\subsection{Tensor Contraction}\label{sec:summation-convention}
In the following we shall adopt the {\em Einstein summation convention}:
\begin{quotation}
  If the same index name appears twice in any term, once as an upper index and once as a lower index, that term is understood to be summed over all possible values of that index, generally from \(1\) to the dimension of the space in question.
\end{quotation}
Thus for instance we can rewrite:
\[
\begin{array}{lcl}
  \Sigma_{\nu=1}^n A_{\mu\nu}v_{\nu} & \rightarrow &   A_{\mu\nu}v_{\nu}\\
  \Sigma_{\beta=1}^n \Sigma_{\gamma=1}^n A _{\alpha \beta}B_{\beta \gamma}C_{\gamma \delta} & \rightarrow & A_{\alpha \beta}B_{\beta \gamma}C_{\gamma \delta}
\end{array}
\]

The {\em contraction} of a tensor is obtained by setting unlike indices equal, thus indicating a summation per the convention above. The result of contracting a \((k,l)\)-tensor  is a \((k-1,l-1)\) tensor.

The contraction operation is invariant under coordinate changes.

\subsection{Working with tensors}

Tensors can be ``partially'' evaluated. For \(T\) a tensor of rank \(k\) representing a predicate \(p\), and \(a\) a vector representing the value of argument \(i\), \(T^{1\ldots k}a_i\) represents the predicate \(\lambda x_1, \ldots, x_{i-1},x_{i+1},\ldots x_k. p(x_1, \ldots, x_{i-1}, a, x_{i+1}, \ldots, x_k)\). It can be thought of as {\em contracting} \(T\) and \(a\) on index \(i\).

\begin{example}\label{ex:tc-1}
  Consider the tensor representation of the boolean polynomial \(p(x,y) = x(1-y)\). Because \(x,y\in \{0,1\}\) with the two values independent of each other, we will embed them in the two dimensional vector space \(U=\{0,1\}^2\), with ``one hot'' basis vectors \(\epsilon_1=(1,0)\) and \(\epsilon_2=(0,1)\) representing \(0\) and \(1\) respectively. Now the tensor \(T^{1,2}\) representing the binary predicate \(\lambda x,y. p(x,y)\) is represented by the table:
  \[
  \begin{array}{l}
    T(\epsilon_1,\epsilon_1)=0\\
    T(\epsilon_1,\epsilon_2)=0\\
    T(\epsilon_2,\epsilon_1)=1\\
    T(\epsilon_2,\epsilon_2)=0
  \end{array}
  \]
  We can use this representation to evaluate the predicate at different points through tensor contraction. For instance, \(p(0,1)\) is given, for \(a=\epsilon_1,b=\epsilon_2\) by \(T^{1,2}a_1 b_2\), which through the Einstein convention expands out to:\footnote{Recall that for \(x\) a vector \(x^j\) represents its \(j\)th coordinate.}
  \[\begin{array}{ll}
  T^{1,2}a_1 b_2 &= \Sigma_{j=1}^2 \Sigma_{i=1}^2 T(\epsilon_i,\epsilon_j)a_1^ib_2^j \\
  &= T(\epsilon_1,\epsilon_1)*1*0 + T(\epsilon_1,\epsilon_2)*1*1 +
  T(\epsilon_2,\epsilon_1)*0*0 + T(\epsilon_2,\epsilon_2)*0*1 \\
  & =T(\epsilon_1,\epsilon_2)\\
  & = 0
  \end{array}
  \]
\end{example}

More generally, we can compute tensor contraction symbolically. Let \(x,y\) be  unknown vectors in \(U\), and let \(S\) be a rank-2 tensor over \(U\). Then if \(S\) represents the predicate \(q\), the predication \(q(x,y)\) is represented by \(S^{1,2}x_1 y_2\) which expands out to:
\[\begin{array}{ll}
  [S(x,y)]&\defeq  S^{1,2}x_1 y_2\\
  & = \Sigma_{j=1}^2 \Sigma_{i=1}^2 S(\epsilon_i,\epsilon_j)x_1^iy_2^j \\
  & =S(\epsilon_1,\epsilon_1)x^1y^1 + S(\epsilon_1,\epsilon_2)x^1y^2 +
      S(\epsilon_2,\epsilon_1)x^2y^1 + S(\epsilon_2,\epsilon_2)x^2y^2 \\
  \end{array}
  \]

  \begin{example}[Example~\ref{ex:tc-1} contd]
    Taking the value of \(S\) above to be \(T\), we get:
    \[\begin{array}{ll}
        [T(x,y)] & = x^2y^1\\
         & = [x=1][y=0]
    \end{array}
    \]
  \end{example}

The representation of an arbitrary $k$-are predicate is similar, a tensor \(T^k\) represented by \(n^k\) numbers, for \(U=\{0,1\}^n\).


\subsection{Features}
\label{sec:methods_features}

In this section, all the features of the model are introduced. We will present
univariate and bivariate analysis of the features as well as different
techniques to transform them, in particular fractional differentiation.

When working with inference models, features should be stationary. Common
procedures to features like prices involve integer differentiation to end up
working with price returns instead. The latter would remove entirely the price
series memory which is required by the model to effectively predict the output.
Other methods involve applying power transformations such as logarithms, square
roots or box-cox transformations. We are not interested in those for price series.
They will drastically affect scales and might collapse movements around the trend
while preserving the trend.

In \cite{frac_diff_paper} the fractional differentiation method was introduced,
and Lopez de Prado takes the method and explains the model in chapter 5 of
\cite{lopez_de_prado}. I will present the mathematical model and explain the
algorithm in what follows.

Let $B$ be a backshift operator, i.e. delay operator, to be applied to a matrix
of real valued features $X_{t}$ such that $B^k X_t = X_{t-k}$. Also, we can
express the positive integer powers of a binomial as $(x+y)^n = \sum_{k=0}^n {n \choose k} x^k y^{n-k}$.
When considering real valued exponents, combinatorial number ${n \choose k}$
becomes (after substitution of $n$ an integer by $d$ a real number) ${d \choose k} = \frac{d (d-1) ... (d-k+1)}{k!}$
which coincides with the integer formula. Thus,

\begin{equation}
  \label{eqn:binomial_expantion_frac_power}
  (x+y)^d = \sum_{k=0}^{\infty} {d \choose k} x^k y^{d-k}
\end{equation}

Equation \ref{eqn:binomial_expantion_frac_power} presents an infinite series, a
key difference with respect to the integer counterpart. If we replace $x$ by $1$
and $y$ by $-B$, the backshift operator, one can write from \ref{eqn:binomial_expantion_frac_power}:

\[(1-B)^d = \sum_{k=0}^{\infty} {d \choose k} (-B)^k \]

\[(1-B)^d = \sum_{k=0}^{\infty} \frac{\prod_{i=0}^{k-1}(d-i)}{k!} (-B)^k \]

\begin{equation}
  \label{eqn:binomial_expantion_diff_operator}
  (1-B)^d = 1 - dB + \frac{d(d-1)}{2!}B^2 - \frac{d(d-1)(d-2)}{3!}B^3 + ...
\end{equation}

Equation \ref{eqn:binomial_expantion_diff_operator} presents the foundation of
the fractional differentiation method. If one could find the value of $d$ such
that a series $X$ gets differentiated and becomes stationary while preserving
as much memory as possible a later model would be able to exploit that memory to
predict the output. In particular, when the value of $d$ ends up being less than 1.
Equation \ref{eqn:binomial_expantion_diff_operator} also
presents a problem, it is a infinite series when $d$ is noninteger which makes
the operation to be non exact for the general case due to the impossibility of
applying infinite multiplications and sums. Lopez de Prado proposes a solution
for both issues.

A fractionally differentiated series $X$ can be expressed for a given $d$ as:

\[ X_t^d = \sum_{k=0}^{\infty} w_k X_{t-k} \]

The vector $w$ of weights in the above equation follows:

\begin{equation}
  \label{eqn:ffd_weights}
	w = {1, -d, \frac{d(d-1)}{2!}, -\frac{d(d-1)(d-2)}{3!}, ..., (-1)^k \prod_{i=0}^{k-1} \frac{(d-i)}{k!}}
\end{equation}

One can derive by inspection of equation \ref{eqn:ffd_weights} a generative and
recursive expression for each item in the series:

\begin{equation}
  \label{eqn:ffd_weights_generative}
  w_k = -w_{k-1} \frac{d-k+1}{k}
\end{equation}

Equation \ref{eqn:ffd_weights_generative} is easy to implement as a programming
function which is ideal for this application. It is also interesting to evaluate
the tendency of $w_k$ as $k$ tends to infinite, see figure \ref{fig:w_k_vs_k}.

\begin{figure}[!htb]
    \centering
    \includegraphics[width=\textwidth]{methods/images/weights_vs_k.png}
    \caption{Weight values vs. $k$ for different $d$ values.}
    \label{fig:w_k_vs_k}
\end{figure}

As it can be seen in figure \ref{fig:w_k_vs_k}, coefficients tend to zero as $k$
increases. It can also be proved the convergence of coefficients $w_k$. Let's
analyze the following when $k > d$ and $w_{k-1} \ne 0$:

\[|\frac{w_k}{w_{k-1}}| = |\frac{d-k+1}{k}| < 1 \]

what makes $|w_k| < |w_{k-1}|$ leading to $\lim_{k \to \infty} w_k = 0$.

When implementing fractional differentiation on a real time series, one has two
options:

\begin{enumerate}
  \item Adjust the length of $w_k$ vector by weight loss with a certain
        threshold.
  \item Work with a fixed number of coefficients.
\end{enumerate}

Option 1 requires the operation to compute the size of $w_k$ vector to account
for weight loss given a certain threshold. Weight loss cam be computed:

\begin{equation}
  \lambda_l = \frac{\sum_{j=T-l}^{T}|w_j|}{\sum_{i=0}^{T-1}|w_i|}
\end{equation}

where:

\begin{itemize}
  \item $T$ is the length of the time series,
  \item $l$ is the index of the sample from the end where the fractional
        differentiation occurs.
  \item $\lambda_l$ the weight loss at $l$ index
\end{itemize}

One should discard all samples whose $\lambda_l < \tau$ and $\tau$ is the
threshold. As $d \to 0$, the energy of the weights decreases leading to more
weight loss and more discarded samples.

Option 2 comes with the simplicity of having always the same vector of
coefficients $w_k$ such that $|w_k| > \tau$ and $\tau$ is a user defined
threshold. It comes with the advantage of having no drift as option 1 and just
needs to drop $l$ samples at the beginning, being $l$ the value of $l$ that
makes $w_k$ less or equal to $\tau$. In this thesis, option 2 is used.

So far, how to compute the weights vector was explained. Now, we just need to
address the value of $d$. $X_t$ might be stationary already which leads to
$d^* = 0$ with $d^*$ the value of $d$ that preserves most memory making the
time series stationary. When $X_t$ has a \emph{unit root} (see chapter 15 of \cite{time_series_analysis}),
$0 \le d^* \leq 1$. And when $X_t$ exhibits an explosive (bubble) behavior,
$d^* > 1$. Unit roots can be tested with Dickey Fuller hypothesis test
(see chapter 17 of \cite{time_series_analysis}). The null hypothesis of the test claims the series has a unit root.
After determining a certain confidence level one can derive an \emph{optimum}
$d^*$ by:

\begin{enumerate}
  \item Define a vector of $d$ values in range of 0 to 1.
  \item For each value of $d$:
  \begin{enumerate}
    \item Fractionally differentiate $X_t$ with $d$ given a certain amount of
          weights. Obtain $X_t^d$.
    \item Compute the Dickey Fuller statistic, ADF, for $X_t^d$.
    \item Compute the p-value of the test.
  \end{enumerate}
  \item Choose $d^*$ that yields the maximum p-value between all p-values that
        are less or equal to the confidence level.
\end{enumerate}

This process has two flaws:

\begin{itemize}
  \item It is computationally time complex. Each time we apply the fractional
        differentiation, we are processing a $O(n^2)$ algorithm. Computing the
        ADF of the differentiated series requires a differentiation and model
        estimation which yields at least another $O(n^2)$ process. Finally, we
        iterate through a vector of $d$ adding another dimension.
  \item \emph{Optimum} $d$ is subject to the granularity of the $d$ vector of
        samples. The smaller the step, the more information one could preserve
        in the final time series, but the more iterations are required which
        impacts directly in the aforementioned item.
\end{itemize}

Regardless, all features that expose non-stationary characteristics could be
transformed and stabilized while preserving memory.


\section{Principle of Strong Induction}

Here is the Principle of Strong Induction:

\begin{theorem}
Suppose $P(n)$ is a proposition, and that the following statements hold:
\begin{itemize}
    \item $P(0)$ is true,
    \item If $P(i)$ is true for all $0 \leq i \leq n$, then we can prove that $P(n + 1)$ is true.
\end{itemize}
Then $P(n)$ holds for all $n \in \mathbb{N}$.
\end{theorem}
\begin{proof}
In fact, this theorem actually directly follows from the Principle of Weak Induction. To see this clearly, define the proposition $Q(n)$ to be equivalent to the claim that $P(i)$ is true for $0 \leq i \leq n$. Then by the hypotheses:
\begin{itemize}
	\item $Q(0)$ is true,
	\item If $Q(n)$ is true, then $P(n + 1)$ is true. This essentially means that if $Q(n)$ is true, then $P(i)$ is true for all $0 \leq i \leq n + 1$, implying that $Q(n + 1)$ is true.
\end{itemize}

So by the Principle of Weak Induction $Q(n)$ is true for all natural numbers $n$, which implies that $P(n)$ is true for all natural numbers $n$ as well.
\end{proof}

%note: define inductive hypothesis in weak_ind.tex

The Principle of Strong Induction is seemingly ``stronger'' than the Principle of Weak Induction. The reason behind this is because of the nature of the theorem's ``induction hypothesis'', which assumes the truth of $P(i)$ for all $i$ up to a value $n$. We may apply this principle to prove certain statements where in order to complete the inductive step assumptions about more than just the previous steps are needed. Here is a particular example of this.

\section{Exercises}
\begin{enumerate}
    \item \input{Ch5/2_ex/problem_1}
    \item \input{Ch5/2_ex/problem_2}
    \item \input{Ch5/2_ex/problem_3}
    \item \input{Ch5/2_ex/problem_4}
    \item \input{Ch5/2_ex/problem_5}
\end{enumerate}


\subsection{Lines}
\noindent
A straight line is probably the simplest 3D VVF. We can form any straight line using a point that the line passes through and the direction vector of the line.
Letting $P$ be the point and $\vec{v}$ be the direction, a straight line has the form $\vec{r}(t) = \vec{P}+t\vec{v}$, where $\vec{P}$ is the vector with components the same as $P$.
The function's output starts at $\vec{P}$ when $t=0$ and moves in the direction of $\vec{v}$ as $t$ increases.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.45\textwidth]{./Images/vectorValuedFunctions/VectorLine.png}
	\caption{A line can be formed by a single point and a direction vector or two points.}
\end{figure}

\noindent
A line can also be formed using two points.
To find the equation of the line in this case, let $\vec{v}$ be the vector connecting the two points, and let $\vec{P}$ be a vector point from the origin to one of the two points.
We now have an origin point and direction and can write our function as
\begin{equation*}
	\vec{r}(t) = \vec{P_0} + t\left(\vec{P_1} - \vec{P_0}\right),
\end{equation*}
where $P_0$ and $P_1$ are the two points on the line.


\section*{Exercises}

\begin{ex}
  Suppose $V$ and $W$ are vector spaces, $T,S:V\to W$ and $R,Q:W\to W$
  are linear transformations, and $k$ is a scalar. Which of the
  following equalities are valid?
  \begin{enumerate}
  \item $R\circ (T+kS) = R\circ T + kR \circ S$.
  \item $(R+Q)\circ (R+Q) = R\circ R + 2R\circ Q + Q\circ Q$.
  \item $(R+Q)\circ (T+S) = R\circ T + R\circ S + Q\circ T + Q\circ S$.
  \item If $T$, $S$, and $T+S$ are invertible, then $(T+S)^{-1} =
    T^{-1} + S^{-1}$.
  \end{enumerate}
  \begin{sol}
    Yes, no, yes, no.
  \end{sol}
\end{ex}

\begin{ex}
  Finish the proof of
  Proposition~\ref{prop:addition-linear-transformations}, i.e., prove
  that if $T:V\to W$ is a linear transformation and $k$ a scalar, then
  $kT:V\to W$ is a linear transformation.
\end{ex}