\section{Recursion Equations}

\begin{frame}{Recursion Equations}{Recursion Equation}
  \textbf{Recursion equation:}
  \begin{itemize}
    \item
      Runtime description for recursive functions:
  \end{itemize}
  \vspace{2em}
  \begin{displaymath}
    {\color{MainA}T(n)} =
    \begin{cases}
      \hfill
      \smash{
        \overbrace{f_0(n)}^\text{%
          \footnotesize%
          trivial case for $n_0$
        }
      }
      \vphantom{f_0(n)} & n = n_0\\[0.5em]
      \smash{
        \underbrace{
          a \cdot {\color{MainA}T\left(\frac{n}{b}\right)}
        }_\text{
          \footnotesize%
          \begin{tabular}{c}
            \text{solving of {\color{MainA}$a$}}\\
            \text{subproblems}\\
            \text{with reduced}\\
            \text{input size {\color{MainA}$\frac{n}{b}$}}
          \end{tabular}
        }
      } \hspace{0.5em} + \hspace{1.5em}
      \smash{
        \underbrace{
          f(n)
          \vphantom{\left(\frac{n}{b}\right)}
        }_\text{
          \makebox[0pt][c]{
            \footnotesize%
            \begin{tabular}{c}
              slicing and\\
              splicing of\\
              subsolutions
            \end{tabular}
          }
        }
      }
      \vphantom{\left(\frac{n}{b}\right)}
      &n > n_0
    \end{cases}
  \end{displaymath}
  \vspace{4em}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Recursion Equation}
  \textbf{Recursion equation:}
  \begin{itemize}
    \item<2->
    Runtime descripion for recursive functions:
    \begin{displaymath}
    {\color{MainA}T(n)} = \begin{cases}
      f_0(n) & n = n_0\\
      a \cdot {\color{MainA}T\left(\frac{n}{b}\right)} + f(n) & n > n_0
    \end{cases}
    \end{displaymath}
    \item<3->
      {\color{MainA}$n_0$} is usually small, {\color{MainA}$f_0(n_0) \; \in \; \Theta(1)$}
    \item<4->
      Usually, {\color{MainA}$a > 1$} and {\color{MainA}$b > 1$}
    \item<5->
      Dependent on the strategy of solving {\color{MainA}$T(n)$}
      {\color{MainA}$f_0$} is ignored
    \item<6->
      {\color{MainA}$T(n)$} is only defined for integers of
      {\color{MainA}$\frac{n}{b}$}, which is often ignored in benefit of
      a simpler solution
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\subsection{Substitution Method}

\begin{frame}{Recursion Equations}{Substitution Method}
  \textbf{Substitution Method:}
  \begin{itemize}
    \item<2->
      Guess the solution and prove it with induction
    \item<3->
      Example:        \color{MainA}
      \begin{displaymath}
        T(n) = \begin{cases}
          \hfill 1 & n = 1\\
          2 \cdot T\left(\frac{n}{2}\right) + n & n > 1
        \end{cases}
      \end{displaymath}        \color{black}
    \item<4->
      Assumption:  {\color{MainA}$T(n) = n + n \cdot \log_2 n$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Substitution Method}
  \textbf{Induction:}
  \begin{itemize}
    \item<2->
      Induction basis (for {\color{MainA}$n = 1$}):
      ${\color{MainA}T(1)} = 1 + 1 \cdot \log_2 1 = 1$
    \item<3->
      Induction step (from {\color{MainA}$\frac{n}{2}$}
      to {\color{MainA}$n$}):
      \begin{align*}
        \onslide<4->{{\color{MainA}T(n)} & =
          2 \cdot {\color{MainA}T\left(\dfrac{n}{2}\right)} + n\\}
        \onslide<5->{{} & \stackrel{IA}{=} 2 \cdot \left(
            \dfrac{n}{2} + \dfrac{n}{2} \cdot \log_2 \dfrac{n}{2}
          \right) + n \\}
        \onslide<6->{{} & = 2 \cdot \left(
            \dfrac{n}{2} + \dfrac{n}{2} \cdot \left(\log_2 n - 1\right)
          \right) + n}\\
        \onslide<7->{{} & = n + n \, \log_2 n - n + n\\}
        \onslide<8->{{} & = n + n \, \log_2 n}
      \end{align*}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Substitution Method}
  \textbf{Substitution Method:}
  \begin{itemize}
    \item<2->
      Alternative assumption
    \item<3->
      Example:
      \begin{displaymath}
        \color{MainA}
        T(n) = \begin{cases}
          \hfill 1 & n = 1\\
          2 \cdot T\left(\frac{n}{2}\right) + n & n > 0
        \end{cases}
      \end{displaymath}
    \item<4->
      Assumption:
      {\color{MainA}$T(n) \, \in \, O(n \, \log n)$}
    \item<5->
      Solution: Find {\color{MainA}$c > 0$} with
      {\color{MainA}$T(n) \leq c \cdot n \, \log_2 n$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Substitution Method}
  \textbf{Induction:}
  \begin{itemize}
    \item<2->
      Solution: Find {\color{MainA}$c > 0$} with
      {\color{MainA}$T(n) \leq c \cdot n \, \log_2 n$}
    \item<3->
      Induction step (from {\color{MainA}$\frac{n}{2}$} to
      {\color{MainA}$n$}):
      \begin{align*}
        \onslide<4->{{\color{MainA}T(n)} & =
          2 \cdot {\color{MainA}T\left(\dfrac{n}{2}\right)} + n\\}
        \onslide<5->{{} & \leq 2 \cdot \left(
          c \cdot \dfrac{n}{2} \, \log_2 \dfrac{n}{2}\right) + n\\}
        \onslide<6->{{} & = c \cdot n \, \log_2 n - c \cdot n \, \log_2 2 + n\\}
        \onslide<7->{{} & = c \cdot n \, \log_2 n - c \cdot n + n\\}
        \onslide<8->{{} & \leq c \cdot n \, \log_2 n,
          \hspace{0.5em} {\color{MainA}c \geq 1}}
      \end{align*}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\subsection{Recursion Tree Method}

\begin{frame}{Recursion Equations}{Recursion Tree Method}
  \textbf{Recursion tree method:}
  \begin{itemize}
    \item<2->
      Can be used to make assumptions about the runtime
    \item<3->
      Example:\\
      \begin{displaymath}
        {\color{MainA}T(n)}
        = 3 \cdot {\color{MainA}T\left(\dfrac{n}{4}\right)} + \Theta(n^2)
        \leq 3 \cdot {\color{MainA}T\left(\dfrac{n}{4}\right)} + c \cdot
        n^2
      \end{displaymath}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Recursion Tree Method}
  \begin{figure}
    \begin{adjustbox}{width=\linewidth}
      \input{Images/RecursionEquations/Example_RecursionTree.tikz}
    \end{adjustbox}
    \caption{recursion tree of example}
    \label{fig:recursion_equations:example_recursion_tree}
  \end{figure}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Recursion Tree Method}
  \begin{figure}
    \begin{adjustbox}{width=\linewidth}
      \input{Images/RecursionEquations/Example_RecursionTree_Levels.tikz}
    \end{adjustbox}
    \caption{levels of the recursion tree}
    \label{fig:recursion_equations:example_recursion_tree_layer}
  \end{figure}
\end{frame}
%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Recursion Tree Method Costs}
  \textbf{Costs of connecting the partial solutions:}\\
  \hspace{1.5em}(excludes the last layer)
  \begin{itemize}
    \item<2->
      Size of partial problems on {\color{MainA}level $i$}:
      \begin{math}
        \color{MainA}{
          \mathrm{s}_i(n) = \left(\frac{1}{4}\right)^i \cdot n
        }
      \end{math}
    \item<3->
      Costs of partial problems on {\color{MainA}level $i$}:
      \begin{displaymath}
        \color{MainA}{
          T_{i_p}(n) =
          c \cdot \left(\left(\tfrac{1}{4}\right)^i \cdot n\right)^2
        }
      \end{displaymath}
    \item<4->
      Number of partial problems on {\color{MainA}level $i$}:
      {\color{MainA}$n_{i} = 3^{i}$}
    \item<5->
      Costs on {\color{MainA}level $i$}:
        \color{MainA}
      \begin{displaymath}
        T_i(n) = 3^{i} \cdot c \cdot \left(
          \left(\tfrac{1}{4}\right)^{i} \cdot n
        \right)^2
        = \left(\tfrac{3}{16}\right)^{i} \cdot c \cdot n^2
      \end{displaymath}
        \color{black}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Recursion Tree Method Costs}
  \textbf{Costs of solving partial solutions:} (only the last layer)
  \begin{itemize}
    \item<2->
      Size of partial problems on the {\color{MainA}last level}:
      {\color{MainA}$\mathrm{s}_{i+1}(n) = 1$}
    \item<3->
      Costs of partial problem on the {\color{MainA}last level}:
      {\color{MainA}$T_{i+1_p}(n) = d$}
    \item<4->
      With this the depth of the tree is:
      \begin{displaymath}
        \left(\tfrac{1}{4}\right)^i \cdot n = 1
        \hspace{1.5em}\Rightarrow n = 4^i
        \hspace{1.5em}\Rightarrow\color{MainA} i = \log_4 n
      \end{displaymath}
      \vspace{-1em}
    \item<5->
      Number of partial problems on the {\color{MainA}last level}:
      \begin{displaymath}
        \color{MainA}{n_{i+1} = 3^{\log_4 n}    \onslide<6-> \quad=
          n^{\log_4 3}}  \quad \leftarrow \text{next slide}
      \end{displaymath}
    \item<7->
      Costs on the {\color{MainA}last level}:
      {\color{MainA}$T_{i+1}(n) = d \cdot n^{\log_4 3}$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Fun with logarithm}{Logarithm}
  \begin{itemize}
  \item
    Transforming $\color{MainA}3^{\log_4 n}$ using general log rules
    {\color{MainA}
    \begin{align*}
      \log_4 n & = \log_4 \left(3^{\log_3 n} \right) &&
      \qquad \text{\color{black}using } n= 3^{\log_3 n}\\
      \onslide<2->{& = {\log_3 n \cdot \log_4 3} &&
        \qquad \text{\color{black}using } \log a^b = b \cdot \log a}
    \end{align*}}
  \item<3->
    This proves the general log rule
    {\color{MainA}$\log_b c = \log_a c \cdot \log_b a$}
  \item<4->
    Now the whole expression:
\color{MainA}
    \begin{align*}
      3^{\log_4 n} & = 3^{\log_3 n \cdot \log_4 3} &&
      \qquad \text{\color{black}using reformulation above}\\
      \onslide<5->{& = {\left(3^{\log_3 n}\right)}^{\log_4 3} &&
        \qquad \text{\color{black}using }x^{a \cdot b} = (x^a)^b}\\
      \onslide<6->{& = n^{\log_4 3}}
    \end{align*}
  \item<7->
    {\color{gray}This term will recur in the master theorem}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Total costs}
  \textbf{Total costs:}
  \begin{itemize}
    \item<2->
      Costs of {\color{MainA}level i}:
      $T_i(n) = \left(\frac{3}{16}\right)^i \cdot c \cdot n^2$
    \item<3->
      Costs of {\color{MainA}last level}:
      $T_{i+1}(n) = d \cdot n^{\log_4 3}$
  \end{itemize}
  \vspace{0.5em}
  \onslide<4->
  \begin{displaymath}
    \mathclap{T(n) =}\underbrace{
      \sum\limits_{i = 0}^{(\log_4 n) - 1} \left(\tfrac{3}{16}\right)^i
      \mathrlap{\cdot c \cdot n^2}
    }_{
      \text{
        \footnotesize
        \begin{tabular}{c}
          geometric series,\\
          constant\\
          \color{gray}
          $\left(\begin{array}{c}
            \text{even with}\\
            \text{infinite elements}
          \end{array}\right)$
        \end{tabular}
      }
    } \hspace{1.25em} + \hspace{0.5em} \underbrace{
      d \cdot n^{\log_4 3}
      \vphantom{\sum\limits_{i = 0}^{(\log_4 n) - 1}}
    }_{
      \text{
        \makebox[0pt][c]{
          \footnotesize
          \begin{tabular}{c}
            {\color{MainA}$\log_4 3 < 1$},\\
            grows a lot\\
            slower than {\color{MainA}$n^2$}
          \end{tabular}
        }
      }
    } ~ \in \mathcal{O}(n^2)
  \end{displaymath}
  \begin{itemize}
    \item<5->
      Here: The costs of connecting the partial problems dominate
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Geometric Series}
  \begin{itemize}
    \item
      \textbf{Geometric progression:}\\
      Quotient of two neighboring sequence parts is constant
      \begin{displaymath}
        2^0, 2^1, 2^2, \ldots, 2^k
      \end{displaymath}
    \item<2->
      \textbf{Geometric series:}\\
      The series (cumulative sum) of a geometric sequence\\
    \item<3->
      For $\mid q \mid < 1$:
      \begin{displaymath}
        \sum\limits^{\infty}_{k=0} a_0 \cdot q^k = \dfrac{a_0}{1 - q}
        \hspace{1.5em}\Rightarrow\text{constant}
      \end{displaymath}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Proof of $O(n^2)$}
  \textbf{Proof of $\mathcal{O}(n^2)$:}
  \begin{itemize}
    \item<2->
      We know:
      \begin{align*}
        T(n) &= 3T\left(\frac{n}{4}\right) + \Theta(n^2)\\
        {} &\leq 3T \left(\frac{n}{4}\right) + c \cdot n^2
      \end{align*}
    \item<3->
      Assumption:
      {\color{MainA}$T(n) \, \in \, \mathcal{O}(n^2)$},
      so there exists a {\color{MainA}$k > 0$} with
      \begin{displaymath}
        \color{MainA}
        T(n) \leq k \cdot n^2
      \end{displaymath}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Proof of $O(n^2)$}
  \textbf{Proof of $\mathcal{O}(n^2)$:}
  \begin{itemize}
    \item<2->
      Presumption:
      {\color{MainA}$T(n) \, \in \, \mathcal{O}(n^2)$},
      so there exists a {\color{MainA}$k > 0$} with
      \begin{displaymath}
        \color{MainA}{T(n) < k \cdot n^2}
      \end{displaymath}
    \item<3->
      Substitution method:
      \begin{align*}
        \onslide<4->{T(n) & \leq 3 \cdot T \left( \frac{n}{4}\right)  + c \cdot n^2\\}
        \onslide<5->{{} & \leq 3 \, k \cdot \left( \frac{n}{4}\right)^2  + c \cdot n^2\\}
        \onslide<6->{{} & = \frac{3}{16} \, k \cdot n^2  + c \cdot n^2\\}
        \onslide<7->{{} & \leq k \cdot n^2
        \hspace{6em}\text{for } k \geq \frac{16}{13} \, c}
      \end{align*}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\subsection{Master theorem}

\begin{frame}{Recursion Equations}{Master theorem}
  \textbf{Master theorem:}
  \begin{itemize}
    \item<2->
      Solution approach for a recursion equation of the form:
      \begin{displaymath}
        \color{MainA}{
          T(n) = a \cdot T\left(\frac{n}{b}\right) + f(n),%
          \hspace{1.5em}a \geq 1, b > 1
        }
      \end{displaymath}
    \item<3->
      {\color{MainA}$T(n)$} is the runtime of an algorithm $\ldots$
      \begin{itemize}
        \item<4->
          $\ldots$ which divides a {\color{MainA}problem of size $n$}
          in {\color{MainA}$a$ partial problems}
        \item<5->
          $\ldots$ which solves each partial problem recursively\newline
          \hphantom{$\ldots$} with a
          {\color{MainA}runtime of $T\left(\tfrac{n}{b}\right)$}
        \item<6->
          $\ldots$ which takes {\color{MainA}$f(n)$} steps to
          merge all partial solutions
      \end{itemize}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\subsubsection{Master theorem (Simple Form)}

\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
  \textbf{Master theorem:}
  \begin{itemize}
    \item<2->
      In the examples we have seen that $\ldots$
      \begin{itemize}
        \item<3->
          Either the runtime of {\color{MainA}connecting the solutions}
          dominates
        \item<4->
          Or the runtime of {\color{MainA}solving the problems} dominates
        \item<5->
          Or both have {\color{MainA}equal influence on runtime}
      \end{itemize}
    \item<6->
      \textbf{Simple form:}
      Special case with runtime of connecting the solutions
      {\color{MainA}$f(n) \in O(n)$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
  \textbf{Simple form:}
  \onslide<2->
  \begin{displaymath}
    \color{MainA}{
      T(n) = a \cdot T\left(\frac{n}{b}\right) +
      \underbrace{c \cdot n}_{
        \text{
          \clap{
            \footnotesize\color{black}
            \begin{tabular}{c}
              Is any $f(n)$\\
              in general form
            \end{tabular}
          }
        }
      },
      \hspace{1.5em}a \geq 1, b > 1, c > 0
    }
  \end{displaymath}
  \begin{itemize}
     \item<3->
       This yields a runtime of:
  \end{itemize}
  \onslide<4->
  \vspace{2em}
  \begin{displaymath}
    T(n) = \begin{cases}
      \Theta({\color{MainA}
        \smash{\overbrace{n^{\log_b a}}^{
          \text{\clap{Number of leaves}}
        }}
      }) & \text{if } a > b\\
      \Theta(n \, \log n) & \text{if } a = b\\
      \Theta(n) & \text{if } a < b
    \end{cases}
  \end{displaymath}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
  \begin{figure}[!h]
    \begin{adjustbox}{width=\linewidth}
      \def\AlgoREDivide{3}% 3
      \def\AlgoRESize{0.5}% 1/2
      \def\AlgoREScale{4.444}% 15 / (3/2)^3
      \input{Images/RecursionEquations/ProblemSizeTree.tikz}
    \end{adjustbox}
    \caption{simple recursion equation with {\color{MainA}$a = 3, b = 2$}}
    \label{fig:recursion_equations:master_theorem_tree_3_2}
  \end{figure}
  \onslide<2->
  \textbf{Case 1:} {\color{MainA}$a > b$}
  \begin{itemize}
    \item<3->
      Three partial problems with $\frac{1}{2}$ the size
    \item<4->
      Solving the partial problems dominates (last layer, leaves)
    \item<5->
      Runtime of {\color{MainA}$\Theta(n^{\log_b a})$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
  \begin{figure}[!h]
    \begin{adjustbox}{width=\linewidth}
      \def\AlgoREDivide{2}% 2
      \def\AlgoRESize{0.5}% 1/2
      \def\AlgoREScale{15}% 15
      \input{Images/RecursionEquations/ProblemSizeTree.tikz}
    \end{adjustbox}
    \caption{simple recursion equation with {\color{MainA}$a = 2, b = 2$}}
    \label{fig:recursion_equations:master_theorem_tree_2_2}
  \end{figure}
  \onslide<2->
  \textbf{Case 2:} {\color{MainA}$a = b$}
  \begin{itemize}
    \item<3->
      Two partial problems with $\frac{1}{2}$ the size
    \item<4->
      Each layer has equal costs, $\log n$ layers
    \item<5->
      Runtime of {\color{MainA}$\Theta(n \, \log n)$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

%\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
%  \begin{figure}[!h]
%    \begin{adjustbox}{width=\linewidth}
%      \def\AlgoREDivide{2}% 2
%      \def\AlgoRESize{0.25}% 1/4
%      \def\AlgoREScale{15}% 15
%      \input{Images/RecursionEquations/ProblemSizeTree.tikz}
%    \end{adjustbox}
%    \caption{simple recursion equation with
%      {\color{MainA}$a = 2, b = 4$}}
%    \label{fig:recursion_equations:master_theorem_tree_2_4}
%  \end{figure}
%  \textbf{Case 3:} {\color{MainA}$a < b$}
%  \begin{itemize}
%    \item
%      Two partial problems with $\frac{1}{4}$ the size
%    \item
%      Connecting all partial solutions dominates (first layer, root)
%    \item
%      Runtime of {\color{MainA}$\Theta(n)$}
%  \end{itemize}
%\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
  \begin{figure}[!h]
    \begin{adjustbox}{width=\linewidth}
      \def\AlgoREDivide{2}% 2
      \def\AlgoRESize{0.33}% 1/3
      \def\AlgoREScale{15}% 15
      \input{Images/RecursionEquations/ProblemSizeTree.tikz}
    \end{adjustbox}
    \caption{simple recursion equation with {\color{MainA}$a = 2, b = 3$}}
    \label{fig:recursion_equations:master_theorem_tree_2_3}
  \end{figure}
  \onslide<2->
  \textbf{Case 3:} {\color{MainA}$a < b$}
  \begin{itemize}
    \item<3->
      Two partial problems with $\frac{1}{3}$ the size
    \item<4->
      Connecting all partial solutions dominates (first layer, root)
    \item<5->
      Runtime of {\color{MainA}$\Theta(n)$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (Simple Form)}
  \textbf{For a recursion equation like}
  \begin{displaymath}
    \color{MainA}{%
      T(n) = a \cdot T\left(\frac{n}{b}\right) + c \cdot n,%
      \hspace{1.5em}a \geq 1, b > 1, c > 0%
    }%
  \end{displaymath}
  \vspace{-1.0em}
  \begin{itemize}
    \item<2->
      $\ldots$ yields a runtime of:
  \end{itemize}
  \onslide<3->
  \begin{displaymath}
    T(n) = \begin{cases}
      \Theta(n^{\log_b a}) & \text{if } a > b\\
      \Theta(n \, \log_b n) & \text{if } a = b\\
      \Theta(n) & \text{if } a < b
    \end{cases}
  \end{displaymath}
  \begin{itemize}
    \item<4->
      Proof with \textit{geometric series}:
      Number of operations per layer grows / shrinks by constant factor
      $\tfrac{a}{b}$
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\subsubsection{Master theorem (General Form)}

\begin{frame}<beamer>{\LectureToC}
  \tableofcontents[currentsection,
    currentsubsection,
    subsubsectionstyle=show/shaded/shaded/shaded]
\end{frame}

\begin{frame}{Recursion Equations}{Master theorem (General Form)}
  \textbf{Master theorem (general form):}
  \onslide<2->
  \begin{displaymath}
  \color{MainA}{%
    T(n) = a \cdot T\left(\frac{n}{b}\right) + f(n),%
    \hspace{1.5em}a \geq 1, b > 1%
  }%
  \end{displaymath}
  \vspace{-1.0em}
  \begin{itemize}
    \item<3->
      \textbf{Case 1:}
      {\color{MainB}$T(n) \in \Theta(n^{\log_b a})$}
      \hfill if {\color{MainA}
        $f(n) \in \mathcal{O}(n^{\log_b a-\varepsilon}), ~ \varepsilon > 0$
      }\\
      {\color{gray}Solving the partial problems dominates\\
        (last layer, leaves)}
      \vspace{1.5em}
    \item<4->
      \textbf{Case 2:}
      {\color{MainB}$T(n) \in \Theta(n^{\log_b a} \log n)$}
      \hspace{1.5em} if {\color{MainA}
        $f(n) \in \Theta(n^{\log_b a})$
      }\\
      {\color{gray}Each layer has equal costs, $\log_b n$ layers}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (General Form)}
  \textbf{Master theorem (general form):}
  \begin{itemize}
    \item
      \textbf{Case 3:}
      {\color{MainB}$T(n) \in \Theta(f(n))$}
      \hfill if {\color{MainA}
        $f(n) \in \Omega(n^{\log_b a+\varepsilon}), ~ \varepsilon > 0$
      }\\
      {\color{gray}Connecting all partial solutions in first layer (root) dominates}\\[1.0em]
      {\color{MainA}Regularity condition:}
      \begin{align*}
        a \cdot f\left(\frac{n}{b}\right) \leq c \cdot f(n), \hspace{1.5em} &
        0 \leq c \leq 1,\\
        {} & n > n_0
      \end{align*}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (General Form) - Case 1}
  \textbf{Case 1 - Example:}
  \onslide<2->{{\color{MainB}$T(n) \in \Theta(n^{\log_b a})$}}
  \hfill if {\color{MainA}
    $f(n) \in O(n^{\log_b a-\varepsilon}), ~ \varepsilon > 0$
  }\\
  {\color{gray}Solving the partial problems dominates (last layer, leaves)}
  \vspace{1.0em}
  \begin{itemize}
    \item<3->
      $T(n) = 8 \cdot T(\tfrac{n}{2}) + 1000 \cdot n^2$
      \vspace{-0.5em}
      \begin{displaymath}
        a = 8, ~ b = 2, ~ f(n) = 1000 \cdot n^2, ~
        \smash{\underbrace{
          \log_b a = \log_2 8 = 3
        }_{\text{$n^3$ leaves}}}
      \end{displaymath}
      {\color{MainA}\begin{math}
        f(n) \in \mathcal{O}(n^{3-\varepsilon})$
        $\Rightarrow$
        $T(n) \in \Theta(n^3)
      \end{math}}
      \vspace{1.5em}
    \item<4->
      $T(n) = 9 \cdot T(\tfrac{n}{3}) + 17 \cdot n$
      \vspace{-0.5em}
      \begin{displaymath}
        a = 9, ~ b = 3, ~ f(n) = 17 \cdot n, ~
        \smash{\underbrace{
          \log_b a = \log_3 9 = 2
        }_{\text{$n^2$ leaves}}}
      \end{displaymath}
      {\color{MainA}\begin{math}
        f(n) \in \mathcal{O}(n^{2-\varepsilon})$
        $\Rightarrow$
        $T(n) \in \Theta(n^2)
      \end{math}}
   \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (General Form) - Case 2}
  \textbf{Case 2:}
  \onslide<2->{{\color{MainB}$T(n) \in \Theta(n^{\log_b a} \log n)$}}
  \hspace{1.5em} if {\color{MainA}
    $f(n) \in \Theta(n^{\log_b a})$
  }\\
  {\color{gray}Each layer has equal costs, $\log n$ layers}
  \vspace{1.0em}
  \begin{itemize}
    \item<3->
      $T(n) = 2 \cdot T(\tfrac{n}{2}) + 10 \cdot n$
      \vspace{-0.5em}
      \begin{displaymath}
        a = 2, ~ b = 2, ~ f(n) = 10 \cdot n, ~
        \smash{\underbrace{
          \log_b a = \log_2 2 = 1
        }_{\text{$n^1$ leaves}}}
      \end{displaymath}
      {\color{MainA}\begin{math}
        f(n) \in \Theta(n^{\log_2 2})$
        $\Rightarrow$
        $T(n) \in \Theta(n \, \log n)
      \end{math}}
      \vspace{1.0em}
    \item<4->
      $T(n) = T(\tfrac{2\,n}{3}) + 1$
      \vspace{-0.5em}
      \begin{displaymath}
        a = 1, ~ b = \tfrac{3}{2}, ~ f(n) = 1, ~
        \smash{\underbrace{
          \log_b a = \log_{\nicefrac{3}{2}} 1 = 0
        }_{\text{$n^0$ leaves $=$ 1 leaf}}}
      \end{displaymath}
      \hfill\\[1.0em]
      {\color{MainA}\begin{math}
        f(n) \in \Theta(n^{\log_{\nicefrac{3}{2}} 1})$
        $\Rightarrow$
        $T(n) \in \Theta(n^0 \log n) = \Theta(\log n)
      \end{math}}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (General Form) - Case 3}
  \textbf{Case 3:}
  \onslide<2->{{\color{MainB}$T(n) \in \Theta(f(n))$}}
  \hfill if {\color{MainA}
    $f(n) \in \Omega(n^{\log_b a+\varepsilon}), ~ \varepsilon > 0$
  }\\
  {\color{gray}Connecting all partial solutions in first layer (root) dominates}
  \vspace{1.0em}
  \begin{itemize}
    \item<3->
      $T(n) = 2 \cdot T(\tfrac{n}{2}) + n^2$
      \vspace{-0.5em}
      \begin{displaymath}
        a = 2, ~ b = 2, ~ f(n) = n^2, ~
        \smash{\underbrace{
          \log_b a = \log_2 2 = 1
        }_{\text{$n^1$ leaves}}}
      \end{displaymath}
      {\color{MainA}$f(n) \in \Omega(n^{1+\varepsilon})$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (General Form) - Case 3}
  \textbf{Case 3:}
  {\color{MainB}$T(n) \in \Theta(f(n))$}
  \hfill if {\color{MainA}
    $f(n) \in \Omega(n^{\log_b a+\varepsilon}), ~ \varepsilon > 0$
  }\\
  {\color{gray}Connecting all partial solutions in first layer (root) dominates}
  \vspace{1.0em}
  \begin{itemize}
    \item
      $T(n) = 2 \cdot T(\tfrac{n}{2}) + n^2$
    \item<3->
      {\color{MainA}$f(n) \in \Omega(n^{1+\varepsilon})$}
    \item<4->
      Check if {\color{MainA}regularity condition} also holds:\\
      \begin{align*}
        a \cdot f\left(\frac{n}{b}\right) &\leq c \cdot f(n)\\
        2 \cdot \left(\frac{n}{2}\right)^2 &\leq c \cdot n^2
        \hspace{1.5em} \Rightarrow \frac{1}{2} \cdot n^2 \leq c \cdot n^2
        \hspace{1.5em} \Rightarrow c \geq \frac{1}{2}
      \end{align*}
      {\color{MainA}$\Rightarrow T(n) \in \Theta(n^2)$}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem (General Form)}
  \textbf{Master theorem:}
  \begin{itemize}
    \item<2->
      Not always applicable:
      {\color{MainA}$T(n) = 2 \cdot T(\frac{n}{2}) + n \, \log n$}
  \end{itemize}
  \onslide<3->
  \vspace{1.0em}
  \begin{displaymath}
    a = 2, ~ b = 2, ~ f(n) = n \, \log n, ~
    \underbrace{\log_b a = \log_2 2 = 1}_{\text{$n^1$ leaves}}
  \end{displaymath}
  \begin{itemize}
    \item<4->
      \textbf{Case 1:}
      $f(n) \notin O(n^{1-\varepsilon})$
    \item<5->
      \textbf{Case 2:}
      $f(n) \notin \Theta(n^1)$
    \item<6->
      \textbf{Case 3:}
      $f(n) \notin \Omega(n^{1+\varepsilon})$
  \end{itemize}
  \onslide<7->
  \begin{center}
    {\color{MainA}$n \, \log n$} is \textit{asymptotically} larger than
    {\color{MainA}$n$},\\ but not \textit{polynominal} larger
  \end{center}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Recursion Equations}{Master theorem - Summary}
  \textbf{Master theorem:}
  \onslide<2->
  \begin{displaymath}
    \color{MainA}{
      T(n) = a \cdot T\left(\frac{n}{b}\right) + f(n)
    }
  \end{displaymath}
  \vspace{-1.0em}
  \begin{itemize}
    \item<3->
      Three cases depending on the dominance of the terms
    \item<4->
      \textbf{Case 1:}
      Solving the partial problems is \textit{polynominal} bigger than merging
      all solutions\\
      {\color{MainA}$T(n) \in \Theta(n^{\log_b a})$}, \hspace{3.5em}
      {\color{gray}$T(n) \in \Theta(\text{number of leaves})$}
      \vspace{0.5em}
   \item<5->
     \textbf{Case 2:}
     Each layer has equal costs\\
     {\color{MainA}$T(n) \in \Theta(n^{\log_b a} \log n)$}, \hspace{1.5em}
     {\color{gray}$\log n$ layers}
     \vspace{0.5em}
   \item<6->
     \textbf{Case 3:}
     Connecting all partial solutions is \textit{polynominal} bigger than
     solving all partial problems\\
     {\color{MainA}$T(n) \in \Theta(f(n))$}
  \end{itemize}
\end{frame}


\section{State Error Objective}

	\begin{frame}
		\frametitle{Robot Models Used}
		\begin{columns}[T]
			\begin{column}{0.45\textwidth}
				\onslide<2->{
				\begin{block}{Cartesian Co-ordinate Model}
					\[
					\begin{bmatrix}
						\dot{x} \\
						\dot{y} \\
						\dot{\theta}
					\end{bmatrix}
					=
					\begin{bmatrix}
						\cos(\theta) & 0 \\
						\sin(\theta) & 0 \\
						0 & 1
					\end{bmatrix}
					\begin{bmatrix}
						\upsilon \\
						\omega
					\end{bmatrix}
					\]
				\end{block}
				}
				\onslide<3->{
				\begin{block}{Kino-dynamic Model}
					\[
					\begin{bmatrix}
						\dot{x} \\
						\dot{y} \\
						\dot{\theta} \\
						\dot{\upsilon} \\
						\dot{\omega}
					\end{bmatrix}
					=
					\begin{bmatrix}
						\cos(\theta) & 0 & 0 & 0 \\
						\sin(\theta) & 0 & 0 & 0 \\
						0 & 1 & 0 & 0 \\
						0 & 0 & 1 & 0 \\
						0 & 0 & 0 & 1
					\end{bmatrix}
					\begin{bmatrix}
						\upsilon \\
						\omega \\
						a \\
						\alpha
					\end{bmatrix}
					\]
				\end{block}
				}
			\end{column}
			\begin{column}{0.49\textwidth}
				\centering
				\onslide<1->{\includegraphics[scale=0.35]{pictures/robot_cart.pdf}}
			\end{column}
		\end{columns}
	\end{frame}

	\begin{frame}
		\frametitle{Robot Models Used}
		\begin{columns}[T]
			\begin{column}{0.45\textwidth}
				\onslide<2->{
				\begin{block}{Polar Co-ordinate Model}
					\begin{align*}
						\rho& = \sqrt{(x-x_g)^2+(y-y_g)^2} \\[0.4cm]
						\phi& = \arctan2\big((y-y_g),(x-x_g)\big) \\[0.4cm]
						\alpha&= \theta - \phi
					\end{align*}
				\end{block}
				}
			\end{column}
			\begin{column}{0.49\textwidth}
				\centering
				\onslide<1->{\includegraphics[scale=0.35]{pictures/robot_polar.pdf}}
			\end{column}
		\end{columns}
	\end{frame}

	\begin{frame}
		\frametitle{NMPC General Formulation}
		\begin{block}{NMPC Optimization Problem}
			\begin{align*}
				&\underset{\mathbf{X}^*,\mathbf{U}^*}{\text{min    }}
				\tikzmark{a9}J_{N_p}(\mathbf{x}_k) = \sum_{j=0}^{N_p}\tikzmark{a11}\ell (\mathbf{x}_{k + j},\mathbf{u}_{k + j}), \\
				&\quad \text{subject to:} \notag  \\
				&\quad\quad \tikzmark{a12}\mathbf{x}_{k+j+1} = \mathcal{F}(\mathbf{x}_{k+j},\mathbf{u}_{k+j}), & \forall\text{ } 0\leq j\leq N_p-1  \\
				&\quad\quad \tikzmark{a13}\mathbf{x}_{k+j} \in \mathbb{X}, & \forall \text{ }0\leq j\leq N_p \\
				&\quad\quad \tikzmark{a14}\mathbf{u}_{k+j} \in \mathbb{U}, & \forall \text{ }0\leq j\leq N_p-1 \\
				&\quad\quad \tikzmark{a15}\mathbf{d}_{\imath,k+j}  \geq \xi_o, & \forall\text{ } 0\leq j\leq N_p-1, \text{ }\forall\text{ } \imath \in I_s
			\end{align*}
		\end{block}

		\begin{tikzpicture}[overlay, remember picture]
			\coordinate (A9) at ($({pic cs:a9})+(1ex, 1ex)$);
			\coordinate (A11) at ($({pic cs:a11})+(1ex, 1ex)$);
			\coordinate (A12) at ($({pic cs:a12})+(0ex, 0.5ex)$);
			\coordinate (A13) at ($({pic cs:a13})+(0ex, 0.5ex)$);
			\coordinate (A14) at ($({pic cs:a14})+(0ex, 0.5ex)$);
			\coordinate (A15) at ($({pic cs:a15})+(1ex, -0.5ex)$);
			\pause
			\node [fit=(A9), baseline] (n9) {};
			\node[cyan, above of= n9, node distance = 3em] (t9_1) {\small cost};
			\node[cyan, above of= n9, node distance = 2em] (t9) {\small function};
			\draw [<-, cyan] (n9.north) to [below] (t9.south);
			\pause
			\node [fit=(A11), baseline] (n11) {};
			\node[cyan, above of= n11, node distance = 3em] (t11_1) {\small regulatory};
			\node[cyan, above of= n11, node distance = 2em] (t11) {\small term};
			\draw [<-, cyan] (n11.north) to [below] (t11.south);
			\pause
			\node [fit=(A12), baseline] (n12) {};
			\node[cyan, left of= n12, node distance = 4em] at (2.8,3) (t12) {\small model};
			\node[cyan, below of= t12, node distance = 1em] (t12_1) {\small constraints};
			\draw [<-, cyan] (n12.west) to [right] (t12.east);
			\pause
			\node [fit=(A13), baseline] (n13) {};
			\node[cyan, left of= n13, node distance = 4em] (t13) {\small state};
			\node[cyan, below of= t13, node distance = 1em] (t13_1) {\small boundries};
			\draw [<-, cyan] (n13.west) to [right] (t13.east);
			\pause
			\node [fit=(A14), baseline] (n14) {};
			\node[cyan, below of= t13, node distance = 3em] (t14) {\small input};
			\node[cyan, below of= t14, node distance = 1em] (t14_1) {\small boundries};
			\draw [<-, cyan] (n14.west) to [right] (t14.east);
			\pause
			\node [fit=(A15), baseline] (n15) {};
			\node[cyan, below of= n15, node distance = 3em] (t15_1) {\small static};
			\node[cyan, below of= n15, node distance = 2em] (t15) {\small obstacles};
			\draw [<-, cyan] (n15.south) to [above] (t15.north);
		\end{tikzpicture}
	\end{frame}

	\begin{frame}
		\frametitle{NMPC General Formulation}
		\begin{block}{Regulatory Terms (e.g., Cartesian coordinate)}
			\[
				\begin{aligned}
					\ell (\mathbf{x}_{k + j},\mathbf{u}_{k + j}) = \enspace
					\tikzmark{a1}
					(\mathbf{x}_{k + j} - \mathbf{x}_{ref})^T \enspace \mathbf{Q_1} \enspace (\mathbf{x}_{k + j} - \mathbf{x}_{ref})
					\tikzmark{a2}
					\enspace + \enspace
					\tikzmark{a3}
					\Delta\mathbf{u}_{k + j}^T \enspace \mathbf{R} \enspace \Delta\mathbf{u}_{k + j}
					\tikzmark{a4}
					\enspace \tikzmark{a7}-\tikzmark{a8} \enspace
					\tikzmark{a5}
					\sum_{i=0}^{I} \mathbf{d}_{i,k + j}^T \enspace \mathbf{Q_2} \enspace \mathbf{d}_{i,k + j}
					\tikzmark{a6}
				\end{aligned}
			\]
		\end{block}
		\vfill
		\vfill
		\vfill
		\begin{itemize}
			\item $\mathbf{d}_{i,k + j}$: distance between the robot and $i^{th}$ detected moving obstacle.
		\end{itemize}

		\begin{tikzpicture}[overlay, remember picture]
			\coordinate (A1) at ($({pic cs:a1})+(+0.1ex, 2ex)$);
			\coordinate (A2) at ($({pic cs:a2})+(-0.1ex,-0.5ex)$);
			\coordinate (A3) at ($({pic cs:a3})+(+0.1ex, 2ex)$);
			\coordinate (A4) at ($({pic cs:a4})+(-0.1ex,-0.5ex)$);
			\coordinate (A5) at ($({pic cs:a5})+(+0.1ex, 3.5ex)$);
			\coordinate (A6) at ($({pic cs:a6})+(-0.1ex,-2.5ex)$);
			\coordinate (A7) at ($({pic cs:a7})+(+1ex, 0.5ex)$);
			\coordinate (A8) at ($({pic cs:a8})+(-1ex, 0.5ex)$);
			\pause
			\node [fit=(A1)(A2), draw=tugreen, thick, rounded corners] (n1) {};
			\node[overlay, below of= n1, node distance = 6em] (t1) {State Error};
			\node[overlay, below of= n1, node distance = 7em] (t1_1) {Objective};
			\draw [thick, -latex, tugreen] (n1.south) to [above] (t1.north);
			\pause
			\node [fit=(A3)(A4), draw=tugreen, thick, rounded corners] (n2) {};
			\node[overlay, below of= n2, node distance = 6em] (t2) {Input Penealty};
			\node[overlay, below of= n2, node distance = 7em] (t2_1) {Objective};
			\draw [thick, -latex, tugreen] (n2.south) to [above] (t2.north);
			\pause
			\node [fit=(A5)(A6), draw=tugreen, thick, rounded corners] (n3) {};
			\node[overlay, below of= n3, node distance = 5.7em] (t3) {Dyn Obstacles};
			\node[overlay, below of= n3, node distance = 6.7em] (t3_1) {Objective};
			\draw [thick, -latex, tugreen] (n3.south) to [above] (t3.north);
			\pause
			\node [fit=(A7)(A8), draw=red, circle] (n4) {};
			\node[overlay, red, above of= n4, node distance = 8em] (t4) {Maximization};
			\draw [thick, -latex, red] (n4.north) to [below] (t4.south);
		\end{tikzpicture}

%		\tikz[baseline=(n1.base)]{
%			\node[draw=tugreen, ,thick, rounded corners, inner sep=5pt]
%			(n1){$xfoz$};
%			\node[overlay, below of= n1, node distance = 6em] (t1) {State Error};
%			\draw [thick, -latex, tugreen] (n1.south) to [above] (t1.north);
%		}
	\end{frame}

%	\begin{frame}
%		\frametitle{Cartesian Coordinate Model Block Diagram}
%		\begin{figure}[hbtp]
%			\centering
%			\includegraphics[scale=0.7]{pictures/block_diagram_cart.pdf}
%			\caption{NMPC Control Loop (Cartesian coordinate Model)}
%		\end{figure}
%	\end{frame}

	\begin{frame}
		\frametitle{NMPC General Formulation}
		\begin{block}{Regulatory Terms (e.g., Polar coordinate)}
			\parbox[c][4\baselineskip][t]{\textwidth}{
			\begin{align*}
				\ell_p (\vartheta_{k + j},\mathbf{u}_{k + j}) &=
				\pmb{[} \ell_{p,1}(\cdot,\cdot) \enspace \ell_{p,2}(\cdot,\cdot) \enspace \ell_{p,3}(\cdot,\cdot) \pmb{]}
				\enspace
				\mathbf{Q_1} \enspace
				\begin{bmatrix}
					\ell_{p,1}(\cdot,\cdot) \\
					\ell_{p,2}(\cdot,\cdot) \\
					\ell_{p,3}(\cdot,\cdot)
				\end{bmatrix}
				+ \cdots
			\end{align*}
			}
		\end{block}
		Error state objective is the minimization of: \\[0.5cm]
		\begin{itemize}
			\item Euclidean distance: $\ell_{p,1}(\cdot,\cdot) = \sqrt{(x_{k+j} - x_{ref})^2+(y_{k+j} - y_{ref})^2}$ \\ [0.2cm]
			\item Heading towards goal: $\ell_{p,2}(\cdot,\cdot) = \Big(\theta_{k+j} - \arctan2\big((y_{k+j} - y_{ref}),(x_{k+j} - x_{ref})\big)\Big)^2$ \\ [0.2cm]
			\item Reference orientation: $\ell_{p,3}(\cdot,\cdot) = (\theta_{k+j} - \theta_{ref})^2$
		\end{itemize}
	\end{frame}

%	\begin{frame}
%		\frametitle{Polar Coordinate Model Block Diagram}
%		\begin{figure}[hbtp]
%			\centering
%			\includegraphics[scale=0.64]{pictures/block_diagram_polar.pdf}
%			\caption{NMPC Control Loop (Polar coordinate Model)}
%		\end{figure}
%	\end{frame}

%	\begin{frame}
%		\frametitle{Kino-dynamic Model Block Diagram}
%		\begin{figure}[hbtp]
%			\centering
%			\includegraphics[scale=0.64]{pictures/block_diagram_kino.pdf}
%			\caption{NMPC Control Loop (Kino-dynamic Model)}
%		\end{figure}
%	\end{frame}

	\begin{frame}
		\frametitle{Potential Collision Zone}
		\centering
		\includegraphics[scale=0.6]{pictures/robots_pot_coll.pdf}
	\end{frame}


\section*{Web Appendix F}

Recall that a density $f(x;\theta)$ is identifiable if $f(x;\theta_{1})=f(x;\theta_{2})$
for all $x$ implies that $\theta_{1}=\theta_{2}.$ Call a density
$f(x;\theta)$ \textit{strongly identifiable} if $f(x;\theta_{1})/f(x;\theta_{2})$
being constant for all $x$ in an open interval $I$ implies that
$\theta_{1}=\theta_{2}$.
\paragraph{Proposition A.}\label{prop:identifiable}

Let $f(x;\theta)$ be a family of densities on $\mathbb{R}$, $-\infty=a_{1}<a_{2}<\ldots<a_{k+1}=\infty$
a sequence of cutoffs, and $f_{[a_{i},a_{i+1})}(x;\theta)$ the density
$f$ truncated to $[a_{i},a_{i+1})$. Let $\lambda_{i},k=1,\ldots k$
be positive numbers satisfying $\sum_{i=1}^{k}\lambda_{i}=1$. Then
the mixture
\[
g(x;\lambda,\theta)=\sum_{i=1}^{k}\lambda_{i}f_{[a_{i},a_{i+1})}(x;\theta)
\]
is identifiable in $(\lambda,\theta)$ if $f(x;\theta)$ is strongly
identifiable in $\theta$.

\paragraph{Proof.}
Assume that $g(x;\lambda_{1},\theta_{1})=g(x;\lambda_{2},\theta_{2})$.
Then $$\lambda_{1i}f_{[a_{i},a_{i+1})}(x;\theta_{1})=\lambda_{2i}f_{[a_{i},a_{i+1})}(x;\theta_{2})$$
for all $i$, thus
\[
\frac{\lambda_{1i}}{\lambda_{2i}}=\frac{f_{[a_{i},a_{i+1})}(x;\theta_{1})}{f_{[a_{i},a_{i+1})}(x;\theta_{2})}.
\]
This implies that $f(x;\theta_{1})/f(x;\theta_{2})$ is constant for
$x\in[a_{i},a_{i+1})$. But since $f(x;\theta)$ is strongly identifiable,
$\theta_{1}=\theta_{2}$, and, consequently, $\lambda_1 = \lambda_2$.

If $f(x;\theta)$ is real analytic and nowhere zero, $f(x;\theta_{1})/f(x;\theta_{2})$
is also real analytic and nowhere zero. By the Identity Theorem \citep[Corollary 1.2.6]{Krantz2002-bt}, if the ratio
$f(x;\theta_{1})/f(x;\theta_{2})$ is constant on some interval $I$,
then $f(x;\theta_{1})/f(x;\theta_{2})$ is constant everywhere, hence
$f(x;\theta_{1})=f(x;\theta_{2})$ everywhere. Thus a family of real
analytic nowhere zero densities is identifiable if and only if it is
strongly identifiable. Every exponential family of densities on the form
\[
f(x;\theta)=h(x)\exp(\eta(\theta)^{T}T(x)-A(\theta))
\]
satisfies this property, provided only that $h$ is nowhere zero real analytic
and $T$ is real analytic. In particular, the normal family satisfies
the properties.

Not every density is strongly identifiable. For instance, mixtures of uniforms are not strongly identifiable. And indeed, Proposition A fails when $f$ is a mixture of uniforms.


\section{Introduction}

Neurons, like all other cells, require proteins to function. Because of their morphological diversity and the fact that synapses can be hundreds of micrometers away from the some, neurons require an exquisite control of protein and mRNA distributions in both time and space.
Experimental evidence indicate the presence of mRNA in dendrites and recent reports highlight the importance of local translation in various forms of synaptic plasticities \supercite{Hafner2019, Younts2016}.

In their article, Fonkeu et al. (2019) develop a model framework to capture the spatial profiles of mRNA and protein in dendrites while taking into account: production, degradation, uni- and bi-directional transport \supercite{Fonkeu2019}. Validation of the model is obtained  by comparing with experimental data obtained from the analysis of CaMKII$\alpha$ mRNA, a major kinase involved in synaptic plasticity.

Here, we aimed at replicating model-based figures (therefore excluding Figures 2, 4, S1, S2, S3 that are either experimental or explanatory models). Code was not available on public repositories, except small pieces of scripts. Overall, the replication was successfull with only minor quantitative differences.

\section{Methods}

The model is based on the following equations:

\textbf{Equation 1} describes the mRNA dynamics using transcription rate ($\beta_{R}$), passive diffusion ($D_{R}$), transport velocity ($\nu_{R}$), degradation rate ($k_{R}$) and half-life ($T_{1/2}$).
\begin{equation}
  R^{den}_{ss}(x) = \frac{\beta_{R}\lambda_{R}}{k_{R}} e^{-\lambda_{R}x}x
\end{equation}
where $\lambda_{R} = \frac{\sqrt{\nu^{2}_{R} + 4D_{R}k_{R}}-\nu_{R}}{2D_{R}}$ and $k = \frac{\ln{2}}{T_{1/2}}$, $T_{1/2}$ being in seconds.

\textbf{Equation 2} describes protein dynamics using mRNA and protein transcription and translation rates ($\beta_{R}$ and $\beta_{P}$), diffusion coefficients ($D_{R}$ and $D_{P}$ respectively), transport velocities ($\nu_{R}$ and $\nu_{P}$) and degradation rates ($k_{R}$ and $k_{P}$).

\begin{equation}
  P^{den}_{SS}(x) = \frac{\beta_{P}\beta_{R}\lambda_{R}}{k_{R}(D_{P}\lambda^{2}_{R}+\nu_{P}\lambda_{R} - k_{P})}(-e^{-\lambda_{R}x} + \frac{D_{P}\lambda_{P}\lambda_{R} + \nu_{P}\lambda_{P}}{k_{P}}e^{-\lambda_{P}x})
\end{equation}

where $\lambda_{P} = \frac{\sqrt{\nu^{2}_{P} + 4D_{P}k_{P}}-\nu_{P}}{2D_{P}}$

In the case of proteins with 3'UTR localizing them to the soma, the \textbf{Equation 3} describes their distribution:
\begin{equation}
  P^{som}_{ss}(x) = \frac{\beta_{P}\beta_{R}\lambda_{P}}{k_{R}k_{P}} e^{-\lambda_{P}x}
\end{equation}

In neurons, both somatically and dendritically synthesized proteins contribute to the dendritic protein distribution. Thus, the total distribution of protein is a mixture of both \textbf{Equation 2} and \textbf{Equation 3} in the form of \textbf{Equation 4}:

\begin{equation}
  P^{tot}_{ss}(x) = S_{mRNA} \cdot z \cdot P^{som}_{ss}(x) + (1-S_{mRNA})  \cdot P^{den}_{ss}(x)
\end{equation}

where $S_{mRNA}$ denotes the fraction of mRNAs found in the soma and $z$ the fraction of somatically synthesized proteins that is transported to the dendrite.

\textbf{Table 1} provides values of rates used for simulations.

\begin{center}
  \begin{tabular}{|c|c|c|}
    \hline
    & mRNA & Protein \\
    \hline
    Transcription (mRNA/s) & $\beta_{R} = 0.001$ & \\
    \hline
    Translation (protein/s) & & $\beta_{P} = 0.021$ \\
    \hline
    Diffusion ($\mu m^{2}/s$) & $D_{R}=3.4e^{-3}$ & $D_{P}=0.24$ \\
    \hline
    Velocity ($\mu m/s$) & $\nu_{R}=6.7e^{-3}$ & $\nu_{P}=0$ \\
    \hline
    Half life & $T_{1/2}=16$ hours & $T_{1/2}=6.67$ days \\
    \hline
  \end{tabular}
\end{center}

\textbf{Table 2} provides values for other parameters. Values were rounded to 2 decimals. Of note, using these values can lead to slight changes in the quantitative results. Full values can be found in the code files.

\begin{center}
  \begin{tabular}{|c|c|}
    \hline
    Name & Value \\
    \hline
    $S_{mRNA}$ & 0.557 \\
    \hline
    $z$ & 0.22 \\
    \hline
  \end{tabular}
\end{center}

\section{Results}

In \textbf{Figure 1}, we show the range of possible dendritic mRNA and protein distributions according to the smallest and largest experimental values from the literature (Figure 1C,D in the orignal article). We observed small quantitative differences with values reported in the original article (31.1\% instead of 29\% reduction of protein cncentration in conditions of reduced protein mobility).

\begin{figure}[H]
  \includegraphics[width=\textwidth]{Figure1.pdf}
  \caption{Replication of the possible range of dendritic mRNA and protein distribution according to the literature.  Color code denotes the upper bound (dark blue and red) and lower bound (light blue and orange) for the respective distributions (\textbf{A}) mRNA distribution obtained for candidate values from the literature. Upper bound: $D_{R} = 0.0038\mu m^{2}/s$ and $\nu_{R}=1.3\mu m/s$. Lower bound:  $D_{R} = 0.003\mu m^{2}/s$ and $\nu_{R}=0.0058\mu m/s$. (\textbf{B}) Protein distribution obtained for candidate values from the literature.Upper bound: $D_{P} = 4.5\mu m^{2}/s$ and $\nu_{P}=0\mu m/s$. Lower bound:  $D_{P} = 0.023\mu m^{2}/s$ and $\nu_{R}=0\mu m/s$.}
\end{figure}

In \textbf{Figure 2}, we show the influence of single parameter variations on mRNA and protein distribution. The initial parameters used are the one experimentally confirmed in the study: $D_{R}=3.4\cdot10^{-3} \mu m^{2}/s, \nu_{R}=6.7 \cdot 10^{-3} \mu m/s$. Qualitative inspection of the obtained graphs showed similar behavior and allowed to reach similar conclusions.

\begin{figure}[H]
  \includegraphics[width=\textwidth]{Figure2.png}
  \caption{Consequences of modulating mRNA parameters on mRNA and protein distributions. (\textbf{A, B}) Impact of mRNA velocity. (\textbf{D, E}) Impact of mRNA diffusion.(\textbf{F, G}) Impact of mRNA lifetime. In all panels, distributions were normalizxed to the original value. Alternative normalization were also explored and were successfully replicated (LINK ??)}
\end{figure}


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{fmlin}
\section*{\hspace*{-1.6cm} fmlin}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Signal with linear frequency modulation.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
[y,iflaw] = fmlin(N)
[y,iflaw] = fmlin(N,fnormi)
[y,iflaw] = fmlin(N,fnormi,fnormf)
[y,iflaw] = fmlin(N,fnormi,fnormf,t0)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty fmlin} generates a linear frequency modulation, going from
{\ty fnormi} to {\ty fnormf}.  The phase of this modulation is such that
{\ty y(t0)=1}.\\

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8.5cm} c}
Name & Description & Default value\\
\hline
        {\ty N }      & number of points\\
        {\ty fnormi}  & initial normalized frequency & {\ty 0.0}\\
        {\ty fnormf}  & final   normalized frequency & {\ty 0.5}\\
        {\ty t0}      & time reference for the phase & {\ty N/2}\\
  \hline {\ty y}       & signal\\
        {\ty iflaw }  & instantaneous frequency law  \\
\hline
\end{tabular*}

\end{minipage}
\vspace*{1cm}

{\bf \large \sf Example}
\begin{verbatim}
         z=amgauss(128,50,40).*fmlin(128,0.05,0.3,50);
         plot(real(z));
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
fmconst, fmsin, fmodany, fmhyp, fmpar, fmpower.
\end{verbatim}
\end{minipage}


% \documentclass{article}
% \usepackage[utf8]{inputenc}
% \usepackage{amsmath}

% \title{Toda Lattice, Double Bracket}
% \author{Lusa Zhan}
% \date{September 2016}

% \begin{document}

% \maketitle

% \section{Equivalence between ODE and Lie bracket notation}

The central idea of analog sorting as described in~\cite{bloch, brockett} makes use of the Toda lattice. The system of Hamiltonian equations in (1) associated with the Hamiltonian in (2) gives us the following set of equations
\begin{align*}
\dot{p_k} &= \exp(q_{k-1}-q_{k})-\exp(q_{k}-q_{k+1}) \\
\dot{q_k} &= p_k
\end{align*}
where the boundary conditions are set so that $\exp(x_0-x_1)=\exp (x_n-x_{n+1})=0$.

This system is also analogous to the double bracket notation
\begin{align}
\dot{H} = [H,[H,N]]
\end{align}

where the brackets stand for the Lie bracket $[A,B] = AB-BA$. The connection between the Hamiltonian system  and the double bracket notation of the Toda lattice will be discussed and outlined in the following sections.

\subsubsection{Toda lattice - ODE form}

The above system of ordinary differential equations can be transformed into the system described in~\cite{harvard_robo} using a change of variables.
\begin{align}
x_k &= -\frac{1}{2}p_k \nonumber \\
y_k &= \frac{1}{2}\exp(\frac{q_k-q_{k+1}}{2})
\end{align}

In that case,
\begin{align*}
    \dot{x_k} &= -\frac{1}{2}\dot{p_k} \\
              &= -\frac{1}{2}(exp(q_{k-1}-q_{k})-exp(q_{k}-q_{k+1})) \\
              &= -\frac{1}{2}(4y^2_{k-1}-4y^2_k) \\
              &= 2y^2_k-2y^2_{k-1}
\end{align*}

and
\begin{align*}
    \dot{y_k} &= \frac{1}{2}\exp(\frac{q_k-q_{k+1}}{2})\frac{\dot q_k-\dot q_{k+1}}{2} \\
              &= y_k\frac{p_k-p_{k+1}}{2} \\
              &= y_k(x_{k+1}-x_k)
\end{align*}

Taking into account the boundary conditions $y_0=y_n=0$, we get the desired system of ODEs with
\begin{align}
    \dot{x_k} &= 2y^2_k-2y^2_{k-1} \nonumber \\
    \dot{y_k} &= y_k(x_{k+1}-x_k) \\
    y_0 &= y_n = 0 \nonumber
\end{align}

This is the system of ODEs we will solve in the analog chip and in our simulations.

\subsubsection{Toda lattice - Jacobi matrix}

The connection between the Toda lattice and the double bracket notation $\dot{H} = [H,[H,N]]$ can be made through the Jacobi Matrix form of the Toda lattice. The Jacobi matrix for the Hamiltonian system after the change of variables (4) is given by
\begin{align}
 H = \begin{bmatrix}
    x_{1} & y_{1} & 0  & \dots & 0 \\
    y_{1} & x_{2} & y_{2} & \dots & 0 \\
     & & \ddots & \\
          &       & y_{n-2} & x_{n-1} & y_{n-1}\\
    0 & \hdots & & y_{n-1} & x_{n}
\end{bmatrix}
\end{align}
This is the form of $H$ required for analog sorting as outlined by Brockett in ~\cite{brockett}.

In order to get the double bracket form, we need a diagonal matrix $N = \text{diag}(n, n-1, \dots, 1)$ whose role will be discussed in 2.5.1.

\begin{align*}
N = \begin{bmatrix}
        n & 0 & \hdots & 0 \\
        0 & n-1 & \\
        \vdots &  & \ddots & \vdots \\
        0 & \hdots & & 1
    \end{bmatrix}
\end{align*}

From this, we can calculate $[H[H,N]] = H[H,N]-[H,N]H$ step by step:

\[
HN = \begin{bmatrix}
        nx_1 & (n-1)y_1 & \hdots & 0 \\
        ny_1 & (n-1)x_2 & \hdots & 0 \\
        \vdots & & \ddots & 0\\
         &  & 2x_{n-1} & y_{n-1} \\
        0 & \hdots & 2y_{n-1} & x_n

    \end{bmatrix}
\]

\[
NH = \begin{bmatrix}
        nx_1 & ny_1 & \hdots & 0 \\
        (n-1)y_1 & (n-1)x_2 & \hdots & 0 \\
        \vdots & & \ddots & 0\\
         &  & 2x_{n-1} & 2y_{n-1} \\
        0 & \hdots & y_{n-1} & x_n

    \end{bmatrix}
\]

\[
HN-NH = \begin{bmatrix}
        0 & -y1 & \hdots &  & 0 \\
        y_1 & 0 & -y_2 & \hdots & 0 \\
        \vdots & \ddots & \ddots & \ddots & 0\\
         & & y_{n-2}& 0 & -y_{n-1} \\
        0 & & \hdots & y_{n-1} & 0

    \end{bmatrix}
\]

\[
H[H,N] = \begin{bmatrix}
        y_1^2 & -x_1y_1 & \hdots & 0 \\
        x_2y_1 & -y_1^2+y^2_2 &\hdots & 0 \\
        y_1y_2 & \ddots & & 0\\
        \vdots & & y^2_{n-1}-y^2_{n-2} & -x_{n-1}y_{n-1} \\
        0 & \hdots & x_ny_{n-1} & -y^2_{n-1}

    \end{bmatrix}
\]

\[
[H,N]H = \begin{bmatrix}
        y_1^2 & -x_2y_1 & \hdots & 0 \\
        x_1y_1 & y_1^2-y^2_2 &\hdots & 0 \\
        y_1y_2 & \ddots & & 0\\
        \vdots &  & -y^2_{n-1}+y^2_{n-2} & -x_{n}y_{n-1} \\
        0 & \hdots & x_{n-1}y_{n-1} & y^2_{n-1}

    \end{bmatrix}
\]
\\

Therefore, we get \\

\[[H,[H,N]] =
\begin{bmatrix}
    2y_1^2 & y_1(x_2-x_1) & 0 \\
    y_1(x_2-x_1) & -2(y_1^2-y^2_2) & 0\\
    0 & \ddots & 0\\
    \vdots & & y_{n-1}(x_{n}-x_{n-1}) \\
    0 & \hdots & -2y^2_{n-1}
\end{bmatrix}
\]

This is equivalent to the result we get from combining the matrix form of $H$ with the values for $\dot{x}$ and $\dot{y}$
\[\dot{H} = \begin{bmatrix}
    \dot{x}_{1} & \dot{y}_{1} & 0  & \dots & 0 \\
    \dot{y}_{1} & \dot{x}_{2} & \dot{y}_{2} & \dots & 0 \\
     & & \ddots & \\
     & & \dot{y}_{n-2} & \dot{x}_{n-1} & \dot{y}_{n-1}\\
    0 & \hdots & & \dot{y}_{n-1} & \dot{x}_{n}

\end{bmatrix}\]

In total, we conclude that the double bracket notation of the Toda lattice used in other papers is analogous to the system of ODEs. We use the equivalent system of ODEs to construct the analog sorter.


\documentclass[11pt]{article}

\usepackage{array}
\usepackage{clrscode3e}
\usepackage{amsmath}
\usepackage{kbordermatrix}

\setlength{\parindent}{0em}
\setlength{\parskip}{1em}

\begin{document}

\title{Efficient Calculation of Interaction Features on Sparse Matrices}
\author{Andrew Nystrom}
\date{}

\maketitle

\begin{abstract}%   <- trailing '%' for backward compatibility of .sty file
FILL THIS IN
\end{abstract}

\section{Introduction}

Introduction
Interaction features are a way of capturing correlations between features in a machine
learning setting. A feature vector $\vec{x}$ of dimensionality $D$ has second degree interaction features
$\{x_i \cdot x_j : i, j \in \{0,1,..., D-1\} \land i < j\}$,
so a $D$ dimensional vector has $\binom{D}{2} = \frac{D^2-D}{2}$ second degree interaction features. A naive
approach to calculating these features is to simply iterate through the combinations of the column indices.
For a sparse vector, many of the resulting interaction features would be zero, and could therefore be ignored.
This work describes a method to efficiently calculate second degree interaction features for a sparse matrix
that has time and space complexities that decrease quadratically with the density of the input matrix with respect to the naive approach.

$\{x_i \cdot x_j : i, j \in \{0,1,..., D-1\} \land i \le j\}$  \\
\\
$\{(i,j) : i, j \in \{0,1,..., D-1\} \land i \le j\} \to \{0, 1, ..., \frac{D^2+D}{2}-1\}$ \\

$i$
$j$

$\vec{x}$

\begin{codebox}
\Procname{$\proc{Dense Polynomials}(A)$}
    \zi $N \gets$ row count of $A$
    \zi $D \gets$ column count of $A$
    \zi $B$ $\gets$ Matrix of size $N \times \frac{D^2+D}{2}$
    \zi \For $\id{row}$ in $A$ \Do
    \zi     $k \gets 0$
    \zi     \For $\i \gets 0 \To D$ \Do
    \zi         \For $\j \gets i \To D$ \Do
    \zi             $B[i, k] \gets \id{row}[i] \cdot \id{row}[j]$
    \zi             $k \gets k + 1$
                \End
            \End
       	\End
\end{codebox}

$[a, b, c, d, e]$

$[ab, ac, ad, ae, bc, bd, be, cd, ce, de]$

\renewcommand{\kbldelim}{(}% Left delimiter
\renewcommand{\kbrdelim}{)}% Right delimiter
\[
  \kbordermatrix{
        & x_0 & x_1 & x_2 & x_3 & x_4 \\
    x_0 & 0 & 1 & 2 & 3 & 4\\
    x_1 & - & 5 & 6 & 7 & 8\\
    x_2 & - & - & 9 & 10 & 11\\
    x_3 & - & - & - & 12 & 13\\
    x_4 & - & - & - & - & 14\\
  }
\]

$\text{polynomial-index}(i, j | D) = \frac{2Di-i^2+2j-3i-2}{2}+i+1$

%Consider the following matrix:

%\[ \left( \begin{array}{cccc}
%3 & 0 & 0 & 3 \\
%0 & 4 & 2 & 0 \\
%1 & 2 & 0 & 3 \end{array} \right)\]
\[ \left( \begin{array}{ccccc}
A & b & c & d \end{array} \right)\]

%\[ \left( \begin{array}{cccccccccc}
%1 & 2 & 0 & 3 & 0 \end{array} \right)\]

%If each row is an instance vector, then the interaction feature matrix is

%\[ \left( \begin{array}{cccccc}
%0 & 0 & 9 & 0 & 0 & 0 \\
%0 & 0 & 0 & 8 & 0 & 0 \\
%2 & 0 & 3 & 0 & 6 & 0 \end{array} \right)\]

%Note that there are $\binom{D}{2} = \frac{D^2-D}{2}$ columns in the interaction matrix, which is D choose 2,
%since we generate products of all combinations of 2 features in the original matrix.
%Which column corresponds with which product pair is not important so long as it’s
%consistent. In this example, each column in the interaction matrix corresponded to the
%following product pairs: $(1, 2),  (1, 3),  (1, 4),  (2, 3), ( 2, 4), (3, 4)$.

%Notice that the interaction matrix contains many zero entries. This is of course because
%the original matrix contained zero entries, so the interaction features, which are
%products of pairs of features, contain many zeros. This means that the only products that
%need to actually be calculated or those for which both features in the combination are
%nonzero. If the original matrix is sparse and represented in a sparse matrix format
%(e.g. compressed sparse row), a list of nonzero column indices are stored for each row
%and are easily retrievable in $O(1)$ time. Interaction features can be generated from this
%list via the following method.

\section{Approach}
Let the list of nonzero columns for a given row $\vec{x}$ be denoted by $N_{zc}$. The nonzero second degree
interaction features are simply the products of all combinations of two elements whose
columns are in $N_{zc}$. However, to properly place an interaction feature into the correct column, a mapping from the column
index pairs of $N_{zc}$ into the columns of the interaction matrix is needed. The mapping is
from the space (a, b) where a,b are in $1,2,..., D$ onto the space $1,2,..., \frac{D^2-D}{2}$. This
is isomorphic to mapping the coordinates of the upper triangle of a matrix onto a flat
list. The following is a proof by construction for such a mapping.

INSERT JOHN'S PROOF HERE

With this mapping, an algorithm for generating second degree interaction features on a
matrix $A$ can be formulated as follows:

\begin{codebox}
\Procname{$\proc{Sparse Polynomials}(A)$}
    \zi $\func{map}(a, b) = \frac{2Da-a^2+2b-3a-2}{2}+a+1$
    \zi $N \gets$ row count of $A$
    \zi $D \gets$ column count of $A$
    \zi $B$ $\gets$ Compressed Sparse Row Matrix of size $N \times \frac{D^2+D}{2}$
    \zi \For $\id{row}$ in $A$ \Do
    \zi     $N_{zc} \gets$ nonzero columns of $row$
    \zi     \For $i \gets 0 \To |N_{zc}|$ \Do
    \zi         \For $j \gets i \To |N_{zc}|$ \Do
    \zi             $k \gets \func{map}(i, j)$
    \zi             $r \gets$ index of $\id{row}$
    \zi             $B[r, k] \gets \id{row}[i] \cdot \id{row}[j]$
                \End
            \End
       	\End
\end{codebox}

\section{Complexity Analysis}
Assume that A is a matrix with sparsity $0 < d < 1$, $N$ rows, and $D$ columns. Finding
interaction features with the proposed algorithm has time and space complexity
$\func{O}(d N D^2)$,
whereas a naive approach of using non-sparse matrices and multiplying all column
combinations has time and space complexity $\func{O}(N D^2)$. The algorithm is therefore an
improvement by a factor of the density factor of $A$.

This can represent a large gain in speed and time. For example, the 20 Newsgroups dataset
has density $d$ of 0.12 when its unigrams are represented in a vector space model. This
means the proposed approach would take less than $\frac{1}{8}$ time and memory.

The real benefit of this method is revealed when the average complexity is analysed. The
number of interactions calculated for a given row are $\binom{|N_{zc}|}{2}$. If the matrix has
density $d$, then on average, $N_{zc} = D d$, so the number of interaction features
calculated in total is

\begin{align*}
N \binom{d D}{2} &= N \frac{(Dd)!}{2!(Dd-2)!}\\
    \\
    &= N \frac{(D^2d^2-Dd)}{2}
\end{align*}

This means that the average complexity decreases quadratically with the density.

\section{Future Work}
The approach for generating second degree interaction features required a mapping from
combinations of two to the space $1,2,...,\frac{D^2-D}{2}$, which is isomorphic to a mapping from
the indices of an upper triangular matrix to the indices of a flat list of the same size.
To generate third degree interaction features, a mapping from combinations of three
$(a,b,c)$ to the space $1,2,...\frac{D^3-3D^2+2D}{6}$ (which is $\binom{D}{3}$), or the upper $3$-simplex of a tensor to a flat
list of the same size $\frac{D^3-3D^2+2D}{6}$ would be required. In general, for interaction
features of degree k, the upper $k$-simplex of a $k$-dimensional tensor must be mapped to the
space $1,2,...\frac{D!}{k!(D-k)!}$. A similar approach for finding these mappings could be taken
as the one used here for $k=2$.

Motivation for deriving mapping functions for higher orders
of interaction features is that the average complexity of generating degree $k$ interaction
features is $N \binom{Dd}{k}$, which decreases polynomially with respect to k compared to
generating the features naively.

\vskip 0.2in
\bibliography{sample}

\end{document}


The is the first paragraph of the Softcover article template. It shows how to write a document in \PolyTeX, a subset of the \LaTeX\ typesetting language optimized for writing ebooks.\footnote{Pronunciations of ``LaTeX'' differ, but \emph{lay}-tech is the one I prefer.}

This is the second paragraph, showing how to \emph{emphasize} text. You can also make text \textbf{bold}.

\section{A section}
\label{sec:a_section}

This is a section. We'll take a look at some of the features supported by Softcover.

\subsection{Source code}

In plain Markdown, you can typeset code samples and other verbatim text using four spaces of indentation:

\begin{verbatim}
def hello
  puts "hello, world"
end
\end{verbatim}

Softcover also supports GitHub-style ``code fencing'' with language-specific syntax highlighting:

%= lang:ruby
\begin{code}
# "Hello, world!" in Ruby.
def hello
  puts "hello, world!"
end
\end{code}

The second of these can be combined with Softcover's \kode{codelisting} environment to make code listings via embedded \LaTeX, as shown in Listing~\ref{code:hello}.

\begin{codelisting}
\codecaption{Hello, world.}
\label{code:hello}
%= lang:ruby
\begin{code}
# "Hello, world!" in Ruby.
def hello
  puts "hello, world!"
end
\end{code}
\end{codelisting}

\subsection{Mathematics}

Softcover supports mathematical typesetting via embedded \LaTeX. This includes both inline math, such as \( \phi^2 - \phi - 1 = 0, \) and centered math, such as
\[ \phi^2 - \phi - 1 = 0. \]
Softcover also supports numbered equations via embedded \LaTeX, as seen in Eq.~\eqref{eq:phi} and Eq.~\eqref{eq:gauss}.

\begin{equation}
\label{eq:phi}
\phi = \frac{1+\sqrt{5}}{2} \approx 1.618
\end{equation}

\begin{equation}
\label{eq:gauss}
\mathbf{\nabla}\cdot\mathbf{B} = 0 \qquad\mbox{Gauss's law}
\end{equation}

\section{Images and tables}
\label{sec:images_and_tables}

This is the second section. Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

\subsection{Images}

Softcover supports the inclusion of images, like this:

\image{images/figures/01_michael_hartl_headshot.jpg}

Using \LaTeX\ labels, you can also include a caption (as in Figure~\ref{fig:captioned_image}) or just a figure number (as in Figure~\ref{fig:figure_number}).

\begin{figure}[h]
\begin{center}
\image{images/figures/01_michael_hartl_headshot.jpg}
\end{center}
\caption{Some dude.\label{fig:captioned_image}}
\end{figure}

\begin{figure}[h]
\begin{center}
\image{images/figures/01_michael_hartl_headshot.jpg}
\end{center}
\caption{\label{fig:figure_number}}
\end{figure}

\subsection{Tables}

Softcover supports raw tables via the \LaTeX\ \kode{table} or \kode{longtable} environments.

\begin{longtable}{|l|l|l|l|}
\hline
\textbf{HTTP request} & \textbf{URL} & \textbf{Action} & \textbf{Purpose}\\
\kode{GET} & /users & \kode{index} & page to list all users\\
\kode{GET} & /users/1 & \kode{show} & page to show user with id \kode{1}\\
\kode{GET} & /users/new & \kode{new} & page to make a new user\\
\kode{POST} & /users & \kode{create} & create a new user\\
\kode{GET} & /users/1/edit & \kode{edit} & page to edit user with id \kode{1}\\
\kode{PATCH} & /users/1 & \kode{update} & update user with id \kode{1}\\
\kode{DELETE} & /users/1 & \kode{destroy} & delete user with id \kode{1}\\
\hline
\end{longtable}

Softcover also supports \kode{tabular} environments, as shown in Table~\ref{table:figure_placement}.

\begin{table}
\caption{Options for a float placement specifier.\label{table:figure_placement}}
\begin{tabular}{l|l}
\textbf{Specifier} & \textbf{Placement} \\ \hline
\kode{h} & Place the float \emph{approximately} here \\
\kode{h!} & Place the float \emph{(almost) exactly} here \\
\kode{t} & Place at the top of the page \\
\kode{b} & Place at the bottom of the page \\
\kode{p} & Put on a special page for floats only
\end{tabular}
\end{table}

\section{Final section}

This is the final section. The previous sections were Section~\ref{sec:a_section} and Section~\ref{sec:images_and_tables}.


\section{The predicate \Unchanged}
\Label{sec:unchanged}

Many of the algorithms in this section iterate sequentially over one or several sequences.
For the verification of such algorithms it is often important to express that a section
of an array, or the complete array, have remained \emph{unchanged}.
As this cannot always be expressed by an \inl{assigns} clause,
we introduce in the following listing the overloaded predicate
\logicref{Unchanged}.
The expression \inl{Unchanged\{K,L\}(a,m,n)} is true if the range
\inl{a[m..n-1]} in state~\inl{K} is element-wise equal to that range in state~\inl{L}.

\input{Listings/Unchanged.acsl.tex}

In some situations we use the predicate \ArrayUpdate,
which relies on the predicate \Unchanged and the the logic function \logicref{At},
to concisely describe which parts of an array have changed or remained unchanged
when updating an individual array element.

\input{Listings/ArrayUpdate.acsl.tex}

\clearpage

In the following listing we show a few lemmas for \logicref{Unchanged}
that we need for the verification of various algorithms.

\input{Listings/UnchangedLemmas.acsl.tex}

\begin{itemize}
\item
Lemma~\logicref{UnchangedShrink} states that if the range~\inl{a[m..n-1]} does
not change when going from state~\inl{K} to state~\inl{L}, then~\inl{a[p..q-1]}
does not change either, provided the latter is a subrange of the
former, i.e.\ provided $0 \leq m \leq p \leq q \leq n$ holds.

\item
Lemma~\logicref{UnchangedExtend} expresses the simple fact that ``unchangedness'' is an inductive property.

\item
Lemma~\logicref{UnchangedShift} states how \Unchanged behaves under pointer additions.

\item
Lemmas~\logicref{UnchangedSymmetric} and~\logicref{UnchangedTransitive} express respectively
the symmetry and transitivity of \Unchanged with respect to program states.

\end{itemize}

\clearpage


\chapter{Frequentist statistics}

Attempts have been made to devise approaches to statistical inference that avoid treating parameters like random variables, and which thus avoid the use of priors and Bayes rule. Such approaches are known as \textbf{frequentist statistics}, \textbf{classical statistics} or \textbf{orthodox statistics}. Instead of being based on the posterior distribution, they are based on the concept of a sampling distribution.

\section{Sampling distribution of an estimator}
In frequentist statistics, a parameter estimate $\hat{\vec{\theta}}$ is computed by applying an \textbf{estimator} $\delta$ to some data $\mathcal{D}$, so $\hat{\vec{\theta}}=\delta(\mathcal{D})$. The parameter is viewed as fixed and the data as random, which is the exact opposite of the Bayesian approach. The uncertainty in the parameter estimate can be measured by computing the \textbf{sampling distribution} of the estimator. To understand this

\subsection{Bootstrap}
We might think of the bootstrap distribution as a “poor man’s” Bayes posterior, see (Hastie et al. 2001, p235) for details.

\subsection{Large sample theory for the MLE *}

\section{Frequentist decision theory}
In frequentist or classical decision theory, there is a loss function and a likelihood, but there is no prior and hence no posterior or posterior expected loss. Thus there is no automatic way of deriving an optimal estimator, unlike the Bayesian case. Instead, in the frequentist approach, we are free to choose any estimator or decision procedure $f: \mathcal{X} \rightarrow \mathcal{Y}$ we want.

Having chosen an estimator, we define its expected loss or \textbf{risk} as follows:
\begin{equation}\begin{split}
R_{\mathrm{exp}}(\theta,f) & \triangleq \mathbb{E}_{p(\tilde{\mathcal{D}}|\theta^*)}[L(\theta^*, f(\tilde{\mathcal{D}}))] \\
    & =\int L(\theta^*, f(\tilde{\mathcal{D}}))p(\tilde{\mathcal{D}}|\theta^*)\mathrm{d}\tilde{\mathcal{D}}
\end{split}\end{equation}
where˜$\tilde{\mathcal{D}}$ is data sampled from “nature’s distribution”, which is represented by parameter $\theta^*$. In other words, the expectation is wrt the sampling distribution of the estimator. Compare this to the Bayesian posterior expected loss:
\begin{equation}
\rho(f|\mathcal{D},)
\end{equation}

\section{Desirable properties of estimators}

\section{Empirical risk minimization}

\subsection{Regularized risk minimization}

\subsection{Structural risk minimization}

\subsection{Estimating the risk using cross validation}

\subsection{Upper bounding the risk using statistical learning theory *}

\subsection{Surrogate loss functions}
\label{sec:Surrogate-loss-functions}

\textbf{log-loss}
\begin{equation}\label{eqn:log-loss}
L_{\mathrm{nll}}(y,\eta)=-\log p(y|\vec{x},\vec{w})=\log(1+e^{-y\eta})
\end{equation}

\section{Pathologies of frequentist statistics *}


\documentclass[12pt]{article}

\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage[latin1]{inputenc}
\usepackage{geometry}
\usepackage{graphicx}
\usepackage{titlesec}
\usepackage{pagecolor}

\date{}

\begin{document}
\pagecolor{white}

\section{Question 1}

$$n + d = 2011 <=> d = 2011 - n$$
$$\frac{n} {d} < \frac{1}{3}$$
$$\frac{n}{2011 - n} <  \frac{1}{3}$$
$$3n < 2011 - n$$
$$4n < 2011$$
$$n < 502.75$$
Due to the constraint that $n$ and $d$ are positive integers, the maximum value for $n$ is $502$. Therefore $d = 2011 - n = 2011 - 502 = 1509$.

\section{Question 2}

Let X be the random variable modeling the outcome of the bet, where $x_1$ is the amount by which one's money, $m$, is increased in the event that the bet is favorable, and $x_2$ is the amount by which $m$ is increased if the bet is not favorable. We know that $x_1$ and $x_2$ are complements. Therefore, the expectation of such a variable $X$ is:

$$E[X; m] = \sum_i x_iP(x_i) = 3mP(x_1) - mP(x_2) = 3mP(x_1) - m(1 - P(x_1)) = 4mP(x_1) - m$$
$$= m(4P(x_1) - 1)$$
In order for the bet to be favorable, we would need the expectation of $X$ to be positive, i.e.,:

$$E[X; m] = m(4P(x_1) - 1) > 0$$
$$P(x_1) > 0.25$$
Let $n$ be the number of face cards in the first 5 cards, and let $c$ be the event where the sixth card is a face card, i.e., one's money is increased. Then:

$$P(c | n = i) = \frac{12 - i}{47} $$
Therefore:
$$P(c | n = 0) = 0.255$$
$$P(c | n = 1) =  0.234$$
Thus we can see that the only way the bet is favorable (i.e., $P(x_1) = P(c | n = i) > 0.25$) is when $i = 0$. We now need to figure out the probability of such an occurrence:

$$P(n = 0) = \frac{{12 \choose 0}{40 \choose 5}}{{52 \choose 5}} = 0.25318$$

\section{Question 3}

We have the following parameters for the Poisson distribution:

$$\lambda = 100 / 5 = 20$$
$$\alpha = 0.05$$
$$n = 60$$
$$k = n\lambda = (60)(20) = 1200$$

$$CI = [\frac{1}{2}\chi^2(\frac{\alpha}{2}; 2k), \frac{1}{2}\chi^2(1 - \frac{\alpha}{2}; 2k + 2)]
= [\frac{1}{2}\chi^2(0.025; 2400), \frac{1}{2}\chi^2(0.975; 2402)]
= [1133.1, 1269.9]$$

\section{Question 4}

This is a one-sided Z-test.

$$Z^* = \frac{\hat{p} - p_0}{\sqrt{\frac{p_0(1-p_0)}{n}}}  = \frac{(\frac{26}{200} - 0.1)}{\sqrt{\frac{0.1(0.9)}{200}}} = 1.414$$
$$p = P(Z \geq Z^*) = 0.0787 > 0.05$$
Therefore we fail to reject the null hypothesis.

\end{document}