%%% lecture 06 %%%
\documentclass{beamer}
\usepackage[utf8]{inputenc}
\usepackage{algorithm2e, amsmath, amssymb, amsfonts, graphicx}
% allow section.equation numbering
\numberwithin{equation}{section}
% use boadilla theme
\usetheme{Boadilla}
% remove navigation symbols
\usenavigationsymbolstemplate{}
% get numbered figure captions
\setbeamertemplate{caption}[numbered]
% changes itemize to circle + other things
\useoutertheme{split}
\useinnertheme{circles}

% command for the title string. change for each lecture
\newcommand{\lecturetitle}{Intro to Optimization, Part 1}
% allow automatic alert-highlighted references and hyperlinks
\newcommand{\aref}[1]{\alert{\ref{#1}}}
\newcommand{\ahref}[2]{\href{#1}{\alert{#2}}}
% title page stuff. brackets content displayed in footer bar
\title[\lecturetitle]{\lecturetitle}
% metadata. content in brackets is displayed in footer bar
\author[Derek Huang (BAC Advanced Team)]{Derek Huang}
\institute{BAC Advanced Team}
\date{December 30, 2021}

% change "ball" bullet to numbered bullet and section title for section
\setbeamertemplate{section in toc}{\inserttocsectionnumber.~\inserttocsection}
% change ball to gray square (copied from stackoverflow; \par needed for break)
\setbeamertemplate{subsection in toc}{
    \hspace{1.2em}{\color{gray}\rule[0.3ex]{3pt}{3pt}}~\inserttocsubsection\par}
% use default enumeration scheme
\setbeamertemplate{enumerate items}[default]
% required line that fixes the problem of \mathbf, \bf not working in beamer
% for later (post-2019) TeX Live installations. see the issue on GitHub:
% https://github.com/josephwright/beamer/issues/630
\DeclareFontShape{OT1}{cmss}{b}{n}{<->ssub * cmss/bx/n}{}

\begin{document}

% title slide
\begin{frame}
    \titlepage
    \centering
    % relative path may need to be updated depending on .tex file location
    \includegraphics[scale = 0.1]{../bac_logo1.png}
\end{frame}

% table of contents slide
\begin{frame}{Overview}
    \tableofcontents
\end{frame}

\section{Convex optimization}

\begin{frame}{Motivation}
    \begin{itemize}
        \item
        Let $ \mathbf{X} \in \mathbb{R}^{N \times d} $ be the input matrix,
        $ \mathbf{y} \in \mathbb{R}^N $ the response vector. Consider
        fitting a linear regression model s.t. we minimize the
        absolute value of the residuals. Optimal $ \hat{\mathbf{w}} \in
        \mathbb{R}^d $, $ \hat{b} \in \mathbb{R} $ solve
        \begin{equation*}
            \begin{array}{ll}
                \displaystyle\min_{\mathbf{w}, b} &
                    \Vert\mathbf{y} - \mathbf{Xw} - b\mathbf{1}\Vert_1
            \end{array}
        \end{equation*}

        \item
        Model parameter estimation is an optimization problem.

        \item
        Quite hard to solve. May be recast as the linear program (LP)
        \cite{bv_convex_opt}
        \begin{equation*}
            \begin{array}{ll}
                \displaystyle\min_{\mathbf{w}, b, \mathbf{t}} &
                    \mathbf{1}^\top\mathbf{t} \\
                \text{s.t.} &
                    -\mathbf{t} \preceq \mathbf{y} - \mathbf{Xw} - b\mathbf{1}
                    \preceq \mathbf{t} \\
                & \mathbf{t} \succeq \mathbf{0}
            \end{array}
        \end{equation*}
        Here $ \mathbf{t} \in \mathbb{R}^N $. Algorithms for solving LPs
        are quite reliable \cite{bv_convex_opt}.
    \end{itemize}
\end{frame}

\subsection{Convex sets and functions}

\begin{frame}{Convex sets and functions}
    \begin{itemize}
        \item
        Optimization problems broadly categorized as convex vs. nonconvex
        $ \Rightarrow $ we need to know what convexity means.

        \item
        \textit{Definition.} Let $ \mathbf{x}_1, \mathbf{x}_2 \in
        \mathbb{R}^n $, $ \mathbf{x}_1 \ne \mathbf{x}_2 $. The \textit{line}
        passing through points $ \mathbf{x}_1, \mathbf{x}_2 $ is
        $ \{\theta\mathbf{x}_1 + (1 - \theta)\mathbf{x}_2 :
        \theta \in \mathbb{R}\} $. The \textit{line segment} connecting
        points $ \mathbf{x}_1, \mathbf{x}_2 $ is $ \{\theta\mathbf{x}_1 +
        (1 - \theta) \mathbf{x}_2 : \theta \in [0, 1]\} $
        \cite{bv_convex_opt}.
    \end{itemize}
    \begin{figure}
        \centering
        \includegraphics[scale = 0.3]{bv_fig_2.1.png}
        % remove excess space
        \vspace{-10 pt}
        \caption{
            The $ \mathbf{x}_1, \mathbf{x}_2 $ line and line segment
            parametrized by $ \theta $\footnote{
                Figure 2.1 from Boyd and Vandenberghe's
                \textit{Convex Optimization}.
            }.
        }
    \end{figure}
\end{frame}

\begin{frame}{Convex sets and functions}
    \begin{itemize}
        \item
        \textit{Definition.} A set $ C \subseteq \mathbb{R}^n $ is
        \textit{affine} if $ \forall \mathbf{x}, \mathbf{y} \in C $,
        $ \alpha \in \mathbb{R} $, $ \alpha\mathbf{x} + (1 - \alpha)\mathbf{y}
        \in C $, i.e. $ C $ contains the $ \mathbf{x}, \mathbf{y} $ line.

        \item
        \textit{Definition.} Let $ C \subseteq \mathbb{R}^n $. The
        \textit{affine hull} of $ C $, denoted $ \operatorname{aff} C $, is
        s.t. $ \operatorname{aff} C \triangleq \{
            \sum_{i = 1}^k\theta_i\mathbf{x}_i : \mathbf{x}_1, \ldots
            \mathbf{x}_k \in C, \theta_1, \ldots \theta_k \in \mathbb{R},
            \sum_{i = 1}^k\theta_i = 1
        \} $ \cite{bv_convex_opt}.

        \item
        \textit{Definition.} A set $ C \subseteq \mathbb{R}^n $ is
        \textit{convex} if $ \forall \mathbf{x}, \mathbf{y} \in C $,
        $ \alpha \in [0, 1] $,
        $ \alpha\mathbf{x} + (1 - \alpha)\mathbf{y} \in C $, i.e. $ C $
        contains the $ \mathbf{x}, \mathbf{y} $ line segment.
    \end{itemize}
    \begin{figure}
        \centering
        \includegraphics[scale = 0.2]{bv_fig_2.2.png}
        % remove excess space
        \vspace{-10 pt}
        \caption{
            Convex and nonconvex sets. Only the leftmost set is
            convex\footnote{
                Figure 2.2 from Boyd and Vandenberghe's
                \textit{Convex Optimization}.
            }.
        }
        % remove excess space
        \vspace{-10 pt}
    \end{figure}
    \begin{itemize}
        \item
        \textit{Definition.} Let $ C \subseteq \mathbb{R}^n $. The
        \textit{convex hull} of $ C $, denoted $ \operatorname{conv} C $, is
        s.t. $ \operatorname{conv} C \triangleq \{
            \sum_{i = 1}^k\theta_i\mathbf{x}_i : \mathbf{x}_1, \ldots
            \mathbf{x}_k \in C, \theta_1, \ldots \theta_k \in [0, 1],
            \sum_{i = 1}^k\theta_i = 1
        \} $.
    \end{itemize}
    % adjust spacing since there is a footnote
    \bigskip
\end{frame}

\begin{frame}{Convex sets and functions}
    \begin{itemize}
        \item
        If $ C $ affine, then $ \operatorname{aff} C = C $, while if $ C $
        convex, then $ \operatorname{conv} C = C $.
    \end{itemize}
    \begin{figure}
        \centering
        % remove excess space
        \vspace{-3 pt}
        \includegraphics[scale = 0.24]{bv_fig_2.3.png}
        % remove excess space
        \vspace{-10 pt}
        \caption{
            Examples of $ \mathbb{R}^2 $ convex hulls\footnote{
                Figure 2.3 from Boyd and Vandenberghe's
                \textit{Convex Optimization}.
            }.
        }
        % remove excess space
        \vspace{-10 pt}
    \end{figure}
    \begin{itemize}
        \item
        \textit{Definition.} Let $ C \subseteq \mathbb{R}^n $. The
        \textit{relative interior} of $ C $, denoted
        $ \operatorname{relint} C $, is s.t. $ \operatorname{relint} C
        \triangleq \{\mathbf{x} \in C : B(\mathbf{x}, r) \cap
        \operatorname{aff} C \subseteq C, r \in (0, \infty)\} $
        \cite{bv_convex_opt}. Here
        $ B(\mathbf{x}, r) \triangleq \{\mathbf{x}' \in \mathbb{R}^n :
        \Vert\mathbf{x}' - \mathbf{x}\Vert \le r\} $ for some norm
        $ \Vert\cdot\Vert $\footnote{
            All norms define the same relative interior \cite{bv_convex_opt}.
        }.

        \item
        \textit{Definition.} $ f : \mathcal{M} \rightarrow
        \mathbb{R} $, $ \mathcal{M} \subseteq \mathbb{R}^n $, is
        \textit{convex} if $ \mathcal{M} $ convex and if
        $ \forall \mathbf{x}, \mathbf{y} \in \mathcal{M} $,
        $ \forall \alpha \in [0, 1] $,
        $ f(\alpha\mathbf{x} + (1 - \alpha)\mathbf{y}) \le
        \alpha f(\mathbf{x}) + (1 - \alpha)f(\mathbf{y}) $.

        \item
        \textit{Definition.} $ f : \mathcal{M} \rightarrow \mathbb{R} $ is
        \textit{concave} if $ -f $ is convex.
    \end{itemize}

    % spacing for footnote
    \bigskip
\end{frame}

\begin{frame}{Convex sets and functions}
    \begin{itemize}
        \item
        Affine and linear functions are both convex and concave.
    \end{itemize}
    \begin{figure}
        \centering
        % remove extra space
        \vspace{-3 pt}
        \includegraphics[scale = 0.24]{bv_fig_3.1.png}
        % remove extra space
        \vspace{-5 pt}
        \caption{Graph of a convex function\footnote{
            Figure 3.1 from Boyd and Vandenberghe's
            \textit{Convex Optimization}.
        }. }
        % remove excess space
        \vspace{-10 pt}
    \end{figure}
    \begin{itemize}
        \item
        \textit{Examples.}
        \begin{itemize}
            \item
            \textit{Exponential.} $ \forall a \in \mathbb{R} $, $ e^{ax} $
            convex on $ \mathbb{R} $ \cite{bv_convex_opt}.

            \item
            \textit{Powers.} $ x^a $ convex on $ (0, \infty) $ if
            $ a \in (\infty, 0] \cup [1, \infty) $, concave if
            $ a \in [0, 1] $. $ \forall a \in [1, \infty) $, $ |x|^a $ convex
            on $ \mathbb{R} $ \cite{bv_convex_opt}.

            \item
            \textit{Logarithms.} $ \log x $ concave on $ (0, \infty) $.

            \item
            \textit{Norms.} Any norm on $ \mathbb{R}^n $ is convex
            \cite{bv_convex_opt}, e.g. $ \ell^p $-norm.
        \end{itemize}
    \end{itemize}

    % spacing for footnote
    \bigskip
\end{frame}

\begin{frame}{Convex sets and functions}
    \begin{itemize}
        \item
        \textit{Theorem.} Let $ f : \mathcal{M} \rightarrow \mathbb{R} $ be
        differentiable $ \forall \mathbf{x} \in \mathcal{M} $. $ f $
        convex $ \Leftrightarrow \mathcal{M} $ convex, $ \forall \mathbf{x},
        \mathbf{y} \in \mathcal{M} $, $ f(\mathbf{y}) \ge f(\mathbf{x}) +
        \nabla f(\mathbf{x})^\top(\mathbf{y} - \mathbf{x}) $
        \cite{bv_convex_opt}.
    \end{itemize}
    \begin{figure}
        \centering
        % remove extra space
        \vspace{-5 pt}
        \includegraphics[scale = 0.3]{bv_fig_3.2.png}
        % remove extra space
        \vspace{-10 pt}
        \caption{Convex function bounded below by tangent line\footnote{
            Figure 3.2 from Boyd and Vandenberghe's
            \textit{Convex Optimization}.
        }. }
        % remove excess space
        \vspace{-15 pt}
    \end{figure}
    \begin{itemize}
        \item
        \textit{Theorem.} Let $ f : \mathcal{M} \rightarrow \mathbb{R} $ be
        twice differentiable $ \forall \mathbf{x} \in \mathcal{M} $.
        $ f $ convex $ \Leftrightarrow \mathcal{M} $ convex, $ \forall
        \mathbf{x} \in \mathcal{M} $, $ \nabla^2f(\mathbf{x}) \succeq
        \mathbf{0} $ \cite{bv_convex_opt}.

        \item
        \textit{Remark.} If $ \mathcal{M} \subseteq \mathbb{R} $ convex,
        reduces to $ \forall x \in \mathcal{M}, f''(x) \ge 0 $.
    \end{itemize}

    % spacing for footnote
    \medskip

\end{frame}

\subsection{Optimization problems}

\begin{frame}{Optimization problems}
    \begin{itemize}
        \item
        \textit{Definition.} Let $ \mathcal{M}_f, \mathcal{M}_\mathbf{u},
        \mathcal{M}_\mathbf{v} \subseteq \mathbb{R}^n $. For
        $ f: \mathcal{M}_f \rightarrow \mathbb{R} $, $ \mathbf{u} :
        \mathcal{M}_\mathbf{u} \rightarrow \mathbb{R}^p $,
        $ \mathbf{v} : \mathcal{M}_\mathbf{v} \rightarrow \mathbb{R}^q $, a
        \textit{standard form} optimization problem is\footnote{
            Assume the problem is well-defined, i.e. \textit{feasible}
            (solvable) and bounded below.
        }
        \cite{bv_convex_opt}
        \begin{equation} \label{opt_prob_std}
            \begin{array}{ll}
                \displaystyle\min_\mathbf{x} & f(\mathbf{x}) \\
                \text{s.t.} & \mathbf{u}(\mathbf{x}) \preceq \mathbf{0} \\
                & \mathbf{v}(\mathbf{x}) = \mathbf{0}
            \end{array}
        \end{equation}
        $ \mathbf{x} \in \mathbb{R}^n $ is the \textit{optimization
        variable} \cite{bv_convex_opt}. $ \mathbf{u} \triangleq
        [ \ u_1 \ \ldots \ u_p \ ]^\top $ gives $ p $ \textit{inequality
        constraints}, $ \mathbf{v} \triangleq [ \ v_1 \ \ldots \ v_q \ ]^\top $
        gives $ q $ \textit{equality constraints}.

        \item
        \textit{Definition.} The \textit{domain} of (\aref{opt_prob_std}) is
        $ \tilde{\mathcal{M}} \triangleq \mathcal{M}_f \cap
        \mathcal{M}_\mathbf{u} \cap \mathcal{M}_\mathbf{v} \ne \emptyset $
        \cite{bv_convex_opt}.

        \item
        Problem is \textit{unconstrained} if no constraints,
        \textit{constrained} otherwise. Note maximization of $ f $
        equivalent to minimization of $ -f $.

        \item
        \textit{Definition.} (\aref{opt_prob_std}) is a
        \textit{convex optimization problem} if functions
        $ f, u_1, \ldots u_p $ are convex and functions $ v_1, \ldots v_q $
        are affine \cite{bv_convex_opt}.
    \end{itemize}

    % spacing for footnote
    \medskip
\end{frame}

\begin{frame}{Optimization problems}
    \begin{itemize}
        \item
        \textit{Examples.}
        \begin{itemize}
            \item
            \textit{Weighted linear least squares.} Let $ \mathbf{\Gamma}
            \triangleq \operatorname{diag}(\gamma_1, \ldots \gamma_N) \succ
            \mathbf{0} \in \mathbb{R}^{N \times N} $ be the data weighting
            matrix. The unconstrained problem to solve is
            \begin{equation*}
                \begin{array}{ll}
                    \displaystyle\min_{\mathbf{w}, b} &
                    \Vert
                        \mathbf{\Gamma}^{1 / 2}(\mathbf{y} - \mathbf{Xw} -
                        b\mathbf{1})
                    \Vert_2^2
                \end{array}
            \end{equation*}

            \item
            \textit{SVM dual problem.} Note $ \mathbf{X} \triangleq
            [ \ \mathbf{x}_1 \ \ldots \mathbf{x}_N \ ]^\top $. The problem is
            \begin{equation*}
                \begin{array}{ll}
                    \displaystyle\max_\alpha & \mathbf{1}^\top\alpha -
                    \frac{1}{2}\alpha^\top\mathbf{H}\alpha \\
                    \text{s.t.} & \alpha^\top\mathbf{y} = 0 \\
                    & \mathbf{0} \preceq \alpha \preceq C\mathbf{1}
                \end{array}
            \end{equation*}
            Here $ \alpha \in \mathbb{R}^N $,
            $ \mathbf{H} \in \mathbb{R}^{N \times N} $ is such that
            $ h_{ij} = y_iy_j\mathbf{x}_i^\top\mathbf{x}_j $, $ C > 0 $.

            \item
            Both problems are \textit{quadratic programs}. Quadratic programs
            have convex, quadratic objectives and affine constraints (if any)
            \cite{bv_convex_opt}.
        \end{itemize}
    \end{itemize}
\end{frame}

\subsection{Feasibility and optimality}

\begin{frame}{Feasibility and optimality}
    \begin{itemize}
        \item
        \textit{Definition.} Let $ \mathbf{u}, \mathbf{v},
        \tilde{\mathcal{M}} $ be defined as in (\aref{opt_prob_std}).
        $ \mathbf{x}' \in \tilde{\mathcal{M}} $ is \textit{feasible} if
        $ \mathbf{x}' \in \mathcal{X}^*
        \triangleq \{\mathbf{x} \in \tilde{\mathcal{M}} :
        \mathbf{u}(\mathbf{x}) \preceq \mathbf{0}, \mathbf{v}(\mathbf{x}) =
        \mathbf{0}\} $. $ \mathcal{X}^* $ is the \textit{feasible set}
        \cite{bv_convex_opt}.

        \item
        \textit{Definition.} The \textit{optimal value} $ p^* $
        of an optimization problem, as defined in (\aref{opt_prob_std}), is
        such that $ p^* = \inf\{f(\mathbf{x}) : \mathbf{x} \in
        \mathcal{X}^*\} $ \cite{bv_convex_opt}.

        \item
        \textit{Definition.} $ \mathbf{x}^* $ is \textit{[globally] optimal}
        if $ \mathbf{x}^* \in \mathcal{X}^* $ and $ f(\mathbf{x}^*) = p^* $
        \cite{bv_convex_opt}.

        \item
        \textit{Definition.} $ \mathbf{x}' $ is \textit{locally optimal} if
        $ \mathbf{x}' \in \mathcal{X}^* $ and $ \exists r \in (0, \infty) $
        such that $ f(\mathbf{x}') = \inf\{
            f(\mathbf{x}) : \mathbf{x} \in \mathcal{X}^*,
            \Vert\mathbf{x}' - \mathbf{x}\Vert_2 \le r
        \}$ \cite{bv_convex_opt}.

        \item
        \textit{Theorem.} Suppose (\aref{opt_prob_std}) is a \alert{convex}
        optimization problem. If $ \mathbf{x}' $ is locally optimal, then
        $ \mathbf{x}' $ is [globally] optimal.

        \item
        \textit{Theorem.} Suppose (\aref{opt_prob_std}) is a \alert{convex}
        optimization problem and objective $ f $ differentiable.
        $ \mathbf{x}^* \in \tilde{\mathcal{M}} $ optimal
        $ \Leftrightarrow \mathbf{x}^* \in \mathcal{X}^* $,
        $ \forall \mathbf{x} \in \mathcal{X}^* $,
        $ \nabla f(\mathbf{x}^*)^\top(\mathbf{x} - \mathbf{x}^*) \ge 0 $.
        If no constraints, reduces to $ \nabla f(\mathbf{x}^*) = \mathbf{0} $
        \cite{bv_convex_opt}.
    \end{itemize}
\end{frame}

\section{Duality}

\subsection{The Lagrangian dual}

\begin{frame}{The Lagrangian dual}
    \begin{itemize}
        \item
        \textit{Definition.} The \textit{Lagrangian} $ \mathcal{L}_f :
        \tilde{\mathcal{M}} \times \mathbb{R}^p \times \mathbb{R}^q \rightarrow
        \mathbb{R} $ of (\aref{opt_prob_std}) is s.t.
        \begin{equation} \label{std_lagrangian}
            \mathcal{L}_f(\mathbf{x}, \lambda, \nu) \triangleq f(\mathbf{x}) +
            \lambda^\top\mathbf{u}(\mathbf{x}) + \nu^\top\mathbf{v}(\mathbf{x})
        \end{equation}
        $ \lambda \in \mathbb{R}^p $, $ \nu \in \mathbb{R}^q $ are the
        \textit{Lagrange multipliers} or \textit{dual variables}.

        \item
        \textit{Definition.} The \textit{[Lagrangian] dual} $ f_d :
        \tilde{\mathcal{M}}_d \rightarrow \mathbb{R} $ of
        (\aref{opt_prob_std}), is s.t.
        \begin{equation} \label{std_dual}
            f_d(\lambda, \nu) \triangleq
            \inf_{\mathbf{x} \in \tilde{\mathcal{M}}}
            \{\mathcal{L}_f(\mathbf{x}, \lambda, \nu)\}
        \end{equation}
        Here $ \tilde{\mathcal{M}}_d \subseteq \mathbb{R}^p \times
        \mathbb{R}^q $. $ f_d $ is \alert{always} concave, even if the
        problem (\aref{opt_prob_std}) is nonconvex \cite{bv_convex_opt}.
        $ (\lambda, \nu) \in \tilde{\mathcal{M}}_d $ with $ \lambda \succeq
        \mathbf{0} $ is called \textit{dual feasible} \cite{bv_convex_opt}.

        \item
        \textit{Theorem.} $ \forall (\lambda, \nu) \in \tilde{\mathcal{M}}_d $,
        $ \lambda \succeq \mathbf{0} $, $ f_d(\lambda, \nu) \le p^* $.

        \item
        For any dual feasible $ (\lambda, \nu) $, $ f_d(\lambda, \nu) $
        gives a lower bound to the optimal value of the original optimization
        problem.
    \end{itemize}

    % more footnote spacing
    \medskip
\end{frame}

\begin{frame}{The Lagrangian dual}
    \begin{itemize}
        \item
        If we allow infinite values, (\aref{opt_prob_std}) can be written as
        the unconstrained
        \begin{equation*}
            \begin{array}{ll}
                \displaystyle\min_\mathbf{x} &
                \displaystyle f(\mathbf{x}) + \tilde{\mathbb{I}}^\infty_{\{
                    \mathbf{x}' \in \mathbb{R}^p : \mathbf{x}' \preceq
                    \mathbf{0}
                \}}\circ\mathbf{u}(\mathbf{x}) +
                \tilde{\mathbb{I}}^\infty_{\{\mathbf{0}\}} \circ
                \mathbf{v}(\mathbf{x})
            \end{array}
        \end{equation*}
        % \displaystyle used inline to raise the superscript more
        $ \displaystyle\tilde{\mathbb{I}}_A^\infty $  is such that for set
        $ A $, $ \displaystyle\tilde{\mathbb{I}}_A^\infty(x) = 0 $ if
        $ x \in A $, else
        $ \displaystyle\tilde{\mathbb{I}}_A^\infty(x) = \infty $.

        \item
        $ \lambda^\top\mathbf{u} $, $ \nu^\top\mathbf{v} $ are linear
        underestimators of $ \displaystyle\tilde{\mathbb{I}}^\infty_{\{
            \mathbf{x}' \in \mathbb{R}^p : \mathbf{x}' \preceq \mathbf{0}
        \}}\circ\mathbf{u} $, $ \displaystyle
        \tilde{\mathbb{I}}^\infty_{\{\mathbf{0}\}}\circ\mathbf{v} $ when
        $ \lambda \succeq \mathbf{0} $, intuitively justifying why $ f_d $
        yields a lower bound for $ p^* $ \cite{bv_convex_opt}.

        \item
        A natural question is to find the closest underestimator to the
        original problem, i.e. $ \lambda^*, \nu^* $ s.t.
        $ p^* - f_d(\lambda^*, \nu^*) $ is minimized.

        \item
        \textit{Definition.} Let $ f_d $ be the dual for
        (\aref{opt_prob_std}). The \textit{dual problem} for
        (\aref{opt_prob_std}) is
        \begin{equation} \label{opt_prob_std_dual}
            \begin{array}{ll}
                \displaystyle\max_{\lambda, \nu}& f_d(\lambda, \nu) \\
                \text{s.t.} & \lambda\succeq \mathbf{0}
            \end{array}
        \end{equation}
        $ \lambda^*, \nu^* $ is \textit{dual optimal} if optimal for
        (\aref{opt_prob_std_dual}). (\aref{opt_prob_std}) is the
        \textit{primal problem}.
    \end{itemize}

    % spacing for the footnote
    \bigskip
\end{frame}

\begin{frame}{The Lagrangian dual}
    \begin{itemize}
        \item
        \textit{Example.} Consider solving an undetermined linear system with
        a minimum $ \ell^2 $-norm solution\footnote{
            Squared $ \ell^2 $-norm is differentiable and does not change the
            solution.
        }. I.e. for $ q < n $, we want to
        solve
        \begin{equation*}
            \begin{array}{ll}
                \displaystyle\min_\mathbf{x} & \Vert\mathbf{x}\Vert_2^2 \\
                \text{s.t.} & \mathbf{Ax} = \mathbf{b}
            \end{array}
        \end{equation*}
        Here $ \mathbf{A} \in \mathbb{R}^{q \times n} $, $ \mathbf{b} \in
        \mathbb{R}^q $. The Lagrangian $ \mathcal{L} : \mathbb{R}^n \times
        \mathbb{R}^q \rightarrow \mathbb{R} $ is s.t.
        $ \mathcal{L}(\mathbf{x}, \nu) \triangleq \mathbf{x}^\top\mathbf{x}
        + \nu^\top(\mathbf{Ax} - \mathbf{b}) $. Fixing $ \nu $, $ \mathcal{L} $
        is convex in $ \mathbf{x} $, so at its minimizer $ \mathbf{x}_\nu $,
        $ \nabla_\mathbf{x}\mathcal{L}(\mathbf{x}_\nu, \nu) =
        2\mathbf{x}_\nu + \mathbf{A}^\top\nu = \mathbf{0} \Rightarrow
        \mathbf{x}_\nu = -\frac{1}{2}\mathbf{A}^\top\nu $. Then,
        \begin{equation*}
            f_d(\nu) \triangleq \inf_{\mathbf{x} \in \mathbb{R}^n}
                \mathcal{L}(\mathbf{x}, \nu) =
                \mathcal{L}(\mathbf{x}_\nu, \nu) =
                -\frac{1}{4}\nu^\top\mathbf{AA}^\top\nu - \mathbf{b}^\top\nu
        \end{equation*}
        $ \forall \nu \in \mathbb{R}^q $, $ f_d(\nu) \in \mathbb{R} $, so
        $ \operatorname{dom}f_d = \mathbb{R}^q $. $ -f_d $ is convex,
        $ \mathbf{AA}^\top \succeq \mathbf{0} $\footnote{
            $ \forall \mathbf{x} \in \mathbb{R}^n,
            \mathbf{x}^\top\mathbf{AA}^\top\mathbf{x} =
            \big(\mathbf{A}^\top\mathbf{x}\big)^\top\mathbf{A}^\top\mathbf{x} =
            \Vert\mathbf{A}^\top\mathbf{x}\Vert_2^2 \ge 0 $.
        }.
    \end{itemize}

    % spacing for footnote
    \medskip
\end{frame}

\subsection{Strong duality}

\begin{frame}{Strong duality}
    \begin{itemize}
        \item
        \textit{Definition.} Let $ d^* $ denote the optimal value of the dual
        problem (\aref{opt_prob_std_dual}). The property $ d^* \le p^* $ is
        \textit{weak duality}, which always holds \cite{bv_convex_opt}.

        \item
        \textit{Definition.} If $ d^* = p^* $, we say that
        \textit{strong duality} holds.

        \item
        \textit{Theorem.} If (\aref{opt_prob_std}) is a convex problem and
        Slater's condition holds, i.e. $ \exists \mathbf{x} \in
        \operatorname{relint}\tilde{\mathcal{M}} $ s.t.
        $ \mathbf{u}(\mathbf{x}) \prec \mathbf{0} $,
        $ \mathbf{v}(\mathbf{x}) = \mathbf{0} $, strong duality holds
        \cite{bv_convex_opt}.

        \item
        If (\aref{opt_prob_std}) convex and $ u_1, \ldots u_k $ affine,
        $ k \le p $, Slater's condition can be refined s.t. if
        $ \exists \mathbf{x} \in
        \operatorname{relint}\tilde{\mathcal{M}} $ s.t.
        $ u_1(\mathbf{x}) \le 0, \ldots u_k(\mathbf{x}) \le 0 $,
        $ u_{k + 1}(\mathbf{x}) < 0, \ldots u_p(\mathbf{x}) < 0 $,
        $ \mathbf{v}(\mathbf{x}) = \mathbf{0} $, strong duality
        holds \cite{bv_convex_opt}.

        \item
        If (\aref{opt_prob_std}) convex, $ \mathbf{u} $ affine,
        $ \tilde{\mathcal{M}} = \mathbb{R}^n $, strong duality holds if
        $ |\mathcal{X}^*| > 0 $.

        \item
        \textit{Definition.} Suppose strong duality holds for
        (\aref{opt_prob_std}). Let $ \mathbf{x}^* $ be primal optimal and
        $ \lambda^*, \nu^* $ be
        dual optimal. Then, $ \forall i \in \{1, \ldots p\} $,
        $ \lambda_i^* > 0 \Rightarrow u_i(\mathbf{x}^*) = 0 $, i.e.
        \textit{complementary slackness} holds \cite{bv_convex_opt}.

        \item
        Equivalently, one can write $ u_i(\mathbf{x}^*) < 0 \Rightarrow
        \lambda_i^* = 0 $.
    \end{itemize}
\end{frame}

\subsection{Karush-Kuhn-Tucker conditions}

\begin{frame}{Karush-Kuhn-Tucker conditions}
    \begin{itemize}
        \item
        Many problems have differentiable objectives and constraints.

        \item
        \textit{Definition.} Suppose strong duality holds for
        (\aref{opt_prob_std}) and $ f $, $ \mathbf{u} $, $ \mathbf{v} $
        differentiable. Then, the following conditions must hold.
        \begin{enumerate}
            \item
            \textit{Stationarity.} $ \nabla f(\mathbf{x}^*) +
            \nabla\mathbf{u}(\mathbf{x}^*)^\top\lambda^* +
            \nabla\mathbf{v}(\mathbf{x}^*)^\top\nu^* = \mathbf{0} $.

            \item
            \textit{Primal feasibility.} $ \mathbf{u}(\mathbf{x}^*) \preceq
            \mathbf{0} $, $ \mathbf{v}(\mathbf{x}^*) = \mathbf{0} $.

            \item
            \textit{Dual feasibility.} $ \lambda^* \succeq \mathbf{0} $.

            \item
            \textit{Complementary slackness.} $ \mathbf{u}(\mathbf{x}^*)^\top
            \lambda^* = 0 $.
        \end{enumerate}
        These are the \textit{Karush-Kuhn-Tucker conditions}
        \cite{bv_convex_opt}.

        \item
        Note $ \mathbf{x}^* =
        \arg\min_{\mathbf{x} \in \tilde{\mathcal{M}}}
        \mathcal{L}_f(\mathbf{x}, \lambda^*, \nu^*) \Rightarrow
        \nabla_\mathbf{x}\mathcal{L}_f(\mathbf{x}^*, \lambda^*, \nu^*) =
        \mathbf{0} $, where
        \begin{equation*}
            \nabla_\mathbf{x}\mathcal{L}_f(\mathbf{x}^*, \lambda^*, \nu^*)
            \triangleq  \nabla f(\mathbf{x}^*) +
            \nabla\mathbf{u}(\mathbf{x}^*)^\top\lambda^* +
            \nabla\mathbf{v}(\mathbf{x}^*)^\top\nu^*
        \end{equation*}
        By strong duality, $ f(\mathbf{x}^*) = f_d(\lambda^*, \nu^*)
        \triangleq \inf_{\mathbf{x} \in \tilde{\mathcal{M}}}\{
            \mathcal{L}_f(\mathbf{x}, \lambda^*, \nu^*)
        \} $.

        \item
        Complementary slackness is equivalently written as
        $ \lambda_i^*u_i(\mathbf{x}^*) = 0 $,
        $ \forall i \in \{1, \ldots p\} $, as
        $ \mathbf{u}(\mathbf{x}^*)^\top\lambda^* = 0 \Leftrightarrow
        u_i(\mathbf{x}^*) < 0 \Rightarrow \lambda_i^* = 0 $.
    \end{itemize}
\end{frame}

\begin{frame}{Karush-Kuhn-Tucker conditions}
    \begin{itemize}
        \item
        KKT conditions are \alert{necessary} optimality conditions for
        differentiable optimization problems where strong duality holds.

        \item
        But if (\aref{opt_prob_std}) is convex and $ \mathbf{x}^*, \lambda^*,
        \nu^* $ satisfy the KKT conditions, then strong duality holds,
        $ \mathbf{x}^* $ is primal optimal, $ (\lambda^*, \nu^*) $ is dual
        optimal.

        \item
        KKT conditions are \alert{sufficient} for optimality if the problem is
        convex.

        \item
        Furthermore, if $ \mathbf{x}^* $ also satisfies Slater's
        condition, then KKT conditions become \alert{necessary and sufficient}
        for optimality.

        \item
        In summary:
        \begin{enumerate}
            \item
            Strong duality $ \Rightarrow $ KKT conditions satisfied.

            \item
            Convex problem, KKT conditions $ \Rightarrow $ strong duality.

            \item
            Convex problem, Slater's condition, KKT conditions
            $ \Leftrightarrow $ strong duality.
        \end{enumerate}
    \end{itemize}
\end{frame}

% BibTeX slide for references. should use either acm or ieeetr style
\begin{frame}{References}
    \bibliographystyle{acm}
    % relative path may need to be updated depending on .tex file location
    \bibliography{../master_bib}
\end{frame}

\end{document}


\section*{NP-completeness - Ada Nzulumike}

Introduction:
\\
- Purpose (identifying hard problems)
\\
- The class P (decision problems, language framework, decided)
\\
- The class NP (verification)

NP-completeness:
\\
- Reducibility (Definition of NP-complete languages)
\\
- Proof of Theorem 34.9.


% !BIB TS-program = biber
% !BIB program = biber

\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[top= 2cm, bottom=2cm, left=2cm, right=2cm]{geometry}
\usepackage{amsmath, amsfonts, graphicx}
\usepackage{url}

\usepackage[backend=biber, sorting=none]{biblatex}
\addbibresource{references.bib}

\begin{document}

\title{CS291D Final Report: a Basic Zerocash Implementation}
\author{Gwyneth Allwright, Karl Wang, Dewei Zeng}

\maketitle

\section*{Abstract}
In this project, we attempt a basic implementation of Zerocash \cite{zerocash} in Python. Zerocash is a ledger-based digital currency that makes use of zero-knowledge Succinct Non-Interactive Arguments of Knowledge (zk-SNARKs) to provide stronger privacy guarantees than currencies such as Bitcoin \cite{bitcoin} and Zerocoin \cite{zerocoin}. This functionality is provided through a decentralized anonymous payment (DAP) scheme that hides a transaction's origin, destination and amount. We follow \cite{zerocash} to implement the following core functions: \texttt{Setup}, \texttt{CreateAddress}, \texttt{Receive}, \texttt{Mint}, \texttt{VerifyTransaction}, \texttt{Pour}, \texttt{KeyGen}, \texttt{Prove} and \texttt{Verify}, which form the foundations of Zerocash.

\tableofcontents

\newpage

\section{Introduction}
Data on blockchains such as Bitcoin is public, including the sender, receiver and the amount of money transferred in a payment. While Bitcoin users often utilize different identities to disguise their transactions, it is possible to gain access to both the structure of the transaction graph and the values and dates of transactions. Zerocoin, a cryptographic extension to Bitcoin, aims to introduce better privacy guarantees without requiring new trusted parties, but it still cannot hide the location that money is sent to, as well as the amount of money that is transferred \cite{zerocoin}. In addition, it lacks some features of fully-fledged cryptocurrencies, such as payments of exact values.

In order to solve these problems with Bitcoin and Zerocoin, a new digital currency known as Zerocash was devised. Zerocash makes the sender, receiver and amount of money transferred in a payment anonymous, while also improving on the efficiency of Zerocoin \cite{zerocash}. These outcomes are achieved with the help of zk-SNARKs, which are efficient variants of zero-knowledge proofs of knowledge. Zero-knowledge proofs allow the prover of a certain statement to demonstrate that the statement in question is true without revealing additional information about the statement that could result in a compromise of privacy.

One of the primary objectives of this project is to explore zk-SNARKs and their potential applications in the world of blockchains and cryptocurrencies. To achieve this, we use existing zk-SNARK tooling to implement a minimal version of Zerocash in Python with the purpose of gaining a better understanding of Zerocash's theoretical underpinnings. This would be a first step towards demonstrating that zk-SNARKs are a feasible method of enhancing the privacy and performance of transactions on a simple blockchain. Next steps would include benchmarking and comparisons to a blockchain with similar functionality that does not make use of zk-SNARKs.

\section{Problem Definition}
We wish to understand how to incorporate zk-SNARKs into a basic blockchain in order to improve the blockchain's privacy guarantees. The setup of our scheme must not require any trust beyond a one-time trusted setup of public parameters. The implementation needs to support the minting, merging and splitting of coins without exposing the identities of the users who perform the transactions and the amounts of the currency involved.

\section{Solution}
The above objectives can be achieved through the combination of zk-SNARKs and a decentralized anonymous payment (DAP) scheme. As part of the Zerocash DAP scheme, we implement the following core functions: \texttt{Setup}, \texttt{CreateAddress}, \texttt{Receive}, \texttt{Mint}, \texttt{VerifyTransaction} and \texttt{Pour} \cite{zerocash}. For the zk-SNARK, we require the additional functions (\texttt{KeyGen}, \texttt{Prove}, \texttt{Verify}) \cite{zerocash}. In the sections that follow, we provide an overview of the DAP scheme, zk-SNARK and their core functions.

\subsection{DAP Scheme}
\subsubsection{Basecoin}
The Zerocash system is applied on top of a ledger-based currency (e.g.\ Bitcoin). This ledger-based currency is referred to as the \emph{basecoin}. All basecoin transactions are recorded in an append-only ledger, which can be accessed by all Zerocash users at all times.

In addition to the basecoin transactions, Zerocash includes two new kinds of transactions --- minting and pouring --- which will be described later. Mint and pour transactions are also recorded in the basecoin ledger.

\subsubsection{Public Parameters}
In addition to the ledger, users have access to a set of public parameters. These are part of the one-time trusted setup that takes place before other functions are allowed to execute.

\subsubsection{Address Key Pairs}
Users may generate as many public and private address key pairs as they desire. The public address keys are published with the purpose of allowing users to make payments among themselves. The secret keys are used for receiving payments.

\subsubsection{Coins}
Coins are data structures that encapsulate the following information:
\begin{itemize}
\item A coin commitment, which is a string that we append to the ledger once the coin is minted.
\item A coin value (between 0 and some parameter $v_{\text{max}}$) that specifies the coin denomination in basecoin units.
\item A coin serial number, which is a string that uniquely identifies the coin and is used to prevent double-spending.
\item A coin address --- the public address key of the user who owns the coin.
\end{itemize}

\subsubsection{Coin-Related Data Structures}
The Zerocash protocol requires us to maintain the following coin-related information:
\begin{itemize}
\item A Merkle tree over coin commitments.
\item A list of coin commitments that appear in mint and pour transactions.
\item A list of all coin serial numbers that appear in pour transactions.
\end{itemize}
For efficiency reasons, it is useful to store the latter two lists (which could also be obtained from the ledger) separately.

\subsubsection{New Transaction 1: Mint}
Mint transactions are used to create coins. At its most basic, a mint transaction can be described as a tuple $(\text{cm}, v)$, where $\text{cm}$ is the commitment of the minted coin and $v$ is its value. Whenever a coin is minted, this tuple is placed on the ledger.

\subsubsection{New Transaction 2: Pour}
Pour transactions record the pouring of two input coins into two new output coins (thereby spending the two initial coins). At its most basic, a pour transaction can be described as a tuple $(\text{rt}, \, {\text{sn}}^{\text{old}}_1, \, {\text{sn}}^{\text{old}}_2, \, {\text{cm}}^{\text{new}}_1, \, {\text{cm}}^{\text{new}}_2, \, v_{\text{pub}}, \, \text{info})$, where rt is the root of the Merkle tree over coin commitments, the sn are the serial numbers of the old coins, the cm are the commitments of the new coins, $v_{\text{pub}}$ is a coin value and info is an arbitrary string. Pour transactions may also include implementation-specific information.

\subsection{zk-SNARK}
The zk-SNARK construction consists of a tuple of polynomial-time functions (\texttt{KeyGen}, \texttt{Prove}, \texttt{Verify}). In what follows below, we give a high-level overview of these functions and their properties.

\subsubsection{Arithmetic Circuits}
For a given field $\mathbb{F}$, an arithmetic circuit $C$ takes as input $n$ field elements $\in \mathbb{F}$ and returns $m$ field elements $\in \mathbb{F}$. We can therefore think of $C$ as a map $\mathbb{F}^n \longrightarrow \mathbb{F}^m$.

In the Zerocash construction, we decompose the circuit input that lives in $\mathbb{F}^n$ into a main input and auxiliary input, where the latter is known as the \emph{witness}. If the dimensions of these two subinputs are $u$ and $v$ respectively, then we can write $C$: $\mathbb{F}^u \times \mathbb{F}^v \longrightarrow \mathbb{F}^m$.

\subsubsection{Circuit Satisfiability}
zk-SNARKs can be described in terms of arithmetic circuit satisfiability. The key relationship involved is the following:

\begin{equation}
\text{For a given} \,\,\, X \in \mathbb{F}^u, \,\,\, \exists A \in \mathbb{F}^v \,\,\, \text{such that} \,\,\, C(X, A) = 0^{m}.
\label{zk-snark-definition}
\end{equation}
\newline

The set of all $X$ that satisfy Equation \ref{zk-snark-definition} form the set $\mathbb{L}_C$. The statement that a prover would want to demonstrate is that for a given $X$, we have $X \in \mathbb{L}_C$.

\subsubsection{Important zk-SNARK Functions}

\begin{enumerate}

\item {\texttt{KeyGen}}:

The function \texttt{KeyGen} is used to sample a proving key and a verification key for the zk-SNARK. These keys are both public parameters. Their purpose is to help prove that a certain $X$ is a member of the set $\mathbb{L}_C$.

\texttt{KeyGen} takes as input the security parameter and zk-SNARK circuit $C$, and returns a key pair.

\item {\texttt{Prove}}:

The function \texttt{Prove} takes as input a proving key, as well as a pair $(X, A)$ --- where $X$ represents a main input for the circuit $C$ and $A$ the witness. It returns a proof $\Pi$ for the statement that $X \in \mathbb{L}_C$.

\item {\texttt{Verify}}:

The function \texttt{Verify} takes as input a verification key, the circuit's main input $X$ and a proof $\Pi$. It outputs $1$ if there is sufficient evidence that $x \in \mathbb{L}_C$, and 0 otherwise.

\end{enumerate}

\subsubsection{zk-SNARK Properties}
\begin{itemize}
\item \emph{Completeness.} Intuitively, this property means that an honest prover can convince the verifier that $X \in \mathbb{L}_C$. More mathematically, it means that with probability

\begin{equation}
P(\lambda) = 1 - \text{negl}(\lambda),
\end{equation}

where $\lambda$ is the security parameter, the output of \texttt{Verify} will be $1$ after going invoking \texttt{KeyGen} and \texttt{Prove} to correctly generate a proof for a circuit input $X \in \mathbb{L}_C$.

\item \emph{Succinctness.} This property has two components. First, it means that a proof $\Pi$ that was generated from \texttt{Prove} has $\mathcal{O}(1)$ bits for a given security parameter. Second (again for a fixed security parameter), \texttt{Verify} has time complexity $\mathcal{O}(X)$.

\item \emph{Proof of knowledge} (intuitive idea). If a proof is verified to be correct, then the prover ``knows" (is able to extract) a witness that corresponds to the instance, with certain guarantees around the time complexity of the extraction.

\item \emph{Zero knowledge}. The proof does not leak information about the witness.
\end{itemize}

\subsection{Key Zerocash Functions}
The DAP scheme described above is implemented by means of a tuple of polynomial-time algorithms (\texttt{Setup}, \texttt{CreateAddress}, \texttt{Mint}, \texttt{Pour}, \texttt{VerifyTransaction}, \texttt{Receive}). In this section, we describe the arguments, outputs and interrelation of these functions.

\subsubsection{\texttt{Setup}}
The purpose of \texttt{Setup} is to perform the one-time trusted setup of public parameters. It takes as input a security parameter and produces the following list of public parameters as output:
\begin{itemize}
\item (pk$_{\text{POUR}}$, vk$_{\text{POUR}}$): a proving and verification key pair for the zk-SNARK. These are sampled from \texttt{KeyGen}.
\item pp$_{\text{enc}}$: parameters for the encryption scheme.
\item pp$_{\text{sig}}$: parameters for the digital signature scheme.
\end{itemize}
All three of the above are functions of the provided security parameter.

\subsubsection{\texttt{CreateAddress}}
The purpose of \texttt{CreateAddress} is to generate public-private address key pairs for users. It takes as input the public parameters generated by \texttt{Setup} and produces a key pair as output.

\subsubsection{\texttt{Mint}}
The purpose of a call to \texttt{Mint} is the creation of a coin. It takes as input the public parameters generated by \texttt{Setup}, the value of the coin to be minted and the public address key of the coin's owner. It returns a coin data structure for the minted coin, along with the associated mint transaction. The mint transaction is a tuple containing (at minimum) a coin commitment and value.

\subsubsection{\texttt{Pour}}
\texttt{Pour} is easily one of the most complicated functions in the Zerocash system. It is used to ``pour" two old coins into two new ones, such that the sum of the two old coins equals the sum of the two new ones. The latter sum could potentially an additional value $v_{\text{pub}}$ that is publicly spent in the transaction. In this case, the balance equation would be

\begin{equation}
v_1^{\text{old}} + v_2^{\text{old}} = v_1^{\text{new}} + v_2^{\text{new}} + v_{\text{pub}}.
\label{balance}
\end{equation}
\newline

The \texttt{Pour} operation could have multiple purposes, including switching coin denominations, making public payments and transferring coin ownership.

\texttt{Pour} takes the following as input:

\begin{itemize}
\item The public parameters generated by \texttt{Setup}.
\item The root of the Merkle tree over coin commitments.
\item The coin data structures for two old coins.
\item The secret address keys for the owners of the two old coins.
\item The authentications paths for each of the old coin commitments to the root of the Merkle tree.
\item Two new coin values.
\item The public address keys for the owners of the two new coins.
\item A coin value $v_\text{pub}$. This is amount that will be \emph{publicly} spent in the pour transaction --- for example, to pay a transaction fee or to purchase coins.
\item An information string.
\end{itemize}

One of the primary tasks performed in \texttt{Pour} is to prove or disprove the following: \newline

\fbox{\parbox{\textwidth}{
\medskip
For the provided Merkle tree root, serial numbers of the old coins and coin commitments for the two new coins, there are two old coins, two news coins and a secret address key such that the following statements hold:

\begin{itemize}
\item All four of the coins have the correct commitments.
\item For both of the old coins, the provided secret address key matches the secret address key that is generated from the public address key that forms part of the old coin's data structure.
\item The serial numbers of the old coins are correct.
\item The two commitments of the old coins feature in the provided Merkle tree. This is to ensure that the old coins have been previously minted.
\item The balance equation for the old and new coin values, Equation \eqref{balance}, is preserved.
\end{itemize}}}
\medskip

In order to perform the generation and verification of the proof, we turn to zk-SNARKs. Recall that the circuit $C$ was generated in the \texttt{Setup} step. $C$ takes a main input $X$ and witness $A$ that are constructed as follows:

\begin{align}
X &= \left( \text{rt}, \, {\text{sn}}^{\text{old}}_1, \, {\text{sn}}^{\text{old}}_2, \, {\text{cm}}^{\text{new}}_1, \, {\text{cm}}^{\text{new}}_2, \, v_{\text{pub}}, \, h_{\text{Sig}}, \, h_1, \, h_2 \right) \\
A &= \left( \text{path}_1, \, \text{path}_2, \, {\text{c}}^{\text{old}}_1, \, {\text{c}}^{\text{old}}_2, \, {\text{addr}}^{\text{old}}_{\text{sk}, 1}, \, {\text{addr}}^{\text{old}}_{\text{sk}, 2}, \, {\text{c}}^{\text{new}}_1, \, {\text{c}}^{\text{new}}_2 \right),
\end{align}
\newline

where rt is the root of the Merkle tree over coin commitments, the sn are the coin serial numbers, the cm are the coin commitments, $v_{\text{pub}}$ is the publicly spent amount in the pour transaction, the $h$ are parameters used to ensure non-malleability, the paths are Merkle tree authentication paths for the two old coins, the c are the coins themselves and the addr are the secret address keys associated with the old coins.

\texttt{Pour} also appends its transaction to the ledger. Note that a transaction tuple does not reveal any of the coin values or recipient addresses besides from $v_{\text{pub}}$ --- it contains only the following information:

\begin{equation}
\left(\text{rt}, \, {\text{sn}}^{\text{old}}_1, \, {\text{sn}}^{\text{old}}_2, \, {\text{cm}}^{\text{new}}_1, \, {\text{cm}}^{\text{new}}_2, \, v_{\text{pub}}, \, \text{info}, \, *\right),
\end{equation}

where $*$ is implementation-specific, but would likely include the zero-knowledge proof $\Pi$ generated by $\texttt{Prove}$.

\texttt{Pour} returns the transaction data structure and the two new coins.

\subsubsection{\texttt{Receive}}
\texttt{Receive} is used to receive payments. It takes as input the ledger and the address key pair of the recipient. It returns a set of coin objects that have not been spent by the recipient, but that were paid to the recipient through \texttt{Pour} transactions. This excludes coins that were minted by the recipient.

\subsubsection{\texttt{VerifyTransaction}}
The purpose of \texttt{VerifyTransaction} is to determine whether or not a provided transaction is valid. It takes as input the ledger, the public parameters generated by \texttt{Setup} and a mint or pour transaction. It returns $1$ if the transaction is valid, and $0$ otherwise. In theory, \texttt{VerifyTransaction} could be utilized by both individual users and the nodes of the distributed system.

\section{Related Work}
Although there has been an extensive amount of research related to zk-SNARKs, only a few have made their way into cryptographic tools \cite{zk-snarks-blockchains-chapter}. Nevertheless, zk-SNARKs are especially useful for blockchains --- for reasons other than the mere improvement of privacy guarantees. For example, zk-SNARK proofs are non-interactive, which means that verifiers can check a proof at their leisure, without collaborating with the prover. In addition, zk-SNARKs proofs are concise, which means that they can be verified efficiently. These properties can be used to improve the scalability of blockchains \cite{zk-snarks-blockchains-chapter}.

It should therefore come as no surprise the zk-SNARKs have been used in blockchain systems other than Zerocash. An example of such a system is CODA \cite{coda}, where one of the main ideas is to bundle up a group of transactions, calculate a zero-knowledge proof for each, and then provide a single proof that can be used to verify them all \cite{zk-snarks-blockchains-chapter}. There has also been some Zerocash-inspired work that explores integrating zk-SNARKs into Ethereum \cite{zeth}.

In addition to the various blockchain systems that capitalize on zk-SNARKs, there are also variations of zero-knowledge proofs themselves. These include zero-knowledge Succinct Transparent Arguments of Knowledge (zk-STARKs) and bulletproofs.

One of the main advantages of zk-STARKs over zk-SNARKs is the excellent security provided by zk-STARKs --- for example, they do not require a trusted setup. However, due to their large proof size, which grows as $\mathcal{O}(\log^ 2{|C|})$ with respect to the circuit size $|C|$, zk-STARKs are currently not as practical as zk-SNARKs. For the latter, the proof size remains constant as $|C|$ increases \cite{zk-snarks-blockchains-chapter}.

Bulletproofs also do not require a trusted setup. However, their proof size scales as $\mathcal{O}(\log{|C|})$, which again makes them (generally) less performant than zk-SNARKs \cite{zk-snarks-blockchains-chapter}.

\section{Evaluation}
Our attempt to implement a basic version of Zerocash in Python was partially successful. First, we were able to build a basic blockchain from scratch. Second, we made our blockchain distributed through the implementation of a \texttt{Node} class, where different nodes could be run on different ports of the same machine. Third, we managed to build most Zerocash functionalities on top of our blockchain. Lastly, we integrated our code with the C++ zk-SNARK library libsnark and constructed the portion of the NP Statement POUR that checks for coin values. However, we struggled to construct the rest of POUR, which was mostly due to the lack of documentation and tooling on zk-SNARKs.

\printbibliography

\end{document}


\subsection{Encoding Functions in Process Calculi}
\label{sec:blue}
Process Calculi are used to describe the structure and behaviour of concurrent processes.  As a basis for the design of programming languages, a calculus should be able to encode canonical calculations with functions.  Encoding the $\lambda$-calculus, the canonical form of functional programming, to a processes calculus could be done either indirectly or directly.

\subsubsection{Indirect Encoding}
In \cite{function_as_process}, Milner gave translation rules from both the lazy $\lambda$-calculus and the call-by-value $\lambda$-calculus to his $\pi$-calculus.  Based on this work, in an higher-order $\pi$-calculus, functions could be encoded into processes and then passed around the network as a value.  Later, Sangiorgi pointed out that the higher-order approach is unnecessary since the plain $\pi$-calculus could simulate this higher-order feature by passing a name that points to the encoding process\cite{HOPI}.

The main drawback of the two indirect encoding approaches is its inefficiency in translation and reduction.  Indirect encoding will yield a mass of intermediate variables.  Moreover, without a direct representation for functions, complex substitutions of variables for functions are unavoidable during reductions.

\subsubsection{The Blue Calculus: Encoding Functions in a Direct Style}
Boudol's blue calculus, $\pi^*$, is a direct extension of both the $\lambda$-calculus and the $\pi$-calculus.  In fact, Boudol defined two calculi in \cite{Blue}: a name-passing $\lambda$-calculus ($\lambda^*$) and an extended $\pi$-calculus without summation and matching ($\pi^*$).  The $\lambda^*$-calculus has $\lambda$-style syntax and $\pi$-style reduction relation (Table \ref{lambda_star}).  It is used as an intermediate language when translating $\lambda$-terms to $\pi^*$-terms.  The $\pi^*$-calculus contains primitives from both the $\lambda^*$-calculus and the $\pi$-calculus so that the translation from both languages is straightforward.

\begin{table} [h]
  \begin{center}
  \includegraphics[scale=0.5]{lambda_star.png}
  \end{center}
  \caption{The $\lambda ^*$-calculus}
  \label{lambda_star}
\end{table}

The $\lambda^*$-calculus differs from the $\lambda$-calculus in two aspects:
\begin{inparaenum}[(i)]
  \item The argument ($N$) in an application ($M N$) must be a variable.
  \item A convenient notation for name declaration, $def\ x\ =\ N\ in\ M$, is allowed.
\end{inparaenum}
It is important to note that the $\lambda^*$-calculus only contains the call-by-name evaluation.  This simplifies subsequent studies on relationship between the $\lambda$-calculus and other calculi.  Translations from $\lambda$ to $\lambda^*$ is similar to Launchbury's encoding in \cite{Launchbury93anatural}:

\begin{center}
  \begin{tabular}{ r c l  c}
$x^*$&=&$x$&\\
$(\lambda xM)^*$&=&$\lambda xM^*$&\\
$(M N)^*$&=&$(def\ v = N^*\ in\ (M^*v))$&($v$ fresh)\\
  \end{tabular}
\end{center}

As mentioned earlier, both the $\lambda^*$-calculus and the $\pi$-calculus (without summation and matching) could be translated to the  $\pi^*$-calculus (see Table \ref{trans_blue}).  In addition, a CPS \footnote{continuation passing style} transform from the $\pi^*$-calculus to the $\pi$-calculus is given in \cite{Blue} as well.  Lastly, Silvano Dal-zilio \cite{Dal-Zilio97implicitpolymorphic} proposed a implicit polymorphic type sytem for the $\pi^*$-calculus as an improvement of the original simple type system in \cite{Blue}.

\begin{table}[h]
  \begin{center}
  \begin{tabular}{ l r c l r}
syntax:\\
&$P$&::=& $A\ |\ D\ |\ (P\ |\ P)\ |\ (\nu x)P$&processes\\
&$A$&::=& $u\ |\ (\lambda u)P\ |\ (Pu)$&agents\\
&$D$&::=& $\langle u\ =\ P\rangle\ |\ \langle u\ \Leftarrow P\rangle$& declarations\\
structural equivalence:\\
&$(P\ |\ Q)$&$\equiv$&$(Q\ |\ P)$&commutativity\\
&$((P\ |\ Q)\ |\ R)$&$\equiv$&$(P\ |\ (Q\ |\ R))$&associativity\\
&$((\nu u)P\ |\ Q)$&$\equiv$&$(\nu v)(P\ |\ Q)\ \ (u\ is\ not \ free\ in\ Q)$&scope migration\\
&$(P\ |\ Q)u$&$\equiv$&$(Pu\ |\ Qu)$&distributivity\\
&$((\nu u)P)v$&$\equiv$&$(\nu u)(Pv)\ \ (u\ \neq\ v)$&\\
&$Du$&$\equiv$&$D$&\\
&$\langle u\ =\ P\rangle$&$\equiv$&$\langle u\ \Leftarrow\  (P\ |\ \langle u\ =\ P\rangle )\rangle$&duplication\\
reduction:\\
&$((\nu u)P)v$&$\rightarrow$&$[v/u]P$&$\beta$\\
&$(u\ |\ \langle u\ \Leftarrow\ P \rangle)$&$\rightarrow$&$P$&resource fetching
  \end{tabular}
  \end{center}
  \caption{The $\pi^*$-Calculus}
  \label{blue}
\end{table}

\begin{table}[h]
  \begin{center}
  \begin{tabular}{ r c l r c l}
$[x]u$&=&$\overline{x}u$&$[\overline{u}v_1\cdots v_k]$&=&$uv_1\cdots v_k$\\
$[\lambda xL]u$&=&$u(x,v)[L]v$&$[u(v_1, \cdots ,v_k)P]$&=&$\langle u\ \Leftarrow\ (\lambda v_1 \cdots v_k)[P]\rangle$\\
$[Lx]u$&=&$(\nu x)([L]v\ | \ \overline{v}xu)$&$[!u(v_1, \cdots ,v_k)P]$&=&$\langle u\ =\ (\lambda v_1 \cdots v_k)[P]\rangle$\\
$[def\ x\ =\ N\ in\ L]u$&=&$(vx)([L]u\ |\ !x(v)[N]v)$&$[P\ |\ Q]$&=&$([P]\ |\ [Q])$\\
&&&$[(\nu u)P]$&=&$(\nu u)[P]$
  \end{tabular}
  \end{center}
  \caption{Translation from $\lambda^*$-calculus and $\pi$-calculus to $\pi^*$-calculus}
  \label{trans_blue}
\end{table}


\chapter{Methodology}

In this chapter, the experiment's methodology is summarized and abstracted in regards to the research goals, experimental setup and program implementation of the work. The process of preparing and benchmarking neural networks with varying arithmetic precisions, as well as the creation of an end-to-end system for carrying out these tests, is detailed. In brief, trained models are obtained from the scientific community for each architecture form various resources, model weights are extracted and inference calculations are made at different precisions.

\section{Experimental summary}
The goal of this work is to study how performing inference calculations with reduced arithmetic precision affects two measures: runtime and accuracy. Given a convolutional neural network model $M$, runtime $R$ and accuracy $A$ of inference with $M$ are calculated with a varying GEMM mode $C \in{eigen, gemmlowp}$ where $eigen$ \cite{eigen} uses 32-bit floating-point representation and $gemmlowp$ \cite{gemmlowp} uses quantized 8-bit integers. $gemmlowp$ also performs quantization, the process of which is detailed in Section \ref{sec:quantization}. A reference implementation using Caffe \cite{caffe} was implemented, as described in Section \ref{sec:caffe}.

Concretely, formula \ref{eqn:conv} on page \pageref{eqn:conv} is either computed in full float precision with eigen or with quantized values with gemmlowp as described in the formulas in Section \ref{sec:quantization}.

\section{Model architectures}
The models used in this work were chosen because they represent small, medium and large model sizes in terms of *the number of trainable weight parameters*. Additionally, in their number of convolutions, they trend upwards, generally exponentially, which creates a natural environment for the comparison of runtimes of convolutions (see Figure \ref{fig:trend}). ``LeNet'' \cite{mnist} and ``cifar-10\_quick'' \cite{cifar} represent the two smaller models. The larger models include the popular ``VGG-16'' and the even larger ``VGG-19'' \cite{return}. The medium-sized models included in experimentation consist of the revolutionary ``ResNet50'' and ``ResNet101'' \cite{resnets2} models. For the purpose of the experiments the convolution layers are of most interest. Each model has a different number of these layers. LeNet has a mere two such layers, cifar-10\_quick has three, and VGG-16 and VGG-19 have 13 and 16 convolution layers, respectively. Refer to Table \ref{tbl:cnns} on page \pageref{tbl:cnns} for the layer outline of these model architectures. ResNet50 and ResNet101 have---by far---the most number of layers, totaling 53 such operations for ResNet50, and 104 operations for ResNet101.

\bildklein{figures/trend.png}{Convolution counts}{The relationship between convolution counts for the models used.}{fig:trend}

The model architectures and their relevant properties are highlighted in Table \ref{tbl:models} on page \pageref{tbl:models}, while some diagrams of varying model designs are displayed in Figures \ref{fig:vgg16_design} and \ref{fig:resnet_design}.

\bildgross{figures/vgg16.png}{Design of the VGG-16 architecture}{Design of the VGG-16 architecture. [Image courtesy of book.paddlepaddle.org]}{fig:vgg16_design}

\bildgross{figures/resnet.png}{Design of the ResNet architecture}{Design of the ResNet architecture. [Image courtesy of book.paddlepaddle.org]}{fig:resnet_design}

\begin{table}[]
\centering
\caption[Model properties]{The models and their properties.}
\label{tbl:models}
\begin{tabular}{lllllll}
\textbf{Model}     & \textbf{layers} & \textbf{params} & \textbf{input} & \textbf{convs} & \textbf{pools} & \textbf{products} \\
LeNet     & 4               & 430.5K          & 1x28x28        & 2              & 2              & 2                 \\
cifar-10  & 5               & 145.4K          & 3x32x32        & 3              & 3              & 2                 \\
VGG-16    & 16              & 138.3M          & 3x224x224      & 13             & 5              & 3                 \\
VGG-19    & 19              & 143.7M          & 3x224x224      & 16             & 5              & 3                 \\
ResNet50  & 50              & 25.6M           & 3x224x224      & 53             & 2              & 1                 \\
ResNet101 & 101             & 44.5M           & 3x224x224      & 104            & 4              & 1
\end{tabular}
\end{table}

\section{Inference inputs}
\label{sec:infinputs}
The size and nature of inputs also differs amongst models used. The LeNet models uses its famous MNIST handwritten database dataset of hand-written digits \cite{mnist}. These are one-channel (gayscale) images of size $28\times28$. The cifar-10\_quick model is fed with $32\times32$ 3-channel (RGB) images. They are mean-image normalized. All four other models take ImageNet images as input, specifically the first 1000 images of the ILSVRC 2012 competition's validation set \cite{imagenet}. They are either mean-pixel or mean-image normalized, depending on what was done at training-time, and are of size $224\times224$.

\section{Breaking down the low-precision process}
The requirements for integrating low precision into convolutions can be broken down into several components. Some of these can---and are---obtained or performed offline, while others are strictly online run-time processes. These components are, briefly: result matrix quantization parameters, input quantization parameters, weight quantization parameters, the actual quantization process for inputs, activations and weights, the GEMM operation, and dequantization. Table \ref{tbl:gemmlowp} on page \pageref{tbl:gemmlowp} shows the different steps involved, whether they can be performed offline, and whether they are involved in calculating timings for the experiments. The quantization method is highlighted in Section \ref{sec:quantization}.

\section{Implementation details}
\subsection{Caffe feature and weight extraction}
\label{sec:caffe}
Caffe is a popular deep-learning framework born out of Berkeley College \cite{caffe}. It has a vast number of publicly-accessible models made available, and processes in 32-bit float \cite{caffe}, thus making it a good candidate for building into our custom system as a means to select models, provide model weights and baseline features, and reference against the custom implementation to make sure it is bug-free.

At this point, it may also be of use to keep track of the number representations of the various libraries involved in extracting weights and features, as to be able to effectively compare to the Caffe baseline, and to keep our weights in the right precision.

Weights are extracted from the forward-pass of the Caffe network, instantiated through Caffe's ``Caffe.Net()'' method, taking a .prototxt and .caffemodel file and called with ``Caffe.Net.().forward()'' \cite{caffe}. Caffe processes in single floating-point precision, so the current state is 32-bit. After extraction, weights are saved via Python's Numpy package, which uses double precision \cite{scipy}, so our precision is maintained.

First in the pipeline of the Caffe feature extraction is, after the forward pass, the saving of activations by Python's Numpy \cite{scipy}. Therefore the reference features are of the correct precision and are comparable to the 32-bit experiment results.

The inputs, having values typically ranging from 0-255, are inherently integers and remain so until activated in the first layer of the network (or rather are represented by floats with a trailing ``.0'', unless quantized), where they either remain integers (in integer mode) or become floats due to multiplication with weights.

\subsection{Inference}
All experiments were run on an Intel Core i5-2520M CPU @ 2.50GHz x 4 with 3.7 GB of RAM. The operating system was Ubuntu 16.04 LTS 64-bit. Additionally, auxiliary experiments were run on a Raspberry Pi 3 (see Section \ref{sec:pi}.

In both modes, input is run through the custom system related to this work. The system includes bare-bones implementations of all relevant layers: convolution, pooling, ReLU, fully-connected, batch-normalization, scale, and eltwise. The inference files and makefiles are automatically generated via a script-generation script written in Python, which parses the Caffe prototxt files and outputs a C++ script in the most minimal way possible. Great care has been taken in ensuring that the script is minimal: all large objects are passed-by-reference when possible, eigen routines are called conservatively, the loading of parameters, weights and inputs, handled by the Armadillo package \cite{eigen_vs_armadillo}, is optimized, and the compiler is set to optimize for speed. The options used at compile time are as follows:

\lstset{language=make}
-c -O3 -march=native -std=c++11

The option ``-O3'' sets the compiler to make full optimization: it attempts to reduce code size and execution time, and performs all other optimizations possible. Furthermore, ``-march=native'' tells the compiler to use the platform-specific assembly code instructions.

\subsection{Integrating low precision}
The low-arithmetic precision module was taken from gemmlowp, a ``small self-contained low-precision GEMM library'' \cite{gemmlowp}. Gemmlowp performs several tasks needed for the experiments in this work. Apart from performing the actual GEMM procedure, gemmlowp quantizes inputs, dequantizes, retrieves parameters needed for these quantizations, and quantizes weights. This section will detail the processes involved in replacing regular floating-point GEMM operations with a custom call to the gemmlowp module.

\begin{table}[]
\centering
\caption[Low-precision GEMM broken down by process]{Subprocesses of low-precision GEMM. Although low-precision arithmetic is in theory faster, computation time can quickly add up with other online operations.}
\label{tbl:gemmlowp}
\begin{tabular}{lll}
\textbf{Process}                                       & \textbf{possible offline?} & \textbf{incl. in timing?} \\
Calculate input matrix MIN/ MAX     & yes                        & yes                       \\
Calculate weight matrix MIN/ MAX     & yes                        & no                        \\
Calculate activation matrix MIN/ MAX & can estimate               & no                        \\
Quantize input matrix                & yes                        & yes                       \\
Quantize weight matrix               & yes                        & no                        \\
GEMM                                 & no                         & yes                       \\
Dequantize activation matrix         & no                         & yes
\end{tabular}
\end{table}

One of the convenient realities that makes low-precision GEMM work is the fact that many---and sometimes all---of the parameters can be calculated offline \cite{warden_quantize}. For the weights of a cnn, this is always the case, as the weight parameters are learned at training-time. Thus, the minimum and maximum of these matrices can be stored as constants. For the inputs, it is also possible to collect parameters offline. This is straightforward for standard RGB images, known to hold values in the range [0-255]. The exact processes that accompany these calculations are detailed in the equations in Section \ref{sec:quantization}.

For result/ activation parameters, this is a bit more complex. One cannot know beforehand the resulting matrices at each convolution. However, it's possible to collect estimations of these parameters using the training images. In this experiment, these parameters are gathered prior to runtime using a "hold-off" set whose distinct purpose is to provide runtime parameters. This is all done offline, as it is equivalent to collecting the same parameters at training time. This cuts down on timing and apparently doesn't cost much in terms of accuracy, as is explained in the next chapter.

Quantizing inputs and weights, like finding their parameters, can also be done offline. However here, quantization parameters and quantizations of the input layer have been treated equally with other activation layers, and thus are computed online. We assume for the purposes of these experiments that the input matrix range was not known beforehand.

GEMM itself is an online operation, as well as dequantizing activation matrices, as results are of course not known until after GEMM.


% !TEX root = altosaar-2020-thesis.tex
\chapter{Introduction}\label{ch:intro}
\lettrine[image=true,lines=3]{design/F}{rom} the development of novel antibiotics~\citep{stokes2020a-deep} to cataloging sources of light in the night sky~\citep{regier2019cataloging}, many domains in science can benefit from applied machine learning methods. However, the utility of such methods hinges on building the structure of a problem---knowledge about the data or task---into a machine learning solution. Whether the setting is statistical physics or recommender systems, an off-the-shelf machine learning method can serve as a starting point. But performance is sacrificed when a method cannot be customized to the specifics of an applied scientific problem. This thesis focuses on probabilistic modeling, where what is known about a problem can be molded into assumptions about a probability distribution. We develop probabilistic modeling methods that use the structure of a problem to yield meaningful solutions in the study of models with large numbers of random variables in statistical physics systems and recommender systems. In tandem, this thesis develops an algorithm to improve the accuracy of approximations to probabilistic models, through the use of the structure of a probability model during optimization. By both building scalable probabilistic modeling methods tailored to answer scientific questions, and improving flexible probabilistic modeling methods themselves, we highlight the reciprocal relationship between these aims.% suiting machine learning methods to a scientific domain and developing flexible probabilistic modeling techniques.

One example of an applied problem in statistical physics is the study of probable configurations of atoms in a material. Simulating a material to find likely configurations of atoms with statistical physics models can be expensive, but designing materials with improved properties is valuable~\citep{schmidt2019recent}. The computational cost of these simulations for studying statistical physics models can be reduced by doing math, for example in analytical calculations to develop approximations or to incorporate knowledge of how neighboring atoms interact into simulations~\citep{swendsen1987nonuniversal}. A challenge in studying statistical physics models is balancing problem-specific customization with the resulting computational savings. Machine learning techniques applied to statistical physics systems can be used to develop generic methods that exploit problem structure for better performance. These methods can be re-used across models, saving practitioners time.

Where statistical physics concerns probable configurations of interacting atoms, recommender systems find items a user is likely to interact with~\citep{koren2009matrix}. For example, humans eat. A meal recommender system can predict which meals someone is likely to consume. Such a recommendation model might inform its predictions using the history of meals a user has eaten, namely which foods comprise those meals. A property of this type of data is that items (meals) are associated with unordered collections of attributes (sets of foods). This means that the number of possible meals a user might consume is very large. Existing methods for this type of data either cannot scale to large numbers of datapoints, or fail to accurately predict which items a user is likely to consume. This highlights the need to imbue a recommendation model with both properties of the data (such as meals represented as unordered sets) and the goals of the recommendation problem, or accurate prediction of which items a user will consume.

% define variational approximation here?
Both statistical physics models and recommender systems can be framed as probability models, the former as probable configurations of atoms, and the latter as probable items users may consume. Probabilistic modeling relies on inferring the parameters of a probability model using knowledge about the structure of the problem. For example, knowledge in the form of data regarding which meals someone has eaten can inform the predictions of a recommendation model; or, knowledge of how neighboring atoms interact in a material can be used in a probabilistic model of that material.

Practitioners that work with probability models seek probabilistic inferences. For example, the goals of such inferences include computing probabilities, summing over the random variables in a probability model, or finding likely configurations of random variables. Common inference methods are Markov Chain Monte Carlo~\citep{metropolis1953equation}, variational inference~\citep{blei2017variational}, and maximum likelihood estimation~\citep{bishop2006pattern}. This thesis uses variational inference and maximum likelihood estimation, as both algorithms can be scaled to large probability models~\citep{hoffman2013stochastic,robbins1951a-stochastic}. In particular, the variational inference algorithm can aid probabilistic inference in interacting systems of random variables found in statistical physics models. However, variational inference is sensitive to the initial choice of parameters governing the probability of the random variables under study. The accuracy of inferences of likely configurations of random variables can suffer, depending on this choice of initial parameters. Inference algorithms such as variational inference are utile in applied domains insofar as their performance is independent of the initial choice of parameters.

This thesis is organized as follows. \Cref{ch:background} introduces probabilistic models and gives examples of their use in statistical physics and recommender systems. We also review two approaches for statistical inference in probability models: variational inference and maximum likelihood estimation. \Cref{ch:hvm} develops and applies variational inference methods for statistical physics models. We show that exploiting the structure of a statistical physics model in a variational inference method is advantageous. This work was presented in \citet{altosaar2019hierarchical}. \Cref{ch:rfs} develops probability models for recommending items with sets of attributes and is based on \citet{altosaar2020rankfromsets:}. Similar to variational methods in physics, accounting for the structure of the problem helps: probability models that represent set-valued datapoints and the goals of the recommendation task are accurate and scalable. \Cref{ch:pvi} develops \gls{PVI} based on \citet{altosaar2018proximity}. \gls{PVI} is an inference algorithm that is imbued with information about a probability distribution we wish to infer. In this case, the structure of the problem is information about a probability distribution, which is used to inform an algorithm for fitting a probability model. We show how this enables \gls{PVI} to obtain accurate solutions. Finally, \Cref{ch:discussion} reviews how knowledge about a problem is useful in building probabilistic models for science solutions. We apply the \gls{PVI} algorithm developed in \Cref{ch:pvi} to the statistical physics setting of \Cref{ch:hvm} and the recommender systems application of \Cref{ch:rfs}. This highlights that methods development goes hand-in-hand with the aims of applied probabilistic modeling. We close with a discussion of extensions of this line of work.


In this section, we consider a particular application of the proposed approach
to probabilistic transient analysis presented in \sref{chaos-transient-analysis}
in order to illustrate its usage in practice. We begin by describing the
scenario being considered.

\subsection{\problemtitle}

At Stage~1 in \fref{chaos-overview}, the quantity of interest \g is the
transient power and temperature profiles corresponding to a given workload. Let
us now specify the parameters \vu that make \g uncertain to the designer of the
system.

As discussed in \sref{power-model}, the total dissipation of power is composed
of two major components: dynamic and static. The influence of process variation
on dynamic power is known to be negligibly small \cite{srivastava2010}; on the
other hand, the variability in static power is substantial, with the
subthreshold leakage current contributing the most \cite{juan2011, juan2012}.
With this in mind, we focus our attention on the subthreshold leakage and, more
specifically, on the effective channel length, which is denoted by \u, since it
has the strongest influence on this leakage (including its impact on other
important parameters such as the threshold voltage) and is severely deteriorated
by process variation \cite{chandrakasan2000}.

\inputfigure{chaos-beta-gaussian}
It is well known that the dispersion of the effective channel length around its
nominal value resembles the bell shape of Gaussian distributions. Therefore,
such variations are often conveniently modeled using Gaussian random variables
\cite{bhardwaj2006, ghanta2006, huang2009a, shen2009, chandra2010,
srivastava2010, juan2011, juan2012, lee2013}. Due to the underlying physics and
for demonstration purposes, we take a step further and embed into the model the
fact that the effective channel length---occupying the space between the drain
and source of a transistor---cannot be arbitrarily large or take negative
values, which Gaussian distributions allow it to do. In other words, we require
the model of \u to have a bounded support. To this end, we propose to model the
effective channel length and other physically bounded parameters using the
four-parameter family of beta distributions as
\begin{equation} \elab{beta-distribution}
  \u \sim \mathrm{Beta}(a, b, c, d)
\end{equation}
where $a$ and $b$ are the shape parameters, and $c$ and $d$ are the left and
right bounds of the support, respectively. The parameters $a$ and $b$ can be
chosen so that the frequently observed bell shape is preserved. An illustration
is given in \fref{chaos-beta-gaussian} where a beta distribution is fitted to
the standard Gaussian distribution; alternatively, one can match probabilistic
moments. It can be seen that the curves are nearly indistinguishable; however,
the beta one has a bounded support $[-4, 4]$, which can potentially lead to more
realistic models.

The variability in \u is split into global and local parts \cite{shen2009,
chandra2010, juan2012}, which are denoted by $\u_\overall$ and $\u_\local$,
respectively. The former can be treated as a composition of inter-lot,
inter-wafer, and inter-die variations, and the latter as a composition of
intra-die variations. The variability $\u_\overall$ is assumed to be shared by
all the \np processing elements, whereas each processing element is assumed to
have its own local parameter $\u_{\local, i}$. The effective channel length of
processing element~$i$ is then modeled using the following formula:
\[
  \u_i = \u_\nominal + \u_\overall + \u_{\local, i}
\]
where $\u_\nominal$ is the nominal value of the effective channel length.
Consequently, the uncertain parameters of the problem are
\[
  \vu = (\u_{\local, 1}, \dotsc, \u_{\local, \np}, \u_\overall): \Omega \to \real^{\np + 1}.
\]

Global variations are typically assumed to be uncorrelated with respect to the
local ones. The latter, however, are known to have high spatial correlations.
Similarly to the treatment in \cref{uncertainty-process-fabrication}, we model
these correlations using the composite correlation function given in
\eref{bayes-correlation}, which is inspired by the variation patterns induced by
the fabrication process \cite{friedberg2005, chandrakasan2000, cheng2011}.
Specifically, the correlation function imposes similarities between those
locations on the die that are close to each other as well as between those
locations that are at the same distance from the center of the die; see also
\cite{ghanem1991, ghanta2006, bhardwaj2008, huang2009a, lee2013}.

Although \eref{bayes-correlation} captures certain features that are
characteristic of the fabrication process, it is still an idealization. In
practice, it can be difficult to make a justifiable choice and tune such a
formula, which is a prerequisite for techniques based on the continuous \ac{KL}
decomposition, such as those discussed in \sref{chaos-past}. A correlation
matrix, on the other hand, can be readily estimated from measurements and thus
is a more probable input to probabilistic analysis. Hence, we use
\eref{bayes-correlation} for the sole purpose of constructing a correlation
matrix of $\set{\u_{\local, i}}_{i = 1}^\np$. For convenience, this correlation
matrix is extended by one dimension in order to accommodate $\u_\overall$ along
with $\set{\u_{\local, i}}_{i = 1}^\np$. Thus, the matrix acquires one
additional nonzero diagonal element equal to unity; the resulting matrix is the
correlation matrix of \vu denoted by $\correlation{\vu}$.

Let us now be more specific about the power model in \eref{chaos-power-model}.
In the ongoing scenario, \f can be rewritten as the following summation:
\[
  \f(i, \vq, \vu) = \f_\dynamic(i) + \f_\static(\vq, \vu)
\]
where $\f_\dynamic: \natural[+] \to \real^\np$ and $\f_\static: \real^\np \times
\real^\nu \to \real^\np$. Without loss of generality, the dynamic component
$\f_\dynamic$ is assumed to be given as a dynamic power profile (recall
\eref{power-profile}) denoted by $\mp_\dynamic$. Similarly to
\sref{bayes-results}, the modeling of the static component $\f_\static$ is based
on \up{SPICE} simulations of a reference electrical circuit composed of
\up{BSIM4} devices \cite{bsim} configured according to the 45-nm \up{PTM}
\up{HP} model \cite{ptm}; specifically, we use a series of \up{CMOS} invertors.
The simulations are performed with respect to a sufficiently wide fine-grained
two-dimensional grid---the effective channel length against temperature---and
the results are tabulated. An interpolation technique is then utilized whenever
it is necessary to calculate $\f_\static$ at a point within the range of the
grid.

Lastly, in order to be able to perform temperature calculations, an adequate
thermal \up{RC} circuit should be constructed. Given the specification of the
platform under consideration---including the floorplan of the die and the
configuration of the thermal package---this circuit is obtained by means of
HotSpot \cite{skadron2003}. The structure of the circuit is the one described in
\sref{temperature-model}.

To conclude, in this section, we address the variability in the effective
channel length. The input to our analysis is composed of the marginal
distributions of the uncertain parameters \vu, which are beta distributions, and
the corresponding correlation matrix $\correlation{\vu}$. Let us now go over the
other stages of our methodology presented in \sref{chaos-uncertainty-analysis}
and depicted in \fref{chaos-overview}.

\subsection{Probability Transformation}

At Stage~2 in \fref{chaos-overview}, \vu should be processed in order to extract
a vector of mutually independent random variables \vz via a suitable
transformation $\transform$; see \eref{chaos-transformation}. Following the
guidance given in \sref{chaos-transformation}, the most apposite $\transform$ in
the ongoing scenario is the Nataf transformation. The whole procedure is
described in detail in \xref{probability-transformation} and can be seen in
\eref{probability-transformation}.

Using this specific $\transform$, arbitrary marginal distributions can be
prescribed for \vz. There are no restrictions in this regard as long as a
suitable polynomial basis can be constructed, which is discussed in
\sref{chaos-construction}. We let \vz have beta distributions, keeping \vu and
\vz in the same family of distributions.

Since the number of stochastic dimensions, which is $\nu = \np + 1$ in the case
of \vu, directly impacts the computational cost of \ac{PC} expansions, which is
noted in \sref{chaos-construction}, one should consider the possibility of model
order reduction before constructing these expansions. Therefore, the reduction
procedure described in \xref{probability-transformation} in connection with
$\transform$ is assumed to be engaged in this transformation. The reduced
dimensionality is denoted by \nz.

\subsection{Surrogate Construction}

At Stage~3 in \fref{chaos-overview}, the uncertain parameters, power model, and
temperature model developed in the previous subsections are to be fused together
under the desired workload $\mp_\dynamic$ in order to produce the corresponding
stochastic power and temperature profiles denoted by \mp and \mq, respectively.

In the current scenario, the construction of \ac{PC} expansions is based on the
Jacobi polynomial basis, since it is preferable in situations involving
beta-distributed parameters \cite{xiu2010}. To give a concrete example, for a
dual-core platform ($\np = 2$) with two stochastic dimensions ($\nz = 2$), the
second-level \ac{PC} expansion ($\lc = 2$) of temperature at time step~$i$ is as
follows:
\begin{equation} \elab{chaos-expansion-example}
  \begin{split}
    \chaos{2}{2}{\vq_i}
    =    {} & \hat{\vq}_{i, (0, 0)} \psi_{(0, 0)} +
              \hat{\vq}_{i, (1, 0)} \psi_{(1, 0)} +
              \hat{\vq}_{i, (0, 1)} \psi_{(0, 1)} \\
    {} + {} & \hat{\vq}_{i, (1, 1)} \psi_{(1, 1)} +
              \hat{\vq}_{i, (2, 0)} \psi_{(2, 0)} +
              \hat{\vq}_{i, (0, 2)} \psi_{(0, 2)}
  \end{split}
\end{equation}
where the coefficients $\set{\hat{\vq}_{i \vj}}$ are vectors with two elements
corresponding to the two processing elements. Regarding the basis,
\begin{align*}
  & \psi_{(0, 0)}(\vz) = 1, \\
  & \psi_{(1, 0)}(\vz) = 2 z_1, \\
  & \psi_{(0, 1)}(\vz) = 2 z_2, \\
  & \psi_{(1, 1)}(\vz) = 4 z_1 z_2 \\
  & \psi_{(2, 0)}(\vz) = \frac{15}{4} z_1^2 - \frac{3}{4}, \text{ and} \\
  & \psi_{(0, 2)}(\vz) = \frac{15}{4} z_2^2 - \frac{3}{4}.
\end{align*}
The Jacobi polynomials have two parameters \cite{xiu2010}, and the ones shown
above correspond to the case where both parameters are equal to two. Such a
series can be shorter or longer, depending on the accuracy requirements given by
\lc. The expansion of power has the same structure but different coefficients.

The next step is to compute the coefficients of power $\set{\hat{\vp}_{i \vj}}$
in \eref{chaos-recurrence}, which subsequently yield the coefficients of
temperature $\set{\hat{\vq}_{i \vj}}$. As discussed in
\sref{chaos-construction}, these computations involve multidimensional
integration with respect to the distribution of \vz, and they should be
performed numerically using an adequate quadrature $\quadrature{\nz}{\lq}$. When
beta distributions are involved, the natural choice is Gauss--Jacobi
quadratures, which belong to the class of Gaussian quadratures introduced in
\xref{numerical-integration}. Given $\quadrature{\nz}{\lq}$, the coefficients
are computed as shown in \eref{chaos-coefficient}. It is important to note that
\lq should be chosen in such a way that the quadrature is exact for polynomials
of total order up to at least $2 \lc$, that is, twice the level of \ac{PC}
expansions, which is discussed in \sref{chaos-construction}. Consequently, $\lq
\geq \lc$, since the quadrature is Gaussian.

To summarize, we have completed four out of five stages of the proposed
framework depicted in \fref{chaos-overview}. The result is a lightweight
surrogate for the entire system. At each time step, the surrogate is composed of
two \np-valued polynomials---one is for power, and the other one for
temperature---which are defined in terms of \nz mutually independent random
variables.

\subsection{Post-Processing}

At Stage~4 in \fref{chaos-overview}, the constructed expansions are utilized in
order to assist the designer in analyzing the impact of process variation on
power- and temperature-related characteristics of the system that is being
developed. Consider, for example, \eref{chaos-expansion-example}. It can be seen
that the surrogate model has a negligibly small computational cost: for any
outcome of \vz, one can readily calculate the corresponding temperature by
plugging this outcome into \eref{chaos-expansion-example}; the same applies to
power. Therefore, the representation can be trivially analyzed in order to
retrieve various statistics about the system. Let us illustrate a few of them
using the expansion given in \eref{chaos-expansion-example}.

\inputfigure{chaos-application-power}
\inputfigure{chaos-application-temperature}
Assume that the dynamic power profile $\mp_\dynamic$ is the one shown in
\fref{chaos-application-power}. Having constructed a surrogate with respect to
this profile, we can calculate, for instance, the expectation and variance of
the temperature that the system has at a certain moment in time, which is a
trivial operation given the formulae in \eref{chaos-moments}. For the whole time
span of $\mp_\dynamic$, these quantities are plotted in
\fref{chaos-application-temperature} where the dashed lines correspond to one
standard deviation above the corresponding expectations. The displayed curves
closely match those obtained via \ac{MC} sampling with $\no = 10^4$ samples;
however, our method takes less than a second, whereas \ac{MC} sampling takes
more than a day, which will be discussed further in
\sref{chaos-transient-results}. In addition, the \ac{PDF} of the temperature at
that moment can be estimated. This operation is performed by sampling the
surrogate, in which case we might obtain curves similar to those shown in
\fref{chaos-application-density}, which is a part of a different example given
in \sref{chaos-transient-results}.


% Summary of using velocity dispersions as an age proxy -- lit review
\subsection{Kinematic ages}

The star forming molecular gas clouds observed in the Milky Way have a low
out-of-plane, or vertical, velocity \citep[\eg][]{stark1989, stark2005,
aumer2009, martig2014, aumer2016}.
In contrast, the vertical velocities of older stars are observed to be larger
in magnitude on average \citep{stromberg1946, wielen1977, nordstrom2004,
holmberg2007, holmberg2009, aumer2009, casagrande2011, ting2019, yu2018}.
There are two possible explanations for this observed increase in velocity
dispersion with age: either stars are born kinematically `cool' and their
orbits are heated over time via interactions with giant molecular clouds
\citep[see][for a review of secular evolution in the MW]{sellwood2014}, or
stars formed kinematically `hotter' in the past \citep[\eg][]{bird2013}.
Either way, the vertical velocity dispersions of thin disk stars are observed
to increase with stellar age.
This behavior is codified by Age-Velocity dispersion Relations (AVRs), which
typically express the relationship between age and velocity dispersion as a
power law: $\sigma_v \propto t^\beta$, with free parameter, $\beta$
\citep[\eg][]{holmberg2009, yu2018}.
These expressions can be used to infer the ages of groups of stars from their
velocity dispersions, as we did in \citet{lu2021}

Kinematic ages have been used to explore the evolution of cool dwarfs for over
a decade.
\citet{west2004, west2006} found that the fraction of magnetically active M
dwarfs decreases over time, by using the vertical distances of stars from the
Galactic mid-plane as an age proxy, and \citet{west2008} used kinematic ages
to calculate the expected activity lifetime for M dwarfs of different spectral
types.
\citet{faherty2009} used tangential velocities to infer the ages of M, L and T
dwarfs, and showed that dwarfs with lower surface gravities tended to be
kinematically younger, and \citet{kiman2019} used velocity dispersion as an
age proxy to explore the evolution of H$\alpha$ equivalent width (a magnetic
activity indicator), in M dwarfs.

% After the development of a radiative core, little angular momentum is
% transported between the core and convective envelope and, if wind-braking
% slows the surface substantially before the two zones recouple, the core may
% rotate much more quickly than the envelope.
% The Sun rotates almost as a solid body \citep[\eg][]{thompson1996}, so it is
% expected that the cores and envelopes of Solar-like stars eventually
% re-couple, with angular momentum efficiently transported between them.
% The timescales for recoupling have been studied extensively and explored in
% theoretical models for decades \citep[\eg][]{endal1981, macgregor1991,
% denissenkov2010, gallet2013, lanzafame2015}.

AVRs are usually calibrated in Galactocentric velocity coordinates (\vx, \vy,
\vz\ or $UVW$), and these velocities can only be calculated with full 6D
positional and velocity information, however most \kepler\ rotators do not
have RV measurements\footnote{Although RVs for most will be released in \gaia\
DR3}.
In Angus \etal\ (2020) we used velocity in the direction of Galactic latitude
(\vb) as a stand-in for \vz\ because, in the {\it Galactic} coordinate system,
velocities can be calculated from 3D positions and {\it 2D} proper motions.
The \kepler\ field lies at low Galactic latitude, so \vb\ is a close
approximation to \vz.
Though \vb\ velocity dispersion does not equal \vz\ velocity dispersion, it
still increases monotonically over time and provides accurate age rankings for
\kepler\ stars.
Unfortunately however, given that AVRs are calibrated in {\it Galactocentric}
coordinates (\vx, \vy, \vz), we could not directly translate \vb\ velocity
dispersions to ages.

In this paper, our aim was to use kinematic ages to calibrate a new
gyrochronology relation, for which four main steps were required.
Firstly, we inferred {\it vertical} velocity, \vz, for each star without an RV
measurement by marginalizing over missing RVs using a hierarchical Bayesian
model (see section \ref{sec:velocity_inference}).
Secondly, we calculated velocity dispersion for every star using a moving, or
rolling dispersion method (see section \ref{sec:velocity_dispersion}).
Thirdly, these velocity dispersions were converted into ages using an AVR
\citep[][section \ref{sec:avr}]{yu2018}.
Finally, we used a Gaussian process model to capture the complexities of
stellar rotational evolution and calibrated a new gyrochronology relation
using our kinematic ages, plus benchmark cluster and asteroseismic stars in
section \ref{sec:gp_model}.