%%\documentclass[handout]{beamer}
%\documentclass[aspectratio=169,13pt]{beamer}

%\input{./preamble}

\subtitle{Numerical Optimization}

\date{}
\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\section{Numerical Optimization Problems}

\subsection{Constrained and Unconstrained Optimization}

\begin{frame}{Numerical Optimization}

\begin{itemize}
\item Our focus will be on \coloremph{continuous} rather than \coloremph{combinatorial} optimization:
\[\min_{\B x} f(\B x)\quad \text{subject to} \quad \B g(\B x) = \B 0 \quad \text{and} \quad \B{h}(\B x) \leq \B 0\]

\lgcond{
where $f\in \mathbb{R}^n\to \mathbb{R}$ is assumed to be differentiable.
\begin{itemize}
\sitem Without the constraints, i.e. with $\B g = \B 0$ and $\B h = \B 0$, the problem is \coloremph{unconstrained}.
\sitem With constraints, the \coloremph{constrained} optimization problem restricts the solution to elements of the \coloremph{feasible region}: $\{\B x:\B g(\B x) = \B 0 \ \text{and} \ \B{h}(\B x) \leq \B 0\}$.
\end{itemize}
}

\item We consider linear, quadratic, and general nonlinear optimization problems:

\lgcond{
\begin{itemize}
\item If $f$, $\B g$, and $\B h$ are affine (linear and constant terms only) then we have \coloremph{linear programming} problem.
\sitem If $f$ is quadratic while $\B g$ and $\B h$ are linear, then we have a \coloremph{quadratic programming} problem, for which specialized methods exist.
\sitem Generally, we have a \coloremph{nonlinear programming} problem.
\end{itemize}
}

\end{itemize}

\end{frame}

\subsection{Existence and Uniqueness of Solutions}

\begin{frame}{Local Minima and Convexity}

\begin{itemize}
\item Without knowledge of the analytical form of the function, numerical optimization methods at best achieve convergence to a \coloremph{local} rather than \coloremph{global} minimum:

\lgcond{
If the input domain is infinite or the global minimum is in an infinitesimally narrow trough, it may be impossible to find the global minimum in finite time.
}

\item A set is \coloremph{convex} if it includes all points on any line, while a function is (strictly) convex if its (unique) local minimum is always a global minimum:

\lgcond{
\begin{itemize}
\item Set $S$ is convex if \[\forall \B x,\B y \in S,\quad \alpha \in [0,1], \quad \alpha \B x + (1-\alpha)\B y \in S.\]
\item Function $f$ is convex if \[f(\alpha \B x + (1-\alpha)\B y)\leq \alpha f(\B x) + (1-\alpha)f(\B y).\]
\item A function may have a unique global minima but not be convex.
\end{itemize}
}

\end{itemize}

\end{frame}

\begin{frame}{Existence of Local Minima}

\begin{itemize}
\item \coloremph{Level sets} are all points for which $f$ has a given value, \coloremph{sublevel sets} are all points for which the value of $f$ is less than a given value:

\lgcond{
\[L(z) = \{ \B x: f(\B x)=z\}\]
\[S(z) = \{ \B x: f(\B x)\leq z\}\]
}

\item If there exists a closed and bounded sublevel set in the domain of feasible points, then $f$ has has a global minimum in that set:

\lgcond{
Need a value $z$ such that $S(z)$ has finite size, is contiguous, and includes its own boundary.
}

\end{itemize}

\end{frame}

\subsection{Properties of Local Minima}

\begin{frame}{Optimality Conditions}

\begin{itemize}
\item If $\B x$ is an interior point in the feasible domain and is a local minima,
\[\nabla f(\B x) = \begin{bmatrix} \frac{df}{dx_1}(\B x) & \cdots \frac{df}{dx_n}(\B x)\end{bmatrix}^T = \B 0:\]

\lgcond{
\begin{itemize}
\item If $\frac{df}{dx_i}(\B x) < 0$ an infinitesimal increment to $x_i$ improves the solution,
\sitem if $\frac{df}{dx_i}(\B x) > 0$
an infinitesimal decrement to $x_i$ improves the solution.
\end{itemize}
}
\item \coloremph{Critical points} $\B x$ satisfy $\nabla f(\B x) = \B 0$ and can be minima, maxima, or saddle points:

\lgcond{
For scalar function $f$, can distinguish the three by considering sign of $f''(x)$.
}

\end{itemize}

\end{frame}

\begin{frame}{Hessian Matrix}

\begin{itemize}
\item To ascertain whether a critical point $\B x$, for which $\nabla f(\B x) = \B 0$, is a local minima, consider the \coloremph{Hessian matrix}:
\mdcond{
\[\B{H}_f(\B x) = \B{J}_{\nabla f}(\B x) = \begin{bmatrix}
                                  \frac{d^2 f}{dx_1^2}(\B x) & \cdots & \frac{d^2 f}{dx_1dx_n}(\B x) \\
                                  \vdots & \ddots & \vdots  \\
                                  \frac{d^2 f}{dx_ndx_1}(\B x) & \cdots & \frac{d^2 f}{dx_n^2}(\B x) \end{bmatrix}\]

The Hessian matrix is always symmetric.
}

\item If $\B x^*$ is a minima of $f$, then $\B{H}_f(\B x^*)$ is positive semi-definite:

\lgcond{
If $\B{H}_f(\B x^*)$ is not positive semi-definite, there exists normalized vector $\B{s}$ such that $\B s^T \B{H}_f(\B x^*)\B s < 0$, which means that
for a sufficiently small $\alpha$, $\B{\hat{x}} = \B x^* + \alpha \B s$ will have be a better solution, $f(\B{\hat{x}})<f(\B{x^*})$,
since the gradient is zero at $\B x^*$ and decreases for an infinitesimal perturbation of $\B x^*$ in the direction $\B s$.
}
\mdcond{}

\end{itemize}

\end{frame}

\begin{frame}{Optimality on Feasible Region Border}

\begin{itemize}
\item Given an equality constraint $\B g(\B x)=\B 0$, % implies that a minimum $\B x^*$ must satisfy $\B g(\B x^*)=\B 0$.
      it is no longer necessarily the case that $\nabla f(\B x^*) =\B 0$.
      Instead, it may be that directions in which the gradient decreases lead to points outside the feasible region:

\[\exists \B \lambda\in\mathbb{R}^n, \quad -\nabla f(\B x^*) = \B J^T_{\B g}(\B x^*) \B \lambda\]

\lgcond{
\begin{itemize}
\item $\B \lambda$ are referred to as the Lagrange multipliers.
\sitem This condition implies that at $\B x^*$, the direction in which $f$ decreases is in the span of directions moving along which would exit the feasible region.
\end{itemize}
}

\item Such \coloremph{constrained minima} are critical points of the Lagrangian function $\mathcal{L}(\B x, \B \lambda) = f(\B x) + \B \lambda^T\B g(\B x)$, so they satisfy: %, described by the nonlinear equation,

\[\nabla \mathcal{L}(\B x^*, \B \lambda) = \begin{bmatrix} \nabla f(\B x^*) + \B J_{\B g}^T(\B x^*) \B \lambda \\ \B g(\B x^*)\end{bmatrix} = \B 0\]

\lgcond{
Seeking $\B \lambda^*$ to obtain a function $k(\B x) = \mathcal{L}(\B x, \B \lambda^*)$ with maximum global minimum is the \coloremph{dual optimization problem}.
}

\end{itemize}

\end{frame}

\subsection{Conditioning of Minima}

\begin{frame}{Sensitivity and Conditioning}

\begin{itemize}
\item The condition number of solving a nonlinear equations is $1/f'(x^*)$, however for a minimizer $x^*$, we have $f'(x^*)=0$, so conditioning of optimization is inherently bad:

\lgcond{Consider perturbation of function values for a function that changes slowly near the minimum.}

\item To analyze worst case error, consider how far we have to move from a root $\B x^*$ to perturb the function value by $\epsilon$:

\lgcond{
\[\epsilon = f(x^*+h)-f(x^*)=  \underbrace{f'(x^*)h}_{0} + \frac{1}{2}f''(x^*)h^2 + O(h^3)\]
\begin{itemize}
\item so the function value changes by $\frac{1}{2}f''(x^*)h^2$, which implies we need $h=O(\sqrt{\epsilon})$,
\sitem  a perturbation to the function value in the $k$th significant digit, could result in the solution changing in the $k/2$th significant digit.
\end{itemize}
}
%For an absolute perturbation to the function of size $\epsilon$, we can check how far away If we perturb $x^*$ by $h^*$

\end{itemize}

\end{frame}

\subsection{Algorithms for 1D Optimization}

\begin{frame}{Golden Section Search}

\urcornerlinkdemo{06-optimization}{Golden Section Proportions}

\begin{itemize}
\item Given bracket $[a,b]$ with a unique minimum ($f$ is \coloremph{unimodal} on the interval), \coloremph{golden section search} considers consider points $f(x_1),f(x_2)$, $a<x_1<x_2<b$ and discards subinterval $[a,x_1]$ or $[x_2,b]$:

\lgcond{
\begin{itemize}
\sitem If a function is strictly convex and bounded on $[a,b]$, it is unimodal on that interval, but a unimodal function may be non-convex.
\sitem Because the function is unimodal, if we have $f(x_1)<f(x_2)$ then the unique local minima $f$ in $[a,b]$ has to be in the interval $[a,x_2]$.
\sitem So, if $f(x_1)<f(x_2)$ can restrict search to $[a,x_2]$ and otherwise to $[x_1,b]$.
\end{itemize}
}

\item Since one point remains in the interval, golden section search selects $x_1$ and $x_2$ so one of them can be effectively reused in the next iteration:

\lgcond{
\begin{itemize}
\sitem For example, when $f(x_1)>f(x_2)$,  $x_2$ is inside $[x_1,b]$ and we would like $x_2$ to serve as the $x_1$ for the next iteration.
\sitem To ensure this, and minimize resulting interval length, we pick $x_2=a+(b-a)(\sqrt{5}-1)/2$ and $x_1=b-(b-a)(\sqrt{5}-1)/2$.
\sitem Consequently, the convergence of golden secetion search is linear with constant $(\sqrt{5}-1)/2$ per function evaluation.
\end{itemize}
}

\end{itemize}

\end{frame}

\begin{frame}{Newton's Method for Optimization}

\urcornerlinkdemo{06-optimization}{Newton's Method in 1D}

\begin{itemize}
\item At each iteration, approximate function by quadratic and find minimum of quadratic function:

\lgcond{
Pick quadratic function $\hat{f}$ as first three terms of Taylor expansion of $f$ about $x_k$, matching value and first two derivatives of $f$ at $x_k$.
}

\item The new approximate guess will be given by $x_{k+1}-x_k = -f'(x_k)/f''(x_k)$:

\lgcond{
\[f(x) \approx \hat{f}(x) = f(x_k)+f'(x_k)(x-x_k) + \frac{1}{2}f''(x_k)(x-x_k)^2\]
since the function is quadratic, we can find its unique critical point to find its minima,
\[\hat{f}'(x_{k+1}) = f'(x_k) + f''(x_k)(x_{k+1}-x_k) = 0.\]
}

\end{itemize}

\end{frame}

%\begin{frame}{Successive Parabolic Interpolation}
%
%\begin{itemize}
%\item Like in secant methods, we can use previous points, rather than derivatives of $f$ to define quadratic approximation to $f$:
%
%\lgcond{
%Given three points, there is a unique quadratic function interpolating them.
%}
%
%\item The convergence rate of the resulting method is roughly $1.324$
%
%\lgcond{
%By comparison, the convergence of bisection is linear with a constant of $0.618$, while Newton's method converges quadratically.
%}
%
%\end{itemize}
%
%\end{frame}
%\end{document}

\begin{frame}{Successive Parabolic Interpolation}

\begin{itemize}
\item Interpolate $f$ with a quadratic function at each step and find its minima:

\lgcond{
Given three points, there is a unique quadratic function interpolating them.
}

\item The convergence rate of the resulting method is roughly $1.324$

\lgcond{
By comparison, the convergence of golden section search is linear with a constant of $0.618$, while Newton's method converges quadratically.
}

\end{itemize}

\end{frame}

%\begin{frame}{Newton's Method for Optimization}
%
%\begin{itemize}
%\item Instead of interpolating points, match derivatives at current approximate point:
%
%\lgcond{
%Pick quadratic function $\hat{f}$ as first three terms of Taylor expansion of $f$ about $x_k$, matching value and first two derivatives of $f$ at $x_k$.
%}
%
%\item The new approximate guess will be given by $x_{k+1}-x_k = -f'(x_k)/f''(x_k)$:
%
%\lgcond{
%\[f(x_{k+1}-x_k) \approx \hat{f}(x_{k+1}-x_k) = f(x_k)+f'(x_k)(x_{k+1}-x_k) + \frac{1}{2}f''(x_k)(x_{k+1}-x_k)^2\]
%since the function is quadratic, we can find its unique critical point to find its minima,
%\[\hat{f}'(x_{k+1}-x_k) = f'(x_k) + f''(x_k)(x_{k+1}-x_k) = 0.\]
%}
%
%\end{itemize}
%
%\end{frame}

\begin{frame}{Safeguarded 1D Optimization}

\begin{itemize}
\item Safeguarding can be done by bracketing via golden section search:

\lgcond{
Combination of Newton and golden section search
\begin{itemize}
\sitem achieves quadratic convergence locally,
\sitem is guaranteed convergence provided unimodality of function.
\end{itemize}
}

\item Backtracking and step-size control:

\lgcond{
\begin{itemize}
\sitem Can take smaller step $x_{k+1} = x_k - \alpha_k f'(x_k)/f''(x_k)$ for some $\alpha_k<1$.
\sitem Can backtrack and choose smaller $\alpha_k$ if $f(x_{k+1})>f(x_k)$.
\end{itemize}
}

\end{itemize}

\end{frame}

\section{Multidimensional Quadratic Optimization}

\subsection{Steepest Descent}

\begin{frame}{General Multidimensional Optimization}

\urcornerlinkdemo{06-optimization}{Nelder-Mead Method}

\begin{itemize}
\item Direct search methods by simplex (\coloremph{Nelder-Mead}):

\lgcond{
Form a $n$-point polytope in $n$-dimensional space and adjust worst point (highest function value) by moving it along a line passing through the centroid of the remaining points.
}

\item Steepest descent: find the minimizer in the direction of the negative gradient:

\lgcond{
\[\B{x}_{k+1} = \B{x}_k - \alpha_k \nabla f(\B x_k)\]
such that $f(\B{x}_{k+1})=\min_{\alpha_k} f(\B{x}_k - \alpha_k \nabla f(\B x_k))$, i.e. perform a line search (solve 1D optimization problem) in the direction of the negative gradient.
}

\end{itemize}

\end{frame}

\begin{frame}{Convergence of Steepest Descent}

\urcornerlinkdemo{06-optimization}{Steepest Descent}

\begin{itemize}
\item Steepest descent converges linearly with a constant that can be arbitrarily close to $1$:

\lgcond{
%See Michael Heath's notes chapter 6 slide 35 for an example.
\begin{itemize}
\sitem Convergence is slow locally, in the worst case, and generally depends on the Hessian near the minima.
\sitem If the gradient is changing quickly, it serves as good approximation only within a small local neighborhood, so the line search may result in arbitrarily small steps.
\end{itemize}
}
\item Given quadratic optimization problem $f(\B x)= \frac{1}{2}\B x^T\B A \B x + \B c^T \B x$ where $\B A$ is symmetric positive definite, the error $\B e_k = \B x_k - \B x^{*}$ satisfies
\lgcond{
\[||\B e_{k+1}||_{\B A} = \B e_{k+1}^T \B A \B e_{k+1} = \frac{\sigma_\text{max}(\B A) -\sigma_\text{min}(\B A)}{\sigma_\text{max}(\B A) +\sigma_\text{min}(\B A)}||\B e_k||_{\B A}\]

\begin{itemize}
\item When sufficiently close to a local minima, general nonlinear optimization problems are described by such an SPD quadratic problem.
\sitem Convergence rate depends on the conditioning of $\B A$, since
\[\frac{\sigma_\text{max}(\B A) -\sigma_\text{min}(\B A)}{\sigma_\text{max}(\B A) +\sigma_\text{min}(\B A)}=\frac{\kappa(\B A)-1}{\kappa(\B A)+1}.\]
\end{itemize}
}
\end{itemize}

\end{frame}

\begin{frame}{Gradient Methods with Extrapolation}

\begin{itemize}
\item We can improve the constant in the linear rate of convergence of steepest descent by leveraging \coloremph{extrapolation methods}, which consider two previous iterates (maintain \coloremph{momentum} in the direction $\B x_k -\B x_{k-1}$):
\lgcond{
\[\B x_{k+1} = \B x_k - \alpha_k \nabla f(\B x_k) + \beta_k(\B x_k - \B x_{k-1})\]

}

\item The \coloremph{heavy ball method}, which uses constant $\alpha_k=\alpha $ and $\beta_k=\beta$, achieves better convergence than steepest descent:

\lgcond{
\[||\B e_{k+1}||_{\B A} = \frac{\sqrt{\kappa(\B A)} -1}{\sqrt{\kappa(\B A) }+1}||\B e_k||_{\B A}\]
Nesterov's gradient optimization method is another instance of an extrapolation method that provides further improved optimality guarantees.

}

\end{itemize}

\end{frame}

\subsection{Conjugate Gradient}

\urcornerlinkdemo{06-optimization}{Conjugate Gradient Method}

\begin{frame}{Conjugate Gradient Method}

\begin{itemize}
\item The \coloremph{conjugate gradient method} is capable of making the optimal choice of $\alpha_k$ and $\beta_k$ at each iteration of an extrapolation method:

\lgcond{
\[(\alpha_k,\beta_k)=\argmin_{\alpha_k,\beta_k}\bigg[f\Big(\B x_k - \alpha_k \nabla f(\B x_k) + \beta_k(\B x_k - \B x_{k-1})\Big)\bigg]\]
\begin{itemize}
\item For SPD quadratic programming problems, conjugate gradient is an optimal 1st order method, converging in $n$ iterations.
\sitem It implicitly computes Lanczos iteration, searching along $\B A$-orthogonal directions at each step.
\end{itemize}
}

\item \coloremph{Parallel tangents} implementation of the method proceeds as follows

\lgcond{
\begin{enumerate}
\sitem Perform a step of steepest descent to generate $\B{\hat{x}}_{k}$ from $\B x_k$.
\sitem Generate $\B x_{k+1}$ by minimizing over the line passing through $\B x_{k-1}$ and $\B{\hat{x}}_{k}$.
\end{enumerate}
}

\end{itemize}

\end{frame}

\begin{frame}{Krylov Optimization}

\urcornerlinkdemo{06-optimization}{Conjugate Gradient Parallel Tangents as Krylov Subspace Method}

\begin{itemize}
\item Conjugate Gradient finds the minimizer of $f(\B x) = \frac{1}{2} \B x^T \B A \B x + \B c^T \B x $ within the Krylov subspace of $\B A$:
%Generally, Krylov subspaces describe the information available from $k$ matrix-vector products, and can be used to find an approximation $\B x_k$ to the minima of $\B c^T \B x - \B x^T \B A \B x $:

\lgcond{
\begin{itemize}
\sitem It constructs Krylov subspace $\mathcal{K}_k(\B A, \B c) = \text{span}(\B c,\B A\B c, \ldots, \B A^{r-1}\B c)$.
\sitem At the $k$th step conjugate gradient yields iterate
\[\B x_k = -||\B c||_2\B Q_k \B T_k^{-1} \B e_1 ,\]
where $\B Q_k$ are the Lanczos vectors associated with $\mathcal{K}_k(\B A, \B c)$ and $\B T_k =\B Q_k^T \B A \B Q_k$.
\sitem This choice of $\B x_k$ minimizes $f(\B x)$ since
\begin{align*}
\min_{\B x \in \mathcal{K}_k(\B A, \B c)} f(\B x) &= \min_{\B y \in \mathbb{R}^k} f(\B Q_k \B y)  \\
&=\min_{\B y \in \mathbb{R}^k} \B y^T \B Q_k^T \B A \B Q_k \B y + \B c^T \B Q_k \B y \\
&= \min_{\B y \in \mathbb{R}^k} \B y^T \B T_k \B y + ||\B c||_2\B e_1^T \B y
\end{align*}
is minimized by $\B y = -||\B c||_2\B T_k^{-1} \B e_1 $.
%\sitem MINRES/GMRES (usually used to solve least squares problems) minimize residual of optimality equation: $||\B A \B x + \B c||_2^2$.
%\sitem Our goal is to obtain $\B x_r = \argmin_{\B x \in \mathcal{K}_r}f(\B x)$:
%\sitem The $r$th iteration conjugate gradient gives
%\item there is insufficient information to minimize $||\B x_k - \B x^*||_2^2$
%\sitem The $r$th iterate of conjugate gradient minimizes this residual in $\B A^{-1}$ norm:
%\[\B x_r = \argmin_{\B x \in \mathcal{K}_r(\B A, \B c)}||\B A \B x + \B c||_{\B A^{-1}}^2=\argmin_{\B x \in \mathcal{K}_r(\B A, \B c)} \B x^T \B A \B x + 2 \B c^T\B x = \argmin_{\B x \in \mathcal{K}_r(\B A, \B c)} f(\B x).\]
%\[ = (\B c^T - \B A \B x_k)^T\B{A}^{-1}(\B c^T - \B A \B x_k)\]
\end{itemize}
}
\lgcond{}
\end{itemize}
%\item Conjugate gradient can be derived from vectors (columns of $\B Q_r$) generated by the Lanczos algorithm, in particular if $\B T_r = \B Q_r^T \B A \B Q_r$,
%\[\B x_r = -\B Q_r \B T_r^{-1} \B e_1 ||\B c||_2.\]
%
%\lgcond{
%This iterate minimizes the projection of the residual onto $\B Q_r=\begin{bmatrix} \frac{\B c}{||\B c||_2} & \cdots \end{bmatrix}$,
%\begin{align*}
%\B 0 =
%\B Q^T_r (\B c + \B A \B x_r) & = \B e_1 ||\B c||_2  + \underbrace{\B Q_r^T \B A\B Q_r}_{\B T_r} \B T_r^{-1} \B e_1||\B c ||_2
%\end{align*}
%}
\end{frame}

\section{General Nonlinear Optimization Methods}

\subsection{Newton's Method}

\begin{frame}{Newton's Method}

\urcornerlinkdemo{06-optimization}{Newton's Method in n dimensions}

\begin{itemize}
\item Newton's method in $n$ dimensions is given by finding minima of $n$-dimensional quadratic approximation:
\lgcond{
\[f(\B x_k+\B s)\approx \hat{f}(\B s) = f(\B x_k) + \B s^T \nabla f(\B x_k) + \frac{1}{2}\B s^T \B H_{f}(\B x_k)\B s.\]

The minima of this function can be determined by identifying critical points
\[\B 0=\nabla \hat{f}(\B s) = \nabla f(\B x_k) + \B H_{f}(\B x_k)\B s,\]
thus to determine $\B s$ we solve the linear system,
\[\B H_{f}(\B x_k)\B s =- \nabla f(\B x_k).\]
Assuming invertibility of the Hessian, we can write the Newton's method iteration as
\[\B x_{k+1} = \B x_k - \underbrace{\B H_{f}(\B x_k)^{-1}\nabla f(\B x_k)}_{\B s}.\]
Quadratic convergence follows by equivalence to Newton's method for solving nonlinear system of optimality equations \(\nabla f(\B x) = \B 0.\)
}
\lgcond{
}

\end{itemize}

\end{frame}

%\begin{frame}{Convergence of Newton's Method}
%
%\begin{itemize}
%\item Newton's method is quadratically convergent in $n$-dimensions as well:
%
%\lgcond{
%%Let $\B g(\B x)$ be the fixed-point iteration corresponding to Newton's method, using a Taylor expansion at the minima, which is valid in the limit of increasing $k$, we obtain
%\begin{align*}
%\B e_{k+1} &= \B x_{k+1} - \B x^* \\
%           &= \B x_k - \B x^* - \B H_{f}(\B x_k)^{-1}\nabla f(\B x_k) \\
%           &= -\B H_{f}(\B x_k)^{-1}(\B H_{f}(\B x_k)(\B x_k -\B x^*)- \nabla f(\B x_k)) \\
%           &= -\B H_{f}(\B x_k)^{-1}(\B H_{f}(\B x_k)(\B x_k -\B x^*)- (\nabla f(\B x_k) - \nabla f(\B x^*))), \\
%|| \B e_{k+1}|| &\leq ||\B H_{f}(\B x_k)^{-1}|| \underbrace{|| \B H_{f}(\B x_k)(\B x_k -\B x^*)- (\nabla f(\B x_k) - \nabla f(\B x^*) ||}_{O(||\B x_k - \B x^*||^2)} \\
%           &= C ||\B e_{k}||^2
%\end{align*}
%}
%\lgcond{}
%
%
%\end{itemize}
%
%\end{frame}

\subsection{Quasi-Newton Method}

\begin{frame}{Quasi-Newton Methods}

\begin{itemize}
\item \coloremph{Quasi-Newton} methods compute approximations to the Hessian at each step:

\lgcond{
\[\B x_{k+1}  = \B{x}_k -\alpha_k\B{B}_k^{-1}\nabla f(\B x_k)\]
where $\alpha_k$ is a line search parameter.
Quasi-Newton methods can be more robust than Newton's method, as the Newton's method step can lead to a direction in which the objective function is strictly increasing.
%Let $\B g(\B x)$ be the fixed-point iteration corresponding to Newton's method, using a Taylor expansion at the minima, which is valid in the limit of increasing $k$, we obtain
}

\item   The \coloremph{BFGS} method is a secant update method, similar to Broyden's method:

\lgcond{
\begin{itemize}
\sitem At each iteration, perform a rank-2 update to $\B B_{k}$ using $\B{s}_k = \B{x}_{k+1}-\B{x}_k$ and $\B{y_k} =\nabla f(\B{x}_{k+1})-\nabla f(\B{x}_{k})$:
\[\B{B}_{k+1} = \B{B}_k + \frac{\B{y}_k\B{y}_k^T}{\B{y}_k^T\B{s}_k} - \frac{\B{B}_k\B{s}_k \B{s}_k^T\B{B}_k}{\B{s}_k^T\B{B}_k \B{s}_k}\]
\sitem Can update inverse with $O(n^2)$ work, but its more stable and efficient to update a symmetric indefinite factorization.
\sitem The BFGS method also preserves symmetry of the Hessian approximation.
\end{itemize}
}

\end{itemize}

\end{frame}

\section{Nonlinear Least Squares}

\begin{frame}{Nonlinear Least Squares}

\begin{itemize}
\item An important special case of multidimensional optimization is \coloremph{nonlinear least squares}, the problem of fitting a nonlinear function $f_{\B x}(t)$ so that $f_{\B x}(t_i)\approx y_i$:

\mdcond{
For example, consider fitting $f_{[x_1,x_2]}(t) = x_1 \sin(x_2t)$ so that
\[\begin{bmatrix} f_{[x_1,x_2]}(1.5) \\f_{[x_1,x_2]}(1.9)\\f_{[x_1,x_2]}(3.2) \end{bmatrix} \approx \begin{bmatrix} -1.2 \\ 4.5 \\ 7.3 \end{bmatrix}.\]
}
\item We can cast nonlinear least squares as an optimization problem and solve it by Newton's method:

\lgcond{
Define residual vector function $\B r(\B x)$ so that $r_i(\B x)= y_i-f_{\B x}(t_i)$ and minimize
\[\phi(\B x)=\frac{1}{2}||\B r(\B x)||_2^2=\frac{1}{2}\B r(\B x)^T\B r(\B x).\]
Now the gradient is $\nabla \phi(\B x) = \B J_{\B r}^T(\B x)\B r(\B x)$ and the Hessian is
\[\B{H}_{\phi}(\B x) = \B{J}^T_{\B r}(\B x)\B{J}_{\B r}(\B x) + \sum_{i=1}^m r_i(\B x)\B{H}_{r_i}(\B x).\]
}
\mdcond{}

\end{itemize}

\end{frame}

\begin{frame}{Gauss-Newton Method}

\begin{itemize}
\item The Hessian for nonlinear least squares problems has the form:

\lgcond{
%Note that the Hessian used in Newton's method is given by
\[\B{H}_{\phi}(\B x) = \B{J}^T_{\B r}(\B x)\B{J}_{\B r}(\B x) + \sum_{i=1}^m r_i(\B x)\B{H}_{r_i}(\B x).\]
The second term is small when the residual function $\B r(\B x)$ is small, so approximate
\[\B H_{\phi}(\B x) \approx \B{\hat{H}}_{\phi}(\B x) = \B J_{\B r}^T(\B x)\B J_{\B r}(\B x).\]
%and we can observe that the error is small if $||\B r(\B x)||_2$ is small,
%\[||\B H_{\phi}(\B x) - \B{\hat{H}}_{\phi}(\B x)||_F = ||\B r(\B x)||_2 \sum_{i=1}^m ||\B H_{r_i}(\B x)||_F\]
}
\item The \coloremph{Gauss-Newton} method is Newton iteration with an approximate Hessian:
\lgcond{
  \begin{align*}
  \B x_{k+1} &= \B x_{k} - \B{\hat{H}}_{\phi}(\B x_k)^{-1} \nabla f(\B x_k)
             = \B x_{k} - (\B J_{\B r}^T(\B x_k)\B J_{\B r}(\B x_k))^{-1} \B J_{\B r}^T(\B x_k)\B r(\B x_k).
  \end{align*}
Recognizing the normal equations, we interpret the Gauss-Newton method as solving  linear least squares problems $\B J_{\B r}(\B x_k)\B s_k  \cong \B r(\B x_k), \B x_{k+1}= \B x_k + \B s_k$.
}
\item The Levenberg-Marquardt method incorporates Tykhonov regularization into the linear least squares problems within the Gauss-Newton method.
\end{itemize}

\end{frame}
%\begin{frame}{Levenberg-Marquardt Method}
%
%\begin{itemize}
%\item The \coloremph{Levenberg-Marquardt} method modifies the Gauss-Newton method to use Tykhonov regularization:
%
%\lgcond{
%  \begin{align*}
%  \B x_{k+1} &= \B x_{k} - (\B J_{\B r}^T(\B x_k)\B J_{\B r}(\B x_k) - \mu \B I)^{-1} \B J_{\B r}(\B x_k)\B r(\B x_k).
%  \end{align*}
%}
%
%\item The scalar $\mu$ controls the step size through the least squares problem:
%
%
%\lgcond{
%  \[\begin{bmatrix} \B J_{\B r}(\B x_k) \\ \sqrt{\mu} \B I \end{bmatrix} \B s_k \cong \begin{bmatrix} \B r(\B x_k) \\ \B 0 \end{bmatrix}\]
%By increasing $\mu$ we force the step $\B s$ to be smaller.
%}
%\end{itemize}
%
%\end{frame}

\section{Constrained Optimization Methods}

\subsection{Overview of Approaches}

%\end{document}

\begin{frame}{Constrained Optimization Problems}

\begin{itemize}
\item We now return to the general case of \coloremph{constrained} optimization problems:
\lgcond{
\[\min_{\B x} f(\B x)\quad \text{subject to} \quad \B g(\B x) = \B 0 \quad \text{and} \quad \B{h}(\B x) \leq \B 0\]

When $f$ is quadratic, while $h$, $g$ is linear, this is a \coloremph{quadratic optimization} problem.
}

\item Generally, we will seek to reduce constrained optimization problems to a series of unconstrained optimization problems:

%\lgcond{
\begin{itemize}
\mitem \coloremph{sequential quadratic programming}: \filufil{solve an unconstrained quadratic optimization problem at each iteration,}{\medskip}
\mitem \coloremph{penalty-based methods}: \filufil{solve a series of more complicated (more ill-conditioned) unconstrained optimization problems,}{\medskip}
\mitem \coloremph{active set methods}: \filufil{define sequence of optimization problems with inequality constrains ignored or treated as equality constraints.}{\medskip}
\end{itemize}
%}
\end{itemize}

\end{frame}

%\begin{frame}{Constrained Optimality}
%
%\begin{itemize}
%\item In equality-constrained optimization $\B g(\B x)=\B 0$, minimizers $\B x^*$ are on the border of the feasible region (set of points satisfying constraints),
%      in which case we must ensure any direction of decrease of $f$ from $\B x^*$ leads to an infeasible point, which gives us the condition:
%
%\[\exists \B \lambda\in\mathbb{R}^n, \quad -\nabla f(\B x^*) = \B J^T_{\B g}(\B x^*) \B \lambda\]
%
%\mdcond{
%$\B \lambda$ are referred to as the Lagrange multipliers.
%}
%
%\item Seek critical points in the Lagrangian function $\mathcal{L}(\B x, \B \lambda) = f(\B x) + \B \lambda^T\B g(\B x)$, described by the nonlinear equation,
%
%\[\nabla \mathcal{L}(\B x, \B \lambda) = \begin{bmatrix} \nabla f(\B x) + \B J_{\B g}^T(\B x) \B \lambda \\ \B g(\B x)\end{bmatrix} = \B 0\]
%
%\lgcond{
%Seeking $\B \lambda$ that maximizes the global minimum of $\mathcal{L}(\B x, \B \lambda)$ defines the \coloremph{dual optimization problem}.
%}
%\mdcond{}
%
%
%\end{itemize}
%
%
%\end{frame}

\subsection{Optimization with Equality Constraints}

%\begin{frame}{Sequential Quadratic Programming}
%
%\begin{itemize}
%\item \coloremph{Sequential quadratic programming (SQP)} reduces a nonlinear equality constrained problem to a sequence of constrained quadratic programs
%via a Taylor expansion of the Lagrangian function $\mathcal{L}_f(\B x, \B \lambda)=f(\B x)+\B \lambda ^T\B g(\B x)$:
%\lgcond{
%\begin{align*}
%q(\B x_k+ \B s,\B \lambda_k + \B \delta ) =& \mathcal{L}_f(\B x_k, \B \lambda_k) + \B s^T (\nabla f(\B x_k) + \B J_{\B g}^T(\B x_k)\B \lambda_k) + \frac{1}{2}\B s^T \B B(\B x_k, \B \lambda_k)\B s \\
%&+\B \delta^T (\B J_{\B g}(\B x_k) \B s + \B g(\B x_k))
%\end{align*}
%where $\B B(\B x, \B \lambda) = \B H_{f}(\B x) + \sum_{i=1}^m \lambda_i \B H_{g_i}(\B x)$
%}
%\item SQP ignores the constant term $\mathcal{L}_f(\B x_k, \B \lambda_k)$ and minimizes $\B s$ while treating $\B \delta$ as a Lagrange multiplier:
%
%\lgcond{
%The above unconstrained quadratic program corresponds to the Lagrangian form of the constrained quadratic program
%\begin{align*}
%\max_{\B s} \B s^T (\nabla f(\B x_k) + \B J_{\B g}^T(\B x_k)\B \lambda_k) + \frac{1}{2}\B s^T \B B(\B x_k, \B \lambda_k)\B s
%\end{align*}
%with constraint $\B J_{\B g}(\B x_k) \B s = -\B g(\B x_k)$.
%
%%and use a linear approximation of the constraint $\B g(\B x_k + \B s)=\B 0$,
%%subject to the linear constraint approximation $\B J_{\B g}(\B x_k) \B s = -\B g(\B x_k)$.
%}
%
%%\item We can reduce a constrained quadratic program to an unconstrained one, using the Lagrangian function of $q$:
%%\lgcond{
%%\[\mathcal{L}_q(\B x_k+ \B s,\B \lambda + \B \delta, \B \mu ) = f(\B x_k) + \B s^T (\nabla f(\B x_k) + \B J_g^T(\B x_k)\B \lambda_k) + \frac{1}{2}\B s^T \B B(\B x_k)\B s + \mu(\B J_{\B g}(\B x_k) \B s +\B g(\B x_k))\]
%%which we can solve directly as a linear system.
%%%\[\mathcal{L}_{\hat{f}}(\B x_k+ \B s,\B \lambda) = f(\B x_k) + \B s^T \nabla f(\B x_k) + \frac{1}{2}\B s^T \B H_{f}(\B x_k)\B s + \B \lambda^T(\B J_{\B g}(\B x_k) \B s + \B g(\B x_k)) \]
%%The minima of $q$ will correspond to a critical point of $\mathcal{L}_{\hat{f}}$, which may or may not be the true minima.
%%}
%%
%\end{itemize}
%
%
%\end{frame}

\begin{frame}{Sequential Quadratic Programming}

\urcornerlinkdemo{06-optimization}{Sequential Quadratic Programming}

\begin{itemize}
\item \coloremph{Sequential quadratic programming} (SQP) corresponds to using Newton's method to solve the equality constrained optimality conditions, by finding critical points of the Lagrangian function $\mathcal{L}(\B x, \B \lambda) = f(\B x) + \B \lambda^T\B g(\B x)$,
\mdcond{
\[\nabla \mathcal{L}(\B x, \B \lambda) = \begin{bmatrix} \nabla f(\B x) + \B J_{\B g}^T(\B x) \B \lambda \\ \B g(\B x)\end{bmatrix} = \B 0\]
}

\item At each iteration, SQP computes $\begin{bmatrix} \B x_{k+1} \\ \B \lambda_{k+1}\end{bmatrix} = \begin{bmatrix} \B x_{k}\\ \B \lambda_{k}\end{bmatrix} + \begin{bmatrix} \B s_k \\ \B\delta_k \end{bmatrix}$ by solving
\lgcond{
\[\B H_{\mathcal{L}}(\B x_k, \B \lambda_k)\begin{bmatrix} \B s_k \\ \B\delta_k \end{bmatrix} =  -\nabla \mathcal{L}(\B x_k, \B \lambda_k)\]
% = -\begin{bmatrix} \nabla f(\B x_k) + \B J_{\B g}^T(\B x_k) \B \lambda_k \\ \B g(\B x_k)\end{bmatrix}
where
\[\B H_{\mathcal{L}}(\B x_k, \B \lambda_k) = \begin{bmatrix} \B B(\B x_k, \B\lambda_k) & \B J^T_{\B g}(\B x_k) \\ \B J_{\B g}(\B x_k) & \B 0\end{bmatrix} \quad \text{with} \quad \B B(\B x, \B \lambda) = \B H_{f}(\B x) + \sum_{i=1}^m \lambda_i \B H_{g_i}(\B x)\]
}

\end{itemize}

\end{frame}

%\begin{frame}{Quadratic Programming Problems}
%
%\begin{itemize}
%\item An equality-constrained quadratic programming problem has the form
%\[\min_{\B x} f(\B x), \quad f(\B x)=\frac{1}{2} \B x^T \B Q \B x + \B c^T \B x \quad \text{subject to} \quad \B A \B x = \B b\]
%where $\B Q$ is symmetric.
%
%\lgcond{
%Its first-order optimality condition in the unconstrained case is
%\[\B 0 = \nabla f(\B x)  = \B Q \B x + \B c \quad \Rightarrow \quad \B Q \B x =-\B c\]
%and in the constrained case, becomes
%\[ \begin{bmatrix} \B Q & \B A^T \\ \B A & \B 0 \end{bmatrix} \begin{bmatrix} \B x \\ \B \lambda \end{bmatrix} = - \begin{bmatrix} \B c \\ \B b \end{bmatrix} \]
%This allows us to observe, that at the $k$th iteration SQP solves a QP with $\B Q =\B B(\B x_k, \B\lambda_k)$, $\B A =\B J_g(\B x_k)$, $\B c =\nabla f(\B x_k) + \B J_{\B g}^T(\B x_k)$, and $\B b = \B g(\B x_k)$.
%}
%\end{itemize}
%
%
%\end{frame}
%
%\begin{frame}{Steepest Descent for Quadratic Programming}
%
%\begin{itemize}
%\item Near the minima, all smooth nonlinear programming problems look like quadratic programming problems, where $\B Q$ converges to the Hessian at the minima, $\B H_f(\B x^*)$.
%
%\lgcond{}
%
%\item Consequently, we can analyze local convergence of methods by considering their convergence for a QP, e.g. for steepest descent:
%
%\lgcond{
%Assume $\B b =\B 0$ and $\B c =\B 0$, then we have $\nabla f(\B x) = \B Q \B x$.
%Consequently, steepest descent will seek a step-size $\alpha_k$ to perform the step
%\[\B x_{k+1} = \B x_{k} -\alpha_k \B Q \B x_k = (\B I - \alpha_k \B Q) \B x_k\]
%for this fixed-point iteration to converge, it suffices that $\alpha_k<2/\lambda_{\text{max}}(\B Q)$.
%The optimal value can be shown to be $\alpha^*=2/(\lambda_{\text{max}}(\B Q) + \lambda_{\text{min}}(\B Q))$, leading to convergence rate
%\[||\B e_{k+1}|| = \frac{\kappa(\B Q) -1}{\kappa(\B Q) +1}||\B e_k||\]
%}
%
%\end{itemize}
%
%\end{frame}

\subsection{Optimization with Inequality Constraints}

\begin{frame}{Inequality Constrained Optimality Conditions}

\begin{itemize}

\item The \coloremph{Karush-Kuhn-Tucker (KKT)} conditions hold for local minima of a problem with equality and inequality constraints, the key conditions are

\lgcond{
\begin{itemize}
\sitem First, any minima $\B x^*$ must be a feasible point, so $\B g(\B x^*) =\B 0$ and $\B h(\B x^*) \leq \B 0$.
\sitem We say the $i$th inequality constraint is \coloremph{active} at a minima $\B x^*$ if $h_i(\B x^*) = 0$.
\sitem The collection of equality constraints and active inequality constraints $\B q^*$, satisfies $\B q^*(\B x^*) = \B 0$.
\sitem The negative gradient of the objective function at the minima must be in the row span of the Jacobian of this collection of constraints:
\[-\nabla f(\B x^*)= \B J_{\B q^*}^T(\B x^*)\B \lambda^* \quad \text{where $\B \lambda^*$ are Lagrange multiplers of constraints in $\B q^*$} .\]
\end{itemize}
}
%for the general Lagrangian function $\mathcal{L}(\B x, \B \mu, \B \nu) = f(\B x)+\B \mu^T \B g(\B x) + \B \nu^T \B h(\B x)$ are
%%\lgcond{
%\begin{align*}
%\nabla_{\B x}\mathcal{L} (\B x, \B \lambda) & = \B 0\\
%\B g(\B x) &= \B 0\\
%\B h(\B x) &\leq \B 0\\
%\B \nu &\geq \B 0\\
%\B \nu^T \B h (\B x)  &= \B 0
%\end{align*}
%\smcond{
%At an optimal point, we must have that for either the $i$th inequality constraint is \coloremph{active}, so $h_i(\B x)=\B 0$ or it is inactive, but its Lagrange multiplier $\nu_i=0$.
%}
\item %SQP can handle only equality constraints,
To use SQP for an inequality constrained optimization problem, consider at each iteration an \coloremph{active set} of constraints:

\lgcond{
\begin{itemize}
\sitem Active set $\B q_k$ contains all equality constraints and all inequality constraints that are exactly satisfied or violated at $\B x_k$.
\sitem Perform one step of Newton's method to minimize $\mathcal{L}_k(\B x, \B \lambda) = f(\B x) + \B \lambda^T \B q_k(\B x)$ with respect to $\B x$ and $\B \lambda$, then update active set.
%\sitem A minima $\B x^*$ will be a feasible point and if the constraints in the active set at $x^*$ are $\B q(\B x)$ with Lagrange multipliers $\B \lambda$, then
\end{itemize}
%A constraint is active if the current iterate is at a boundary satisfies the inequality constraint as an equality (i.e. $i$th constraint is active if $i$th component of $\B h(\B x_k)$ is zero).
}

\end{itemize}

\end{frame}

%\begin{frame}{Optimality Conditions for Inequality Constraints}
%
%\begin{itemize}
%\item The Lagrangian function with constraints $\B g(\B x) = \B 0$ and $\B{h}(\B x) \leq \B 0$ is
%\lgcond{
%\[\mathcal{L}(\B x, \B \lambda) = f(\B x) + \B \lambda^T \begin{bmatrix} \B h(\B x) \\ \B g(\B x)\end{bmatrix}\]
%
%%\[\mathcal{L}(\B x, \B \lambda) = f(\B x) +  \B \lambda_1^T \B h(\B x)  + \B \lambda_2^T \B g(\B x)\]
%}
%
%\item The Lagrangian dual problem is an unconstrained optimization problem:
%\[\max_{\B \lambda} q(\B \lambda), \quad q(\B \lambda) =   \begin{cases}  \min_{\B x} \mathcal{L}(\B x, \B \lambda) & \text{if } \B \lambda \geq \B 0 \\   -\infty & \text{otherwise}\end{cases}\]
%The unconstrained optimality condition $\nabla q(\B \lambda^*) = \B 0$, implies
%\lgcond{
%\[\max\left(\B \lambda^*,\begin{bmatrix} \B h(\B x) \\ \B g(\B x)\end{bmatrix}\right) = \B 0\]
%when $\lambda_i^*= 0$, we say the $i$th constraint is inactive at the minimum point.
%Equality constraints are active at any feasible point.
%}
%
%
%\end{itemize}
%
%
%\end{frame}

%\begin{frame}{Merit Functions}
%
%\begin{itemize}
%\item We can reduce constrained optimization problems to unconstrained ones by modifying the objective function. \coloremph{Penalty} methods are effective for equality constraints $\B g(\B x)=0$:
%
%\lgcond{
%A simple penalty function parameterized by $\rho >0$ is
%\[\phi_\rho(\B x) = f(\B x) + \frac{1}{2} \rho \B g(\B x)^T \B g(\B x)\]
%a more robust variant is the augmented Lagrangian function
%\[\mathcal{L}_\rho(\B x,\B \lambda) = f(\B x) + \B \lambda^T \B g(\B x)+\frac{1}{2} \rho \B g(\B x)^T \B g(\B x)\]
%by increasing $\rho$, in theory convergence to an optimal solution can be expected, however, we must insure the Hessian of $\phi$ is not too ill-conditioned.
%
%}
%
%\item Barrier functions provide an effective way (\coloremph{interior point methods}) of working with inequality constraints $\B h(\B x)\leq \B 0$:
%
%\lgcond{
%To enforce inequality constraints via the objective function, we can introduce barriers, like
%\[\phi_\mu(\B x) = f(\B x) - \mu \sum_{i=1}^p\frac{1}{h_i(\B x)}\]
%which is called the inverse barrier function, or the logarithmic barrier function
%\[\phi_\mu(\B x) = f(\B x) - \mu \sum_{i=1}^p\log(-h_i(\B x))\]
%hoping to obtain convergence by decreasing $\mu$ and ensuring the step size is sufficiently small.
%
%}
%
%
%
%\end{itemize}
%
%\end{frame}

\begin{frame}{Penalty Functions}

\begin{itemize}
\item Alternatively, we can reduce constrained optimization problems to unconstrained ones by modifying the objective function. \coloremph{Penalty} functions are effective for equality constraints $\B g(\B x)=0$:

\lgcond{
\[\phi_\rho(\B x) = f(\B x) + \frac{1}{2} \rho \B g(\B x)^T \B g(\B x)\]
is a simple merit function, and its solutions $\B x^*_\rho$ satisfy $\lim_{\rho \to \infty} \B x_\rho^* = \B x^*$.
However, the Hessian of $\phi_\rho$ becomes increasingly ill-conditioned for large $\rho$, leading to slow convergence.
}

\item The augmented Lagrangian function provides a more numerically robust approach:
\lgcond{
\[\mathcal{L}_\rho(\B x, \B \lambda) = f(\B x) + \B \lambda^T\B g(\B x) + \frac{1}{2} \rho \B g(\B x)^T \B g(\B x)\]
}
\end{itemize}

\end{frame}

\begin{frame}{Barrier Functions}

\begin{itemize}

\item \coloremph{Barrier functions} (\coloremph{interior point methods}) provide an effective way of working with inequality constraints $\B h(\B x)\leq \B 0$:

\lgcond{
\begin{itemize}
\item Provided we start at a feasible point, modify objective function so it diverges to $\infty$ when approaching border of feasible region.
\mitem Inverse barrier function: \[\phi_\mu(\B x) = f(\B x) - \mu\sum_{i=1}^m\frac{1}{h_i(\B x)}.\]
\mitem Logarithmic barrier function: \[\phi_\mu(\B x) = f(\B x) - \mu\sum_{i=1}^m\log(-h_i(\B x)).\]
%The minimum $\phi_\mu$ is guaranteed to be in the feasible domain and
\mitem When using sufficiently small steps, we have $\B x^*_\mu\to \B x^*$ as $\mu \to 0$.
\end{itemize}
}
\lgcond{}

\end{itemize}

\end{frame}

%\begin{frame}{Linear Programming}
%
%\begin{itemize}
%
%\item A linear program is a quadratic program with $\B Q = \B O$:
%
%\mdcond{}
%
%\item The simplex method is effective for linear programs, and operates by using the fact that the minimum over a convex polyhedron must occur at one of its vertices:
%
%\lgcond{}
%
%\item The sequential linear-quadratic programming algorithm provides an effective way of solving linear programs iwth inequality constraints:
%
%\mdcond{}
%
%\end{itemize}
%
%\end{frame}
%\begin{frame}{Optimization with Data}
%
%\begin{itemize}
%
%\item Lets consider the matrix completion problem, which is a nonlinear optimization problem:
%
%\lgcond{
%Given a subset of entries
%\[\Omega\subseteq \{1,\ldots,n\}\times \{1,\ldots, n\}\]
%of the entries of matrix $\Mat{A}\in \rmn nn$, seek rank-$k$ approximation
%\[\argmin_{\Mat{W}\in \rmn mk, \Mat{H}\in \rmn nk} \sum_{(i,j)\in \Omega}\Big(\underbrace{a_{ij}-\sum_l w_{il}h_{jl}}_{(\Mat{A}-\Mat{W}\Mat{H}^T)_{ij}}\Big)^2 + \lambda(||\Mat{W}||_F + ||\Mat{H}||_F)\]
%}
%\lgcond{}
%\end{itemize}
%
%\end{frame}
%
%
%\begin{frame}{Alternating Least Squares}
%
%\begin{itemize}
%\item Fix $\Mat{W}$ and solving for $\Mat{H}$ by quadratic optimization:
%%\[\argmin_{\Mat{W}\in \rmn mk} \sum_{(i,j)\in \Omega} \Big(a_{ij}-\sum_l {w}_{il}{h}_{jl}\Big)^2 + \lambda||\Mat{W}||_F\]
%%\lgcond{The optimization problem is independent for rows of $\Mat{W}$. Letting $\Vec{w}_i=w_{i\ALL}$, $\Vec{h}_i=h_{i\ALL}$, $\Omega_i =\{j : (i,j)\in\Omega\}$, seek
%\[\argmin_{\Vec{w}_i\in \rn k}  \sum_{j\in \Omega_i} \Big(a_{ij}-\Vec{w}_i\Vec{h}_j^\T\Big)^2 + \lambda||\Vec{w}_i||_2\]
%%}
%%\item
%\lgcond{
%Seek minimizer $\Vec{w}_i$ for quadratic vector equation
%\[f(\Vec{w}_i) = \sum_{j\in \Omega_i} \Big(a_{ij}-\Vec{w}_i\Vec{h}_j^\T\Big)^2 + \lambda||\Vec{w}_i||\]
%
%Differentiating with respect to $\Vec{w}_i$ gives
%\[\frac{\partial f(\Vec{w}_i)}{\partial \Vec{w}_i} = 2\sum_{j\in \Omega_i}\Vec{h}_j^\T\Big( \Vec{w}_i\Vec{h}_j^\T-a_{ij}\Big) + 2\lambda \Vec{w}_i=0\]
%Rotating $\Vec{w}_i\Vec{h}_j^\T=\Vec{h}_j\Vec{w}_i^\T$ and defining $\Mat{G}^{(i)}=\sum_{j\in\Omega_i}\Vec{h}_j^\T \Vec{h}_j$,
%%$\in \rmn{|\Omega_i|}{k}$
%%=[\Vec{h}_{j_1}; \Vec{h}_{j_2};...]$ $$
%\[(\Mat{G}^{(i)}+\lambda \Mat{I})\Vec{w}_i^T = \sum_{j\in \Omega_i}\Vec{h}_j^\T a_{ij}\]
%which is a $k\times k$ symmetric linear system of equations
%}
%\lgcond{}
%\end{itemize}
%
%\end{frame}
%
%\begin{frame}{Gradient Descent for Matrix Completion}
%
%\begin{itemize}
%\mitem Recall that we seek to minimize
%\[f(\Vec{w}_i) = \sum_{j\in \Omega_i} \Big(a_{ij}-\Vec{w}_i\Vec{h}_j^\T\Big)^2 + \lambda||\Vec{w}_i||\]
%ALS minimizes $\Vec{w}_i$, gradient descent methods only improve it using the partial derivative
%\[\frac{\partial f(\Vec{w}_i)}{\partial \Vec{w}_i} = 2\sum_{j\in \Omega_i}\Vec{h}_j^\T\Big( \Vec{w}_i\Vec{h}_j^\T-a_{ij}\Big) + 2\lambda \Vec{w}_i
%%\item we can use $r_{ij}=a_{ij}-\Vec{h}_j^\T \Vec{w}_i$ to write this as
%%\[\frac{\partial f(\Vec{w}_i)}{\partial \Vec{w}_i}
%=   2\bigg(\lambda \Vec{w}_i-\sum_{j\in \Omega_i}r_{ij}\Vec{h}_j\bigg) \]
%\mdcond{}
%\item \coloremph{Gradient descent} method updates
%\[\Vec{w}_i=\Vec{w}_i-\eta \frac{\partial f(\Vec{w}_i)}{\partial \Vec{w}_i}\]
%\lgcond{
%where parameter $\eta$ is our step-size
%}
%\end{itemize}
%
%\end{frame}
%
%\begin{frame}{Stochastic Gradient Descent (SGD)}
%
%\begin{itemize}
%\item \coloremph{Stochastic gradient descent (SGD)} performs fine-grained updates based on a component of the overall gradient
%\[\frac{\partial f(\Vec{w}_i)}{\partial \Vec{w}_i}
%=   2\bigg(\lambda \Vec{w}_i-\sum_{j\in \Omega_i}r_{ij}\Vec{h}_j\bigg)
%=   2\sum_{j\in \Omega_i}\lambda \Vec{w}_i/|\Omega_i|-r_{ij}\Vec{h}_j \]
%where $\B R$ is the residual on entries $\Omega$.
%
%\lgcond{
%SGD selects random $(i,j)\in\Omega$ and updates $\Vec{w}_i$ using $\Vec{h}_j$
%\[\Vec{w}_i\leftarrow \Vec{w}_i-\eta(\lambda \Vec{w}_i/|\Omega_i| - r_{ij}\Vec{h}_j)\]
%}
%%\item SGD randomly selects pairs $(i,j)\in\Omega$ and updates $\Vec{w}_i$ (and $\Vec{h}_j$ in a dual fashion)
%\item SGD then updates $r_{ij}=a_{ij}-\Vec{w}_i^\T \Vec{h}_j$, for a low overall cost:
%
%\lgcond{
%Each update costs $O(k)$ operations if $k$ is the rank.
%Stochastic properties make the method very effective in practice.
%Techniques such as using momentum and extrapolation apply also in this context.
%}
%%\item $O(|\Omega|)$ yield the same total cost as CCD-based updates of $W$ and $H$
%\end{itemize}
%
%\end{frame}
%
%
%\begin{frame}{Coordinate Descent}
%
%Rather than whole rows $\Vec w_i$ solve for elements of $\Mat{\Mat{W}}$, recall
%\[\argmin_{\Mat{W}\in \rmn mk} \sum_{(i,j)\in \Omega} \Big(a_{ij}-\sum_l w_{il}h_{jl}\Big)^2 + \lambda||\Mat{W}||_F\]
%\begin{itemize}
%\item \coloremph{Coordinate descent} finds the best replacement $\prm$ for $w_{it}$
%\[\prm = \argmin_{\prm} \sum_{j\in \Omega_i} \Big(a_{ij}-\prm h_{jt}-\sum_{l\neq t} w_{il}h_{jl}\Big)^2 + \lambda \prm^2\]
%
%\mdcond{}
%
%\item The solution is given by
%%\[\prm= \frac{\sum_{j\in \Omega_i} h_{jt}\Big(a_{ij}-\sum_{l\neq t} w_{il}h_{jl}\Big)}{
%%\lambda +\sum_{j\in\Omega_i}h_{jt}^2}\]
%\[\prm= \frac{\sum_{j\in \Omega_i} h_{jt}\Big(a_{ij}-\sum_{l\neq t} w_{il}h_{jl}\Big)}{
%\lambda +\sum_{j\in\Omega_i}h_{jt}^2}
%%
%%can be computed as
%%\[\prm=
%=\frac{\sum_{j\in \Omega_i} h_{jt}\Big(r_{ij}+ w_{it}h_{jt}\Big)}{
%\lambda +\sum_{j\in\Omega_i}h_{jt}^2}\]
%
%\mdcond{
%after which we can update $\Mat{R}$ via
%\[r_{ij}\gets r_{ij}-(\prm-w_{it})h_{jt}\quad \forall j\in \Omega_i\]
%}
%
%
%\end{itemize}
%
%\end{frame}

%\end{document}


\begin{appendix}

\chapter{Appendix A: Gas Case Data File Format}
\label{app:gas_format}

All details about the gas case (\code{mgc}) format are provided in the tables below. For the sake of convenience and code portability, \code{idx\_node} defines a set of constants (positive integers) to be used as named indices into the columns of the \code{node.info} matrix. Similarly, \code{idx\_well}, \code{idx\_pipe}, \code{idx\_comp}, and \code{idx\_sto} defines names for the columns in \code{well}, \code{pipe}, \code{comp}, and \code{sto}, respectively. On the other hand, \code{mgc\_PU} converts from real to per-unit (P.U) quantities, while \code{mgc\_REAL} converts from P.U to real values. Moreover, the \code{pbase}, \code{fbase}, and \code{wbase} fields are simple scalar values to define the gas system pressure, flow and power bases, respectively.

\begin{table}[!ht]
	\centering
	\begin{threeparttable}
		\caption{Node Information Data (\code{mgc.node.info})}
		\label{tab:nodedata}
		\footnotesize
		\begin{tabular}{lcl}
			\toprule
			name & column & description \\
			\midrule
			\code{NODE\_I}	& 1	& node number (positive integer)\\
			\code{NODE\_TYPE}	& 2	& node type (1 = demand node, 2 = extraction node)\\
			\code{PR}	& 3	& pressure [psia]\\
			\code{PRMAX}	& 4	& maximum pressure [psia]\\
			\code{PRMIN}	& 5	& minimum pressure [psia]\\
			\code{OVP}	& 6	& over-pressure [psia]\\
			\code{UNP}	& 7	& under-pressure [psia]\\
			\code{COST\_OVP}	& 8	& over-pressure cost [\$/psia$^2$]\\
			\code{COST\_UNP}	& 9	& under-pressure cost [\$/psia$^2$]\\
			\code{GD}	& 10	& full nodal demand [MMSCFD]\tnote{\dag}\\
			\code{NGD}	& 11	& number of different nodal users (positive integer)\\
			\bottomrule
		\end{tabular}
		\begin{tablenotes}
			\scriptsize
			\item [\dag] {MMSCFD: Million Standard Cubic Feet Per Day.}
		\end{tablenotes}
	\end{threeparttable}
\end{table}

\begin{table}[!ht]
	\centering
	\begin{threeparttable}
		\caption{Well Information Data (\code{mgc.well})}
		\label{tab:welldata}
		\footnotesize
		\begin{tabular}{lcl}
			\toprule
			name & column & description \\
			\midrule
			\code{WELL\_NODE}	& 1	& well number (positive integer)\\
			\code{G}	& 2	& well gas production [MMSCFD]\\
			\code{PW}	& 3	& known well pressure [psia]\\
			\code{GMAX}	& 4	& maximum gas injection [MMSCFD]\\
			\code{GMIN}	& 5	& minimum gas injection [MMSCFD]\\
			\code{WELL\_STATUS}	& 6	& well status (0 = disable, 1 = enable)\\
			\code{COST\_G}	& 7	& well production cost [\$/MMSCFD]\\
			\bottomrule
		\end{tabular}
	\end{threeparttable}
\end{table}

\begin{table}[!ht]
	\centering
	\begin{threeparttable}
		\caption{Pipeline Information Data (\code{mgc.pipe})}
		\label{tab:pipedata}
		\footnotesize
		\begin{tabular}{lcl}
			\toprule
			name & column & description \\
			\midrule
			\code{F\_NODE}	& 1	& from node number (positive integer)\\
			\code{T\_NODE}	& 2	& to node number (positive integer)\\
			\code{FG\_O}	& 3	& known gas pipeline flow [MMSCFD]\\
			\code{K\_O}	& 4	& Weymouth constant [MMSCFD/psia]\\
			\code{DIAM}	& 5	& diameter [inches]\\
			\code{LNG}	& 6	& longitude [km]\\
			\code{FMAX\_O}	& 7	& maximum flow [MMSCFD]\\
			\code{FMIN\_O}	& 8	& minimum flow [MMSCFD]\\
			\code{COST\_O}	& 9	& pipeline transportation cost [\$/MMSCFD]\\
			\bottomrule
		\end{tabular}
	\end{threeparttable}
\end{table}

\begin{table}[!ht]
	\centering
	\begin{threeparttable}
		\caption{Compressor Information Data (\code{mgc.comp})}
		\label{tab:compdata}
		\footnotesize
		\begin{tabular}{lcl}
			\toprule
			name & column & description \\
			\midrule
			\code{F\_NODE}	& 1	& from node number (positive integer)\\
			\code{T\_NODE}	& 2	& to node number (positive integer)\\
			\code{TYPE\_C}	& 3	& compressor type (1 = power-driven, 2 = gas-driven)\\
			\code{FG\_C}	& 4	& gas flow through compressor [MMSCFD]\\
			\code{PC\_C}	& 5	& consumed compressor power [MVA]\\
			\code{GC\_C}	& 6	& gas consumed by the compressor [MMSCFD]\tnote{\dag}\\
			\code{RATIO\_C}	& 7	& maximum compressor ratio\\
			\code{B\_C}	& 8	& compressor-dependent constant [MVA/MMSCFD]\\
			\code{Z\_C}	& 9	& compresibility factor\\
			\code{X}	& 10	& independent approximation coefficient [MMSCFD]\\
			\code{Y}	& 11	& linear approximation coefficient [MMSCFD/MVA]\\
			\code{Z}	& 12	& quadratic approximation coefficient [MMSCFD/MVA$^2$]\\
			\code{FMAX\_C}	& 13	& maximum flow through compressor [MMSCFD]\\
			\code{COST\_C}	& 14	& compressor cost [\$/MMSCFD]\\
			\bottomrule
		\end{tabular}
		\begin{tablenotes}
			\scriptsize
			\item [\dag] {Only relevant for a gas-driven compressor.}
		\end{tablenotes}
	\end{threeparttable}
\end{table}

\begin{table}[!ht]
	\centering
	\begin{threeparttable}
		\caption{Storage Information Data (\code{mgc.sto})}
		\label{tab:storedata}
		\footnotesize
		\begin{tabular}{lcl}
			\toprule
			name & column & description \\
			\midrule
			\code{STO\_NODE}	& 1	& node number (positive integer)\\
			\code{STO}	& 2	& end of day storage level [MSCF]\tnote{\dag}\\
			\code{STO\_0}	& 3	& initial storage level [MSCF]\\
			\code{STOMAX}	& 4	& maximum storage [MSCF]\\
			\code{STOMIN}	& 5	& minimum storage [MSCF]\\
			\code{FSTO}	& 6	& storage outflow difference [MMSCFD]\tnote{\ddag}\\
			\code{FSTO\_OUT}	& 7	& storage outflow [MMSCFD]\\
			\code{FSTO\_IN}	& 8	& storage inflow [MMSCFD]\\
			\code{FSTOMAX}	& 9	& maximum storage outflow difference [MMSCFD]\\
			\code{FSTOMIN}	& 10	& minimum storage outflow difference [MMSCFD]\\
			\code{S\_STATUS}	& 11	& storage status \\
			\code{COST\_STO}	& 12	& storage cost [\$/MSCF]\\
			\code{COST\_OUT}	& 13	& storage outflow cost [\$/MMSCFD]\\
			\code{COST\_IN}	& 14	& storage inflow cost [\$/MMSCFD]\\
			\bottomrule
		\end{tabular}
	\begin{tablenotes}
		\scriptsize
		\item [\dag] {Volume in Million Standard Cubic Feet (MSCF).}
		\item[\ddag] {Storage outflow minus storage inflow. See Section \ref{chap:formulation} for more details.}
	\end{tablenotes}
	\end{threeparttable}
\end{table}

\chapter{Appendix B: Interconnection Case Data File Format}
\label{app:connect_format}

A detailed description about the interconnection case (\code{connect}) is provided in Table \ref{tab:connectcase}. As seen, some additional information is required for the power system besides the input data given in the \matpower{}-case. For the sake of clarity and readability, we decided to include such an additional information in the interconnection case rather than the \matpower{}-case. In short, different periods that are modeled using an island-based approach are allowed for the power system, where each island defines the network conditions at each period. On the other hand, the power-driven compressors and the gas-fired generator units set the coupling features between the power and natural gas systems. The user could define any of these two coupling options as empty arrays when they are not to be considered for a specific analysis. See Section \ref{chap:examples} for details.

\begin{table}[!ht]
	\centering
	\begin{threeparttable}
		\caption{Connection Data (\code{mpgc.connect})}
		\label{tab:connectcase}
		\footnotesize
		\begin{tabular}{lcp{0.6\textwidth}}
			\toprule
			name & domain & description \\
			\midrule
			\code{.power.time}	& $\Real^{n_t}$	& vector to define the number of $n_t$ periods to be considered in the power system. Each component in the vector represents the number of hours for each period such that \code{sum(power.time)=24}\tnote{\dag}.\\
			\code{.power.demands}	& 	& \\
			\hspace{2.5cm} \code{.pd}       & $\Real^{n_b\times n_t}$  & matrix to define the active power demand for $n_b$ buses over $n_t$ periods of time.\\
			\hspace{2.5cm} \code{.qd}       & $\Real^{n_b\times n_t}$  & matrix to define the reactive power demand for $n_b$ buses over $n_t$ periods of time.\\
			\code{.power.cost} & $\Real^+$ & non-supplied power demand cost.\\
			\code{.power.sr} & $\Real^{n_a\times n_t}$ & matrix to define the spinning reserve of $n_a$ areas over $n_t$ periods.\\
			\code{.power.energy} & $\Real^{n_{g_h}\times 2}$ & matrix to define the maximum energy available for the $n_{g_h}\subseteq n_g$ hydroelectric power generators, holding columns as follows:\tnote{\ddag}
			\begin{tabular}{c @{ -- } p{0.4\textwidth}}
				column 1  & generator number (positive integer)\\
				column 2  & maximum energy for hydroelectric unit [MW$\cdot$h]\\
			\end{tabular}\\
			\code{.interc.comp} & $\Real^{n_{c_p}\times 2}$ & index matrix to locate the $n_{c_p}\subseteq n_c$ power-driven compressors at some specific buses, holding columns as below\tnote{\S}:
			\begin{tabular}{c @{ -- } p{0.4\textwidth}}
				column 1  & compressor number (positive integer)\\
				column 2  & bus number to locate the power-driven compressor (positive integer)\\
			\end{tabular}\\
			\code{.interc.term} & $\Real^{n_{g_g}\times 3}$	& matrix to locate the $n_{g_g} \subseteq n_g$ gas-fired generators at some specific nodes and buses, holding the following columns:\tnote{\ddag}
			\begin{tabular}{c @{ -- } p{0.4\textwidth}}
				column 1  & bus number to locate the gas-fired unit as generator (positive integer)\\
				column 2  & node number to locate the gas-fired unit as demand (positive integer)\\
				column 3  & thermal efficiency of the gas-fired unit (positive real)\\
			\end{tabular}\\
			\bottomrule
		\end{tabular}
		\begin{tablenotes}
			\scriptsize
			\item [\dag] {The gas temporal resolution is one \textit{day}, while the electrical resolution is in terms of \textit{hours}. See Section \ref{chap:formulation} for more details.}
			\item [\ddag] {$n_g$ is the number of all power generator units installed in the power system.}
			\item[\S] {$n_c$ is the number of all compressors installed in the gas system.}
		\end{tablenotes}
	\end{threeparttable}
\end{table}

\end{appendix}


%  Example for the ETH Beamer template
%  Copyright 2014 by
%  Dr. Antonios Garas,
%  Chair of Systems Design, ETH Zurich
%  Weinbergstrasse 56/58 CH-8092 Zurich
%
\documentclass[
%aspectratio=169,
first,
%handout,
%compress,
%Helv,
ETH1,
navigation
]{ETHbeamerclass}

% Options for beamer:
%
% compress: navigation bar becomes smaller
% t       : place contents of frames on top (alternative: b,c)
% handout : handoutversion
% notes   : show notes
% notes=onlyslideswithnotes
%
\setbeamertemplate{note page}{\ \\[.3cm]
\textbf{\color{colorSG}Notes:}\\%[0.1cm]
{\footnotesize %\tiny
\insertnote}}

\setbeameroption{hide notes}
%\setbeameroption{show notes}

\setbeamertemplate{navigation symbols}{} % suppresses all navigation symbols:
% \setbeamertemplate{navigation symbols}[horizontal] % Organizes the navigation symbols horizontally.
% \setbeamertemplate{navigation symbols}[vertical] % Organizes the navigation symbols vertically.
% \setbeamertemplate{navigation symbols}[only frame symbol] % Shows only the navigational symbol for navigating frames.

\usepackage{amsmath, amssymb}
\usepackage{pifont}
\usepackage{multimedia}

%--------------------------------------------------------------------------------------------------------
%--- Some custom definitions...

\definecolor{cbf}{RGB}{199,100,95}%{163,15,0}
\newcommand{\cbf}{\color{cbf}}

\newcommand{\mean}[1]{\left\langle #1 \right\rangle}
\newcommand{\abs}[1]{\left| #1 \right|}

%--------------------------------------------------------------------------------------------------------
%--- Inputs for the title page...

\title{Title of the talk\\
And a much larger title spanning two lines
}
\newcommand{\shorttitle}{Shorter title}

\author{Author's Name}
\date{\today}
\institute{Chair of Systems Design}

\newcommand{\collaborators}{Collaborators: A list of names..}
\newcommand{\event}{Event Name}
\newcommand{\place}{Place of Event}

\newcommand{\Prob}{\mathrm{Pr}}
\newcommand{\sign}{\mathrm{sign}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\nbin}[1]{\mathrm{nb}_\mathrm{in}(#1)}
\newcommand{\nbout}[1]{\mathrm{nb}_\mathrm{out}(#1)}
\newcommand{\one}{\mathbb{\bf 1}}

\newcommand{\sref}[1]{SLIDE \ref{#1}}

%--------------------------------------------------------------------------------------------------------
%--- Here begins the presentation...

\begin{document}

\frame{
\maketitle
}
\note{}

\section{This is section one}

\frame{

  \frametitle{Contents}
    \tableofcontents[currentsection]
}
\note{}

\frame{

  \frametitle{This is the title of the first frame}

  In this frame we use a block:
  \begin{block}{Title of the block}
  Contents of the block...\\
  In one or more lines
    \begin{itemize}
    \item It can also include environments...
     \begin{itemize}
      \item It can also include environments...
     \end{itemize}
    \end{itemize}
  \end{block}

  \begin{block}{}
  Contents of the block...\\
  In one or more lines
    \begin{enumerate}
     \item It can also include environments...
    \end{enumerate}
  \end{block}

}
\note{}

\frame{

  \frametitle{This is the title of the first frame}

  In one or more lines
    \begin{itemize}
    \item It can also include environments...
     \begin{itemize}
      \item It can also include environments...
     \end{itemize}
%    \end{itemize}

%    \begin{itemize}
    \item It can also include environments...
     \begin{itemize}
      \item It can also include environments...
     \end{itemize}
    \end{itemize}

    \begin{enumerate}
     \item It can also include environments...
    \end{enumerate}

}
\note{}

\section{This is section two}

\frame{

  \frametitle{Let's use a long frame title that will be
    displayed in two (or more) lines... This is not a good practice, but it may happen :-)}

  \begin{columns}
   \column{0.45\textwidth}
   Now lets split the presentation using columns..

  \column{0.45\textwidth}
    In this frame we use a block:
    \begin{block}{Title of the block}
    Contents of the block...\\
    In one or more lines
    \begin{itemize}
    \item It can also include environments...
     \begin{itemize}
      \item It can also include environments...
      \end{itemize}
     \end{itemize}
    \end{block}
  \end{columns}
}
\note{}

\frame{

  \frametitle{Contents}
    \tableofcontents[currentsection]
}
\note{}

\frame{

  \frametitle{Summary..}

  }
\note{}

\end{document}


\documentclass[a4paper,12pt]{article}
\usepackage[top=2cm,bottom=2cm,left=2cm,right=2cm]{geometry}
\usepackage{amsmath}
\usepackage{hyperref}
\usepackage{listings}

\lstset{
  language=C++,                % choose the language of the code
  numbers=left,                   % where to put the line-numbers
  stepnumber=1,                   % the step between two line-numbers.
  numbersep=5pt,                  % how far the line-numbers are from the code
  showspaces=false,               % show spaces adding particular underscores
  showstringspaces=false,         % underline spaces within strings
  showtabs=false,                 % show tabs within strings adding particular underscores
  tabsize=2,                      % sets default tabsize to 2 spaces
  captionpos=b,                   % sets the caption-position to bottom
  breaklines=true,                % sets automatic line breaking
  breakatwhitespace=true,         % sets if automatic breaks should only happen at whitespace
  title=\lstname,                 % show the filename of files included with \lstinputlisting;
}

\title{Solving a Nonlinear Equation - Mechanical Engineering}
\author{Lucia Mingotti, Giacomo Sentimenti, Eugenio Tampieri, Kilian Ziegler}
%\renewcommand{\arraystretch}{1.5}
\begin{document}
\maketitle
\section{Problem}
    \par A trunnion has to be cooled before it is shrink fitted into a steel hub.
    \par We are given the equation to find out at which temperature we need to cool down the trunnion in order to get the desired shrinking.
    \begin{equation}
        f(T_f) = -0,50598\cdot 10^{-10}T_f^3 + 0,38292 \cdot 10^-7T_f^2+0,74363\cdot 10^-4 T_f + 0,88318\cdot 10^-2
    \end{equation}
\section{Solution}
    \par To solve the problem we have to find a solution to equation 1. In order to do this, we can do it by using a computer algorithm that uses one of the following methods:
    \begin{itemize}
        \item Newton's method
        \item Bisection method
        \item Secants method
    \end{itemize}
    The code is reported below.
    \lstinputlisting{trunnion.cpp}
\section{Discussion}
    \par Firstly, we have applied all methods with $T_{f0} \in [-130;-110]$, taking the leftmost poiint as the initial guess for Newton's method. The output follows.
    \begin{verbatim}
Newton's method: A root has been found in 1 iterations: f(-128.753)=9.00726e-08
Bisection method: A root has been found in 4 iterations: f(-129.375)=-3.84176e-05
Secants method: A root has been found in 2 iterations: f(-128.755)=-2.21485e-08
    \end{verbatim}
    \par Then, we chose this interval: $T_{f0} \in [-1\cdot 10^3;1\cdot 10^3]$.
    \begin{verbatim}
Newton's method: A root has been found in 3 iterations: f(-802.927)=1.8381e-06
Bisection method: A root has been found in 9 iterations: f(-126.953)=0.00011187
Secants method: A root has been found in 7 iterations: f(1688.45)=1.86587e-11
    \end{verbatim}
    \par Then, we chose this interval: $T_{f0} \in [-5\cdot 10^2;1]$.
    \begin{verbatim}
Newton's method: A root has been found in 10 iterations: f(-802.905)=2.46225e-09
Bisection method: A root has been found in 10 iterations: f(-128.898)=-8.86823e-06
Secants method: A root has been found in 5 iterations: f(-128.755)=-2.86784e-10
    \end{verbatim}
    \par From the listings above, we can observe that the Newton's algorithm converges typically faster, usually followed by the secants' and by bisection. Also, by increasing the interval the algorithms converge to different roots. Therefore, to make the search effective, we should place boundaries lowerly limited by the minimum value we can reach and superiorly limited by the initial temperature of the trunnion, since we want to cool it down, thus excluding the solutions greater than the starting temperature.
\end{document}


\section{Persistence}
\label{sec:persistence}

\emph{Persistence}\index{persistence} is a distinctive feature of
purely functional languages, meaning that values are
constants. Functions update a data structure by creating a new version
of it, instead of modifying it in place and thereby erasing its
history. We saw in section~\vref{sec:functional} that subtrees common
to both sides of the same rule are shared. Such a sharing is sound
because of persistence: there is no logical way to tell apart a copied
subtree from the original.

\paragraph{Maximum sharing}
\index{sharing}

An obvious source of sharing is the occurrence of a variable in the
pattern of a rule and its right\hyp{}hand side, as seen in
\fig~\vref{fig:cat_dag} for instance. But this does not always lead to
maximum sharing as the definition of
\fun{red/1}\index{red@\fun{red/1}} (\emph{reduce}) in
\fig~\ref{fig:red} shows. This function copies a stack without its
consecutively repeated items. For example, we might have \(\fun{red}([4,1,2,2,2,1,1]) \twoheadrightarrow [4,1,2,1]\).
% Wrapping figure better declared before a paragraph
%
\setlength{\intextsep}{0pt}
\begin{wrapfigure}[6]{r}[0pt]{0pt}
% [6] vertical lines
% {r} mandatory right placement (better because of a list)
\centering
\includegraphics[bb=71 672 203 726]{red}
\caption{Reducing\label{fig:red}}
\end{wrapfigure}
A
directed acyclic graph of the second rule is depicted in
\fig~\vref{fig:red_dag1}.
\begin{figure}
\centering
\subfloat[Sharing variables\label{fig:red_dag1}]{%
  \includegraphics[bb=70 645 158 723]{red_dag1}
}
\qquad\qquad
\subfloat[Sharing subtrees\label{fig:red_dag2}]{%
  \includegraphics[bb=70 645 158 723]{red_dag2}
}
\caption{Directed acyclic graphs (DAG) of the second rule of
  \fun{red/1}}
\end{figure}
In this picture, sharing is based on commonly occurring variables, but
we can see that \(\cons{x}{s}\) is not completely shared. Consider the
same rule in \fig~\vref{fig:red_dag2} with maximum sharing, where a
whole subtree is shared.
\setlength{\intextsep}{12pt}

When discussing memory management, \emph{we assume that sharing is
  maximum for each rule}, so, for instance, \fig~\vref{fig:red_dag2}
would be the default. But this property is not enough to insure that
sharing is maximum between the arguments of a function call and its
value. For example,
\begin{equation*}
\fun{cp}(\el) \rightarrow \el;\qquad
\fun{cp}(\cons{x}{s}) \rightarrow \cons{x}{\fun{cp}(s)}.
\end{equation*}
makes a copy of its argument, but the value of \(\fun{cp}(s)\) only
shares its items with~\(s\), despite \(\fun{cp}(s) \equiv s\).

\paragraph{Revision control}

A simple idea to implement data structures enabling backtracking
consists in keeping all successive
versions\index{persistence!version-based}. A stack can be used to keep
such a record, called \emph{history}\index{persistence!history}, and
backtracking is reduced to linear search\index{linear search}. For
example, we may be interested in recording a series of stacks, each
one obtained from its predecessor by a push or a pop, like the history
\([[4,2,1],[2,1],[3,2,1],[2,1],[1],\el]\), where the initial stack was
empty; next, \(1\)~was pushed, followed by~\(2\) and~\(3\); then
\(3\)~was popped and \(4\)~pushed instead. This way, the \emph{last
  version} is the top of history, like \([4,2,1]\) in our
example. Furthermore, we want two successive versions to share as much
structure as possible. (We speak of `version' instead of `state',
because the latter applies when values are not persistent.)  These
requirements are at the heart of \emph{revision control} software,
used by programmers to keep track of the evolution of their source
code.

As a simple case study, we shall continue with our example of
recording the different versions of a stack, while keeping in mind
that the technique described further applies to any data structure. We
need to write two functions, \fun{push/2}\index{push@\fun{push/2}}
(different from the one defined in section~\ref{sec:cutting}) and
\fun{pop/1}\index{pop@\fun{pop/1}}, which, instead of processing one
stack, deal with a history of stacks. Consider the following
definitions:
\begin{equation}
\begin{array}{@{}r@{\;}c@{\;}l@{\quad}r@{\;}c@{\;}l@{}}
\fun{push}(x,\el) & \rightarrow & [[x],\el];
& \fun{pop}(\cons{\cons{x}{s}}{h}) & \rightarrow &
                                     \cons{s,\cons{x}{s}}{h}.\\
\fun{push}(x,\cons{s}{h}) & \rightarrow &
\cons{\cons{x}{s},s}{h}. &
\fun{top}(\cons{\cons{x}{s}}{h}) & \rightarrow & x.
\end{array}
\label{eq:push_pop_persistence}
\end{equation}
The corresponding directed acyclic graphs are found in
\fig~\vref{fig:push_pop_dag}.
\begin{figure}
\centering
\includegraphics[bb=70 639 294 723]{push_pop_dag}
\caption{DAGs of \fun{push/2} and \fun{pop/1} with maximum sharing
\label{fig:push_pop_dag}}
\end{figure}
History \([[4,2,1],[2,1],[3,2,1],[2,1],[1],\el]\) is displayed in
\fig~\ref{fig:history}
\begin{figure}[!b]
\centering
\includegraphics[bb=70 575 247 723]{history}
\caption{History \([[4,2,1],[2,1],[3,2,1],[2,1],[1],\el]\)
\label{fig:history}}
\end{figure}
as a directed acyclic graph as well. It is obtained as the evaluation
of
\begin{equation}
%\abovedisplayskip=4pt
%\belowdisplayskip=4pt
\fun{push}(4, \fun{pop}(\fun{push}(3, \fun{push}(2,
\fun{push}(1, \el))))).
\label{ex:history}
\end{equation}

Let \(\fun{ver}(k,h)\) \index{ver@\fun{ver/2}} evaluate in the~\(k\)th
previous version in the history~\(h\), so \(\fun{ver}(0,h)\) is the
last version. As expected, \fun{ver/2} is just a linear search with a
countdown:
\begin{equation*}
%\abovedisplayskip=4pt
%\belowdisplayskip=4pt
\fun{ver}(0,\cons{s}{h}) \rightarrow s;\qquad
\fun{ver}(k,\cons{s}{h}) \rightarrow \fun{ver}(k-1,h).
\end{equation*}
Our encoding of history allows the last version to be easily modified,
but not older ones. When a data structure allows every version in its
history to be modified, it is said \emph{fully
  persistent}\index{persistence!full $\sim$}; if only the last version
can be replaced, it is said \emph{partially persistent}
\citep{MehlhornTsakalidis_1990}.\index{persistence!partial $\sim$}

\paragraph{Backtracking updates}
\index{persistence!backtracking}

In order to achieve full persistence, we should keep a history of
updates\index{persistence!update-based},
\(\ufun{push}(x)\)\index{push@\fun{push/1}} and
\(\ufun{pop}()\)\index{pop@\fun{pop/0}}, instead of successive,
maximally overlapping versions. In the following, the underlining
avoids confusion with the functions
\fun{push/2}\index{push@\fun{push/2}} and
\fun{pop/1}\index{pop@\fun{pop/1}}. Instead of
equation~\eqref{ex:history}, we have now the history
\begin{equation}
[\ufun{push}(4), \ufun{pop}(), \ufun{push}(3), \ufun{push}(2),
\ufun{push}(1)].
\label{ex:history1}
\end{equation}
But not all series of \(\ufun{push}(x)\) and \(\ufun{pop}()\) are
valid, as \([\ufun{pop}(), \ufun{pop}(), \ufun{push}(x)]\) and
\([\ufun{pop}()]\). In order to characterise valid histories, let us
consider a graphical representation of updates in
\fig~\vref{fig:push_pop1}.
\begin{figure}
\centering
\subfloat[$\protect\fun{push}(x)$]{%
  \includegraphics[bb=71 662 130 721,scale=0.8]{push}}
\qquad
\subfloat[$\protect\fun{pop}()$]{%
  \includegraphics[bb=71 662 130 721,scale=0.8]{pop}}
\caption{Stack updates\label{fig:push_pop1}}
\end{figure}
This is the same model we used in section~\ref{sec:queueing} where we
studied queueing (see in particular \fig~\vref{fig:enq_deq}), except
that we choose here a leftwards orientation to mirror the notation for
stacks, whose tops are laid out on the left of the page. Consider for
instance the history in \fig~\vref{fig:history1}.
\begin{figure}[b]
\centering
\includegraphics[bb=69 577 300 726,scale=0.9]{history1}
\caption{$[\ufun{pop}(), \ufun{pop}(), \ufun{push}(4),\ufun{push}(3),
      \ufun{pop}(), \ufun{push}(2), \ufun{push}(1)]$
\label{fig:history1}}
\end{figure}
It is clear that \emph{a valid history is a line which never
  crosses the absissa axis.}

Programming \fun{top/1}\index{top@\fun{top/1}} with a history of
updates is more difficult because we must find the top item of the
last version without constructing it. The idea is to walk back the
historical line and determine the last push leading to a version whose
length is equal to that of the last version. In
\fig~\ref{fig:history1}, the last version is~(\(\bullet\)). If we draw
a horizontal line from it rightwards, the first push ending on the
line is \(\ufun{push}(1)\), so the top of the last version
is~\(1\).

This thought experiment is depicted in \fig~\ref{fig:topmost}.
\begin{figure}
\centering
\includegraphics[bb=69 577 300 726,scale=0.9]{topmost}
\caption{Finding the top of the last version
\label{fig:topmost}}
\end{figure}
Notice that we do not need to determine the length of the last
version: the difference of length with the current version, while
walking back, is enough.

Let \fun{top\(_0\)/1}\index{top0@\fun{top\(_0\)/1}} and
\fun{pop\(_0\)/1}\index{pop0@\fun{pop\(_0\)/1}} be the equivalent of
\fun{top/1} and \fun{pop/1} operating on updates instead of versions.
Their definitions are found in \fig~\vref{fig:pop_top}.
\begin{figure}[h]
\begin{equation*}
\boxed{%
\begin{array}{@{}r@{\;}l@{\;}lr@{\;}l@{\;}l@{}}
\fun{pop}_0(h)   & \rightarrow          & \cons{\ufun{pop}()}{h}. &
\fun{top}_0(0,\cons{\ufun{push}(x)}{h}) & \rightarrow             & x;\\
\fun{top}_0(h)   & \rightarrow          & \fun{top}_0(0,h).       &
\fun{top}_0(k,\cons{\ufun{push}(x)}{h}) & \rightarrow             & \fun{top}_0(k-1,h);\\
                 & & &
\fun{top}_0(k,\cons{\ufun{pop}()}{h})   & \rightarrow            & \fun{top}_0(k+1,h).
\end{array}}
\end{equation*}
\caption{The top and the rest of an history of updates
\label{fig:pop_top}}
\end{figure}
An ancillary function \fun{top\(_0\)/2}\index{top0@\fun{top\(_0\)/2}}
keeps track of the difference between the lengths of the last and
current versions. We have found the item when the difference is~\(0\)
and the current update is a push.

As previously, we want the call
\(\fun{ver}_0(k,h)\)\index{ver0@\fun{ver\(_0\)/2}} to evaluate in
the~\(k\)th previous version in~\(h\). Here, we need to walk back
\(k\)~updates in the past,
\begin{equation*}
\fun{ver}_0(0,h)           \rightarrow \fun{lst}_0(h);\qquad
\fun{ver}_0(k,\cons{s}{h}) \rightarrow \fun{ver}_0(k-1,h).
\end{equation*}
and build the last version from the rest of the history with
\fun{lst\(_0\)/1}\index{lst0@\fun{lst\(_0\)/1}}:
\begin{mathpar}
\inferrule*{}{\fun{lst}_0(\el) \rightarrow \el};
\quad
\inferrule*{}{\fun{lst}_0(\cons{\ufun{push}(x)}{h})
              \rightarrow \cons{x}{\fun{lst}_0(h)}};
\quad
\inferrule
  {\fun{lst}_0(h)                      \twoheadrightarrow \cons{x}{s}}
  {\fun{lst}_0(\cons{\ufun{pop}()}{h}) \twoheadrightarrow s}.
\end{mathpar}
Let \(\C{\fun{ver}_0}{k,n}\)~be the cost of the call
\(\fun{ver}_0(k,h)\) and \(\C{\fun{lst}_0}{n}\)~the cost
of~\(\fun{lst}_0(h)\), where \(n\)~is the length of~\(h\):
\(\C{\fun{lst}_0}{i} = i + 1\) and \(\C{\fun{ver}_0}{k,n} = (k+1) +
\C{\fun{lst}_0}{n-k} = n + 2\).

What is the total amount of memory
allocated?\index{sharing}\index{memory|see{sharing}} More precisely,
we want to know the number of pushes performed. The only rule of
\fun{lst\(_0\)/1} featuring a push in its right\hyp{}hand side is the
second one, so the number of cons\hyp{}nodes is the number of push
updates. But this is a waste in certain cases, for example, when the
version built is empty, like the history
\([\ufun{pop}(),\ufun{push}(6)]\). The optimal situation is to
allocate only as much as the computed version actually contains.

% Wrapping figure better declared before a paragraph
%
\setlength{\intextsep}{0pt}
\begin{wrapfigure}[]{r}[0pt]{0pt}
% {r} mandatory right placement
\centering
\includegraphics[bb=71 644 257 721]{lst1}
\caption{Last version \label{fig:lst1}}
\end{wrapfigure}
\hspace*{-5.6pt} We can achieve this memory optimality with
\fun{lst\(_1\)/1} in \fig~\ref{fig:lst1} by retaining features of both
\fun{top\(_0\)/1}\index{top0@\fun{top\(_0\)/1}} and \fun{lst\(_0\)/1}.
We have\index{lst1@$\C{\fun{lst}_1}{n}$} \(\C{\fun{lst}_1}{n} =
\C{\fun{lst}_0}{n} = n + 1\), and the number of
cons\hyp{}nodes\index{sharing|(} created is now the length of the last
version itself. This is another instance of an output\hyp{}dependent
cost\index{cost!output-dependent $\sim$}, like
\fun{flat\(_0\)/1}\index{flat0@\fun{flat$_0$/1}} in
section~\vref{sec:flattening}. There is still room for improvement if
the historical line meets the absissa axis, as there is no need to
visit the updates \emph{before} a pop resulting into an empty version;
for instance, in \fig~\vref{fig:back2zero},
\begin{figure}[b]
\centering
\includegraphics[bb=69 606 300 726,scale=0.9]{back2zero}
\caption{Last version $[3]$ found in four steps\label{fig:back2zero}}
\end{figure}
it is useless to go past \(\ufun{push}(3)\)\index{push@\fun{push/1}}
to find the last version to be~\([3]\). But, in order to detect
whether the historical line meets the abscissa axis, we need to
augment the history~\(h\) with the length~\(n\) of the last version,
that is, work with \(\pair{n}{h}\), and modify
\fun{push/2}\index{push@\fun{push/2}}
and~\fun{pop/1}\index{pop@\fun{pop/1}} accordingly:
\begin{equation*}
\fun{push}_2(x,\pair{n}{h}) \! \rightarrow \!
\pair{n\!+\!1}{\cons{\ufun{push}(x)}{h}}.\quad
\fun{pop}_2(\pair{n}{h}) \! \rightarrow \!
\pair{n\!-\!1}{\cons{\ufun{pop}()}{h}}.
\end{equation*}
We have to rewrite~\fun{ver/2}\index{ver2@\fun{ver\(_2\)/2}} so it keeps
track of the length of the last version:
\begin{equation*}
\begin{array}{@{}r@{\;}l@{\;}l@{}}
\fun{ver}_2(0,\pair{n}{h}) & \rightarrow & \fun{lst}_1(0,h);\\
\fun{ver}_2(k,\pair{n}{\cons{\ufun{pop}()}{h}})
                      & \rightarrow & \fun{ver}_2(k-1,\pair{n+1}{h});\\
\fun{ver}_2(k,\pair{n}{\cons{\ufun{push}(x)}{h}})
                      & \rightarrow & \fun{ver}_2(k-1,\pair{n-1}{h}).
\end{array}
\end{equation*}
We can reduce the memory\index{sharing|)} footprint by separating the
length~\(n\) of the last version from the current history~\(h\), so no
pairs are allocated, and we can stop when the current version
is~\(\el\), as planned, in \fig~\ref{fig:ver_no_pair}.
\begin{figure}
\begin{equation*}
\boxed{%
\begin{array}{r@{\;}l@{\;}l}
\fun{ver}_3(k,\pair{n}{h}) & \rightarrow & \fun{ver}_3(k,n,h).\\
\fun{ver}_3(0,n,h) & \rightarrow & \fun{lst}_3(0,n,h);\\
\fun{ver}_3(k,n,\cons{\ufun{pop}()}{h})
                      & \rightarrow & \fun{ver}_3(k-1,n+1,h);\\
\fun{ver}_3(k,n,\cons{\ufun{push}(x)}{h})
                      & \rightarrow & \fun{ver}_3(k-1,n-1,h).\\
\fun{lst}_3(\pair{n}{h}) & \rightarrow & \fun{lst}_3(0,n,h).\\
\fun{lst}_3(k,0,h) & \rightarrow & \el;\\
\fun{lst}_3(0,n,\cons{\ufun{push}(x)}{h}) & \rightarrow
                      & \cons{x}{\fun{lst}_3(0,n-1,h)};\\
\fun{lst}_3(k,n,\cons{\ufun{push}(x)}{h}) & \rightarrow
                      & \fun{lst}_3(k-1,n-1,h);\\
\fun{lst}_3(k,n,\cons{\ufun{pop}()}{h}) & \rightarrow
                      & \fun{lst}_3(k+1,n+1,h).
\end{array}}
\end{equation*}
\caption{Querying a version without pairs\label{fig:ver_no_pair}}
\end{figure}

One may wonder why bother pairing~\(n\) and~\(h\) just to separate
them again, defeating the purpose of data abstraction. This example
demonstrates that abstraction is desirable for callers, but called
functions may break it due to pattern matching. We may also realise
that the choice of a stack for storing updates is not the best in
terms of memory usage. Instead, we can directly chain updates with an
additional argument denoting the previous update, so, for instance,
instead of equation~\eqref{ex:history1}:
\begin{equation*}
\pair{3}{\ufun{push}(4, \ufun{pop}(\ufun{push}(3, \ufun{push}(2, \ufun{push}(1,\el)))))}.
\end{equation*}
This new encoding\index{stack!encoding with tuples} closely mirrors
the call~\eqref{ex:history} on page~\pageref{ex:history} and saves
\(n\)~cons\hyp{}nodes in a history of length~\(n\). See
\fig~\ref{fig:ver}.
\begin{equation*}
\fun{push}_4(x,\pair{n}{h}) \rightarrow
\pair{n\!+\!1}{\ufun{push}(x,h)}.\quad
\fun{pop}_4(\pair{n}{h}) \rightarrow \pair{n\!-\!1}{\ufun{pop}(h)}.
\end{equation*}
\begin{figure}[t]
\begin{equation*}
\boxed{%
\begin{array}{r@{\;}l@{\;}l}
\fun{ver}_4(k,\pair{n}{h}) & \rightarrow & \fun{ver}_4(k,n,h).\\
\fun{ver}_4(0,n,h) & \rightarrow & \fun{lst}_4(0,n,h);\\
\fun{ver}_4(k,n,\ufun{pop}(h))
                      & \rightarrow & \fun{ver}_4(k-1,n+1,h);\\
\fun{ver}_4(k,n,\ufun{push}(x,h))
                      & \rightarrow & \fun{ver}_4(k-1,n-1,h).\\
\fun{lst}_4(\pair{n}{h}) & \rightarrow & \fun{lst}_4(0,n,h).\\
\fun{lst}_4(k,0,h) & \rightarrow & \el;\\
\fun{lst}_4(0,n,\ufun{push}(x,h)) & \rightarrow
                      & \cons{x}{\fun{lst}_4(0,n-1,h)};\\
\fun{lst}_4(k,n,\ufun{push}(x,h)) & \rightarrow
                      & \fun{lst}_4(k-1,n-1,h);\\
\fun{lst}_4(k,n,\ufun{pop}(h)) & \rightarrow
                      & \fun{lst}_4(k+1,n+1,h).
\end{array}}
\end{equation*}
\caption{Querying a version without a stack\label{fig:ver}}
\end{figure}

There is now a minimum and maximum cost. The worst case is when the
bottommost item of the last version is the first item pushed in the
history, so \fun{lst\(_4\)/3}\index{lst4@\fun{lst\(_4\)/3}} has to
recur until the origin of time. In other words, the historical line
never reaches the abscissa axis after the first push. We have same
cost as before: \(\W{\fun{lst}_4}{n} = n +
1\). \index{lst4@$\W{\fun{lst}_4}{n}$} The best case happens when the
last version is empty. In this case, \(\B{\fun{lst}_4}{n} =
1\)\index{lst4@$\B{\fun{lst}_4}{n}$}, and this is an occurrence of the
kind of improvement we sought.

%\mypar{Average cost}

\paragraph{Full persistence}
\index{persistence!full $\sim$}

The update\hyp{}based approach to history is fully persistent by
enabling the modification of the past as follows: traverse history
until the required moment, pop up the update at that point, push
another one and simply put back the previously traversed updates,
which must have been kept in some accumulator. But changing the past
must not create a history with a non\hyp{}constructible version in it,
to wit, the historical line must not cross the absissa axis after the
modification. If the change consists in replacing a pop by a push,
there is no need to worry, as this will raise by~\(2\) the end point
of the line. It is the converse change that requires special
attention, as this will lower by~\(2\) the end point. The \(\pm
2\)~offset comes from the vertical difference between the end points
of a push and a pop update of same origin, as can be easily seen in
\fig~\vref{fig:push_pop1}. As a consequence, in
\fig~\ref{fig:history1}, the last version has length~\(1\), which
implies that it is impossible to replace a push by a pop, anywhere in
the past.

Let us consider the history in \fig~\vref{fig:history2}.
\begin{figure}
\centering
\includegraphics[bb=69 578 272 726,scale=0.9]{history2}%[]
\caption{$\ufun{pop}(\ufun{push}(4,\ufun{push}(3,
          \ufun{pop}(\ufun{push}(2,\ufun{push}(1,\el))))))$
\label{fig:history2}}
\end{figure}
Let \(\fun{chg}(k,u,\pair{n}{h})\)\index{chg@\fun{chg/3}} be the
changed history, where \(k\)~is the index of the update we want to
change, indexing the last one at~\(0\); \(u\)~is the new update we
want to set; \(n\)~is the length of the last version of
history~\(h\). The call
\begin{equation*}
\fun{chg}(3,\ufun{push}(5),
  \pair{2}{\ufun{pop}(\ufun{push}(4,\ufun{push}(3,
           \ufun{pop}(\ufun{push}(2,\fun{push}(1,\el))))))})
\end{equation*}
results in
\(\pair{4}{\ufun{pop}(\ufun{push}(4,\ufun{push}(3,\ufun{push}(5,
  \ufun{push}(2,\ufun{push}(1,\el))))))}\). \!\!This call succeeds
because in \fig~\ref{fig:pop2push}
\begin{figure}
\centering
\subfloat[A pop becomes a push\label{fig:pop2push}]{
  \includegraphics[bb=69 521 270 726,scale=0.8]{pop2push}}
\quad
\subfloat[A push becomes a pop\label{fig:push2pop}]{
  \includegraphics[bb=69 577 270 726,scale=0.8]{push2pop}}
\caption{Changing updates}
\end{figure}
the new historical line does not cross the absissa axis. We can see in
\fig~\ref{fig:push2pop} the result of the call
\begin{equation*}
\fun{chg}(2,\ufun{pop}(),
            \pair{2}{\ufun{pop}(\ufun{push}(4,\ufun{push}(3,
                     \ufun{pop}(\ufun{push}(2,\ufun{push}(1,\el))))))}).
\end{equation*}
All these examples help in guessing the characteristic property for a
replacement to be valid:
\begin{itemize}

  \item the replacements of a pop by a push, a pop by a pop, a push by
    a push are always valid;

  \item the replacement of a push by a pop as update~\(k>0\) is valid
    if and only if the historical line between updates
    \(0\)~and~\(k-1\) remains above or reaches without crossing the
    horizontal line of ordinate~\(2\).

\end{itemize}
We can divide the algorithm in two phases. First, the update to be
replaced must be found, but, the difference with \fun{ver\(_4\)/3} is
that we may need to know if the historical line before reaching the
update lies above the horizontal line of ordinate~\(2\). This is easy
to check if we maintain across recursive calls the lowest ordinate
reached by the line. The second phase substitute the update and checks
if the resulting history is valid.

Let us implement the first phase. First, we project \(n\)~and~\(h\)
out of \(\pair{n}{h}\) in order to save some memory, and the lowest
ordinate is~\(n\), which we pass as an additional argument to another
function \fun{chg/5}\index{chg@\fun{chg/5}|(}:
\begin{equation*}
\fun{chg}(k,u,\pair{n}{h}) \rightarrow \fun{chg}(k,u,n,h,n).
\end{equation*}
Function~\fun{chg/5} traverses~\(h\) while decrementing~\(k\), until
\(k=0\), which means that the update to be changed has been found. At
the same time, the length of the current version is computed (third
argument) and compared to the previous lowest ordinate (the fifth
argument), which is updated according to the outcome. We may try the
following canvas:
\begin{equation*}
\begin{array}{@{}r@{\;}l@{\;}l@{}}
\fun{chg}(0,u,n,h,m) & \rightarrow & \fbcode{CCCCCCC}\,;\\
\fun{chg}(k,u,n,\ufun{pop}(h),m) & \rightarrow
                                 & \fun{chg}(k-1,u,n+1,h,m);\\
\fun{chg}(k,u,n,\ufun{push}(x,h),m) & \rightarrow
                   & \fun{chg}(k-1,u,n-1,h,m),\qquad \text{if \(m < n\)};\\
\fun{chg}(k,u,n,\ufun{push}(x,h),m) & \rightarrow
                                    & \fun{chg}(k-1,u,n-1,h,n-1).
\end{array}
\end{equation*}
\index{chg@\fun{chg/5}|)} The problem is that we forget the history
down to the update we are looking for. There are two methods to record
it: either we use an accumulator\index{functional
  language!accumulator} and stick with a definition in tail
form\index{functional language!tail form} (small\hyp{}step
design\index{design!small-step $\sim$}), or we put back a visited
update after returning from a recursive call (big\hyp{}step
design\index{design!big-step $\sim$}). The latter is faster, as there
is no need to reverse the accumulator when we are done; the former
allows us to share the history up to the update when it leaves history
structurally invariant, at the cost of an extra parameter being the
original history. This same dilemma was encountered when we compared
\fun{sfst/2}\index{sfst@\fun{sfst/2}} and
\fun{sfst\(_0\)/2}\index{sfst0@\fun{sfst\(_0\)/2}} in
section~\ref{sec:skipping} on page~\pageref{sec:skipping}. Let us opt
for a big\hyp{}step design in \fig~\ref{fig:chg}.
\begin{figure}[t]
\centering
\framebox[\columnwidth]{\vbox{%
\begin{gather*}
\inferrule*{}{\fun{chg}(0,u,n,h,m) \rightarrow \fun{rep}(u,h,m)}
\;\;
\inferrule
  {\fun{chg}(k-1,u,n+1,h,m) \twoheadrightarrow \pair{n'}{h'}}
  {\fun{chg}(k,u,n,\ufun{pop}(h),m) \twoheadrightarrow
                                    \pair{n'}{\ufun{pop}(h')}}
\\
\inferrule
  {\fun{chg}(k-1,u,n-1,h,m) \twoheadrightarrow \pair{n'}{h'}\\
   m < n}
  {\fun{chg}(k,u,n,\ufun{push}(x,h),m) \twoheadrightarrow
                              \pair{n'}{\ufun{push}(x,h')}}
\\
\inferrule
  {\fun{chg}(k-1,u,n-1,h,n-1) \twoheadrightarrow \pair{n'}{h'}}
  {\fun{chg}(k,u,n,\ufun{push}(x,h),m) \twoheadrightarrow
                               \pair{n'}{\ufun{push}(x,h')}}
\end{gather*}
}}
\caption{Changing a past version (big-step design)\label{fig:chg}}
\end{figure}
Notice how the length~\(n'\) of the new history is simply passed down
the inference rules. It can simply be made out:
\begin{itemize*}

  \item replacing a pop by a pop or a push by a push leaves the
    original length invariant;

  \item replacing a pop by a push increases the original length
    by~\(2\);

  \item replacing a push by a pop, assuming this is valid, decreases
    the original length by~\(2\).

\end{itemize*}
This task is up to the new
function~\fun{rep/3}\index{rep@\fun{rep/3}|(} (\emph{replace}), which
implements the second phase. The design is to return a pair made of
the differential in length~\(d\) and the new history~\(h'\):
\begin{equation*}
\begin{array}{@{}r@{\;}l@{\;}l@{}}
\fun{rep}(\ufun{pop}(),\ufun{pop}(h),m)
     & \rightarrow & \pair{0}{\ufun{pop}(h)};\\
\fun{rep}(\ufun{push}(x),\ufun{push}(y,h),m)
     & \rightarrow & \pair{0}{\ufun{push}(x,h)};\\
\fun{rep}(\ufun{push}(x),\ufun{pop}(h),m)
     & \rightarrow & \pair{2}{\ufun{push}(x,h)};\\
\fun{rep}(\ufun{pop}(),\ufun{push}(y,h),m)
     & \rightarrow & \pair{-2}{\ufun{pop}(h)}.
\end{array}
\end{equation*}
\index{rep@\fun{rep/3}|)} This definition implies that we need to
redefine~\fun{chg/3}\index{chg@\fun{chg/3}} as follows:
\begin{mathpar}
\inferrule{\fun{chg}(k,u,n,h,n) \twoheadrightarrow \pair{d}{h'}}
          {\fun{chg}(k,u,\pair{n}{h}) \twoheadrightarrow \pair{n+d}{h'}}.
\end{mathpar}


\documentclass[12pt]{cdblatex}
\usepackage{exercises}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% --------------------------------------------------------------------------------------------
\section*{Exercise 3.10 Example of {\tt repeat=True} in a substitution}

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,u#}::Indices(position=independent).

   foo := A B + A B A B + A B A B A B + A B A B A B A B .   # cdb(ex-0310.foo.001,foo)
   bah := @(foo).                                           # cdb(ex-0310.bah.001,bah)

   substitute (foo,$A B -> A$)                              # cdb(ex-0310.foo.002,foo)
   substitute (bah,$A B -> A$,repeat=True)                  # cdb(ex-0310.bah.002,bah)
\end{cadabra}

\vskip 1cm

Without {\tt repeat=True} only the first match in a product will be susbstituted.
\begin{align*}
   &\cdb*{ex-0310.foo.001}\\
   &\cdb*{ex-0310.foo.002}
\end{align*}

But with {\tt repeat=True} then all matches in a product will be susbstituted.
\begin{align*}
   &\cdb*{ex-0310.bah.001}\\
   &\cdb*{ex-0310.bah.002}
\end{align*}

\end{document}


\documentclass[a4paper,english]{article}
\usepackage{tgtermes}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{hyperref}
\usepackage{amssymb}
\usepackage[cm]{fullpage}
\usepackage{babel}

\newcommand{\universe}{\mathcal{P}}
\newcommand{\closure}{\mathcal{C}}
\newcommand{\installation}{\mathcal{I}}
\newcommand{\removed}{\mathcal{R}}
\newcommand{\optimization}{\mathcal{O}}
\newcommand{\name}{\mathrm{name}}
\newcommand{\version}{\mathrm{version}}
\newcommand{\conflicts}{\mathrm{conflicts}}
\newcommand{\recommends}{\mathrm{recommends}}
\newcommand{\depends}{\mathrm{depends}}
\newcommand{\reason}{\mathrm{reason}}
\newcommand{\eval}{\mathrm{eval}}
\newcommand{\funsatrecom}{\mathrm{unsat\_recommends}}
\newcommand{\fcount}{\mathrm{count}}
\newcommand{\fsum}{\mathrm{sum}}
\newcommand{\faligned}{\mathrm{aligned}}
\newcommand{\fnotuptodate}{\mathrm{notuptodate}}
\newcommand{\selmax}{S^\infty}
\newcommand{\reasonmax}[1]{S^+(#1)}
\newcommand{\reasonmin}[1]{S^-(#1)}
\newcommand{\attr}[1]{\mathit{#1}}
\newcommand{\ssolution}{\mathrm{solution}}
\newcommand{\schanged}{\mathrm{changed}}
\newcommand{\snew}{\mathrm{new}}
\newcommand{\sremoved}{\mathrm{removed}}
\newcommand{\selup}{\mathrm{up}}
\newcommand{\sdown}{\mathrm{down}}
\newcommand{\sinstallrequest}{\mathrm{installrequest}}
\newcommand{\supgraderequest}{\mathrm{upgraderequest}}
\newcommand{\srequest}{\mathrm{request}}

\begin{document}

\section{Preliminaries}

\begin{itemize}
  \item See: \url{http://www.mancoosi.org/misc-2012/criteria/}
  \item Package universe: $\universe$
  \item Installation: $\installation\subseteq\universe$
  \item Closure: $\closure\subseteq\universe$
  \item Removed (by explicit request): $\removed\subseteq\universe$
  \item Optimization criteria: $\optimization$
  \item Names: $X.\name=\{p.\name\mid p\in X\}$
  \item Versions: $X.\version(p)=\{q.\version\mid q\in X,p.\name=q.\name\}$
\end{itemize}

\section{Closure}

\begin{itemize}
  \item Table \ref{tab:selectors} lists available selectors
    \begin{itemize}
      \item $\selmax$ is an upper bound for packages in a selector
      \item $\reasonmin{q}$ for $q\in\selmax$ is the set of packages whose installation might remove $q$ from the selector
      \item $\reasonmax{q}$ for $q\in\selmax$ is the set of packages whose installation might add $q$ to the selector
    \end{itemize}
  \item a package $p \in \universe \setminus \removed$ is relevant if at least one of the following conditions holds
    \begin{itemize}
      \item $+f \in \optimization$ where
        \begin{itemize}
          \item $f = \fcount(S)$, $q \in \selmax$, and $p \in \reasonmax{q}$,
          \item $f = \fsum(S,\attr{attr})$, $q \in \selmax$, $q.\attr{attr} > 0$, and $p \in \reasonmax{q}$,
          \item $f = \fsum(S,\attr{attr})$, $q \in \selmax$, $q.\attr{attr} < 0$, and $p \in \reasonmin{q}$,
          \item $f = \fnotuptodate(S)$, $q \in \selmax$, $q.\version \neq \max(\universe.\version(q))$, and $p \in \reasonmax{q}$,
          \item $f = \funsatrecom(S)$, $q \in \selmax$, $q.\recommends \neq \emptyset$, and $p \in \reasonmax{q}$, or
          \item $f = \faligned(S,\attr{group},\attr{value}))$, $q\in \selmax$, $q' \in \selmax$, $q.\attr{group} = q'.\attr{group}$, $q.\attr{value} \neq q'.\attr{value}$, and $p \in \reasonmax{q}$\\
        \end{itemize}
      \item $-f \in \optimization$ where
        \begin{itemize}
          \item $f = \fcount(S)$, $q \in \selmax$, and $p \in \reasonmin{q}$,
          \item $f = \fsum(S,\attr{attr})$, $q \in \selmax$, $q.\attr{attr} > 0$, and $p \in \reasonmin{q}$,
          \item $f = \fsum(S,\attr{attr})$, $q \in \selmax$, $q.\attr{attr} < 0$, and $p \in \reasonmax{q}$,
          \item $f = \fnotuptodate(S)$, $q \in \selmax$, $q.\version \neq \max(\universe.\version(q))$, and $p \in \reasonmin{q}$,
          \item $f = \funsatrecom(S)$, $q \in \selmax$, $q.\recommends \neq \emptyset$, and $p \in \reasonmin{q}$,
          \item $f = \funsatrecom(S)$, $q \in \selmax$ and $p \in q.\recommends\setminus\removed$, or
          \item $f = \faligned(S,\attr{group},\attr{value}))$, $q\in \selmax$, $q' \in \selmax$, $q.\attr{group} = q'.\attr{group}$, $q.\attr{value} \neq q'.\attr{value}$, and $p \in \reasonmin{q}$\\
        \end{itemize}
    \end{itemize}
  \item the closure $\closure$ is the least set that
    \begin{itemize}
      \item contains all relevant packages, and
      \item if $p\in \closure$ then $p.\depends\setminus\removed\subseteq\closure$
    \end{itemize}
\end{itemize}

\begin{table}
  \scriptsize
  \centering{}\begin{tabular}{r|l|l|l}
    selector $S$       & $\selmax$ & $\reasonmin{q}$ & $\reasonmax{q}$ \\
    \hline
    $\ssolution$       & $\universe\setminus\removed$ & $\emptyset$ & $\{q\}$ \\
    $\schanged$        & $\universe\setminus\removed\cup\installation$ & $\{q\}\cap\installation\setminus\removed$ & $\{q\}\setminus\installation$ \\
    $\snew$            & $\{p\in \universe\setminus\removed\mid p.\name\notin \installation.\name\}$ & $\emptyset$ & $\{q\}$ \\
    $\sremoved$        & $\installation$ & $\{p\in\universe\setminus\removed\mid p.\name = q.\name\}$ & $\emptyset$ \\
    $\selup$           & $\{p\in \universe\setminus\removed\mid p.\name\in \installation.\name,p.\version>\max(\installation.\version(p))\}$ & $\emptyset$ & $\{q\}$ \\
    $\sdown$           & $\{p\in \universe\setminus\removed\mid p.\name\in \installation.\name,p.\version<\max(\installation.\version(p))\}$ & $\emptyset$ & $\{q\}$ \\
    $\sinstallrequest$ & $\{ p \in P\setminus\removed \mid q \mbox{ mentioned in install request } p.\name = q.\name \}$ & $\emptyset$ & $\{q\}$ \\
    $\supgraderequest$ & $\{ p \in P\setminus\removed \mid q \mbox{ mentioned in upgrade request } p.\name = q.\name \}$ & $\emptyset$ & $\{q\}$  \\
    $\srequest$        & $\sinstallrequest^\infty \cup \supgraderequest^\infty$ & $\emptyset$ & $\{q\}$ \\
  \end{tabular}\caption{Selectors\label{tab:selectors}}
\end{table}

\section{Fact Format}

\begin{itemize}
  \item \texttt{unit($p.\name$,$p.\version$,in).} for $p\in\closure$
  \item \texttt{unit($p.\name$,$p.\version$,out).} for $p\in\removed$
  \item \texttt{installed($p.\name$,$p.\version$).} for $p\in\universe$
  \item \texttt{maxversion($p.\name$,$p.\version$).} for $p\in\universe$ and $p.\version=\max(\universe.\version(p))$
  \item \texttt{satisfies(Name,Version,Condition).} for $p\in\closure$
    \begin{itemize}
      \item applies to various parts of the CUDF-document
        \begin{itemize}
          \item dependencies
          \item conflicts
          \item requests
          \item recommendations
          \item keep flags
        \end{itemize}
    \end{itemize}
  \item \texttt{depends($p.\name$,$p.\version$,Condition).} for $p\in\closure$
  \item \texttt{conflict($p.\name$,$p.\version$,Condition).} for $p\in\closure$
  \item \texttt{recommends($p.\name$,$p.\version$,Condition,Weight).}
    \begin{itemize}
      \item $\pm f\in\optimization$, $f=\funsatrecom(S)$, and $p\in\selmax$
    \end{itemize}
  \item \texttt{request(Condition).}
    \begin{itemize}
      \item requests
      \item keep flags
    \end{itemize}
  \item \texttt{attribute($p.\name$,$p.\version$,Attribute,Value).}
    \begin{itemize}
      \item $\pm f\in\optimization$, $f=\fsum(S,\attr{attr})$, and $p\in\selmax$, or
      \item $\pm f\in\optimization$, $f=\faligned(S,\attr{group},\attr{value})$, and $p\in\selmax$, or
    \end{itemize}
  \item \texttt{installrequest(Name,Version).}
  \item \texttt{upgraderequest(Name,Version).}
  \item
    \texttt{clique($I$,$p.\name$,$p.\version)$} where $p\in I$ and $I$ is a non-singular clique in the graph\\
    $(\closure, \{ (p,q) \mid p,q\in\closure, p\neq q, q\in p.\conflicts \})$
    \begin{itemize}
      \item no overlapping cliques will be added
    \end{itemize}
  \item \texttt{criterion(Maximize,Selector,$\fcount$,Priority).}
  \item \texttt{criterion(Maximize,Selector,$\fsum(\attr{attr})$,Priority).}
  \item \texttt{criterion(Maximize,Selector,$\fnotuptodate$, Priority).}
  \item \texttt{criterion(Maximize,Selector,$\funsatrecom$,Priority).}
  \item \texttt{criterion(Maximize,Selector,$\faligned(\attr{group},\attr{value})$,Priority).}
\end{itemize}

\end{document}


A section where note will be made on open issus. These aissues are ment to be discussed with Peter or Thivyian and ultimatly solved. No open issues should remain at the end of November.

\subsection{Graph matching}

% NetworkX offers graph matching algorithms as do other repos. The drawback is that they are programmed for smaller graph such as molecule graphs with undirected edges and no edge atribues, or these attributes are not taken into account and only the structure is compared.
% My implementation of the max-pooling graph matching algorithm from \cite{simonovsky_graphvae_2018} return a $n\times n\times k\times k$ matrix with integer values ${0,2}$. Most probably something went wrong here since this makes little sense regarding interpretation.
% When graphs are compared with NetworkX using the greedy-matching or minimum distance, the similarity matrix is a $2\times 2$ matrix for comparing two graphs.
% My question is which sort of graph matching makes more sense for our usecase.
% Further I struggle to interprete the similarity matrix - is that my job tho? Maybe its just meant to be used in the loss function as described!

Hungarian algorithm:
Should we assume the affinity matrix is a cost or a profit matrix. If we assume it is a cost matrix and n != k the algorithm returns an error while padding. If we assume it is a profit matrix and convert it to a profit matrix it can handle any dimension. The shortest path varies with both approaches.
Further I assume the shortest path is what we are looking for and mask these entries with 1 and the rest of the matrix with 0.
\\
Diagonal of A is zeros. As soon as I do this, code crashes. Think it is because of divided zero, Martin adds 1e-7 to the X norm
\\
Do we backpropagate through the graph matching? then all torch functions, otherwise np.
\\
Martin takes 300 iterations for hungarian. also not looping over pairs.