\documentclass{standalone}
\begin{document}
	\chapter{The Remainder and Factor Theorems}
	\section{The Remainder Theorem}
	Consider the polynomial $f(x)$. Suppose that this polynomial is to be divided by the linear expression $x-a$. This gives:
	\begin{alignat*}{3}
		&&\frac{f(x)}{x-a}A&\equiv Q(x) + \frac{R}{x-a}\\
		&\implies&f(x) &\equiv Q(x) \cdot (x-a) + R\\
		\text{Let $x-a=0$}&\implies &x &= a\\
		& \therefore & f(a) & = R;\quad & \text{Where R is the remainder of $\frac{f(x)}{x-a}$} \\
		&            &      &           & \text{and Q is the quotient of $\frac{f(x)}{x-a}$}    \\
	\end{alignat*} \\
	\begin{example}
		Find the remainder when the cubic polynomial $f(x) = 2x^3-3x-5$ is divided by $x-2$
	\end{example}

	If $f(x)$ is to be divided by $x-2$, then $f(2)$ is equal to the remainder of $\frac{2x^3-3x-5}{x-2}$
	\begin{alignat*}{2}
		&        & 2x^3 -3x-5       \\
		& =\quad & 2(2)^3 - 3(2) -5 \\
		&\boxed{\therefore\quad R\quad=\quad5}
	\end{alignat*}
	\section{The Factor theorem}

	The factor theorem states that:
	\begin{itemize}
		\item{If the polynomial $f(x)$ is divided by $x-a$, then $f(a) = 0$ (i.e $R=0$}), therefore it can also be concluded that $x-a$ is a factor of $f(x)$
	\end{itemize}
	\begin{example}
		Determine whether $2x+3$ is a factor of $2x^3+x^2-5x+6$
	\end{example}

	\begin{flalign*}
		&  & \text{Let } f(x) &= 2x^3+x^2-5x+6& \\
		& \rlap{If $2x+3$ is a factor of $f(x)\colon$} &0 &=f\left(\frac{-3}{2}\right)\\
		&\rlap{However,} &0 &\neq 2\left(\frac{-3}{2}\right)^3 + \left(\frac{-3}{2}\right)^2 -5\left(\frac{-3}{2}\right) + 6 \\
		&&&\neq -3\\
		&&&\boxed{\therefore \quad \text{$2x+3$ is \textbf{not} a factor of $f(x)$}}
	\end{flalign*}
	\newpage
	\end{document}


\section{On Spectral Clustering: Analysis and an algorithm}
\label{ch:ng01}

\textit{On Spectral Clustering: Analysis and an algorithm} by Andrew Ng, Michael I. Jordan, and Francis R. Bach. \\
Cited by 3787. \textit{Advances in neural information processing systems 2 (2002)}.
\newline

\textbf{Main point} is that the author \begin{inparaenum}[\itshape a\upshape)]
\item suggest the method nomalize the rows of the chosen eigenvector.
\end{inparaenum}

\subsection{Algorithm}
Form a similarity matrix $S \in R^{n \times n}$ defined by $A_{ij} = exp(- \| s_i - s_j \|^2 / 2 \sigma^2)$ if $i \neq j$, and $S_{ii} = 0$.
\begin{figure}[ht]
\begin{mdframed}
\begin{enumerate}
\item[Input] : Similarity matrix $S \in R^{n \times n}$, number $k$ of clusters. \\
\item[Step 1] : Construct the normalized graph Laplacian $L_{sym} = I - D^{-\frac{1}{2}} S D^{-\frac{1}{2}}$. \\
\item[Step 2] : Find the $k$ eigenvectors $u_1, \cdots, u_k$ corresponding to the largest $k$ eigenvalues of $L_{sym}$. \\
\item[Step 3] : Let $U = R^{n \times k}$ be the matrix containing the eigenvectors $u_i$ as columns, and let $y_i \in R^k$ be the $i$th row of $U$.\\
\item[Step 4] : Let $T$ be the row-normalized $U$ matrix where $t_{ij} = \frac{u_{ij}}{(\sum_k u_{ik}^2)^2}$ and let $y_i \in R^k$ be the $i$th row of $T$.\\
\item[Step 5] : Cluster the points $(y_i)_{i=1,\cdots,n}$ in $R^k$ into clusters $C_1,\cdots,C_k$ via $k$-means clustering or any other algorithm that attempts to minimize distortion. \\
\item[Output] : clusters $A_1, \cdots, A_k$ where $A_k - {v_j|y_j \in C_i}$
\end{enumerate}
\end{mdframed}
\caption{Normalized spectral clustering according to Ng}
\end{figure}

Then assign the original point $s_i$ to cluster $j$ if and only if row $i$ of the matrix $Y$ was assigned to cluster $j$. The scaling parameter $\sigma^2$ controls how rapidly the affinity $S_{ij}$ falls off with the distance between $s_i$ and $s_j$.


\section{Quadratics and Polynomials}

\subsection{Basic factoring}

Given some $ax^2+bx+c$, try and find the factorization $(sx+u)(tx+v)$ by expanding this template expression.

\begin{remark}
    If one root is 0, the product of roots is 0 so
    the equation is of form $ax^2+bx=0$.
\end{remark}

\begin{remark}
    Difference of squares are of the form $x^2-a^2=0\implies (x-a)(x+a)=0$.
\end{remark}

\begin{remark}
    A perfect square is of the form $(x+a)^2=x^2+2ax+a^2$. This is a case of a "double root."
\end{remark}

\subsection{Quadratic Formula}

By manipulating $ax^2+bx+c=0$ by completing the square, the quadratic formula can be found.

\begin{equation}
    x=\frac{-b\pm \sqrt{b^2-4ac}}{2a}
\end{equation}

\subsection{Expansions}

\begin{eqnarray}
    (a+b)^2=a^2+2ab+b^2\\
    (a+b)^3 = a^3 + 3a^2b+3ab^2+b^3
\end{eqnarray}

\subsection{Factoring}

\begin{definition}[Difference of squares]
    \begin{equation}
        a^2-b^2=(a+b)(a-b)
    \end{equation}
\end{definition}

\begin{definition}[Sum of squares]
    \begin{equation}
        a^2+b^2=(a+b)^2-2ab
    \end{equation}
\end{definition}

\begin{definition}[Sum of cubes]
    \begin{equation}
        a^3+b^3=(a+b)(a^2-ab+b^2)
    \end{equation}
\end{definition}

\begin{definition}[Difference of cubes]
    \begin{equation}
        a^3-b^3=(a-b)(a^2+ab+b^2)
    \end{equation}
\end{definition}

\begin{definition}[Some cube identity]
    \begin{equation}
        a^3+b^3+c^3-3abc=(a+b+c)(a^2+b^2+c^2-ab-ac-bc)
    \end{equation}
\end{definition}

\begin{definition}[Simon factoring trick]
    Used to factor in a diophantine equation.
    If $ab + ka + nb = c$, then $(a+n)(b+k)=c+nk$.
\end{definition}

\subsection{Vieta's Formulas}

\begin{eqnarray}
    x^2+ax+b=(x-p)(x-q)\\
    x^2+ax+b=x^2-(p+q)x+pq
\end{eqnarray}

Thus, $a=p+q$ and $b=pq$. Generally,

\begin{align*}
    s_1&= & r_1+r_2+r_3&+\cdots+r_n & &=-\frac{a_{n-1}}{a_n} \\	 s_2&= & r_1r_2+r_1r_3+r_1r_4&+\cdots+r_{n-2}r_{n-1} & &=\phantom{-}\frac{a_{n-2}}{a_n} \\	 s_3&= & r_1r_2r_3+r_1r_2r_4&+\cdots+r_{n-2}r_{n-1}r_n & &=-\frac{a_{n-3}}{a_n} \\	 & & &\vdots & & \\	 s_n&= & r_1r_2r_3&\cdots r_n & &=(-1)^n\frac{a_0}{a_n}.\\
\end{align*}

\begin{definition}[AM-GM Inequality]
    \begin{equation}
        \frac{a_1+\ldots+a_n}{n}\geq \sqrt[n]{a_1a_2\ldots a_n}
    \end{equation}
\end{definition}


\section{Introduction}

\noindent
Consider the following arithmetic from Vladimir Nabokov's autobiography ``Speak, Memory.''

\begin{quote}
A foolish tutor had explained logarithms to me much too early, and I had
read (in a British publication, the {\it Boy's Own Paper}, I believe)
about a certain Hindu calculator who in exactly two seconds could find the
seventeenth root of, say,
352947114576027513 2301897342055866171392
(I am not sure I have got this right; anyway the root was 212).
\end{quote}

\noindent
Let us compute $212^{17}$ and check the result.
In the field shown below, enter

{\color{blue}
\begin{verbatim}
212^17
\end{verbatim}
}

\begin{center}
\begin{tikzpicture}
\node at (0,0) {\includegraphics[scale=0.2]{1st.png}};
\draw[red,thick] (2.3,-1.95) ellipse (2.5cm and 0.5cm);
\end{tikzpicture}
\end{center}

\noindent
After pressing the return key, Eigenmath displays the following result.

\bigskip
\noindent
$3529471145760275132301897342055866171392$

\bigskip
\noindent
Now let us compute the seventeenth root of this number.

{\color{blue}
\begin{verbatim}
N = 212^17
N^(1/17)
\end{verbatim}
}

\noindent
Eigenmath displays the following result.

\bigskip
\noindent
$212$

\bigskip
\noindent
When a symbol is assigned a value, such as $N$ above,
no result is printed.
To see the value of a symbol, just evaluate it.

{\color{blue}
\begin{verbatim}
N
\end{verbatim}
}

\noindent
$N=3529471145760275132301897342055866171392$

\bigskip
\noindent
The previous example shows a convention that will be used throughout
this manual.
That is, the color blue indicates something that the user should type.
The computer response is shown in black.


\subsection{Progressions}\label{subsec:progressions}

Progressions are an elementary concept that happens to be useful quite often. There is no definition of progression, but rather the term \enquote{progression} refers to specific recursively defined \hyperref[def:sequence]{sequences}.

\begin{definition}\label{def:arithmetic_progression}
  The \term{arithmetic progression} with \term{base} \( a_0 \) and \term{difference} \( d \) is the sequence
  \begin{equation}\label{eq:def:arithmetic_progression}
    a_k \coloneqq \begin{cases}
      a_0,         & k = 0, \\
      a_{k-1} + d, & k > 0.
    \end{cases}
  \end{equation}

  Clearly every index \( k \geq 0 \) we have the closed form representation \( a_k = a_0 + kd \).
\end{definition}

\begin{proposition}\label{thm:arithmetic_progression_partial_sums}
  The \hyperref[def:convergent_series]{series} constructed from the arithmetic progression \eqref{eq:def:arithmetic_progression} has partial sums
  \begin{equation}\label{eq:thm:arithmetic_progression_partial_sums}
    \sum_{k=0}^n a_k = \frac {(n + 1) (a_n - a_0)} 2.
  \end{equation}

  In the special case where \( a_0 = 0 \) and \( d = 1 \), this reduces to
  \begin{equation}\label{eq:thm:arithmetic_progression_partial_sums/integers}
    \sum_{k=0}^n k = \sum_{k=1}^n k = \frac {n (n + 1)} 2.
  \end{equation}
\end{proposition}
\begin{proof}
  \begin{balign*}
    2 \sum_{k=0}^n a_k
     & =
    2 \sum_{k=0}^n (a_0 + kd)
    =    \\ &=
    \sum_{k=0}^n (a_0 + kd) + \sum_{k=0}^n (a_0 + (n-k)d)
    =    \\ &=
    \sum_{k=0}^n (2 a_0 + nd)
    =    \\ &=
    (n + 1) (a_0 + a_n).
  \end{balign*}
\end{proof}

\begin{definition}\label{def:geometric_progression}
  The \term{geometric progression} with \term{base} \( a_0 \) and \term{denominator} \( q \) is the sequence
  \begin{equation}\label{eq:def:geometric_progression}
    a_k \coloneqq \begin{cases}
      a_0,       & k = 0, \\
      a_{k-1} q, & k > 0.
    \end{cases}
  \end{equation}

  Clearly every index \( k \geq 0 \) we have the closed form representation \( a_k = a_0 q^k \).

  The \hyperref[def:convergent_series]{series}
  \begin{equation}\label{eq:def:geometric_progression/series}
    \sum_{k=0}^\infty a_k = a_0 \sum_{k=0}^\infty q^k.
  \end{equation}
  is called the \term{geometric series} for \( q \). Without loss of generality, we will assume \( a_0 = 1 \) when speaking about geometric series.
\end{definition}

\begin{proposition}\label{thm:geometric_series_properties}
  The geometric series \eqref{eq:def:geometric_progression/series} has the following basic properties:
  \begin{thmenum}
    \thmitem{thm:geometric_series_properties/finite_sum} For all \( q \in \BbbC \setminus \{ 1 \} \), the geometric series \eqref{eq:def:geometric_progression/series} has partial sums
    \begin{equation}\label{thm:geometric_progression/partial_sum}
      \sum_{k=0}^n q^k = \frac {1 - q^{n+1}} {1 - q}.
    \end{equation}

    Compare this to \fullref{thm:xn_minus_yn_factorization}.

    \thmitem{thm:geometric_series_properties/degenerate} In the degenerate case \( q = 1 \), the progression itself is constant, and its partial sums are instead
    \begin{equation}\label{thm:geometric_progression/degenerate}
      \sum_{k=0}^n q^k = n + 1.
    \end{equation}

    \thmitem{thm:geometric_series_properties/series_sum_exterior} For \( \abs{q} \geq 1 \), the geometric series diverges.

    \thmitem{thm:geometric_series_properties/series_sum_interior} For \( 0 < \abs{q} < 1 \), the geometric series converges absolutely with sum
    \begin{equation}\label{thm:geometric_progression/series_sum}
      \sum_{k=0}^\infty q^k = \frac 1 {1 - q}.
    \end{equation}
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:geometric_series_properties/finite_sum} Follows from \fullref{thm:xn_minus_yn_factorization}.
  \SubProofOf{thm:geometric_series_properties/degenerate} Obvious.

  \SubProofOf{thm:geometric_series_properties/series_sum_exterior} For \( q = 1 \), \fullref{thm:geometric_series_properties/degenerate} implies that the series diverges because it grows indefinitely. If \( \abs{q} = 1 \) and \( q \neq 1 \), the integer powers \( q^k \) are rotations around the complex plane unit circle, which do not tend to a limit. Hence, the series diverges again.

  When \( \abs{q} > 1 \), \( \abs{q^n} \) grows indefinitely with \( n \), and it follows that
  \begin{equation*}\label{thm:geometric_progression/cauchy_partial_sum}
    \sum_{k=m}^n q^k
    =
    q^m \sum_{k=0}^{n-m} q^k
    =
    q^m \frac {1 - q^{n-m+1}} {1 - q}
    =
    \frac {q^m - q^{n+1}} {1 - q}.
  \end{equation*}
  can get arbitrarily large. Therefore, in this case the series also diverges.

  \SubProofOf{thm:geometric_series_properties/series_sum_interior} Fix \( q \in B(0, 1) \). Since only \( q^{n + 1} \) depends on \( n \) in \eqref{thm:geometric_progression/partial_sum}, we obtain \eqref{thm:geometric_progression/series_sum} by simply noting that \( q^n \to 0 \) when \( n \to \infty \).
\end{proof}

\begin{example}\label{ex:n_ary_decomposition}
  A simple but important practical example of a \hyperref[eq:def:geometric_progression/series]{geometric series} is
  \begin{equation}\label{eq:ex:n_ary_decomposition/binary}
    \sum_{k=0}^\infty \frac 1 {2^k} = \frac 1 {1 - \sfrac 1 2} = 2.
  \end{equation}

  Note that if the series starts at \( k = 1 \) instead of \( k = 0 \), it sums to \( 1 \). This is often applied in analysis indirectly via \fullref{thm:continuous_function_series_powers_of_two}.

  Another application of \eqref{eq:ex:n_ary_decomposition/binary} is showing that \( 0.\overline{1} = 2 \) in the binary number system. More generally, for the \( n \)-ary number system we have
  \begin{equation}\label{eq:ex:n_ary_decomposition/general}
    \sum_{k=0}^\infty \parens*{ \frac {n-1} n }^k = \frac 1 {1 - \ifrac {(n-1)} n} = n.
  \end{equation}
\end{example}

\begin{remark}\label{rem:progressions_and_interest}
  In this example we exploit the equivalence between the closed form representations in \fullref{def:arithmetic_progression} and \fullref{def:geometric_progression} and the corresponding inductive definitions. The equivalences are obvious from a mathematical standpoint, however outside of mathematics they have highly nontrivial consequences. Indeed, they highlight the difference between simple interest and compound interest.

  As an example, a savings account with \( 1000\$ \) with a simple monthly interest of \( 2\% \) will earn \( 240\$ \) over a year:
  \begin{equation*}
    1000 (1 + 12 \cdot \sfrac 2 {100}) = 1240.
  \end{equation*}

  The same account with a compound interest of \( 2\% \) will earn a bit more - about \( 268\$ \):
  \begin{equation*}
    1000 (1 + \sfrac 2 {100})^{12} \approx 1268.24.
  \end{equation*}

  Over the course of ten years, however, simple interest will earn a total of \( 2400\$ \), while compound interest will earn \( \approx 9765\$ \).

  The difference between linear and exponential growth appears staggering in a real world situation even though the difference may not be very noticeable short-term.
\end{remark}

\begin{definition}\label{def:harmonic_progression}
  The \term{harmonic progression} with \term{base} \( a_0 \) and \term{difference} \( d \) is the sequence
  \begin{equation}\label{eq:def:harmonic_progression}
    a_k \coloneqq \frac 1 {a_0 + kd}.
  \end{equation}

  That is, each term is the reciprocal of the corresponding term in an \hyperref[def:arithmetic_progression]{arithmetic progression} with the same base and difference. In order for \eqref{eq:def:harmonic_progression} to be well-defined, either
  \begin{itemize}
    \item \( d = 0 \) and \( a_0 \neq 0 \), which turns \eqref{eq:def:harmonic_progression} into the constant sequence \( \seq{ \sfrac 1 {a_0} }_{k=0}^\infty \).
    \item \( d \neq 0 \), in which case
    \begin{equation*}
      a_k = \frac d {\sfrac {a_0} d + k}.
    \end{equation*}

    Thus, if \( d \neq 0 \), \( \ifrac {a_0} d \) must not be a negative integer unless we are satisfied with only the first \( -\ifrac {a_0} d \) terms of the progression existing.
  \end{itemize}

  Furthermore, the series may only start at \( k = 0 \) if \( a_0 \neq 0 \).

  For series related to harmonic progressions, see \fullref{ex:harmonic_series}
\end{definition}

\begin{remark}\label{rem:harmonic_progression_recursive_form}
  Unlike \fullref{def:arithmetic_progression} and \fullref{def:geometric_progression}, we have defined the harmonic progressions via closed-form expressions. Indeed, the equivalent inductive definition is more awkward to work with:
  \begin{equation*}
    a_k \coloneqq \begin{cases}
      \ifrac 1 {a_0},                    & k = 0, \text{ only defined if } a_0 \neq 0, \\
      \ifrac 1 {(a_0 + d)},                & k = 1,                                      \\
      \ifrac 1 {(\sfrac 1 {a_{k-1}} + d)}, & k > 0.
    \end{cases}
  \end{equation*}
\end{remark}


\documentclass[12pt]{cdblatex}
\usepackage{examples}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% ============================================================================================
\bgroup
\CdbSetup{action=hide}
\begin{cadabra}
   import cdblib
   checkpoint_file = 'tests/semantic/output/example-06.json'
   cdblib.create (checkpoint_file)
   checkpoint = []
\end{cadabra}
\egroup

\clearpage

% ============================================================================================
\section*{Example 6-01 Evaluating components}

\begin{cadabra}
   {\theta, \varphi}::Coordinate.
   {a,b,c,d,e,f,g,h#}::Indices(values={\theta, \varphi}, position=independent).

   \partial{#}::PartialDerivative.

   V  := { V_{\theta} = \varphi, V_{\varphi} = \sin(\theta) }.   # cdb(ex-06.100,V)
   dV := \partial_{b}{V_{a}} - \partial_{a}{V_{b}}.              # cdb(ex-06.101,dV)

   evaluate (dV, V)   # cdb(ex-06.102,dV)
\end{cadabra}

\begin{align*}
             V_{a} &= \Cdb{ex-06.100}\\[10pt]
   \cdb{ex-06.101} &= \Cdb{ex-06.102}
\end{align*}

\clearpage

% ============================================================================================
\section*{Example 6-02 Riemann tensor of a 2-sphere}

% LCB: the dimensionality of the space is inferred from the number of distinct coordinates
%      so use just two symbols for the coordinates

\begin{cadabra}
   {\theta, \varphi}::Coordinate.
   {a,b,c,d,e,f,g,h#}::Indices(values={\theta, \varphi}, position=independent).

   \partial{#}::PartialDerivative.

   Gamma := \Gamma^{a}_{b c} -> 1/2 g^{a d} (   \partial_{b}{g_{d c}}
                                              + \partial_{c}{g_{b d}}
                                              - \partial_{d}{g_{b c}}).

   Rabcd := R^{a}_{b c d} ->   \partial_{c}{\Gamma^{a}_{b d}}
                             - \partial_{d}{\Gamma^{a}_{b c}}
                             + \Gamma^{e}_{b d} \Gamma^{a}_{c e}
                             - \Gamma^{e}_{b c} \Gamma^{a}_{d e}.

   gab := { g_{\theta\theta}   = r**2,
            g_{\varphi\varphi} = r**2 \sin(\theta)**2 }.      # cdb(ex-06.201,gab)

   iab := { g^{\theta\theta}   = 1/r**2,
            g^{\varphi\varphi} = 1/(r**2 \sin(\theta)**2) }.  # cdb(ex-06.202,iab)

   substitute (Rabcd, Gamma)                                  # cdb(ex-06.203,Gamma)

   evaluate   (Gamma, gab+iab, rhsonly=True)                  # cdb(ex-06.204,Gamma)
   evaluate   (Rabcd, gab+iab, rhsonly=True)                  # cdb(ex-06.205,Rabcd)

   # convert from a rule to a simple expression
   Riem := R^{a}_{b c d}.
   substitute (Riem, Rabcd)                                   # cdb(ex-06.206,Riem)

   from cdb.core.component import *

   RiemCompt = get_component (Riem, $\theta, \varphi, \theta, \varphi$)   # cdb(ex-06.207,RiemCompt)

\end{cadabra}

\begin{align*}
   &\Cdb{ex-06.201}\\[10pt]
   &\Cdb{ex-06.202}\\[10pt]
   &\Cdb{ex-06.204}\\[10pt]
   &\Cdb{ex-06.205}\\[10pt]
   &\Cdb{ex-06.206}
\end{align*}

\begin{align*}
   R^{\theta}{}_{\varphi\varphi\theta} &= \Cdb{ex-06.207}
\end{align*}

\clearpage

% ============================================================================================
\section*{Example 6-03 Using {\tt complete} to compute the inverse metric}

This version uses \verb|complete| to compute the inverse metric.

\begin{cadabra}
   {\theta, \varphi}::Coordinate.
   {a,b,c,d,e,f,g,h#}::Indices(values={\theta, \varphi}, position=independent).

   \partial{#}::PartialDerivative.

   g^{a b}::InverseMetric.  # essential when using complete (gab, $g^{a b}$)

   Gamma := \Gamma^{a}_{b c} -> 1/2 g^{a d} (   \partial_{b}{g_{d c}}
                                              + \partial_{c}{g_{b d}}
                                              - \partial_{d}{g_{b c}}).

   Rabcd := R^{a}_{b c d} ->   \partial_{c}{\Gamma^{a}_{b d}}
                             - \partial_{d}{\Gamma^{a}_{b c}}
                             + \Gamma^{e}_{b d} \Gamma^{a}_{c e}
                             - \Gamma^{e}_{b c} \Gamma^{a}_{d e}.

   gab := { g_{\theta\theta}   = r**2,
            g_{\varphi\varphi} = r**2 \sin(\theta)**2 }.      # cdb(ex-06.301,gab)

   complete   (gab, $g^{a b}$)                                # cdb(ex-06.302,gab)

   substitute (Rabcd, Gamma)

   evaluate   (Gamma, gab, rhsonly=True)                      # cdb(ex-06.303,Gamma)
   evaluate   (Rabcd, gab, rhsonly=True)                      # cdb(ex-06.304,Rabcd)

\end{cadabra}

\begin{align*}
   &\Cdb{ex-06.301}\\[10pt]
   &\Cdb{ex-06.302}\\[10pt]
   &\Cdb{ex-06.303}\\[10pt]
   &\Cdb{ex-06.304}
\end{align*}

\clearpage

% ============================================================================================
\section*{Example 6-04 Components by scalar projection}

This example shows how one component of the Riemann tensor can be computed using a scalar
projection.

\begin{cadabra}
   {\theta, \varphi}::Coordinate.
   {a,b,c,d,e,f,g,h#}::Indices(values={\theta, \varphi}, position=independent).

   theta{#}::LaTeXForm{"\theta"}.
   varphi{#}::LaTeXForm{"\varphi"}.

   # usual definitions for the connection and Riemann tensor

   Gamma := \Gamma^{a}_{b c} -> 1/2 g^{a d} (   \partial_{b}{g_{d c}}
                                              + \partial_{c}{g_{b d}}
                                              - \partial_{d}{g_{b c}}).

   Rabcd := R^{a}_{b c d} ->   \partial_{c}{\Gamma^{a}_{b d}}
                             - \partial_{d}{\Gamma^{a}_{b c}}
                             + \Gamma^{e}_{b d} \Gamma^{a}_{c e}
                             - \Gamma^{e}_{b c} \Gamma^{a}_{d e}.

   gab := { g_{\theta \theta}   = r**2,
            g_{\varphi \varphi} = r**2 \sin(\theta)**2 }.   # cdb(ex-06.400,gab)

   iab := { g^{\theta\theta}   = 1/r**2,
            g^{\varphi\varphi} = 1/(r**2 \sin(\theta)**2) }.

   substitute (Rabcd, Gamma)
   evaluate   (Rabcd, gab+iab, rhsonly=True)

   # above code just to compute Rabcd
   # following code is all that is needed for the scalar projection method

   # define the basis for vectors and dual vectors

   basis := {theta^{\theta} = 1, varphi^{\varphi} = 1}.
   dual  := {theta_{\theta} = 1, varphi_{\varphi} = 1}.

   # obtain components by contracting with basis

   compt := R^{a}_{b c d} theta_{a} varphi^{b} theta^{c} varphi^{d}.   # cdb(ex-06.401,compt)
   substitute (compt,Rabcd)

   evaluate (compt,basis+dual)                                         # cdb(ex-06.402,compt)

   compt_sympy = compt._sympy_()

   # cdbBeg(print.ex-06.04)
   print ('type compt   = ' + str(type(compt)))        # shows that compt is a Cadabra object
   print ('type gphiphi = ' + str(type(compt_sympy)))  # shows that gphiphi is a Python object
   print ('     compt   = ' + str(compt))              # will contain LaTeX markup
   print ('     gphiphi = ' + str(compt_sympy))        # will be pure Python/SymPy
   # cdbEnd(print.ex-06.04)

   checkpoint.append (compt)
\end{cadabra}

\begin{align*}
   R^{\theta}{}_{\varphi\theta\varphi} &= \Cdb{ex-06.401}\\
                                       &= \Cdb{ex-06.402}
\end{align*}

\IfFileExists{example-06-04.cdbcopy}%
{\lstinputlisting[backgroundcolor=\color{white}]{example-06-04.cdbcopy}}%
{Where is {\tt example-06-04.cdbcopy}?}

\clearpage

% ============================================================================================
\section*{Example 6-05 Components by selection}

This example shows how one component of the metric tensor can be computed by indexing
the result of a call to \verb|evaluate|.

\begin{cadabra}
   {\theta, \varphi}::Coordinate.
   {a,b,c,d,e,f,g,h#}::Indices(values={\theta, \varphi}, position=independent).

   theta{#}::LaTeXForm{"\theta"}.
   varphi{#}::LaTeXForm{"\varphi"}.

   gab := { g_{\theta \theta}   = r**2,
            g_{\varphi \varphi} = r**2 \sin(\theta)**2 }.   # cdb(ex-06.500,gab)

   metric := g_{a b}.

   evaluate (metric,gab)

   indcs = metric[2][1][0]                                  # cdb(ex-06.501,indcs)
   compt = metric[2][1][1]                                  # cdb(ex-06.502,compt)

   # cdbBeg(print.ex-06.05)
   print ('metric = ' + str(metric.input_form())+'\n')  # reveals Cadabra's internal structure for storing metric

   print ('metric[0] = ' + str(metric[0]))
   print ('metric[1] = ' + str(metric[1]))
   print ('metric[2] = ' + str(metric[2])+'\n')

   print ('metric[2][1] = '+ str(metric[2][1]))
   print ('metric[2][1][0] = '+ str(metric[2][1][0]))
   print ('metric[2][1][1] = '+ str(metric[2][1][1]))
   # cdbEnd(print.ex-06.05)

   checkpoint.append (indcs)
   checkpoint.append (compt)
\end{cadabra}

\begin{align*}
   g_{\varphi\varphi} &= g_{\Cdb{ex-06.501}}\\
                      &= \Cdb{ex-06.502}
\end{align*}

\IfFileExists{example-06-05.cdbcopy}%
{\lstinputlisting[backgroundcolor=\color{white}]{example-06-05.cdbcopy}}%
{Where is {\tt example-06-05.cdbcopy}?}

\clearpage

% ============================================================================================
% export to json format

\bgroup
\CdbSetup{action=hide}
\begin{cadabra}
   for i in range( len(checkpoint) ):
      cdblib.put ('check{:03d}'.format(i),checkpoint[i],checkpoint_file)
\end{cadabra}
\egroup

\end{document}


\subsection{Dual space}

The dual space \(V^*\) of vector space \(V\) is the set of all linear forms, \(\hom(V,F)\).

\subsubsection{The dual space is itself a vector space}

\(v\in V\)

\(f\in F\)

\(av = f\)

\(bv = g\)

\((a\oplus b)v=f+g\)

\((a\oplus b)v=av + bv\)

So there is some operation we can do on two members of dual space

Linear in addition. That is, if we have two dual "things", we can define the addition of functions as the operation which results int he outputs being added.

what about linear in scalar? same approach.

Well we define

\((c\odot a)=cav\)

\subsubsection{The dual space has the same dimension as the underlying vector space}


\chapter{Abstract algebra}

\section{Group}

\index{binary operation}%
The type of a \emph{binary operation} is \(a \to b \to c\).
\index{binary operation!closed}%
\index{closed binary operation}%
The type of a \emph{closed binary operation} is \(a \to a \to a\).
A binary operation \(\cdot\)
is \emph{associative} iff \(x\cdot(y\cdot z) = (x\cdot y)\cdot z\),
is \emph{commutative} iff \(x \cdot y = y \cdot x\).

\index{magma}%
A \emph{magma} is a set and a closed binary operation.
\index{semigroup}%
A \emph{semigroup} is an associative magma (a magma whose operation is associative).
\index{monoid}%
A \emph{monoid} is a semigroup with an identity element.
\index{group}%
A \emph{group} is a monoid where each element has an inverse.
\index{commutative!group}%
\index{group!abelian}%
\index{group!commutative}%
An \emph{abelian group} or a \emph{commutative group}
is a group whose operation is commutative.

\(-x\) is the additive inverse of \(x\).

\section{Ring}

A
\index{ring}%
\emph{ring} is \((S,+,\cdot,-,0,1)\) where
\((S,+,-,0)\) is a commutative group,
\((S,\cdot,1)\) is a monoid,
and multiplication distributes addition:
\begin{align*}
    x \cdot (y+z) &= (x \cdot y) + (x \cdot z)
    \\
    (x+y) \cdot z &= (x \cdot z) + (y \cdot z).
\end{align*}
A ring is
\index{commutative!ring}%
\index{ring!commutative}%
\emph{commutative} iff its multiplicative monoid is commutative.

If \(R\) is a ring and \(A \to R\) is a function space,
then the \emph{natural ring of the function space} is:
\begin{align*}
    0_{A\to R}(x) &= 0_R
    \\ 1_{A\to R}(x) &= 1_R
    \\ (- f)(x) &= - f(x)
    \\ (f + g)(x) &= f(x) + g(x)
    \\ (f \cdot g)(x) &= f(x) \cdot g(x).
\end{align*}

\section{Field}

\(x^{-1}\) is the multiplicative inverse of \(x\).

\index{field}%
\((F,+,\cdot,-,{}^{-1},0,1)\) is a \emph{field} iff
\((F,+,-,0)\) is a commutative group,
\((F-\{0\},\cdot,{}^{-1},1)\) is a commutative group,
and \(\cdot\) distributes \(+\) (that is \(x \cdot (y + z) = (x \cdot y) + (x \cdot z)\)) \cite{wpfield}.
For brevity, if \(F\) is a set, ``\(F\) is a field'' means that \((F,+,\cdot,-,{}^{-1},0,1)\) is a field.
That is, for brevity, we blur the distinction between a field and its underlying set.


\section{Intuition of Euler-Lagrange equation}

Euler-Lagrange equation is a \textbf{partial differential equation} fundamental
to the study of the \textbf{calculus of variations}, especially in the field of
optimizing a \textbf{functional}. There are many methods to derive this
equation rigourously, but this section will give readers a basic understanding
of where this equation comes from.

\subsection{Functional}

According to calculus of variations, a typical functional is defined as:

\begin{equation}
	y\mapsto J[y]=\int_{x_0}^{x_1}F(x,y,\dot{y})dx
	\label{functional-def}
\end{equation}

where $y$ is a function dependent on $x$. Arc length, for instance, is a
typical functional:

$$
y\mapsto L[y]=\int_{x_0}^{x_1}\sqrt{1+\dot{y}^2}dx
$$

in which $F$ is called the \textbf{core} of the functional.

\subsection{Optimize functional}

In single-variable calculus, we know that a function reaches an extremum if its
derivative becomes zero, and this is same for functionals. Since the solution
of the optimization of a problem is a function instead of a real number, we
assume $y$ is the optimal function and define $f$ as follows:

\begin{equation}
	f=y+\varepsilon\mu
	\label{var-y}
\end{equation}

In equation (\ref{var-y}), $\mu$ is an arbitrary function satisfying
$\mu(x_0)=\mu(x_1)=0$, and $\varepsilon\in\mathbb{R}$. Since $y$ denotes the
result of optimization of $J[f]$, the function $N(\varepsilon)=J[f]$ reaches
its extremum at $\varepsilon=0$. Therefore, we can deduce:

\begin{equation}
	\left.dN\over d\varepsilon\right|_{\varepsilon=0}=0
	\label{optimal-functional}
\end{equation}

In order to find out $y$, we first need to express the derivative in
(\ref{optimal-functional}). According to \textbf{Leibniz's rule}, we can swap
the differential operator and integral if the limits of the integral are
not depending on $\varepsilon$.

$$
\begin{aligned}
	{dN\over d\varepsilon}
	&={d\over d\varepsilon}\int_{x_0}^{x_1}F(x,f,\dot{f})dx \\
	&=\int_{x_0}^{x_1}{d\over d\varepsilon}F(x,f,\dot{f})dx
\end{aligned}
$$

Since $F$ is a multivariable function, we take its total derivative with
respect to $\varepsilon$. Since $x$ does not depend on $\varepsilon$,
$dx/d\varepsilon$ is automatically set to zero.

$$
\begin{aligned}
	{dN\over d\varepsilon}
	&=\int_{x_0}^{x_1}{\partial F\over\partial x}
	{dx\over d\varepsilon}+{\partial F\over\partial f}
	{df\over d\varepsilon}+{\partial F\over\partial\dot{f}}
	{d\dot{f}\over d\varepsilon}dx \\
	&=\int_{x_0}^{x_1}{\partial F\over\partial f}{df\over d\varepsilon}
	+{\partial F\over\partial\dot{f}}{d\dot{f}\over d\varepsilon}dx
\end{aligned}
$$

According to the definition of $f$ in (\ref{var-y}), $df/d\varepsilon$ is
simply $\mu$, and $d\dot{f}/d\varepsilon=\dot{\mu}$.

$$
{dN\over d\varepsilon}=\int_{x_0}^{x_1}\mu{\partial F\over\partial f}
+\dot{mu}{\partial F\over\partial\dot{f}}dx
$$

Using the technique of integration by parts, $\mu$ became factored out. Because
$\mu(x_0)=\mu(x_1)=0$, there is nothing left outside the integral.

$$
\begin{aligned}
	{dN\over d\varepsilon}
	&=\int_{x_0}^{x_1}\mu{\partial F\over\partial f}dx+
	\left.\mu{\partial F\over\partial\dot{f}}\right|_{x_0}^{x_1}
	-\int_{x_0}^{x_1}\mu{d\over dx}{\partial F\over\partial\dot{f}}dx \\
	&=\int_{x_0}^{x_1}\mu\left({\partial F\over\partial f}-
	{d\over dx}{\partial F\over\partial\dot{f}}\right)dx
\end{aligned}
$$

According to (\ref{optimal-functional}) and \ref{var-y}, we replace
$dN/d\varepsilon$, $f$, and $\dot{f}$ with zero, $y$, and $\dot{y}$
respectively.

$$
0=\int_{x_0}^{x_1}\mu\left({\partial F\over\partial y}-{d\over dx}
{\partial F\over\partial\dot{y}}\right)dx
$$

According to the \textbf{fundamental lemma of calculus of variations}, we know
that the coefficient of $\mu$ must be zero if the integral is zero.

$$
{\partial F\over\partial y}-{d\over dx}{\partial F\over\partial\dot{y}}=0
$$

Therefore, we obtain the Euler-Lagrange equation:

\begin{equation}
	{\partial F\over\partial y}={d\over dx}{\partial F\over\partial\dot{y}}
	\label{el-def}
\end{equation}

\subsection{Shortcut for specific functional}

If the core $F$ of a functional does not depend on $x$ ($F=F(y,\dot{y})$), a
simpler approach is possible without taking the derivative of $\partial
F/\partial\dot{y}$, facilitating the whole optimization problem.

\begin{flushleft}
	Step 1: Eliminate the differential operator via integrating factor
\end{flushleft}

If we multiply (\ref{el-def}) by $\dot{y}$, we can formulate:

$$
{\partial F\over\partial y}\dot{y}=
{d\over dx}{\partial F\over\partial\dot{y}}\dot{y}
$$

According to product rule of differential calculus, the right part of the
equation is equivalent to:

$$
{d\over dx}\left({\partial F\over\partial\dot{y}}\dot{y}\right)
-\ddot{y}{\partial F\over\partial\dot{y}}
$$

As a result, we can gather a dot product in the left side.

$$
\left({\partial F\over\partial y}\hat{\i}
+{\partial F\over\partial\dot{y}}\hat{\j}\right)^\top
\left({dy\over dx}\hat{\i}+{d\dot{y}\over dx}\hat{\j}\right)
={d\over dx}\left({\partial F\over\partial\dot{y}}\dot{y}\right)
$$

We can discover that the left side is the total derivative of $F$ with respect
to $x$, so we find the antiderivative of the both side in order to eliminate
differential operator.

$$
F(y,\dot{y})+C={\partial F\over\partial\dot{y}}\dot{y}
$$

Therefore, when optimizing functionals with cores only depending on $y$ and
$\dot{y}$, we can use the following equation instead of the Euler-Lagrange
equation:

$$
F(y,\dot{y})-\dot{y}{\partial F\over\partial\dot{y}}=C
$$

where $C$ is a constant. This corollary is known as \textbf{Beltrami identity}.


%---------------------------Aspect Gamma-----------------------------
\section{Aspect Gamma}

This metric compares root-mean-square edge length to volume.
The root-mean-square edge length is
\[
R = \sqrt{\frac{\sum_{i=0}^{5}\normvec{L_i}^2}{6}}
\]
and so, normalizing the metric to a value of 1 for equilateral tetrahedra, we have
\[
q = \frac{R^3\sqrt{2}}{12|V|}.
\]

Note that if  $|V| < DBL\_MIN$, we set $q = DBL\_MAX$.

\tetmetrictable{aspect $\gamma$}%
{$1$}%                  Dimension
{$[1,3]$}%              Acceptable range
{$[1,DBL\_MAX]$}%       Normal range
{$[1,DBL\_MAX]$}%       Full range
{$1$}%                  Equilateral tet
{\cite{par:93}}%        Citation
{v\_tet\_aspect\_gamma}%                            Verdict function name


%\section{Mathematical framework (for QM)}

%	\begin{enumerate}
%	\item Dirac's braket notation
%	\item Inner/(Outer) product
%	\item Linear operator
%	\item Adjoints and Hermitian operators
%	\item Pauli matrices
%	\item Tensor Product and tensor space
%	\end{enumerate}

	Unless stated otherwise, we consider each coefficient to be complex (as in elements of $\mathbb{C}$). Vectors have complex components.
	\section{Inner product spaces}
	In standard vector notation we define the inner (scalar) product of complex vectors as
	\begin{equation*}
	 ( \vec{v}, \vec{w} ) =  \begin{pmatrix} \bar{v_1} & \bar{v_2}\end{pmatrix} \begin{pmatrix} w_1 \\ w_2 \end{pmatrix} = \begin{pmatrix} \bar{w_1} & \bar{w_2}\end{pmatrix} \begin{pmatrix} v_1 \\ v_2 \end{pmatrix} = ( \vec{w}, \vec{v} )^{\dagger}
	\end{equation*}
	Where $\dagger$ represents the conjugate transpose.\\
%	This property is fundamental in the sense that it will allows us to go from a state space --- that can be many dimensional --- to a \textit{measurement} space, which assumes real values.\\ %0 and 1 in our case??

	It is important also to note that through the inner product of two vectors we also define the norm $\|\ket{v}\|  =  \sqrt{( \vec{v}, \vec{v} )} $.\\

	\section{Tensor product spaces}
	The tensor product $V\otimes W$ is an operation between vector spaces that combines every element of the first vector space and every element of the second vector space in a bigger vector space. Tensor product is linear and from its properties emerges the famous phenomenon of quantum entanglement, which simply is that not all vectors in $\H = V\otimes W$ can be divided into $\ket{v}\otimes\ket{w}$ with $\ket{v}\in V,\; \ket{w}\in W$. This will later be explained in the next section.
	Notation and abbreviation for the tensor product is
	$$ \ket{v}\otimes\ket{w} = \ket{v}\ket{w} = \ket{v,w} = \ket{vw}$$
	It has the following properties:
	\begin{align*}
		 \forall\ket{v}\in V ,\; \forall\ket{w}\in W, \; \forall z\in \mathbb{C}&\\
					&z(\ket{v}\otimes\ket{w}) = (z\ket{v})\otimes\ket{w} = \ket{v}\otimes(z\ket{w} \\
		\forall\ket{v_1},\ket{v_2}\in V ,\; \forall\ket{w}\in W	&\\
					 &(\ket{v_1} + \ket{v_2})\otimes\ket{w} = \ket{v_1w} + \ket{v_2w} \\
		\forall\ket{v}\in V ,\; \forall\ket{w}\in W, \; A:V\rightarrow V' \; B:W\rightarrow W'&\\
					&(A\otimes B) \left(\sum_i a_i \ket{v_i w_i} \right) = \sum_i a_i A\ket{v_i}\otimes B\ket{w_i}
	\end{align*}
	The inner product on $V$ and $W$ can be used to define (linearly) an inner product on $V\otimes W$.


\section{Policy Gradients}
Policy gradient algorithms directly optimize the policy $\pi_\theta(\cdot \mid  s)$. The goal is to maximize the expected return under $\pi_\theta$:
\[
    \max_\theta J(\theta) = \E_{\tau \sim \pi_\theta} [R(\tau)]
\]
Using the log-derivative trick ($\nabla f = f \nabla \log f$) and the likelihood of a trajectory $\tau$:
\[
    \pi_\theta(\tau) = \rho_0(s_0) \prod_{t=0}^T P(s_{t+1} \mid  s_t, a_t) \pi_\theta(a_t \mid  s_t)
\]
the \emph{policy gradient} can be derived:
\begin{align}
    \nabla_\theta J(\theta)
    &= \E_{\tau \sim \pi_\theta} [\nabla_\theta \log \pi_\theta(\tau) R(\tau)] \nonumber \\
    &= \E_{\tau \sim \pi_\theta} \left[\sum_{t=0}^T R(\tau) \nabla_\theta \log \pi_\theta(a_t \mid  s_t) \right]
    \label{eq:policy-gradient}
\end{align}

Learning the optimal policy can be easier than learning all the action values $Q(s,a)$, and directly optimizing $\pi_\theta$ makes it possible to have smoother updates and more stable convergence than Q-learning. In addition, all policy gradient algorithms work with continuous/infinite action spaces, and can encourage exploration with additional rewards (entropy). However, they can be less adapted to tabular environments and are often less sample-efficient than value-based approaches, which typically use memory buffers.

\subsection{On-Policy}
\paragraph{REINFORCE}
Based on eq. \ref{eq:policy-gradient}, the REINFORCE algorithm simply uses a Monte-Carlo estimate of $\nabla_\theta J(\theta)$ to optimize $\pi_\theta$, and increase the likelihood of trajectories with high rewards.

\begin{algorithm}[H]
\DontPrintSemicolon
\For{a number of epochs}{
    \While{not enough samples}{
        collect trajectory $\tau_i=(s_0^i, a_0^i, \dots, s_{T_i}^i)$ by running active policy $\pi_\theta$
    }
    $\hat{g} \leftarrow \sum_i \sum_t R(\tau^i) \nabla_\theta \log \pi_\theta(a_t^i \mid  s_t^i)$ \tcp*{compute policy gradient estimate}
    $\theta \leftarrow \theta + \alpha \hat{g}$ \;
}
\caption{Vanilla REINFORCE}
\label{algo:reinforce}
\end{algorithm}

It is unbiased but typically has very high variance and is slow to converge. Variance can be reduced using alternatives expressions of the gradient (all three can be combined):
\begin{itemize}
    \item \emph{Causality (don't let the past distract you)}
    Intuitively, rewards obtained before taking an action do not bring any information, they are just noise. Indeed $R(\tau)$ can be replaced by the \emph{reward-to-go} $R_t(\tau) = \sum_{t'=t}^T r_t'$ without making the policy gradient expression biased \demo.
    \item \emph{Baseline}
    For any function $b:\mathcal{S} \rightarrow \mathbb{R}$, we have:
    \[
        \E_{\tau \sim \pi_\theta} \left[\sum_{t=0}^T b(s_t) \nabla_\theta \log \pi_\theta(a_t \mid  s_t) \right] = 0
    \]
    Hence \demo we can use $(R(\tau) - b(s_t))$ instead of $R(\tau)$, and the variance is minimal when choosing $b(s) = V^\pi(s)$  (e.g. by fitting $b(s_t)$ to  $R_t(\tau)$).
    \item \emph{Discount}
    Using a discounted return $R(\tau) = \sum_{t=0}^T \gamma r_t$ can also reduce the variance but, unlike the 2 previous techniques, it makes the gradient biased.
\end{itemize}

More generally, the policy gradient can be expressed as:
\begin{equation}
    \nabla_\theta J(\theta)
    = \E_{\tau \sim \pi_\theta} \left[\sum_{t=0}^T \Psi_t \nabla_\theta \log \pi_\theta(a_t \mid  s_t) \right]
    \label{eq:policy-gradient2}
\end{equation}

Where $\Psi_t$ may be the reward $R(\tau)$, the reward-to-go $R_t(\tau)$, with or without a baseline (e.g. $R_t(\tau) - b(s_t)$).

\paragraph{Actor-Critic}
The policy gradient (eq. \ref{eq:policy-gradient2}) can also be expressed \demo with:
\begin{itemize}
    \item $\Psi_t = Q^\pi(s_t,a_t)$: state-action value function
    \item $\Psi_t = A^\pi(s_t,a_t)$: advantage function
    \item $\Psi_t = r_t + \gamma V^\pi(s_{t+1}) - V^\pi(s_t) = \delta_t$: TD error
\end{itemize}
Actor-critic methods combine the strengths of policy gradient and value-based methods by estimating a critic ($Q^\pi$, $V^\pi$ and/or $A^\pi$) in order to better optimize the actor (policy $\pi_\theta$). Adding a critic reduces the variance, but estimation errors introduce a bias as soon as the critic is not perfect.

\paragraph{Actor-Critic with compatible functions}
We can try to approximate $A^\pi$ with a function $f_\phi: \mathcal{S} \times \mathcal{A} \rightarrow \mathbb{R}$. According to the \emph{Compatible Function Approximation Theorem} \cite{sutton2000policy} \demo, if $f_\phi$ satisfies
\begin{equation}
    \forall s, a, \nabla_\phi f_\phi(s,a) = \nabla_\theta \log \pi_\theta(a\mid s)
    \qquad \text{(i.e. $f_\phi$ is \emph{compatible})}
\label{eq:compat-fct-1}
\end{equation}
\begin{equation}
    \E_{\tau \sim \pi_\theta} \left[ \sum_{t=0}^T \gamma^t (Q^\pi(s_t,a_t) - f_\phi(s_t,a_t)) \nabla_\theta \log \pi_\theta(a_t\mid s_t)\right] = 0
\label{eq:compat-fct-2}
\end{equation}
then the policy gradient is exact:
\[
    \nabla_\theta J(\theta)
    = \E_{\tau \sim \pi_\theta} \left[\sum_{t=0}^T \gamma^t f_\phi(s_t,a_t) \nabla_\theta \log \pi_\theta(a_t \mid  s_t) \right]
\]
Eq. \ref{eq:compat-fct-1} can be satisfied with $f_\phi(s,a) = \nabla_\theta \log \pi_\theta(s,a)^T \phi$, and eq. \ref{eq:compat-fct-2} by solving:
\[
    \min_{\phi, w} \E_{s, a \sim \pi_\theta}\left[
    \left(Q^\pi(s,a) - f_\phi(s,a) - v_w(s)\right)^2
    \right]
\]
for some function $v_w:\mathcal{S} \rightarrow \mathbb{R}$ \demo. In practice, TD is used to estimate $V^\pi$ with $v_w$, and $Q^\pi$ with $f_\phi + v_w$, so that $f_\phi(s,a) \approx A^\pi(s,a)$.

\paragraph{Actor-Critic with Generalized Advantage Estimation (GAE) \cite{schulman2015high}}
This method only requires estimating $V^\pi$ (easier to learn than $Q^\pi(s,a) \forall s,a$, especially in high dimension) to obtain an estimate of $A^\pi$.
Similar to multi-steps learning, we can define the \emph{n-steps advantage estimate}:
\begin{equation}
    A_t^{(n)} = r_t + \gamma r_{t+1} + \dots + \gamma^{n-1} r_{t+n-1} + \gamma^n V^\pi(s_{t+n}) - V^\pi(s_t)
\label{eq:n-steps-advantage}
\end{equation}
and control the bias-variance trade-off with $n$ ($n=1$: lower variance but higher bias when the estimate of $V^\pi$ is wrong, $n\rightarrow \infty$: Monte-Carlo estimate, no bias but higher variance). Like TD($\lambda$) learning, GAE averages the estimates and uses $\lambda \in [0,1]$ to control the trade-off:
\[
    A_t^{GAE(\gamma,\lambda)} = (1-\lambda) \sum_{n=0}^\infty \lambda^n A_t^{(n)} = \sum_{n=0}^\infty (\lambda \gamma)^n \delta_{t+n}
\]
Here as well, eligibility traces are a way to implement GAE$(\gamma,\lambda)$. Indeed, we can re-write the sample estimate of the policy gradient:
\[
    \sum_{t=0}^\infty A_t^{GAE(\gamma,\lambda)} \nabla_\theta \log \pi_\theta(a_t \mid  s_t)
    = \sum_{t=0}^\infty \nabla_\theta \log \pi_\theta(a_t \mid  s_t) \sum_{n=0}^\infty (\gamma \lambda)^n \delta_{t+n}
\]

\begin{algorithm}[H]
\DontPrintSemicolon
\For{a number of episodes}{
    initialize $s_0$ and set $e_0(s)=0, \forall s$\;
     \While{episode not done}{
        take action $a_t \sim \pi(\cdot \mid s_t)$, observe $r_t$ and $s_{t+1}$ \;
        \For{all $s$}{
          $e_t(s) \leftarrow \lambda \gamma e_{t-1}(s) + \mathbbm{1}_{s_t=s}$ \tcp*{this leads to $e_t(s) = \sum_{k=0}^t(\lambda \gamma)^{t-k} \mathbbm{1}_{s_k=s}$}
          $V(s_t) \leftarrow V(s_t) + \alpha e_t(s) \delta_t$ \;
        }
    }
}
\caption{Actor-critic with GAE, using eligibility traces}
\label{algo:gae-eligibility-traces}
\end{algorithm}

% todo: add algos, e.g. learn $V^\pi ?$

\paragraph{A2C / A3C}
\emph{Asynchronous Advantage Actor Critic} (A3C) \cite{mnih2016asynchronous} and its synchronous version A2C are two widely used actor-critic methods. They both run $n$ environments in parallel to get better estimates of the returns (more samples = less variance).
\begin{itemize}
    \item Both a policy $\pi_\theta$ and a value function $V^{\pi}_{\theta_v}$ are learned. Most of the layers are shared (fig. \ref{fig:actor-critic-architecture}).
    \item \emph{Asynchronous}. The A3C updates are done asynchronously: each agent sends its gradient to the global network every $k$ steps and updates its local weights after that.
    \item \emph{Advantage}. Use the $n$-steps return (forward view, eq. \ref{eq:n-steps-advantage}) to learn $V^\pi_{\theta_v}$ and to estimate the advantage function $A^\pi$.
\end{itemize}
The A2C algorithm works the same way but with synchronous, deterministic updates. It waits for all agents to be done with the $k$ steps, before performing a batch update and updating all weights at the same time.

% todo: give actual formula (and algo) of A3C/A2C, cf https://arxiv.org/pdf/1611.01224.pdf
\begin{figure}[H]
    \centering
    \includegraphics[width=0.8\linewidth]{figures/shared-actor-critic-network.png}
    \caption{Example of a shared network architecture for actor-critic methods.}
    \label{fig:actor-critic-architecture}
\end{figure}

\paragraph{Entropy}
To encourage exploration and avoid converging on a deterministic policy too fast, an entropy cost can be added to the policy gradient update:
\[
    \theta \leftarrow \theta + \alpha (\nabla_\theta J(\theta) + \nabla_\theta H_\theta(s_t))
\]
With
\[
    H_\theta(s_t) = - \sum_a \pi_\theta(a_t\mid s_t) \log \pi_\theta(a_t\mid s_t)
\]
the entropy term to maximize.

\paragraph{Relative Policy Performance Bound}
Instead of looking directly for $\max_\pi J(\pi)$, we can optimize the \emph{relative performance} between $\pi$ and and arbitrary policy $\pi_{old}$: $\max_\pi J(\pi) - J(\pi_{old})$. For any policies $\pi = \pi_\theta$, $\pi_{old} = \pi_{\theta_{old}}$, we have the \emph{relative performance bound}:
\begin{align}
    J(\pi) - J(\pi_{old})
    &= \E_{\tau \sim \pi} \left[ \sum_{t=0}^\infty \gamma^t A^{\pi_{old}}(s_t,a_t) \right] \nonumber\\
    &= \frac{1}{1-\gamma} \E_{s\sim d^\pi(s), a \sim \pi_{old}(a\mid s)} \left[ \frac{\pi(a\mid s)}{\pi_{old}(a\mid s)} A^{\pi_{old}}(s,a)\right] \nonumber\\
    &\geq \underbrace{\frac{1}{1-\gamma} \E_{s\sim d^{\pi_{old}}(s), a \sim \pi_{old}(a\mid s)} \left[
        \frac{\pi(a\mid s)}{\pi_{old}(a\mid s)} A^{\pi_{old}}(s,a)
    \right]}_{=L_{\theta_{old}}(\theta)} - C D_{KL}^{max}(\pi \mid \mid  \pi_{old})
    \label{eq:performance-bound}
\end{align}
with
\begin{itemize}
    \item The \emph{discounted state distribution} of policy $\pi$
    \begin{equation}
        d^\pi(s) = (1-\gamma) \sum_{t=0}^\infty \gamma^t P(s_{t}=s\mid \pi)
    \label{eq:stationary-state-distribution}
    \end{equation}
% According to Sylvain Lamprier,
% FALSE:   According to \href{https://eva.fing.edu.uy/pluginfile.php/343525/mod_resource/content/1/modern_methods.pdf}{this}, \href{https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html#policy-gradient-theorem}{this}, \href{https://www.alexirpan.com/rl-derivations/#state-visitation--occupency-measure}{this} and \href{https://proceedings.neurips.cc//paper/2020/file/69bfa2aa2b7b139ff581a806abf0a886-Paper.pdf}{this} (to check?),
    This is \emph{not} the same as the \emph{stationary distribution}:
    \[
        d^\pi(s) = \lim_{t\rightarrow \infty} P(s_t = s \mid \pi)
    \]
    \item $D_{KL}^{max}(\pi \mid \mid  \pi_{old}) = \max_s D_{KL}[\pi(\cdot\mid s) \mid \mid  \pi_{old}(\cdot\mid s)]$
    \item $C$ a factor that depends on $\gamma$ and $\pi_{old}$
\end{itemize}
A number of algorithms (e.g. TRPO, PPO) maximize this lower bound of the relative performance, by keeping $\pi$ (current policy) and $\pi_{old}$ (old policy) close and maximizing objective $L_{\theta_{old}}(\theta)$. This has the advantages of:
\begin{enumerate}
    \item Re-using past samples, since $s,a$ are sampled from the old policy $\pi_{old}$ in the training objective $L_{\theta_{old}}(\theta)$. This is more sample-efficient.
    \item Controlling the policy updates in the space of distributions instead of the space of parameters, thanks to the KL divergence term. This makes updates smoother and training more stable.
\end{enumerate}

\paragraph{Trust Region Policy Optimization (TRPO) \cite{schulman2015trust}}
TRPO considers an approximate objective to eq. \ref{eq:performance-bound}. Since the penalty coefficient $C$ can be very large and lead to very small updates, it uses a hard constraint of the $D_{KL}$ (trust region), and since $D_{KL}^{max}$ is hard to optimize it uses $\overline{D}_{KL}$ instead.
\begin{equation}
    \max_\theta L_{\theta_{old}}(\theta)
    \quad \text{subject to} \quad
    \underbrace{
        \E_{s \sim d^{\pi_{old}}}[D_{KL}(\pi(\cdot\mid s) \mid \mid  \pi_{old}(\cdot\mid s)]
    }_{=\overline{D}_{KL}(\pi\mid \mid \pi_{old})}
    \leq \delta
\label{eq:trpo-exact}
\end{equation}
for some hyper-parameter $\delta$.
Using Taylor approximation, this training objective (eq. \ref{eq:trpo-exact}) is replaced by
\begin{equation}
    \max_\theta g^T (\theta - \theta_{old})
    \quad \text{s.t.} \quad
    \frac{1}{2}(\theta-\theta_{old})^T F (\theta-\theta_{old}) \leq \delta
\label{eq:trpo-taylor-approx}
\end{equation}
with
\begin{align*}
g
    &= \nabla_\theta L_{\theta_{old}}(\theta)\big|_{\theta=\theta_{old}} \\
    &= \nabla_\theta J(\theta)\big|_{\theta=\theta_{old}}
    \qquad \text{(only at $\theta=\theta_{old}$)} \\
    &= \frac{1}{1-\gamma}
    \E_{s\sim d^{\pi_{old}}, a \sim \pi_{old}(\cdot\mid s)}\left[
        \nabla_\theta \log \pi_\theta(a\mid s)\big|_{\theta=\theta_{old}} A^{\pi_k}(s,a)
    \right]
\end{align*}
and
\begin{align*}
F
    &= \nabla^2_\theta \overline{D}_{KL}(\pi\mid \mid \pi_{old})\big|_{\theta=\theta_{old}} \\
    &= \E_{s\sim d^{\pi_{old}}}\left[
        \nabla^2_\theta D_{KL}(\pi(\cdot\mid s)\mid \mid \pi_{old}(\cdot\mid s))\big|_{\theta=\theta_{old}}
    \right]
\end{align*}

At each iteration, sample estimates of the gradient $g$ and Fisher matrix $F$ can be computed, and the constrained optimization objective of eq. \ref{eq:trpo-taylor-approx} is solved approximately. The solution
\footnote{This update step does not take into account architectures using dropout or shared parameters between policy and value function.}
(obtained by deriving KKT conditions) is:
\[
    \theta = \theta_{old} + \beta F^{-1} g
    \qquad \text{with} \qquad
    \beta = \sqrt{\frac{2\delta}{g^T F^{-1} g}}
\]
Instead of directly inverting $F$ (impossible with deep and large models), the conjugate gradient method solves $Fx=g$ (max $d$ steps with $\theta \in \mathbb{R}^d$) and obtains an estimate of $x=F^{-1}g$, called the natural gradient
\footnote{The natural gradient is effectively a gradient in the space of the distributions (using distance $D_{KL}(\pi_{\theta}\mid \mid \pi_{\theta_{old}})$), rather than the space of parameters (with euclidean distance on parameters $\theta$). More details \href{https://julien-vitay.net/deeprl/NaturalGradient.html}{here}.}.
The update steps becomes:
\[
    \theta = \theta_{old} + \sqrt{\frac{2\delta}{x^T F x}} x
\]
Finally, a \emph{backtracking line search} adjusts the step size in order to make the largest update that effectively improves the objective $L_{\theta_{old}}(\theta)$ and respects the constraint $\overline{D}_{KL}(\pi\mid \mid \pi_{old}) \leq \delta$ (which may be violated due to the approximations).

\paragraph{Natural Gradient with compatible functions}
When using a compatible function for the advantage estimation (eq. \ref{eq:compat-fct-1} and \ref{eq:compat-fct-2}), it follows that $F \phi = \nabla_\theta J(\theta)$ \demo, and $\phi = F^{-1} \nabla_\theta J(\theta)$ is actually the natural gradient (cf TRPO).

An incremental algorithm is suggested by \cite{bhatnagar2007incremental}, using learning rate $\alpha_i$ for the update of the critic (parameters $\phi$ and $w$, cf \emph{Actor-Critic with compatible functions}), and learning rate $\beta_i$ for the policy: $\theta \leftarrow \theta + \beta_i \phi$. This ensures that the critic converges faster than the actor. In practice however, the learning rates are hard to tune (approaches doing batch updates and adding an entropy cost have been suggested to make training more stable \cite{pajarinen2019compatible}).

\paragraph{Proximal Policy Optimization (PPO) \cite{schulman2017proximal}}
Second-order optimization methods like natural gradients perform well, but are computationally expensive and complex to implement. PPO mimics the reliable trust-region update of TRPO, but using a simpler first-order method. Two versions are proposed:
\begin{itemize}
    \item \emph{Adaptive KL penalty}. Consider the following unconstrained objective to maximize:
    \begin{align*}
        \mathcal{L}^{KL}(\theta)
        &= L_{\theta_{old}}(\theta) -
        \beta_k \overline{D}_{KL}(\pi\mid \mid \pi_{old}) \\
        &= \E_{s,a\sim \pi_{old}} \left[
            \frac{\pi(a\mid s)}{\pi_{old}(a\mid s)} A^{\pi_{old}}(s,a)
            - \beta_k D_{KL}[\pi(\cdot\mid s)\mid \mid \pi_{old}(\cdot\mid s)]
        \right]
    \end{align*}
    With $\beta_k$ an adaptive KL penalty term ensuring that $\overline{D}_{KL}(\pi\mid \mid \pi_{old})$ stays close to $\delta$ most of the time.
    % todo: algo
    \item \emph{Clipped objective}.
    \[
        \mathcal{L}^{CLIP}(\theta)
        = \E_{s,a\sim \pi_{old}} \left[
            \min(r_t(\theta) A^{\pi_{old}}_t, \text{clip}(r_t(\theta), 1-\epsilon, 1+\epsilon) A^{\pi_{old}}_t)
        \right]
    \]
    With $r_t(\theta) = \frac{\pi_\theta(a_t\mid s_t)}{\pi_{\theta_{old}}(a_t,s_t)}$ and $A^{\pi_{old}}_t = A^{\pi_{old}}(s_t,a_t)$. As illustrated in figure \ref{fig:ppo-clip}, this objective is a pessimistic lower bound of $\frac{\pi(a\mid s)}{\pi_{old}(a\mid s)} A^{\pi_{old}}(s,a)$, and effectively discourage too large improvements of policy $\pi_\theta$ by clipping the probability ratio. Updates in the wrong direction, i.e. that deteriorate $\pi_\theta$, remain fully penalized.
\end{itemize}
Compared to TRPO, PPO makes it straightforward to share parameters between the policy and value functions. Also, re-using the sampled data $\mathcal{D}_k$ for $K$ update steps makes it more data-efficient.

\begin{figure}
    \centering
    \includegraphics[width=0.5\linewidth]{figures/ppo-clip.png}
    \caption{\emph{PPO w/ clipped objective}. Plots showing $\mathcal{L}^{CLIP}$ as a function of the probability ratio $r$, for positive advantages (left) and negative advantages (right). The red circle on each plot shows the starting point for the optimization, i.e., $r=1$. More detailed explanations \href{https://stackoverflow.com/a/50663200}{here}. Figure taken from \cite{schulman2017proximal}.}
    \label{fig:ppo-clip}
\end{figure}

\subsection{Off-Policy}
Although actor-critic methods such as PPO or A2C/A3C are already re-using a couple recent samples, they are not as sample-efficient as off-policy value-based methods. \emph{Off-policy policy gradient} methods improve this, and make it possible to define better exploration strategies.

In this section, samples are collected with a \emph{behavior policy} $\mu$, typically different from our learned policy $\pi_\theta$.

\paragraph{Off-policy policy gradient theorem}
Given a behavior policy $\mu$ and its stationary distribution $d^\mu(s) = \lim_{t\rightarrow \infty} P(s_t = s \mid \mu)$ (see eq. \ref{eq:stationary-state-distribution}), we can define an \emph{off-policy performance objective} \footnote{
This is slightly different from the on-policy reward that we have been considering so far: $J(\theta) = \E_{\tau \sim \pi_\theta}[R(\tau)] = \E_{s_0 \sim \rho_0}[V^{\pi_\theta}(s_0)]$.
}:

\[
    J_\mu(\theta) = \E_{s_0 \sim \mu}[V^{\pi_\theta}(s_0)]
\]
The off-policy policy gradient can then be expressed as:
\begin{align}
    \nabla_\theta J_\mu(\theta)
    &= \nabla_\theta \E_{s\sim d^\mu} \left[ \sum_a \pi_\theta(a\mid s) Q^{\pi_\theta}(s,a) \right] \nonumber \\
    &= \E_{s\sim d^\mu} \left[
        \sum_a \nabla_\theta \pi_\theta(a\mid s) Q^{\pi_\theta}(s,a) +
        \underbrace{\pi_\theta(a\mid s) \nabla_\theta Q^{\pi_\theta}(s,a)}_{\approx 0}
    \right] \nonumber \\
    &\approx \E_{s,a \sim \mu} \left[
        \frac{\pi_\theta(a\mid s)}{\mu(a\mid s)} Q^{\pi_\theta}(s,a) \nabla_\theta \log \pi_\theta(a\mid s)
    \right]
    \label{eq:off-policy-policy-gradient}
\end{align}

The approximation is necessary since $\nabla_\theta Q^{\pi_\theta}(s,a)$ is typically hard to compute. However, this biased gradient still improves the policy and allows converging to the right solution, at least in the tabular case (off-policy policy gradient theorem, \cite{degris2012off}).

\emph{Note}. Eq. \ref{eq:off-policy-policy-gradient} closely resembles the on-policy gradient (eq. \ref{eq:policy-gradient}), but with an added weight $\frac{\pi_\theta(a\mid s)}{\mu(a\mid s)}$, and using the stationary distribution.
The \emph{policy gradient theorem} \footnote{Proof can be found \href{https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html\#policy-gradient-theorem}{here} or \href{https://stanford.edu/~ashlearn/RLForFinanceBook/PolicyGradient.pdf}{here}} actually states that:
\begin{align}
    \nabla_\theta J(\theta)
    &= \E_{s\sim d^{\pi_\theta}} \left[ \sum_a Q^{\pi_\theta}(s,a) \nabla_\theta \pi_\theta(a\mid s) \right]
    \nonumber \\
    &= \E_{s,a\sim \pi_\theta} \left[ Q^{\pi_\theta}(s,a) \nabla_\theta \log \pi_\theta(a\mid s) \right]
\end{align}

\paragraph{Estimating $Q^\pi$ in off-policy}

\emph{Tabular case (discrete action and state spaces)}.
Using a slightly different TD error $\delta_t = r_t + \gamma \E_{a\sim \pi(\dot \mid s_{t+1}}[Q(s_{t+1})] - Q(s_t,a_t)$, we can estimate $Q^\pi$ with TD-0 learning, even if our samples $(s_t, a_t, s_{t+1})$ are not collected with policy $\pi$:
\[
Q(s_t, a_t) \leftarrow Q(s_t, a_t) + \alpha \delta_t
\]

Multi-step learning is harder however, since the next actions $a_{t+k}$ are sampled from the behavior policy $\mu$:
\[
Q(s, a) \leftarrow Q(s, a) + \alpha \E_{s_t,a_t,s_{t+1} \sim \mu, \forall t} \left[
    \sum_{t=0}^k \gamma^t \left( \prod_{i=0}^t c_i \right) \left(
        r_t + \gamma \E_{a_{t+1}\sim \pi} [Q(s_{t+1}, a_{t+1})] - Q(s_t,a_t)
    \right) \mid s_0 = s, a_0 = a
\right]
\]
With $c_i$ a coefficient that can be
\begin{itemize}
    \item $c_i = \lambda$: regular $Q(\lambda)$ algorithm, but biased since we are off-policy, and only converges if $\mu$ and $\pi$ are close
    \item $c_i = \frac{\pi(a_i\mid s_i)}{\mu(a_i\mid s_i)}$: unbiased thanks to importance sampling, but not stable (high variance)
    \item $c_i = \lambda \pi(a_i\mid s_i)$: \emph{Tree Backup} \cite{precup2000eligibility} works even when $\mu$ and $\pi$ are different, but not efficient due to premature “cuts” if $\pi(a_i\mid s_i)$ becomes small.
    \item $c_i = \lambda \min(1, \frac{\pi(a_i\mid s_i)}{\mu(a_i\mid s_i)})$: \emph{Retrace} \cite{munos2016safe} combines the best of both worlds, with controlled variance $c_i \leq \lambda$), no premature “cuts”, and even has guarantees of convergence.
\end{itemize}

\emph{Approximate case (e.g. continuous state space)}.
Here as well we can use the Retrace coefficient $c_i$, but learning $Q_\theta(s,a)$ requires a target value $Q^{ret}$ in order to compute an error (e.g. least square) and a gradient. For each trajectory $\tau$, the target can be defined recursively (starting from the final value $Q^{ret}(s_{T-1}, a_{T-1}) = r_{T-1}$):
\[
Q^{ret}(s_t, a_t) = r_t + \gamma c_{t+1} [Q^{ret}(s_{t+1}, a_{t+1}) - Q_\theta(s_{t+1}, a_{t+1})] + \gamma E_{a\sim \pi} Q_\theta(s_{t+1}, a)
\]

\paragraph{Actor-Critic with Experience Replay (ACER)} \cite{wang2016sample}
Similar to on-policy actor critic methods (e.g. A3C/A2C), learning an estimate of $Q^\pi$ in eq. \ref{eq:off-policy-policy-gradient} can reduce the variance. ACER decomposes $\nabla_\theta J_\mu(\theta)$ into
\begin{align*}
\nabla_\theta J_\mu(\theta) = \E_{s_t\sim \mu} \biggl[
    & \E_{a_t\sim \mu}[
        \min(c, \omega_t(a_t)) Q^\pi(s_t, a_t) \nabla_\theta \log \pi_\theta(a_t\mid s_t)
    ] \\
    & + \E_{a\sim \pi}\left[
        \max\left(0, \frac{\omega_t(a)-c}{\omega_t(a)}\right) Q^\pi(s_t, a) \nabla_\theta \log \pi_\theta(a\mid s_t)
    \right]
\biggr]
\end{align*}

With $c$ a hyper-parameter and $\omega_t(a) = \frac{\pi(a\mid s_t)}{\mu(a\mid s_t)}$ the importance sampling ratio. The left term clips this ratio to bound the variance, the right term ensures that the estimate is unbiased. Using our estimates of $Q^\pi$, and substracting our estimate $V_{\theta_v}$ of $V^\pi$ to further reduce the variance, the gradient estimate becomes:
\begin{align*}
    g_t^\text{acer} =
    & \min(c, \omega_t(a_t)) (Q^{ret}(s_t, a_t) - V_{\theta_v}(s_t)) \nabla_\theta \log \pi_\theta(a_t\mid s_t) \\
    &+ \E_{a\sim \pi}\left[
        \max\left(0, \frac{\omega_t(a)-c}{\omega_t(a)}\right)
        (Q_{\theta_v}(s_t, a) - V_{\theta_v}(s_t)) \nabla_\theta \log \pi_\theta(a\mid s_t)
    \right]
\end{align*}

In addition to this, ACER does trust-region updates (like TRPO) but actually considers the gradient $g_{\phi_\theta(s_t)}$ w.r.t. the policy distribution parameters $\phi_\theta(s)$ (e.g. logits or probability vector of discrete actions) instead of the model parameters $\theta$. With $\pi_{\theta_a}$ a smoothly updated average policy, the following optimisation problem is solved:
\[
    \min_z \frac{1}{2} \mid \mid g_{\phi_\theta(s_t)} - z\mid \mid ^2 \quad \text{s.t.} \quad
    \nabla_{\phi_\theta(s_t)} D_{KL}[
        \pi_{\theta_a}(\cdot \mid s_t) \mid \mid \pi_\theta(\cdot \mid s_t)
    ]^T z \leq \delta
\]
Which has a closed form solution $z^*$. Policy parameters $\theta$ are updated using the gradient $g_\theta = z^* \nabla_\theta \phi_\theta(s_t)$.

\subsection{Continuous Action Spaces}
Previous policy gradients methods can be used with either a discrete or continuous action space (e.g. $\mathcal{A} \in \mathbb{R}^d, \pi(a\mid s) = \mathcal{N}(a; \mu_\theta(s), \Sigma_\theta(s))$). But continuous action spaces actually allow taking the gradient w.r.t. actions $\nabla_a Q_\theta(s,a)$ and deriving specific algorithms.

\paragraph{Deterministic Policy Gradients (DPG/DDPG)} \cite{silver2014deterministic} \cite{lillicrap2015continuous}
In addition to a continuous action space, DPG considers a non-stochastic policy taking actions $a = \mu_\theta(s)$. The on-policy policy gradient becomes:
\begin{equation}
    \nabla_\theta J(\theta) = \E_{s\sim d^{\pi_\theta}} \left[
        \nabla_a Q^{\pi_\theta}(s,a) \Bigr|_{a=\mu_\theta(s)} \nabla_\theta \mu_\theta(s)
    \right]
\label{eq:ddpg-on-policy}
\end{equation}
And critic $Q^{\pi_\theta}$ can be learned with SARSA updates.

In order to add exploration and to be more sample-efficient, an off-policy version is also proposed:
\begin{equation}
    \nabla_\theta J_\beta(\theta) \approx \E_{s\sim d^\beta} \left[
        \nabla_a Q^{\pi_\theta}(s,a) \Bigr|_{a=\mu_\theta(s)} \nabla_\theta \mu_\theta(s)
    \right]
\label{eq:ddpg-off-policy}
\end{equation}
Doing the same approximation as in \ref{eq:off-policy-policy-gradient} (and this time we call $\beta$ the behavior policy). Critic can be learned with Q-learning updates.

DDPG simply implements the off-policy version of DPG with deep neural networks for $Q^\pi$ and $\mu_\theta$, and adding noise to $\mu_\theta$ to construct the exploration policy. Stability is achieved by using tricks from DQN: replay buffer, target networks and soft updates. A couple extensions use additional tricks: 2 value networks $Q^\pi$ like Double DQN (TD3 \cite{fujimoto2018addressing}), Prioritized replay and a distributional critic (D4G \cite{barth2018distributed}).

\paragraph{Policy gradient with an off-policy critic (Q-Prop)} \cite{gu2016q}
While off-policy methods like DDPG are more sample efficient than REINFORCE, the off-policy gradient (eq. \ref{eq:ddpg-off-policy}) is biased and typically less stable (requiring fine hyper-parameter optimization). The Q-Prop algorithm uses an unbiased on-policy gradient (more stable) but reduces the variance (more sample efficient) thanks a control variate $\eta(s_t)$ and an off-policy critic $Q_w$ learned like in DDPG.

\begin{align}
    \nabla_\theta J(\theta) =
    & \E_{s_t\sim d^\pi, a_t \sim \pi} \underbrace{
        \left[
            \nabla_\theta \log \pi_\theta(a_t\mid s_t) (\hat{A}(s_t,a_t) - \eta(s_t) \overline{A_w}(s_t,a_t))
        \right]
    }_{
        \text{REINFORCE correction term ($\approx 0$ if $Q_w$ is good)}
    }
    \nonumber \\
    & + \E_{s_t\sim d^\pi} \underbrace{
        \left[
            \eta(s_t) \nabla_a Q_w(s_t,a) \Bigr|_{a=\mu_\theta(s_t)} \nabla_\theta \mu_\theta(s_t)
        \right]
    }_{
        \text{On-policy policy gradient with control variate $\eta(s_t)$}
    }
\end{align}

With
\begin{itemize}
    \item $\overline{Q}(s_t,a_t) = Q_w(s_t, \mu_\theta(s_t)) + \nabla_a Q_w(s_t,a) \Bigr|_{a=\mu_\theta(s_t)}(a_t-\mu_\theta(s_t))$ (1st order Taylor approximation)
    \item $\overline{A}(s_t,a_t) = \overline{Q}(s_t,a_t) - \E_{a \sim \pi_\theta}\left[\overline{Q}(s_t,a)\right] = \nabla_a Q_w(s_t, a) \Bigr|_{a=\mu_\theta(s_t)} (a_t-\mu_\theta(s_t))$
    \item $\hat{A}(s_t,a_t)$ estimated with GAE (using $V_w$ learned on-policy)
    \item $\eta(s_t) = 1$ if $\hat{A}(s_t,a_t) \overline{A}(s_t,a_t) > 0$ (i.e. our estimate of $A^\pi$ according to $Q_w$ at least has the same sign than the observed advantage $\hat{A}$), and $\eta(s_t) = 0$ otherwise.
\end{itemize}


\chapter{Real tuple spaces}

Earlier, we defined \enquote{space} as another word for \enquote{set} because we want
that definition to include every real tuple space.

The set \(\Real^n\) is the set of all real \(n\)-tuples.

The dimension of \(\Real^n\) is \(n\).

An example element of \(\Real^3\) is \((1,2,3)\).

\section{An example basis}

Imagine a flat sheet of paper.

Draw a point \(A\).

Draw a vector named \(i\), from \(A\), \SI{1}{cm} long, pointing right.

Draw another vector named \(j\), also from \(A\), \SI{1}{cm} long, but pointing up.

Thus, the vectors \(i\) and \(j\) are orthogonal.

Then, we declare the basis
\( e : \Real^2 \to E^2 \) as \( e(x,y) = xi + yj \).%

\section{Cartesian coordinate system}

Let \(E^n\) mean the \(n\)-dimensional Euclidean space.

The \(n\)-dimensional Cartesian coordinate system matches
a point in \(E^n\) and a point in \(\Real^n\).

\footnote{\url{https://en.wikipedia.org/wiki/Cartesian_coordinate_system}}

For example...

What mapping is drawn in \FigureRef{fig:a-2d-cartesian-coordinate-system}?

\begin{figure}[h]
    \centering
    \url{https://en.wikipedia.org/wiki/File:Cartesian-coordinate-system.svg}
    \caption{A two-dimensional Cartesian coordinate system}
    \label{fig:a-2d-cartesian-coordinate-system}
\end{figure}

\section{Example of geometry with numbers: describing a circle in \(\Real^2\)}

Every variable is a real number unless specified otherwise.

Assume a two-dimensional Cartesian coordinate system.

Consider a circle \(C\) with center \((0,0)\) and radius 1.
That circle can be described in two ways:
the algebraic description \eqref{eq:set-circle-algebraic},
and the parametric description \eqref{eq:set-circle-parametric}.
Both sets have the same members and describe the same circle.
\begin{align}
    \label{eq:set-circle-algebraic}
    C_{\text{algebraic}} &= \{ (x,y) ~|~ x^2 + y^2 = 1 \}
    \\
    \label{eq:set-circle-parametric}
    C_{\text{parametric}} &= \{ (r \cos t, r \sin t) ~|~ t \in [0,2\pi] \}
\end{align}

The parametric description \eqref{eq:set-circle-parametric}
tells us that the dimension of \(C\) is one
because the description uses one parameter \(t\).

\FIXME{Rename \enquote{parametric description} to \enquote{chart}?}

The derivative of the parametric function is the curve's velocity.

A shape's dimension is the number of parameters in its parametric description.

To see that both sets does describe a circle (and the same circle),
we can plot the graph of both sets.

\section*{Describing a curve parametrically or algebraically}

\paragraph{Using parametric equations}

We can describe a curve with a function of type \( \Real \to \Real^n \).
For example, we can write \(x(t) = (r \cos t, r \sin t)\) to describe a circle of radius \(r\).
We can also write the same equation as a set of parametric equations:
\(x(t) = r \cos t\) and \(y(t) = r \sin t\).

\paragraph{Using algebraic equations}

We can describe a circle as \( \{ (x,y) ~|~ (x,y) \in \Real^2, ~ x^2 + y^2 = r^2 \} \).
The equation describes a circle because the set of all points satisfying the equation forms a circle.

\paragraph{Comparing the approaches}

The parametric equation of a circle simplifies computing the derivative.

The algebraic equation of a circle simplifies computing the distance.

\section*{Calculating the length of a parametric curve}

Let the function \(x : \Real \to \Real^n\) describe a curve.

Define \(X(T) = \{ x(t) ~|~ t \in T \}\) as the \emph{image of \(T\) under \(x\)}.
\footnote{\url{https://en.wikipedia.org/wiki/Image_(mathematics)}}

The segment between \(a\) and \(b\) is \( X([a,b]) = \{ x(k) ~|~ a \le k \le b \} \).
What is the length of this segment?

Divide the curve into many segments.
Approximate each segment as a straight line segment.

The \emph{arc length}.

The length of the segment of \(x\) in \(T\) is
\begin{align*}
    L(x,T) &= \sum_k \norm{x(t_{k+1}) - x(t_k)}
    \\ &= \sum_k \frac{\norm{x(t_k + h_k) - x(t_k)}}{h_k} \cdot h_k
    \\ &= \int_T \abs{x'(t)} \dd{t}
\end{align*}

Wikipedia\footnote{\url{https://en.wikipedia.org/wiki/Arc_length\#Definition_for_a_smooth_curve}}
explains how to derive that equation.
The key is to multiply by \(h_k / h_k\).

\footnote{\url{https://en.wikipedia.org/wiki/Arc_length}}%
\footnote{\url{https://en.wikipedia.org/wiki/Curve\#Length_of_a_curve}}%
\footnote{\url{http://mathworld.wolfram.com/ArcLength.html}}

\section*{Describing a surface}

An example of a space is \( \{ (x,y,z) ~|~ x^2+y^2+z^2 = 1 \} \), the skin of the unit sphere in a 3-dimensional Euclidean space.

A \emph{smooth space} looks smooth (no discontinuities).
It has something to do with differentiability.

\section*{Describing spaces using vectors}

\section*{Describing a line, a plane, and a hyperplane}

A line is a two-dimensional hyperplane.
A plane is a three-dimensional hyperplane.

The \emph{line} that connects point \(A\) and point \(B\) is the set
\( \{ A + k \cdot AB ~|~ k \in \Real \} \).

To define a hyperplane, we need a \emph{normal vector} \(n\) and a point \(C\) on the plane.
The \emph{hyperplane} is then the set of every point \(P\) such that \(CP\) is orthogonal to \(n\).

Try convincing yourself that a line is a two-dimensional hyperplane.
In two-dimensional Euclidean space, fix a point \(C\),
and draw a vector \(n\) whose origin is \(C\).
Pick a \(P\) such that \(CP\) and \(n\) form a right angle.
Pick another such \(P\).
Pick yet another such \(P\).
Pick as many such \(P\) as you need to see that those points form a line.

\section*{Defining lines as cotangent spaces and tangent spaces?}

We have just shown that there are two ways to define a line:
by \emph{cotangent space} and by \emph{tangent space}?

\section*{Describing a circle, a sphere, and a hypersphere}

A circle has a center and a radius.
The circumference of a circle is the set of all points
whose distance from the center is the radius.

A circle with center \(C\) and radius \(r\)
is described by the set \( \{ P : \norm{CP} = r \} \).
Note that this set only describes the circumference.

With coordinates: \( \{ (x,y) : \norm{xi + yj} = r \} \).

With coordinates and orthonormal basis: \( \{ (x,y) : x^2 + y^2 = r^2 \} \).

The two-dimensional circle generalizes to the three-dimensional sphere,
which generalizes to the higher-dimensional hypersphere.

\section*{Describing a shape}

\paragraph{As a set of points}

A shape is a set of points.

\paragraph{With numbers}

Here we get used to using numbers to describe shapes.

Let's say we have a line that connects \((0,0)\) and \((1,1)\).
Some other points on that line are \((2,2)\) and \((3,3)\).
However, we want to describe \emph{all} points on that line.
The way we do it is: ``For every \((x,y)\) in \(\Real^2\), iff \(x=y\), then \((x,y)\) is on the line.''
We can write it in math notation as \( \{ (x,y) ~|~ (x,y) \in \Real^2, ~ x = y\} \).

We generalize.
A line that passes \((0,c/b)\) and \((c/a,0)\)
is described by the set \(\{ (x,y) ~|~ (x,y) \in \Real^2, ~ ax + by = c \}\).
If this is not obvious to you, try replacing \(x\) and \(y\) in the equation with some numbers
without violating the equation.

The set \(\{ (x,y) ~|~ (x,y) \in \Real^2, ~ x^2 + y^2 = r^2 \}\) describes a circle.

The set \(\{ (x,y,z) ~|~ (x,y,z) \in \Real^3, ~ x^2 + y^2 + z^2 = r^2 \}\) describes a sphere.

\index{definitions!dimension of a shape}%
\index{dimension of a shape}%
The number of the tuple component is the \emph{dimension} of the shape.

\paragraph{With functions}

A function \(\Real \to \Real^3\) describes a curve in a three-dimensional space.

\index{definitions!dimension of a shape}%
\index{dimension of a shape}%
The function's parameter count is the \emph{dimension} of the shape.

We can use a function \(\Real^2 \to \Real^3\) such as \( (u,v) \to (0,u,v^2) \).
This describes a \emph{surface} in a three-dimensional space.

We can use a vector equation such as \( m \cdot x + n = 0 \).

We can use an equation such as \( x^2 + y^2 + z^2 = 1 \).

The distance of two points on the surface is the length of the shortest one-dimensional submanifold
of that surface such that this submanifold connects those points.

What is the length of a one-dimensional submanifold?
The curve is described by \( x : \Real \to \Real^n \).
The length of a small segment around \(t\) is \(\norm{x(t+dt) - x(t)}\).

\section{We haven't understood yet}

\subsection{Using charts and atlases}

TODO
Motivate differential geometry:
How do we make a map of the Earth?
How do we project the surface of a sphere to a paper?
Curvilinear coordinates. Space curvature.
How do you describe a (curved) surface?
What is curvature?

Manifold generalizes surface.
A surface is a two-dimensional thing in a three-dimensional space.
There can be \(m\)-dimensional manifold in \(n\)-dimensional space, if \(m \le n\).

We describe the surface of the unit sphere as \( \{ (x,y,z) ~|~ x^2 + y^2 + z^2 = 1, ~ (x,y,z) \in \Real^3 \} \).
We can also in cylindrical coordinates \( \{ (1,a,b) ~|~ 0 \le a,b < 2\pi \} \).
The \emph{surface} of that sphere is a \emph{two}-dimensional manifold.

How do we describe a line on the sphere?
A great circle?

A \emph{chart} is a mapping (a function) between two manifolds?

An \emph{atlas} is a set of charts?

\subsection{Describing a cotangent bundle}

The cotangent bundle of a space \(S\) is the vector bundle of all the cotangent spaces at every point in \(S\).

The cotangent bundle of a space \(S\) is the dual bundle of the tangent bundle of \(S\).

\subsection{Treating a phase space as a cotangent bundle}

This system of two equations describes a system that consists of one free particle: \( x(t) = t \) and \( v(t) = 1 \).
The phase space of that system is \( \{ (t,mv) ~|~ t \in \Real \} \).
A set of \emph{canonical coordinates} describes a point in the phase space.
What physicists call \emph{phase space},
mathematicians call \emph{cotangent bundle of a manifold} (\enquote{Phase space}, Wikipedia).

\subsection{Understanding \enquote{locally}}

\enquote{locally}\footnote{\url{https://en.wikipedia.org/wiki/Local_property}}

\subsection{Understanding parallel transports}

To understand Riemann curvature tensor.
\cite{arnold1989mathematical}

\subsection{Describing smooth deformation}

A map from a sheet to a bent sheet.

\subsection{Describing strain using tensor}

\footnote{\url{https://en.wikipedia.org/wiki/Infinitesimal_strain_theory}}
\footnote{\url{https://en.wikipedia.org/wiki/Continuum_mechanics}}

\section{Generalizing two-dimensional shapes to higher dimensions}

\subsection{Generalizing line to plane and hyperplane}

\subsection{Generalizing circle to sphere and hypersphere}

\section{The word \enquote{manifold}}

You can skip this section.

Riemann defines manifold?
Poincar\'e's \emph{Analysis situs} defines \emph{manifold}%
\footnote{\url{http://www.maths.ed.ac.uk/~aar/papers/poincare2009.pdf}}

Manifolds are confusing
because the word \enquote{manifold} means \enquote{variety},
which doesn't help students guess anything about locally flat spaces.
History can explain this mess.%
\footnote{\url{https://en.wikipedia.org/wiki/Manifold\#History}}%
\footnote{\url{https://en.wikipedia.org/wiki/History_of_manifolds_and_varieties}}

\section*{Replacing \enquote{manifold} with \enquote{space}}

Sometimes we can replace \enquote{manifold} with \enquote{space}.
For example, if we are discussing about spaces that are smooth and locally flat,
then we can write \enquote{cotangent bundle of a space}
instead of \enquote{cotangent bundle of a manifold}.

\section*{Redefining \enquote{manifold} to mean \enquote{locally flat spaces}}

Every time we meet the word \enquote{manifold},
we should by reflex think \enquote{locally flat space}.

A manifold is a space that looks flat if we zoom close enough.
\enquote{Flat} means Euclidean, that is resembling an Euclidean space.

\ExerciseAnswer{Say to yourself until it becomes a reflex: \enquote{A manifold is a locally flat space.}}{A manifold is a locally flat space.}

\ShowAnswers

Some examples of one-dimensional manifolds are lines and circles.%
\footnote{\url{https://en.wikipedia.org/wiki/Manifold}}
Some examples of two-dimensional manifolds are flat sheets and bent seets.

\section{Raw thought}

A line is a set of points.
A sheet is a set of lines.
A cube is a set of sheets.
Is this thought useful?

We will study mappings of the form \(\Real^a \to \Real^b\).
For example, \( \Real \to \Real^n \) is the type of a curve,
and \( \Real^n \to \Real \) is the type of a scalar field.
Is there a deeper connection between curves and scalar fields?
Curves and embeddings?
Scalar fields and projections?

With the real tuple space and the Cartesian coordinate systems,
we can marry infinitesimal calculus and geometry into differential geometry.

With analytic geometry, we can describe shapes using real numbers.

\section{Embedding a space in another space}

An \(n+1\)-dimensional space is bigger than an \(n\)-dimensional space.

An \(n+1\)-dimensional space can contain an \(n\)-dimensional space.

An \(n\)-dimensional space can be embedded in an \(n+1\)-dimensional space.

\footnote{\url{https://en.wikipedia.org/wiki/Embedding}}

\section*{Ambient spaces}

An \(m\)-dimensional object in an \(n\)-dimensional ambient space is a function \(\Real^m \to \Real^n\).
It tells you how to embed the \(m\)-dimensional object into the \(n\)-dimensional ambient space.


\section{\texorpdfstring{Algebra and Number theory}{Algebra and Number theory}}
\vspace{5mm}
\large

\begin{theorem}[Invertibility of element]
	$a \in \Z_n$ is invertible $\iff (a, n) = 1$.
\end{theorem}
\begin{proof}
	For coprime $a, n$ we can use Extended Euclid algorithm and get the inverse using Bezout coefficients.

	Otherwise, no solution exists.
\end{proof}

\begin{definition}
	\[ \Z_N^{\ast} := \{ a \in \Z_N | (a, N) = 1 \} \]
	Group of all invertible elements.
\end{definition}

\begin{theorem}[Lagrange]
	If G is finite group, $H \leq G \Rightarrow |H|\ |\ |G|$.
\end{theorem}

\begin{theorem}[Fermat little]
	For $p$ prime, $(x, p) = 1: x^{p-1} \equiv 1\mod p $.
\end{theorem}

\begin{theorem}[Euler]
	For $N > 1$, $(x, N) = 1: x^{\varphi(N)} \equiv 1 \mod N$.
\end{theorem}
\begin{proof}
	Consider
	\[ 1 = x^0, x^1,..., x^k = 1 \]
	All of them distinct and form a subgroup of $\Z_N^{\ast}$.

	By Lagrange $k = |H|$ is a divisor of order of the group $\varphi(N)$.
	\[ \varphi(N) = k\cdot l\]
	So:
	\[ x^{\varphi(N)} \equiv x^{kl} = (x^k)^l \equiv 1^l = 1\]

\end{proof}

\begin{theorem}[Chinese Remainder Theorem (CRT)]
	For $n_1,..., n_k$ pairwise coprime, $n := \prod_i n_i$. Then
	\[ \Z_{n_1} \times \Z_{n_2} \times ... \times \Z_{n_k} \simeq \Z_n \]
\end{theorem}
\begin{proof}
	Proof for $k = 2$, continue by induction.

	1) Consider
	\[ f:\Z_n \to \Z_{n_1} \times \Z_{n_2}, f(x) = (x \mod n_1, x \mod n_2) \]
	Firstly $f$ is injective:
	\[ f(x) = f(y) \Rightarrow f(x - y) = (0,0) \]
	Also $(x - y)$ is divisible by both $n_1, n_2 \Rightarrow (x - y)$ is divisible by $n_1 \cdot n_2 = n \Rightarrow y = x$.

	Secondly $f$ is bijective as both sets have the same cardinality.

	2) Constructive proof: given $(a,b)$, find $f(x) = (a,b)$.\\
	Firslty, find
	\[ u,v: f(u) = (0, 1) \land f(v) = (1, 0) \Rightarrow f(au + bv) = af(u) + b f(v) = (a,0) + (0,b) = (a,b) \]

	\[ f(n_1) = (0, c_1) \]
	If $c_1 = 1, v = n_1$. Otherwise
	\[ f(c_1^{-1} \cdot n_1) = c_1^{-1} \cdot (0, c_1) = (0, 1) \]
	Similarly obtain $f(n_2)$.
\end{proof}
Primality test (sketch)
\begin{enumerate}
	\item Generate $a \ne 0 \in_R \Z_n$
	\item if $(a, n) \ne 1$ (Euclid witness)\\
		return NO
	\item if $a^{n - 1} \mod n \neq 1$ (Fermat witness)\\
		retutn NO
	\item YES.
\end{enumerate}

\begin{definition}
	Carmichael number: $n$ is composite and
	\[ a \in \Z_n^{\ast}: a^{n - 1} \mod n = 1 \]
	Smallest 561.
\end{definition}

\begin{theorem}[Prime probability]
	If $n$ is composite, but not Carmichael $\Rightarrow Pr_a[YES] \leq 1/2$.
\end{theorem}
\begin{proof}
	\[ H := \{ a \in \Z_n^{\ast} | a^{n - 1} \mod n = 1 \} \]
	Then $H \leq G$, as $H$ is not trivial subgroup. Therefore
	\[ |H| \leq 1/2 \Z_n^{\ast} \]
\end{proof}


\subsection{T-test for variable significance}

\subsubsection{T-statistic}

In practice we don't know the population standard deviation and so must estimate it instead.

We use the standard deviation on the sample.

\(t=\dfrac{\bar x-x_0}{s_0 }\)

\subsubsection{Student's t-distribution}

As we have used the sample standard deviation we have lost a degree of freedom, and can no longer model the variable as a normal distribution, as we did for the z-statistic.

We now have a distribution with an addition parameter, the number of degrees of freedom.

The number of degrees of freedom is \(n-1\).

As the sample size tends towards infinity, the distribution tends towards the normal distribution.

\subsubsection{Student's t-test}

\subsubsection{Confidence interval}