\subsection{Continuity}

\subsubsection{Transitivity}

Axioms of continuity and transitivity

Continuous

Independence of irrelevant alternatives

\subsubsection{Axiom 3: Continuity}

In order to find optimal points for a utility function we want these functions to be differentiable. This requires complete sets of choices. That is, if \(a\) if preferred to \(b\), points very close to \(a\) will also be preferred to \(b\).

Agents often make choices discretely. How much of good \(x\) to consume, whether to go on a holiday. We treat these as continuous. This is generally not problematic as agents choices become less discrete over longer time spans, and most economic areas of interest do not rest on discrete consumption.

\subsubsection{Marginal utility}

We can differentiate our utility function with respect to a good. For example, for:

\(f=2(x-1)^2-10\)

\(\dfrac{\delta f}{\delta x}=4x-1\)

This last term is the marginal utility of \(x\), often shown as \(MU_x\).

\subsubsection{Solving}


\section{Statistics}

\begin{definition}[Bernoulli distribution]\label{bernoulli}
    A distribution of parameters, where $\forall x, 0 \leq x \leq 1$,
    Typically dual, e.g.\ in coin toss there is heads and tails.
    If $\Pr_{heads} = k$, then $\Pr_{tails} = 1 - \Pr_{heads}$
\end{definition}

\begin{definition}[Bigvee]
    $\bigvee\limits_{a \in I} P_{a}$ says that at least one $P_{a}$ is true.
    It can also be used for the maximum value in a set.
\end{definition}

\begin{definition}[Central moment]
    The distribution of variable $[$values$]$ around the mean.
    This way, you can now if the distribution is spread out or not.

\end{definition}

\begin{definition}[Cauchy distribution]
    A family of distributions with no expected value.
    They usually look like:

    \includegraphics[scale=.5]{cau.png}

\end{definition}

\begin{definition}[Chernoff bound]\label{chernoff}
    In a nutshell, it determines a bound on how many times we must perform
    a trial to know that our random variables represent a majority.
    E.g.\ if we trying to determine that a coin is biased (heads/tails),
    a chernoff bound will say how many times we must flip the coin to know
    that we have unraveled a bias. In this case, for example, simply flipping it
    twice will not suffice.
\end{definition}

\begin{definition}[Conditional expectations]
    $$
        E(X | Y=y) = \sum\limits_{x \in X} x \
        P(X=x | Y=y) = \sum_{x \in X} x
        \frac{P(X=x,Y=y)}{P(Y = y)}
    $$
\end{definition}

\begin{definition}[Conditional probability]
    \includegraphics[scale=0.3]{prob_form.png}
\end{definition}

\begin{definition}[Covariance]
    A measure of how much two random variables change together.

\end{definition}

\begin{definition}[Expected value]\label{expectedvalue}
    $E[X] = \sum\limits_{s \in S}^{\dots} X(s) \cdot \Pr(\{s\}) $ \newline
    $E[X] = \sum\limits_{s \in S}^{\dots} X(s) \cdot \Pr(X = x) $
\end{definition}

\begin{definition}[hyperplane]
    A plane (surface) that has one less dimension than it's ambient space,
    i.e.\ the space around it. E.g.\ a hyperplane for 3-d dims is only defined
    2D.

    A hyperplane will therefore act as a separator. Imagine a holding a
    square in the middle of a ball.
\end{definition}

\begin{definition}[Gold standard]
In medicine and statistics, gold standard test refers to a diagnostic test or
benchmark that is the best available under reasonable conditions. It does
not have to be necessarily the best possible test for the condition in absolute
terms. For example, in medicine, dealing with conditions that require an
autopsy to have a perfect diagnosis, the gold standard test is less accurate
than the autopsy.

\end{definition}

\begin{definition}[Ground truth]
    to the accuracy of the training set's classification for supervised
    learning techniques. This is used in statistical models to prove or
    disprove research hypotheses. The term "ground truthing" refers to the
    process of gathering the proper objective data for this test.

\end{definition}

\begin{definition}[Normal (Gaussian) distribution]
    A distribution that is centered around a value.
    E.g.\ when you collect people's height, there will be many around the
    average, and fewer and fewer to the sides.

\end{definition}

\begin{definition}[Indicator variable]
    Indicator variable: 0 or 1 for whether an element is selected or not.
\end{definition}

\begin{definition}[Linearity of expectation]\label{lin_expect}
    $ E[X] + E[Y] = E[X + Y] \newline
    [\sum\limits_{x \in S}^{\dots} X(s) \cdot \Pr(X = x) +
    \sum\limits_{y \in S}^{\dots} X(s) \cdot \Pr(Y = y) ] \newline
    = [\sum\limits_{s \in S}^{\dots} a \cdot \Pr(Y = a) + a \cdot \Pr(X = a) ]
    $
\end{definition}

\begin{theorem}[Likelyhood that both X and Y occur in S]
    $ \newline E[X] \cdot E[Y] = E[X \cdot Y]$
\end{theorem}
\begin{proof}
    From~\nameref{expectedvalue}: \newline
    $
    E[X \cdot Y] = \newline \sum\limits_{z \in S}^{\dots} z \cdot
        \Pr(X = z \text{ and } Y = z) = \newline
    \sum\limits_{x \in S}^{\dots}\sum\limits_{y \in S}^{\dots} x \cdot y
    \cdot \Pr(X = x \text{ and } Y=y) = \newline
    [\sum\limits_{x \in S}^{\dots} x \cdot \Pr(X = x)] \cdot
    [\sum\limits_{y \in S}^{\dots} y \cdot \Pr(Y = y)]  = \newline
    E[X] \cdot E[Y]
    $
\end{proof}

\begin{definition}[Markov Property]
    Memoryless stochastic process - what happened at point of time X
    does not matter for times $Y \geq X$.
\end{definition}

\begin{definition}[Poisson distribution]
    expresses the probability of a given number of events occurring in a fixed
    interval of time and/or space if these events occur with a known average
    rate and independently of the time since the last even.

\end{definition}

\begin{definition}[Probability Mass function]
    A function that gives a probability that a variable is exactly equal to
    some value.
\end{definition}

\begin{definition}[Supervised learning]
    the machine learning task of inferring a function from labeled training
    data. The training data consist of a set of training examples. In
    supervised learning, each example is a pair consisting of an input object
    (typically a vector) and a desired output value (also called the
    supervisory signal).

\end{definition}


\subsection{Determinants}

From invertible matrix section in endo

A matrix can only be inverted if it can be created from a combination of elementary row operations.

How can we identify if a matrix is invertible? We want to create a scalar from the matrix which tells us if this possible. We can this scalar the determinant.

For a matrix \(A\) we label the determinant \(|A|\), or \(\det A\)

We propose \(|A|=0\) when the matrix is not invertible.

So how can we identify the function we need to undertake on the matrix?

\subsubsection{New 1}

We know that linear dependence results in determinants of \(0\).

We can model this as a function on the columns of the matrix.

\(\det M = \det ([M_1, ...,M_n)\)

If there is linear depednence, for example if two columns are the same then:

\(\det ([M_1,...,M_i,...,M_i,...,M_n])=0\)

Similarly, if there is a column of \(0\) then the determinant is \(0\).

\(\det ([M_1,...,0,...,M_n])=0\)
\subsubsection{New 2}

Show linear in addition

How can we identify the determinant of less simple matrices? We can use the multilinear form.

\(\sum c_i\mathbf M_i=\mathbf 0\)

Where \(\mathbf c \ne \mathbf 0\)

Or:

\(M\mathbf c=\mathbf 0\)
\subsubsection{Rule 1: Columns of matrices can be the input to a multilinear form}

A matrix can be shown in terms of its columns.
\(A=[v_1,...,v_n]\)

\(\det A=\det [v_1,...,v_n]\)

\(\det A=\sum_{k_1=1}^m...\sum_{k_n=1}\prod_{i=1}^ma_{ik_i}\det ([e_{k_1},...,e_{k_n}])\)

\subsubsection{Multiplying a matrix by a constant multiplies the determinant by the same amount}

If a whole row or columns is \(0\) then:

\(\det A=\det [v_1,...,v_i,...,v_n]\)

\(\det A'=\det [v_1,...,cv_i,...,v_n]\)

\(\det A=\det [v_1,...,v_i,...,v_n]\)

\(\det A'=\det [v_1,...,cv_i,...,v_n]\)

\(\det A'=c\det [v_1,...,v_i,...,v_n]\)

\(\det A'=c\det A\)

As a result, multiplying a column by \(0\) makes the determinant \(0\).

A matrix with a column of \(0\) therefore has determinant \(0\)

\subsubsection{Rule 2: A matrix with equal columns has a determinant of \(0\).}

\(A=[a_1,...,a_i,...,a_i,...,a_n]\)

\(D(A)=D([a_1,...,a_i,...,a_i,...,a_n])\)

We know from Result 3 that swapping columns reverses the sign. Reversing columns results in the same matrix, so the determinant must be unchanged.

\(D(A)=-D(A)\)

\(D(A)=0\)

\subsubsection{Linear dependence}

If a column is a linear combination of other columns, then the matrix cannot be inverted.

\(A=[a_1,...,\sum_{j\ne i}^{n}c_ja_j,...,a_n]\)

\(\det A=\det ([v_1,...,\sum_{j\ne i}^{n}c_jv_j,...,v_n])\)

\(\det A=\sum_{j\ne i}^{n}c_j\det ([v_1,...,v_j,...,v_n])\)

\(\det A=\sum_{j\ne i}^{n}c_j\det ([v_1,...,v_j,,...,v_j,...,v_n])\)

As there is a repeating vector:

\(\det A=0\)

\subsubsection{Swapping columns multiplies the determinant by \(-1\)}

\(A=[v_1,...,v_i+v_j,...,v_i+v_j,...,v_n]\)

We know.

\(\det A=0\)

\(\det A=\det ([a_1,...,a_i,...,a_i,...,a_n])+\det([a_1,...,a_i,...,a_j,...,a_n])+\det([a_1,...,a_j,...,a_i,...,a_n])+\det([a_1,...,a_j,...,a_j,...,a_n])\)

So:

\(\det ([a_1,...,a_i,...,a_i,...,a_n])+\det ([a_1,...,a_i,...,a_j,...,a_n])+\det([a_1,...,a_j,...,a_i,...,a_n])+\det([a_1,...,a_j,...,a_j,...,a_n])=0\)

As \(2\) of these have equal columns these are equal to \(0\).

\(\det ([a_1,...,a_i,...,a_j,...,a_n])+\det ([a_1,...,a_j,...,a_i,...,a_n])=0\)

\(\det ([a_1,...,a_i,...,a_j,...,a_n])=-\det ([a_1,...,a_j,...,a_i,...,a_n])\)

\subsubsection{Calculating the determinant}

We have

\(\det A=\sum_{k_1=1}^m...\sum_{k_n=1}\prod_{i=1}^ma_{ik_i}\det ([e_{k_1},...,e_{k_n}])\)

So what is the value of the determinant here?

We know that the determinant of the identity matrix is \(1\).

We know that the determinant of a matrix with identical columns is \(0\).

We know that swapping columns multiplies the determinant by \(-1\).

Therefore the determinants where the values of \(k\) are not all unique are \(0\).

The determinants of the others are either \(-1\) or \(1\) depending on how many swaps are required to restore to the identity matrix.

This is also shown as the Leibni formula.

\(\det A = \sum_{\sigma \in S_n}sgn (\sigma )\prod_{i=1}^na_{i,\sigma_i}\)


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 5}

Determine the cost and structure of an optimal binary search tree for a set of $n = 7$ keys with the following probabilities.

\begin{table}[H]\centering
\begin{tabular}{c|c c c c c c c c}
$i$ & 0 & 1 & 2 & 3 & 4 & 5 & 6 & 7\\
\hline
$p_i$ &      & 0.04 & 0.06 & 0.08 & 0.02 & 0.10 & 0.12 & 0.14\\
$q_i$ & 0.06 & 0.06 & 0.06 & 0.06 & 0.05 & 0.05 & 0.05 & 0.05\\
\end{tabular}
\caption{Probabilities for searching for different nodes}\label{tab51}
\end{table}

\subsection*{Solution}

To obtain the expected cost and the form of the optimal binary search tree based on the probabilities given in Table \ref{tab51}, $w(i,j)$ is calculated based on Equation \ref{eq51}.

\begin{equation}\label{eq51}
\begin{aligned}
w(i,j) &= \sum_{l = i}^{j} p_l + \sum_{l = i-1}^{j} q_l\\
&= w(i, j-1) + p_j + q_j
\end{aligned}
\end{equation}

Table \ref{tab52} provides $w(i,j)$ for all $0 \leq j \leq 7$ and $1 \leq i \leq 8$.

\begin{table}[H]\centering
\begin{tabular}{c c c c c c c c c}
  & 0    & 1    & 2    & 3    & 4    & 5    & 6    & 7\\\hline
1 & 0.06 & 0.16 & 0.28 & 0.42 & 0.49 & 0.64 & 0.81 & 1.00\\
2 &      & 0.06 & 0.18 & 0.32 & 0.39 & 0.54 & 0.71 & 0.90\\
3 &      &      & 0.06 & 0.20 & 0.27 & 0.42 & 0.59 & 0.78\\
4 &      &      &      & 0.06 & 0.13 & 0.28 & 0.45 & 0.64\\
5 &      &      &      &      & 0.05 & 0.20 & 0.37 & 0.56\\
6 &      &      &      &      &      & 0.05 & 0.22 & 0.41\\
7 &      &      &      &      &      &      & 0.05 & 0.24\\
8 &      &      &      &      &      &      &      & 0.05\\\hline
\end{tabular}
\caption{$w(i,j)$ for $0 \leq j \leq 7$ and $1 \leq i \leq 8$}\label{tab52}
\end{table}

We now calculate expected search cost $e[i,j]$ for $0 \leq j \leq 7$ and $1 \leq i \leq 8$.
We know that $e[i,j] = q_{i-1}$ when $j = i - 1$.
For the general case however where $i \leq j$, Equation \ref{eq52} can be used.

\begin{equation}\label{eq52}
\begin{aligned}
e[i,j] = \min_{i \leq r \leq j} \{e[i, r-1] + e[r+1, j] + w(i,j)\}
\end{aligned}
\end{equation}

where $r$ is the root in the optimal subtree $n_i$ to $n_j$.
Using this equation and following the \textsc{Optimal-BST} algorithm described in the textbook, $e[i, j]$ and $root[i,j]$ are obtained as shown in Table \ref{tab53} and Table \ref{tab54}, respectively.

\begin{table}[H]\centering
\begin{tabular}{c c c c c c c c c}
  & 0    & 1    & 2    & 3    & 4    & 5    & 6    & 7\\\hline
1 & 0.06 & 0.28 & 0.50 & 0.76 & 0.90 & 1.18 & 1.50 & 1.86\\
2 &      & 0.06 & 0.30 & 0.56 & 0.70 & 0.98 & 1.30 & 1.66\\
3 &      &      & 0.06 & 0.32 & 0.46 & 0.74 & 1.06 & 1.42\\
4 &      &      &      & 0.06 & 0.30 & 0.46 & 0.78 & 1.14\\
5 &      &      &      &      & 0.05 & 0.24 & 0.62 & 0.98\\
6 &      &      &      &      &      & 0.05 & 0.32 & 0.68\\
7 &      &      &      &      &      &      & 0.05 & 0.34\\
8 &      &      &      &      &      &      &      & 0.05\\\hline
\end{tabular}
\caption{$e[i,j]$ for $0 \leq j \leq 7$ and $1 \leq i \leq 8$}\label{tab53}
\end{table}

\begin{table}[H]\centering
\begin{tabular}{c c c c c c c c c}
  & 1    & 2    & 3 & 4 & 5 & 6 & 7\\\hline
1 & 1 & 2 & 3 & 3 & 5 & 6 & 7\\
2 &   & 2 & 3 & 3 & 5 & 6 & 7\\
3 &   &   & 3 & 3 & 5 & 6 & 7\\
4 &   &   &   & 4 & 5 & 6 & 7\\
5 &   &   &   &   & 5 & 6 & 7\\
6 &   &   &   &   &   & 6 & 7\\
7 &   &   &   &   &   &   & 7\\\hline
\end{tabular}
\caption{$root[i,j]$ for $1 \leq i,j \leq 7$}\label{tab54}
\end{table}

And the binary tree can now be constructed based on Table \ref{tab54}, as shown in Figure \ref{fig51}.

\begin{figure}[H]\centering
\begin{tikzpicture}[level distance=1.5cm,
  level 4/.style={sibling distance=3cm},
  level 5/.style={sibling distance=1.5cm},scale=0.8]
\node[circle,draw]{$k_7$}
child{
  node[circle,draw]{$k_6$}
  child{
    node[circle,draw]{$k_5$}
    child {
      node[circle,draw]{$k_3$}
      child {
        node[circle,draw]{$k_2$}
        child {
          node[circle,draw]{$k_1$}
          child {
            node[circle,draw,fill=black!20]{$d_0$}
          }
          child {
            node[circle,draw,fill=black!20]{$d_1$}
          }
        }
        child {
          node[circle,draw,fill=black!20]{$d_2$}
        }
      }
      child {
        node[circle,draw]{$k_4$}
        child {
          node[circle,draw,fill=black!20]{$d_3$}
        }
        child {
          node[circle,draw,fill=black!20]{$d_4$}
        }
      }
    }
    child {
      node[circle,draw,fill=black!20]{$d_5$}
    }
  }
  child{
    node[circle,draw,fill=black!20]{$d_6$}
  }
}
child{
  node[circle,draw,fill=black!20]{$d_7$}
};
\end{tikzpicture}
\caption{Optimal binary search tree based on given probabilities}\label{fig51}
\end{figure}

The total cost of the tree can also be easily computed using Table \ref{tab55} given below.

\begin{table}[H]\centering
\begin{tabular}{c c c c | c c c c}
node & depth & probability & contribution & node & depth & probability & contribution\\\hline
$k_1$ & 5 & 0.04 & 0.24 & $d_0$ & 6 & 0.06 & 0.42\\
$k_2$ & 4 & 0.06 & 0.30 & $d_1$ & 6 & 0.06 & 0.42\\
$k_3$ & 3 & 0.08 & 0.32 & $d_2$ & 5 & 0.06 & 0.36\\
$k_4$ & 4 & 0.02 & 0.10 & $d_3$ & 5 & 0.06 & 0.36\\
$k_5$ & 2 & 0.10 & 0.30 & $d_4$ & 5 & 0.05 & 0.30\\
$k_6$ & 1 & 0.12 & 0.24 & $d_5$ & 3 & 0.05 & 0.20\\
$k_7$ & 0 & 0.14 & 0.14 & $d_6$ & 2 & 0.05 & 0.15\\
      &   &      &      & $d_7$ & 1 & 0.05 & 0.10\\\hline
\end{tabular}
\caption{$root[i,j]$ for $1 \leq i,j \leq 7$}\label{tab55}
\end{table}

By summing over the contribution of each node, the entire cost of this binary search tree is found to be 3.95.


\section{Multiple View Geometry}

\includegraphics[width=\columnwidth]{pictures/epipolarplane}

All points on $\pi$ project on l and l’

\includegraphics[width=\columnwidth]{pictures/epipolarplane2}

epipoles e,e’
\begin{itemize}
	\item intersection of baseline with image plane
	\item projection of projection center in other image
	\item vanishing point of camera motion direction\\
	\end{itemize}

an epipolar plane ($\pi$)
\begin{itemize}
	\item plane containing baseline (1-D family)\\
\end{itemize}

an epipolar line ($l$)
\begin{itemize}
	\item intersection of epipolar plane with image (always come in corresponding pairs)\\
\end{itemize}

epipolar lines from motion

\includegraphics[width=0.8\columnwidth]{pictures/motion}

parallel motion

\includegraphics[width=0.8\columnwidth]{pictures/parallelmotion}

forward motion

\includegraphics[width=0.4\columnwidth]{pictures/forwardmotion}

\subsection{Fundamental matrix}

$$l' = e' \times x' = e' \times Hx = Fx$$

The fundamental matrix F is independent of the point.

\includegraphics[width=\columnwidth]{pictures/epipolarplane3}

correspondence condition:
$$x'^TFx=x'^T l'=0$$

\begin{itemize}
	\item transpose: if F is fundamental matrix for (P,P'), then transp(F) is fundamental matrix for (P',P)
	\item Epipolar lines: l'=Fx and l=transp(F)x'
	\item Epipoles: on all epipolar lines, thus transp(e')Fx=0 for all x therefore transp(e')F = 0, similarly Fe=0
	\item F has 7 DOF i.e. 3x3-1(homogeneous)-1(rank2)
	\item F is a correlation, projective mapping from a point x to a line l'=Fx (not a proper correlation, i.e. not invertible)
\end{itemize}

for pure translation F only has 2 degrees of freedom

\subsubsection{Eight-Point Algorithm}
Since the fundamental matrix F is a 3x3 matrix determined up to an arbitrary scale factor, 8 linear equations are required to obtain a unique solution. It is possible with 7.

\subsection{Essential Matrix}
for the calibrated case

5 linear equations are needed.

Same as Fundamental matrix
\begin{itemize}
	\item Ep' is the epipolar line associated with p'
	\item transp(E)p is the epipolar line associated with p
	\item Ee'=0 and transp(E)e=0
	\item E is singular
\end{itemize}

New

\begin{itemize}
	\item E has two equal non-zero singular values
\end{itemize}

\subsection{Multi-view geometry}

\subsection{Dealing with Wide FOV Camera}
\begin{itemize}
	\item Two-step linear approach to compute radial distortion
	\item estimates distortion polynomial of arbitrary degree
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%
%
%   Thesis template by Youssif Al-Nashif
%
%   May 2020
%
%%%%%%%%%%%%%%%%%%%%%%%%

\section{Clustering with Graph Kernels}

\hspace*{0.3cm} The resulting graph kernel matrix, with dimensions $n \times n$, is high dimensional data. We can take this kernel and perform a principal component analysis (PCA) to reduce the high dimensionality of this data. Each row, can be considered as an observation, and each column as its similarity value or in the $j$-th graph's dimension. PCA then allows us to express the data in a lower dimensionality, hopefully in 2 or 3 dimensions which can be visualized much better. \\

Then, this transformed kernel data is then used in place of a distance matrix for hierarchical clustering. Since the kernel is a similarity kernel, i.e. larger values mean values are closer or more similar, and not a distance kernel, we need to adjust the kernel. The kernel's elements are passed through the function $f(x) = \frac{1}{x}$ to get the reciprocal value for each similarity value\textemdash thus converting the value to a value representative of distance. The hierarchical clustering is performed with the manhattan distance: \\

\begin{equation}
d(\vec{p},\vec{q}) = \sum_{i=1}^n |p_i - q_i|
\end{equation}

Since the edge label histograms have entries of either 1 or 0, the manhattan distance was a better match for the space. The hierarchical clustering also uses a linkage method of Ward's method, for minimizing variance. Ward's method is defined as:\\

\begin{equation}
d(\vec{p},\vec{q}) = || \vec{p} - \vec{q} ||^2
\end{equation}

The resulting dendrograms are then able to be visualized, analyzed, and cut to form clusters of the documents. In the dendrograms, each document is a ``leaf", or end of the tree, and each dendrogram represents the corpus as a whole. Clusters are then formed when the tree is ``cut" at some height; the clusters are then what are of interest to the researcher, as they are the result of this unsupervised learning method.


\section{Stochastic Gradient Descent (SGD)}
\frame{\tableofcontents[currentsection, hideothersubsections]}

\begin{frame}
\frametitle{SGD: Intro}

WHAT:\\
Stochastic Gradient Descent (SGD);
\vspace{5mm}

WHY:\\
do not know $D$, so do not know the gradient of $L_D(w)$.
\vspace{5mm}

HOW:\\
take a step along a \textbf{random} direction (vector), as long as \\
its \textbf{expected value} at each iteration will \textbf{equal the gradient direction} \\
(more generally, a subgradient of the function at the current vector)
\end{frame}

\begin{frame}
\frametitle{SGD: Intro}

\begin{figure}
    \centering
    \includegraphics[scale=0.3]{fig_14_3}
\end{figure}

\end{frame}

\begin{frame}
\frametitle{SGD: Intro}

\begin{figure}
    \centering
    \includegraphics[scale=0.325]{sgd}
\end{figure}

\end{frame}

\begin{frame}
\frametitle{SGD: Analysis for Convex-Lipschitz-Bounded Fn}

\begin{figure}
    \centering
    \includegraphics[scale=0.2]{theorem_14_8}
\end{figure}

\end{frame}


\chapter{}

\rmk{1} In general, for any sampling distribution, there is a natural family of prior distributions, called the conjugate family. For a class of distributions \(\mathcal{P}=\{f(\mathbf{x} \mid \theta)\}_{\theta \in \Theta}\), a class \(\Pi\) of priors is called a conjugate family for \(\mathcal{P}\) if the posterior is in the class \(\Pi\) for any \(\theta \in \Theta\), all priors in \(\Pi\) and all \(x \in \mathcal{X}\).

\begin{ex}
    An experiment consists in flipping a coin independently \(n\) times, i.e., the total number of successes \(X\) follows a \(B(n, \theta)\) distribution. Suppose \(\theta \sim U[0,1]\) is our prior distribution (which is quite uninformative/conservative). Show that
    \[
        \theta \mid x \sim \operatorname{Beta}(x+1, n-x+1).
    \]
\end{ex}

\begin{solution}
    From Bayes' theorem,
    \begin{align*}
        p(\theta|x) &= \frac{p(x|\theta)p(\theta)}{\int p(x|\theta) \der x} \\
        &\propto p(x|\theta)p(\theta) \\
        &\propto \binom{n}{x} \theta^x(1-\theta)^{n-x}\\
        &\propto \theta^x(1-\theta)^{n-x}.
    \end{align*}
    So, $\theta|x\sim Beta(x+1, n-x+1)$.
\end{solution}

\begin{ex}
    Suppose we collect a sample \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \operatorname{Poi}(\lambda), \lambda>0\). Develop the posterior distribution \(\lambda \mid \mathbf{x}\) when the prior is given by \(\lambda \sim \Gamma(\alpha, \nu)\), i.e.,
    \[
        \pi(\lambda)=\frac{\nu^{\alpha}}{\Gamma(\alpha)} \lambda^{\alpha-1} e^{-\nu \lambda}.
    \]
\end{ex}

\begin{solution}
    Because $x_i\sim Poi(\lambda)$,
    \[
        f(\mathbf{x}|\lambda) = \frac{e^{-n\lambda}\lambda^{\sum x_i}}{\prod x_i!}.
    \]
    \begin{align*}
        p(\theta|x) &= \frac{p(x|\theta)p(\theta)}{\int p(x|\theta) \der x} \\
        &\propto p(x|\theta)p(\theta) \\
        &\propto \frac{e^{-n\lambda}\lambda^{\sum x_i}}{\prod x_i!} \frac{\lambda^{\alpha-1}v^\alpha e^{-v\lambda}}{\Gamma(\alpha)} \\
        &\propto \lambda^{\sum x_i+\alpha-1} e^{-\lambda (n+v)}.
    \end{align*}
    So, $\lambda|\mathbf{x}\sim \Gamma(\sum x_i +\alpha, n+v)$.
\end{solution}

\begin{ex}
    We saw in class the following example. Suppose \(\Theta=\mathcal{A}=\mathbb{R}, L(\theta, a)=(\theta-a)^{2}\) and \(X \mid \theta=t \sim \mathcal{N}(t, 1)(n=1\) observation \() .\) Then, it is clear that the decision rule \(d_{\mathrm{ML}}(X)=X\) is the ML estimator of \(\theta\). In this problem, we show that \(d_{\mathrm{ML}}(X)\) is not a Bayes rule, irrespective of the prior. By contradiction, suppose it is.
    \begin{enumerate}[(a)]
        \item Show that
        \[
            r\left(\pi, d_{\mathrm{ML}}(X)\right)<\infty
        \]
        \item Conclude that \(d_{\mathrm{ML}}(X)=\mathbb{E}(\theta \mid X)\).
        \item On one hand, show that
        \[
        \mathbb{E}\left[\theta d_{\mathrm{ML}}(X)\right]=\mathbb{E} d_{\mathrm{ML}}^{2}(X)
        \]
        (hint: condition on \(X\) ).
        \item  On the other hand, show that
        \[
        \mathbb{E}\left[\theta d_{\mathrm{ML}}(X)\right]=\mathbb{E} \theta^{2}
        \]
        (hint: condition on \(\theta\) ).
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item From the definition, for some prior distribution $\pi$, $r(\pi, d_{\mathbf{ML}})$ is bayes risk. So,
        \[
            r(\pi, d_{\mathrm{ML}}(X)) = \inf_{d\in D^\star} r(\pi, d)<\infty.
        \]
        \item
    \end{enumerate}
\end{solution}

\begin{ex}
    Let \(\mathbf{X} \sim f_{X}(\mathbf{x} \mid \theta), \pi(d \theta)=\pi(\theta) d \theta\) be a.c. distributions, and let \(T\) be a sufficient statistic for \(\theta\) with a.c. density function \(g(t \mid \theta) d t\) and a.c. marginal density \(f_{T}(t)\). Suppose the factorization theorem holds. Show that
\[
\pi(\theta \mid \mathbf{x})=\pi(\theta \mid t)=\frac{\pi(\theta) g(t \mid \theta)}{f_{T}(t)}
\]
(n.b.: the reason for determining \(\pi(\theta \mid \mathbf{x})\) from a sufficient statistic is that \(g(t \mid \theta)\) and \(f_{T}(t)\) are usually easier to handle than \(f_{X}(\mathbf{x} \mid \theta)\) and the marginal \(f_{\mathbf{X}}(\mathbf{x})\) - see next problem).
\end{ex}

5. Assume \(\mathbf{X}=\left(X_{1}, \ldots, X_{n}\right) \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\theta, \sigma^{2}\right)\), where \(\theta\) is unknown but \(\sigma^{2}\) is known. Let the prior distribution \(\pi(\theta)\) be a \(\mathcal{N}\left(\mu, \tau^{2}\right)\) density, where \(\mu\) and \(\tau^{2}\) are known. Show that
\[
\theta \mid \mathbf{x} \sim \mathcal{N}\left(\mu(\mathbf{x}), \rho^{-1}\right)
\]
where
\[
\rho=\frac{n \tau^{2}+\sigma^{2}}{\tau^{2} \sigma^{2}}, \quad \mu(\mathbf{x})=\frac{\sigma^{2} / n}{\tau^{2}+\sigma^{2} / n} \mu+\frac{\tau^{2}}{\tau^{2}+\sigma^{2} / n} \bar{x}
\]
(hint: use the previous problem and show that
\[
\frac{(\theta-\mu)^{2}}{\tau^{2}}+\frac{(t-\theta)^{2}}{\sigma^{2} / n}=\rho\left[\theta-\frac{1}{\rho}\left(\frac{\mu}{\tau^{2}}+\frac{t}{\sigma^{2} / n}\right)\right]^{2}+\frac{(\mu-t)^{2}}{\left(\sigma^{2} / n+\tau^{2}\right)}
\]
Note that \(f_{T}(t)\) plays no role in determining the nature of \(\left.\pi(\theta \mid \mathbf{x})\right)\).
6. If \(S^{2}\) is the sample variance based on sample of size \(n\) from a Normal population, we know that \(\frac{(n-1) S^{2}}{\sigma^{2}} \sim \chi_{n-1}^{2}\). The conjugate prior for \(\sigma^{2}\) is the inverse Gamma distribution \(I G(\alpha, \lambda)\), which is given by
\[
\pi\left(\sigma^{2} \mid \alpha, \lambda\right)=\frac{\lambda^{\alpha}}{\Gamma(\alpha)}\left(\sigma^{2}\right)^{-\alpha-1} e^{-\lambda / \sigma^{2}} 1_{\left\{0<\sigma^{2}<\infty\right\}}, \quad \alpha, \lambda>0
\]
(a) Show that the posterior distribution of \(\sigma^{2}\) is \(I G\left(\alpha+\frac{n-1}{2}, \frac{(n-1) S^{2}}{2}+\lambda\right)\).
(b) Find the Bayes estimator of \(\sigma^{2}\) assuming quadratic error loss.
7. Please answer the following questions.
(a) Show that \(\operatorname{Beta}(\alpha, \beta)\) is a conjugate family for \(B(n, p)\). In other words, if \(X \sim B(n, p)\) and we assume a prior distribution \(p \sim \operatorname{Beta}(\alpha, \beta)\), then \(p \mid X\) also follows a Beta distribution.
(b) In the situation described in (a), compute the Bayes rule (estimator of \(p\) ) assuming a quadratic loss function.


\subsection{Structural supply and demand functions}

Supply:

\(Q_s=\alpha_1 + \beta_1P+\gamma_1I + \epsilon_1\)

Demand:

\(Q_d=\alpha_2 + \beta_2P+\gamma_2I + \epsilon_2\)

Can't estimate because the equations are simulataneous.


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{September 19, 2014}
\maketitle
\section*{assignment}
number of cycles of length  $n$ in $S_n$ is $(n-1)!$ because you fix the first entry to eliminate duplicates.

number of cycles of length $m$ of $S_n$. pick $\binom{n}{m}$ for the first element. $\binom{n}{m}m!/m=\frac{n!}{(n-m)!}\frac{1}{m}$

\#12

(ab) is odd, length three cycles are even, two evens multiplied together is even.

\section*{3.1 groups}
$S$ is a set.
\subsubsection*{definiton}a binary operation on $S$ is a function $S\times S\to S$, or $(x,y)\to x*y$

interesting binary operations satisfy: associativity, identity($\exists e\in S$ such that $x*e=e*x=x$), inverse ($a*b=b*a=e$). if an element has an inverse, we say that it is invertible.

\section*{example}
$\mathbb{Z}$ with binary operation is usual addition, $(\mathbb{Z},+)$, then it is associative, $0$ is identity, and all elements are invertible.

$(2\mathbb{Z},+)$, nothing is different

$(2\mathbb{Z},\cdot)$. No identity element.o

$(2\mathbb{Z}+1,+)$, this operation is not closed, it's not a binary operation.

\section*{propostion}
let $*$ be an associative opertion on $S$, let $a,b,c\in S$ be invertible elements. then
\begin{enumerate}
\item
the $*$ operation has at most one identity element
\item
if it has an identity elemen, then an element a in S has at most one inverse.
\end{enumerate}
\subsubsection*{proof}
assume that $e,e'$ identity elements, $x\star e=e\star x=x$ and $x\star e'=e'\star x=x$for all $x\in S$.

take $x=e'$ then $e'=e'\star e=e$

now if $a$ has two inverses $b,b'$ then $b=b\star e=b\star (a\star b')=(b\star a)\star b'=e\star b'=b'$

\section*{propostion}
let $*$ be an associative opertion on $S$, let $a,b,c\in S$ be invertible elements. then
\begin{enumerate}
\item
$a^{-1}$ is invertible

$a\star a^{-1}=a^{-1}\star a=e$
\item
$a\star b$ is invertible and $(a\star b)^{-1}=a^{-1}\star b^{-1}$.

$(a*b)*(b^{-1}*a^{-1})=a*e*a^{-1}=e$ and similarly $(b^{-1}*a^{-1})*(a*b)=e$
\end{enumerate}

\section*{definition of group}
let $G$ be a set and $\star$ be a binary operation on $G$. we say that $(G,\star)$ is a group if
\begin{enumerate}
\item
$\star$ is associative
\item
$\star$ as an identity element
\item
every element of $G$ is invertible.
\end{enumerate}
\subsubsection*{examples}
$(\mathbb{Z},+)$ is a group, $(\mathbb{Z},\cdot)$ is not, $(\mathbb{Q},\cdot)$ is not because zero is not invertible, $(\mathbb{Q}*,\cdot)$ is because the $*$ means throw out zero. $(S_n,\circ)$ where $\circ$ is a permutation, is a group, $(\mathbb{Z}_n,+)$ is a group. $\mathbb{Z}_n^*$ is all the elements of $\mathbb{Z}_n^*$ is all elements of $\mathbb{Z}_n$ that are invertible

\section*{proposition}
$(G,*)$ is a group, and $a,b,c\in G$ thenn if $ab=ac$ then $b=c$ and if $ba=ca$ then $b=c$

$a^{-1}ab=a^{-1}ac=b=c$

\section*{abelian groups (commutative groups)}
a group $(G,*)$ is called abelian if $*$ is commutative.

for example $(S_n,\circ)$ is not abelian.

another example is $GL_n(\mathbb{R})$ the invertible $n\times n$ matrices with entries in $\mathbb{R}$. $(GL_n(\mathbb{R}),\cdot)$ is not commutative.

\end{document}


\section{Dynamical System Scaling}
\label{sec:dssdoc}

The DSS approach to system scaling is based on transforming the typical view of processes to a special coordinate system in terms of the parameter of interest and its agents of change \cite{DSS2015}.
By parameterizing using a time term that will be introduced later in this section, data reproduced can be converted to the special three coordinate system (also called the phase space)
and form a geometry with curves along the surface containing invariant and intrinsic properties. The remainder of this section is a review of DSS theory introduced in publications
by Reyes \cite{DSS2015,Reyes2015,Martin2019} and is used in this analysis for FR scaling. The parameter of interest is defined to be a conserved quantity within a control volume:
\begin{equation}
  \label{eq_1}
  \beta(t)=\frac{1}{\Psi_{0}}\iiint_{V}{\psi\left(\vec{x},t\right)}dV
\end{equation}
$\beta$ is defined as the volume integral of the time and space dependent conserved quantity $\psi$ normalized by a time-independent value, $\Psi_{0}$, that characterizes the process. The agents of change are defined as the first derivative of the normalized parameter of interest:
\begin{equation}
  \label{eq_2}
  \omega=\frac{1}{\Psi_{0}}\frac{d}{dt}\iiint_{V}{\psi\left(\vec{x},t\right)}dV=\iiint_{V}{\left(\phi_{v}+\phi_{f}\right)}dV+\iint_{A}{\left(\vec{j}\cdot\vec{n}\right)}dA-\iint_{A}{\psi\left(\vec{v}-\vec{v}_{s}\cdot\vec{n}dA\right)}dA
\end{equation}
The change is categorized into three components; volumetric, surface, and quantity transport. The agents of change is also the sum of the individual agent of change:
\begin{equation}
  \omega=\frac{1}{\Psi_{0}}\frac{d}{dt}\iiint_{V}{\psi\left(\vec{x},t\right)}dV=\sum^{n}_{i=1}{\omega_{i}}
\end{equation}
The relation of $\omega$ and $\beta$ is the following:
\begin{equation}
  \label{eq_3}
  \omega(t)=\left.\frac{d\beta}{dt}\right|_{t}=\sum^{n}_{i=1}{\omega_{i}}
\end{equation}
Where $\omega$ is the first derivative of reference time. As defined in Einstein and Infeld, time is a value stepping in constant increments \cite{Einstein1966}. The process dependent term in DSS is called process time:
\begin{equation}
  \label{eq_4}
  \tau(t)=\frac{\beta(t)}{\omega(t)}
\end{equation}
To measure the progression difference between reference time and process time in respect to reference time, the idea of temporal displacement rate (D) is adopted:
\begin{equation}
  \label{eq_5}
  D=\frac{d\tau-dt}{dt}=-\frac{\beta}{\omega^{2}}\frac{d\omega}{dt}
\end{equation}
The interval of process time is:
\begin{equation}
  \label{eq_8}
  d\tau=\tau_{s}=\left(1+D\right)dt
\end{equation}
Applying the process action to normalize the phase space coordinates gives the following normalized terms:
\begin{equation}
  \label{eq_10}
  \tilde{\Omega}=\omega\tau_{s},\qquad \tilde{\beta}=\beta,\qquad \tilde{t}=\frac{t}{\tau_{s}},\qquad \tilde{\tau}=\frac{\tau}{\tau_{s}},\qquad
  \tilde{D}=D
\end{equation}
The scaling relation between the prototype and model can be defined both for $\beta$ and $\omega$ and represents the scaling of the parameter of interest and the corresponding agents of change (or frequency given from the units of per time):
\begin{equation}
  \label{eq_11}
  \lambda_{A}=\frac{\beta_{M}}{\beta_{P}},\qquad \lambda_{B}=\frac{\omega_{M}}{\omega_{P}}
\end{equation}
The subscripts $M$ and $P$ stand for the model and prototype. Applying these scaling ratios to equations (\ref{eq_4}), (\ref{eq_5}), and (\ref{eq_10}) provides the scaling ratios for other parameters as well:
\begin{equation}
  \label{eq_12}
  \frac{t_{M}}{t_{P}}=\frac{\lambda_{A}}{\lambda_{B}},\qquad \frac{\tau_{M}}{\tau_{P}}=\frac{\lambda_{A}}{\lambda_{B}},\qquad \frac{\tilde{\beta}_{M}}{\tilde{\beta}_{P}}=\lambda_{A},\qquad \frac{\tilde{\Omega}_{M}}{\tilde{\Omega}_{P}}=\lambda_{A},\qquad \frac{\tilde{\tau}_{M}}{\tilde{\tau}_{P}}=1,\qquad \frac{D_{M}}{D_{P}}=1
\end{equation}
Normalized agents of change is the sum in the same respect:
\begin{equation}
  \label{eq_18}
  \Omega=\sum^{k}_{i=1}{\Omega_{i}}
\end{equation}
The ratio of $\Omega$ is expressed in the following alternate form:
\begin{equation}
  \label{eq_19}
  \Omega_{R}=\frac{\Omega_{M}}{\Omega_{P}}=\frac{\sum^{k}_{i=1}{\Omega_{M,i}}}{\sum^{k}_{i=1}{\Omega_{P,i}}}=\frac{\Omega_{M,1}+\Omega_{M,2}+...+\Omega_{M,k}}{\Omega_{P,1}+\Omega_{P,2}+...+\Omega_{P,k}}
\end{equation}
By the law of scaling ratios, The following must be true:
\begin{equation}
  \label{eq_13}
  \lambda_{A}=\frac{\Omega_{M,1}}{\Omega_{P,1}},\lambda_{A}=\frac{\Omega_{M,2}}{\Omega_{P,2}},...,\lambda_{A}=\frac{\Omega_{M,k}}{\Omega_{P,k}}
\end{equation}
Depending on the scaling ratio values, From Reyes, the scaling methods and similarity criteria is subdivided into five categories; 2-2 affine, dilation, $\beta$-strain, $\omega$-strain, and identity \cite{DSS2015}.
Table \ref{DSS:table_1} summarizes the similarity criteria. Despite the five categories, in essence, all are 2-2 affine with exceptions of partial scaling ratios values being 1.
\begin{table}[H]
\centering
\begin{tabular}{c|c|c|c|c}
\hline
%\rowcolor{lightgray}
\multicolumn{5}{c}{Basis for Process Space-time Coordinate Scaling}\\
\hline
Metric & \multirow{2}{*}{$d\tilde{\tau}_{P}=d\tilde{\tau}_{P}$} & \multirow{2}{*}{And} & Covariance & \multirow{2}{*}{$\frac{1}{\omega_{P}}\frac{d\beta_{P}}{dt_{P}}=\frac{1}{\omega_{M}}\frac{d\beta_{M}}{dt_{M}}$} \\
Invariance   & & & Principle & \\
\hline
\multicolumn{5}{c}{$\beta-\omega$ Coordinate Transformations}\\
\hline
2-2 Affine  & Dilation  & $\beta$-Strain & $\omega$-Strain & Identity \\
$\beta_{R}=\lambda_{A}$ & $\beta_{R}=\lambda$ & $\beta_{R}=\lambda_{A}$ & $\beta_{R}=1=\lambda_{B}$ & $\beta_{R}=1$ \\
$\omega_{R}=\lambda_{B}$ & $\omega_{R}=\lambda$ & $\omega_{R}=1$ & $\omega_{R}=\lambda_{B}$ & $\omega_{R}=1$ \\
\hline
\multicolumn{5}{c}{Similarity Criteria}\\
\hline
$\tilde{\Omega}_{R}=\lambda_{A}$ & $\tilde{\Omega}_{R}=\lambda$ & $\tilde{\Omega}_{R}=\lambda_{A}$ & $\tilde{\Omega}_{R}=1$ & $\tilde{\Omega}_{R}=1$ \\
$\tau_{R}=t_{R}=\frac{\lambda_{A}}{\lambda_{B}}$ & $\tau_{R}=t_{R}=1$ & $\tau_{R}=t_{R}=\lambda_{A}$ & $\tau_{R}=t_{R}=\frac{1}{\lambda_{B}}$ & $\tau_{R}=t_{R}=1$ \\
\hline
\end{tabular}
\caption{Scaling Methods and Similarity Criteria Resulting from Two-Parameter Transformations \cite{DSS2015}}\label{DSS:table_1}
\end{table}
The separation between both process curves along the constant normalized process time is the local distortion \cite{Martin2019}:
\begin{equation}
  \label{eq_15}
  \tilde{\eta}_{k}=\beta_{P_{k}}\sqrt{\varepsilon D_{P_{k}}}\left[\frac{1}{\Omega_{P_{k}}}-\frac{\lambda_{A}}{\Omega_{M_{k}}}\right]
\end{equation}
Where $\epsilon$ is a sign adjuster ensuring positive values within the square root. The total distortion is:
\begin{equation}
  \label{eq_16}
  \tilde{\eta}_{T}=\sum^{N}_{k=1}{\left|\tilde{\eta}_{k}\right|}
\end{equation}
And, the equivalent standard deviation is:
\begin{equation}
  \label{eq_17}
  \sigma_{est}=\sqrt{\frac{1}{N}\sum^{N}_{k=1}{\tilde{\eta}^{2}_{k}}}
\end{equation}


\subsection{Trigonometric functions}\label{subsec:trigonometric_functions}

\begin{definition}\label{def:trigonometric_functions}
  We define the two basic \term{trigonometric functions}. They are also called \term{circular trigonometric functions} to distinguish them from the hyperbolic trigonometric functions defined and motivated in \fullref{def:hyperbolic_trigonometric_functions}.

  \begin{thmenum}
    \thmitem{def:trigonometric_functions/sine} The \term{sine} function, also called the \term{sinus} function, is
    \begin{equation*}
      \sin(z)
      \coloneqq
      -i \sum_{m \text{ is odd}}^\infty \frac {i^m z^m} {m!}
      =
      -i \sum_{k=0}^\infty \frac {i^{2k+1} z^{2k+1}} {(2k + 1)!}
      =
      \sum_{k=0}^\infty \frac {i^{2k} z^{2k+1}} {(2k + 1)!}
    \end{equation*}

    \thmitem{def:trigonometric_functions/cosine} The \term{cosine} function, also called the \term{cosinus} function, is
    \begin{equation*}
      \cos(z)
      \coloneqq
      \sum_{m \text{ is even}}^\infty \frac {i^m z^m} {m!}
      =
      \sum_{k=0}^\infty \frac {i^{2k} z^{2k}} {(2k)!}.
    \end{equation*}
  \end{thmenum}

  \Fullref{def:geometric_trigonometric_functions} justifies the term \enquote{angle} for the \hyperref[def:multi_valued_function/arguments]{parameter} of the trigonometric functions.
\end{definition}
\begin{proposition}\label{thm:def:trigonometric_function}
  The \hyperref[def:trigonometric_functions]{main trigonometric functions} have the following basic properties:
  \begin{thmenum}
    \thmitem{thm:def:trigonometric_function/convergence} Both \( \sin(z) \) and \( \cos(z) \) converge in the entire complex plane.
    \thmitem{thm:def:trigonometric_function/parity} \( \sin(z) \) is an odd function and \( \cos(z) \) is an even function.
    \thmitem{thm:def:trigonometric_function/derivative} \( \sin'(z) = \cos(z) \) and \( \cos'(z) = -\sin(z) \) for all \( z \in \BbbC \).
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:def:trigonometric_function/convergence} Note that the zero coefficients in the expansion of either \( \sin \) or \( \cos \) do not alter convergence. Therefore, by \fullref{thm:power_series_radius_of_convergence}, the radius of convergence is
  \begin{equation*}
    \limsup_{k \to \infty} \frac {\abs{i^{k-1} k!}} {\abs{i^k (k-1)!}}
    =
    \limsup_{k \to \infty} k
    =
    +\infty.
  \end{equation*}

  \SubProofOf{thm:def:trigonometric_function/parity} Follows from \fullref{thm:power_series_parity}.

  \SubProofOf{thm:def:trigonometric_function/derivative} Follows from \fullref{thm:power_series_are_locally_uniform_convergent} and \fullref{thm:derivative_limit_exchange}.
\end{proof}

\begin{proposition}\label{thm:trigonometric_identities}
  We have the following basic trigonometric identities:
  \begin{thmenum}
    \thmitem{thm:trigonometric_identities/pythagorean_identity} (Pythagorean identity) For any \( z \in \BbbC \),
    \begin{equation}\label{eq:thm:trigonometric_identities/pythagorean_identity}
      \sin(z)^2 + \cos(z)^2 = 1.
    \end{equation}

    \thmitem{thm:trigonometric_identities/products} (Products) For \( x, y \in \BbbC \),
    \begin{balign}
      2 \sin(x) \sin(y) & = \cos(x - y) - \cos(x + y) \label{eq:thm:trigonometric_identities/products/ss}  \\
      2 \cos(x) \cos(y) & = \cos(x - y) + \cos(x + y) \label{eq:thm:trigonometric_identities/products/cc}  \\
      2 \sin(x) \cos(y) & = \sin(x - y) + \sin(x + y) \label{eq:thm:trigonometric_identities/products/sc}  \\
      2 \cos(x) \sin(y) & = -\sin(x - y) + \sin(x + y) \label{eq:thm:trigonometric_identities/products/cs}
    \end{balign}

    \thmitem{thm:trigonometric_identities/sums} (Sums) For \( x, y \in \BbbC \),
    \begin{balign}
      \sin(x) + \sin(y) & = 2 \cos\left(\frac{x - y} 2 \right) \sin\left(\frac{x + y} 2 \right) \label{eq:thm:trigonometric_identities/sums/sin_sum}   \\
      \sin(x) - \sin(y) & = 2 \sin\left(\frac{x - y} 2 \right) \cos\left(\frac{x + y} 2 \right) \label{eq:thm:trigonometric_identities/sums/sin_diff}  \\
      \cos(x) + \cos(y) & = 2 \cos\left(\frac{x - y} 2 \right) \cos\left(\frac{x + y} 2 \right) \label{eq:thm:trigonometric_identities/sums/cos_sum}   \\
      \cos(x) - \cos(y) & = -2 \sin\left(\frac{x - y} 2 \right) \sin\left(\frac{x + y} 2 \right) \label{eq:thm:trigonometric_identities/sums/cos_diff}
    \end{balign}

    \thmitem{thm:trigonometric_identities/sum_of_angles} (Sum of angles) For \( x, y \in \BbbC \),
    \begin{balign}
      \sin(x + y) & = \cos(x) \sin(y) + \cos(x) \sin(y) \label{eq:thm:trigonometric_identities/sum_of_angles/sin} \\
      \cos(x + y) & = \cos(x) \cos(y) - \sin(x) \sin(y) \label{eq:thm:trigonometric_identities/sum_of_angles/cos}
    \end{balign}
  \end{thmenum}
\end{proposition}
\begin{proof}
  We first use \hyperref[def:algebra_of_polynomials/polynomial_multiplication]{Cauchy multiplication} for the power series \( \cos(v) \) and \( \cos(w) \):
  \begin{balign}
    \cos(v) \cos(w)
    &=
    \left( \sum_{k=0}^\infty \frac {i^{2k} v^{2k}} {(2k)!} \right) \Ast \left( \sum_{k=0}^\infty \frac {i^{2k} w^{2k}} {(2k)!} \right)
    = \nonumber \\ &=
    \sum_{k=0}^\infty \sum_{m=0}^k \frac {i^{2m} v^{2m}} {(2m)!} \frac {i^{2(k-m)} w^{2(k-m)}} {(2(k-m))!}
    = \nonumber \\ &=
    \sum_{k=0}^\infty \frac {i^{2k}} {(2k)!} \sum_{m=0}^k \binom {2k} {2m} v^{2m} w^{2(k-m)}. \label{eq:thm:trigonometric_identities/cos_product}
  \end{balign}

  Analogously,
  \begin{balign}
    \sin(v) \sin(w)
    &=
    (-i) (-i) \left( \sum_{k=0}^\infty \frac {i^{2k+1} v^{2k+1}} {(2k+1)!} \right) \Ast \left( \sum_{k=0}^\infty \frac {i^{2k+1} w^{2k+1}} {(2k+1)!} \right)
    = \nonumber \\ &=
    -\sum_{k=0}^\infty \sum_{m=0}^k \frac {i^{2m+1} v^{2m+1}} {(2m+1)!} \frac {i^{2(k-m)+1} w^{2(k-m)+1}} {(2(k-m)+1)!}
    = \nonumber \\ &=
    -\sum_{k=0}^\infty \frac {i^{2(k+1)}} {(2(k+1))!} \sum_{m=0}^k \binom {2(k+1)} {2m+1} v^{2m+1} w^{2(k-m)+1}
    = \nonumber \\ &=
    -\sum_{k=1}^\infty \frac {i^{2k}} {(2k)!} \sum_{m=0}^{k-1} \binom {2k} {2m+1} v^{2m+1} w^{2k-(2m+1)}. \label{eq:thm:trigonometric_identities/sin_product}
  \end{balign}

  \SubProofOf{thm:trigonometric_identities/pythagorean_identity} From \eqref{eq:thm:trigonometric_identities/cos_product} and \eqref{eq:thm:trigonometric_identities/sin_product} we have
  \begin{equation*}
    \sin(z)^2 + \cos(z)^2
    =
    1 + \sum_{k=1}^\infty \frac {i^{2k} z^{2k}} {(2k)!} \underbrace{\left[-\sum_{m=0}^{k-1} \binom {2k} {2m+1} + \sum_{m=0}^k \binom {2k} {2m} \right]}_{\eqqcolon a_k}.
  \end{equation*}

  It remains to show that the expression \( a_k \) equals zero for all \( k = 1, 2, \ldots \). We have
  \begin{equation*}
    a_k
    =
    \sum_{m=0}^k \binom {2k} {2m} - \sum_{m=0}^{k-1} \binom {2k} {2m+1}
    =
    \sum_{m=0}^k (-1)^m \binom {2k} m
    \reloset {\ref{thm:binomial_theorem}} =
    1 - 1 = 0.
  \end{equation*}

  \Fullref{eq:thm:trigonometric_identities/pythagorean_identity} follows.

  \SubProofOf{thm:trigonometric_identities/products} We will only prove \eqref{eq:thm:trigonometric_identities/products/cc} because the other identities are proved analogously. We have
  \begin{balign*}
    \cos(v - w) + \cos(v + w)
    &=
    \sum_{k=0}^\infty \frac {i^{2k}} {(2k)!} \left[(v - w)^{2k} + (v + w)^{2k} \right]
    \reloset {\ref{thm:binomial_theorem}} = \\ &=
    \sum_{k=0}^\infty \frac {i^{2k}} {(2k)!} \sum_{m=0}^{2k} \binom {2k} m v^{2k-m} w^m \left[ (-1)^m + 1 \right]
    = \\ &=
    2 \sum_{k=0}^\infty \frac {i^{2k}} {(2k)!} \sum_{m=0}^{2k} \binom {2k} {2m} v^{2(k-m)} w^{2m}
    \reloset {\eqref{eq:thm:trigonometric_identities/cos_product}} = \\ &=
    2 \cos(v) \cos(w).
  \end{balign*}

  \SubProofOf{thm:trigonometric_identities/sums} Fix some \( v, w \in \BbbC \) and define
  \begin{balign*}
    x \coloneqq \frac {v + w} 2
    &&
    y \coloneqq \frac {v - w} 2
  \end{balign*}
  so that \( v = x + y \) and \( w = x - y \).

  The identity \eqref{eq:thm:trigonometric_identities/sums/sin_sum} the follows from \eqref{eq:thm:trigonometric_identities/products/sc} applied to \( x \) and \( y \). The other identities are proved analogously.

  \SubProofOf{thm:trigonometric_identities/sum_of_angles} We will only prove \eqref{eq:thm:trigonometric_identities/sum_of_angles/sin} because \eqref{eq:thm:trigonometric_identities/sum_of_angles/cos} is proved analogously. From \eqref{eq:thm:trigonometric_identities/products/cs},
  \begin{balign*}
    \sin(x + y)
     & =
    2 \cos(x) \sin(y) + \sin(x - y)
    \reloset {\eqref{eq:thm:trigonometric_identities/products/sc}} = \\ &=
    2 \cos(x) \sin(y) + 2 \cos(x) \sin(y) - \sin(x + y).
  \end{balign*}

  After dividing by \( 2 \), we obtain \eqref{eq:thm:trigonometric_identities/sum_of_angles/sin}.
\end{proof}

\begin{lemma}\label{thm:trigonometric_function_basic_roots}
  We have the following important special values:
  \begin{balign}
    \sin(0) = 0,   &  & \cos(0) = 1,    \label{eq:thm:trigonometric_function_basic_roots/zero} \\
    \sin(\pi) = 0, &  & \cos(\pi) = -1. \label{eq:thm:trigonometric_function_basic_roots/pi}
  \end{balign}
\end{lemma}
\begin{proof}
\eqref{eq:thm:trigonometric_function_basic_roots/zero} follows directly from \fullref{def:trigonometric_functions}.
  Now consider the \hyperref[def:multi_valued_function/restriction]{restriction} of \( \cos \) to the real line. Since \( \cos(0) \neq 0 \) and \( \cos \) is continuously differentiable as a power series, in some neighborhood \( U \) of \( 0 \) we have \( 0 \not\in \cos(U) \). Therefore, the inverse function theorem holds and there exists a neighborhood \( V \subseteq U \) of \( 1 \) such that the continuously differentiable function \( f: V \to \BbbR \) is the inverse of \( \cos \) in \( V \) (we have not yet defined \hyperref[def:inverse_trigonometric_functions/arccos]{\( \arccos \)}). If \( y = \cos(x) \), then
  \begin{equation*}
    Df(y)
    =
    \frac 1 {D\cos(x)}
    =
    \frac 1 {-\sin(x)}
    \reloset {\ref{thm:trigonometric_identities/pythagorean_identity}} =
    -\frac 1 {\sqrt{1 - y^2}},
    \quad y \in \cos(V).
  \end{equation*}

  The derivative is actually well-defined and continuous anywhere except for \( y \in \{ -1, 1 \} \). Therefore, for any \( \alpha \in (-1, 1) \),
  \begin{equation*}
    f(y) = f(\alpha) - \int_{\alpha}^y \frac 1 {\sqrt{1 - t^2}} dt, \quad y \in [\alpha, 1).
  \end{equation*}

  We already know that \( \cos(0) = 1 \), hence \( f(1) = 0 \) and, since \( f(y) \) is given by a convergent integral in \( [\alpha, 1) \), we can extend this interval to \( [\alpha, 1] \).

  By taking \( y = \alpha \), we obtain
  \begin{equation*}
    f(y) - f(-y) = -\int_{-y}^y \frac 1 {\sqrt{1 - t^2}} dt, \quad y \in [-1, 1].
  \end{equation*}

  Note that by \hyperref[def:pi]{our definition} of \( \pi \),
  \begin{equation*}
    \pi
    =
    \int_{-1}^1 \frac 1 {\sqrt{1 - t^2}} dt
    =
    -[\underbrace{f(1)}_{=0} - f(-1)]
    =
    f(-1).
  \end{equation*}

  Hence, \( \cos(\pi) = -1 \). From \fullref{thm:trigonometric_identities/pythagorean_identity},
  \begin{equation*}
    \abs{\sin(\pi)} = \sqrt{1 - \cos(\pi)^2} = 0,
  \end{equation*}
  proving that \( \sin(\pi) = 0 \).

  This concludes the proof of \eqref{eq:thm:trigonometric_function_basic_roots/pi}.
\end{proof}

\begin{definition}\label{def:periodic_function}
  A function \( f: G \to H \) between \hyperref[def:abelian_group] abelian groups is called \term{periodic} with \term{period} \( p \in G \) if, for all \( x \in G \), we have \( f(x) = f(x + r) \).

  The \term{base period} of a function is the \hyperref[def:partially_ordered_set_extremal_points/maximum_and_minimum]{least} of all periods, if a minimum exists. When referring to \enquote{the period}, we mean the base period.

  We can define periods for arbitrary magmas rather than abelian groups, but the definition would make it difficult to talk about the base period.
\end{definition}

\begin{theorem}\label{thm:trigonometric_function_period}
  Both \( \sin(z) \) and \( \cos(z) \) are \( 2\pi \)-periodic.
\end{theorem}
\begin{proof}
  We will temporarily restrict ourselves to the real line. Since \( \cos(x) \) is continuous, \( \cos^{-1}(\{ 0 \}) \) is a closed set by \fullref{thm:weierstrass_extreme_value_theorem} there exists a minimum \( \gamma \) of \( [0, \pi] \cap \cos^{-1}(\{ 0 \}) \).

  Since, by \fullref{thm:trigonometric_function_basic_roots}, \( \cos(0) = 1 \), it follows that \( \cos(x) > 0, x \in (-\gamma, \gamma) \). Therefore, its primitive function \( \sin(x) \) increases on the same interval. It is also continuous, hence by \fullref{thm:trigonometric_identities/pythagorean_identity}, \( \sin(\gamma) = 1 \) because \( \cos(\gamma) = 0 \).

  Because \( \sin \) is an odd function, \( \sin(-\gamma) = \sin(\gamma) = -1 \).

  From \hyperref[def:pi]{our definition} of \( \pi \) it follows that
  \begin{equation*}
    \pi
    =
    \int_{-1}^1 \frac 1 {1 - t^2} dt
    =
    \int_{-\gamma}^\gamma \frac {\cos(\varphi)} {1 - \sin(\varphi)^2} d\varphi
    \reloset {\ref{thm:trigonometric_identities/pythagorean_identity}} =
    \int_{-\gamma}^\gamma d\varphi
    =
    2\gamma.
  \end{equation*}

  In order for a number \( p \) to be a period of \( \sin \), we need to have \( \sin(p) = \sin(0) = 0 \). But we showed that \( \sin(x) \) is increasing from \( 0 \) to \( \tfrac \pi 2 \) and cannot possibly contain zeros in that interval. Hence, \( p > \tfrac \pi 2 \).

  We also have \( \cos(\tfrac \pi 2) = 0 \). By \fullref{thm:trigonometric_identities/sums},
  \begin{equation*}
    \sin(\tfrac \pi 2 + x)
    =
    \sin(\tfrac \pi 2) \cos(x) + \cos(\tfrac \pi 2) \sin(x)
    =
    \cos(x).
  \end{equation*}

  Since \( \cos \) is positive on \( [0, \tfrac \pi 2) \), \( \sin \) is positive on \( [\tfrac \pi 2, \pi) \).

  We already showed in \fullref{thm:trigonometric_function_basic_roots} that \( \sin(\pi) = 0 \).

  It follows that the minimal period of \( \sin \) is either \( \pi \) or a multiple of \( \pi \). It cannot be \( \pi \) since \( \cos(\pi) \neq \cos(0) \), therefore it must be \( 2\pi \).
\end{proof}

\begin{definition}\label{def:hyperbolic_trigonometric_functions}
  In analogy with \fullref{thm:exponential_trigonometric_identities/inverse_eulers_formula}, we define \term{hyperbolic trigonometric functions}.

  \begin{thmenum}
    \thmitem{def:hyperbolic_trigonometric_functions/sine} The \term{hyperbolic sine} function:
    \begin{equation*}
      \sinh(x) \coloneqq - \frac {e^x - e^{-x}} 2 \\
    \end{equation*}

    \thmitem{def:hyperbolic_trigonometric_functions/cosine} The \term{hyperbolic cosine} function:
    \begin{equation*}
      \cosh(x) \coloneqq \frac {e^x + e^{-x}} 2
    \end{equation*}
  \end{thmenum}

  Compare \fullref{def:quadratic_plane_curve/ellipse/parametric_equations} and \fullref{def:quadratic_plane_curve/hyperbola/parametric_equations} for a justification of the naming.
\end{definition}

\begin{definition}\label{def:derived_trigonometric_functions}
  In addition to \( \sin(z) \) and \( \cos(z) \), we define two additional functions, also called \enquote{trigonometric}.

  \begin{thmenum}
    \thmitem{def:derived_trigonometric_functions/tan} The \hyperref[def:partial_function]{partial} \term{tangent} function, also called \term{tangens}, is
    \begin{equation*}
      \tan(z) \coloneqq \frac {\sin(z)} {\cos(z)}.
    \end{equation*}

    It is defined in \( \BbbC \setminus (\tfrac \pi 2 + \pi\BbbZ) \).

    \thmitem{def:derived_trigonometric_functions/cot} The \hyperref[def:partial_function]{partial} \term{cotangent function}, also called \term{cotangens}, is
    \begin{equation*}
      \cot(z) \coloneqq \frac {\cos(z)} {\sin(z)}.
    \end{equation*}

    It is defined in \( \BbbC \setminus \pi\BbbZ \).
  \end{thmenum}
\end{definition}

\begin{definition}\label{def:inverse_trigonometric_functions}
  We can define \term{inverse trigonometric functions}. We will thus restrict ourselves only to real numbers. Fix an integer \( k \). Unless noted otherwise, we assume \( k = 0 \).

  \begin{thmenum}
    \thmitem{def:inverse_trigonometric_functions/arcsin} The \term{arcus sinus} function \( \arcsin(x) \) is defined as the \hyperref[def:multi_valued_function/inverse]{inverse function} of \( \sin(x) \) (see \fullref{def:trigonometric_functions/sine}) from \( [-1, 1] \) to \( \left[(k - \tfrac 1 2) \pi, (k + \tfrac 1 2) \pi \right) \).

    \thmitem{def:inverse_trigonometric_functions/arccos} The \term{arcus cosinus} function \( \arccos(x) \) is defined as the inverse of \( \cos(x) \) (see \fullref{def:trigonometric_functions/cosine}) from \( [-1, 1] \) to \( (k\pi, (k + 1)\pi) \).

    \thmitem{def:inverse_trigonometric_functions/arctan} The \term{arcus tangens} function \( \arctan(x) \) is defined as the inverse of \( \tan(x) \) (see \fullref{def:derived_trigonometric_functions/tan}) from \( \BbbR \) to \( \left((k - \tfrac 1 2) \pi, (k + \tfrac 1 2) \pi \right) \).

    \thmitem{def:inverse_trigonometric_functions/arccot} The \term{arcus cotangens} function \( \arccot(x) \) is defined as the inverse of \( \cot(x) \) (see \fullref{def:derived_trigonometric_functions/cot}) from \( \BbbR \) to \( (k\pi, (k + 1)\pi) \).

    \thmitem{def:inverse_trigonometric_functions/arctantwo} The \term{two-argument arcus tangens} function \( \arctantwo(y, x) \) is a bit special, however it is very useful in practice - see \fullref{thm:arctantwo}. It is defined as
    \begin{align*}
       &\arctantwo: \BbbR^2 \setminus \{ 0 \} \to [2k\pi, 2k\pi + 2)  \\
       &\arctantwo(y, x) \coloneqq \begin{dcases}
        \arctan \parens*{ \tfrac y x },       &x > 0                  \\
        \arctan \parens*{ \tfrac y x } + \pi, &x < 0 \T{and} y \geq 0 \\
        \arctan \parens*{ \tfrac y x } - \pi, &x < 0 \T{and} y < 0    \\
        \pi,                                 &x = 0 \T{and} y \geq 0 \\
        -\pi,                                &x = 0 \T{and} y < 0.
      \end{dcases}
    \end{align*}
  \end{thmenum}
\end{definition}

\begin{proposition}\label{thm:arctantwo}
  Fix an integer \( k \). Given \( (x_0, y_0) \in S_{\BbbR^2} \), \( t_0 \coloneqq \arctantwo(y_0, x_0) \) is the unique, solution to the equation
  \begin{equation}\label{thm:arctantwo/equation}
    \begin{cases}
      x_0 = \cos(t) \\
      y_0 = \sin(t)
    \end{cases}
  \end{equation}
  in \( t \in [2k\pi, 2k\pi + 2) \).
\end{proposition}