\documentclass{beamer}
\usetheme{CambridgeUS}
\title[Process]{The Koopman operator identification algorithm}
\subtitle{So far}
\institute[Polimi]{Politecnico di Milano}
\author{Sergio Vanegas}
\date{\today}

\usepackage{listings}
\usepackage[framed,numbered,autolinebreaks,useliterate]{mcode/mcode}

\usepackage{caption}
\usepackage{subcaption}

\usepackage{siunitx}

\begin{document}

\begin{frame}[plain,noframenumbering]
    \maketitle
\end{frame}

\begin{frame}{Table of Contents}
    \tableofcontents
\end{frame}

\section{Pre-requisites}

\begin{frame}[fragile]{The forced Van Der Pol oscillator}
    \begin{equation}
        \begin{cases}
            \dot{x}_1 = 2*x_2 \\
            \dot{x}_2 = -0.8*x_1 + 2*x_2 + 10*x_1^2*x_2 + u
        \end{cases}
    \end{equation}

    \begin{lstlisting}[language=Matlab]
function dxdt = VanDerPol(t,x,u_t,u)
    % Interpolation just for the sake of indexing
    u = interp1(u_t,u,t);

    dxdt = [2*x(2);
            -0.8*x(1)+2*x(2)-10*(x(1)^2)*x(2) + u];
end
    \end{lstlisting}
\end{frame}

\begin{frame}{Data generation - Parameters}
    \begin{itemize}
        \item $T_s = \SI{1}{\milli \second}$
        \item $T = \SI{20}{\second}$
        \item $N_\text{Sim} = \num{100}$
        \item $\textbf{u}$: random Gaussian signal with the same sampling rate and length as the simulation
        \item $\sigma = 1$: std deviation of the Gaussian noise
        \item $\mu$: average of the Gaussian noise (randomly selected per trajectory)
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{Data generation - Noise as input}
    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
Data_Source = "~/Documents/Thesis/VanDerPol_Unsteady_Input/";
ts = 1e-3;
T = 20;
N_Sim = 40;

u_t = 0:ts:T;
sigma = 1; % 0 for constant input
mu = 0;

system("mkdir -p "+Data_Source);
delete(Data_Source+"Data_*.mat");

parfor f=1:N_Sim
    mu = randn(1); % 0 for zero-mean noise
    u = sigma^2*randn(size(u_t)) + mu;
    z0 = 4*rand(2,1) - 2;
    [~,z] = ode113(@(t,z) VanDerPol(t,z,u_t,u), u_t, z0);

    z = z';
    L = length(z)-1;
    parsave(sprintf(Data_Source+'Data_%i.mat',f),L,z,u,T,ts);
end
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Polynomial observables}
    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
function [g,n] = Poly_Obs(z,P,Nx,Nu)
    % Number of monomials with degree lesser than P
    n = factorial(Nx+P)/(factorial(Nx)*factorial(P));
    g = ones(n+Nu,size(z,2));

    exponents = zeros(n,Nx);
    current = zeros(1,Nx);

    [exponents,~] = Recursive_Monomial(1,1,exponents,current,P);

    for i=1:n
        for j=1:Nx
            g(i,:) = g(i,:).*(z(j,:).^exponents(i,j));
        end
    end

    % Inputs returned as additional observables
    g(n+1:end,:) = z(Nx+1:end,:);
end
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Recursive exponent generation}
    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
function [exponents,i] = Recursive_Monomial(i,j,exponents,current,P)
    while sum(current) <= P
        % Decide wether or not to register the exponent combination only on the deepest level of recursion
        if j==size(exponents,2)
            exponents(i,:) = current;
            i = i+1;
        else
            [exponents,i] = Recursive_Monomial(i,j+1,exponents,current,P);
        end

        % Reset exponent count
        current(j) = current(j)+1;
    end
    current(j) = 0;
end
    \end{lstlisting}
\end{frame}

\section{First step: data observables}

\begin{frame}[fragile]{Data reading \& Polynomial degree selection}
    Up to fourth order polynomials used for the presentation (lower order more stable but just as inaccurate).

    \begin{lstlisting}[language=Matlab, basicstyle=\tiny]
data = dir(Data_Source+"Data_*.mat");
load(Data_Source+data(1).name);

K = 0;
for f=1:length(data)
    load(Data_Source+data(f).name,"L");
    K = K+L;
end

P = 12;
[g_t,n] = Poly_Obs(z,P);

Px = zeros(n,K);
Py = Px;
Z = zeros(size(z,1),K);
U = zeros(size(u,1),K);
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Data concatenation}
    \begin{lstlisting}[language=Matlab]
Px(:,1:L) = g_t(:,1:end-1);
Py(:,1:L) = g_t(:,2:end);
Z(:,1:L) = z(:,2:end);
U(:,1:L) = u(:,1:end-1);

idx = L;
for f=2:length(data)
    load(Data_Source+data(f).name);
    [g_t,~] = Poly_Obs(z,P);

    Px(:,idx+1:idx+L) = g_t(:,1:end-1);
    Py(:,idx+1:idx+L) = g_t(:,2:end);
    Z(:,idx+1:idx+L) = z(:,2:end);
    U(:,idx+1:idx+L) = u(:,1:end-1);

    idx = idx+L;
end
    \end{lstlisting}
\end{frame}

\section{Second step: operator matrix}

\begin{frame}{Matrix calculation - Notation}
    \begin{itemize}
        \item $\mathbf{u}\left[k\right]$: input vector at sample $k \in \left\{1,\dots,K+1\right\}$.
        \item $\left(\mathbf{x}\left[k\right],\mathbf{y}\left[k\right]\right)$: state pairs such that, for a generic causal DT system $\mathbf{x}\left[k+1\right] = \mathbf{T}(\mathbf{x}\left[k\right] , \mathbf{u}\left[k\right])$, $\mathbf{y}\left[k\right] = \mathbf{T}(\mathbf{x}\left[k\right], \mathbf{u}\left[k\right])$. States are of dimension $N_x$, and the input is of dimension $N_u$.
        \item $P_N$: Projection over the N-Dimensional truncated space of observables, where $\tilde{N} = \frac{\left(N_x + P\right)!}{N_x! P!}$ and $N = \tilde{N} + N_u$.
        \item $p_j$: Monomial observing the original states, with $j \in {1,\dots,\tilde{N}}$.
        \item $\mathbf{P_x},\mathbf{P_y},\mathbf{Z}$: State data collections of the form
            \begin{align*}
                \mathbf{P_x} &=
                \begin{pmatrix}
                    \mathbf{p}\left(\mathbf{x}\left[1\right]\right)   &
                    \cdots  &
                    \mathbf{p}\left(\mathbf{x}\left[K\right]\right)
                \end{pmatrix}
                \\
                \mathbf{P_y} &=
                \begin{pmatrix}
                    \mathbf{p}\left(\mathbf{y}\left[1\right]\right)   &
                    \cdots  &
                    \mathbf{p}\left(\mathbf{y}\left[K\right]\right)
                \end{pmatrix}
                \\
                \mathbf{Z} &=
                \begin{pmatrix}
                    \mathbf{x}\left[1\right]    &
                    \cdots  &
                    \mathbf{x}\left[K\right]
                \end{pmatrix}
            \end{align*}
        \item $\mathbf{U}$: Input data collection of the form $\begin{pmatrix} \mathbf{u}\left[1\right] & \cdots & \mathbf{x}\left[K\right] \end{pmatrix}$.
    \end{itemize}
\end{frame}

\begin{frame}{Matrix calculation - I}

    We denote the associated Koopman operator with sampling time $T_s$ by $\mathcal{K}^{T_s}$, derived from the solution to the following minimization problem:

    \begin{align}
        P_N & g = \text{arg min}_{\tilde{g} \in \text{span}\left\{p_1 , \dots , p_N , u_1 , \dots , u_m\right\}} \sum_{k=1}^K \left|\tilde{g}\left(\mathbf{x}_k\right) - g\left(\mathbf{x}_k\right)\right|^2 \\
        &\implies P_N g =
        \begin{pmatrix}
            g\left(\mathbf{x}_1\right) &
            \cdots &
            g\left(\mathbf{x}_K\right)
        \end{pmatrix}
        \begin{pmatrix}
            \mathbf{P}_x \\
            \mathbf{U}
        \end{pmatrix}^\dagger
        \begin{pmatrix}
            \mathbf{p} \\
            \mathbf{u}
        \end{pmatrix}\\
        & \implies P_N \left(\mathcal{K}^{T_s} p_j\right) = \mathbf{p}^T \mathbf{P_x}^\dagger
        \begin{pmatrix}
            \mathcal{K}^{T_s} p_j\left(\mathbf{x}_1\right) \\
            \vdots \\
            \mathcal{K}^{T_s} p_j\left(\mathbf{x}_K\right)
        \end{pmatrix}
        \approx
        \begin{pmatrix}
            p_j\left(\mathbf{y}_1\right) \\
            \vdots \\
            p_j\left(\mathbf{y}_K\right)
        \end{pmatrix}
        \begin{pmatrix}
            \mathbf{P}_x \\
            \mathbf{U}
        \end{pmatrix}^\dagger
        \begin{pmatrix}
            \mathbf{p} \\
            \mathbf{u}
        \end{pmatrix}
    \end{align}
\end{frame}

\begin{frame}[fragile]{Matrix calculation - II}
    We get each row of the truncation (and, by consequence, the whole matrix) as below, where $A$ and $B$ follow the notation from a classical State Space representation:

    \begin{equation} \label{eq:Matrix_Operator}
        \overline{\mathcal{K}}_{N,j} \approx \mathbf{P}_{y,j} \begin{pmatrix}
            \mathbf{P}_x \\
            \mathbf{U}
        \end{pmatrix}^\dagger \implies \overline{\mathcal{K}}_N =
        \begin{bmatrix}
            A & B
        \end{bmatrix}
        \approx \mathbf{P}_y
        \begin{pmatrix}
            \mathbf{P}_x \\
            \mathbf{U}
        \end{pmatrix}^\dagger
    \end{equation}

    We implement Equation~\ref{eq:Matrix_Operator} as follows:

    \begin{lstlisting}
[A,B] = Koopman(Px,Py,U);
% Generic way to recover original states
C = Unobserver(Px,Z);
% Recovery of original states independent from input
D = zeros(size(Z,1),size(U,1));
save(sprintf(Data_Source+'Operator_P_%i.mat',P), ...
    "A","B","C","D","ts");
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Matrix calculation - III}
    The function \texttt{Koopman} called above is defined as follows:

    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
function [A,B] = Koopman(X,Y,U)
% Following "Practical Considerations" from Korda-Mezic
G = [X;U]*[X;U]';
V = Y*[X;U]';
A = zeros(size(Y,1),size(X,1));
B = zeros(size(Y,1),size(U,1));

%     Matricial division segmented because of memory limits
%     M0 = V/G;
%     A = M0(:,1:size(X,1));
%     B = M0(:,size(X,1)+1:end);

for i=1:size(Y,1)
    m0 = V(i,:)/G;
    A(i,:) = m0(1:size(X,1));
    B(i,:) = m0(size(X,1)+1:end);
end
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Spectral analysis - Code}
    Originally done for DMD purposes, but it keeps being useful for stability analysis.

    \begin{lstlisting}[language=Matlab]
Lambda = eig(A);
figure(1);
scatter(real(Lambda),imag(Lambda));
hold on;
rectangle('Position', [-1 -1 2 2], 'Curvature', 1);
hold off;
    \end{lstlisting}
\end{frame}

\section{Third step: trajectory prediction}

\begin{frame}[fragile]{Training data - Visualization}
    \begin{lstlisting}[language=Matlab]
load(Data_Source+data(1).name);
L = length(0:ts:T);

markerDecay = 0.001;
figure(2);
scatter(z(1,:),z(2,:),36*exp(-markerDecay*(0:L-1)));
hold on;
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Training data - Trajectory prediction}
    \begin{lstlisting}[language=Matlab]
g_p = zeros(size(g_t,1),L);
[g_p(:,1),~] = Poly_Obs(z(:,1),P);

for i=1:L-1
    g_p(:,i+1) = A*g_p(:,i) + B*u(:,i);
end

z_p = C*g_p;

scatter(z_p(1,:),z_p(2,:),36*exp(-markerDecay*(0:L-1)));
legend("Original Data","Trajectory Prediction");
hold off;
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{New signal - Data generation}
    \begin{lstlisting}[language=matlab]
T = 20;

z0 = 4*rand(2,1) - 2;
sigma = randn(1);
u_t = 0:ts:T;
u = sigma * cos(u_t);

[t,z] = ode113(@(t,z) VanDerPol(t,z,u_t,u), u_t, z0);
z = z';
L = length(0:ts:T);

markerDecay = 0.001;
figure(1);
scatter(z(1,:),z(2,:),36*exp(-markerDecay*(0:L-1)));
hold on;
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{New signal - Trajectory prediction}
    \begin{lstlisting}
g_p = zeros(size(g_t,1),L);
[g_p(:,1),~] = Poly_Obs(z(:,1),P);

for i=1:L-1
    g_p(:,i+1) = A*g_p(:,i) + B*u(:,i);
end

z_p = C*g_p;

scatter(z_p(1,:),z_p(2,:),36*exp(-markerDecay*(0:L-1)));
legend("New Simulation","Trajectory Prediction");
hold off;
    \end{lstlisting}
\end{frame}

\section{Alternative Observables}

\begin{frame}[fragile]{Thin-Plate Spline-Radial functions I}
    Instead of using polynomials, we can use $\psi_j \left(\mathbf{x}\right) = \left|\left|\mathbf{x}-\mathbf{x}_j\right|\right|^2 \log{\left(\left|\left|\mathbf{x}-\mathbf{x}_j\right|\right|\right)}$ as basis function, where $\mathbf{x}_j$ is a bounded random vector of the same dimension as the original states. Since this collection of reference vectors has to be constant once it has been selected, we implement the observables as follows:

    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
function [g,n] = Spline_Radial_Obs(z,X0)
    L = size(z,2);
    n = size(X0,1);
    N = size(X0,2);

    % Preservation of original states
    g = [z;zeros(N,L)];

    for i=1:N
        x0 = X0(:,i);
        g(i+n,:) = vecnorm(z-x0).^2 .* log(vecnorm(z-x0));
    end

    n=n+N;
end
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Thin-Plate Spline-Radial functions II}
    The following modifications to the code had to be made in order to accommodate the new observables:

    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
% P = 12;
% [g_t,n] = Poly_Obs(z,P);
M = 100;
X0 = 2*rand(size(z,1),M)-1;
[g_t,n] = Spline_Radial_Obs(z,X0);
...
    %     [g_t,~] = Poly_Obs(z,P);
    [g_t,~] = Spline_Radial_Obs(z,X0);
...
% save(sprintf(Data_Source+'Operator_P_%i.mat',P), ...
%     "A","B","C","D","ts");
save(sprintf(Data_Source+'Operator_M_%i.mat',M), ...
    "A","B","C","D","ts","X0");
...
% [g_p(:,1),~] = Poly_Obs(z(:,1),P);
[g_p(:,1),~] = Spline_Radial_Obs(z(:,1),X0);
    \end{lstlisting}
\end{frame}

\section{Results}

\begin{frame}{Spectral analysis - Results}
    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Eigen_Poly.png}
            \caption{Polynomial observables (P=13) - Operator eigenvalues}
            \label{fig:eigen_poly}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Eigen_Radial.png}
            \caption{Radial observables (M=100) - Operator eigenvalues}
            \label{fig:eigen_radial}
        \end{subfigure}
        \caption{Operator eigenvalues (highest calculated order) - Observable to observable}
    \end{figure}
\end{frame}

\begin{frame}{Noisy input prediction - Results}
    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Training_Poly.png}
            \caption{Polynomial observables (P=13) - Predicted trajectory}
            \label{fig:training_poly}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Training_Radial.png}
            \caption{Radial observables (M=100) - Predicted trajectory}
            \label{fig:training_radial}
        \end{subfigure}
        \caption{Reference and predicted trajectories (highest calculated order) - Training data}
    \end{figure}
\end{frame}

\begin{frame}{Deterministic - Results}
    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Verification_Poly.png}
            \caption{Polynomial observables - Predicted trajectories}
            \label{fig:verification_poly}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Verification_Radial.png}
            \caption{Radial observables}
            \label{fig:verification_radial}
        \end{subfigure}
        \caption{Reference and predicted trajectories - Predicted trajectories}
    \end{figure}
\end{frame}

\section{Conclusions}

\begin{frame}{Conclusions}
    \begin{itemize}
        \item Lower sampling periods have shown to yield a better prediction horizon, since in practice it is bounded by the variations in shape between different trajectories getting wider throughout physical time.
        \item A classical optimization approach was implemented as part of the development process; nevertheless, it did not produce different results from the pseudo-inverse approach and instead was not able to keep up with the increase in dimension of the observable space. As a consequence, it was removed.
        \item Spline-radial functions yielded an overall better prediction horizon than the polynomial observables, which opens the question of wether or not better observable structures should be researched.
        \item Finally, higher-degree observables yielded better approximations of the original system, but required a higher than theoretical data library in order to converge when applying the pseudo-inverse.
    \end{itemize}
\end{frame}

\end{document}


\section{Data Mining through RAVEN}
\label{sec:DMraven}

Data mining is the computational process of discovering patterns in large data sets (``big data'') involving methods at the intersection of artificial intelligence, machine learning, statistics, and database systems. The overall goal of the data mining process is to extract information from a data set and transform it into an understandable structure for further use.
\\RAVEN has support of several different data mining algorithms,
such as:
\begin{enumerate}
  \item \textit{Hierarchical methodologies}
  \item \textit{K-Means}
  \item \textit{Mean-Shift}, etc.
\end{enumerate}

The goals of this section are about learning how to:
 \begin{enumerate}
   \item Set up a sampling strategy to apply clustering algorithms, perturbing a driven code
  \item Analyze the data using clustering algorithms.
\end{enumerate}
To accomplish these tasks, the following RAVEN \textbf{Entities} (XML blocks in the input files) need to be defined:
\begin{enumerate}
   \item \textbf{\textit{RunInfo}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{RunInfo}
   The \textit{RunInfo} \textbf{Entity} is intended  to set up the analysis sequence that
   needs to be performed. In this specific case, two steps  (\xmlNode{Sequence}) are sequentially run
   using forty processors (\xmlNode{batchSize}).
   \\In the first step, the original physical model is going to be sampled.
   The obtained results are going to be analyzed with data mining
   algorithms.
   \item \textbf{\textit{Files}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{Files}
   Since the driven code uses a single input file, in this section the original input is placed. The attribute  \xmlAttr{name} represents the alias that is going to be
   used in all the other input blocks in order to refer to this file.
   \item \textbf{\textit{Models}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{Models}
 The goal of this example is to show how the
 data mining algorithms in RAVEN can be useful to analyze large data set.
 \\In addition to the previously explained Code
 model, two Post-Processor models ($DataMining|cluster|KMeans$ and $DataMining|decomposition|PCA$) are specified.
Note that the post-processing is
performed on all the output FOMs used in this example ( $A,\, B,\, C \, and \, D$).
   \item \textbf{\textit{Distributions}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{Distributions}
  In the Distributions XML section, the stochastic model for the
  uncertainties are reported. In
  this case 2 distributions are defined:
  \begin{itemize}
    \item $sigma \sim \mathbb{U}(0,1000)$, used to model the uncertainties
    associated with  the Model \textit{sigma-A} and \textit{sigma-B};
    \item  $decayConstant \sim \mathbb{U}(1e-8,1e-7)$,  used to
    model the uncertainties
    associated with  the Model \textit{decay-A} and \textit{decay-B}.
  \end{itemize}
   \item \textbf{\textit{Samplers}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{Samplers}
  In order to obtain the data-set on which the data mining algorithms are going to be applied, a \textit{Grid} sampling approach is here employed.
   \item \textbf{\textit{DataObjects}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{DataObjects}
  Int this block, two \textit{DataObjects} are defined:
  1) PointSet named ``samples'' used to collect the final outcomes of
  the code,
  2) HistorySet named ``histories'' in which the full time responses of the
  variables $A,B,C,D$ are going to be stored.
 %figure samples
 \begin{figure}[h!]
  \centering
  \includegraphics[scale=0.7]{../../tests/framework/user_guide/DataMining/gold/dataMiningAnalysis/1-PlotLabels_dataMining.png}
  \caption{K-means clustering on original dataset.}
  \label{fig:KmeanOrigData}
 \end{figure}
   \item \textbf{\textit{OutStreams}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{OutStreams}
     This workflow uses one Print OutStream and three Plot OutStreams:
     \begin{itemize}
       \item ``samplesDump'', which writes the original sample set with the additional columns from the
         PostProcess steps,
       \item ``PlotKMeans1'', which plots the samples against the Figures of Merit with coloring according to
         the KMeans clustering,
       \item ``PlotLabels'', which plots all the samples and colors them according to the KMeans clustering,
       \item ``PlotPCA1,'' which plots the surrogate principal component dimensions and their associated clustering.
     \end{itemize}
     Note that a special kind of plot, the ``dataMining'' \xmlNode{type}, has been implemented to simplify
     plotting complicated results using RAVEN, and is used in all three of the plots in this workflow.  Also
     note the use of the \xmlNode{range} block to define the data range of the plots created.
   \item \textbf{\textit{Steps}}:
     \xmlExample{framework/user_guide/DataMining/dataMiningAnalysis.xml}{Steps}

 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   Finally, all the previously defined \textbf{Entities} can be combined in
   the \xmlNode{Steps} block;
   3 \xmlNode{Steps} have been inputted:
   \begin{itemize}
     \item \xmlNode{MultiRun} named ``sample'', used to run the
     multiple
     instances of the driven code and
     collect the outputs in the two \textit{DataObjects}.The \xmlNode{Sampler} is inputted to communicate to the
     \textit{Step} that the driven code needs to
     be perturbed through the Grid sampling strategy;
     \item \xmlNode{PostProcess} named ``kmeans'', used
     to analyze the data obtained through the sampling strategy. In
     this step, a K-Means algorithm is going to be employed, plotting
     the clustering results;
     \textit{Step} that the driven code needs to
     be perturbed through the Grid sampling strategy;
     \item \xmlNode{PostProcess} named ``pca'', used
     to analyze the data obtained through the sampling strategy. In
     this Step, a PCA algorithm is going to be employed, plotting
     the decomposition results.
   \end{itemize}
\end{enumerate}
Figure~\ref{fig:KmeanOrigData} shows the clustering on the original
input space.
\\Figure~\ref{fig:KmeanProjected} shows the clustering on the projected
input space. It can be noticed, that the algorithm fully capture the fact
that the parameter $sigma-B$ does not impact the response $A$ (being completely independent).
\\Figure~\ref{fig:PCAplot} shows the PCA decomposition on the data set.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %figure samples
 \begin{figure}[h!]
  \centering
  \includegraphics[scale=0.7]{../../tests/framework/user_guide/DataMining/gold/dataMiningAnalysis/1-PlotKMeans1_dataMining-dataMining-dataMining.png}
  \caption{K-means clustering on projected parameters.}
  \label{fig:KmeanProjected}
 \end{figure}
 %%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

 %%%%%%%%%%%%%%%%%%%%%%%%
  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %figure samples
 \begin{figure}[h!]
  \centering
  \includegraphics[scale=0.7]{../../tests/framework/user_guide/DataMining/gold/dataMiningAnalysis/1-PlotPCA1_dataMining.png}
  \caption{Principal Component Analysis.}
  \label{fig:PCAplot}
 \end{figure}
 %%%%%%%%%%%%%%%%%%%%%%%%


%================================
\section{Product Spaces}
%================================

% https://en.wikipedia.org/wiki/Base_(topology)#Theorems

\begin{definition}
	[product topologies]
	\label{def: product topologies}
\end{definition}


\section{Implementation}

\citet{Nakajima2009} developed and published a software that was used for their calculations using SV with leverage~\citep{nakajima2009code}.
However, that software was written in the proprietary Ox language, which is only freely available with limited features for academic and teaching purposes~\citep{doornik2009object}.
Thus, apart from investigating the behaviour of leverage, this master thesis also partly aims at providing a freely available solution for fitting SV with leverage.
This section provides details and tests about our implementation, which will be published under the GNU GPLv3 License~\citep{gplv3}.
In the meantime, the software is available upon request from the author.

\subsection{Software framework}

The software is written mainly in R, and some parts are in C++ via the package Rcpp for the sake of efficiency~\citep{rlanguage,rcpp2011,iso2016iec}.
There are two parts to the software: an R package that provides a function that runs the algorithm detailed in Section~\ref{sec:estimlev}, and a set of individual R scripts that apply that function on data, evaluate results and create plots.
Other packages used for the model are numDeriv, Matrix, testthat, mvtnorm and MCMCpack.
For data manipulation and visualisation the tidyverse is used~\citep{rmcmcpack,rtestthat,rmatrix,rnumderiv,rmvtnorm,rtidyverse}.

\subsection{Simulation}\label{sec:simulation}

The main task of the software is to estimate leverage, thus tests were centred around different simulated values of $\rho$.
The seven setups that are detailed here all had the same settings except for the simulated value of $\rho$:
\begin{description}
	\item[``True'', simulated values]
	\begin{align*}
	\phi &= 0.95, \\
	\sigma^2 &= 0.01, \\
	\mu &= -9, \\
	T &= 1000,
	\end{align*}
	\begin{equation*}
	\rho\in[-0.9,-0.6,-0.3,0,0.3,0.6,0.9].
	\end{equation*}
	\item[Prior distributions]
	\begin{align*}
	\frac{\phi+1}2 &\sim\text{Beta}(20,1.5), \\
	\sigma^2 &\sim\text{InverseGamma}(2.25,\text{rate}=0.0625), \\
	\rho &\sim\mathcal{U}(-1,1), \\
	\mu &\sim\mathcal{N}(-9,1), \\
	h_1\mid\phi,\sigma,\mu &\sim\mathcal{N}(\mu,\sigma^2/(1-\phi^2)).
	\end{align*}
\end{description}
The simulated true values used here are similar to the ones found in the literature~\citep{Omori2007,Kastner2014}.
Variance is usually highly persistent and has a mean about $\exp(-9/2)\approx 1\%$, these are encoded both in the simulated value and the prior.
The chosen prior for $\sigma^2$ has mean 0.05 and standard deviation 0.1, so the prior is not too far from the simulated value.
The latent vector is initialised by $h_1$, and its prior is chosen to be the stationary distribution of the AR(1) process that models $\bm h$.
Finally, we check a wide range of $\rho$ values, and $\rho$'s prior is intended to be non-informative.

The simulation was based on model~\eqref{form:orig_model}.
All the parameters and $h_1$ need to be initialised, and a realisation of the two white-noise processes has to be generated.
Finally, after substituting the parameters and the errors into the model, $\bm h$ and $\bm y$ are acquired.

Figure~\ref{fig:simdata} illustrates the simulated dataset, the price process and the latent volatility process, with three chosen values of $\rho$.
It also demonstrates how the different correlation signs affect the directly observable relationship.
High values of $\rho$ make the price process and the volatility do mirrored changes, while low values imply parallel movements.

Figure~\ref{fig:volatility} presents how the posterior variance fits the simulated variance.
In general, comovement of the posterior median with the simulated values is observed, and stronger correlation seemingly brings better fit.
That result is not surprising since, intuitively, there is less information in the directly measured return series about the variance when correlation is low.

There are three distributions corresponding to each parameter in the current estimation method: the prior, the approximate posterior and the corrected posterior.
Figure~\ref{fig:rhodensities} shows these and the simulated value for $\rho$.
For low absolute values, the correction outlined in Section~\ref{sec:reweight} has quite low impact, the difference is invisible on the plot, but it is larger for more extreme $\rho$.
Although the ``true'' value is in the 5\%-95\% credible interval in each case, the posterior is closer to the simulated value for negative $\rho$.

\begin{figure}
	\thisfloatpagestyle{empty}
	\centering
	\includegraphics[width=\linewidth]{simulations/data-plot.pdf}
	\caption[Simulated price process and standard deviation]{Simulated price process scaled to 0.03 and its return's standard deviation.
		Top: with high negative correlation, the price process and the volatility move to the opposite direction.
		Middle: with no correlation, there is no visible pattern.
		Bottom: with high positive correlation, the price process and the volatility move together.}
	\label{fig:simdata}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=\linewidth]{simulations/variance-plot.pdf}
	\caption{Simulated values and posterior quantiles of the variance process.}
	\label{fig:volatility}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=\linewidth]{simulations/rho-densities.pdf}
	\caption[Prior, approximate and true posteriors, and simulated $\rho$]{Prior, approximate and corrected posterior distributions, and the simulated value for $\rho$.}
	\label{fig:rhodensities}
\end{figure}


\documentclass[11pt, twocolumn]{article}

\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage[a4paper, total={7in, 9in}]{geometry}

\title{Probability Cheatsheet}
\author{Yahya~Almardeny\\almardeny@gmail.com}
\date{June 2019}

\begin{document}
\begin{titlepage}
\maketitle
\end{titlepage}

\section{Overview}
Probability provides measures for reasoning the likelihood that an event will occur in an experiment.
\section{Terminology}
\begin{enumerate}
\item \textbf{Experiment}: Procedure that yields one of the possible outcomes. E.g. Tossing a coin.
\item \textbf{Sample Space S}: A set of all possible outcomes of an experiment. E.g. S = \{Head, Tail\} when tossing a fair coin.
\item \textbf{Event E}: Set of outcomes of an experiment. Note that if the Event contains only one outcome, it is then called an elementary event. E.g. The event that coin is Head.
\end{enumerate}
\section{Random Variable R.V}
A variable that its possible values are the outcomes of an experiment. It has two types:
\begin{enumerate}
\item \textbf{Discrete R.V}: Take one countable number of distinct values. E.g 1, 2, 3, 4
\item \textbf{Continuous R.V}: Take infinite number of possible values (usually measurements). E.g. all interval values in the range [1 - 2].
\end{enumerate}

\section{Probability of Events}
\begin{enumerate}
\item \textbf{Marginal probability}: Simply $P(A)$ which means "unconditional probability". And for each outcome $s$, it should satisfy: $ 0 \leq P(s) \leq 1$ and $\Sigma P(s_i) = 1$.
\item \textbf{Addition Rule}: $P(A \cup B) = P(A) + P(B) - P(A \cap B)$ where $\cup$ means OR, $\cap$ means AND.
\item \textbf{Joint (Compound) Probability Rule}:
	\begin{enumerate}
	\item Independent Events: $P(A \cap B) = P(A) \times P(B)$.
	\item Dependent Events: $P(A \cap B) = P(A) \times P(B | A)$. where $P(B | A)$ read: Probability of B given A had happened.
	\end{enumerate}
\item \textbf{Conditional Probability}: $P(A | B) = \frac{P(A \cap B)}{P(B)}$
\item \textbf{Bayes Theorem}: $P(A | B) = \frac{P(B | A) \times P(A)}{P(B)}$. It gives the probability of A based on prior knowledge of B.
\end{enumerate}

\section{Probability Distributions}
\begin{enumerate}
\item \textbf{Probability Mass Function (PMF)}: Gives the probability a \textit{discrete} R.V $X$ takes on the value $x$: $P( X = x)$.
\begin{figure}[h!]
  \centering
  \includegraphics[width=0.7\linewidth]{figs/PMF.png}
\end{figure}
\item \textbf{Probability Density Function (PDF)}: Gives the probability a \textit{continuous} R.V $X$ takes on the value $x$: $P( X = x)$ which means how dense the probability of $X$ near $x$, and it will be the definite Integrals between two points.
\begin{figure}[h!]
  \centering
  \includegraphics[width=0.6\linewidth]{figs/PDF.png}
\end{figure}
\item \textbf{Cumulative Density Function (CDF)}: Gives the probability a R.V $X$ is less than or equals the value $x$: $P( X \leq x)$. It is  cumulative because it adds the probabilities up to $x$.
	\begin{enumerate}
	\item For Discrete R.V: It is the summation of previous probabilities $\Sigma p(x_i)$.
	\item For Continuous R.V: It is the integral $\int_{-\infty}^{x} p(x) dx$
	\end{enumerate}
\end{enumerate}

\section{Expected Value of R.V}
Intuitively, it is the long-run average value of repetitions of the same experiment it represents (i.e. what outcome to expect on long run)
\begin{enumerate}
\item \textbf{For a Discrete R.V}: It is the average of R.V values based on their associated probabilities $E(X) = \Sigma (x_i \times p(x_i))$.
\item \textbf{For a Continuous R.V}: $E(X) = \int_{-\infty}^{\infty} x f(x) dx$ where $f(x)$ is probability density function.
\end{enumerate}

\section{Combining Random Variables}
If we have two independent random variables $X$ and $Y$, we can add or subtract to get the total or the difference of their means (expected values) and variances:
$T = X \pm Y \Rightarrow \mu_T = \mu_X \pm \mu_Y  ~~ and ~~ \sigma_T^2 = \sigma_X^2 \pm \sigma_Y^2$.

\section{The Law of Large Numbers}
Let $X$ be R.V of a population where its expected value (i.e Mean) is $E(X)$. Let $\overline{X_n} = \frac{x_1 + x_2 + ... x_n}{n}$ be the mean of $n$ samples $\in X$. As $n \to \infty$ : $\overline{X_n} \to E(X)$. Which means as we do more experiments, the outcomes become closer and closer to the outcomes of the probability theory.

\section{Statistical Distributions}\footnote{Included because it is related directly to probability}
\begin{enumerate}
\item \textbf{Binomial Distribution}: How many success in finite number of trials:
	\begin{itemize}
	\item Made up of \textit{independent} trials.
	\item Each trial has one of two \textit{discrete} outcomes (success or failure).
	\item \textit{Fixed} number of trials
	\item Probability of success in each trial is \textit{constant}.
	\item E.g. X = number of heads after 10 flips of a coin.
	\item \textit{10\% Rule}: We assume independence if the sample $\leq$ 10\% of the population.
	\item PMF: $P(X = x) = (_{n}^{k}) p^x (1 - p)^{n-x}$ which means the probability of $k$ successes in $n$ trials.
	\item $E(X) = \mu_X = n . p$ where $p$ is the probability of success per trial.
	\item $\sigma_X = \sqrt{var(X)} = \sqrt{n . p (1 - p)}$.
	\item The more trials we do, the more Binomial becomes Normal distribution.
	\end{itemize}

\item \textbf{Bernoulli Distribution}: Simply a Binomial Distribution with just \textit{one} trial.
\item \textbf{Geometric Distribution}: How many trials until success?. It is similar to Binomial dist. but here we do not have fixed number of trials because we do not know ahead how many trials until we get the desired outcome:
	\begin{itemize}
	\item PMF: $P(X = x) = (1 - p)^{k - 1} p$.
	\item $E(X) = \mu_X = \frac{1}{p}$.
	\item $\sigma_X = \sqrt{var(X)} = \sqrt{\frac{1 - p}{p^2}}$.
	\end{itemize}

\item \textbf{Hypergeometric Distribution}: It is a \textit{discrete} probability distribution that describes the probability of $k$ successes in $n$ draws, \textit{without} replacement, from a finite population of size $N$ that contains exactly $K$ objects. In contrast, the binomial distribution describes the probability of $k$ successes in $n$ draws \textit{with} replacement.

\item \textbf{Poisson Distribution}: It is a \textit{discrete} probability distribution that predicts \textit{rare} events that are independent of one another and occur with a known constant rate $\lambda$. It is usually used instead of Binomial dist. if the number of trials is very high (fixed interval of time) and the probability (occurrence) of each is relatively low:
	\begin{itemize}
	\item PMF: $P\left( x \right) = \frac{{\lambda ^x }}{{x!}} e^{ - \lambda } $.
	\item Mean is $E(X) = \lambda$. and Variance is $\lambda$.
	\end{itemize}

\item \textbf{Normal Distribution (a.k.a Gaussian)}: It is a \textit{continuous} probability distribution that has the notation  $X \sim \mathcal{N}(\mu,\,\sigma^{2})$. It is symmetric and has a bell shape where most of the values are near to the mean and follows the 68–95–99.7 rule (a.k.a empirical rule). This rule states that 68.27\%, 95.45\% and 99.73\% of the values lie within one, two and three standard deviations of the mean, respectively:
	\begin{itemize}
	\item PDF: $P(x) = \frac{1}{{\sigma \sqrt {2\pi } }}e^{{{ - \left( {x - \mu } \right)^2 } \mathord{\left/ {\vphantom {{ - \left( {x - \mu } \right)^2 } {2\sigma ^2 }}} \right. \kern-\nulldelimiterspace} {2\sigma ^2 }}}$.
	\item Mean is $E(X) = \mu_X$. and Variance is $\sigma_X^2$.
	\end{itemize}

\item \textbf{Log-Normal Distribution}: It is a \textit{continuous} probability distribution of a random variable whose \textit{logarithm is normally distributed}.Because the values in a log-normal distribution are positive, they create a right-skewed curve which is important in determining which distribution is appropriate to use in investment decision-making.  One of the most common applications of log-normal distributions is in the analysis of stock prices. A log-normal distribution can be translated to a normal distribution and vice versa using associated logarithmic calculations.

\item \textbf{Uniform Distribution}: It is a probability distribution that has \textit{constant} probability.
	\begin{enumerate}
	\item \textit{Discrete Uniform Dist.}:  It takes one of a finite $n$ possible values:
		\begin{itemize}
		\item PMF: $P(X = x) = \frac{1}{n}$
		\item $E(X) = \mu_X = \frac{(a + b)}{2}$
		\item $\sigma_X^2 =\frac{(b - a + 1)^2 ~ - 1 }{12}$
		\end{itemize}

	\item \textit{Continuous Uniform Dist.}: It takes values within a specified range $a$ to $b$:
		\begin{itemize}
		\item PDF:
		\begin{align*}
			P(X = x) = \begin{cases}  \frac{1}{b-a}~ if ~ x \in [a , b]\\ 0 ~ otherwise \end{cases}
		\end{align*}
		\item $E(X) = \mu_X = \frac{1}{2} (a+b)$
		\item $\sigma_X^2 =\frac{1}{12} (b - a)^2 $
		\end{itemize}

	\end{enumerate}

\item \textbf{Gamma Distribution}: It is a family of a \textit{continuous}, \textit{positive-only}, \textit{unimodal} distributions that encode the time required for $alpha$ events to occur in a Poisson process with mean arrival time of $beta$. These distributions are useful in real-life where something has a natural minimum of $0$. For example, it is commonly used in finance, for elapsed times, or during Poisson processes. In other words, it is a generalization of both the exponential and chi-squared distributions.

\item \textbf{Exponential Distribution}: Special case of Gamma Distribution. It is a \textit{continuous} analogue of the geometric distribution and often used to model the time elapsed between events. Formally, it is the probability distribution of the time between events in a process where events occur continuously and independently at a \textit{constant} average rate (i.e. Poisson process):
		\begin{itemize}
		\item PDF: $P(X = x) = {\lambda e}^{-\lambda x}$
		\item $E(X) = \mu_X = \lambda^{-1} = \beta$
		\item $\sigma_X^2 =\lambda^{-2} = \beta^2$
		\end{itemize}

\item \textbf{Chi-Square Distribution}:  It is a special case of the Gamma distribution. It is a \textit{continuous} distribution of a sum of the squares of $k$ independent standard normal deviates (where $k$ is known as degree of freedom).  A standard normal deviate is a \textit{random sample} from the standard normal distribution.
		\begin{itemize}
		\item PDF: $P(X = x) = {\frac{1}{2^{\frac{k}{2}}\Gamma(	\frac{K}{2})}}^{x^{\frac{k}{2}-1}~e^{\frac{-x}{2}}}$ where $\Gamma$ is the gamma function
		\item $E(X) = \mu_X = k$
		\item $\sigma_X^2 = 2k$
		\end{itemize}

\item \textbf{Student's t-Distribution}:  It is a probability distribution used to estimate population parameters when the sample size is small and standard deviation is unknown. It is mainly used by applying Student's t-test for assessing the statistical significance of the difference between two sample means, and in linear regression analysis.
		\begin{itemize}
		\item PDF: $$ \frac{\Gamma (\frac{\nu+1}{2})}{\sqrt{\nu \pi} \Gamma (\frac{\nu}{2})} (1 + \frac{x^2}{\nu})^{-\frac{\nu+1}{2}}
$$ where $\nu$ is the number of degrees of freedom and $\Gamma$ is the gamma function.
		\item $E(X) = \mu_X = 	0 ~ for ~ \nu > 1,$ otherwise undefined.
		\item $\sigma_X^2 = \frac{\nu }{\nu -2} ~for~ \nu >2, ~\infty ~for~ 1 < \nu \le 2,$ otherwise undefined.
		\end{itemize}
\end{enumerate}

\end{document}


\documentclass[10pt,letterpaper]{article}
\usepackage[dvipsnames]{xcolor}
\usepackage{outlines}
\usepackage{amsmath}
\usepackage{tikz}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{cancel}
\usepackage{subcaption}
\DeclareCaptionOptionNoValue{centering}{\centering} % Make sure everything is centered in subs
\captionsetup[sub]{centering}

\usepackage{algorithm}
\usepackage[noend]{algpseudocode}
\makeatletter
\def\BState{\State\hskip-\ALG@thistlm}
\makeatother

\usepackage{multirow}
\usepackage{cancel}
\usepackage{float}

\usepackage{parskip}

\usepackage{slantsc,lmodern}

\usepackage{pgfplotstable,booktabs}
\usepackage{framed}
\definecolor{shadecolor}{rgb}{0.9,0.9,0.9}

\usepackage{gensymb}

\usepackage{paralist}

\usepackage[paper=a4paper,margin=1in]{geometry}

\usepackage{etoolbox}

\newcommand{\volume}{{\ooalign{\hfil$V$\hfil\cr\kern0.08em--\hfil\cr}}}

\makeatletter
\g@addto@macro\@floatboxreset\centering
\makeatother

\author{Thaddeus Hughes \\ hughes.thad@gmail.com \\ thaddeus-maximus.github.io}
\date{\today}
\title{Documentation and Validation of EveryCalc's Trajectory Tool}

\begin{document}
	\maketitle

	\begin{abstract}
		Hurling projectiles is something we humans really like doing. We've become exceedingly efficient at it. We make sport of it. We can guess at a trajectory pretty easily- but nailing it down, and tweaking it with an engineering mindset is harder. Lots of tools already exist to do this but I want to roll my own so I can add all the physics I want, along with the swept area of a projectile, and stacking the tolerance.
	\end{abstract}

\section{Basic Projectile Motion}
	Consider a ball of mass $m$, in a vacuum. It has only the force of gravity acting on it. That is to say,

	\begin{align}
		\Sigma F_x &= m \frac{d v_x}{d t} = 0 \\
		\Sigma F_y &= m \frac{d v_y}{d t} = - m g
	\end{align}

	To figure out the path, we'd also need to know the initial conidtions. Let's also say that the ball was launched at an angle $\theta$ from the horizontal at an initial velocity of $\bar{v}_0$, so that the x- and y- velocities would be
	\begin{align}
		v_x(0) &= \bar{v}_0 \ cos(\theta) \\
		v_y(0) &= \bar{v}_0 \ sin(\theta) .
	\end{align}

	We'll start from a height of $y_0$ and at $x=-x_0$ from our target.

	\begin{align}
		x(0) = -x_0 \\
		y(0) = -y_0
	\end{align}

	This is enough to get us a very simple simulation for projectile motion:

	\begin{align}
		\frac{d v_x}{d t} &= 0 \\
		\frac{d v_y}{d t} &= - g \\
		\frac{d x}{d t} &= v_x \\
		\frac{d y}{d t} &= v_y \\
		v_x(0) &= \bar{v}_0 \ cos(\theta) \\
		v_y(0) &= \bar{v}_0 \ sin(\theta) \\
		x(0) &= -x_0 \\
		y(0) &= y_0 \\
		\text{terminate when } x &\geq 0
	\end{align}

\section{Swept Path with Parallel Curves}
	It's also worth knowing the swept zone that the target travels, since the object of firing a projectile may not be to hit a target per se, but to make it \textit{through} a target. This may seem like a trivial task at first blush; just add on the radius of the ball to the y-direction, but then one realizes the projetile may not be striking the target dead-on. Creating an offset path, or \href{https://en.wikipedia.org/wiki/Parallel_curve}{parallel curve} is necessary. The upper swept path ($x_u$, $y_u$) of a ball of radius $r$ can be determined as

	\begin{align}
		x_u = x + r (\hat{x} \cdot \hat{N}) \\
		y_u = y + r (\hat{y} \cdot \hat{N})
	\end{align}

	Where $\hat{x}$, $\hat{y}$, $\hat{N}$, $\hat{T}$ are unit vectors in the x-, y-, normal, and tangent directions.

	\begin{align}
		\text{let } \bar{v} &= \sqrt{v_x^2 + v_y^2} \\
		\hat{x} \cdot \hat{N} &= - v_x / \bar{v} \\
		\hat{x} \cdot \hat{T} &= + v_y / \bar{v} \\
		\hat{y} \cdot \hat{N} &= + v_x / \bar{v} \\
		\hat{y} \cdot \hat{T} &= + v_y / \bar{v}
	\end{align}

	The lower path would be found by reversing the direction of $r$, yielding

	\begin{align}
		x_l &= x - r (\hat{x} \cdot \hat{N}) \\
		y_l &= y - r (\hat{y} \cdot \hat{N}) .
	\end{align}

\section{Aerodynamic Effects}
	There are multiple aerodynamic forces that can act on a ball.

	\begin{align}
		F_{drag}   &= \frac{1}{2}   \ C_{drag} \ \rho \ A \ \bar{v}^2 \\
		F_{lift}   &= \frac{1}{2}   \ C_{lift} \ \rho \ A \ \bar{v}^2 \\
		F_{magnus} &= \frac{\pi}{3} \ C_{magnus} \ \rho \ A \ \bar{v} \ \omega_{\text{+ccw}}
	\end{align}

	The lift and drag forces are standard equations, but the magnus force equation is derived from \href{https://www.grc.nasa.gov/WWW/K-12/airplane/beach.html}{\underline{this NASA page}}. There are undoubtedly better models out there, but this is what I have currently.

	In vector form the conservation of momentum could be written as

	\begin{align}
		m \frac{d \vec{v}}{d t} = - F_{drag} \hat{T} + F_{lift} \hat{N} + F_{magnus} \hat{N} - m g \hat{y}
	\end{align}

	This changes the model, when projected out into the x- and y- components to

	\begin{align}
		\frac{d v_x}{d t} &= \frac{- F_{drag} (\hat{x} \cdot \hat{T}) + F_{lift} (\hat{x} \cdot \hat{N}) + F_{magnus} (\hat{x} \cdot \hat{N})}{m}\\
		\frac{d v_y}{d t} &= \frac{ - F_{drag} (\hat{y} \cdot \hat{T}) + F_{lift} (\hat{y} \cdot \hat{N}) + F_{magnus} (\hat{y} \cdot \hat{N})}{m} - g \\
		\frac{d x}{d t} &= v_x \\
		\frac{d y}{d t} &= v_y \\
		v_x(0) &= \bar{v}_0 \ cos(\theta) \\
		v_y(0) &= \bar{v}_0 \ sin(\theta) \\
		x(0) &= -x_0 \\
		y(0) &= y_0 \\
		\text{terminate when } x &\geq 0 .
	\end{align}

\section{Tolerance Stacking}
	To determine accuracy, multiple iterations of the simulation can be ran with different permutations of input variables.

\section{Reverse Computation}
	A \href{https://en.wikipedia.org/wiki/Bisection_method}{\underline{bisection algorithm}} is used to solve for the appropriate distance/angle/velocity required to propel the projectile into the target. This has benefits over analytical solutions in that it can be used in conjunction with aerodynamic effects.

\section{Validation Against Other Tools}

	I'll compare results to \href{https://www.chiefdelphi.com/t/amb-design-spreadsheet-v5/383857}{\underline{AMB's Design Spreadsheet}}.

	\newpage
	\subsection*{Case A: Metric units, no aero}

	\begin{figure}[H]
		\includegraphics[width=0.8\textwidth]{validation/trajectory_AMB_A.png}
	\end{figure}

	\begin{figure}[H]
		\includegraphics[width=0.65\textwidth]{validation/trajectory_EC_A.png}
	\end{figure}

	Look only at the "Forward" portions of AMB's sheet. Effectively the same result.

	\newpage
	\subsection*{Case B: English units, drag included}
	\begin{figure}[H]
		\includegraphics[width=0.8\textwidth]{validation/trajectory_AMB_B.png}
	\end{figure}

	\begin{figure}[H]
		\includegraphics[width=0.65\textwidth]{validation/trajectory_EC_B.png}
	\end{figure}

	Look only at the "Forward" portions of AMB's sheet. Effectively the same result.

\end{document}


\chapter{First principle methods}

In this section we present our proposed methods for causal inference in the
bivariate setting: The twin test and the residual method. We will introduce each method,
provide a detailed algorithmic description and end by giving a proof of correctness.

\section{The twin test}

Let us recall the setup:
suppose we are given samples $\mathcal{D} = \{x_i, y_i\}_{i \in [n]}$ from an ANM $X \rightarrow Y$, which has the form

\[
    \begin{cases}
        & Y = f(X) + Z \\
        & X \bigCI Z,\quad X \thicksim P_X,\quad Z \thicksim P_{Z}
     \end{cases}
\]

The main strategy of of the ANM methods has been to estimate $f$, and then to compute the estimated
residual $\hat{\mathbf{e}} = \hat{f}(\mathbf{x}) - \mathbf{y}$; the final step is to test the independence between $\hat{\mathbf{e}}$ and $\mathbf{x}$. This exploits
the assumption that $X \bigCI Z$. In practice we often have that the noise is \textit{independent} between each sample
i.e. we produce a sequence $Y_1, ..., Y_n$, where $Z_i \bigCI Z_j$ $\forall i \neq j$; for example when we are
taking measurements, the additive noise of our devices tends to be independent between each sample.

By directly exploiting the i.i.d noise assumption, we will circumvent the need for an independence test. The
idea is to to partition the data ---  for simplicity you can think about splitting it around the median; we
then estimate the residuals for each partition, and we then test if the i.i.d noise assumption holds by comparing
the residuals of each partition. We can apply this procedure to both directions and then we will call the
direction causal if its residuals are more similar.

We explain this idea in more detail by following an example: we are given samples
$\mathcal{D} = \{x_i, y_i\}_{i \in [n]}$ from an ANM $X \rightarrow Y$;
We first visualize the data $\{x_i, y_i\}_{i \in [n]}$, by plotting
X to Y, and vice versa (see Figure \ref{fig:algo_data}).

\begin{figure}[H]
    \captionsetup[subfigure]{labelformat=empty}
    \centering
    \subfloat{{\includegraphics[width=6.5cm]{algo_direct.png} }}%
    % \quad
    \subfloat{{\includegraphics[width=6.5cm]{algo_reverse.png} }}%

    \caption{300 samples of data $X$, $Y$. With \textbf{ANM}: \\
    $f(x) = \tanh(x) + 2\sin(2x) + x^3$}
    $X \thicksim \mathcal{U}_{[-a, a]}$ and $Z \thicksim \mathcal{N}(0, \sigma^2)$
    \label{fig:algo_data}
\end{figure}

For simplicity assume $X \thicksim \mathcal{U}_{[-a, a]}$ (i.e. $X$ is uniformly disributed), then we can split
the data in two\footnote{ The question of how to split the data is important, and we will address this problem in
more detail later, but for now, given the assumptions, splitting it in half is not such a bad idea; another
question one might ask is the following: how many intervals? We will clarify all this later.}
, say $D_1$ and $D_2$, where we place all samples with $x_i < 0$ into $D_1$, and the rest into
$D_2$. To be more precise, $\mathcal{D}_1 = \{(x_i, y_i) : x_i < 0\}$ and $\mathcal{D}_2 = \mathcal{D}
\backslash \mathcal{D}_1$. We also do the same procedure for the reverse set up, i.e. we reverse the roles of x and y
, $\mathcal{\tilde{D}} = \{y_i, x_i\}_{i \in [n]}$ and by the same procedure we obtain $\mathcal{\tilde{D}}_1$
and $\mathcal{\tilde{D}}_2$. We can visualize this partition bellow in Figure \ref{fig:colo_code}.

\begin{figure}[H]
    \captionsetup[subfigure]{labelformat=empty}
    \centering
    \subfloat[Partitions $D_1$ and $D_2$]{{\includegraphics[width=6.5cm]{algo_part.png} }}%
    % \quad
    \subfloat[Partitions $\mathcal{\tilde{D}}_1$ and $\mathcal{\tilde{D}}_2$]{{\includegraphics[width=6.5cm]{algo_part_reverse.png} }}%
    \caption{ We highlight each partition in a different color. On the left we have $D_1$ and $D_2$; and on
    the right we have $\mathcal{\tilde{D}}_1$ and $\mathcal{\tilde{D}}_2$ }
    \label{fig:colo_code}
\end{figure}

If we estimate a fit $\hat{f}_1$ for $\mathcal{D}_1$ and similarly $\hat{f}_2$ for $\mathcal{D}_2$, then we can
compute residuals for each sets, say $\hat{e}_1$ for $\mathcal{D}_1$  and $\hat{e}_2$ for $\mathcal{D}_2$.
Since the noise is ---  not only independent from $X$ but also ---  i.i.d, it follows that $\hat{e}_1$ and $\hat{e}_2$
follow the same distribution ---  assuming a perfect fit $f$. We can visualize this by looking at the
histograms from the residuals.

\begin{figure}[H]
    \captionsetup[subfigure]{labelformat=empty}
    \centering
    \subfloat{{\includegraphics[width=6.5cm]{algo_fit_direct.png} }}%
    % \quad
    \subfloat{{\includegraphics[width=6.5cm]{algo_fit_reverse.png} }}%
    % \caption{Samples from two different sources, $X$ and $Y$, how can we tell if
    % they come from the same distribution?}
    \\

    \subfloat{{\includegraphics[width=6.5cm]{algo_res_direct.png} }}%
    % \quad
    \subfloat{{\includegraphics[width=6.5cm]{algo_res_reverse.png} }}%

    \caption{  We show the the estimated fits $\hat{f}$ for each partition in black. Below each
    partition we plot the histograms of the residuals ---  in the same color. }
    \label{fig:algo_fit}
\end{figure}

Note that for the reverse model, the noise in $\mathcal{\tilde{D}}_1$ appears to be very different from
that of $\mathcal{\tilde{D}}_2$; this is not a coincidence ---  intuitively, it seems very unlikely that
regressing in the other direction will also result in independence noise. Further, as we briefly
mentioned in the early chapters, as \cite{hoyer2009nonlinear} show, it is unlikely that for a non-linear $f$
we might not have identifiability.

One simple idea is then to quantify these observations; from $\mathcal{D}_1$ and $\mathcal{D}_2$ we
compute $\hat{e}_1$, $\hat{e}_2$, and so we can define as a score for these sets:

$$
     \mathcal{C}(\mathcal{D}_1, \mathcal{D}_2) = \norm{p_1 - p_2}_1
$$

Where $p_1$ is the empirical distribution of $\hat{e}_1$, and similarly for $p_2$ and $\hat{e}_2$. We
can then apply the score function to $\mathcal{\tilde{D}}_1$ and $\mathcal{\tilde{D}}_2$ and
infer causality as follows:

\[
     \begin{cases}
        & X \rightarrow Y \quad  \mathcal{C}(\mathcal{D}_1, \mathcal{D}_2)
        \leq \mathcal{C}(\mathcal{\tilde{D}}_1, \mathcal{\tilde{D}}_2) \\
        & Y \rightarrow X \quad \text{otherwise}
     \end{cases}
\]

In the above example, we get that $\mathcal{C}(\mathcal{D}_1, \mathcal{D}_2) = 0.138$ and that
$\mathcal{C}(\mathcal{\tilde{D}}_1, \mathcal{\tilde{D}}_2) = 0.480$ where use bins of size $5$
for discretization; We are able to predict the causal direction with high confidence.

Assuming the regressions are \textbf{suitable}, then as $n \rightarrow \infty$ we know that both
$p_1$ and $p_2$ will converge to the same $p_Z$ and so $\mathcal{C}(\mathcal{D}_1, \mathcal{D}_2) \rightarrow 0$.
On the other hand, it is unlikely that the residuals of $\mathcal{\tilde{D}}_1$ and $\mathcal{\tilde{D}}_2$
follow the same distribution (due to the non-linearlities introduced by f) and
so we can be pretty confident that asymptotically the procedure will correct. In fact, assuming that \textbf{ANM}
$X \rightarrow Y$ is identifiable will be enough to show that this procedure is consistent.

In essence the algorithm consists of the parts:

\begin{enumerate}
    \item Partition the data
    \item Estimate regressions and residuals for each partition
    \item Compute scores between partition
\end{enumerate}

We note that the algorithm is a general framework as we are free to choose the partition, regression method
and score function. As in the ANM methods, one can either form a train/test split to learn the regression, or
instead recycle the data. The algorithmic description can be found in Algorithm \ref{alg:twin_test}.

We will next describe in more detail the core parts of the algorithm.

\subsection{Partition}

Say that we partition
$\mathcal{D}$ into disjoint sets $\mathcal{D}_1, ..., \mathcal{D}_k$; then partition
partitions need to satisfy two requirements:

\begin{enumerate}
    \item The partitions need to be \textbf{dense}:
    $|\mathcal{D}_i| \geq \rho |\mathcal{D}|, \quad \forall i \in [k]$, for some $\rho \in (0, 1)$.
    % \item By disjoint partitions we mean that
    % $$
    % \mathcal{D} = \bigcup_i \mathcal{D}_i \quad \text{and} \quad
    % \mathcal{D}_i \cap \mathcal{D}_j = \emptyset \quad \forall i \neq j
    % $$
    \item We need to be able to order the partitions, say $\mathcal{D}_1, ..., \mathcal{D}_k$, such that
    if $i < j$ then\footnote{The $*$ is to indicate a dummy variable, as we do not care for the value of $y$.}:
    $$
    \operatorname{max} \{ x : (x, *) \in \mathcal{D}_i \} \leq
    \operatorname{min} \{ x : (x, *) \in \mathcal{D}_j \}
    $$
\end{enumerate}

The first condition ---  that of dense partitions ---  is to avoid getting trivial large deviations between residuals
in the subsets; the second reason is that if they are dense, then we can give asymptotic guarantees about each subset.
The second condition simply ensure that we are not mixing data and that it is coherent to make regression
in each subset.

If we use K-means (perhaps the most popular clustering algorithm), then condition 2 are met. The only
doubt is in regards to condition 1. K-means starts by randomly initializing two or n centers (depending
on the number of clusters that we want), and the updates the centers that they locally
minimizes within-cluster variances. If our data is infinite support, and we re-run K-means if there is some cluster
$i$ s.t. $|\mathcal{D}_i| < \rho |\mathcal{D}|$; then if we have enough data and for some $\rho$ we can be quite
certain that the algorithm will eventually terminate.

In practice this has always been the case; we conjecture that the following statement or a slight variant holds true:
given $n$ samples $ \mathcal{D} = x_1, ..., x_n$ from some random variable $X$
that is "well behaved"\footnote{By well behaved we mean that the data should have enough spread, if it is too
concentrated then partitioning will be hard.}, by running k-means once we get two partitions,
$\mathcal{D}_1$ and $\mathcal{D}_2$; we can
restart k-means $r$  times with a different initialization to obtain $\mathcal{D}_1(1), ...\mathcal{D}_1(r)$ and
$\mathcal{D}_2(1), ...\mathcal{D}_2(r)$. Let $s_j = \operatorname{min} \left(\mathcal{D}_1(j), \mathcal{D}_2(j)\right)$

then there is some $\rho > 0$ s.t.

$$
    \Prob \left( \operatorname{max}_j s_j \geq \rho n \right) \rightarrow 1
$$

as $n \rightarrow \infty$, with $r = \log(n)$.

The last question is, "How many clusters do we want?". Obviously for the small data regime we must be content
with only two clusters; but what if we have a lot of data? Experimentally we observed that if we
we choose the number of partition as an increasing function w.r.t. sample size, then we can get better performance.

One crucial aspect is that the benchmark is in the 1-D setting, where clustering is of reasonable difficulty;
in higher dimensions clustering is a much harder problem.

\subsection{Regression}

A benefit of partitioning the is that we are also partitioning the function we are trying to estimate; in
particular one would expect that the regression will be easier, e.g. a low order polynomial might be enough.

For regression we try two different methods:

\begin{enumerate}
    \item Polynomial regression: We perform model selection based on the BIC\footnote{
    The BIC score is defined as follows: $\mathrm{BIC} =k\ln(n)-2\ln({\widehat{L}})$; where $\widehat{L}$
    is the the maximized value of the likelihood function of the model, $n$ the number of samples and
    $k$ the number of parameters in model. In essence it is a score that
    trades off model fit with model complexity. } score. We pick the model with the
    best BIC score with degree at most 6.
    \item Neural networks: We perform regression with a one layer 100 neuron network with ReLu activation
    function, and trained on Adam.
\end{enumerate}

\begin{figure}[H]
    \centering
    \includegraphics[width=.45\textwidth]{partition_smooth.png}
    \caption{An example of how partitioning the data may make the regression task easier.}
\end{figure}

\subsection{Score functions}

We have seen in previous chapters various ways to measure the distances between two distributions say
$p_1$ and $p_2$, via some score function $d(p_1, p_2)$; for example $d$ could be the
MMD metric, or the $l_1$ distance.

Now instead we have a set of distributions, say $P_k = \{p_1, ..., p_k\}$, and we wish to see how homogenous\footnote{
    By homogenous set we simply mean one in which its elements resemble each other, which is precisely what we will try to measure.
}
$P_k$ is compared to some other set $\tilde{P}_j$ ---  recall that we wish to see in which of the two,
 the distributions are more likely to be the same, i.e. we are testing the i.i.d assumption.

There are several simple ways to go about this:

$$
    C(P_k) = \operatorname{max}_{i, j} d(p_i, p_j)
$$

Another option is to take an average of the pairwise score:

$$
    C(P_k) = \frac{1}{\binom{k}{2}} \sum_{i < j} d(p_i, p_j)
$$

or even

$$
    C(P_k) = \frac{1}{\binom{k}{2}} \sum_{i < j} d(p_i, p_\mu)
    , \quad p_\mu = \frac{1}{k} \sum_i p_i
$$

Thus if $C(P_k) < C(\tilde{P}_j)$, we can say that the distributions in $P_k$ are
more homogenous; e.g. they more likely stem from the same noise distribution.

We have tested all of the above and find that the first method ---  using the maximum score between pairs ---
gives the best performance.

\subsection{Proof of consistency}

We show consistency for a simple set up of the twin test ---  but we note that generalizing it to the more
general case should follow with little effort from our proof. We leave it as an exercise for the reader.

The setup was the linear \textbf{ANM}:

\[ \begin{cases}
    & Y = f(X) + Z  \\
    & X \bigCI Z,\; X \thicksim p_x,\; Z \thicksim p_{Z}
 \end{cases}
\]

In practice we are given samples $\mathcal{D} = \{x_i, y_i\}_{i \in [n]}$ from an ANM $X \rightarrow Y$; next
the algorithm will proceed to split the data into sets $\mathcal{D}_1$ and $\mathcal{D}_2$. It then proceeds
to compute residuals and to compute some scores between them.

To simplify the proof, we will \textbf{skip the partition procedure} and assume that
we are directly given $\mathcal{D}_1$ and $\mathcal{D}_2$, each with
$n$ samples (Note that, if $p_x$ is uniform, then having these sets be of equal size would happen exponentially
fast; indeed, in general, we will have dense partitions as $n$ becomes arbitrarily large).

Next, we will assume that on each interval, the data is linear with slope $a_1$ and $a_2$ resp.

\begin{figure}[!h]
    \centering
    \begin{tikzpicture}
        \draw[thin, dashed] (1.5, -0.5) --  (1.5, 2.6) node[above] {};
        \path (0.75, -0.8) -- (0.75, -0.6) node[above] {$\mathcal{D}_1$};
        \path (2.25, -0.8) -- (2.25, -0.6) node[above] {$\mathcal{D}_2$};
        \draw[->] (-1, 0) -- (4.2, 0) node[right] {$x$};
        \draw[->] (0, -1) -- (0, 3) node[above] {$y$};
        \draw[scale=1, domain=0:1.5, smooth, variable=\x, blue] plot ({\x}, {0.5*\x});
        \draw[scale=1, domain=1.5:3, smooth, variable=\y, red]  plot ({\y}, {1*\y - 0.75});
        \draw[decoration={brace,mirror,raise=5pt},decorate] (0.1, -.5) -- node[below=8pt] {$\mathcal{D}$} (2.9, -.5) ;
      \end{tikzpicture}
      \caption{Slopes $a_1$ and $a_2$ in blue and red respectively.}
\end{figure}

Thus our problem can be seen as getting data from two different \textbf{ANM}, both with identical noise, but with
a different scaling of $X$:

$\mathcal{D}_1$ is sampled from

\[ \begin{cases}
    & Y_1 = a_1 X_1 + Z  \\
    & X_1 \bigCI Z,\; X_1 \thicksim p_{X_1},\; Z \thicksim p_{Z}
 \end{cases}
\]

and $\mathcal{D}_2$ is sampled from

\[ \begin{cases}
    & Y_2 = a_2 X_2 + Z  \\
    & X_2 \bigCI Z,\; X_2 \thicksim p_{X_2},\; Z \thicksim p_{Z}
 \end{cases}
\]

We call this scenario the \textbf{simplified Twin Test scenario}.

We next describe the steps of the algorithm after partitioning:

We first split $\mathcal{D}_1$ in two sets of equal size $\mathcal{D}_{1}^{train}$ and
$\mathcal{D}_{1}^{test}$. We first use $\mathcal{D}_{1}^{train}$ to estimate $a_1$, say $\hat{a}_1$ via
regression. Then, using $\mathcal{D}_{1}^{test}$, we estimate the residual:

$$
    \hat{Z}_1 = Y_1 - \hat{a}_1 X
$$

We then discretize $\hat{Z}_1$ and form a distribution say $\hat{p}_1$; we can do the same thing for
$\mathcal{D}_2$, and by doing the same procedure obtain $\hat{p}_2$.

We discretize both with a fixed step size, say $s$; as we will see, the only requirement is that we fix
the size beforehand.

We use the $l_1$ distance as our score:

$$
    \hat{C}_{X \rightarrow Y} = \norm{\hat{p}_1 - \hat{p}_2}_1
$$

As we have seen, $\hat{C}_{Y \rightarrow X} > 0$ holds in general except in very particular situations. So,
to prove that our algorithm is consistent, we need to show that:

$$
    \hat{C}_{X \rightarrow Y} \rightarrow 0 \qquad \text{as }  \qquad n \rightarrow \infty \qquad \text{in probability}
$$

This is precisely what we will show:

\begin{theorem}
    The \textbf{simplified Twin Test scenario} is consistent, i.e.

$$
    \hat{C}_{X \rightarrow Y} \rightarrow 0 \qquad \text{as }  \qquad n \rightarrow \infty \qquad \text{in probability}
$$

Assuming that the noise distribution satisfies the conditions of lemma \ref{lemma:conv_bound}.

\end{theorem}

The assumption about the noise distribution holds for most distributions, for example uniform,
exponential, ...

The idea of the proof is to observe the following:

If we have enough data, i.e. when $n$ is large enough then assuming the regression is
\textbf{suitable}, we can choose any $\alpha$ such that (for the linear regression, one can show that
the events below happen in probability),

$$
    \abs{a_1 - \hat{a}_1} \leq \alpha \qquad \text{and} \qquad \abs{a_2 - \hat{a}_2} \leq \alpha
$$

This means that

$$
    \hat{Z}_1 = Z + (a_1 - \hat{a}_1)X \quad \implies \quad Z - \alpha X \leq \hat{Z}_1 \leq Z + \alpha X
$$

and similarly

$$
    \hat{Z}_2 = Z + (a_2 - \hat{a}_2)X \quad \implies \quad Z - \alpha X \leq \hat{Z}_2 \leq Z + \alpha X
$$

Note that $\hat{Z}_1 \thicksim P_{Z + \Delta_1 X}$ and $\hat{Z}_2 \thicksim P_{Z + \Delta_2 X}$ , where for
brevity we denote $\Delta_1 = a_1 - \hat{a}_1$ and $\Delta_2 = a_2 - \hat{a}_2$. We can visualize the distance
between these distributions as follows: (see\footnote{Note that the illustration does not follow the actual
geometry of the space, we draw it solely to gain intuition about the problem.} figure \ref{fig:dist})

\begin{figure}[!h]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[thick] (0,0) circle (1);
        \draw[semithick, fill=black] (0,0) circle (0.02) node[left] {$P_Z$};
        \draw[thick, <->] (0, .05) -- node[right] {$R(\alpha)$} (0, .95);
        \draw[semithick, fill=black] (0,1) circle (0.02) node[above] {$P_{Z + \alpha X}$};
        % p_1 and p_2
        \draw[semithick, fill=black] (.5, -.5) circle (0.02) node[above, color=red] {$P_{Z + \Delta_2 X}$};;
        \draw[semithick, fill=black] (-.58, .114) circle (0.02) node[above, color=blue] {$P_{Z + \Delta_1 X}$};;
      \end{tikzpicture}
      \caption{The $l_1$ "ball" around $P_Z$, for brevity we denote
      $\Delta_1 = a_1 - \hat{a}_1$ and $\Delta_2 = a_2 - \hat{a}_2$}
      \label{fig:dist}
\end{figure}

The idea is then the following, given some $\epsilon > 0$, we want to show that asymptotically

$$
    \norm{\hat{p}_1 - \hat{p}_2}_1 > \epsilon
$$

cannot happen. The game plan will be to find a bound on the $R(\alpha)$; once we have one, we are done, we
will simply pick an $\alpha$ s.t. $R(\alpha) < \epsilon$, where $R(\alpha) = \norm{P_{Z}- P_{\alpha X + Z}}_{1}$
. Recall that $\alpha$ is the error in of our $\hat{a}$
estimate, which we can get arbitrarily small with enough samples.

We begin by proving lemmas to find bounds for the radius $R(\alpha)$.

\begin{lemma}

    Given $\alpha > 0$, random variables $Z$ and $X$, with $X \bigCI Z$,
$X \thicksim P_X$, $Z \thicksim P_Z$, such that\footnote{
    we remark that this
    condition holds for most distributions such as the uniform, exponential and Gaussian.} there is
    some $g$ s.t.
    $ \forall \theta \in (-\sqrt{\alpha}, \sqrt{\alpha}), \; \left| P_{Z}^{\prime}(t - \theta) \right| \leq g(t) \; \forall t$
    and $\int \left| g(t) \right| d t < L$, for some $L > 0$.
 then

$$
\norm{P_{Z}- P_{\alpha X + Z}}_{1} \leq \sqrt{\alpha} L + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right)
$$

Where $P_{\alpha X + Z}$ is the distribution of the sum: $\alpha X + Z \thicksim  P_{\alpha X + Z}$.

\label{lemma:conv_bound}
\end{lemma}

\begin{proof}
    ~

First note that $\alpha X \thicksim \frac{1}{\alpha}P_{X} \left( \frac{\tau}{\alpha} \right)$ by applying the
change of variable rule. Let $P_{\alpha X + Z}$ be the distribution of the sum: $\alpha X + Z \thicksim  P_{\alpha X + Z}$.

Next, since $X \bigCI Z$, we may write $P_{\alpha X + Z}$ as a convolution:

$$
    P_{\alpha X + Z} (t) = \int P_{Z}(t-x) \frac{1}{\alpha}P_{X} \left( \frac{x}{\alpha} \right) d x =
     \int P_{Z}(t-\alpha x) P_{X}(x) d x
$$

Let $T^{\alpha X} P_Z(t) := P_{Z}(t-\alpha x)$ (we use the notation introduced by Lagrange for the shift operator).

Hence we may write (with $P_X$ as the underlying measure):

$$
    P_{\alpha X + Z} (t) = \E\left(P_{Z}(t-\alpha X ) \right)
$$

We proceed as follows:

$$
\begin{aligned}
\norm{P_{Z}- P_{\alpha X + Z}}_{1} &=\int\abs{ P_{Z}(t)-\E\left(P_{Z}(t-\alpha X ) \right)} d t \\
&\leq \int \E \abs{ P_{Z}(t)-P_{Z}(t-\alpha X )} d t \\
&=\E \int\abs{ P_{Z}(t)-P_{Z}(t-\alpha X ) } d t \\
&= \E \left( \norm{P_{Z}- T^{\alpha X} P_Z }_{1} \, | \, \abs{X} \leq \frac{1}{\sqrt{\alpha}} \right)
\Prob\left( \abs{X} \leq \frac{1}{\sqrt{\alpha}} \right) + \E \left( \norm{P_{Z}- T^{\alpha X} P_Z }_{1} \, | \, \abs{X} > \frac{1}{\sqrt{\alpha}} \right)
\Prob\left( \abs{X} > \frac{1}{\sqrt{\alpha}} \right) \\
&\leq \E \left( \norm{P_{Z}- T^{\alpha X} P_Z }_{1} \, | \, \abs{X} \leq \frac{1}{\sqrt{\alpha}} \right)
 + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right) \\
&\leq \operatorname{max}_{\eta \in (-\sqrt{\alpha}, \sqrt{\alpha})} \int\abs{ P_{Z}(t)-P_{Z}(t-\eta ) } d t + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right) \\
&\leq \operatorname{max}_{\eta \in (-\sqrt{\alpha}, \sqrt{\alpha})} \int \abs{ \eta g(t) } d t  + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right)\\
% &\leq \int\abs{ P_{Z}(t)- \left( P_{Z}(t) + \sum_{n \geq 1} \left( \alpha C^* \right)^n P_{Z}(t)^{(n)} \right) } d t  + 2 P(\abs{ X} > \frac{1}{\sqrt{\alpha}})\\
% &\leq \alpha C^* L + 2 P(\abs{ X} > \frac{1}{\sqrt{\alpha}})\\
&\leq \sqrt{\alpha} \int \abs{ g (t)} dt + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right)\\
&\leq \sqrt{\alpha} L + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right)
\end{aligned}
$$

The first equality follows by the aforementioned observation. The first inequality follows from the triangle
inequality;
the equality that comes after is due to Fubini's theorem, we can swap the expectation (which is also an
integration) since all measures are measurable. We next use the law of total probability by splitting
the expectation w.r.t $\frac{1}{\sqrt{\alpha}}$.

The next upper bounds follows by noting that
$\norm{p - q}_1 \leq 2$ for any distributions $p$ and $q$. Next observe that
$\E \left( \norm{P_{Z}- T^{\alpha X} P_Z }_{1} \, | \, \abs{X} \leq \frac{1}{\sqrt{\alpha}} \right)$ is the average $l_1$ distance
between $\norm{P_{Z}}$ and random shifts of itself. Hence we can bound this by the maximum shift.

Since $\eta \approx 0$, and since
$  \eta \in (-\sqrt{\alpha}, \sqrt{\alpha}) \implies  \left| P_{Z}^{\prime}(t - \eta) \right| \leq g(t) \; \forall t$
, using Taylor's theorem we obtain that:

$$
    \abs{P_{Z}(t- \eta ) - P_{Z}(t) }  \leq \eta g(t)
$$

The rest follows from the assumption that $\int g(t) dt \leq L$.

% % Using taylor expansions we obtain

% % $$
% %     P_{Z}(t-\alpha C^* ) = P_{Z}(t) + \sum_{n \geq 1} \left( \alpha C^* \right)^n P_{Z}(t)^{(n)}
% % $$

% % then  we upper bound what we get after as follows

% % \begin{align*}
% %     \int \left| \sum_{n \geq 1} \left( \alpha C^* \right)^n P_{Z}(t)^{(n)} \right|
% %     &\leq L \sum_{n \geq 1} \left( \alpha C^* \right)^n \\
% %     &\leq L \frac{\alpha C^*}{1 - \alpha C^*} \\
% %     &\approx \alpha C^* L
% % \end{align*}

% % We will later pick $\alpha \ll 1$, and so it follows from $\frac{x}{1 - x} \approx x$ for small $x$.

% Finally, since are free to pick $k$, we choose it large enough s.t. $2P(\abs{ X} > k) \leq \delta$, we then let
% $C = C^* L$ and we are done.

% Note that as $k$ increases so does $C^*$, since we are growing the space to optimise for $C^*$; this is
% where the dependence between $\delta$ and $C$ comes from.

\end{proof}

So we have found a bound on the $l_1$ distance between two continuous distributions; however in our
application, these will be quantized versions of these distributions. The following lemma tells us
that this is not a problem, the $l_1$ distance between the quantized version cannot be bigger than
that of their continuous counter parts. The only requirement is that we fix the quantization scheme
beforehand and use the same one for both.

\begin{lemma}
    Let $P$ and $Q$ be two continuous distributions, then let $P^*$, $Q^*$ resp. be discretized versions.
    Then

    $$
        \norm{P^* - Q^*}_1 \leq \norm{P - Q}_1
    $$
    \label{lemma:quantisation}
\end{lemma}

\begin{proof}
    ~
    We first quantize $\R$ in bins of length $w$, say $I_i = [wi, w(i + 1))$, note
    $\bigcup_{i \in \Z} I_i = \R$.

    Given continuous distributions $P$ and $Q$, we form their quantized counter parts as
    follows:

    $$
        P^*(k) := \sum _{i \in \Z} \; \int_{I_i} P(t) dt \; \Ind_{k  = i}
        , \quad   Q^*(k) := \sum _{i \in \Z} \; \int_{I_i} Q(t) dt \; \Ind_{k  = i}
    $$

    We then conclude as follows by a applying the triangle inequality twice:

    \begin{align*}
        \norm{P^* - Q^*}_1^2 &= \sum _{k \in \Z} \; \left| P^*(k) - Q^*(k)\right|  \\
        &\leq \sum _{k \in \Z} \sum _{i \in \Z} \; \left| \int_{I_i} \left(P(t) - Q(t)\right) dt \right|  \; \Ind_{k  = i} \\
        &\leq \sum _{k \in \Z} \sum _{i \in \Z} \;  \int_{I_i} \left|P(t) - Q(t)\right| dt  \; \Ind_{k  = i} \\
        &= \sum _{i \in \Z} \;  \int_{I_i} \left|P(t) - Q(t)\right| dt  \; \sum _{k \in \Z} \Ind_{k  = i} \\
        &= \sum _{i \in \Z} \;  \int_{I_i} \left|P(t) - Q(t)\right| dt \\
        &= \int_{\R} \left|P(t) - Q(t)\right| dt \\
        &= \norm{P - Q}_1^2
    \end{align*}

\end{proof}

We can now conclude by proving consistency, recall that we want to show that:

$$
    P \left(\norm{\hat{p}_1 - \hat{p}_2}_1 > \epsilon \right) \rightarrow 0
$$

\begin{proof}

\begin{align*}
        \Prob\left( \norm{ \hat{p}_1 - \hat{p}_2 }_1 \geq \epsilon \right) &\leq
        \Prob\left( \norm{ \hat{p}_1 -  P^*_Z}_1 + \norm{ \hat{p}_2 -  P^*_Z}_1 \geq \epsilon \right) \\
        &\leq \Prob\left( \norm{ \hat{p}_1 -  P^*_Z}_1 + \norm{ \hat{p}_2 -  P^*_Z}_1 \geq \epsilon
        \; | \; \abs{a_1 - \hat{a}_1} \leq \alpha \; \text{,} \; \abs{a_2 - \hat{a}_2} \leq \alpha \right) \\
                        &\qquad + \Prob\left( \abs{a_1 - \hat{a}_1} > \alpha \; \text{or} \; \abs{a_2 - \hat{a}_2} > \alpha  \right)
\end{align*}

Where $P^*_Z$ is the quantized version of $P_Z$.

The first inequality follows by the triangle inequality, and the second one by using the law of total probability.

Observe that

$$
\Prob\left( \abs{a_1 - \hat{a}_1} > \alpha \; \text{or} \; \abs{a_2 - \hat{a}_2} > \alpha  \right)  =
\Prob\left( \abs{a_1 - \hat{a}_1} > \alpha \right) + P \left( \abs{a_2 - \hat{a}_2} > \alpha  \right)
$$

Both of which go to zero for any $\alpha > 0$ assuming that our regression is \textit{suitable}.

It remains to bound

\begin{equation}
    \Prob\left( \norm{ \hat{p}_1 -  P^*_Z}_1 + \norm{ \hat{p}_2 -  P^*_Z}_1 \geq \epsilon
        \; | \; \abs{a_1 - \hat{a}_1} \leq \alpha \; \text{,} \; \abs{a_2 - \hat{a}_2} \leq \alpha \right)
\end{equation}

Note that $\hat{p}_1 \rightarrow P^*_{Z + \Delta_1 X}$ as $n \rightarrow \infty$; where
$P^*_{Z + \Delta_1 X}$ is the discretized distribution of $P_{Z + \Delta_1 X}$; recall that
we form $\hat{p}_1$ by creating a discretized histogram from the residuals.

Then, by combining lemma \ref{lemma:conv_bound} and \ref{lemma:quantisation} we have that:

For any $\alpha$, there is some $L > 0$ s.t.
$$
    \norm{P^*_{Z} - P^*_{Z + \Delta_1 X}}_1 \leq \sqrt{\alpha} L + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right)
$$

Thus if we are given some $\epsilon > 0$, pick\footnote{We can do this since $\sqrt{\alpha}$ is increasing
in $\alpha$ and $\Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right)$ is decreasing in $\alpha$.}
some $\alpha$ such that

$$
    \sqrt{\alpha} L + 2 \Prob \left(\abs{ X} > \frac{1}{\sqrt{\alpha}} \right) < \frac{\epsilon}{2}
$$

By applying the same idea to $\hat{p}_2$, we get that as $n \rightarrow \infty$

$$
   \norm{\hat{p}_1 -  P^*_Z}_1 + \norm{ \hat{p}_2 -  P^*_Z}_1 < \frac{\epsilon}{2} + \frac{\epsilon}{2} = \epsilon
$$

And so as $n \rightarrow \infty$

$$
    \Prob\left( \norm{ \hat{p}_1 -  P^*_Z}_1 + \norm{ \hat{p}_2 -  P^*_Z}_1 \geq \epsilon
        \; | \; \abs{a_1 - \hat{a}_1} \leq \alpha \; \text{,} \; \abs{a_2 - \hat{a}_2} \leq \alpha \right)
        \rightarrow 0
$$

\end{proof}

\subsection{Algorithm}

\begin{algorithm}[H]

    \caption{\textbf{Twin Test (TT)}: General procedure to decide whether $P_{X, Y}$ satisfies and ANM $X \rightarrow Y$
        or $Y \rightarrow X$}

    \textbf{Input}:

    \begin{enumerate}
        \item I.i.d samples $\mathcal{D} = \{ (x_i, y_i )\}_{i \in [N]}$ of $X$ and $Y$
        \item Partition procedure
        \item Regression method
        \item Score estimator $\hat{C}: R^{* \times *} \rightarrow \R$, where E is a set of vectors.
    \end{enumerate}

    \textbf{Output}: $\hat{C}_{X \rightarrow Y}$, $\hat{C}_{Y \rightarrow X}$, dir

    \begin{enumerate}

        \item $\tilde{\mathcal{D}} := \{ ( y_i, x_i )\}_{i \in [N]}$

        \item \textbf{Partition} the data into subsets\footnote{The subsets are disjoint and their union equals the data}:
        \begin{itemize}
            \item[--] $\{ \mathcal{D}_i \}_{i \in [k]} \quad \text{s.t.} \quad \mathcal{D}_i \subset \mathcal{D},
            \forall i \in [k]$
            \item[--] $\{ \tilde{\mathcal{D}}_i \}_{i \in [j]} \quad \text{s.t.} \quad \tilde{\mathcal{D}}_i
            \subset \tilde{\mathcal{D}}, \forall i \in [j]$
            \item[--] Where integers $j, k > 1$ are determined by the partition procedure.
        \end{itemize}

        \item \textbf{Estimate regressions} and residuals for each subset

        for $i \in [k]:$

        \begin{itemize}
            \item[--] Let $\mathbf{x}$, $\mathbf{y}$ be the vectors formed from $\mathcal{D}_i$
            \item[--] $\hat{f}_Y$ of the regression function $x \mapsto \E(Y | X=x)$
            \item[--] $ \mathbf{\hat{e}_{Y}}(i) := \mathbf{y} - \hat{f}_Y(\mathbf{x})$
        \end{itemize}

        end for

        $\mathbf{E_Y} := \{ \mathbf{\hat{e}_{Y}}(i) \}_{i \in [k]} $

        for $i \in [j]:$

        \begin{itemize}
            \item[--] Let $\mathbf{x}$, $\mathbf{y}$ be the vectors formed from $\tilde{\mathcal{D}}_i$
            \item[--] $\hat{f}_X$ of the regression function $y \mapsto \E(X | Y=y)$
            \item[--] $ \mathbf{\hat{e}_{X}}(i) := \mathbf{x} - \hat{f}_X(\mathbf{y})$
        \end{itemize}

        end for

        $\mathbf{E_X} := \{ \mathbf{\hat{e}_{X}}(i) \}_{i \in [j]} $

        \item \textbf{Compute scores} to measure the difference between the residuals
        \begin{itemize}
            \item[--] $\hat{C}_{X \rightarrow Y}: = \hat{C}( \mathbf{E_Y} )$
            \item[--] $\hat{C}_{Y \rightarrow X}: = \hat{C}( \mathbf{E_X} )$
        \end{itemize}

        \item Output $\hat{C}_{X \rightarrow Y}$, $\hat{C}_{Y \rightarrow X}$, and

        \[
        \text{dir} :=
         \begin{cases}
            & X \rightarrow Y \quad \text{if} \; \hat{C}_{X \rightarrow Y} \leq \hat{C}_{Y \rightarrow X}\\
            & Y \rightarrow X \quad \text{otherwise}
         \end{cases}
        \]

    \end{enumerate}

  \label{alg:twin_test}
  \end{algorithm}

\newpage
\section{The residual method}

When given an ANM $X \rightarrow Y $, the traditional method based on ANM is to regress $X$ on
$Y$ and then vice versa in order to see which residual is more independent from it's input. A
very basic --- but restrictive --- idea is to assume knowledge about the additive noise, $P_Z$.
In some sense this idea was the precursor to the twin test, where we check if the noise is consistent
in the different intervals of the data. Instead here, since we assume knowledge about $P_Z$, we
will test and see which residual is more likely to be drawn from the actual noise distribution $P_Z$.

This is however quite a strong assumption, so it is more of a theoretical curiosity.

We start again we typical setup:
suppose we are given samples $\mathcal{D} = \{x_i, y_i\}_{i \in [n]}$ from an ANM $X \rightarrow Y$, which has the form

\[
    \begin{cases}
        & Y = f(X) + Z \\
        & X \bigCI Z,\quad X \thicksim P_X,\quad Z \thicksim P_{Z}
     \end{cases}
\]

In addition, we will know $P_{Z}$, in some cases it is not such an unreasonable assumption; for example,
a lot of thermal noise in measurements is usually very well modeled by a Gaussian. Another example
is when a real value x from a sensor, is discretized with a uniform quantizer, the error is
likely to be uniformly distributed. (\cite{uniformCond})

For the ANM methods, the first step was to

1. Regress $\mathbf{x}$ on $\mathbf{y}$, to find an estimate say $\hat{f}_Y$

2. Estimate residual via $\mathbf{\hat{e}}_Y = \hat{f}_Y(\mathbf{x}) - \mathbf{y}$

The next step was to then compute the same thing for the reverse model, that is, swapping the
roles of $\mathbf{x}$ and $\mathbf{y}$. We would then compute some score of independence
between the residual and their respective inputs ($\mathbf{x}$ for the direct model And
 $\mathbf{y}$ for the reverse).

 Since we have knowledge of $P_Z$ we can avoid this last step all together, and instead simply
 compute a score to see how close $\mathbf{\hat{e}}_Y$ is to $P_Z$.

 One simple idea is the following:

 1. $\mathbf{b} :=$ histogram of $\mathbf{\hat{e}}_Y$

 2. $P^*_Z :=$  discretized distribution of $P_Z$

 For both we pick the same discretization size, say $m$.

 Our score is then
 $
     \hat{C}_{X \rightarrow Y} := d\left(\mathbf{b}, P^*_Z \right)
 $
 Where $d$ is some statistical distance, such as an f-divergence. We compute the score for reverse model
 by swapping the roles of $\mathbf{x}$ and $\mathbf{y}$.

\subsection{Proof of consistency: A tale of two bounds}

The setup was the linear \textbf{ANM}:

\[ \begin{cases}
    & Y = aX + Z  \\
    & X \bigCI Z,\; X \thicksim P_X,\; Z \thicksim P_{Z}
 \end{cases}
\]

Given data $\mathcal{D} = \{x_i, y_i\}_{i \in [n]}$ from this ANM, we
estimate $\hat{f}_Y$ by regressing $X$ on $Y$ and
$\hat{f}_X$ for the reverse model. We then compute the residuals (as always we can
either recycle data or perform a test/train partition).

\begin{align}
    &  \hat{\mathbf{e}}_Y = \mathbf{y} - \hat{f}_Y(\mathbf{x})\\
    &  \hat{\mathbf{e}}_X = \mathbf{x} - \hat{f}_X(\mathbf{y})
\end{align}

We note that for the ease of analysis, it would first be wise to use some fraction
of the data to first estimate the regression, and then use the remaining for the test.

The idea is the very simple, test which of the residuals $\hat{\mathbf{e}}_Y$ or
$\hat{\mathbf{e}}_X$ is more likely to be distributed according to $P_Z$.
To ease computation in the bounds we will assume $P_Z$ to be the uniform
distribution, but we remark that the analysis will hold in general.

To do so we first discretize\footnote{We do so in a naive manner by splitting
it uniformly into $m$ bins.} $P_{Z}$ into $m$ bins, call this discrete distribution
$Q$. We apply the same discretization to obtain $\mathbf{b} = (b_1, ..., b_m)$ from $\hat{\mathbf{e}}_Y$
and $\tilde{\mathbf{b}} = (\tilde{b}_1, ..., \tilde{b}_m)$ from $\hat{\mathbf{e}}_X$.

We then decide the causal direction as follows

\[ \begin{cases}
    & X \rightarrow Y \quad \text{if} \quad C \leq W  \\
    & Y \rightarrow X \quad \text{if} \quad C > W
 \end{cases}
\]

Where

$$
    C = \norm{\mathbf{b} - \mathbf{u}}_{1}
$$
$$
    W = \norm{\tilde{\mathbf{b}} - \mathbf{u}}_{1}
$$

s.t. $\mathbf{u} = (\frac{1}{m}, ..., \frac{1}{m})$.

Given our assumption about the \textbf{ANM}, the probability to output the correct causal direction is:

$$
   P_{\text{correct}} = \mathbb{P}\left(C \leq W\right)
$$

We will assume that we have perfect regression estimates to simplify the proof; as we have seen in the
previous chapter, with a little bit of work we can incorporate the error terms of the regression in
the probability of correctness.

We next upper bound this quantity in order to show consistency

\begin{align}
    \Prob\left(C \leq W\right) &\geq \Prob\left( \underset{\tau \in \mathbb{Q}}{\bigcup} C \leq \tau \cap W > \tau \right) \\
    &\geq \Prob\left(C \leq \tau \cap W > \tau \right) \\
    &\geq \Prob\left(C \leq \tau \right) - \Prob\left(W \leq \tau \right)
\end{align}

The first inequality is due to the fact that we are only taking the union in the rationals\footnote{We note that
we can only take unions over countable sets; recall also that the rationals are dense in the irrationals, so the
inequality is very close to equality (and in practice and among friends it would be).}. The second inequality is done by
looking at the probability of a fixed $\tau$; and the final one follows by:

$$
    1 \geq \Prob\left( C \leq \tau \cup W > \tau \right) =
    \Prob\left( C \leq \tau \right) + \Prob\left( W > \tau \right) - \Prob\left( C \leq \tau \cap W > \tau \right)
$$

We will next find appropriate bounds for $\Prob\left( C \leq \tau \right)$ and $\Prob\left(W \leq \tau \right)$.

We will first lower bound $\Prob\left( C \leq \tau \right)$ by upper bounding the complement event.

\begin{align}
    \Prob\left( C \geq \tau \right) &= \Prob\left( \sum_{i = 1}^{m} \abs{b_i - \frac{1}{m}} \geq \tau \right)  \\
    &\leq \Prob\left( m \operatorname{max}_{i} \abs{b_i - \frac{1}{m}} \geq \tau \right)  \\
    &= \Prob\left( \bigcup_{i} \; \abs{b_i - \frac{1}{m}} \geq \frac{\tau}{m} \right)  \\
    &\leq m \; \Prob\left( \abs{b_0 - \frac{1}{m}} \geq \frac{\tau}{m} \right) \\
    &\leq m2\exp \left( -2n \frac{\tau^2}{m^2} \right)
\end{align}

The second to last inequality follows by the union bound and by noting that all $b_i$s are the same since they
are discretized empirical distribution coming from a uniform source. For the final inequality we use Hoeffding's
inequality.

Recall that what is left to bound is the following quantity, $\Prob\left(W \leq \tau \right)$;
for this we first define the following set of probability distributions:

$$
    \Gamma_\tau = \{ \pi \in  \Delta_m : \norm{\pi - U}_{L_1} \leq \tau \}
$$

Where the $\Delta_m$ is the $m$ dimensional simplex and $U$ the uniform vector as before.

Observe that:

$$
    \{ W \leq \tau \} = \{ \tilde{\mathbf{b}} \in \Gamma_\tau \}
$$

In essence, we are asking: "what is the chance that the realization of $\tilde{\mathbf{b}}$ --- which is the
empirical distribution of some distribution $Q$ --- lies inside some set of distributions $\Gamma_\tau$.

We note that bounding this kind of event is exactly what Sanov's theorem\footnote{See the section on Information Theory and statistics in
\cite{cover1999elements}} gives us, an important
result from large deviation theory that also exploits concentration of measure.

Let $\mathbf{x} = (x_1, ..., x_n)$ be a sequence of $n$ each drawn independently from
a finite universe $U$ with $|U| = m$. Denote by $P_\mathbf{x}$ the empirical distribution ---
or type --- for a given sequence $\mathbf{x}$. Let $Q^{n}$ be the product distribution $n$
independent samples of $Q$.

\begin{theorem}[Sanov’s theorem]\label{sanov}

    Let $\Pi$ be a convex set of distributions on $U,$ and $m=|U| .$ Let

    \[
        P^{*}=\operatorname{argmin}_{P \in \Pi} D(P \| Q)
    \]

    Then
    \[
        \underset{Q^{n}}{\Prob}\left(P_{\mathbf{x}} \in \Pi\right) \leq(n+1)^{m} 2^{-nD\left(P^{*} \| Q\right)}
    \]

\end{theorem}

Applying the above theorem, and noting that $\Gamma_\tau$ takes the place of $\Pi$, $\tilde{\mathbf{b}}$ that of $P_{\mathbf{x}}$
and the discretized distribution $\hat{e}_X = X - \hat{f}_X(Y)$ that of $Q$ we get:

\begin{equation}
    \Prob\left(W \leq \tau \right) = \Prob\left( \tilde{\mathbf{b}} \in \Gamma_\tau \right)
    \leq (n+1)^{m} 2^{-nD\left( \tau \right)}
\end{equation}

Where $D\left( \tau \right) := D\left(P^{*} \| Q\right)$, we make the $\tau$ relation explicit to
keep in mind that the minimization is constrained to the set $\Gamma_\tau$ which depends on $\tau$.

We remark that the only place of concern is if $D\left(P^{*} \| Q\right) = 0$; assuming however that $Q \neq U$, then
there will be some $\tau$ s.t. $Q \notin \Gamma_\tau$ and thus $D\left(P^{*} \| Q\right) \neq 0$.

We can now conclude by putting everything together; recall that we had shown that we could bound the success probability
as follows:

\begin{align}
    \Prob\left(C \leq W\right) &\geq \Prob\left(C \leq \tau \right) - \Prob\left(W \leq \tau \right) \\
    &\geq 1 - 2m\exp \left( -2n \frac{\tau^2}{m^2} \right) - (n+1)^{m} 2^{-nD\left( \tau \right)}
\end{align}

This, if we fix $m$, and if there exists some $\tau$ s.t. $D\left( \tau \right) > 0$ then we get consistency
by letting $n \rightarrow \infty$.

We note that to get the best bound we may maximizes the right hand side w.r.t. $\tau$.

\subsection{Algorithm}

\begin{algorithm}[H]

    \caption{\textbf{Residual method}: Method to decide whether $P_{X, Y}$ satisfies and ANM $X \rightarrow Y$
        or $Y \rightarrow X$ , for an ANM given that the additive noise $P_Z$ is known.}

    \textbf{Input}:

    \begin{enumerate}
        \item I.i.d samples $\mathcal{D} = \{ (x_i, y_i )\}_{i \in [N]}$ of $X$ and $Y$
        \item Noise distribution $P_Z$
        \item Regression method
        \item Score estimator $\hat{C}: R^{m \times m} \rightarrow \R$
    \end{enumerate}

    \textbf{Output}: $\hat{C}_{X \rightarrow Y}$, $\hat{C}_{Y \rightarrow X}$, dir

    \begin{enumerate}

        \item \textbf{Estimate regressions}

        \begin{itemize}
            \item[--] $\hat{f}_Y$ of the regression function $x \mapsto \E(Y | X=x)$
            \item[--] $\hat{f}_X$ of the regression function $y \mapsto \E(X | Y=y)$
        \end{itemize}

        \item \textbf{Estimate residuals}

        \begin{itemize}
            \item[--] $ \mathbf{\hat{e}_{Y}} := \mathbf{y} - \hat{f}_Y(\mathbf{x})$
            \item[--] $ \mathbf{\hat{e}_{X}} := \mathbf{x} - \hat{f}_X(\mathbf{y})$
        \end{itemize}

        \item \textbf{Discrete distribution}

        \begin{itemize}
            \item[--] $P^*_Z :=$ discretized distribution of $P_Z$
            \item[--] $b :=$ histogram of $ \mathbf{\hat{e}_{Y}}$
            \item[--] $\tilde{b} :=$ histogram of $ \mathbf{\hat{e}_{X}}$
        \end{itemize}

        \item \textbf{Compute scores} to measure the difference between the residuals
        \begin{itemize}
            \item[--] $\hat{C}_{X \rightarrow Y}: = \hat{C}( b, P^*_Z  )$
            \item[--] $\hat{C}_{Y \rightarrow X}: = \hat{C}( \tilde{b}, P^*_Z  )$
        \end{itemize}

        \item Output $\hat{C}_{X \rightarrow Y}$, $\hat{C}_{Y \rightarrow X}$, and

        \[
        \text{dir} :=
         \begin{cases}
            & X \rightarrow Y \quad \text{if} \; \hat{C}_{X \rightarrow Y} \leq \hat{C}_{Y \rightarrow X}\\
            & Y \rightarrow X \quad \text{otherwise}
         \end{cases}
        \]

    \end{enumerate}

  \label{alg:residual}
  \end{algorithm}