%
%  untitled
%
%  Created by Johan Boissard [] on 2010-02-27.
%  Copyright (c) Johan Boissard. All rights reserved.
% hhh
\documentclass[a4paper]{amsart}

% Packages
\usepackage[french]{babel}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{comment}

%special math typos - see page 65 (table 196) of comprehensive latex tut
\usepackage{amsmath,amssymb,mathrsfs}
%allows to write on whole page
\usepackage{fullpage}

%package to display degree symbol
\usepackage{gensymb}

%package for page numbering and footer
\usepackage{fancyhdr}
%to get last page
\usepackage{lastpage} % \pageref{LastPage}
%allows inclusion of graphics
\usepackage{graphics}
\usepackage{ulem}
\DeclareGraphicsExtensions{.pdf, .jpeg,.jpg}
%allows drawing
%ref:http://www.texample.net/
\usepackage{tikz}

%allows inclusion of url (hyperref is better than url)
%ref: http://www.fauskes.net/nb/latextips/
\usepackage{hyperref}

%package for chemistry ie: \ce{(NH4)2SO4 -> NH4+ + 2SO4^2-}
%ref:www.ctan.org/tex-archive/macros/latex/contrib/mhchem/mhchem.pdf
\usepackage[version=3]{mhchem}

%comments (begin{comment})
\usepackage{comment}

% Title
\title{BioFluids}
\author{Johan Boissard}
\date\today

\pagestyle{fancyplain}
%delete current header & footer configuration
\fancyhf{}
\renewcommand{\headrulewidth}{0pt}
%\fancyhead[EL]{Titel} %Kopfzeile links
%\fancyhead[C]{} %zentrierte Kopfzeile
%\fancyhead[LE,RO]{Name} %Kopfzeile rechts
%head separation line
%\renewcommand{\headrulewidth}{0.4pt}
%foot separation line
%\renewcommand{\footrulewidth}{0.4pt}
%page number
\fancyfoot[OL]{\thepage\ of \pageref{LastPage}}
%\fancyfoot[ER]{\thepage}

% Header
\setlength{\voffset}{0pt}
\setlength{\topmargin}{0in}
\setlength{\headheight}{0in}
\setlength{\headsep}{0in}

\begin{document}
%\maketitle
\twocolumn

\section{Fluid Properties}
\subsection{Fluid properties}
\subsubsection{Density}
\begin{eqnarray*}
	\rho=\frac{m}{V}&
	\begin{cases}
		\rho_\text{air}=1.23 [\text{kg/m$^3$}]\\
		\rho_\text{water}=999 [\text{kg/m$^3$}]
	\end{cases}
\end{eqnarray*}
\subsubsection{Specific Volume}
\begin{eqnarray*}
	\dot v=\frac{1}{\rho}
\end{eqnarray*}
\subsubsection{Specific weight}
\begin{eqnarray*}
	\gamma=\rho g
\end{eqnarray*}
\subsubsection{Specific gravity}
\begin{eqnarray*}
	SG=\frac{\rho}{\rho_{H_20@4\degree}}
\end{eqnarray*}
\subsection{Ideal Gas Law}
\begin{eqnarray*}
	p=\rho RT
\end{eqnarray*}
$p$ absolute pressure, $\rho$ density, $R$ gas constant (dependant of fluid), $T$ absolute Temperature.
\begin{eqnarray*}
	&\text{Absolute pressure} \\&=& \\
	&\underbrace{\text{Atmospheric pressure}}_{101 [\text{kPa}]}
	 + \text{gage pressure}
\end{eqnarray*}

\begin{comment}
	The behavior of a flowing fluid depends on various fluid properties. Viscosity, one
	of the important properties, is responsible for the shear force produced in a moving
	fluid.
	Although the two fluids shown look alike (both are clear liquids and have a specific
	gravity of 1), they behave very differently when set into motion. The very viscous
	silicone oil is approximately 10,000 times more viscous than the water
	\subsection{No-slip condition}
	As a fluid flows near a solid surface, it "sticks" to the surface, i.e., the fluid matches the
	velocity of the surface. This so-called "no-slip" condition is a very important one that must
	be satisfied in any accurate analysis of fluid flow phenomena.
	Dye injected at the bottom of a channel through which water is flowing forms a stagnant
	layer near the bottom due to the noslip condition. As the dye filament is moved away from
	the bottom, the motion of the water is clearly apparent. A significant velocity gradient is
	created near the bottom.
\end{comment}
\subsection{Fluidity}

\subsection{Viscosity}
Speed profile:
\begin{eqnarray*}
	u(y)=U\frac{y}{b}
\end{eqnarray*}
Rate of shearing strain
\begin{eqnarray*}
	\dot \gamma = \frac{U}{b}=\frac{du}{dy}
\end{eqnarray*}
Shearing stress
\begin{eqnarray*}
	\tau \propto \dot\gamma\\
	\tau=\mu\frac{du}{dy}=\frac{F}{A}\\
	T=r\times F
\end{eqnarray*}

Newtonian Fluids: $\mu$ is constant
\\
Non-Newtonian Fluids: $\mu$ is not constant and $\tau\propto\dot\gamma$ is no longer applicable.
\subsubsection{Sutherland Equation}
\begin{eqnarray*}
	\mu=\frac{CT^{3/2}}{T+S}
\end{eqnarray*}
\subsubsection{Andrade's Equation}
\begin{eqnarray*}
	\mu=De^{B/T}\\
	\ln{\mu}=\ln{D}+\frac{B}{T}
\end{eqnarray*}

\subsubsection{Kinematic viscosity}
\begin{eqnarray*}
	v=\frac{\mu}{\rho}
\end{eqnarray*}

%%%%%%% 2
\section{Fluid statics}
\textbf{Pressure is equal at every point in a fluid at rest}

\begin{eqnarray*}
	\frac{\partial p}{\partial x}=\frac{\partial p}{\partial y}=0\\
	\frac{\partial p}{\partial x}=\frac{dp}{dz}=-\gamma=-\rho g\\
	p=\gamma h+p_0
\end{eqnarray*}
\subsection{Hydrostatic Force on a Plane Surface}
\begin{eqnarray*}
	F_R=\gamma h_cA&h_c=\text{centroid}\\
	y_R=\frac{I_{xc}}{h_cA}+h_c & I_x=I_{xc}+Ah_c^2\\
	I_{xc}=\frac{1}{12}h^3w
\end{eqnarray*}

\subsection{Buoyancy}
\begin{eqnarray*}
	\mathbf{F_B}=\rho gV_{\text{immerged}}\\
	F=A\cdot P & (\text{exos})\\
	P_iV_i=P_fV_f
\end{eqnarray*}

%%%% 3
\section{Elementary Fluid Dynamics - The Bernoulli Equation}
\subsection{Bernoulli Equation}
\begin{eqnarray*}
	\frac{P}{\rho}+\frac{V^2}{2}+gz=\text{constant along a streamline}
\end{eqnarray*}
\begin{eqnarray*}
	\frac{P}{\rho}+\int\frac{V^2}{R}dn+gz=\text{constant across a streamline}
\end{eqnarray*}
\subsection{Continuity Equation}
\begin{eqnarray*}
	Q=Q_i=A_iV_i&[m^3/s]&i=1,2,\dots
\end{eqnarray*}

%%%%%%%
\section{Fluid Kinematics}
\subsection{Velocity Fields}
\begin{eqnarray*}
	\frac{dy}{dx}=\frac{v}{u}
\end{eqnarray*}
\subsection{Material Derivative}
\begin{eqnarray*}
	\frac{D( )}{Dt}=\frac{\partial ( )}{\partial t}+(\mathbf{V}\cdot\nabla)( )
\end{eqnarray*}
\subsection{Acceleration}
\begin{eqnarray*}
	\mathbf{a}
	&=&
	\begin{pmatrix}
		a_x\\
		a_y\\
		a_z\\
	\end{pmatrix}
	\nonumber\\
	&=&\frac{D \mathbf{V}}{Dt}=a_s\hat s+a_n\hat n\nonumber\\
	&=&
	\underbrace{\frac{\partial \mathbf{V}}{t}}_{\text{local}}
	+\underbrace{\mathbf{V}\cdot\nabla( \mathbf{V} )}_{\text{convective}}\nonumber\\
	&=&\frac{\partial \mathbf{V}}{t}+
	\begin{pmatrix}
		u\\
		v\\
		w
	\end{pmatrix}
	\cdot
	\begin{pmatrix}
		\frac{\partial\mathbf{V}}{\partial x} &
		\frac{\partial\mathbf{V}}{\partial y} &
		\frac{\partial\mathbf{V}}{\partial z}
	\end{pmatrix}
	\nonumber\\
	&=&\frac{\partial \mathbf{V}}{t}+u\frac{\partial\mathbf{V}}{\partial x}+v\frac{\partial \mathbf{V}}{\partial y}+w\frac{\partial\mathbf{V}}{\partial z}\nonumber\\
	&=&
	\begin{pmatrix}
		\frac{\partial u}{\partial t} + u\frac{\partial u}{\partial x}+ v\frac{\partial u}{\partial y}+ w\frac{\partial u}{\partial z}\\
		\frac{\partial v}{\partial t} + u\frac{\partial v}{\partial x}+ v\frac{\partial v}{\partial y}+ w\frac{\partial v}{\partial z}\\
		\frac{\partial w}{\partial t} + u\frac{\partial w}{\partial x}+ v\frac{\partial w}{\partial y}+ w\frac{\partial w}{\partial z}

	\end{pmatrix}
\end{eqnarray*}
with $\mathbf{V}=(u,v,w)$
\subsubsection{Acceleration in streamline coordinates}
\begin{eqnarray*}
	\mathbf{a}=V\frac{\partial V}{\partial s}\mathbf{\hat s}+\frac{V^2}{R}\mathbf{\hat n}\\
	a_s=V\frac{\partial V}{\partial s}\\a_n=\frac{V^2}{R}
\end{eqnarray*}

\subsection{Reynolds Transport Theorem (RTT)}
%\includegraphics[width=.4]{img1}
\begin{eqnarray*}
	\frac{D B_{sys}}{Dt}=\frac{\partial B_{CV}}{\partial t}+\int_{CS}\rho b \mathbf{v}\cdot\mathbf{n}dA
\end{eqnarray*}

\section{Finite Control Volume Analysis}
\subsection{Continuity Equation}
In RTT, $B=m$ and $b=B/m=1$
\begin{eqnarray*}
	\frac{D M_{sys}}{Dt}&=&0\\
	%\frac{D}{Dt}\int_{sys}\rho dV&=&\frac{\partial}{\partial t}\int_{CV}\rho dV+\int_{CS}\rho \mathbf{V}\cdot \mathbf{\hat n}dA
	\frac{\partial}{\partial t}\int_{CV}\rho dV+\int_{CS}\rho \mathbf{V}\cdot \mathbf{\hat n}dA&=&0
\end{eqnarray*}

\subsection{Linear Momentum Equation}
\begin{eqnarray*}
	\frac{\partial}{\partial t}\int_{CV}\mathbf{V}\rho dV+\int_{CS}\mathbf{V}\rho \mathbf{V}\cdot \mathbf{\hat n}dA=\sum \mathbf{F_{\text{ext}}}
\end{eqnarray*}

\subsection{Energy Equation}
\begin{eqnarray*}
	\frac{\partial}{\partial t}\int_{CV}e\rho dV
	+
	\int_{CS}e\rho \mathbf{V}\cdot \mathbf{\hat n}dA
	=
	\left(\dot Q_{\text{net}}+\dot W_{\text{net}}\right)_{CV}
	\\\dot =\text{[J/s]}\dot =\text{[F$\cdot$m/s]}\dot =\text{[kg$\cdot$m/s$^2$]}
\end{eqnarray*}
\begin{eqnarray*}
	\frac{\partial}{\partial t}\int_{CV}e\rho dV
	+
	\int_{CS}\left(\check u+\frac{p}{\rho}+\frac{v^2}{2}+gz\right)\rho \mathbf{V}\cdot \mathbf{\hat n}dA
	\\
	=
	\left(\dot Q_{\text{net}}+\dot W_{\text{net}}\right)_{CV}
\end{eqnarray*}
\begin{eqnarray*}
	\dot m
	[
		\check u_{out}
		-\check u_{in}
		+
			\left(\frac{p}{\rho}\right)_{out}
		-
			\left(\frac{p}{\rho}\right)_{in}\\
		+
			\frac{1}{2}
				\left(V_{out}^2-V_{in}^2\right)
		+
			g(z_{out}-z_{in})
	]
	\\
	=\dot Q_{net}+\dot W_{\text{net}}
\end{eqnarray*}

\begin{eqnarray*}
	\frac{P_1}{\rho}+\frac{v_1^2}{2}+gz_1
	+{w_{\text{shaft}}}
	=\frac{P_2}{\gamma}+\frac{v_2^2}{2g}+gz_2
	+\text{loss}&
	\text{ [m$^2$/s$^2$]}
	\\
	\frac{P_1}{\gamma}+\frac{v_1^2}{2g}+z_1
	+\underbrace{\frac{\dot W_{s}}{\gamma Q}}_{h_s}
	=\frac{P_2}{\gamma}+\frac{v_2^2}{2g}+z_2
	+\underbrace{\frac{\text{loss}}{g}}_{h_l}
\end{eqnarray*}
\begin{eqnarray*}
	\text{turbine head}&h_T=-(h_s+h_L)_T\\
	\text{pump head}&h_P=(h_s-h_L)_P
\end{eqnarray*}
\begin{eqnarray*}
	v=\frac{\int v^2\rho \mathbf{v}\cdot\mathbf{\hat n}dA}{\overline v A}
\end{eqnarray*}

\onecolumn
\newpage
\twocolumn
\section{Differential Analysis of Fluid Flow}
\subsection{Volumetric dilatation rate}
\begin{eqnarray*}
	\frac{1}{V}\frac{d(\delta V)}{dt}=\mathbf{\nabla}\cdot \mathbf{V}
	\\
	\mathbf{\nabla}\cdot \mathbf{V}=0\Rightarrow&\text{Incompressible fluid}
\end{eqnarray*}

\subsection{Vorticity}
\begin{eqnarray*}
	\mathbf{J}=\nabla\times \mathbf{V}&\\
	\nabla\times \mathbf{V}=0&\Rightarrow\text{Irrotational fluid}
\end{eqnarray*}
Irrotational, Bernoulli applies otherwise Euler.

\subsection{Rate rotation Vector}
\begin{eqnarray*}
	\mathbf{\omega}=\frac{1}{2} \mathbf{J}
\end{eqnarray*}

\subsection{Rate of angular deformation}
\begin{eqnarray*}
	\dot \gamma=\frac{\partial v}{\partial x} + \frac{\partial u}{\partial y}
\end{eqnarray*}

\subsection{Stream Function}
\begin{eqnarray*}
	u=\frac{\partial \psi}{\partial y}
&&	v=-\frac{\partial \psi}{\partial x}
\\
	v_r=\frac{1}{r}\frac{\partial \psi}{\partial \theta}
&&	v_\theta=-\frac{\partial \psi}{\partial r}
\end{eqnarray*}
\subsubsection{Rate of flow}
\begin{eqnarray*}
	Q=\Psi(B) - \Psi(A) & \text{[m$^3$/s]}
\end{eqnarray*}
\subsection{Velocity Potential}
\begin{eqnarray*}
	\mathbf{v}=\nabla\phi
\end{eqnarray*}
\subsubsection{Conservation of Mass}
\begin{eqnarray*}
	\nabla\cdot \mathbf{V}=\nabla^2\phi=0
\end{eqnarray*}

\subsection{Euler's equations of Motion}
\begin{eqnarray*}
	\rho \mathbf{g}-\nabla p=\rho\left(\frac{\partial \mathbf{v}}{\partial t}+(\mathbf{v}\cdot\nabla)\mathbf{v}\right)=\rho\frac{D \mathbf{v}}{Dt}
\end{eqnarray*}

\subsection{Navier-Stokes Equations}
\subsubsection{Cartesian Coordinates}
\begin{eqnarray*}
	\rho \mathbf{g}-\nabla p+\mu\nabla^2 \mathbf{v}=\rho\left(\frac{\partial \mathbf{v}}{\partial t}+(\mathbf{v}\cdot\nabla)\mathbf{v}\right)
\end{eqnarray*}

\subsubsection{Shearing stresses}
\begin{eqnarray*}
	\begin{cases}
		\tau_{xy}=\mu\left(\frac{\partial u}{\partial y}+\frac{\partial v}{\partial x}\right)\\
		\tau_{yz}=\mu\left(\frac{\partial v}{\partial z}+\frac{\partial w}{\partial y}\right)\\
		\tau_{xz}=\mu\left(\frac{\partial w}{\partial x}+\frac{\partial u}{\partial z}\right)
	\end{cases}
\end{eqnarray*}
%\begin{comment}

\subsubsection{Cylindrical Coordinates}
\begin{eqnarray*}
	r:
	\rho g_r
	-\frac{\partial p}{\partial r}
	\\
	\mu \Bigg[\frac{1}{r}\frac{\partial}{\partial r}\left(r \frac{\partial u_r}{\partial r}\right)
+ \frac{1}{r^2}\frac{\partial^2 u_r}{\partial \phi^2}
	+ \frac{\partial^2 u_r}{\partial z^2}-\frac{u_r}{r^2}-\frac{2}{r^2}\frac{\partial u_\phi}{\partial \phi}\Bigg]
	\\=
	\rho\left(
	\frac{\partial u_r}{\partial t} + u_r \frac{\partial u_r}{\partial r} + \frac{u_{\phi}}{r} \frac{\partial u_r}{\partial \phi}
	+ u_z \frac{\partial u_r}{\partial z} - \frac{u_{\phi}^2}{r}
	\right)
	\\
	%%%
	\phi:
	\rho g_{\phi}
	-\frac{1}{r}\frac{\partial p}{\partial \phi}
	\\
	+ \mu
		\left[\frac{1}{r}\frac{\partial}{\partial r}\left(r \frac{\partial u_{\phi}}{\partial r}\right)
		+ \frac{1}{r^2}\frac{\partial^2 u_{\phi}}{\partial \phi^2}
		+ \frac{\partial^2 u_{\phi}}{\partial z^2}
		+ \frac{2}{r^2}\frac{\partial u_r}{\partial \phi}
		- \frac{u_{\phi}}{r^2}\right]
	\\
	=
	\rho\left(
		\frac{\partial u_{\phi}}{\partial t}
		+ u_r \frac{\partial u_{\phi}}{\partial r}
		+ \frac{u_{\phi}}{r} \frac{\partial u_{\phi}}{\partial \phi}
		+ u_z \frac{\partial u_{\phi}}{\partial z}
		+ \frac{u_r u_{\phi}}{r}\right)
	\\
	%%%
	z:
	\rho g_z
	-\frac{\partial p}{\partial z}
	\\
	+ \mu \left[\frac{1}{r}\frac{\partial}{\partial r}\left(r \frac{\partial u_z}{\partial r}\right) + \frac{1}{r^2}\frac{\partial^2 u_z}{\partial \phi^2} + \frac{\partial^2 u_z}{\partial z^2}\right]
	\\=
	\rho
	\left(\frac{\partial u_z}{\partial t}
	+ u_r \frac{\partial u_z}{\partial r}
	+ \frac{u_{\phi}}{r} \frac{\partial u_z}{\partial \phi}
	+ u_z \frac{\partial u_z}{\partial z}\right)
\end{eqnarray*}
%\end{comment}

\subsubsection{Nabla operator in cylindrical coordinates}
\begin{eqnarray*}
	\nabla=(
	{\partial f \over \partial \rho},
	{1 \over \rho}{\partial f \over \partial \phi},
	{\partial f \over \partial z}\boldsymbol{\hat z})
	&
	r=\sqrt{\rho^2+z^2}
\end{eqnarray*}

\subsubsection{Poiseuille's Law}
\begin{eqnarray*}
	Q=\frac{\pi R^4\Delta p}{8\mu l}\\
	\overline v = \frac{R^2\Delta p}{8\mu l}&Q=\pi R^2\\
	v_{\text{max}}=-\frac{R^2}{4\mu}\frac{\partial p}{\partial z}=\frac{R^2\Delta p}{4\mu l}
	& v_{\text{max}}=2\overline v\\
	\frac{v_z}{v_{\text{max}}}=1-\left(\frac{r}{R}\right)^2
\end{eqnarray*}

\section{Dimensional Analysis}
\subsection{Buckingham Pi Theorem}
\begin{eqnarray*}
	u_1&=&f(u_2,\dots,u_k)\\
	&\rightarrow&
	\\
	\Pi_1&=&\phi(\Pi_2,\dots,\Pi_k-r)
\end{eqnarray*}
where $r$ is the number of repeating variables.
\subsection{Similitude and Models}
\begin{eqnarray*}
	\Pi_{i_{\text{model}}}=\Pi_i&\forall i
\end{eqnarray*}
$\mu\dot =FL^{-2}T, \rho=FL^{-4}T^2$

\section{Viscous Flow in Pipes}
\subsection{Reynold's Number}
\begin{eqnarray*}
	Re=\frac{\rho vD}{\mu}=\frac{vD}{\nu}<2100
\end{eqnarray*}
\subsubsection{Fully Developed Profile}
\begin{eqnarray*}
	\text{laminar} &\frac{l_e}{D}=0.06 Re
	\\
	\text{turbulent} &\frac{l_e}{D}=4.4(Re)^{1/6}
\end{eqnarray*}

\subsection{Laminar Pipe Flow (Poiseuille improved)}
\begin{eqnarray*}
	v=\frac{(\Delta p -\gamma l \sin{\theta})D^2}{32\mu l}
\end{eqnarray*}
\subsection{Major Losses (to be added to Bernoulli)}
\begin{eqnarray*}
	h_{L_{\text{Major}}}=f\frac{l}{D}\frac{v^2}{2g}
\end{eqnarray*}
\subsection{Minor Losses}
\begin{eqnarray*}
	h_{L_{\text{Minor}}}=K_L\frac{v^2}{2g}
	&
	K_L=\frac{\Delta p}{\frac{1}{2}\rho v^2}
\end{eqnarray*}
\subsection{Moody Chart}
\begin{eqnarray*}
	f \leftrightarrow \frac{\epsilon}{D} \leftrightarrow Re
\end{eqnarray*}

\section{Boundary Layers, Drag \& Lift}
\subsection{Lift \& Drag}
\subsubsection{Drag}
\begin{eqnarray*}
	\mathscr D&=&\int dFx = \int p\cos\theta dA+\int\tau_\omega\sin\theta dA
	\\
	c_{\mathscr D}&=&\frac{\mathscr D}{\frac{1}{2}\rho U^2A}
\end{eqnarray*}
\subsubsection{Lift}
\begin{eqnarray*}
	\mathscr L&=&\int dFy = -\int p\sin\theta dA+\int\tau_\omega\cos\theta dA
	\\
	c_{\mathscr L}&=&\frac{\mathscr L}{\frac{1}{2}\rho U^2A}
\end{eqnarray*}
\subsection{Boundary Layer for Laminar Flow}
\begin{eqnarray*}
	\delta = c\sqrt{x} \propto \sqrt{\frac{\nu}{U}}\sqrt{x}
	\\
	\tau_\omega=0.332U^{3/2}\sqrt{\frac{\rho\mu}{x}}\\
	\mathscr{D}=\int \tau_\omega dA
	\\
	\text{Power} =\frac{\text{Energy}}{t}= \mathscr{D}v
\end{eqnarray*}

\end{document}


\input{../../_common/preamble}

\title{Introduction to classification}

\begin{document}

\maketitle

\begin{frame}{Contents}
    \tableofcontents[hideallsubsections]
\end{frame}

\section{Classification}

\begin{frame}{Regression versus classification}
    \begin{block}{Regression}
        \begin{description}
            \item[Aim]  Predict a \alert{continuous} value
            \item[Loss] How `off' (numerically) our predictions are
        \end{description}
    \end{block}
    \vfill
    \begin{block}{Classification}
        \begin{description}
            \item[Aim]  Predict a \alert{class}
            \item[Loss] How `inaccurate' the predicted classes are
        \end{description}
    \end{block}
\end{frame}

\section{$k$-nearest neighbours classifier}

\begin{frame}{$k$-nearest neighbours classifier}
    \only<1>{%
        Given a new observation\ldots
        \begin{itemize}
            \item Find the $k$ `most similar' training sample(s)
            \item Use the most common class among them as prediction
        \end{itemize}
        \vfill
        \begin{block}{Questions}
            \begin{itemize}
                \item How do we define similarity?
                \item How many neighbours do we use?
            \end{itemize}
        \end{block}}
    \only<2>{%
        \begin{center}
            \includegraphics[height=0.8\textheight]{figures/knn} \\
        {\scriptsize%
         From Burton DeWilde's blog}
        \end{center}}
\end{frame}

\begin{frame}{Choice of $k$}
    \begin{itemize}
        \item Larger $k$ $\rightarrow$ smoother boundaries, less `noisy'
        \item If $k = N$, we always predict the majority class
    \end{itemize}
    \vfill
    \begin{center}
        \includegraphics[width=\textwidth]{figures/k_effect} \\
        {\scriptsize%
         From \textit{CS231n: Convolutional Neural Networks for Visual Recognition}}
    \end{center}
\end{frame}

\begin{frame}{Minkowski distance}
    \[
        \left( \sum_{i} \left| x_{i} - y_{i} \right|^{p} \right)^{\!1/p}
    \]
    \vfill
    \begin{center}
        \renewcommand*{\arraystretch}{1.5}
        \begin{tabular}{lll}
            \toprule
            $p = 1$ & Manhattan distance & $\sum_{i} \left| x_{i} - y_{i} \right|$ \\
            $p = 2$ & Euclidean distance & $\sqrt{\sum_{i} \left( x_{i} - y_{i} \right)^{2}}$ \\
            \bottomrule
        \end{tabular}
    \end{center}
\end{frame}

\begin{frame}{Weight of neighbours}
    \begin{block}{Uniform weights}
        \begin{itemize}
            \item All $k$ neighbours contribute equally to the prediction
            \item Actual distance to each is ignored
        \end{itemize}
    \end{block}
    \vfill
    \begin{block}{Distance weights}
        \begin{itemize}
            \item Contributions are weighted by $1 / \text{distance}$
            \item Closer neighbours influence the prediction more
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{Curse of dimensionality}
    As the number of variables (coordinates) increases\ldots
    \begin{itemize}
        \item The volume of the space increases
        \item Pairwise distances become more similar $\rightarrow$ sparsity
        \item Some samples have huge neighbourhoods $\rightarrow$ `hubs'
    \end{itemize}
\end{frame}

\section{Metrics}

\begin{frame}{Classification accuracy}
    \begin{block}{Classification accuracy}
        \begin{itemize}
            \item Percentage of \alert{correct} predictions
            \item Higher is better
        \end{itemize}
    \end{block}
    \vfill
    \begin{block}{Classification error}
        \begin{itemize}
            \item Percentage of \alert{incorrect} predictions
                  (inverse of accuracy)
            \item Lower is better
        \end{itemize}
    \end{block}
\end{frame}

\newsavebox{\TP}
\savebox{\TP}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.25cm}
        {\Large\color{mDarkTeal}\cmark} \\[-0.25cm]
        {\tiny%
         True positive}
        \vspace{0.25cm}
    \end{minipage}}

\newsavebox{\TN}
\savebox{\TN}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.25cm}
        {\Large\color{mDarkTeal}\cmark} \\[-0.25cm]
        {\tiny%
         True negative}
        \vspace{0.25cm}
    \end{minipage}}

\newsavebox{\FP}
\savebox{\FP}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.25cm}
        {\Large\color{mDarkBrown}\xmark} \\[-0.25cm]
        {\tiny%
         False positive}
        \vspace{0.25cm}
    \end{minipage}}

\newsavebox{\FN}
\savebox{\FN}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.25cm}
        {\Large\color{mDarkBrown}\xmark} \\[-0.25cm]
        {\tiny%
         False negative}
        \vspace{0.25cm}
    \end{minipage}}

\begin{frame}[fragile]{Confusion matrix}
    \begin{center}
        \begin{tabular}{cc|cc}
                &     & \multicolumn{2}{c}{\textbf{Predicted}} \\
                &     & $1$          & $0$ \\ \hline
            \multirow{2}{*}[-0.75em]{\rotatebox[origin=c]{90}{\textbf{Actual}}}%
                & $1$ & \usebox{\TP} & \usebox{\FN} \\
                & $0$ & \usebox{\FP} & \usebox{\TN} \\
        \end{tabular}
    \end{center}
    \vfill
    \begin{itemize}
        \item Gives a better understanding of behaviour
        \item Can be used to define multiple performance metrics
    \end{itemize}
\end{frame}

\begin{frame}{Sensitivity and specificity}
    \vspace{-1em}
    \begin{columns}[T]
        \begin{column}{0.5\textwidth}
            \begin{center}
                \textbf{Sensitivity} \\[-0.25em]
                {\small%
                 (a.k.a.\ true positive rate)}
                \[
                    \frac{\sum\,\text{True positive}}{\sum\,\text{Actual} = 1}
                \]
            \end{center}
            \vfill
            \onslide<2->{%
                \begin{block}{Perfect sensitivity}
                    \begin{itemize}
                        \item All sick identified as sick
                        \item Negative test result definitely rules out disease
                    \end{itemize}
                \end{block}}
        \end{column}
        \begin{column}{0.5\textwidth}
            \begin{center}
                \textbf{Specificity} \\[-0.25em]
                {\small%
                 (a.k.a.\ true negative rate)}
                \[
                    \frac{\sum\,\text{True negative}}{\sum\,\text{Actual} = 0}
                \]
            \end{center}
            \vfill
            \onslide<3->{%
                \begin{block}{Perfect specificity}
                    \begin{itemize}
                        \item No healthy identified as sick
                        \item Positive test result useful for ruling in disease
                    \end{itemize}
                \end{block}}
        \end{column}
    \end{columns}
    \vfill
    \onslide<4->{%
        \begin{center}
            \alert{Can we maximise both at the same time?}
        \end{center}}
\end{frame}

\begin{frame}{`Perfect' border control}
    \begin{block}{100\% sensitivity}
        \begin{itemize}
            \item `Everyone is a terrorist!'
            \pause
            \item All terrorists are stopped $\rightarrow$ 100\% sensitivity
            \pause
            \item No one can enter the country! $\rightarrow$ 0\% specificity
        \end{itemize}
    \end{block}
    \vfill\pause
    \begin{block}{100\% specificity}
        \begin{itemize}
            \item `No one is a terrorist!'
            \pause
            \item All non\hyp{}terrorists are allowed in $\rightarrow$ 100\% specificity
            \pause
            \item All terrorists are also allowed into the country! $\rightarrow$ 0\% sensitivity
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{ROC and AUC}
    \vspace{-1em}
    \begin{columns}[T]
        \begin{column}{0.5\textwidth}
            \begin{center}
                \textbf{Receiver Operating Characteristic (ROC) curve} \\
                \includegraphics[height=0.6\textheight]{figures/roc} \\
                Sensitivity \textit{vs} ($1 - \text{specificity}$) \\
                $\rightarrow$ TP rate \textit{vs} FP rate
            \end{center}
        \end{column}
        \pause
        \begin{column}{0.5\textwidth}
            \begin{center}
                \vspace{1ex}
                \textbf{Area Under the Curve (AUC)} \\[\medskipamount]
            \end{center}
            \begin{itemize}
                \item Probability that
                      $\text{Prediction}\!\left( \text{actual 1} \right) > \text{Prediction}\!\left( \text{actual 0} \right)$
                \item Random guess \\
                      $\rightarrow$ AUC = 50\% (diagonal)
                \item Higher is better
            \end{itemize}
        \end{column}
    \end{columns}
\end{frame}

\section{Cost\hyp{}benefit analysis}

\begin{frame}{Cost\hyp{}benefit analysis}
    \begin{itemize}
        \item Assume that the four possible outcomes of a classification problem
              have (numerical) \alert{benefits and costs}
              \begin{description}
                  \item[$> 0$] desirable (e.g.\ profit)
                  \item[$= 0$] neutral
                  \item[$< 0$] undesirable (e.g.\ loss) \\[\bigskipamount]
              \end{description}
        \item These `benefits and costs' needn't be symmetrical
    \end{itemize}
\end{frame}

\newsavebox{\TPexA}
\savebox{\TPexA}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        20\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\TNexA}
\savebox{\TNexA}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        50\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\FPexA}
\savebox{\FPexA}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        10\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\FNexA}
\savebox{\FNexA}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        20\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\TPexB}
\savebox{\TPexB}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        30\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\TNexB}
\savebox{\TNexB}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        55\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\FPexB}
\savebox{\FPexB}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        10\%
        \vspace{0.5cm}
    \end{minipage}}

\newsavebox{\FNexB}
\savebox{\FNexB}{%
    \begin{minipage}[c]{2cm}
        \centering
        \vspace{0.5cm}
        5\%
        \vspace{0.5cm}
    \end{minipage}}

\begin{frame}{Planning an outdoor activity}
    \only<1>{%
        You have 20 people enrolled in an outdoor activity paying £30 each
        \begin{itemize}
            \item Before the activity, you check the weather forecast and either:
                  \begin{itemize}
                      \item Go ahead, which costs you £5 per participant
                      \item Cancel and refund the participants in full
                  \end{itemize}
            \item If you decide to go ahead, the day of the activity it will
                  either:
                  \begin{itemize}
                      \item Be sunny, in which case you get to keep the profit
                      \item Rain, in which case you'll have to refund the
                            participants in full
                  \end{itemize}
        \end{itemize}
        What is the `benefits and costs' matrix?}
    \only<2>{%
        \begin{columns}[t]
            \begin{column}{0.5\textwidth}
                \begin{center}
                    \textbf{Free weather forecast} \\[\bigskipamount]
                    \begin{tabular}{cc|cc}
                            &       & \multicolumn{2}{c}{\textbf{Forecast}} \\
                            &       & \Sun           & \Rain \\ \hline
                        \multirow{2}{*}[-0.75em]{\rotatebox[origin=c]{90}{\textbf{Actual}}}%
                            & \Sun  & \usebox{\TPexA} & \usebox{\FNexA} \\
                            & \Rain & \usebox{\FPexA} & \usebox{\TNexA} \\
                    \end{tabular}
                \end{center}
            \end{column}
            \begin{column}{0.5\textwidth}
                \begin{center}
                    \textbf{Weather forecast costing £15} \\[\bigskipamount]
                    \begin{tabular}{cc|cc}
                            &       & \multicolumn{2}{c}{\textbf{Forecast}} \\
                            &       & \Sun           & \Rain \\ \hline
                        \multirow{2}{*}[-0.75em]{\rotatebox[origin=c]{90}{\textbf{Actual}}}%
                            & \Sun  & \usebox{\TPexB} & \usebox{\FNexB} \\
                            & \Rain & \usebox{\FPexB} & \usebox{\TNexB} \\
                    \end{tabular}
                \end{center}
            \end{column}
        \end{columns}
        \begin{itemize}
            \item What are the sensitivity and specificity?
            \item What is the expected profit?
        \end{itemize}}
\end{frame}

\end{document}


\chapter{Understanding variable importances}\label{ch:importances}

\begin{remark}{Outline}
In this chapter, we study variable importance measures as computed from forests of
randomized trees. In Section~\ref{sec:6:importances}, we first present how
random forests can be used to assess the importance of input variables.  We
then derive in Section~\ref{sec:6:theory} a characterization in asymptotic
conditions and show that variable importances derived from totally randomized trees
offer a three-level decomposition of the information jointly  contained in the
input variables about the output. In Section~\ref{sec:6:variable-relevance}, we
show that this  characterization only depends on the relevant variables and
then discuss these ideas in Section~\ref{sec:6:variants} in the context of
variants closer to the Random Forest algorithm. Finally, we illustrate these
results on an artificial problem in Section~\ref{sec:6:illustration}.
\textit{This chapter is based on previous work published in \citep{louppe:2013}.}
\end{remark}

An important task in many scientific fields is the prediction of  a response
variable based on a set of predictor variables. In many situations though, the
aim is not only to make the most accurate predictions of the response but also
to identify which predictor variables are the most important to make these
predictions, e.g. in order to understand the underlying process. Because of
their applicability to a wide range of problems and their capability to
build accurate models and, at the same time, to provide variable importance
measures, random forests have become a major data analysis tool used with
success in various scientific areas.

Despite their extensive use in applied research, only a couple of works have
studied the theoretical properties and statistical mechanisms of these
algorithms. \citet{zhao:2000}, \citet{breiman:2004}, \citet{meinshausen:2006},
\citet{lin:2006} and \citet{biau:2008,biau:2012} investigated simplified to
very close variants of these algorithms and proved  the consistency of
those variants. Little is known however regarding the variable importances
computed by random forests, and -- as far as we know -- the work
of~\citet{ishwaran:2007} is indeed the only theoretical study of tree-based
variable importance measures. In this chapter, we aim at filling this gap and
present a theoretical  analysis of the Mean Decrease Impurity importance
derived from ensembles of randomized trees.

% A forest of trees is impenetrable
% as far as simple interpretations of its mechanims go~\citep{breiman:2001}.

\section{Variable importances}
\label{sec:6:importances}

\subsection{Importances in single decision trees}

In the context of single decision trees, \cite{breiman:1984} first defined
the measure of importance of a variable $X_j$ as
\begin{equation}
\text{Imp}(X_j) = \sum_{t\in \varphi} \Delta I(\tilde{s}^j_t, t),
\end{equation}
where $\tilde{s}^j_t$\label{ntn:s-surrogate} is the best \textit{surrogate split}
for $s_t$, that is the closest split defined on variable $X_j$ that can mimic the actual
split $s_t$ defined at node $t$. The use of surrogate splits was proposed to
account for masking effects: it may indeed happen that some variable $X_{j_2}$
never occurs in any split because it leads to splits that are slightly worse,
and therefore not selected, than those of some other variable $X_{j_1}$.
However, if $X_{j_1}$ is removed and another tree is grown, $X_{j_2}$ may now
occur prominently within the splits and the resulting tree may be nearly as good
as the original tree. In such a case, a relevant measure should detect the
importance of $X_{j_2}$. Accordingly, if $X_{j_2}$ is being masked at $t$ by
$X_{j_1}$ (i.e., if $X_{j_1}$ is used to split $t$), but if $\tilde{s}^{j_2}_t$ is similar to
$s_t$, but not quite as good, then $\Delta I(\tilde{s}^{j_2}_t, t)$ will be
nearly as large as $\Delta I(s_t, t)$ and therefore the proposed measure will
indeed account for the importance of $X_{j_2}$.

\subsection{Importances in forests}

Thanks to randomization, masking effects are dampened within forests of
randomized trees. Even if $X_{j_2}$ is being masked by $X_{j_1}$ there is indeed
still a chance for $X_{j_2}$ to be chosen as a split if $X_{j_1}$ is not
selected among the $K$ variables chosen at random. Depending on the value $K$,
masking effects do not disappear entirely though. The use of bootstrap samples
also helps reduce masking effects, making $X_{j_1}$ or $X_{j_2}$ just slightly
better than the other due to the variations in the bootstrap samples.

For this reason, \cite{breiman:2001,breiman:2002} proposed to evaluate the importance of
a variable $X_j$  for predicting  $Y$ by adding up the weighted impurity decreases $p(t) \Delta
i(s_t, t)$ for all nodes $t$ where $X_j$ is used, averaged over all trees $\varphi_m$ (for $m=1,\dots,M$)
in the forest:
\begin{equation}\label{eq:mdi}
\text{Imp}(X_j) = \frac{1}{M} \sum_{m=1}^M \sum_{t \in \varphi_{m}} 1(j_t = j) \Big[ p(t) \Delta i(s_t, t) \Big],
\end{equation}
where $p(t)$ is the proportion $\tfrac{N_t}{N}$ of samples
reaching $t$ and where $j_t$ denotes the identifier of the variable used for splitting node $t$. When
using the Gini index as impurity function, this measure is known as the
\textit{Gini importance} or \textit{Mean Decrease Gini}. However, since it can
be defined for any impurity measure $i(t)$, we will refer to Equation~\ref{eq:mdi}
as the \textit{Mean Decrease Impurity} importance (MDI), no matter the impurity
measure $i(t)$. We will characterize and derive results for this measure in the
rest of this text.

In addition to MDI, \cite{breiman:2001,breiman:2002} also proposed to evaluate
the importance of a variable $X_j$ by measuring the \textit{Mean Decrease
Accuracy} (MDA) or, equivalently, by measuring the \textit{Mean Increase Error},
of the forest when the values of $X_j$ are randomly permuted in the out-of-bag
samples. For that reason, this latter measure is also known as the
\textit{permutation importance}. Formally, in regression, the permutation
importance of $X_j$ is given as:
\begin{align}\label{eq:mda}
\text{Imp}(X_j) &=  \mathbb{E}_{\pi_j} \Bigg\{ \frac{1}{N} \sum_{(\mathbf{x}_i^\prime,y_i) \in \pi_j({\cal L})} L(\frac{1}{M^{-i}} \sum_{l=1}^{M^{-i}} \varphi_{{\cal L}^{m_{k_l}}}(\mathbf{x}_i^\prime), y_i) \Bigg\} \nonumber  \\
&\hookrightarrow - \frac{1}{N} \sum_{(\mathbf{x}_i,y_i) \in {\cal L}} L(\frac{1}{M^{-i}} \sum_{l=1}^{M^{-i}} \varphi_{{\cal L}^{m_{k_l}}}(\mathbf{x}_i), y_i)
\end{align}
where $\pi_j({\cal L})$ denotes a replicate of ${\cal L}$ in which in the
values of $X_j$ have been randomly permuted, and where
$m_{k_1},\dots,m_{k_{M^{-i}}}$ denote the indices of the trees that have been
built from bootstrap replicates that do not include $(\mathbf{x}_i, y_i)$. For
classification, the permutation importance is derived similarly as in
Equation~\ref{eq:mda}, except  that the out-of-bag average predictions are
replaced with the class which is the most likely, as computed from the out-of-bag
class probability estimates. Its rationale is that randomly permuting the
input variable $X_j$ should break its association with the response $Y$.
Therefore, if $X_j$ is in fact associated to $Y$, permuting its values should
also result in a substantial increase of error, as here measured by the
difference between the out-of-bag estimates of the generalization error. That
is, the larger the increase of error, the more important the variable, and
vice-versa.

Thanks to popular machine learning
softwares~\citep{breiman:2002,liaw:2002,pedregosa:2011},
both of these variable importance measures have shown their practical utility in
an increasing number of experimental studies. Little is known however regarding
their inner workings. \cite{strobl:2007b} compare both MDI and MDA and show
experimentally that the former is biased towards some predictor variables. As
explained by~\cite{white:1994} in case of single decision trees, this
bias stems from an unfair advantage given by the usual impurity functions $i(t)$
towards predictors with a large number of values (see Section~\ref{sec:7:bias:high}). \cite{strobl:2008}
later showed  that MDA is biased as well, and that it overestimates the
importance of correlated variables -- although this effect was not confirmed in
a later experimental study by~\cite{genuer:2010}.  From a theoretical
point of view, \cite{ishwaran:2007} provides a detailed theoretical
development of a simplified version of MDA, giving key insights for the
understanding of the actual MDA.

\section{Theoretical study}
\label{sec:6:theory}

\subsection{Background}

To be self-contained, we first recall several definitions from information
theory (see \citep{cover:2012}, for further properties).

We suppose that we are given a probability space $(\Omega, {\cal E},
\mathbb{P})$ and  consider random variables defined on it taking a finite
number of possible values. We use upper case letters to denote such random
variables (e.g. $X, Y, Z, W \ldots$)  and calligraphic letters (e.g. $\cal X,
Y, Z, W \ldots$) to denote their image sets (of finite cardinality), and lower
case letters (e.g. $x, y, z, w \ldots$) to denote one of their possible values.
For a (finite) set of (finite) random variables $ X = \{X_{1}, \ldots ,
X_{p}\}$, we denote by $P_{X}(\mathbf{x}) = P_{X}(x_{1}, \ldots , x_{p})$ the
probability $\mathbb{P}(\{ \omega \in \Omega \mid  \forall j : 1, \ldots, p:
X_j(\omega) =x_j\})$, and by ${\cal X} = {\cal X}_{1} \times \cdots
\times {\cal X}_{p}$ the set of joint configurations of these random variables.
Given two sets of random variables, $X = \{X_{1}, \ldots , X_{p}\}$ and
$Y=\{Y_{1}, \ldots , Y_{q}\}$, we denote by $P_{X \mid Y}(\mathbf{x} \mid \mathbf{y}) = {P_{X, Y}
(\mathbf{x},  \mathbf{y})}/ {P_{Y}(\mathbf{y})}$ the conditional density of $X$ with respect to
$Y$.\footnote{To avoid problems, we suppose that all probabilities are strictly
positive, without fundamental limitation.}

With these notations, the joint (Shannon) entropy of a set of random variables
$X =\{X_{1}, \ldots , X_{p}\}$ is thus defined by
\begin{equation}\label{eqn:6:entropy}
H(X)  = - \sum_{\mathbf{x} \in {\cal X}}P_{X} (\mathbf{x})\log_{2}P_{X}(\mathbf{x}),
\end{equation}
while the mean conditional entropy of a set of random variables $X = \{X_{1},
\ldots , X_{p}\}$, given the values of another set of random variables
$Y=\{Y_{1}, \ldots , Y_{q}\}$ is defined by
\begin{equation}\label{eqn:6:entropy-cond}
H(X\mid Y) = - \sum_{\mathbf{x} \in {\cal X}} \sum_{\mathbf{y} \in {\cal Y}} P_{X, Y} (\mathbf{x}, \mathbf{y}) \log_{2} P_{X \mid Y} (\mathbf{x} \mid \mathbf{y}).
 \end{equation}
The mutual information among the set of random variables $X =\{X_{1}, \ldots ,
X_{p}\}$ and the set of random variables $Y=\{Y_{1}, \ldots , Y_{q}\}$ is
defined by
 \begin{align}\label{eqn:6:mi}
 I(X; Y) &= - \sum_{\mathbf{x} \in {\cal X}} \sum_{\mathbf{y} \in {\cal Y}} P_{X, Y} (\mathbf{x}, \mathbf{y}) \log_{2} \frac{P_{X}(\mathbf{x}) P_{Y}(\mathbf{y})}{P_{X,Y}(\mathbf{x},\mathbf{y})} \\
 &= H(X) - H(X \mid Y) \nonumber \\
 &=  H(Y) - H(Y \mid X) \nonumber
 \end{align}

The mean conditional mutual information among the set of random variables $X
=\{X_{1}, \ldots , X_{p}\}$ and the set of random variables $Y=\{Y_{1}, \ldots
, Y_{q}\}$, given the values of a third set of random variables $Z=\{Z_{1},
\ldots , Z_{r}\}$, is defined by
 \begin{align}
 I(X; Y \mid Z) &= H(X \mid Z) - H(X \mid Y, Z)\\
 & = H(Y \mid Z) - H(Y \mid X, Z) \nonumber\\
& = - \sum_{\mathbf{x} \in {\cal X}} \sum_{\mathbf{y} \in {\cal Y}} \sum_{\mathbf{z} \in {\cal Z}} P_{X, Y, Z} (\mathbf{x}, \mathbf{y}, \mathbf{z}) \log_{2} \frac{P_{X \mid Z}(\mathbf{x} \mid \mathbf{z}) P_{Y\mid Z}(\mathbf{y} \mid \mathbf{z})}{P_{X,Y \mid Z}(\mathbf{x},\mathbf{y} \mid \mathbf{z})}\nonumber
\end{align}

We also recall the chaining rule
\begin{equation}
I(X, Z ; Y \mid W ) = I(X; Y \mid W  ) + I( Z ; Y \mid W, X),
\end{equation}
and the symmetry of the (conditional) mutual information among sets of random variables
\begin{equation}
I(X ; Y \mid Z) = I(Y ;  X  \mid Z).
\end{equation}

\subsection{Asymptotic analysis}

%\todo{Add figures}

Let us now consider the MDI  importance as defined by Equation~\ref{eq:mdi},
and let us assume a set $V= \{X_1, ..., X_p\}$\label{ntn:V}  of {\em categorical} input
variables and a {\em categorical} output $Y$. For the sake of simplicity we
will  use the Shannon entropy as impurity measure, and focus on totally
randomized trees; later on we will discuss other impurity measures and tree
construction algorithms.

Given a training sample ${\cal L}$ of $N$ joint observations of $X_1, ..., X_p,
Y$ independently drawn from the joint distribution $P(X_1, ..., X_p, Y)$, let us
assume that we infer from it an infinitely large ensemble of \textit{totally
randomized and fully developed trees}.

\begin{definition} A \emph{totally randomized and
fully developed tree} is a decision tree in which each node $t$ is
partitioned using a variable $X_j$ picked uniformly at random among those not
yet used at the parent nodes of $t$, where each node $t$ is split into $|{\cal
X}_j|$ sub-trees, i.e., one for each possible value of ${\cal X}_j$, and where
the recursive construction process halts only when all $p$ variables have been
used along the current branch.
\end{definition}

In such a tree, leaves are all at the same depth $p$, and the set of leaves of
a fully developed tree is in bijection with the set $\cal X$ of all possible
joint configurations of the $p$ input variables. For example, if the input
variables are all binary, the resulting tree will have exactly $2^{p}$ leaves.
As a result, totally randomized trees, as considered here, are in fact closer
to trees grown with ID3~\citep{quinlan:1986} than trees built from the CART
procedure (as described throughout Chapters~\ref{ch:cart} and \ref{ch:forest}).

\begin{theorem}\label{thm:imp}
The MDI importance of $X_j \in V$ for $Y$ as computed
with an   infinite ensemble of fully developed totally randomized trees and an
infinitely large training sample is:
  \begin{equation}\label{eqn:imp-full}
  \text{Imp}(X_j)=\sum_{k=0}^{p-1} \frac{1}{C_p^k} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B),
  \end{equation}
\end{theorem}
\noindent where $V^{-j}$\label{ntn:V-j} denotes the subset $V \setminus \{X_j\}$, ${\cal
P}_k(V^{-j})$\label{ntn:P_k} is the set of subsets of  $V^{-j}$ of cardinality $k$, and
$I(X_j;Y|B)$ is the conditional mutual information of $X_{j}$ and $Y$ given the
variables in $B$\label{ntn:B}.

\begin{proof}
By expanding $\Delta i(s, t) = i(t) - p_L i(t_L) - p_R i(t_R)$ into
Equation~\ref{eq:mdi} and using the entropy $H(Y|t) = -\sum_j p(j|t) \log_2(p(j|t))$
as impurity measure $i(t)$, Equation~\ref{eq:mdi} can be rewritten in terms of
mutual information:
\begin{equation}
\text{Imp}(X_j) = \frac{1}{M} \sum_{m=1}^M \sum_{t \in \varphi_m} 1(j_t = j) p(t) I(Y;X_j|t)
\end{equation}

As the size $N$ of the training sample grows to infinity, $p(t)$ becomes the
(exact) probability (according to $P(X_1,\ldots,X_p,Y)$) that an object reaches
node $t$, i.e., a probability $P(B(t)=b(t))$ where $B(t)=(X_{i_1}, ...,
X_{i_{k}})$ is the subset of $k$ variables tested in the branch from the root
node to the parent of $t$ and $b(t)$ is the vector of values of these
variables. As the the number $M$ of totally randomized trees also grows to
infinity, the importance of a variable $X_j$ can then be written:
\begin{equation}
\text{Imp}(X_j)=\sum_{B\subseteq V^{-j}} \sum_{b\in {\cal X}_{i_1}\times ... \times {\cal X}_{i_{k}} } \alpha(B,b,X_j,p) P(B=b) I(Y;X_j|B=b),
\end{equation}
where $b$ is a set of values for the variables in $B$ and $\alpha(B,b,X_j,p)$ is
the probability that a node $t$ (at depth $k$) in a totally randomized tree
tests the variable $X_j$ and is such that $B(t)=B$ and $b(t)=b$.

Let us compute $\alpha(B,b,X_j,p)$. First, let us consider the probability that
a node $t$ tests the variable $X_j$ and is such that the branch leading to $t$
follows a path defined, in that particular order, by all $k$ variables $X_{i_1},
..., X_{i_{k}} \in B$ and their corresponding values in $b$. The probability of
that branch is the probability of picking (uniformly at random)  $X_{i_1}$  at
the root node times the probability of testing, in that order, the remaining
$X_{i_2}, ..., X_{i_{k}}$ variables in the sub-tree corresponding to the value
$x_{i_1}$ of $X_{i_1}$ defined in $b$. Note that, by construction, it is certain
that this particular sub-tree exists since the root node is split into $|{\cal
X}_{i_1}|$ sub-trees.  Then, the
probability of testing $X_j$ at the end of this branch is the probability of
picking $X_j$ among the remaining $p-k$ variables. By recursion, we thus have:
\begin{equation}
\frac{1}{p} \frac{1}{p-1} ... \frac{1}{p-k+1} \frac{1}{p-k} = \frac{(p-k)!}{p!} \frac{1}{p-k}
\end{equation}

Since the order along which the variables appear in the branch is of no
importance, $\alpha(B,b,X_j,p)$ actually includes all $k!$ equiprobable ways of
building a branch composed of the variables and values in $B$ and $b$. Then,
since a tree may at most contain a single such branch, whatever the order of the
tests, the probabilities may be added up and it comes:
\begin{equation}
\alpha(B,b,X_j,p)=k! \frac{(p-k)!}{p!} \frac{1}{p-k} = \frac{1}{C_p^k} \frac{1}{p-k}
\end{equation}\label{ntn:C_k_p}

From the above expression, it appears that $\alpha(B,b,X_j,p)$ depends only on
the size $k$ of $B$ and on the number $p$ of variables. As such, by grouping in
the previous equation of $\text{Imp}(X_j)$ conditioning variable subsets $B$ according
to their sizes and using the definition of conditional mutual information,
$\alpha$ can be factored out, hence leading to the form foretold by Theorem~\ref{thm:imp}:
\begin{equation}
\text{Imp}(X_j)=\sum_{k=0}^{p-1} \frac{1}{C^k_p} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B).
\end{equation}
\end{proof}

\begin{theorem}\label{thm:sum-of-imp}
For any ensemble of fully developed trees in asymptotic learning sample size
conditions (e.g., in the same conditions as those of Theorem~\ref{thm:imp}), we
have that
\begin{equation}\label{eqn:sum-of-imp}
\sum_{j=1}^{p} \text{Imp}(X_j) = I(X_{1}, \ldots, X_{p} ; Y).
\end{equation}
\end{theorem}

\begin{proof}
For any tree $\varphi$, we have that the sum of all importances estimated by
using an infinitely large sample $\cal L$ (or equivalently, by assuming perfect
knowledge of the joint distribution $P(X_1, ..., X_p, Y)$) is equal to $H(Y) -
\sum_{t \in \widetilde{\varphi}} p(t) H(Y|b(t))$, where $\widetilde{\varphi}$
denotes the set of all leaves of $\varphi$, and where $b(t)$  denotes the joint
configuration of all input variables leading to leaf $t$. This is true because
the impurities of all test nodes intervening in the computation of the variable
importances, except the impurity $H(Y)$ at the root node of the tree, cancel
each other when summing up the importances.

Since, when the tree is fully developed, $\sum_{t \in \widetilde{\varphi}} p(t)
H(Y|b(t))$ is obviously equal to the mean conditional entropy $H(Y | X_{1},
\ldots, X_{p})$ of $Y$ given all input variables, this implies that for any
fully developed tree we have that the sum of variable importances is equal to
$I(X_{1}, \ldots, X_{p} ; Y)$, and so this relation also holds when averaging
over an infinite ensemble of totally randomized trees. Note that, for the
same reasons as in the proof of Theorem~\ref{thm:imp}, this result
holds  in asymptotic conditions only, i.e., when $p(t)$ becomes the exact probability
$P(B(t)=b(t))$.
\end{proof}

Together, theorems \ref{thm:imp} and \ref{thm:sum-of-imp} show that  variable
importances derived from totally randomized trees in asymptotic conditions
provide a three-level decomposition of the information $I(X_{1}, \ldots, X_{p}
; Y)$ contained in the set of input variables about the output variable. The
first level is a decomposition among input variables (see Equation~\ref{eqn:sum-of-imp}
of Theorem~\ref{thm:sum-of-imp}),  the second level is a
decomposition along the degrees $k$ of interaction terms of a variable with the
other ones (see the outer sum in Equation~\ref{eqn:imp-full} of
Theorem~\ref{thm:imp}), and the third level is a decomposition along the
combinations $B$ of interaction terms of fixed size $k$ of possible interacting
variables (see the inner sum in Equation~\ref{eqn:imp-full}).

We observe that the decomposition includes, for each variable, each and every
interaction term of each and every degree weighted in a fashion resulting only
from the combinatorics of possible interaction terms. In particular, since all
$I(X_j;Y|B)$ terms are at most equal to $H(Y)$, the prior entropy of $Y$,  the
$p$ terms of the outer sum of Equation~\ref{eqn:imp-full} are each upper
bounded by
\begin{equation}
\frac{1}{C^k_p}\frac{1}{p-k}\sum_{B \in {\cal P}_k(V^{-j})}
H(Y)=\frac{1}{C^k_p} \frac{1}{p-k} {C^k_{p-1}} H(Y) = \frac{1}{p}H(Y).
\end{equation}
As such,
the second level decomposition resulting from totally randomized trees makes the
$p$ sub-importance terms
\begin{equation}
\frac{1}{C^k_p} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)
\end{equation}
to equally contribute (at most) to the total
importance, even though they each include a combinatorially different number of
terms.

\section{Relevance of variables}
\label{sec:6:variable-relevance}

Following~\citet{kohavi:1997}, let us define relevant and irrelevant variable as follows:

\begin{definition}
A variable $X_j$ is {\em relevant to $Y$ with
respect to $V$} if there exists at least one subset $B
\subseteq V$ (possibly empty) such that $I(X_j;Y|B)>0$.
\end{definition}

\begin{definition} A variable $X_i$ is {\em
irrelevant to $Y$ with respect to $V$} if, for all $B
\subseteq V$, $I(X_i; Y|B)=0$.
\end{definition}

Remark that if $X_i$ is irrelevant to $Y$ with
respect to $V$, then by definition it is also irrelevant to $Y$ with respect to
any subset of $V$. However, if $X_j$ is relevant to $Y$ with respect to $V$,
then it is not necessarily relevant to $Y$ with respect to all  subsets of $V$.
Among the relevant variables, we also distinguish the {\em marginally} relevant
ones, for which $I(X_{j}; Y) > 0$, the  {\em strongly} relevant ones, for which
$I(X_{j}; Y | V^{-j}) > 0$,  and the {\em weakly} relevant variables, which are
relevant but not strongly relevant.

\begin{theorem}\label{thm:irrelevant}
  $X_i \in V$ is irrelevant to $Y$ with respect to $V$ if and only if  its
  infinite sample size importance as computed with an infinite ensemble of fully
  developed totally randomized trees built on $V$ for $Y$ is $0$.
\end{theorem}

\begin{proof}
The proof directly results from the definition of irrelevance. If $X_i$ is
irrelevant with respect to $V$, then $I(X_i;Y|B)$ is zero for all $B \subseteq
V^{-i} \subset V$ and Equation~\ref{eqn:imp-full} reduces to $0$. Also, since
$I(X_i;Y|B)$ is non-negative for any $B$, $\text{Imp}(X_i)$ is zero if and only if all
its $I(X_i;Y|B)$ terms are zero. Since $\text{Imp}(X_i)$ includes all $I(X_i;Y|B)$
terms for $B \subseteq V^{-i}$, and since all of them are therefore null if
$\text{Imp}(X_i)=0$,  $X_i$ is thus, by definition, irrelevant with respect to
$V^{-i}$. $X_i$ is then also trivially irrelevant with respect to $V=V^{-i} \cup
\{X_i\}$ since $I(X_i;Y|B\cup\{X_i\})=0$ for any $B$.
\end{proof}

\begin{lemma}\label{lemma:adding-irrelevant}
  Let $X_i \notin V$ be an irrelevant variable for $Y$ with respect to $V$. The infinite
  sample size importance of $X_j \in V$ as computed with an infinite
  ensemble of fully developed totally randomized trees built on $V$ for $Y$ is the
  same as the importance derived when using $V\cup \{X_i\}$ to build the ensemble of trees for $Y$.
\end{lemma}

\begin{proof}
Let $X_i \notin V$ be an irrelevant variable with respect to $V$. For $X_j \in
V$, $B \subseteq V^{-j}$, using the chain rules of mutual information, we have:
\begin{align}
I(X_j, X_i;Y|B) &= I(X_j;Y|B) + I(X_i;Y|B \cup \{X_j\})  \\
                &= I(X_i;Y|B) + I(X_j;Y|B \cup \{X_i\})
\end{align}
If $X_i$ is irrelevant with respect to $V$, i.e., such that $I(X_i;Y|B)=0$ for
all $B\subseteq V$, then $I(X_i;Y|B \cup \{X_j\})$ and $I(X_i;Y|B)$ both equal
$0$, leading to
\begin{equation}
I(X_j;Y|B \cup \{X_i\}) = I(X_j;Y|B)
\end{equation}

Then, from Theorem~\ref{thm:imp}, the importance of $X_j$ as computed with an
infinite ensemble of totally randomized trees built on $V\cup \{X_i\}$ can be
simplified to:
\begin{align}
  \text{Imp}(X_j)&=\sum_{k=0}^{p-1+1} \frac{1}{C_{p+1}^k} \frac{1}{p+1-k} \sum_{B \in {\cal P}_k(V^{-j} \cup \{X_i\})} I(X_j;Y|B)\nonumber\\
          &=\sum_{k=0}^{p} \frac{1}{C_{p+1}^k} \frac{1}{p+1-k} \left[ \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B) + \sum_{B \in {\cal P}_{k-1}(V^{-j})} I(X_j;Y|B \cup \{X_i\})  \right] \nonumber\\
          &=\sum_{k=0}^{p-1} \frac{1}{C_{p+1}^k} \frac{1}{p+1-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B) + \nonumber\\
          & \hookrightarrow \sum_{k=1}^{p} \frac{1}{C_{p+1}^k} \frac{1}{p+1-k} \sum_{B \in {\cal P}_{k-1}(V^{-j})} I(X_j;Y|B)\nonumber\\
          &=\sum_{k=0}^{p-1} \frac{1}{C_{p+1}^k} \frac{1}{p+1-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B) + \nonumber\\
          & \hookrightarrow \sum_{k'=0}^{p-1} \frac{1}{C_{p+1}^{k'+1}} \frac{1}{p+1-k'-1} \sum_{B \in {\cal P}_{k'}(V^{-j})} I(X_j;Y|B)\nonumber\\
          &=\sum_{k=0}^{p-1} \left[ \frac{1}{C_{p+1}^k} \frac{1}{p+1-k} + \frac{1}{C_{p+1}^{k+1}} \frac{1}{p-k} \right] \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)\nonumber\\
          &=\sum_{k=0}^{p-1} \frac{1}{C^k_p} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)
\end{align}
The last line above exactly corresponds to the importance of $X_j$ as computed
with an infinite ensemble of totally randomized trees built on $V$, which proves
Lemma~\ref{lemma:adding-irrelevant}.
\end{proof}

\begin{theorem}\label{thm:relevant}
  Let $V_R \subseteq V$ be the subset of all variables in $V$ that are relevant to $Y$ with
  respect to $V$. The infinite sample size importance of any variable $X_j \in
  V_R$ as computed with an infinite ensemble of fully developed totally randomized
  trees built on $V_R$ for $Y$ is the same as its importance computed in the same conditions by using all variables in $V$. That is:
    \begin{align}
      \text{Imp}(X_j)&=\sum_{k=0}^{p-1} \frac{1}{C_p^k} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)\nonumber \\
                     &=\sum_{l=0}^{r-1} \frac{1}{C_r^l} \frac{1}{r-l} \sum_{B \in {\cal P}_l(V_R^{-j})} I(X_j;Y|B)
    \end{align}
  where $r$ is the number of relevant variables in $V_R$.
\end{theorem}

\begin{proof}
Let us assume that $V_R$ contains $r \leq p$ relevant variables. If an infinite
ensemble of totally randomized trees were to be built directly on those $r$
variables then, from Theorem~\ref{thm:imp}, the importance of a relevant
variable $X_j$ would be:
\begin{equation}%\label{eqn:imp-rel}
  \text{Imp}(X_j)=\sum_{l=0}^{r-1} \frac{1}{C_r^l} \frac{1}{r-l} \sum_{B \in {\cal P}_l(V_R^{-m})} I(X_j;Y|B)
\end{equation}

Let $X_i \in V \setminus V_R$ be one of the $p-r$ irrelevant variables in $V$
with respect to $V$. Since $X_i$ is also irrelevant with respect to $V_R$, using
Lemma~\ref{lemma:adding-irrelevant}, the importance of $X_j$ when the ensemble
is built on $V_R \cup \{X_i\}$ is the same as the one computed on $V_R$ only
(i.e., as computed by the equation above). Using the same argument,
adding a second irrelevant variable $X_{i'}$ with respect to $V$ -- and therefore
also with respect to $V_R \cup \{X_i\}$ -- and building an ensemble of totally
randomized trees on $V_R \cup \{X_i\} \cup \{X_{i'}\}$ will yield importances
that are the same as those computed on $V_R \cup \{X_i\}$, which are themselves
the same as those computed by an ensemble built on $V_R$. By induction, adding
all $p-r$ irrelevant variables has therefore no effect on the importance of
$X_j$, which means that:
\begin{align}%\label{eqn:imp-rel-equiv}
  \text{Imp}(X_j)&=\sum_{k=0}^{p-1} \frac{1}{C_p^k} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)\nonumber \\
          &=\sum_{l=0}^{r-1} \frac{1}{C_r^l} \frac{1}{r-l} \sum_{B \in {\cal P}_l(V_R^{-m})} I(X_j;Y|B)
\end{align}
\end{proof}

Theorems~\ref{thm:irrelevant} and \ref{thm:relevant} show that the
importances computed with an ensemble of totally randomized trees depends only
on the relevant variables. Irrelevant variables have a zero importance and do
not affect the importance of relevant variables. Practically, we believe that
such properties are desirable conditions for a sound criterion assessing the
importance of a variable. Indeed, noise should not be credited of any importance
and should not make any other variable more (or less) important.

Intuitively, the independence with respect to irrelevant variables can be partly
attributed to the fact that splitting at $t$ on some irrelevant variable $X_i$
should only dillute the local importance $p(t) \Delta i(t)$ of a relevant
variable $X_j$ into the children $t_L$ and $t_R$, but not affect the total sum.
For instance, if $X_j$ was to be used at $t$, then the local importance would be
proportional to $p(t)$. By contrast, if $X_i$ was to be used at $t$ and $X_j$ at
$t_L$ and $t_R$, then the sum of the local importances for $X_j$ would be
proportional to $p(t_L) + p(t_R)=p(t)$, which does not change anything.
Similarly, one can recursively invoke the same argument if $X_j$ was to be used
deeper in $t_L$ or $t_R$.

A second reason comes from the
fact that local importances are collected only in nodes $t$ where $X_j$ is used.
By contrast, if local importances were summed over all nodes (e.g., using
surrogate splits), then it would necessarily depend on the total number of nodes
in a tree, which itself directly depends on $p$ -- that is, not on $r$.

Finally, it is also worth noting that this result is consistent with the work
of~\citet{biau:2012}, who proved that rate of convergence of forests
of randomized trees also only depends on the relevant variables.

\section{Variable importances in random forest variants}
\label{sec:6:variants}

In this section, we consider and discuss variable importances as computed with
other types of ensembles of randomized trees. We first show how our results
extend to any other impurity measure, and then analyze importances computed by
depth-pruned ensemble of randomized trees and those computed by randomized
trees built on random subspaces of fixed size. Finally, we discuss the case of
non-totally randomized trees.

\subsection{Generalization to other impurity measures}

Although our characterization in sections~\ref{sec:6:importances} and
\ref{sec:6:variable-relevance} uses Shannon entropy as the impurity measure,
theorems~\ref{thm:imp}, \ref{thm:irrelevant} and \ref{thm:relevant} hold for
other impurity measures, simply substituting conditional mutual information for
conditional impurity reduction in the different formulas and in the definition
of irrelevant variables.  In particular, our results thus hold for the Gini
index in classification and can be extended to regression problems using
variance as the impurity measure.

Let us consider a generic impurity measure $i(Y|t)$ and, by mimicking the
notation used for conditional mutual information, let us denote by $G(Y;X_j|t)$
the impurity decrease for a split on $X_j$ at node $t$:
\begin{equation}%\label{eqn:impsample}
G(Y;X_j|t) = i(Y|t)-\sum_{x\in{\cal X}_j} p(t_x) i(Y|t_x),
\end{equation}
where $t_{x}$ denotes the successor node of $t$ corresponding to value
$x$ of $X_j$. The importance score associated to a variable $X_j$ (see
Equation~\ref{eq:mdi}) is then rewritten:
\begin{equation}\label{eq:mdigen}
\text{Imp}(X_j) = \frac{1}{M} \sum_{m=1}^M \sum_{t \in \varphi_m} 1(j_t = j) p(t) G(Y;X_j|t).
\end{equation}

As explained in the proof of Theorem~\ref{thm:imp}, conditioning
over a node $t$ is equivalent to conditioning over an event of the
form $B(t)=b(t)$, where $B(t)$ and $b(t)$ denote respectively the set
of variables tested in the branch from the root to $t$ and their
values in this branch. When the learning sample size $N$ grows to
infinity, this yields the following population based impurity decrease
at node $t$:
\begin{align}
&G(Y;X_j|B(t)=b(t)) \\
&=i(Y|B(t)=b(t))-\sum_{x\in{\cal X}_j} P(X_j=x|B(t)=b(t)) i(Y|B(t)=b(t),X_j=x) \nonumber \label{eqn:imppop}
\end{align}
Again by analogy with conditional entropy and mutual
information\footnote{Note however that $G(Y;X_j|B)$ does not share all
  properties of conditional mutual information as for example $G(X_j;
  Y|B)$ might not be equal to $G(Y; X_j|B)$ or even be defined,
  depending on the impurity measure and the nature of the output
  $Y$.}, let us define $i(Y|B)$ and $G(Y;X_j|B)$ for some subset of
variables $B\subseteq V$ as follows:
\begin{align}
i(Y|B)&=\sum_{b} P(B=b) i(Y|B=b)\\
G(Y;X_j|B)&= \sum_{b} P(B=b) G(Y;X_j|B=b)\\
&=i(Y|B)-i(Y|B,X_j) \nonumber
\end{align}
where the sums run over all possible combinations $b$ of values for
the variables in $B$.

With these notations, the proof of Theorem~\ref{thm:imp} can be easily adapted to lead
to the following generalization of Equation~\ref{eqn:imp-full}:
\begin{equation}
\text{Imp}(X_j)=\sum_{k=0}^{p-1} \frac{1}{C_p^k} \frac{1}{p-k} \sum_{B \in
  {\cal P}_k(V^{-j})} G(Y;X_j|B).
\end{equation}
Note that this generalization is valid without any further specific constraints
on the impurity measure $i(Y|t)$.

Let us now define as {\it irrelevant to $Y$ with respect to $V$} a
variable $X_i$ for which, for all $B\subseteq V$,
$G(Y;X_i|B)=0$ (i.e. a variable that neither affects impurity
whatever the conditioning). From this definition, one can deduce the
following property of an irrelevant variable $X_i$ (for all
$B\subseteq V$ and $X_j\in V$):
$$G(Y;X_j|B\cup\{X_i\})=G(Y;X_j|B).$$

Indeed, by a simple application of previous definitions, we have:
\begin{align}
& G(Y;X_j|B)-G(Y;X_j|B\cup\{X_i\})  \nonumber \\
& = i(Y|B)-i(Y|B\cup\{X_j\})-i(Y|B\cup\{X_i\})+i(Y|B\cup\{X_i,X_j\})\nonumber\\
& =  i(Y|B)-i(Y|B\cup\{X_i\})-i(Y|B\cup\{X_j\})+i(Y|B\cup\{X_i,X_j\})\nonumber\\
& =  G(Y;X_i|B)-G(Y;X_i|B\cup\{X_j\})\nonumber\\
& =  0,
\end{align}
where the last step is a consequence of the irrelevance of $X_i$.

Using this property, the proofs of Lemma~\ref{lemma:adding-irrelevant}
and Theorem~\ref{thm:relevant} can be straightforwardly adapted, showing that,
in the general case also, the MDI importance of a variable is
invariant with respect to the removal or the addition of irrelevant
variables.

Given the general definition of irrelevance, all irrelevant variables
also get zero MDI importance but, without further constraints on the
impurity measure $i$, there is no guarantee that all relevant
variables (defined as all variables that are not irrelevant) will get
a non zero importance. This property, and in consequence
theorem~\ref{thm:irrelevant}, will be however satisfied as soon as the
impurity measure is such that $G(Y;X_j|B)\geq 0$ for all $X_j\in V$
and for all $B\subseteq V$.

Previous developments show that all results presented in this chapter remain valid for any impurity measure leading to non negative
impurity decreases, provided that the definition of variable
irrelevance is adapted to this impurity measure. The choice of a
specific impurity measure should thus be guided by the meaning one
wants to associate to irrelevance.

Measuring impurity with Shannon entropy, i.e., taking $i(Y|t)=H(Y|t)$
and $i(Y|B=b)=H(Y|B=b)$, one gets back all previous results. Given the
properties of conditional mutual information, irrelevance for this
impurity measure strictly coincides with conditional independence: a
variable $X_i$ is irrelevant to $Y$ with respect to $V$ if and only if
$X_i\perp Y|B$ for all $B\subseteq V$.

A common alternative to Shannon entropy for growing classification
trees is Gini index, which, in the finite and infinite
sample cases, is written:
\begin{align}
i(Y|t)&= -\sum_j p(j|t) (1-p(j|t))\\
i(Y|B=b)&= -\sum_j P(Y=j|B=b) (1-P(Y=j|B=b)).
\end{align}
Like the Shannon entropy, this measure leads to non negative impurity
decreases and the corresponding notion of irrelevance is also directly
related to conditional independence.

The most common impurity measure for regression is variance, which, in
the finite and infinite sample cases, is written:
\begin{align}
i(Y|t)&=\frac{1}{N_t} \sum_{i\in t} (y_i-\frac{1}{N_t} \sum_{i\in t} y_i)^2\\
i(Y|B=b)&=E_{Y|B=b}\{(Y-E_{Y|B=b}\{Y\})^2\}.
\end{align}
Variance can only decrease as a consequence of a split and therefore,
Theorem~\ref{thm:irrelevant} is also valid for this impurity measure,
meaning that only irrelevant variables will get a zero variance
reduction. Note however that with this impurity measure, irrelevance
is not directly related to conditional independence, as some variable
$X_i$ can be irrelevant in the sense of our definition and still
affects the distribution of output values.

\subsection{Pruning and random subspaces}
\label{sec:6:rs}

In sections~\ref{sec:6:importances} and \ref{sec:6:variable-relevance}, we
considered totally randomized trees that were fully developed, i.e. until all
$p$ variables were used within each branch. When totally randomized trees are
developed only up to some smaller depth $q \leq p$, we show in
Proposition~\ref{proposition:pruning} that the variable importances as computed
by these trees is limited to the $q$ first terms of Equation~\ref{eqn:imp-full}.
We then show in Proposition~\ref{proposition:imp-subspaces} that these
latter importances are actually the same as when each tree of the ensemble is
fully developed over a random subspace~\citep{ho:1998} of $q$ variables
drawn prior to its construction.

\begin{proposition}\label{proposition:pruning}
  The importance of $X_j \in V$ for $Y$ as computed with an
  infinite ensemble of pruned totally randomized trees built up to depth $q \leq p$ and an
infinitely large training sample is:
  \begin{equation}\label{eqn:imp-pruning}
  \text{Imp}(X_j)=\sum_{k=0}^{q-1} \frac{1}{C_p^k} \frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)
  \end{equation}
\end{proposition}

\begin{proof}
The proof of Theorem~\ref{thm:imp} can be directly adapted to prove
Proposition~\ref{proposition:pruning}. If the recursive procedure is stopped at
depth $q$, then it means that $B(t)$ may include up to $q-1$ variables, which is
strictly equivalent to summing from $k=0$ to $q-1$ in the outer sum of
Equation~\ref{eqn:imp-full}.
\end{proof}

\begin{proposition}\label{proposition:imp-subspaces}
  The importance of $X_j \in V$ for $Y$ as computed with an infinite ensemble   of
pruned totally randomized trees built up to depth $q \leq p$ and an infinitely
large training sample is identical to the importance as computed  for $Y$ with an
infinite ensemble of fully developed totally randomized trees built on random
subspaces of $q$ variables drawn from $V$.
\end{proposition}

\begin{proof}
Let us define a random subspace of size $q$ as a random subset $V_S \subseteq V$
such that $|V_S|=q$. By replacing $p$ with $q$ in Equation~\ref{eqn:imp-full}
(since each tree is built on $q$ variables) and adjusting by the probability
$$\frac{C^{q-k-1}_{p-k-1}}{C^q_p}$$ of having selected $X_j$ and the $k$ variables in
the branch when drawing $V_S$ prior to the construction of the tree, it comes:

\begin{equation}
\text{Imp}(X_j)=\sum_{k=0}^{q-1} \frac{C^{q-k-1}_{p-k-1}}{C^q_p} \frac{1}{C_q^k} \frac{1}{q-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)
\end{equation}

The multiplicative factor in the outer sum can then be simplified as follows:
\begin{align}
\frac{C^{q-k-1}_{p-k-1}}{C^q_p} \frac{1}{C_q^k} \frac{1}{q-k} &= \frac{\frac{(p-k-1)!}{(p-k)!(q-k-1)!}}{\frac{p!}{(p-q)!q!}} \frac{1}{C_q^k} \frac{1}{q-k}\nonumber\\
&= \frac{(p-k-1)!q!}{(q-k-1)!p!} \frac{1}{C_q^k} \frac{1}{q-k} \nonumber\\
&= \frac{q(q-1) ... (q-k)}{p(p-1) ... (p-k)} \frac{1}{C_q^k} \frac{1}{q-k}\nonumber\\
&= \frac{q(q-1) ... (q-k)}{p(p-1) ... (p-k)} \frac{k!(q-k)!}{q!} \frac{1}{q-k}\nonumber\\
&= \frac{1}{p(p-1) ... (p-k)} \frac{k!(q-k)!}{(q-k-1)!} \frac{1}{q-k}\nonumber\\
&= \frac{k!}{p(p-1) ... (p-k)}\nonumber \\
&= \frac{k!(p-k)!}{p!} \frac{1}{p-k} \nonumber\\
&= \frac{1}{C^k_p} \frac{1}{p-k} \nonumber\\
\end{align}
which  yields the same importance as in Proposition~\ref{proposition:pruning} and proves the
proposition.
\end{proof}

As long as $q  \geq r $ (where $r$ denotes the number of relevant variables in
$V$), it can easily be shown that all relevant variables will still obtain a
strictly positive importance, which will however differ in general from the
importances computed by fully grown totally randomized trees built over all
variables.  Also, each irrelevant variable of course keeps an importance equal
to zero, which means that, in asymptotic conditions, these pruning and random
subspace methods would still allow us identify the relevant  variables, as long
as we have a good upper bound $q$ on $r$.

% \todo{If r is known (or estimated), can't we built more precise importances from RS for size r?}

\subsection{Non-totally randomized trees}
\label{sec:ntrt}

In our analysis in the previous sections, trees are built totally at random and
hence do not directly relate to those built in Random
Forest~\citep{breiman:2001} or in Extremely Randomized
Trees~\citep{geurts:2006}. To better understand the importances as computed by
those algorithms,  let us consider a close variant of totally randomized trees:
at each node $t$, let us instead draw uniformly at random $1 \leq K \leq p$
variables  and let us choose the one that maximizes $\Delta i(t)$. As
previously, $t$ is split into as many subtrees as the cardinality of the chosen
variable. Asymptotically, for binary variables, this variant exactly matches
Random Forests and Extremely Randomized Trees. For variables with a larger
cardinality, the correspondence no longer exactly holds but the trees still
closely relate. Notice that, for $K=1$, this procedure amounts to building
ensembles of  totally randomized trees as defined before, while, for $K=p$, it
amounts to building classical single trees in a deterministic way.

First, the importance of  $X_j \in V$ as computed with an infinite ensemble of
such randomized trees is not the same as Equation~\ref{eqn:imp-full}. For $K>1$,
masking effects indeed appear: at $t$, some variables are never selected
because there always is some other variable for which $\Delta i(t)$ is larger.
Such effects tend to pull the best variables  at the top of the trees and to
push the others at the leaves.  As a result, the importance of a variable no
longer decomposes into a sum including all $I(X_j;Y|B)$ terms. The
importance of the best variables decomposes into a sum of their mutual
information alone or conditioned only with the best others -- but not
conditioned with all variables since they no longer ever appear at the bottom of
trees. By contrast, the importance of the least promising variables now
decomposes into a sum of their mutual information conditioned only with all
variables -- but not alone or conditioned with a couple of others since they no
longer ever appear at the top of trees. In other words, because of the guided
structure of the trees, the importance of $X_j$ now takes into account only some
of the conditioning sets $B$, which may over- or underestimate its overall
relevance.

To make things clearer, let us consider a simple example. Let $X_1$ perfectly explains $Y$
 and let $X_2$ be a slightly noisy copy of $X_1$ (i.e., $I(X_1;Y)
\approx I(X_2;Y)$, $I(X_1;Y|X_2)=\epsilon$ and $I(X_2;Y|X_1)=0$). Using totally
randomized trees, the importances of $X_1$ and $X_2$ are nearly equal -- the
importance of $X_1$ being slightly higher than the importance of $X_2$:
\begin{align}
\text{Imp}(X_1) &= \frac{1}{2} I(X_1;Y) + \frac{1}{2} I(X_1;Y|X_2) = \frac{1}{2} I(X_1;Y) + \frac{\epsilon}{2}\\
\text{Imp}(X_2) &= \frac{1}{2} I(X_2;Y) + \frac{1}{2} I(X_2;Y|X_1) = \frac{1}{2} I(X_2;Y) + 0
\end{align}
In non-totally randomized trees, for $K=2$, $X_1$ is always selected at the root
node and $X_2$ is always used in its children. Also, since $X_1$ perfectly
explains $Y$, all its children are pure and the reduction of entropy when
splitting on $X_2$ is null. As a result, $\text{Imp}_{K=2}(X_1) = I(X_1;Y)$ and
$\text{Imp}_{K=2}(X_2) = I(X_2;Y|X_1) = 0$. Masking effects are here
clearly visible: the true importance of $X_2$ is masked by $X_1$ as if $X_2$
were irrelevant, while it is only a bit less informative than $X_1$.

As a direct consequence of the example above, for $K>1$, it is no longer true
that a variable is irrelevant if and only if its importance is zero. In the same
way, it can also be shown that the importances become dependent on the number of
irrelevant variables. Let us indeed consider the following counter-example: let
us add in the previous example an irrelevant variable $X_i$ with respect to
$\{X_1, X_2\}$ and let us keep $K=2$. The probability of selecting $X_2$ at the
root node now becomes positive, which means that $\text{Imp}_{K=2}(X_2)$ now includes
$I(X_2;Y)>0$ and is therefore strictly larger than the importance computed
before. For $K$ fixed, adding irrelevant variables dampens masking effects,
which thereby makes importances indirectly dependent on the number of irrelevant
variables.

In conclusion, the importances as computed with totally randomized trees
exhibit properties that do not possess, by extension, neither Random Forests nor
Extremely Randomized Trees. With totally randomized trees, the importance of
$X_j$ only depends on the relevant variables and is $0$ if and only if $X_j$ is
irrelevant. As we have shown, it may no longer be the case for $K>1$.
Asymptotically, the use of totally randomized trees for assessing the importance of
a variable may therefore be more appropriate. In a finite setting (i.e., a
limited number of samples and a limited number of trees), guiding the choice of
the splitting variables remains however a sound strategy. In such a case,
$I(X_j;Y|B)$ cannot be measured neither for all $X_j$ nor for all $B$. It is
therefore  pragmatic to promote those that look the most promising, even if
the resulting importances may be biased.

\section{Illustration}
\label{sec:6:illustration}

In this section, we consider the digit recognition
problem of \citep{breiman:1984} for illustrating variable importances as
computed with totally randomized trees. We verify that they match with our
theoretical developments and that they decompose as foretold. We also compare
these importances with those computed by an ensemble of non-totally randomized
trees, as discussed in section~\ref{sec:ntrt}.%, for increasing values of $K$.

\begin{figure}
    \centering
    \includegraphics[]{figures/ch6_led.pdf}
    \caption{7-segment display}
    \label{fig:digits}
\end{figure}

Let us consider a seven-segment indicator displaying numerals using horizontal and
vertical lights in on-off combinations, as illustrated in
Figure~\ref{fig:digits}. Let $Y$ be a random variable taking its value in $\{0, 1,
..., 9\}$ with equal probability and  let $X_1, ..., X_7$ be binary variables
whose values are each determined univocally given the corresponding value of $Y$
in Table~\ref{table:digits}.

Since Table~\ref{table:digits} perfectly defines the data distribution, and
given the small dimensionality of the problem, it is practicable to directly
apply Equation~\ref{eqn:imp-full} to compute variable importances. To verify our
theoretical developments, we then compare in Table~\ref{table:imp} variable
importances as computed by Equation~\ref{eqn:imp-full} and those yielded by an
ensemble of $10000$ totally randomized trees ($K=1$). Note that given the  known
structure of the problem, building trees on a sample of finite size  that
perfectly follows the data distribution amounts to building them on a sample of
infinite size. At best, trees can thus be built on a 10-sample dataset,
containing exactly one sample for each of the equiprobable outcomes of $Y$. As
the table illustrates, the importances yielded by totally randomized trees match
those computed by Equation~\ref{eqn:imp-full}, which confirms
Theorem~\ref{thm:imp}. Small differences stem from the fact that a finite number
of  trees were built in our simulations, but those discrepancies should
disappear as the size of the ensemble grows towards infinity. It also shows that
importances indeed add up to $I(X_1, ... X_7;Y)$, which confirms
Theorem~\ref{thm:sum-of-imp}.
Regarding the actual importances, they indicate that $X_5$ is stronger than all
others, followed -- in that order -- by $X_2$, $X_4$ and $X_3$ which also show
large importances. $X_1$, $X_7$ and $X_6$ appear to be the less informative. The
table also reports importances for increasing values of $K$. As discussed before,
we see that a large value of $K$ yields importances that can be either
overestimated (e.g., at $K=7$, the importances of $X_2$ and $X_5$ are larger
than at $K=1$) or underestimated due to masking effects (e.g., at $K=7$, the
importances of $X_1$, $X_3$, $X_4$ and $X_6$ are smaller than at $K=1$, as if
they were less important). As Figure~\ref{fig:order} illustrates, it can also be observed that masking effects may even
induce changes in the variable rankings (e.g., compare the rankings at $K=1$ and
at $K=7$), which thus confirms that importances are differently affected.

\begin{table}
    \centering
    \begin{tabular}{| c | c c c c c c c |}
    \hline
    $y$ & $x_1$ & $x_2$ & $x_3$ & $x_4$ & $x_5$ & $x_6$ & $x_7$ \\
    \hline
    \hline
    0 & 1 & 1 & 1 & 0 & 1 & 1 & 1 \\
    1 & 0 & 0 & 1 & 0 & 0 & 1 & 0 \\
    2 & 1 & 0 & 1 & 1 & 1 & 0 & 1 \\
    3 & 1 & 0 & 1 & 1 & 0 & 1 & 1 \\
    4 & 0 & 1 & 1 & 1 & 0 & 1 & 0 \\
    5 & 1 & 1 & 0 & 1 & 0 & 1 & 1 \\
    6 & 1 & 1 & 0 & 1 & 1 & 1 & 1 \\
    7 & 1 & 0 & 1 & 0 & 0 & 1 & 0 \\
    8 & 1 & 1 & 1 & 1 & 1 & 1 & 1 \\
    9 & 1 & 1 & 1 & 1 & 0 & 1 & 1 \\
    \hline
    \end{tabular}
    \caption{Values of $Y, X_1, ..., X_7$}
    \label{table:digits}
\end{table}

\begin{table}
    \begin{tabular}{| c | c c c c c c c c |}
    \hline
        & Eqn.~\ref{eqn:imp-full} & $K=1$ & $K=2$ & $K=3$ & $K=4$ & $K=5$ & $K=6$ & $K=7$ \\
    \hline
    \hline
    $X_1$ & 0.412 & 0.414 & 0.362 & 0.327 & 0.309 & 0.304 & 0.305 & 0.306\\
    $X_2$ & 0.581 & 0.583 & 0.663 & 0.715 & 0.757 & 0.787 & 0.801 & 0.799\\
    $X_3$ & 0.531 & 0.532 & 0.512 & 0.496 & 0.489 & 0.483 & 0.475 & 0.475\\
    $X_4$ & 0.542 & 0.543 & 0.525 & 0.484 & 0.445 & 0.414 & 0.409 & 0.412\\
    $X_5$ & 0.656 & 0.658 & 0.731 & 0.778 & 0.810 & 0.827 & 0.831 & 0.835\\
    $X_6$ & 0.225 & 0.221 & 0.140 & 0.126 & 0.122 & 0.122 & 0.121 & 0.120\\
    $X_7$ & 0.372 & 0.368 & 0.385 & 0.392 & 0.387 & 0.382 & 0.375 & 0.372\\
    \hline
    \hline
    $\sum$& 3.321 & 3.321 & 3.321 & 3.321 & 3.321 & 3.321 & 3.321 & 3.321\\
    \hline
    \end{tabular}
    \caption{Variable importances as computed with an ensemble of randomized trees, for increasing values of $K$. Importances at $K=1$ follow their theoretical values, as predicted by Equation~\ref{eqn:imp-full} in Theorem~\ref{thm:imp}. However, as $K$ increases,  importances diverge due to masking effects. In accordance with Theorem~\ref{thm:sum-of-imp}, their sum is also always equal to $I(X_{1}, \ldots, X_{7}; Y) = H(Y) = \log_{2}(10)= 3.321$ since inputs allow to perfectly predict the output.}
    \label{table:imp}
\end{table}

\begin{figure}
    \centering
    \includegraphics[width=0.9\textwidth]{figures/ch6_order.pdf}
    \caption{Variable importances as computed with an ensemble of randomized trees, for increasing values of $K$. Importances at $K=1$ follow their theoretical values, as predicted by Equation~\ref{eqn:imp-full} in Theorem~\ref{thm:imp}. However, as $K$ increases,  importances diverge due to masking effects.}
    \label{fig:order}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[width=0.9\textwidth]{figures/ch6_imp_led.pdf}
    \caption{Decomposition of variable importances along the degrees $k$ of interactions of one variable with the other ones. At $K=1$, all $I(X_j;Y|B)$ are accounted for in the total importance, while at $K=7$ only some of them are taken into account due to masking effects.}
    \label{fig:decomposition}
\end{figure}

To better understand why a variable is important, it is also insightful to look
at its decomposition into its $p$ sub-importances terms, as shown in
Figure~\ref{fig:decomposition}. Each row in the plots of the figure corresponds
to one the $p$ variables $X_1,\dots,X_7$ and each column to a size $k$ of conditioning sets.
As such, the value at row $j$ and column $k$ corresponds the importance of $X_j$
when conditioned with $k$ other variables (e.g., to the term $\frac{1}{C^k_p}
\frac{1}{p-k} \sum_{B \in {\cal P}_k(V^{-j})} I(X_j;Y|B)$ in Equation~\ref{eqn:imp-full}
in the case of totally randomized trees). In the left plot, for
$K=1$, the figure first illustrates how importances yielded by totally
randomized trees decomposes along the degrees $k$ of interactions terms. We can
observe that they each equally contribute (at most) the total importance of a
variable. The plot also illustrates why $X_5$ is important: it is informative
either alone or conditioned with any combination of the other variables (all of
its terms are significantly larger than $0$). By contrast, it also clearly shows
why $X_6$ is not important: neither alone nor combined with others $X_6$ seems
to be very informative (all of its terms are close to $0$). More interestingly,
this figure also highlights redundancies: $X_7$ is informative alone or
conditioned with a couple of others (the first terms are significantly larger
than $0$), but becomes uninformative  when conditioned with
many others (the last terms are closer to $0$). The right plot, for $K=7$,
illustrates the decomposition of importances when variables are chosen in a
deterministic way. The first thing to notice is masking effects. Some of the
$I(X_j;Y|B)$ terms are indeed clearly never encountered and their contribution
is therefore reduced to $0$ in the total importance. For instance, for $k=0$,
the sub-importances of $X_2$ and $X_5$ are positive, while all others are null,
which means that only those two variables are ever selected at the root node,
hence masking the others. As a consequence, this also means that the importances
of the remaining variables are biased and that they actually only account of their
relevance when conditioned to $X_2$ or $X_5$, but not of their relevance in
other contexts. At $k=0$, masking effects also amplify the contribution of $I(X_2;Y)$
(resp. $I(X_5;Y)$) since $X_2$ (resp. $X_5$) appears more frequently at the root
node than in totally randomized trees. In addition, because nodes become pure
before reaching depth $p$, conditioning sets of size $k\geq4$ are never actually
encountered, which means that we can no longer know whether variables are still
informative when conditioned to many others. All in all, this figure thus indeed
confirms that importances as computed with non-totally randomized trees take
into account only some of the conditioning sets $B$, hence biasing the measured
importances.

% \todo{Insights from the decomposition}

% - Along terms => Identify interactions of a given order (strongly, weakly)
% - Along pairs => Identify interactions between variables
%   Détection des directions

\section{Conclusions}

In this chapter, we made a first step towards understanding variable importances as
computed with a forest of randomized trees. In particular, we derived a
theoretical characterization of the Mean Decrease Impurity importances as
computed by totally randomized trees in asymptotic  conditions.  We showed that
they offer a three-level decomposition of the information jointly provided by
all input variables about the output (Section~\ref{sec:6:theory}). We then demonstrated
(Section~\ref{sec:6:variable-relevance}) that MDI importances as computed by totally randomized
trees exhibit desirable properties for  assessing the relevance of a variable:
it is equal to zero if and only if the variable is irrelevant and it depends
only on the relevant variables. We discussed the case of Random Forests and
Extremely Randomized Trees (Section~\ref{sec:6:variants}) and finally illustrated our
developments on an artificial but insightful example (Section~\ref{sec:6:illustration}).

There remain several limitations to our framework that we would like address in
the future. First, our results should be adapted to binary splits as used within
an actual Random Forest-like algorithm. In this setting, any node $t$ is split
in only two subsets, which means that any variable may then appear one or
several times within a branch, and thus should make variable importances now
dependent on the cardinalities of the input variables. In the same direction,
our framework should also be extended to the case of continuous variables.
Finally, results presented in this work are valid in an asymptotic setting only.
An important direction of future work includes the characterization of the
distribution of variable importances in a finite setting.


\documentclass[main.tex]{subfiles}
\begin{document}

\paragraph{A \(2 \to 3\) process}

\marginpar{Thursday\\ 2021-11-11}

Before looking at three-body decays, let us consider a \(2 \to 3 \) process.

Consider
%
\begin{align}
\ce{K}^{-} + p \to \pi^{+} + \pi^{-} + \Lambda
\,,
\end{align}
%
where the masses are \(m_\pi \approx \SI{1}{GeV}\), \(m_ \pi \approx \SI{140}{GeV}\), \(m_K \approx \SI{500}{MeV}\), and \(m_\Lambda \approx \SI{1.1}{GeV}\).

The experimental setup is a kaon beam impacting upon a proton target.

The invariant mass of the starting particles will be
%
\begin{align}
m_{Kp} = \frac{1}{c} \sqrt{(\sum  p_i)^2}
 = \frac{1}{c} \sqrt{(p_k + p_p)^2}
\,.
\end{align}

We could also define a corresponding energy \(m_{Kp} c^2\).

If we extend this sum to all the particles in the initial state, \(m _{\text{initial}} c^2 =  m_{\text{final}}c^2 = \sqrt{s}\) is the total invariant mass.
The variable \(s\) is written as \(s = c^2 p^2 _{\text{tot}}\).
Its meaning is that it is the total energy in the center-of-mass frame.

We can then think of this as an equivalent decay of a particle with \(m_{Kp}\), therefore the kinematic threshold is \(m_{Kp} \geq m_{\pi^{+}} + m_{\pi^{-}} + m_\Lambda \).
This is a general statement:
%
\begin{align}
\sqrt{s} \geq \sum _{i \in \text{products}} m_i c^2
\,.
\end{align}

Let us approximate the protons as stationary, and we have a kaon with \(E_K\).

The total four-momentum reads
%
\begin{align}
p _{\text{tot}} = (E_K + m_p, \vec{p}_k)^{\top}
\,,
\end{align}
%
so we get
%
\begin{align}
\sqrt{s} &= \sqrt{(E_k + m_p)^2 - \abs{\vec{p}_k}^2}  \\
&=\sqrt{m_K^2 + m_p^2 + 2 E_k m_p}
\,.
\end{align}

We can then straightforwardly compute the energy threshold for \(E_k\).
In this case we have \(E_k^{\text{threshold}} < m_k\), so we have no real lower limit.

This reasoning can be applied, for example, when looking at the reaction
%
\begin{align}
\ce{p} + \ce{p} \to 3 \ce{p} + \overline{p}
\,.
\end{align}

The threshold for the energy of the single proton impacting on the target comes out to be \(7 m_p \approx \SI{6.6}{GeV}\).

This is counterintuitive! It is due to the fact that we lose a lot of energy when boosting back to the CoM frame.

Let us go back to the \(2 \to 3\) reaction.
We can always compute the invariant mass of a subsystem such as \(m_{\pi^{\pm}\Lambda }\):
%
\begin{align}
m_{\pi^{\pm \Lambda }} = \sqrt{(E_{\pi^{\pm}} + E_\Lambda )^2 - \abs{\vec{p}_{\pi^{\pm }} + \vec{p}_\Lambda }^2}
\,,
\end{align}
%
which we can compute once we have the measurement from our detector.

\todo[inline]{See ``dalitz-pi-lambda'' figure in the drive.}

We can look at the histogram of \(m_{\pi^{+}\Lambda }^2\) versus \(m_{\pi^{-}\Lambda }^2\).

There are peaks!
This suggests the presence of something decaying into \(\pi^{\pm }\Lambda \).
This particle is a \(\Sigma^{\pm}\), and what happened was
%
\begin{align}
\ce{K}^{-} + p &\to \Sigma^{\pm} + \pi^{\mp}  \\
\Sigma^{\pm} &\to \pi^{\pm} + \Lambda
\,.
\end{align}

We have not detected the \(\Sigma \) particle, which was short-lived.

We know that there should be a range \(m_2 + m_2 \leq m_{23} \leq M - m_1 \).

This can be extended to a constraint on the two invariant masses.

This is called a \textbf{Dalitz plot}, a common trick to find peaks for new particles.

The (true?) width of the peak is about \SI{36}{MeV}, which corresponds to \SI{1.8e-23}{s}.
Is this a \(\Sigma (1385)\)?

\todo[inline]{But we should also consider experimental error in the determination of the energy\dots}

An alternative way to measure lifetimes is to look at track length: the path length we expect is
%
\begin{align}
\lambda = \beta c \tau \gamma
\,,
\end{align}
%
where we need the Lorentz factor since the particle decays in time \(\tau \) in its own rest frame.

In this case the track length would be about \SI{5.5}{fm}; however we also have the \(\gamma \) factor: if that is large enough we may get something measurable.

We'd want to see at least a few points.
If \(\sigma _x\) is our spatial resolution, we need a track length which is at least \(\gtrsim 5 \sigma _x\).

The best spatial trackers are silicon detectors or emulsion trackers, on the order of \SI{10}{\micro m}.
This means that we need to increase our path length by a factor \(\num{e10}\)\dots
The \(\gamma \) factor is surely not that large: it would mean having a beam with energy on the order of \SI{e19}{eV}.

Let us consider another example: \(\ce{p} + \gamma \to \Delta^{+} \), which can decay into \(\pi^{+} + \ce{n}\) or \(\pi^{0} + \ce{p}\).

What is the kinematic threshold for this reaction?
This is a cosmic ray proton interacting with a CMB photon.

These photons have an average energy of \(\SI{2.7}{K} \approx \SI{230}{\micro eV}\).

This does not precisely match \(\Omega_{0 \gamma } \rho _c c^2\) when multiplied by \(n_{0 \gamma }\), since we'd need to integrate the Planckian.

The value of \(\sqrt{s}\) is
%
\begin{align}
\sqrt{s} &= \sqrt{m_p^2 + 2 \qty(E_p E_\gamma - \vec{p}_p \cdot \vec{p}_\gamma )}  \\
&= \sqrt{m_p^2 + 2 E_p E_\gamma \qty(1 - \cos \theta_{p \gamma })} \geq m_{\Delta^{+}}
\,.
\end{align}

This means that
%
\begin{align}
E_p \geq \frac{m^2_{\Delta^{+}} - m_p^2}{2 E_\gamma \qty(1 - \cos \theta_{p \gamma })}
\,.
\end{align}

If the collision is head-on, we get \(1 - \cos = 2\), while in the other case the threshold diverges since the proton cannot reach the photon.

We should average the formula over all values of \(\theta_{p \gamma }\).

The threshold is about \SI{e19}{eV}.
This is the GZK cutoff, or GZK effect.
In order to properly study this phenomenon we would need
to also look at the probability: \cite[]{groupReviewParticlePhysics2020} says it is of the order of \(\SI{e-1}{mb}\) for the \(\gamma p\) process.

We can study this in the lab simply, we only need \SI{200}{MeV} photons colliding on stationary protons.

With this, we can find that a typical proton above the threshold will travel for only tens of \SI{}{Mpc} before losing energy to this process.

The products from protons being annihilated in this way will produce many secondary particles: photons, neutrinos, electrons, positrons, muons and more, all being very energetic.

Let us do another possibility: could photons from a high-energy source be absorbed by the CMB?
For example, you can have pair production.

We need \(\sqrt{s}\) for the process to be larger or equal than about \SI{1}{MeV}.
This comes out to be about \SI{e14}{eV}.

Let's see how that works. We are looking at the process \(\gamma + \gamma \to e^{+} + e^{-}\); in the lab frame (which is the frame in which the CMB has vanishing dipole moment, the universal rest frame) one photon will have a typical energy of \(E_{\gamma _{\text{CMB}}} \approx \SI{235}{\micro eV}\), while the other will have a large energy \(E_\gamma \).

The invariant mass of the scattering will read
%
\begin{align}
\sqrt{s} &= \sqrt{(p_\gamma +p_{\gamma _{\text{CMB}}})^2}  \\
&= \sqrt{
    (E_\gamma + E_{\gamma _{\text{CMB}}})^2 -
    (\vec{p} _{\gamma } + \vec{p}_{\gamma _{\text{CMB}}})^2}  \\
&= \sqrt{2 E_\gamma E_{\gamma _{\text{CMB}}} (1 - \cos \theta_{\gamma \gamma })}
\marginnote{\(E_\gamma^2 - \vec{p}_\gamma^2 = 0\) for both photons.}
\,,
\end{align}
%
which we want to be \(\geq 2 m_e \approx \SI{1}{MeV}\). This condition reads
%
\begin{align}
E_\gamma \geq \frac{(\SI{1}{MeV})^2}{2 E_{\gamma _{\text{CMB}}}(1 - \cos \theta_{\gamma \gamma })} \approx \frac{\SI{2.2}{PeV}}{(1 - \cos \theta_{\gamma \gamma })}
\,.
\end{align}

\end{document}