\documentclass[10pt, a4paper, twoside]{basestyle}
\usepackage[Mathematics]{semtex}

%%%% Shorthands.

%%%% Title and authors.

\title{Documentation for the symplectic methods}
\date{\printdate{2015-06-06}}
\author{Robin~Leroy (eggrobin)}
\begin{document}
\maketitle
This document expands on the comments at the beginning of\\
\texttt{integrators/symplectic\_runge\_kutta\_nyström\_integrator.hpp}.

\section{Differential equations.}
Recall that the equations solved by this class are
\begin{align}
\tuple{\vq,\vp}\der &=
\vX\of{\vq, \vp, t} = \vA\of{\vq, \vp} + \vB\of{\vq, \vp, t}
\quad\parbox{.4\linewidth}{with $\exp h\vA$ and $\exp h\vB$ known and
$\commutator{\vB}{\commutator{\vB}{\commutator{\vB}{\vA}}}=\nullvec$;}
\label{general}\\
\span\parbox{.7\linewidth}{the above equation, with $\exp h\vA = \Identity+h\vA$,
$\exp h\vB = \Identity+h\vB$,
and $\vA$ and $\vB$ known;}
\label{linear}\\
\vq\dder &= -\matM^{-1} \grad_\vq V\of{\vq, t}\text. \label{rkn}
\end{align}

\section{Relation to Hamiltonian mechanics.}
The third equation above is a reformulation of Hamilton's
equations with a Hamiltonian of the form
\begin{equation}
H\of{\vq,\vp,t} = \frac{1}{2}\Transpose{\vp}\matM^{-1}\vp + V\of{\vq, t}\text,
\end{equation}
where $\vp = \matM\vq\der$.

\section{A remark on non-autonomy.}
Most treatments of these integrators write these differential equations as well
as the corresponding Hamiltonian in an autonomous version, thus
$\vX = \vA(\vq, \vp) + \vB(\vq, \vp)$ and
$H\of{\vq,\vp,t} = \frac{1}{2}\Transpose{\vp}\matM^{-1}\vp + V\of{\vq}$.
It is however possible to incorporate time, by considering it as an
additional variable:\[
\tuple{\vq,\vp,t}\der =
\vX\of{\vq, \vp, t} =
\tuple{\vA\of{\vq, \vp}, 1} +
\tuple{\vB\of{\vq, \vp, t}, 0}\text.\]

For equations of the form (\ref{rkn}) it remains to be shown that Hamilton's
equations with quadratic kinetic energy and a time-dependent potential satisfy
$\commutator{\vB}{\commutator{\vB}{\commutator{\vB}{\vA}}}=\nullvec$.
We introduce $t$ and its conjugate momentum $\gcp$ to the phase space,
and write
\[
\tilde{\vq}=\tuple{\vq, t}\text,\quad
\tilde{\vp}=\tuple{\vp, \gcp}\text,\quad
L\of{\tilde{\vp}} = \frac{1}{2}\Transpose{\vp}\matM^{-1}\vp + \gcp\text.
\]
(\ref{rkn}) follows from Hamilton's equations with\[
H\of{\tilde{\vq},\tilde{\vp}} =
L\of{\tilde{\vp}} + V\of{\tilde{\vq}} =
\frac{1}{2}\Transpose{\vp}\matM^{-1}\vp + \gcp + V\of{\vq, t}
\]
since we then get $t\der = 1$.
The desired property follows from the following lemma:
\begin{lemma}
Let $L\of{\tilde{\vq},\tilde{\vp}}$ be a quadratic polynomial in $\tilde{\vp}$,
$V\of{\tilde{\vq}}$ a smooth function, $\vA=\Poisson\placeholder L$, and
$\vB=\Poisson\placeholder V$.
Then\[
\commutator{\vB}{\commutator{\vB}{\commutator{\vB}{\vA}}}=\nullvec\text.\]
\end{lemma}
\begin{proof}
It suffices to show that $\Poisson V{\Poisson V{\Poisson L V}} = 0$. It is
immediate that every term in that expression will contain a third order
partial derivative in the $\tilde p_i$ of $L$, and since $L$ is quadratic
in $\tilde{\vp}$ all such derivatives vanish.
\end{proof}
See \cite[p.~26]{McLachlanQuispel2006} for a detailed treatment
of non-autonomous Hamiltonians using an extended phase space.
See \cite[p.~8]{McLachlan1993} for a proof that
$\Poisson V{\Poisson V{\Poisson L V}} = 0$ for arbitrary Poisson tensors.
\section{Composition and first-same-as-last property}
Recall from the comments that each step is computed as
\begin{align*}
\tuple{\vq_{n+1}, \vp_{n+1}} &=
\exp a_{r-1}h\vA \exp b_{r-1}h\vB \dotsb \exp a_0h\vA \exp b_0h\vB
\tuple{\vq_n, \vp_n}
\text,
\intertext{thus, when $b_0$ vanishes (type $ABA$) or when $a_{r-1}$ does
(type $BAB$),}
\tuple{\vq_{n+1}, \vp_{n+1}} &=
\exp a_{r-1}h\vA \exp b_{r-1}h\vB \dotsb \exp b_1h\vB \exp a_0h\vA
\tuple{\vq_n, \vp_n}\text{, respectively}\\
\tuple{\vq_{n+1}, \vp_{n+1}} &=
\exp b_{r-1}h\vB \exp a_{r-2}h\vA \dotsb \exp a_0h\vA \exp b_0h\vB
\tuple{\vq_n, \vp_n}\text.
\end{align*}
This leads to performance savings.

Let us consider a method of type $BAB$.
Evidently, the evaluation of $\exp a_0h\vA$
is not required, thus only $r-1$ evaluations of  $\exp \increment t\vA$ are
required.
Furthermore, if output is not needed at step $n$, the computation of
the $\pa{n-1}$th step requires only $r-1$ evaluations of
$\exp \increment t\vB$, since the consecutive evaluations of $\exp b_0h\vB$
and $\exp b_rh\vB$ can be merged by the group property,\[
\exp b_0h\vB\exp b_rh\vB=\exp \pa{b_0+b_r}h\vB\text.
\]

If the equation is of the form \ref{linear}, the latter saving can be achieved even
for dense output, since only one evaluation of $\vB$ is needed to compute the
increments $b_rh\vB$ and $b_0\vB$.

The same arguments apply to type $ABA$.
This motivates the name of the template
parameter \texttt{evaluations}, equal to $r-1$ for methods of type $ABA$ and
$BAB$, and $r$ otherwise.
\end{document}


\section*{Conclusions}
In our first attempts to find a set of feasible solutions, we tried simpler approaches, such as a simple taboo list, a steepest descent using first improvement strategy and a quite sophisticated genetic algorithm performing both swaps and random mutations but they were not effective. Because of this difficulties, we adopted a mixed approach. In fact, in both parts we use some greedy concepts, a taboo list and the idea of a periodic mutation to explore a different region in the solution space.

Our algorithm is very fast in the search of feasible solution. In most cases, it provides a set of feasible solutions in less than a second for all instances, except \emph{instance06}, because of its huge number of exams. The assignation loop cycles for some hundreds of thousands of iterations because it continuously adds and removes exams in different timeslots. In this case, the algorithm is able to find a set of feasible solutions in 8-10 seconds at most.

The optimization phase shows the same problem of the previous phase. In fact, this stage is quite fast for all instances and it produces good results which do not improve much for longer running times while, for \emph{instance06}, we noticed that it continues to improve. The reason for this behavior should be, in this case too, the huge number of exams: a single swap between two different timeslots requires to change a larger number of exams on average. Obviously, this slows down the execution of a single optimization cycle and it also affects the overall performance.

\paragraph*{Benchmark}
In order to evaluate the algorithm performance and gather some statistics we run 60 times each instance using a 60 seconds timeout. We represent with $s\,\%$ the standard deviation weighted on the average value.
\begin{center}
\small
\begin{tabular}{lrrrcc}
\toprule
Instance	&	Benchmark	&	Avg value	&	Max value	&	Avg Gap \%	&	$s\,\%$	\\
\midrule
instance01	&	157.033		&	157.289		&	157.444		&	0.16		&	0.05	\\
instance02	&	34.709		&	37.634		&	38.236		&	8.43		&	0.76	\\
instance03	&	32.627		&	35.039		&	36.359		&	7.39		&	1.35	\\
instance04	&	7.717		&	8.654		&	8.930		&	12.14		&	1.32	\\
instance05	&	12.901		&	14.786		&	15.293		&	14.61		&	1.40	\\
instance06	&	3.045		&	3.790		&	3.916		&	24.48		&	1.55	\\
instance07	&	10.050		&	10.805		&	11.009		&	7.51		&	0.88	\\
\bottomrule
\end{tabular}
\end{center}
It is possible to observe the performance of the algorithm, which in most cases produces good results with a gap < 15\% and appreciate its stability. In fact, on every instance it provides the same result, in terms of objective function, with an error < 2\%.


\subsection{Statement Composition}\label{statements:comp}

\begin{align*}
\intertext{Statements can be sequenced by semicolons. First the statement $S_1$ is run, but is not finished evaluating.}
&\inference[$\text{STATEMENTS}_1$]{\Braket{S_1,e} \Rightarrow_S \Braket{S_1',e'}}
                         {\Braket{S_1;S_2,e} \Rightarrow_S \Braket{S_1';S_2,e'}}
\intertext{In this rule, the statement $S_1$ is run, and finishes, rewriting the rule to simply be $S_2$.}
&\inference[$\text{STATEMENTS}_2$]{\Braket{S_1,e} \Rightarrow_S e'}
                         {\Braket{S_1;S_2,e} \Rightarrow_S \Braket{S_2,e'}}
\end{align*}


\documentclass{article}
\usepackage{amsmath}
\title{Operators in Quantum Mechanics}
\author{Amey Joshi}
\begin{document}
\maketitle
\abstract{One of the postulates of quantum physics states that a dynamical
variable in classical physics is represented by an operator in the quantum
theory. It is hard to imagine what does it mean by `representing' a classical
dynamical variable with an operator. It is even harder to understand how anyone
could think of such a representation. This article attempts to shed light on
the origin of this postulate}

\section{Dynamical variables as matrices}
Heisenberg \cite{heisenberg1925quantum} laid down the foundations of quantum
mechanics by starting to construct its theory based on the quantities that are
observable in an experiment. The position and momentum of an electron are not
observable.  However, the spectral lines are observed. Their frequency and
intensity are measured in experiments. Heisenberg proposes a representation of
the position of an electron in terms of the spectral frequencies and concludes
that the only way to do so is by representing position as a matrix of numbers.
This paper of Heisenberg's is famous for being hard to understand
\cite{aitchison2004understanding}. However, Heisenberg states his thoughts more
lucidly in the appendix `Mathematical Apparatus' of his book
\cite{heisenberg1949physical}.

Heisenberg's cautions \cite{heisenberg1949physical} that his argument for
representing dynamical variables as operators should not be viewed as a
mathematical proof. It is a physicist's insight whose correctness should be
judged on its success in describing natural processes measured in experiments.

\section{The experimental facts}
Heisenberg takes into account the following well-established experimental facts
to build his theory:
\begin{enumerate}
\item The Rydberg-Ritz combination principle that all spectral lines of an
element may be represented as the differences of small number of terms. If
$T_1, T_2, \ldots$ are the terms then the atomic frequencies form a two
dimensional array
\begin{equation}\label{e1}
\nu_{nm} = T_n - T_m
\end{equation}
and obey the combination principle
\begin{equation}\label{e2}
\nu_{nk} + \nu_{km} = \nu_{nm}.
\end{equation}
\item The energy levels of an atom have a discrete sets of values as observed
in the Frank-Hertz experiments.
\item The spectral frequencies are related to energy levels through the
Einstein-Bohr relation
\begin{equation}\label{e3}
\nu_{nm} = \frac{E_n - E_m}{h}.
\end{equation}
\end{enumerate}

One way to relate the dynamical variables to spectral frequencies is to write
them as Fourier series. Thus, a generalised coordinate $q_k$ of a periodic
system of $f$ degrees of freedom is
\begin{equation}\label{e4}
q_k(t)=\sum_{n_1, \ldots, n_f}q^{(k)}_{n_1, \ldots, n_f}\exp[2\pi i(n_1\nu_1 +
\cdots + n_f\nu_f)t],
\end{equation}
where $\nu_1, \ldots, \nu_f$ are the fundamental frequencies of the system.
Heisenberg considered periodic systems because an atom is one such.

If equation \eqref{e4} is to be taken over to quantum mechanics, it should
be changed so that
\begin{enumerate}
\item The amplitudes depend on frequencies of two states of the atom and
\item When we combine quantities like $q_k(t)$, the Rydberg-Ritz combination
principle should be obeyed.
\end{enumerate}

The first condition forces us to write $q_k(t)$ in terms of the quantities
\begin{equation}\label{e5}
q^{(k)}(n_1, \ldots, n_f;m_1, \ldots, m_f)
\exp\left[2\pi i\nu(n_1,\ldots,n_f;m_1,\ldots, m_f)t\right]
\end{equation}
where the amplitude and the frequencies are the ones observed in the atom's
spectrum. Instead of summing numbers of this form in a Fourier series,
Heisenberg postulated that these numbers together represent the generalised
coordinate. Thus, in quantum mechanics,
\begin{equation}\label{e6}
q_k(t) = \begin{bmatrix}
q^{(k)}_{1;1}e^{2\pi i\nu(1;1)t} & \ldots &q^{(k)}_{1;m_f}e^{2\pi i\nu(1;m_f)t}
\\
\vdots & \\
q^{(k)}_{n_f;1}e^{2\pi i\nu(n_f;1)t} & \ldots &q^{(k)}_{1;m_f}
e^{2\pi i\nu(n_f;m_f)t}
\end{bmatrix}
\end{equation}
We now use the classical representation of equation \eqref{e4} to build
analogies to put constraints on the form of the representation of equation
\eqref{e6}.
\begin{enumerate}
\item The quantity $q_k(t)$ on the left hand side of equation \eqref{e4} must
be real. Therefore, the sum on the right hand side should have pairs of
complex cojugate elements. Likewise, if the matrix must represent a dynamical
variable and if its entries are complex then we must insist that the complex
quantities occur in pairs of conjugates. From the Rydberg-Ritz combination
principle we know that $\nu(n, m) = -\nu(m, n)$. Therefore we must also have
\begin{equation}\label{e7}
q_k(m, n) = \overline{q_k(n, m)}.
\end{equation}

\item One can add quantities like \eqref{e4} so that amplitudes corresponding
to the same frequency add up. Likewise, we can add the array of numbers in
equation \eqref{e6} such that the elements corresponding to same frequencies
add up. Thus, the array of number corresponding to $q_1(t) + q_2(t)$ has
elements
\begin{equation}\label{e8}
(q^{(1)}_{m, n} + q^{(2)}_{m, n})e^{2\pi i\nu(m, n)t}.
\end{equation}

\item Each frequency $\nu(n, m)$ in \eqref{e6} must be of the form $\Omega_n
- \Omega_n$ \cite{dirac1925fundamental}. When we multiply arrays of the form
\eqref{e6} we must remember to add together the terms that belong to the
same frequency. Consider the a combination of terms like $q^{(1)}_{l, m}
q^{(2)}_{m, n}$. It corresponds to the frequency $\Omega_m - \Omega_l + \Omega_n
- \Omega_m = \Omega_n - \Omega_l$. Therefore, we must add together terms like
$q^{(1)}_{l, m}q^{(2)}_{m, n}$ for all $m$ to get the term corresponding to
the frequency $\Omega_n - \Omega_l$.
\end{enumerate}
Thus, the arrays of numbers in equation \eqref{e6} are added and multiplied like
matrices. That is how the representation of dynamical variables as matrices
came about to be in quantum mechanics.
\bibliographystyle{plain}
\bibliography{why}
\end{document}


% JuliaCon proceedings template
\documentclass{juliacon}
\usepackage{amsmath}
\setcounter{page}{1}

\begin{document}
\input{header}
\maketitle
\begin{abstract}

The aim of this paper is twofold. The first one is to describe a novel research-project designed for building bridges between machine learning and econometric worlds. The second one is to introduce the main characteristics and comparative performance of the first Julia-native all-subset regression algorithm included in \verb GlobalSearchRegression.jl  (v1.0.3). As other available alternatives, this algorithm allows researchers to obtain the best model specification among all possible covariate combinations - in terms of user defined information criteria-, but up to 3165 and 197 times faster than STATA and R alternatives, respectively.

\end{abstract}

\section{Introduction}
While allowing for better and more accurate analysis (and forecasts), fat-data availability generates a revival of Bellman’s \cite{Bellman1961} -and newer- high-dimensionality concerns:
\begin{enumerate}
    \item Exponential increase in required information and execution time \cite{Yu2003} ;
    \item Overfitting risk \cite{Defernez1999};
    \item Less-informative euclidean distances \cite{Aggarwal2001}; and
    \item Unfeasible estimators \cite{Buhlmann2011}.
\end{enumerate}

Notwithstanding, the advantage of having thousands/millions of features to deal with complex phenomena stimulates an unprecedented number of methodological -and technological- improvements to manage the ‘curse of dimensionality’ \cite{Bolon2016}. \vskip 6pt

In Economics, this process has a dual approach with machine-learning (ML) and econometric (EC) algorithms emerging for different purposes: the former for prediction/forecasts (focusing on $\hat{y}$) and the latter for estimation/causal inference (interested in $\hat{\beta}$). Alternatively, the same distinction can be expressed in Diebold’s terms as non-causal vs causal prediction (see \cite{athey2015}), where ML algorithms are designed to reduce prediction sampling-risks -i.e. learning through cross-validation techniques- and EC methods to identify unbiased multivariate relationships -i.e. avoiding consistency issues through residual and coefficient tests for model selection. \vskip 6pt

In turn, ML feature selection algorithms can be classified into three different families: Filters, Wrappers and Embedded -depending on whether they use some classifier/response variable information or not, or how variable selection is made along with the learning process, see \cite{CHANDRASHEKAR201416}. Similarly, most EC dimensionality reduction approaches can be classified into three different groups: Exhaustive, General-to-Specific and Specific-to-General -depending on the search pattern; see \cite{gluzmann2015}. \vskip 6pt

Despite significant improvements in recent econometric developments -like PCGIVE/Autometrics or Retina algorithms, which combine some ML and EC characteristics-, available alternatives fails to fully exploit cross-validation and model averaging capabilities -see \cite{Doornik09autometrics}, \cite{perezamaral2003}, \cite{DAVIDSON1981}, \cite{derksen1992}, \cite{marinucci2008}, \cite{herwartz2010}, and \cite{castle2006}. \vskip 6pt

Conversely, newer ML algorithms -like Convolutional Neural Networks or Bootstrap-Based LASSO, which improve complex non-linear adjustment or model selection under regularization schemes- achieved unprecedented forecast accuracy but disregarding model interpretability and/or parameter estimation issues -i.e omitting residual and coefficient tests; see \cite{Bzdok2018}. \vskip 6pt

Following Varian’s \cite{varian2014} advices, about ML and EC complementarities -i.e. merging algorithms from different families to reduce both sampling and model uncertainty-, we are developing a novel multi-layer-multi-algorithm methodology combining two reinforcing paradigms: The LSE “Testimation” approach -to obtain information about residual properties, see \cite{abramovich2006}- and the Bayesian-like “Double-model averaging” -across different covariates and sub-samples, see \cite{hoetening1999}. This methodology includes five complementary layers -handling cross-section, time series and panel data-: 1) Pre-processing: with outlier detection, missing values identification, seasonal adjustment and normalization/standardization functions; 2) Feature extraction: creation of logs, squares, inverses and interactions from selected variables; 3) Feature pre-selection: using filter and embedded ML algorithms like CFS, Variance threshold and LASSO functions; 4) Final feature selection: with a modified all-subset regression approach, including residual tests and model averaging capabilities; 5) Post-estimation fine-tuning: coefficient re-evaluation through cross-validation techniques and model averaging across different k-fold results. \vskip 6pt

The objective of this paper is to introduce the main characteristics and comparative performance of a key layer of our methodology: the modified all-subset regression algorithm included in \href{https://github.com/ParallelGSReg/GlobalSearchRegression.jl}{GlobalSearchRegression.jl} (v1.0.3). As other available alternatives (like \href{https://cran.r-project.org/web/packages/MuMIn/MuMIn.pdf}{MuMin-pdredge} in R, or \href{https://www.researchgate.net/profile/Pablo_Gluzmann/publication/264782750_Global_Search_Regression_A_New_Automatic_Model-selection_Technique_for_Cross-section_Time-series_and_Panel-data_Regressions/links/53eed18a0cf23733e812c10d/Global-Search-Regression-A-New-Automatic-Model-selection-Technique-for-Cross-section-Time-series-and-Panel-data-Regressions.pdf?origin=publication_detail}{GSREG} Stata alternatives), this Julia algorithm allows researchers to obtain the best model specification among all possible covariate/feature combinations - in terms of user defined information criteria-, but up to 3165 times faster than Stata and 197 times faster than R. \vskip 6pt

\section{Package's main features}
\label{sec:packagefeatures}

Written in Julia, GlobalSearchRegression is a parallel (and improved) version of the Stata-GSREG all-subset regression command (get the original code \href{https://ideas.repec.org/c/boc/bocode/s457737.html}{here}). The package structure is quite simple, as shown in figure 1:

%%%%%%%%%%%%%%%%%% Figure 1
\begin{figure}[h]
\centerline{\includegraphics[width=10cm]{flowchart.png}}
\caption{ GlobalSearchRegression.jl Structure Flowchart}
	\label{fig:sample_figure}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%

Through the gsreg function of the \verb interface.jl  internal package, users set the appropriate database to be used, the general unrestricted model -GUM, which defines the search space- and additional options for model selection. With this information and complementary supporting functions and definitions provided by \verb strings.jl  -i.e. error messages-, \verb utils.jl  -i.e. equation formatting, combinatorial analysis, database manipulation, sorting results, etc.- and \verb gsreg_results.jl  -i.e. the structure to save estimation results-, the \verb core.jl  package perform the all-subset-regression algorithm explained in the pseudocode below, to obtain the following outputs: 1) a matrix -optionally exported to a csv file through \verb utils.jl-  including regression coefficients, selection criteria, observations and (optionally) t-test, residual tests, averaging-weights and out-of-sample metrics for every alternative model; 2) a text file -also displayed on screen- which contains the best model specification and (optionally) model averaging results in terms of the user-selected information criteria -see multiple examples in \verb runtest.jl-. \vskip 6pt
\break

%%%%%%%%%%%%%%%%%%%%%%% Algorithm 1
\begin{listing}
\hline
\center{ \textbf{Pseudo-code for module Core.jl}}
\vspace{2mm}
\hline
  \begin{enumerate}

    \item[1.] \textit{\textbf{function} gsreg\_single\_proc\_results!(args…)}
    \begin{enumerate}
        \item[1.1.] Select regression covariates
        \item[1.2.] Perform QR Decomposition
        \item[1.3.] Estimate regression results
        \item[1.4.] Calculate additional tests
        \item[1.5.] Save individual results in a Shared Array
    \end{enumerate}
    \item[] \textbf{end} \textit{function gsreg\_single\_proc\_results!(args…)}

\vspace{2mm}

    \item[2.] \textit{\textbf{function} gsreg\_proc\_results!(args…)}
    \begin{enumerate}
        \item[2.1.] \textbf{for} $j=1$ : $((k^2-1)/ nworkers)$ \textit{\# where nworkers is the user defined number of cores, and k is the numbers of covariates in the GUM}
        \begin{enumerate}
            \item[2.1.1.] $order = (j -1) * nworkers + workerID$
            \item[2.1.2.] \textit{gsreg\_single\_proc\_results!(order, args...)}
        \end{enumerate}
        \item[] \textbf{end} for
    \end{enumerate}
    \item[] \textbf{end} \textit{function gsreg\_single\_proc\_results!(args…)}

\vspace{2mm}

    \item[3.] \textit{\textbf{function} proc!(args…)}
    \begin{enumerate}
        \item[3.1.] Create environment (Shared Array and other objects)
        \item[3.2.] for $workerID = 1:nworkers$
        \begin{enumerate}
            \item[3.2.1] spawn jobs among workerIDs to perform gsreg\_proc\_results!
        \end{enumerate}
        \item[] \textbf{end} for
        \item[3.3.] Perform vector operations
        \item[3.4.] Call \verb utils.jl  sort function
        \item[3.5.] Create results array
    \end{enumerate}
    \item[] \textbf{end} \textit{function proc!(args…)}

\vspace{2mm}

    \item[4.] Export results using \verb gsreg\_results.jl   structure
    \item[5.] Export summary results to a txt file
    \item[6.] Optionally send the results structure to \verb utils.jl  to export a csv file.
  \end{enumerate}

\begin{flushleft}
\textbf{end} module \verb Core.jl . \hfill
\end{flushleft}
\hline
\end{listing}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Comparative performance against \textit{R} and \textit{STATA}}
\label{sec:comparativeperf}

%%%%%%%%%%%%%%%%%% Table 1
\begin{table*}[!t]
\centering
\tbl{Average execution time and speed-up metrics -over 5 runs-}{
\scalebox{0.85}{
    \begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|}
    \hline
    \multirow{Cov.} & \multirow{Obs} & \multirow{Threads} & \multicolumn{3}{|c|}{Execution time (in seconds)} & \multicolumn{3}{
    c|}{Parallel Speed-up} & \multicolumn{2}{|c|}{Julia Speed-up} \\
\cline{4-11}      &   &   & Julia & R & Stata & Julia & R & Stata & Over R & Over Stata \\
    \hline
    15 & 100 & 1 & 14.0 & 69.1 & 646.6 & 1.0 & 1.0 & 1.0 & 4.9 & 46.2 \\
    \hline
    15 & 100 & 4 & 17.0 & 33.4 & 80.3 & 0.8 & 2.1 & 8.0 & 2.0 & 4.7 \\
    \hline
    15 & 100 & 16 & 22.0 & 23.6 & 22.1 & 0.6 & 2.9 & 29.2 & 1.1 & 1.0 \\
    \hline
    15 & 100 & 32 & 31.4 & 22.3 & 15.1 & 0.4 & 3.1 & 43.0 & 0.7 & 0.5 \\
    \hline
    15 & 1000 & 1 & 18.6 & 74.9 & 667.5 & 1.0 & 1.0 & 1.0 & 4.0 & 35.9 \\
    \hline
    15 & 1000 & 4 & 18.5 & 35.3 & 81.6 & 1.0 & 2.1 & 8.2 & 1.9 & 4.4 \\
    \hline
    15 & 1000 & 16 & 22.2 & 24.6 & 23.3 & 0.8 & 3.1 & 28.7 & 1.1 & 1.0 \\
    \hline
    15 & 1000 & 32 & 31.5 & 21.5 & 16.0 & 0.6 & 3.5 & 41.6 & 0.7 & 0.5 \\
    \hline
    15 & 10000 & 1 & 61.6 & 142.0 & 809.0 & 1.0 & 1.0 & 1.0 & 2.3 & 13.1 \\
    \hline
    15 & 10000 & 4 & 30.5 & 61.4 & 114.7 & 2.0 & 2.3 & 7.1 & 2.0 & 3.8 \\
    \hline
    15 & 10000 & 16 & 25.7 & 38.4 & 37.3 & 2.4 & 3.7 & 21.7 & 1.5 & 1.5 \\
    \hline
    15 & 10000 & 32 & 33.4 & 35.5 & 26.7 & 1.8 & 4.0 & 30.3 & 1.1 & 0.8 \\
    \hline
    20 & 100 & 1 & 169.2 & 10538.9 & 535693.2 & 1.0 & 1.0 & 1.0 & 62.3 & 3165.1 \\
    \hline
    20 & 100 & 4 & 65.6 & 9275.4 & 75510.2 & 2.6 & 1.1 & 7.1 & 141.4 & 1151.1 \\
    \hline
    20 & 100 & 16 & 44.7 & 8855.2 & 4526.3 & 3.8 & 1.2 & 118.4 & 197.9 & 101.2 \\
    \hline
    20 & 100 & 32 & 51.2 & 8732.4 & 1658.2 & 3.3 & 1.2 & 323.0 & 170.7 & 32.4 \\
    \hline
    20 & 1000 & 1 & 362.4 & 10765.9 &   & 1.0 & 1.0 &   & 29.7 &  \\
    \hline
    20 & 1000 & 4 & 119.0 & 9352.3 &   & 3.0 & 1.2 &   & 78.6 &  \\
    \hline
    20 & 1000 & 16 & 58.8 & 8926.4 &   & 6.2 & 1.2 &   & 151.8 &  \\
    \hline
    20 & 1000 & 32 & 59.8 & 8753.9 &   & 6.1 & 1.2 &   & 146.4 &  \\
    \hline
    20 & 10000 & 1 & 2338.4 & 13218.9 &   & 1.0 & 1.0 &   & 5.7 &  \\
    \hline
    20 & 10000 & 4 & 629.4 & 10287.9 &   & 3.7 & 1.3 &   & 16.3 &  \\
    \hline
    20 & 10000 & 16 & 198.6 & 9392.3 &   & 11.8 & 1.4 &   & 47.3 &  \\
    \hline
    20 & 10000 & 32 & 149.5 & 9041.6 &   & 15.6 & 1.5 &   & 60.5 &  \\
    \hline
    25 & 100 & 1 & 6700.7 &   &   & 1.0 &   &   &   &  \\
    \hline
    25 & 100 & 4 & 1937.7 &   &   & 3.5 &   &   &   &  \\
    \hline
    25 & 100 & 16 & 913.7 &   &   & 7.3 &   &   &   &  \\
    \hline
    25 & 100 & 32 & 803.9 &   &   & 8.3 &   &   &   &  \\
    \hline
    25 & 1000 & 1 & 14257.4 &   &   & 1.0 &   &   &   &  \\
    \hline
    25 & 1000 & 4 & 3954.2 &   &   & 3.6 &   &   &   &  \\
    \hline
    25 & 1000 & 16 & 1471.4 &   &   & 9.7 &   &   &   &  \\
    \hline
    25 & 1000 & 32 & 1159.9 &   &   & 12.3 &   &   &   &  \\
    \hline
    25 & 10000 & 1 & 93450.0 &   &   & 1.0 &   &   &   &  \\
    \hline
    25 & 10000 & 4 & 23783.8 &   &   & 3.9 &   &   &   &  \\
    \hline
    25 & 10000 & 16 & 6825.6 &   &   & 13.7 &   &   &   &  \\
    \hline
    25 & 10000 & 32 & 4660.4 &   &   & 20.1 &   &   &   &  \\
    \hline
    \end{tabular}}}
\end{table*}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In table 1, we present a performance comparison of our GLobalSearchRegression Julia-package against its main alternatives: MuMin-pdredge and GSREG (written in R and Stata\footnote{The parallel version of Stata-GSREG is still under development. A preliminary version is available upon request from authors}, respectively).
Execution times were obtained from a HED architecture using a Threadripper 1950x build, with 16 cores (32 threads) overclocked to 3.8GHz and 64 GiB of DDR4-RAM at 3200Mhz. Comparative scripts were implemented on Julia 1.0.3, R 3.6.0 and Stata 15 IC, running on Ubuntu 18.04.2 LTS -Linux kernel 4.15-\footnote{All these test are available \href{https://github.com/ParallelGSReg/GlobalSearchRegression.jl/tree/master/Reproducibility}{here}} \vskip 6pt

For experimental -random variable- databases with a few covariates -up to 15 explanatory variables-, our Julia algorithm only provides significant time improvement in standard personal computers -e.g. 4 cores-, being up to twice faster than R and 4 times faster than Stata. For HED computers or HPC nodes, there is almost no difference among the best result obtained for each alternative. \vskip 6pt

However, for databases with 20 or more covariates, our Julia all-subset-regression code is always faster, irrespective of the number of observations or threads -up to 3165 times faster than STATA and 197 times faster than R-. Execution time differences exponentially increase with the number of covariates and slightly decrease with available observations. Moreover, Stata and R alternatives become unfeasible for databases with 25 or more covariates. For a confirmatory analysis, in figure 2 we present execution-time kernel densities for the 20 covariates - 100 observations - 32 threads case, obtained from 300 independent (non-cached) runs. Kernel results show that execution time differences are not an artifact obtained from noisy runs. Our Julia algorithm is consistently faster than R and Stata alternatives. \vskip 6pt

A detailed speed-up analysis is also available for the 25-covariate case. In figure 3, it's shown that our Julia all-subset-regression algorithm scales almost linearly for large databases -while the number of threads is not higher than the number of physical cores-. With small databases, Amdhal's law \cite{Amdahl1967} inputs change. Parallel tasks become lighter and speed-up efficiency degrades consistently with additional threads, because the marginal overhead cost of a larger environment creation is not overcompensated by parallelism gains obtained from additional threads. Notwithstanding, using only physical cores speed-up efficiency is always above 45 \%  -with an average of 84\% for 2, 4, 8 and 16 threads-.

%%%%%%%%%%%%%%%%%% Figure 2

\begin{figure*}[!t]
  \begin{subfigure}
    \centering
    \includegraphics[width=5.5cm]{juliatime.png}
  \end{subfigure}
    \centering
  \begin{subfigure}
    \includegraphics[width=5.5cm]{statatime.png}
  \end{subfigure}
    \centering
  \begin{subfigure}
    \includegraphics[width=5.5cm]{rtime.png}
  \end{subfigure}
  \vspace{2mm}
\caption{Execution time Kernel Densities from Julia, R and Stata}
\end{figure*}

%%%%%%%%%%%%%%%%%% Figure 3
\begin{figure*}[!t]
\centering
    \includegraphics[width=16cm]{juliaexe.PNG}
    \vspace{2mm}
    \caption{Execution time, speed-up and speed-up efficiency for the 25-covariate case using GlobalSearchRegression.jl}
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Why GlobalSearchRegression.jl is faster than existing alternatives?}
\label{sec:whygsreg}
\subsection{JULIA platform}
\label{subsub:juliaplat}

Despite some \verb GlobalSearchRegression.jl  specific features to be examined below, our all-subset-regression algorithm benefits from the well-known Julia language efficiency for High Performance Computing tasks.

\vspace{-2mm}

%%%%%%%%%%%%%%%%%% Table 2
\begin{table}[!h]
\centering
\tbl{Julia speed-up over STATA and R in basic functions}{
    \begin{tabular}{|l|c|c|}
    \hline
    Function & Speed-up over R & Speed-up over Stata \\
    \hline
    \hline
    recursion\_fibonacci & 251.28 & 76.92 \\
    \hline
    recursion\_quicksort & 32.36 & 48.54 \\
    \hline
    matrix\_multiply & 11.6 & 14.07 \\
    \hline
    iteration\_pi\_sum & 14.33 & 26.89 \\
    \hline
    matrix\_statistics & 7.65 & 44.92 \\
    \hline
    parse\_integers & 28.4 & 49.38 \\
    \hline
    userfunc\_mandelbrot & 175.44 & 368.42 \\
    \hline
    \hline
    Simple-Average & 74.44 & 89.88 \\
    \hline
    Weighted-Average & 12.15 & 22.01 \\
    \hline
    \multicolumn{3}{p{8.5cm}}{Note: Weighted-average were estimated as the Ratio between total execution time for R or Stata, and Julia's total execution time. Benchmarkings were obtained using a i7-4700MQ quad-core build, with 8 GiB of DDR3 1600 Mhz RAM, under Windows 8 and running Julia 1.1.0, Stata 15 and R 3.5.3}
    \end{tabular}
    }
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%

\break

Julia JIT-compilation allows packages to run faster than those executed using interpreted or byte-compiled languages (like R or Stata-Mata). Indeed, basic functions running on Julia can be up to 251 and 368 times faster than those running in R and Stata, respectively -with an average speed-up of 75 and 90, see Table 2-.

\subsection{Parallel strategy and memory setup}
\label{subsub:parallelst}
It is well know that multicore architectures can be used to speed-up execution times through two main paradigms: data and task-parallelism \cite{zaki2001}. The preferred strategy critically depends on:
\begin{enumerate}
    \item Database structure;
    \item Algorithm serial portion; and
    \item Interthread communication costs.
\end{enumerate}
The first discussion is about tall vs fat data. While tall databases are more suitable for data-parallelism \cite{babu2013}, fat-structures improve relative performance of task-parallelism (because tasks increase exponentially with covariates/columns in feature selection problems, see \cite{foster1994}). \vskip 6pt

Additionally, the choice between alternative paradigms takes into account the Amdahl's Law for the specific algorithm to be used. Task-parallelism is usually better for econometric and machine learning algorithms requiring some specific serial optimization paths (i.e. arima-arfima models), while data-parallelism performs better under linear-algebra solutions (i.e. OLS-family estimators, see \cite{guo2012}). \vskip 6pt

Finally, we have the problem of intercomunication costs. Data-parallelism -generally- involves intense needs of inter-thread-communication. While task-parallelism communication costs depends on Load-Balancing choices (i.e. Dynamic vs Static), they are usually lower than data-parallelism ones \footnote{At least when Static Load Balancing is implemented, because it allows for Coarse-grained granularity \cite{gordon2006}} \cite{MOREANO2017}. \vskip 6pt

For feature selection problems, pros and cons of alternative strategies often determine that available cores should be used for task parallelism. Databases are fat, data-mining algorithms can include large serial portions, and intercomunication costs can be huge for large-multicore architectures. These reasons explain why all available all-subset-regression packages use taks-parallism to speed-up execution times. \vskip 6pt

As for the memory setup, there are also alternative methodologies. First, it's necessary to choose among Static vs. Dynamic memory allocations \cite{prajapati2015}. To improve speed-up, Static "once-and-for-all" allocations are often preferred (even at a cost of higher average memory utilization). Second, shared-memory strategies must be determined. Depending on both object-size and CPU architecture -cache size and its distribution among cores-, it could be optimal to use large shared arrays or -alternatively- smaller core-specific objects. Splitting output matrices to work with smaller non-shared arrays could be useful for cache optimization purposes, but it could also entail additional communication costs and higher memory requirements \cite{ahmed2015}. In practice, feature selection algorithms usually prefer shared-arrays for high performance computing. \vskip 6pt

\verb GlobalSearchRegression.jl  (execution-time) advantages have been obtained combining:
\begin{itemize}
    \item[a)]  Task-parallelism
    \item[b)]  Static Load-Balancing
    \item[c)]  Coarse-grained granularity
    \item[d)]  Static memory allocation
    \item[e)]  Efficient shared-array implementation
\end{itemize}

While some of these characteristics are shared with R and Stata alternatives (\verb MuMin-pdredge  and \verb GSREG,  respectively), our Static Load-Balancing algorithm outperform the round-robin R scheduling (implemented by the clusterapply function included in the \verb parallel  R-package see \cite{deb2013}) and the Shared-Array strategy significantly improves I/O performance against Stata. By construction, pure Static Load-Balancing in GlobalSearchRegression.jl avoids Round-Robin communication costs and execution gaps. This advantage overcompensate minimal\footnote{Minimal because our static scheduler guarantees that average task-complexity will not be too different among workers} load-asymmetries, associated with any static scheduling. In turn, using efficient shared-arrays to store all-subset-regression results allows us to outperform the \verb Stata-GSREG  methodology which heavily relies on slower I/O disk operations (because multiple instances must be launched to enable task-parallelism and, therefore, shared-arrays become unfeasible). \vskip 6pt

\subsection{OLS estimation}
\label{subsub:ols}
Efficient Ordinary Least Squares (OLS) algorithms rely on Linear Algebra operations (matrix decomposition, matrix inversion, etc.).\footnote{Optimization alternatives (i.e. Gradient descent), while quite inefficient for linear models, can also be used for non-linear estimations \cite{abdi2007}} The traditional $(X'X)^{-1}$ operation could be time-expensive with unstable solutions under certain conditions \cite{jameson1981}. A preferred method is the QR-decomposition developed by Francis \cite{francis1961} and Kublanovskaya \cite{kublanovskaya1962s}. The QR factorization allows us to decompose any full rank $N \times p$ matrix $\bar{X}$ as:

\begin{equation}
    X=QR
\end{equation}

where

\begin{itemize}
    \item[•] $Q$ is a $N \times p$ matrix with $Q'Q=I$  ; and
    \item[•] $R$ a $p \times p$ upper triangular matrix.
\end{itemize}

The QR decomposition is fast and provides stable numerical solutions under rank-deficient matrices. Alternative factorizations are either slower (SVD decomposition) or potentially unstable (Cholesky decomposition)\cite{agarwal2014}. \vskip 6pt

\verb GlobalSearchRegression.jl  OLS estimation through QR-decompositions outperform existing Julia alternatives (like \verb GLM.jl)  allowing for large speed-ups compared with \verb R-lm    and \verb Stata-regress  commands. Table 3 shows execution time differences for a 200 covariates - 1000 observations - multivariate linear regression \footnote{Execution times were calculated for alternative OLS algorithms available for all-subset-regression packages in Julia, R and Stata.}.
\vskip 6pt

It must be notice that execution times are compilation-free, because all-subset-regression algorithms must perform thousand to millions of regressions where compilation time is absent (it only affects the first regression).
\vskip 6pt

\break
%%%%%%%%%%%%%%%%%% Table 3
\begin{table}[!h]
  \centering
\tbl{Execution times for OLS algorithms in R, Stata and Julia}{
    \begin{tabular}{|c|c|c|c|c|}
    \hline
      & GlobalSearchRegression.jl &R-lm & Stata-reg & GLM.jl \\
    \hline
    Exec. time & 0.012 & 0.06 & 0.2 & 0.023 \\
    \hline
    Speed-up & 1 & 5 & 16.67 & 1.92 \\
    \hline
    \multicolumn{5}{p{8.5cm}}{Note: Execution times were obtained using a i7-7500 dual-core build, with 8 GiB of DDR4 RAM, under Windows 10 and running Julia 1.1.0, Stata 13 and RStudio 3.6.0. Speed-ups are obtaining dividing each execution time by the GlobalSearchRegression.jl execution time.} \\
    \end{tabular}}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Conclusion}
\label{sec:con}
\vspace{2mm}
\begin{quote}
\textit{"There are a number of areas where there would be opportunities for fruitful collaboration between econometrics and machine learning [... and] the most important area for collaboration involves causal inference"}. \cite{varian2014}
\end{quote}
\vspace{2mm}

As Hal Varian emphasizes, there is a need for mutual collaboration between machine learning developers/practitioners and econometricians. In this paper we describe a research-project aimed at building bridges between machine learning and econometric worlds and introduce the main characteristics of its first outcome: a Julia-native all-subset regression algorithm (\verb GlobalSearchRegression.jl)  which runs up to 3165 and 197 times faster than existing Stata and R alternatives, respectively. \vskip 6pt

Throughout the paper, it has been shown that execution-time gains are explained by multiple efficient strategies combined in \verb GlobalSearchRegression.jl  (i.e. task parallelism, static load-balancing, coarse-grained granularity, static memory allocation, efficient shared array implementation, and OLS estimation using neat QR decompositions). However, the main 'explanatory variable' is the impressive speed-up in atomic operations obtained using the Julia language. \vskip 6pt

Notwithstanding, increasing availability of Big and -more challenging- Fat-data, force us to go beyond pure all-subset-regression approaches and combine it with machine learning feature selection algorithms. Work in progress include a new hybrid package merging LASSO capabilities, all-subset-regression robustness and K-fold cross validation strengths.

\section{Acknowledgments}
\label{sec:ackno}
This research benefited from the financial support of 1) the National Scientific and Technical Research Council and the YPF-Foundation (PIO CONICET-YPF 13320150100020CO); and 2) the National Agency for Scientific and Technological Promotion (PICT-2017-0867). We would also like to thank PhD. Jorge Carrera for his technical assistance on Big-data econometric algorithms.

\input{bib.tex}

\end{document}

% Inspired by the International Journal of Computer Applications template