\documentclass[11pt,oneside,a4paper]{article}

\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{geometry}
\usepackage{layout}
\usepackage{fancyhdr}

\usepackage{hyperref}
%\hypersetup{pdfcolorlinks=true,allcolors=blue}

%\pagestyle{fancy}
%\headheight = 14.5 pt
%\markright{Notes on \textsl{volatilitytrade.pdf} \hfill Peter Zsohar}

\title{Notes on \textsl{volatilitytrade.pdf}}
\author{Peter Zsohar}

\voffset = -10 pt
\textheight = 650 pt
\parindent = 0 pt

\begin{document}
\maketitle

\section{Input Files}
Data files in alphabetical order.
\subsection{\textsl{alfas.csv}}
Contains the $J(=24)$ sectoral $\alpha^j$ values.
\subsubsection{Time varying alphas}
\begin{enumerate}
  \item $w\!L^j_{t} \doteq \sum_n w^j_{nt} L^j_{nt}$ -- Use sectoral value added from the data. (see below)
  \item $s^j_{t} \doteq \frac{w\!L^j_{t}}{\sum_k w\!L^k_{t}}$
  \item $\alpha^j_t \doteq \frac{s^j_{t} / \beta^j}{\sum_k (s^k_{t} / \beta^k)}$ -- Use betas from data. (see below)
\end{enumerate}
\subsubsection{Constant alphas}
\begin{itemize}
  \item Averaged: Take the time average of the time varying alphas.
  \item Pooled
	\begin{enumerate}
  		\item $w\!L^j \doteq \sum_{n, t} w^j_{nt} L^j_{nt}$ -- Use sectoral value added from the data. (see below)
  		\item $s^j \doteq \frac{w\!L^j}{\sum_k w\!L^k}$
  		\item $\alpha^j \doteq \frac{s^j / \beta^j}{\sum_k (s^k / \beta^k)}$ -- Use betas from data. (see below)
	\end{enumerate}
\end{itemize}
\subsubsection{Smooth alphas}
We take the time varying alphas and apply the bandpass filter to get the trend for each sector. Then renormalize so that the sum of alphas is 1 in all periods.

\subsection{\textsl{betts.csv}}
Contains the country, sector and time period specific $\beta^j_{nt}$ values.
Construct sectoral $\beta^j$ values by averaging over $t$ and $n$: $$\beta^j = \frac{\sum_{t, n}\beta^j_{nt}}{TN}$$

\subsection{\textsl{export\_shares.csv}}
This file contains the \emph{import} expenditure shares for each country pair, sector and time period.\\
Manipulations
\begin{itemize}
  \item Negative entries are set to 0 (3379 cases). There are some numbers in the magnitude of 1e2.
  \item Get domestic shares $d_{mmt}  = 1 - \sum_{n \neq m} d_{mnt}$
  \item Set negative domestic shares $\left(1364 \mbox{ cases when }\sum_{n \neq m} d_{mnt} > 1\right)$ to previous year's value or 0, then re-normalize row by $\sum_{n} d_{mnt}$
  \item Remaining zeros are set to 1e-10.
\end{itemize}
Use
\begin{itemize}
  \item Infer trade costs $\kappa_{nmt}^{j}$
  \item The US values are used in the calculation of the sectoral productivities ($Z$s).
\end{itemize}

\subsection{\textsl{Price\_USA.csv}}
This contains a single time series of the US aggregate price index with 1995 as a base. \\
Manipulation
\begin{itemize}
  \item Normalize the base value in 1995 to 1.
\end{itemize}
Use
\begin{itemize}
  \item Price changes for the US ($\hat{P}_{US,t}$) -- the rest of price changes uses also the PWT (see below).
\end{itemize}

\subsection{\textsl{prices.csv}}
Stores price index time series (with base 1995) for each country and sector separately. Only US prices are used.\\
Use
\begin{itemize}
  \item Sectoral prices in the US ($P^j_{US,t}$) are used to calculate the evolution of sectoral prices ($\hat{P}^j_{US,t}$) and the evolution of sectoral productivities in the US.
\end{itemize}

\subsection{\textsl{PWT.csv}}
Stores aggregate prices relative to the US for each country and time period.\\
Use
\begin{itemize}
  \item Deflator of sectoral value added ($w^j_{nt}L^j_{nt}$) to get sectoral output ($y^j_{nt}$).
  \item Calculate sectoral productivities in all countries in period 0 ($Z^j_{n,0}$) based on US sectoral productivities ($Z^j_{US,0}$).
  \item Calculate the aggregate (relative to US) price changes ($\hat{P}_{m,t}$).
\end{itemize}

\subsection{\textsl{VA.csv}}
Contains country, sector and time period specific value added. Quantities are in \emph{current} million USD.\\
Use
\begin{itemize}
  \item Deflate with aggregate prices to get sectoral output ($y^j_{nt} \equiv w^j_{nt}L^j_{nt} / P^j_{n,t} $).
  \item Use the trend of value added share as a proxy for equipped labor share.
  $$\psi^j_{nt} \equiv L^j_{nt} / L_{nt} \approx \widehat{\left(\frac{L^j_{nt} w^j_{nt}}{L_{nt} w_{nt}}\right)}$$
\end{itemize}

\newpage
\section{Mapping the model into observables}
\subsection{Trade Costs}
Following equation (26p) trade costs are backed out from the import expenditure shares.
\begin{align}
\frac{d_{nmt}^{j}d_{mnt}^{j}}{d_{mmt}^{j}d_{nnt}^{j}} &=\left( \kappa_{nmt}^{j}\right) ^{2\theta }  \tag{26p}\\
\kappa_{nmt}^{j} &= \left( \frac{d_{nmt}^{j}d_{mnt}^{j}}{d_{mmt}^{j}d_{nnt}^{j}}\right) ^{\frac{1}{2\theta}}
\end{align}

\subsection{Price normalization}
Problem: From Equation (15p) aggregate price will not equal 1 if all sectoral prices are normalized to 1.
\begin{align}
P_{nt} &= \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} {P_{nt}^j}^{\alpha^j} \tag{15p}\\
%
P_{nt} &= \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} \neq 1 \notag
\end{align}
From now on I will use
\begin{equation}
P_{US, 1972} = \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} \approx 3.3667,
\end{equation}
where I use the $\alpha$ values to compute the \emph{level} of aggregate price in the US in 1972.

\subsection{Compound Shocks (Except for Services)}
First $Z_{0t_0}^j = Z_{US, 1972}^{j}$ is backed out based on (34p) for all sectors except for services. Note that for clarity I make it explicit that country 0 is the US.
\begin{align}
Z_{US, 1972}^{j} &= {B^{j}}^{\theta } {\xi}^\theta d_{US, US, 1972}^{j} \left(y_{US, 1972}^{j}\right) ^{\theta \beta^{j}} \left(\frac{1}{P_{US, 1972}}\right) ^{-\theta} \left(\psi_{US, 1972}^{j}\right) ^{-\theta \beta^{j}} \notag\\
%
&= {B^{j}}^{\theta } {\xi}^\theta d_{US, US, 1972}^{j} \left(\frac{VA_{US, 1972}^j}{P_{US, 1972}}\right) ^{\theta \beta^{j}} \left(\frac{1}{P_{US, 1972}}\right) ^{-\theta} \left(\psi_{US, 1972}^{j}\right) ^{-\theta \beta^{j}}  \notag\\
%
&= {B^{j}}^{\theta } {\xi}^\theta d_{US, US, 1972}^{j} \left(P_{US, 1972}\right) ^{\theta (1 - \beta^{j})} \left(VA^j_{US, 1972}\right)^{\theta \beta^{j}} \left(\psi_{US, 1972}^{j}\right) ^{-\theta \beta^{j}}\notag\\
%
&= {B^{j}}^{\theta } {\xi}^\theta d_{US, US, 1972}^{j} \left(\prod_{k = 1}^J {\alpha^k}^{- \alpha^k}\right) ^{\theta(1 - \beta^{j})} \left(VA^j_{US, 1972}\right)^{\theta \beta^{j}} \left(\psi_{US, 1972}^{j}\right) ^{-\theta \beta^{j}}
\end{align}
%
Then $Z_{nt_0}^j = Z_{n, 1972}^j$ is recovered for all the other countries and all sectors (except for services) based on the $Z_{US, 1972}^{j}$ values. This is done based on (30p).
%
\begin{align}
Z_{n, 1972}^{j} &= Z_{US, 1972}^{j} \frac{d_{US, n, 1972}^{j}}{d_{US, US, 1972}^{j}} \left(\frac{y_{n, 1972}^{j}}{y_{US, 1972}^{j}}\right)^{\theta \beta ^{j}} \left(\kappa_{US, n, 1972}^{j}\right)^{-\theta} \left(\frac{P_{n, 1972}}{P_{US, 1972}}\right)^{\theta} \left(\frac{\psi_{n, 1972}^{j}}{\psi_{US, 1972}^{j}}\right)^{-\theta \beta^{j}} \notag\\
%
&= Z_{US, 1972}^{j} \frac{d_{US, n, 1972}^{j}}{d_{US, US, 1972}^{j}} \left(\frac{VA_{n, 1972}^{j}}{VA_{US, 1972}^{j}}\right)^{\theta \beta ^{j}} \left(\kappa_{US, n, 1972}^{j}\right)^{-\theta} \left(\frac{P_{n, 1972}}{P_{US, 1972}}\right)^{\theta(1 - \beta^j)} \left(\frac{\psi_{n, 1972}^{j}}{\psi_{US, 1972}^{j}}\right)^{-\theta \beta^{j}} \notag\\
%
&=  {B^{j}}^{\theta} {\xi}^\theta d_{US, n, 1972}^{j} \left(VA_{n, 1972}^{j}\right)^{\theta \beta ^{j}} \left(\kappa_{US, n, 1972}^{j}\right)^{-\theta} \left(\prod_{k = 1}^J {\alpha^k_t}^{- \alpha^k_t}\right) ^{\theta(1 - \beta^{j})} \cdot \dots \notag\\
& \qquad \quad \cdot \left(PWT_{n, 1972}\right)^{\theta(1 - \beta^j)} \left(\psi_{n, 1972}^{j}\right)^{-\theta \beta^{j}}
%
\end{align}
%
Then we compute the $Z_{0t}^j = Z_{US, t}^j$ series for all 35 remaining time periods ($t = 1973 \dots 2007$) and all sectors (except for services). To do this, first we obtain the growth rate of these series, $\hat{Z}_{US, t}^j$, based on (33p), and then calculate the whole series based on the initial values $Z_{US, 1972}^j$.
%
\begin{align}
\hat{Z}_{US, t}^{j} &= \hat{d}_{US, US, t}^{j} + \theta\beta^{j} \hat{y}_{US, t}^{j} - \theta \left(\hat{P}_{US, t}^{j} - \hat{P}_{US, t}\right) - \theta\beta^{j} \hat{\psi}_{US, t}^{j} \notag\\
%
&= \hat{d}_{US, US, t}^{j} + \theta(1 - \beta^{j}) \hat{P}_{US, t} - \theta \hat{P}_{US, t}^{j} + \theta\beta^{j} \left(\widehat{VA}^j_{US, t} - \hat{\psi}^j_{US, t}\right)
\end{align}
%
Then use
\begin{equation}
Z_{US, t}^j = Z_{US, t-1}^j e^{\hat{Z}_{US, t}^j} \qquad \mbox{with} \qquad Z_{US, 1972}^j.
\end{equation}
%
The growth rate of $Z_{nt}^j$ values for the rest of the countries is then recovered by using (31p) and then using the exponential formula like for the US.
%
\begin{align}
\hat{Z}_{nt}^{j} &= \hat{Z}_{US, t}^{j} + (\hat{d}_{US, n, t}^{j} - \hat{d}_{US, US, t}^{j}) + \theta \beta^{j} \left(\hat{y}_{n, t}^{j}-\hat{y}_{US, t}^{j}\right) \notag \\
& \qquad \quad - \theta \left(\hat{\kappa}_{US,n,t}^{j} + \hat{P}_{US,t} - \hat{P}_{n,t}\right) + \theta \beta^{j} \left(\hat{\psi}_{US,t}^{j} - \hat{\psi}_{n,t}^{j}\right) \notag \\
%
&= \hat{Z}_{US, t}^{j} + (\hat{d}_{US, n, t}^{j} - \hat{d}_{US, US, t}^{j}) + \theta \beta^{j} \left(\widehat{VA}_{n, t}^{j}-\widehat{VA}_{US, t}^{j} + \hat{P}_{US, t}-\hat{P}_{n, t}\right) \notag \\
& \qquad \quad - \theta \left(\hat{\kappa}_{US,n,t}^{j} + \hat{P}_{US,t} - \hat{P}_{n,t}\right) + \theta \beta^{j} \left(\hat{\psi}_{US,t}^{j} - \hat{\psi}_{n,t}^{j}\right) \notag \\
%
&= \hat{d}_{US, n, t}^{j} - \theta\hat{\kappa}_{US,n,t}^{j} + \theta \beta^{j} \left(\widehat{VA}^j_{n, t} - \hat{\psi}^j_{n, t}\right) + \theta(1 - \beta^{j}) \left(\hat{P}_{US, t} + \widehat{PWT}_{n,t}\right) - \theta \hat{P}_{US, t}^{j}
\end{align}
%
Then use
\begin{equation}
Z_{n, t}^j = Z_{n, t-1}^j e^{\hat{Z}_{n, t}^j} \qquad \mbox{and} \qquad Z_{n, 1972}^j.
\end{equation}

\subsection{Compound Shocks for Services}
Now we have to compute the shocks for services. Generally we use three equations to derive all the formulas below. These three are (15p), (16p) and (27p).
%
\begin{align}
P_{nt} &= \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} {P_{nt}^j}^{\alpha^j} \tag{15p}\\
%
P_{nt}^j &= \xi^j {\Phi_{nt}^j}^{-\frac{1}{\theta}} \tag{16p}\\
%
d^j_{nmt} &= \frac{{B^{j}}^{-\theta} \left(\psi^j_{mt}\right)^{\theta\beta^j} Z_{mt}^j \left(\kappa^j_{nmt}\right)^{\theta} \left(y^j_{mt}\right)^{-\theta\beta^j}}{{P_{mt}}^\theta {\Phi_{nt}^j}} \tag{27p}
\end{align}
%
\begin{enumerate}
  \item Use (27p) and $n = m$ to calculate $\Phi_{n, t}^j$ for all sectors (except for services)
  \begin{equation}
     \Phi_{n, t}^j = {B^j}^{-\theta} Z_{n, t}^j \left(VA^j_{n, t}\right)^{-\theta \beta^j} \left(\psi^j_{n, t}\right)^{\theta \beta^j}  \left(PWT_{n, t} P_{US, t}\right)^{\theta(\beta^j - 1)} \frac{1}{d_{n, n, t}^j}
  \end{equation}
  \item Use (16p) to recover sectoral prices for all countries (except for services)
  \begin{equation}
    P_{n, t}^j = \xi \left(\Phi_{n,t}^j\right)^{-\frac{1}{\theta}}
  \end{equation}
  \item Use (15p) to recover the price of services
  \begin{equation}
    P_{n, t}^s = \left(PWT_{n, t} P_{US, t}\right)^{\frac{1}{\alpha^s}} \left(\prod_{j = 1}^J {\alpha^j}^{- \alpha^j}\right)^{-\frac{1}{\alpha^s}} \left[\prod_{j \neq s} \left(P_{n, t}^j\right)^{\alpha^j}\right]^{-\frac{1}{\alpha^s}}
  \end{equation}
  \item Recover $Z_{n, t}^s$ using (27p), (16p) and $n = m$
  \begin{equation}
    Z_{n, t}^s = \xi^\theta {B^s}^\theta \left(\frac{VA^s_{n, t}}{\psi^s_{n, t}}\right)^{\theta \beta^s} \left(PWT_{n, t} P_{US, t}\right)^{\theta(1 - \beta^s)} {P_{n, t}^s}^{-\theta}
  \end{equation}
\end{enumerate}

\subsection{Alternative Computation of Shocks and Sectoral Prices}
Start with (28p) which is true for all $(n, k)$ pairs of countries and all sectors $j$ except for services:
\begin{align}
Z_{nt}^j &= \underbrace{\xi^\theta {B^j}^\theta d_{k, n, t}^j \left(\kappa_{k, n, t}^j\right)^{-\theta} \left(VA_{n, t}^j\right)^{\theta \beta^j} \left(\psi_{n, t}^j\right)^{-\theta \beta^j} {P_{n, t}}^{\theta (1 - \beta^j)}}_{\exp(\zeta_{k, n, t}^j)} {P_{k, t}^j}^{-\theta} \notag \\
&= \exp(\zeta_{k, n, t}^j) {P_{k, t}^j}^{-\theta}
\end{align}
Note that this relationship is true for any choice of $k$ and that $\exp(\zeta_{k, n, t}^j)$ can be constructed from data.
Decompose $\exp(\zeta_{k, n, t}^j) = Z_{nt}^j {P_{k, t}^j}^{\theta}$ according to the following procedure:
\begin{enumerate}
  \item Take logs and rename terms for brevity.
  \begin{align}
  \zeta_{k, n, t}^j &= \ln{Z_{nt}^j} + \theta\ln{P_{k, t}^j} \\
  &= \lambda_{nt}^j + \tau_{k, t}^j
  \end{align}
  \item Use sectoral prices in the US.
  \begin{equation}
  \tau_{US, t}^j = \theta\ln{P_{US, t}^j}
  \end{equation}
  \item Get $\tau_{k, t}^j$ for all other countries.
  \begin{equation}
  \tau_{k, t}^j = \frac{1}{N} \sum_{n = 1}^N \left(\zeta_{k, n, t}^j- \zeta_{US, n, t}^j\right) + \tau_{US, t}^j
  \end{equation}
  \item Get $\lambda_{nt}^j$ for all other countries.
  \begin{equation}
  \lambda_{nt}^j = \frac{1}{N} \sum_{k = 1}^N \left(\zeta_{k, n, t}^j - \tau_{k, t}^j\right)
  \end{equation}
  \item Recover shocks and prices.
  \begin{align}
  Z_{n, t}^j &= \exp\left(\lambda_{nt}^j\right)\\
  P_{k, t}^j &= \exp\left(\frac{\tau_{kt}^j}{\theta}\right)
  \end{align}
\end{enumerate}

To get shocks and prices for services one can use the same procedure as described in the previous section with the option to directly use the sectoral prices for all other sectors delivered by the decomposition approach from the $\tau$ values.

\subsection{Compute Equipped Labor}
As a last step, we have to compute $L$, the measure of \emph{equipped labor}. We use the identity
$$\frac{1}{\beta^j \theta} \hat{Z}^j_{mt} \equiv \frac{1}{\beta^j} \hat{A}^j_{mt} + \hat{L}_{mt}$$ along with the identifying assumption
$$\sum_j \frac{\alpha^j}{\beta^j} \hat{A}^j_{mt} = 0.$$

This yields
$$L_{mt} = \prod_j \left(Z^j_{mt}\right)^{\frac{\alpha^j}{\beta^j \theta}}.$$
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newpage

\section{Shock Decomposition}
\subsection{Filtering}
To separate per period shocks from trends we use a band pass filter to detrend each $\left\{\log{Z^j_{n,t}}\right\}_{t=1}^T$ series.

The filter uses the moving average weights recommeneded by Table 4 in Baxter and King (1995). In the equilibrium calculations we will have to repeat this filtering several times on each time series and that would result in loss of data at edges of the series in each step. To resolve this issue we reflect the head and tail of the time series at the edges.

Let $y_0$ denote the first observation in the time series. Then we extrapolate to $K$ preceding periods by using
$$y_{-k} = y_0 + (y_0 - y_k)\quad \forall k \leq K, $$ where $K$ is the number of lags used in the moving average.
The same procedure is employed at the end of the time series. Note that in this way the trend value at the edges of the time series will be equal to the actual values. This implies that the reflection procedure yields shocks equal to 0 in the first and last periods.
After the trend is removed we denote the cyclical component of the $\left\{\log{Z^j_{n,t}}\right\}_{t=1}^T$ series with $\hat{Z}^j_{n,t}$.

\subsection{Decomposition}
The decomposition to sector and country specific parts is done in a way described in Koren and Tenreyro (2007).
First, we calculate the time average of the shocks for each $(n,j)$ pair and subtract it from the growth rate to get the object to be decomposed, $\tilde{Z}^j_{nt}$.
$$\tilde{Z}^j_{nt} = \hat{Z}^j_{n,t} - (T - 1)^{-1} \sum_{t=2}^T \hat{Z}^j_{n,t}$$

Then $\tilde{Z}$ is decomposed as
$$\tilde{Z}^j_{nt} = \lambda^j_t + \mu_{nt} + \epsilon^j_{nt},$$

where $\lambda, \mu$ and $\epsilon$ are estimated as
\begin{align*}
\hat{\lambda}^j_t &= N^{-1} \sum_{n=1}^N  \tilde{Z}^j_{nt} \\
\hat{\mu}_{nt} &= J^{-1} \sum_{j=1}^J \left(\tilde{Z}^j_{nt} - \hat{\lambda}^j_t\right)\\
\hat{\epsilon}^j_{nt} &= \tilde{Z}^j_{nt} - \hat{\lambda}^j_t - \hat{\mu}_{nt}.
\end{align*}

\newpage
\section{Equilibrium Algorithm}
%
\subsection{Equilibrium Equations}
Equations (\ref{eq1}) - (\ref{eq7}) describe the equilibrium of the model.
\begin{align}
% \nonumber to remove numbering (before each equation)
  P_{nt} &= \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} {P_{nt}^j}^{\alpha^j} \label{eq1}\\
  %
  P_{nt}^j &= \xi {\Phi_{nt}^j}^{-\frac{1}{\theta}} \label{eq2}\\
  %
  \Phi_{nt}^j &= {B^j}^{-\theta} \sum_{i = 1}^N T_i^j {A_{it}^j}^{\theta} \left(\frac{{P_{it}}^{1 - \beta^j} {w_{it}^j}^{\beta^j}}{\kappa_{nit}^j}\right)^{-\theta} \label{eq3}\\
  %
  d_{mnt}^j &= \frac{{B^j}^{-\theta} T_n^j {A_{nt}^j}^{\theta} \left(\frac{{P_{nt}}^{1 - \beta^j} {w_{nt}^j}^{\beta^j}}{\kappa_{mnt}^j}\right)^{-\theta}} {\Phi_{mt}^j} \label{eq4}\\
  %
  w_{nt}^j L_{nt}^j &= \beta^j \sum_{m = 1}^N d_{mnt}^j \left( \alpha^j w_{mt} L_{mt} + \frac{1 - \beta^j}{\beta^j} w_{mt}^j L_{mt}^j\right) \label{eq5}\\
  %
  w_{nt} L_{nt} &= \sum_{j = 1}^J w_{nt}^j L_{nt}^j \label{eq6}\\
  %
  \frac{L_{nt}^{j}}{L_{nt}} &= E_{t - 1} \left( \frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right) \label{eq7}
\end{align}

Where
\begin{align}
  \nonumber
  \xi &= \Gamma\left(\frac{\theta + 1 - \eta}{\theta}\right) \\
  \nonumber
  B^j &= {\beta^j}^{- \beta^j} (1 - \beta^j)^{-(1 - \beta^j)}
\end{align}

\newpage
\subsection{Inner loop}
For a given pair of sectoral resource allocation $(L_{nt}^j)$ and sectoral wages $(w_{nt}^j)$ solve the system below for the aggregate price indexes $P_{nt}.$
\begin{eqnarray}
% \nonumber to remove numbering (before each equation)
  P_{nt} &=& \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} {P_{nt}^j}^{\alpha^j} \label{aggr1}\\
  P_{nt}^j &=& \xi {\Phi_{nt}^j}^{-\frac{1}{\theta}} \label{aggr2} \\
  \Phi_{nt}^j &=& {B^j}^{-\theta} \sum_{i = 1}^N T_i^j {A_{it}^j}^{\theta} \left(\frac{{P_{it}}^{1 - \beta^j} {w_{it}^j}^{\beta^j}}{\kappa_{nit}^j}\right)^{-\theta}
\end{eqnarray}
Where $$\xi = \Gamma\left(\frac{\theta + 1 - \eta}{\theta}\right)$$ and $$B^j = {\beta^j}^{- \beta^j} (1 - \beta^j)^{-(1 - \beta^j)}.$$

Algebraic manipulations to arrive at a system of $N$ equations and $N$ unknowns.
%
Simplify $\Phi_{nt}^j:$
\begin{eqnarray*}
  \Phi_{nt}^j &=&  {B^j}^{-\theta} \sum_{i = 1}^N \underbrace{T_i^j {A_{it}^j}^{\theta}}_{\frac{Z_{it}^j}{{L_{it}}^{\beta^j\theta}}} \left(\frac{{P_{it}}^{1 - \beta^j} {w_{it}^j}^{\beta^j}}{\kappa_{nit}^j}\right)^{-\theta}\\
  &=& {B^j}^{-\theta} \sum_{i = 1}^N Z_{it}^j {L_{it}}^{- \beta^j\theta} {w_{it}^j}^{-\beta^j\theta} {\kappa_{nit}^j}^{\theta}{P_{it}}^{\theta(\beta^j - 1)}\\
  &=& {B^j}^{-\theta} \sum_{i = 1}^N \underbrace{Z_{it}^j {\left((L_{it}w_{it}^j)^{- \beta^j} \kappa_{nit}^j\right)}^{\theta}}_{D_{nit}^j} {P_{it}}^{\theta(\beta^j - 1)}\\
  &=& {B^j}^{-\theta} \sum_{i = 1}^N D_{nit}^j{P_{it}}^{\theta(\beta^j - 1)}
\end{eqnarray*}
%
Notice that we can compute the coefficients of the equation (the $D$ values) before starting the search for the price vector.

Use equation (\ref{aggr2}) and then the expression for $\Phi_{nt}^j$:
\begin{eqnarray*}
  P_{nt} &=&  \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} {P_{nt}^j}^{\alpha^j}\\
  &=& \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} \left(\xi {\Phi_{nt}^j}^{-\frac{1}{\theta}}\right)^{\alpha^j}\\
  &=& \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} \xi^{\alpha^j} {\Phi_{nt}^j}^{-\frac{\alpha^j}{\theta}}\\
  &=& \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} \xi^{\alpha^j} \left({B^j}^{-\theta} \sum_{i = 1}^N D_{nit}^j{P_{it}}^{\theta(\beta^j - 1)}\right)^{-\frac{\alpha^j}{\theta}}\\
  &=& \prod_{j = 1}^J \underbrace{{\alpha^j}^{- \alpha^j} \xi^{\alpha^j} {B^j}^{\alpha^j}}_{K^j} \left(\sum_{i = 1}^N D_{nit}^j{P_{it}}^{\theta(\beta^j - 1)}\right)^{-\frac{\alpha^j}{\theta}}\\
  &=& \left( \prod_{j = 1}^J K^j \right) \cdot \prod_{j = 1}^J \left(\sum_{i = 1}^N D_{nit}^j {P_{it}}^{\theta(\beta^j - 1)}\right)^{-\frac{\alpha^j}{\theta}}
\end{eqnarray*}
%
Notice that $\prod_{j = 1}^J K^j$ can be computed before the whole procedure.

%
We could simplify this further by solving for $\mathcal{P}_{nt} \equiv P_{nt}^{\theta}$ instead of $P_{nt}$:

\begin{equation}
\mathcal{P}_{nt} = K \prod_{j = 1}^J \left(\sum_{i = 1}^N D_{nit}^j {\mathcal{P}_{it}}^{(\beta^j - 1)}\right)^{-\alpha^j}, \label{price_eq}
\end{equation}

where $$K \equiv \left(\prod_{j = 1}^J K^j\right)^\theta \equiv \left(\prod_{j = 1}^J {\alpha^j}^{- \alpha^j} \xi^{\alpha^j} {B^j}^{\alpha^j} \right)^\theta$$
%
and $$D_{nit}^j \equiv Z_{it}^j {\left((L_{it}w_{it}^j)^{- \beta^j} \kappa_{nit}^j\right)}^{\theta}.$$

Then we solve the system for the vector $\mathcal{P}_{\cdot t}$ by iterating on the right hand side of (\ref{price_eq}) starting from $\mathcal{P}_{\cdot ,t-1}$.

\newpage
\subsection{Middle loop}
For a given resource allocation, $L_{nt}^j$, this loop searches for sectoral wages $w_{nt}^j$ that solve the nonlinear system of equations below.

\begin{eqnarray}
% \nonumber to remove numbering (before each equation)
  w_{nt}^j L_{nt}^j &=& \beta^j \sum_{m = 1}^N d_{mnt}^j \left( \alpha^j w_{mt} L_{mt} + \frac{1 - \beta^j}{\beta^j} w_{mt}^j L_{mt}^j\right)\\
  w_{nt} L_{nt} &=& \sum_{j = 1}^J w_{nt}^j L_{nt}^j
\end{eqnarray}

There are three important remarks:
\begin{itemize}
  \item The system is separable in $t$, so we can solve the corresponding subsystem for each $t$ separately.
  \item The system is nonlinear because $d$ depends on sectoral wages by definition through
    $$d_{mnt}^j \equiv \frac{{B^j}^{-\theta} T_n^j {A_{nt}^j}^{\theta} \left(\frac{{P_{nt}}^{1 - \beta^j} {w_{nt}^j}^{\beta^j}}{\kappa_{mnt}^j}\right)^{-\theta}}{{B^j}^{-\theta} \sum_{i = 1}^N T_i^j {A_{it}^j}^{\theta} \left(\frac{{P_{it}}^{1 - \beta^j} {w_{it}^j}^{\beta^j}}{\kappa_{mit}^j}\right)^{-\theta}}.$$
  \item This is a system of the form $x = A(x) x$, where the matrix $A(x)$ depends on $x$ nonlinearly. To solve for $x$ we can use the following iterative procedure.
    \begin{enumerate}
      \item Start from an initial $x^0$.
      \item Iterate $x^{i + 1} = \lambda A(x^i) x^i + (1 - \lambda) x^i$ $\forall i = 0, 1, \dots$ until $x^i$ converges to some $x^*$, where $\lambda \in \left(0,1\right]$ is a dampening parameter.
    \end{enumerate}
\end{itemize}

\subsubsection{Nonlinear part}
To facilitate computation we introduce $D$, the coefficients from the inner loop.
We can rewrite the definition of $d$ as
\begin{align*}
  d_{mnt}^j &= \frac{\overbrace{T_n^j {A_{nt}^j}^{\theta}}^{\frac{Z_{nt}^j}{{L_{nt}}^{\beta^j\theta}}} \left(\frac{{P_{nt}}^{1 - \beta^j} {w_{nt}^j}^{\beta^j}}{\kappa_{mnt}^j}\right)^{-\theta}}{\sum_{i = 1}^N \underbrace{T_i^j {A_{it}^j}^{\theta}}_{\frac{Z_{it}^j}{{L_{it}}^{\beta^j\theta}}} \left(\frac{{P_{it}}^{1 - \beta^j} {w_{it}^j}^{\beta^j}}{\kappa_{mit}^j}\right)^{-\theta}}\\
%
  &= \frac{Z_{nt}^j {L_{nt}}^{- \beta^j\theta} {w_{nt}^j}^{-\beta^j\theta} {\kappa_{mnt}^j}^{\theta}{P_{nt}}^{\theta(\beta^j - 1)}}{\sum_{i = 1}^N Z_{it}^j {L_{it}}^{- \beta^j\theta} {w_{it}^j}^{-\beta^j\theta} {\kappa_{mit}^j}^{\theta}{P_{it}}^{\theta(\beta^j - 1)}}\\
%
  &= \frac{\overbrace{Z_{nt}^j {L_{nt}}^{- \beta^j\theta} {w_{nt}^j}^{-\beta^j\theta} {\kappa_{mnt}^j}^{\theta}}^{D_{mnt}^j}{P_{nt}}^{\theta(\beta^j - 1)}}{\sum_{i = 1}^N \underbrace{Z_{it}^j {L_{it}}^{- \beta^j\theta} {w_{it}^j}^{-\beta^j\theta} {\kappa_{mit}^j}^{\theta}}_{D_{mit}^j}{P_{it}}^{\theta(\beta^j - 1)}}\\
%
  &= \frac{D_{mnt}^j {P_{nt}}^{\theta(\beta^j - 1)}}{\sum_{i = 1}^N D_{mit}^j{P_{it}}^{\theta(\beta^j - 1)}}
\end{align*}
Note that $d$ does not depend on the resource allocation.

\subsubsection{Linear part}
Substitute the sum from the second equation into the first one to get the expression below:

\begin{align*}
w_{nt}^j L_{nt}^j &= \beta^j \sum_{m = 1}^N d_{mnt}^j \left( \alpha^j \left(\sum_{k = 1}^J w_{mt}^k L_{mt}^k\right) + \frac{1 - \beta^j}{\beta^j} w_{mt}^j L_{mt}^j\right)\\
%
w_{nt}^j L_{nt}^j &= \alpha^j \beta^j \sum_{m = 1}^N  d_{mnt}^j \sum_{k = 1}^J w_{mt}^k L_{mt}^k \quad + \quad (1 - \beta^j)\sum_{m = 1}^N d_{mnt}^j w_{mt}^j L_{mt}^j
\end{align*}

We gather the coefficients on the right hand side to matrix $A$ and then iterate on the sectoral value added term according to the procedure described above. That is in iteration $i + 1$ the new value of the value added term is calculated as
$$\left(w_{nt}^j L_{nt}^j\right)^{i + 1} = \lambda A\!\left(\left(w_{nt}^j\right)^i\right) \left(w_{nt}^j L_{nt}^j\right)^i + (1 - \lambda) \left(w_{nt}^j L_{nt}^j\right)^i.$$

\subsubsection{Wage normalization}
Once the procedure converged, we get sectoral wages from dividing $w_{nt}^j L_{nt}^j$ by $L_{nt}^j$. Then we scale sectoral wages so that the average wage across all countries and sectors in any given period is 1.

\newpage
\subsection{Outer loop}
The goal of this loop is to find the sectoral resource allocations $L_{nt}^j$ that satisfy
$$\frac{L_{nt}^{j}}{L_{nt}} = E_{t - 1} \left( \frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right).$$
This loop runs over iterations of $L_{nt}^j$ until it converges up to a predefined threshold.

We use three methods to approximate the expectation:
\begin{itemize}
  \item An exponential trend that allows for constant growth rate.
  \item A band pass filtered trend that allows for breaks in growth rates.
  \item Perfect foresight to match actual values by neglecting the expectation operator.
\end{itemize}

\subsubsection{Exponential Trend}
%
\begin{enumerate}
  \item We start from the initial value $(L_{nt}^j)^0 = \alpha^j L_{nt}$. \footnote{Would it make sense to start from the average value added share in the data?}
  \item In iteration $i$ for the actual $(L_{nt}^j)^i$ we get sectoral and aggregate wages, $(w_{nt}^j)^i$ and $(w_{nt})^i$, from the middle loop.
  \item We calculate the implied total value added and the sectoral value added shares as
    $$\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i = \frac{(w_{nt}^j)^i (L_{nt}^j)^i}{(w_{nt})^i L_{nt}}.$$
  \item We calculate the exponential trend of the value added share series and use the trend values to approximate the expectation.
    \begin{enumerate}
      \item Estimate each of the $N \cdot J$ equations separately with OLS
        $$\log\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i = \delta_n^{j0} + \delta_n^{j1} t + \epsilon_{nt}^j.$$
      \item Normalize the trend values so that in each period in each country the trend values add up 1:
        $$\widehat{\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i} = \frac{\exp(\hat{\delta}_n^{j0} + \hat{\delta}_n^{j1} t)}{\sum_k \exp(\hat{\delta}_n^{k0} + \hat{\delta}_n^{k1} t)}$$
      \item Replace the expectation with the adjusted trend value.
      $$E_{t - 1} \left( \frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right) = \widehat{\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i}$$
    \end{enumerate}
  \item Update the resource allocations
    $$(L_{nt}^j)^{i + 1} = L_{nt} \widehat{\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i}$$
\end{enumerate}

\subsubsection{Band Pass Filtered Trend}
\begin{enumerate}
  \item We start from the initial value $(L_{nt}^j)^0 = \alpha^j L_{nt}$.
  \item In iteration $i$ for the actual $(L_{nt}^j)^i$ we get sectoral and aggregate wages, $(w_{nt}^j)^i$ and $(w_{nt})^i$, from the middle loop.
  \item We calculate the implied total value added and the sectoral value added shares as
    $$\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i = \frac{(w_{nt}^j)^i (L_{nt}^j)^i}{(w_{nt})^i L_{nt}}.$$
  \item
    \begin{enumerate}
      \item Decompose all $N \cdot J$ value added share series to trend and cycle components using a band pass filter with the same specification as in the shock decomposition.
        $$\log\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i = trend^j_{nt} + cycle_{nt}^j.$$
      \item Normalize the trend values so that in each period in each country the trend values add up 1:
        $$\widehat{\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i} = \frac{\exp(trend^j_{nt} + cycle_{nt}^j)}{\sum_k \exp(trend^k_{nt} + cycle_{nt}^k)}$$
      \item Replace the expectation with the adjusted trend value.
      $$E_{t - 1} \left( \frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right) = \widehat{\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i}$$
    \end{enumerate}
  \item Update the resource allocations
    $$(L_{nt}^j)^{i + 1} = L_{nt} \widehat{\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i}$$
\end{enumerate}

\subsubsection{Perfect Foresight}
\begin{enumerate}
  \item We start from the initial value $(L_{nt}^j)^0 = \alpha^j L_{nt}$.
  \item In iteration $i$ for the actual $(L_{nt}^j)^i$ we get sectoral and aggregate wages, $(w_{nt}^j)^i$ and $(w_{nt})^i$, from the middle loop.
  \item We calculate the implied total value added and the sectoral value added shares as
    $$\left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i = \frac{(w_{nt}^j)^i (L_{nt}^j)^i}{(w_{nt})^i L_{nt}}.$$
  \item Update the resource allocations
    $$(L_{nt}^j)^{i + 1} = L_{nt} \left(\frac{w_{nt}^j L_{nt}^j}{w_{nt} L_{nt}}\right)^i$$
\end{enumerate}

\newpage
\section{Potential Typos}

\subsection{Equilibrium Equations}
Formulas (15p) - (21p) describe the equilibrium in the paper.
\begin{align}
% \nonumber to remove numbering (before each equation)
  P_{nt} &= \prod_{j = 1}^J {\alpha^j}^{- \alpha^j} {P_{nt}^j}^{\alpha^j} \tag{15p}\\
  %
  P_{nt}^j &= \xi^j {\Phi_{nt}^j}^{-\frac{1}{\theta}} \tag{16p}\\
  %
  \Phi_{nt}^j &= {B^j}^{-\theta} \sum_{i = 1}^N T_i^j {A_{it}^j}^{\theta} \left(\frac{{P_{it}}^{1 - \beta^j} {w_{it}^j}^{\beta^j}}{\kappa_{nit}^j}\right)^{-\theta} \tag{17p}\\
  %
  d_{mnt}^j &= \frac{{B^j}^{-\theta} T_n^j {A_{nt}^j}^{\theta} \left(\frac{{P_{nt}}^{1 - \beta^j} {w_{nt}^j}^{\beta^j}}{\kappa_{mnt}^j}\right)^{-\theta}} {\Phi_{mt}^j}; \qquad \sum_n^N d^j_{mnt} = 1 \tag{18p}\\
  %
  w_{nt}^j L_{nt}^j &= \beta^j \sum_{m = 1}^N d_{mnt}^j \left( \alpha^j w_{mt} L_{mt} + (1 - \beta^j) w_{mt}^j L_{mt}^j\right) \tag{19p}\\
  %
  w_{nt} L_{nt} &= \sum_{j = 1}^J w_{nt}^j L_{nt}^j \tag{20p}\\
  %
  \frac{L_{nt}^{j}}{L_{nt}} &= E_{t - 1} \left( \frac{w_{n}^{j} L_{n}^{j}}{\sum_{k = 1}^J w_{n}^k L_{n}^k}\right)  \tag{21p}
\end{align}

\subsection{Proposed Corrections}
\begin{itemize}
  \item (16p): $\xi$ does not depend on $j$
  \item (18p): Adding that the sum of the $d$ values is 1 is redundant
  \item (19p): It should be $\frac{1 - \beta^j}{\beta^j}$ instead of only $(1 - \beta^j)$
  \item (21p): Adding the time subscript $t$ to the terms in the expectation may be better
\end{itemize}

\subsubsection{Equations for Calibration}
If the equilibrium equations (15p-21p) are OK, then (27p) is OK.\\
Equation (28p)
\begin{equation}
Z_{mt}^{j}=\left( B^{j}\right) ^{-\theta }d_{nmt}^{j}\left( y_{m}^{j}\right)
^{\theta \beta ^{j}}\left( \kappa _{nmt}^{j}\right) ^{-\theta }\left( \frac{%
P_{nt}^{j}}{P_{mt}}\right) ^{-\theta }\left( \psi _{mt}^{j}\right) ^{-\theta
\beta ^{j}}  \tag{28p}
\end{equation}%
Correct version
\begin{equation}
Z_{mt}^{j}={B^{j}}^{\theta } {\xi}^\theta d_{nmt}^{j}\left( y_{m}^{j}\right)
^{\theta \beta ^{j}}\left( \kappa _{nmt}^{j}\right) ^{-\theta }\left( \frac{%
P_{nt}^{j}}{P_{mt}}\right) ^{-\theta }\left( \psi _{mt}^{j}\right) ^{-\theta
\beta ^{j}}
\end{equation}%
Equations (29p), (30p) and (31p) are OK.\\
Equation (32p)
\begin{equation}
Z_{0t}^{j}=\left( B^{j}\right) ^{-\theta }d_{00t}^{j}\left(
y_{0t}^{j}\right) ^{\theta \beta ^{j}}\left( \frac{P_{0t}^{j}}{P_{0t}}%
\right) ^{-\theta }\left( \psi _{0t}^{j}\right) ^{-\theta \beta ^{j}} \tag{32p}
\end{equation}%
Correct version
\begin{equation}
Z_{0t}^{j}={B^{j}}^{\theta } {\xi}^\theta d_{00t}^{j}\left(
y_{0t}^{j}\right) ^{\theta \beta ^{j}}\left(\frac{P_{0t}^{j}}{P_{0t}}\right) ^{-\theta }\left( \psi _{0t}^{j}\right) ^{-\theta \beta ^{j}}
\end{equation}%
Equation (33p) is OK.\\
Equation (34p)
\begin{equation}
Z_{01972}^{j}=d_{001972}^{j}\left( y_{1972}^{j}\right) ^{\theta \beta
^{j}}\left( \psi _{01972}^{j}\right) ^{-\theta \beta ^{j}} \tag{34p}
\end{equation}%
Correct version
\begin{equation}
Z_{01972}^{j}= {B^{j}}^{\theta } {\xi}^\theta d_{001972}^{j}\left( y_{01972}^{j}\right) ^{\theta \beta
^{j}}\left(\frac{1}{P_{01972}}\right) ^{-\theta}\left(\psi_{01972}^{j}\right) ^{-\theta \beta ^{j}},
\end{equation}
as $P_{01972}^{j}$ is normalized to 1 for all $j$. (See details in the section on price normalization above.)\\
Equation (35p) is OK.

\end{document}


\section{\textbf{SimpleLinearTest}}

\subsection{Particular Case (problem)}
The problem is to implement a bounded priority queue, which are a
multi-set data structure; each set has an associated priority. The
contract or interface for a priority queue is essentially two methods:
\C{add(x,k)} which adds element $x$ to the set of priority $k$, and
  \C{removeMin} which returns and removes the element with smaller
  (higher) priority from all the sets. This type of priority queues
  are called ``bounded'', because the priority of elements is taken from
  a finite predefined set.

\subsection{Solution}
Given that the problem is to implement a bounded priority queue, an
array shall suffice. So we create an array of sets as big as the range
of values for the priority. The \C{add} and \C{removeMin} methods are
straightforward, but they are built on the assumption that the
auxiliary class \C{Bin} is thread safe (which is the case, as it is
implemented with locks). Below the relevant snippets of code: \\

\begin{lstlisting}[style=numbers]
public class SimpleLinear<T> implements PQueue<T> {
  int range;
  Bin<T>[] pqueue;

  ...

  public void add(T item, int key) {
    pqueue[key].put(item);
  }

  ...

  public T removeMin() {
    for (int i = 0; i < range; i++) {
       T item = pqueue[i].get();
       if (item != null) {
         return item;
       }
    }
    return null;
  }
}
\end{lstlisting}
\hfill

\subsection{Experiment Description}
The test program consists of the following test cases: \\

\begin{itemize}
  \item \C{testAdd}: Serial test which adds 512 random numbers in a
    fixed range (these numbers become the priority), and later validates
    that they are removed in ascending order with \C{removeMin}. \\
  \item \C{testParallelAdd}: Same as \C{testAdd}, with the exception
    that the addition of the elements is done in parallel with 8
    threads. \\
  \item \C{testParallelBoth}: Same as \C{testParallelAdd}, except that
    the removal of items is also performed in parallel with 8 threads.
\end{itemize}

\subsection{Observations and Interpretations}
The test runs fine in 2 and 24 core machines, below a sample output:
\\

\begin{verbatim}
[oraadm@gdlaa008 orig]$ junit priority.SimpleLinearTest
.add
OK.
...
OK.
OK.
.testParallelBoth
OK.
.testParallelAdd
OK.

Time: 0.116

OK (3 tests)
\end{verbatim}
\hfill


% Thesis, Chapter 5
% by Rachel Slaybaugh

\chapter{Summary}
\label{sec:Chp5}
To enhance and improve the design of nuclear systems, high-fidelity neutron fluxes are required. For today's problems, high-fidelity means thousands $\times$ thousands $\times$ thousands of mesh points, up to $\sim$150 energy groups, accurate scattering expansions, and the use of many directions. Leadership-class machines provide platforms on which problems of this size can be solved in a reasonable amount of time. Computing such fluxes accurately and efficiently requires numerical methods with good convergence properties and algorithms that can scale to hundreds of thousands of cores.

Many 3-D deterministic transport codes can scale in space and angle to tens of thousands of cores. They typically rely on methods such as Gauss Seidel for fixed source problems and power iteration wrapped around GS for eigenvalue problems. Within group solvers range from source iteration to Krylov methods. In many cases these methods are accelerated with strategies like CMR, TTG, DSA, and others discussed earlier in this document. Nevertheless, these calculations can be slow to converge for challenging systems like those with highly scattering materials or high dominance ratios.

Three methods have been added to Denovo that are designed to improve convergence and enable the full use of leadership-class computers. The driving concept of this research is to use Rayleigh quotient iteration for solving the $k$-eigenvalue problem in 3-D neutron transport. This objective was not tractable with the tools originally available in Denovo. The multigroup Krylov solver and multigrid-in-energy preconditioner were developed to facilitate RQI, though both of these tools are also useful on their own. This chapter reviews each method, highlights associated results, recommends future work, and closes with some discussion about their potential impact.

\section{Multigroup Krylov Solver}
Before this work began, Denovo had been shown to exhibit good strong scaling in space and angle to about 20,000 cores using Gauss Seidel as its multigroup solver. The first method added was a multigroup Krylov solver that was designed to improve convergence when compared to Gauss Seidel and to dramatically increase the number cores Denovo can use.

Instead of sequentially solving each group with some inner iteration method and then using GS for outer iterations to converge the up-scattering, the MG Krylov solver treats a block of groups (either all groups or just upscattering groups) at once such that the inner-outer iteration structure is removed. Using a Krylov method over the upscattering block instead of GS resulted in much faster convergence, both in terms of time and iteration count. No other transport codes were found that apply Krylov solvers over a block of groups without also needing inner iterations.

In addition, the block Krylov solver allows energy groups to be solved simultaneously because the multigroup-sized matrix vector multiply can be divided up in energy and parallelized. The block is broken into energy sets and each set is handled by a different group of cores. The spatial decomposition does not change, and each set gets the entire spatial mesh such that space-angle scaling remains intact.  Other \Sn transport codes that are parallelized in the energy dimension could not be found. Energy sets extend the number of cores that can be used efficiently by Denovo from tens of thousands to hundreds of thousands.

Three test problems were used to investigate how the new method compares to GS, and how it scales in energy. The iron graphite cube and PWR900 tests showed MG Krylov to be much faster than GS, even when GS is accelerated. Once the multiset communication was optimized, the strong scaling performance of block Krylov was shown to be nearly perfect to 50,000 cores and reasonable to 200,000 cores. Weak scaling in energy was not as good, though it was not tested with the improved multiset communication strategy. The addition of energy sets enabled Denovo to use 100,000 and 200,000 cores with reasonable efficiency; this would not have been possible for the same problems with KBA alone.

%Two test problems were used to investigate the new method's functionality, how it compares to GS, and how it scales in energy. The small fixed source, iron graphite cube showed that the Krylov solver was much faster than unaccelerated GS. The use of graphite made this problem particularly challenging for GS. A full PWR was also studied with power iteration as the eigenvalue solver. The 2-group versions of this problem showed the MG Krylov solver to be much faster than accelerated GS. Both tests confirmed that the new multigroup Krylov solver has the potential to substantially outperform Gauss Seidel in challenging cases and problems of interest.

%The effect of using energy sets was also investigated with both test problems. The iron graphite cube verified that the new MG solver works successfully with multisets. The large PWR problem that had over 1.7 trillion degrees of freedom was used to investigate strong scaling. Three cases that used different combinations of $\sim$100,000 or $\sim$200,000 cores, and 11 or 22 energy sets were computed on Jaguar. This is exactly the type and scale of calculation the MG Krylov solver was made to handle.

%The strong scaling study gave mixed results. Initially, doubling the energy sets with approximately the same spatial decomposition yielded 80\% efficient scaling. Doubling the spatial blocks with the energy sets held constant gave 86\% efficient scaling. Both of those numbers are great. However, some communication was optimized in Denovo and the energy scaling cases were repeated. This reduced the efficiency to only 62\%. Some improvement in the multiset communication has been implemented since these results were found, but the tests have not been repeated to see the effect on energy scaling.

%The PWR problem was also used for a weak scaling study. The ratio of calculated time to ideal time was 1.019 when the problem was expanded from 17,424 cores to 112,200 cores. The weak scaling performance was very good.

A multigroup Krylov solver has not been previously pursued because of computer architecture limitations in the past. It is only now that there are machines with memories large enough to accommodate such large Krylov subspaces and that have enough cores to warrant the capability of using hundreds of thousands of cores. Denovo has been used successfully on a machine of this size for a real problem because of the new energy decomposition. The test problems demonstrated that the new method accomplishes the goal of accelerating transport calculations by fully using new computers.

\section{Rayleigh Quotient Iteration}
Finding the criticality state of a multiplying nuclear system is one of the most important kinds of analysis done for nuclear systems. The standard eigenvalue solution methods can converge slowly in some important instances. To design new reactors, a better eigenvalue solver is needed. One of a class of methods that solves a modified system with identical eigenvectors and shifted eigenvalues, Rayleigh quotient iteration finds the dominant eigenvalue using a mathematically optimal shift. Theory indicates that RQI should converge in fewer iterations than traditional eigenvalue solvers.

The addition of RQI, though, would not be possible without the MG Krylov solver. Within Denovo, the Rayleigh quotient in RQI multiplies the fission matrix which is then subtracted from the scattering matrix. This results in a set of equations that is mathematically equivalent to having up-scattering in every group, so the scattering matrix becomes energy-block dense. Handling energy-block dense systems when there are more than a few energy groups is not tractable with GS as the multigroup solver. It is only the MG Krylov solver that makes RQI a reasonable idea when there are many energy groups.

%In return, RQI takes fuller advantage of the energy parallelization made available by the block Krylov solver than power iteration or fixed source solves can. Every group can be solved simultaneously since every group looks like it has upscattering. In the upper limit, there can be as many energy sets as groups in a problem. This degree of energy parallelization only makes sense with RQI.

There is, however, a drawback that comes from using RQI with a Krylov method. When the Rayleigh quotient is close to the eigenvalue being sought, the system becomes ill-conditioned. The closer RQI comes to solving the problem, the worse the conditioning becomes. Krylov methods can converge very slowly and may not be backwards stable for ill-conditioned systems. If the multigroup iterations do not converge the eigenvector inside the RQI calculations, then RQI itself will not converge as it will not be able to find an accurate RQ.

Three sizes of problems, small, medium, and large, were used to test whether RQI requires fewer iterations than power iteration and whether the shift prevents the Krylov solver from converging. The small problems showed RQI could correctly find the eigenvalue and eigenvector in fewer iterations than power iteration. RQI began to falter with the medium problem, finding an eigenvalue only close to the reference and failing to converge the eigenvalue or vector. With the large problems RQI did not work at all.

%Two small problems, one with reflecting and one with vacuum boundaries, demonstrated that RQI could correctly find the eigenvalue and eigenvector in fewer iterations than power iteration.

%The intermediately sized problem with reflecting boundary conditions did not work as well. The Krylov solver was not able to converge the flux moments in a reasonable number of iterations. Then the eigenvalue part of the calculation did not converge because there was not an accurate eigenvector from which to make the eigenvalue guess. This problem seemed to have an eigenvector guess that was close to correct, though, because the RQI-computed eigenvalue only differed from the reference $k$ by 0.0033. With two benchmark reactor problems, RQI did not work at all. The multigroup iterations did not converge and the eigenvalue was not close to the reference.

Valuable insights were gained from the RQI test results despite the failures. Simple problems demonstrated that RQI can find correct solutions and converge in fewer iterations than power iteration, thus it has promise for accelerating eigenvalue calculations. The failures showed that for real problems the matrix system becomes too ill-conditioned to work with a Krylov solver. If preconditioning can get the Krylov multigroup solves to converge, then RQI will likely be a successful method that can solve at least some challenging problems in fewer eigenvalue and total Krylov iterations than power iteration.

Despite the clear theoretical edge RQI has over many other iterative eigenvalue methods, it has never been used for 3-D neutron transport before, largely because of the application of the shift. Traditional multigroup solvers like Gauss Seidel cannot treat energy-block-dense scattering systems efficiently. The MG Krylov solver lets RQI be used in this context, and enables the parallelization of eigenvalue calculations to hundreds of thousands of cores.

\section{Multigrid in Energy}
Another way to reduce calculation time is by reducing the number of iterations needed for convergence through preconditioning. %If the savings from a lower iteration count is more than the cost of the extra work done in making and applying the preconditioner, the result is a faster code.
A multigrid-in-energy preconditioner was added to Denovo to reduce iteration count for all problem types, and to address the convergence issues arising from RQI.

Iterative methods reduce oscillatory error effectively, but not smoother error. The smooth error can prevent iterative methods from converging. This behavior is characterized by rapid error reduction in the first several iterations followed by very little error reduction. Such a trend was observed when RQI failed. Multigrid methods selectively remove smoother error components and are therefore ideal for accelerating this type of problem. Thus, a multigrid preconditioner should work very well with RQI.

The  preconditioner conducts a multigrid method in the energy dimension. A set of energy grids with increasingly coarse energy group structures are created. To apply the preconditioner, the residual is restricted and prolonged between these grids and weighted Richardson iteration is done on each grid to remove different error modes. The V-cycle(s) result in a vector that corrects the current flux estimate.

%Restricting a problem from a fine energy grid to a coarser energy grid makes the error modes that appeared smooth on the fine grid become oscillatory. An iterative scheme can be applied on the coarse grid to reduce the error components that appear oscillatory there. The problem can then be prolonged back to the fine energy grid where the troublesome error modes have now been removed.

The multigrid preconditioner is implemented in a way that easily and efficiently takes advantage of the new energy decomposition. The multigrid algorithm is applied within each energy set such that the energy groups are only restricted and prolonged between groups on that set. Sets do not have to communicate with one another in the preconditioner, and using more sets creates fewer total grid levels. An alternative strategy that was not investigated is to restrict groups across sets such that the total number of grid levels is set-count independent.

Many tests were done to investigate the behavior of the new preconditioner. All of the tests showed that the preconditioner can be useful for many problem types: reflecting and vacuum boundaries, few and many groups, fixed source and eigenvalue. The preconditioner always worked well for fixed source problems. Multigrid in energy also dramatically reduced iteration count in all tests with power iteration.

The preconditioning did not always work well for problems that RQI could already solve, though these were easy problems that did not need to be preconditioned. More importantly, though, problems that RQI could not solve without preconditioning did converge with it. As problems became more realistic, RQI benefited more from preconditioning. Preconditioned RQI used fewer iterations and was substantially faster than preconditioned power iteration for the full-facility PWR.

There was one test RQI did not handle well. It highlighted that RQI is a somewhat brittle method that does not always work because of the ill-conditioning issue. When it does work, it works very well. As a general rule, the output of RQI should not be trusted if the Krylov iterations do not converge on every iteration, even if $k$ does converge. $k$ should only be considered valid when all of the multigroup iterations converge. To help ensure the eigenvector is converged for real problems, a moderate amount of preconditioning (e.g., $w1r3v3$) should be used.

%The preconditioner always worked well for fixed source problems. The number of iterations were reduced for the vacuum and reflecting unit tests as well as for the iron and graphite cube problem. Multigrid in energy also always worked well with power iteration, dramatically reducing iteration count in all tests.

%For tough fixed source and eigenvalue problems solved with PI, the preconditioner could prove to be very beneficial once it has been optimized. The timing comparisons were not favorable, but there are a several suggestions in the future work section that could remedy this.

%With RQI, the preconditioning did not always work well for problems that RQI could already solve, though these were easy problems that did not need to be preconditioned. In these simple cases preconditioned power iteration worked better than preconditioned RQI. All of the tests where this was the case had low dominance ratios. The spectral properties of these systems are good for PI and are not necessarily as good for RQI.

%Problems that RQI could not solve without preconditioning did converge with it. As problems became more realistic, RQI benefited more from preconditioning. For the 2-D benchmark problem, RQI worked well and was slightly inferior to preconditioned PI. In the 3-D benchmark study RQI performed better than PI for intermediate and large amounts of preconditioning. RQI was substantially better than PI in the full-facility PWR test case.

%The one trouble case was the intermediate infinite medium problem. RQI always had difficultly and preconditioning did not help much. It is unclear why this happened. It seemed that RQI did not perform as well for any problem that had six reflecting boundaries, though this was the only such case where RQI was unreliable. Power iteration, however, solved the problem without issue.

%This test highlights that RQI is a somewhat brittle method. It does not always work because of the ill-conditioning issue. When it does work, it works very well. As a general rule, the output of RQI should not be trusted if the Krylov iterations do not converge on every iteration, even if $k$ does converge. In the cases where this happened the $k$ was usually correct, but it was not always correct. $k$ should only be considered valid when all of the multigroup iterations converge. To help ensure the eigenvector is converged for real problems, a moderate amount of preconditioning (e.g., $w1r3v3$) should be used.

The multigrid method worked efficiently with multisets and scaled very well. %This is likely for two reasons. One is that the preconditioner did not add inter-set communication, so changing the number of sets does not increase the cost of the preconditioner from a communication standpoint. The second is that the reduction in number of grids makes the preconditioner less expensive with increasing set count.
%
Not only were there no negative impacts from using more energy sets in the preconditioner, but scaling was actually improved by its use. The energy scaling performance with this implementation of the multigrid-in-energy method suggests that this parallelization strategy is better than the one that communicates across sets.

A multigrid method has never been applied to the energy portion of phase space for the neutron transport problem before. This is not surprising because the motivation for the preconditioner is relatively new as well. Only in the last few years have energy parallelization and Rayleigh quotient iteration become feasible. Without those drivers there was no reason to do multigrid in energy.

\section{Suggested Future Work}
While many interesting things about these three methods have been found, there are still many questions to be answered. There are also a few remaining implementation details that will make these methods more flexible. This section suggests some ideas for further investigation that would help answer some questions more decisively, could guide how and when these methods should be used, and could lead to their improvement. Some suggestions are research-based, some require implementation, and some are simply tests.

\subsection{Testing}
Most of the problems where direct comparison between PI and RQI was possible did not have characteristics that represented the kinds of problems that scientists want to solve in practice. The cases that did have realistic characteristics could not be meaningfully compared. Tests that conclusively show whether or not preconditioned RQI can be better than unpreconditioned PI are of highest priority.

Calculations with the PWR900 should be done for this purpose, all using the same spatial decomposition, the same number of energy sets, and the same tolerances. The preconditioned RQI calculations should use $S_{2}$ in the preconditioner, only two grids in energy, and a few different preconditioning parameters sets. These results should be compared to unpreconditioned PI. In addition, at least one PI calculation with the same preconditioning options as RQI should be run to completion so a full comparison can be made.

One or two other test problems with parameter consistency across calculations and with realistic properties, such as high dominance ratios, are also needed. Doing sound comparisons between a few different problems should provide enough information to make conclusions about the relative benefit of RQI when compared to PI.

More investigation into weak scaling for all solvers is needed as well. The performance with MG Krylov reported in Chapter~\ref{sec:Chp2} was very poor. Such a study should be repeated with the optimized multiset communication. If the results are not improved, a deeper investigation into the causes and possible remedies is warranted.

No weak scaling studies have been done for the preconditioner or RQI. Once the MG Krylov study has been completed, a preconditioned fixed source weak scaling study should be done. The performance with the preconditioner can be compared to the performance with MG Krylov alone. After that, a preconditioned RQI weak scaling study can be done, informed by the MG Krylov and preconditioned fixed source results.

\subsection{Optimization}
RQI and multigrid in energy can both by improvement through implementation changes. \\Rayleigh quotient iteration can not be used with energy sets when there are reflecting boundaries. This capability should be added as it vastly expands the number of problems for which it is applicable. This also expands the suite of tests available to study energy scaling with RQI. For example, the 2-D and 3-D c5g7 benchmark problems would then be available for these kinds of tests.

Using a reduced number of solution directions inside the preconditioner is also not available when there are reflecting boundaries. This option seems like it may be very useful for reducing the time taken by the preconditioner. Ensuring all problems have this tool available will increase the benefit of preconditioning for problems using anything besides $S_{2}$.

Related to this, the benefit gained by using the smaller angle set in the preconditioner should be quantified. If it is found that the benefit is large and there is little or no impact on iteration reduction, then the default behavior could be changed to always use $S_{2}$, or $S_{N/2}$, etc.\ in the preconditioner with an input option to specify something else if desired.

A crucial area of work is optimizing the preconditioner to make it faster. The multiset tests indicate that reducing the total number of energy grids to two or three regardless of the number of energy groups will not impact the iteration reduction gained from preconditioning. This change alone would greatly reduce the time the preconditioner takes for problems with more than a few energy groups. A few different grid depths should be investigated to assess what is best. Much speed can probably be gained by more efficient C++ coding as well.

Once ideal V-cycle depth has been determined, guidance has been developed about using reduced angle sets, and any C++-related optimization has been added, comparison studies should be considered for time. The tests done in this work can be repeated or similar studies using problems with practical engineering properties can be done for this purpose. Energy scaling studies will also be needed to learn how the preconditioner scales with a different implementation of the number of grid levels.

\subsection{Research}
A more research-oriented line of investigation is to try different ways to test for convergence within RQI. Right now the current Rayleigh quotient is compared to the previous one. A different criterion might be better or lead to more accuracy, for example using the eigenvector instead of the eigenvalue. There has also been some convergence analysis for RQI and research discussing interesting ways to set RQI convergence criteria that would inform this research. See for example Notay \cite{Notay2003} and Szyld \cite{Szyld2011}.

Changing some of the solution details inside the preconditioner is another avenue of research. Relaxation methods besides weighted Richardson should be considered. It may be that something like Jacobi, or successive over relaxation, or another iterative method would work better. The behavior of the weight parameter inside the preconditioner was difficult to characterize in a generic way and occasionally caused poor behavior. Weight is the only parameter that is specifically related to Richardson. Another method would avoid that difficulty and could be more robust as a result. The preconditioner might also benefit from different prolongation and/or restriction operators. A few different methods should be tried and tested to see if any impact is made.

Some of these suggestions require more time and effort than others. A shallower default V-cycle will be easy to implement. Adding reduced angle sets and RQI with multisets for reflecting boundaries may be more work, but will be important to have. Using a different RQI convergence strategy or a different relaxation method are topics that likely require mathematical analysis and more in depth research. Most importantly, though, more testing should be done to more rigorously understand the new methods and their impacts in different scenarios.

\section{Closing Remarks}
The goal of this research was to accelerate transport calculations with methods that can take full advantage of modern leadership-class comuputers, facilitating the design of better nuclear systems. Three complimentary methods were implemented that accomplish this goal to varying degrees.

At the outset of this work, Denovo could be decomposed in five of the six dimensions of phase space over which the steady-state transport equation is solved in a way that restricted it to about 20,000 cores for a problem with 500 million cells. The original suite of solvers included Gauss Seidel as the multigroup solver and power iteration wrapped around GS as the eigenvalue solver. These solvers have some significant limitations in many cases of interest.

The new multigroup Krylov solver converges more quickly than GS and enables energy decomposition such that Denovo can scale to hundreds of thousands of cores. The new multigrid-in-energy preconditioner reduces iteration count for many problem types and takes advantage of the new energy decomposition such that it can scale very efficiently. These two tools are useful on their own, but together they allow the Rayleigh quotient iteration eigenvalue solver to work.

The real motivation of this work was to add RQI, which should converge in fewer iterations that power iteration for large and challenging problems. RQI creates shifted systems that would not be tractable without the MG Krylov solver. It also creates ill-conditioned matrices that cannot converge without the multigrid-in-energy preconditioner. Using these methods, RQI converged in fewer iterations and in less time than preconditioned PI for a full-facility PWR on 200,000 cores.

The methods added in this research accelerated Denovo in multiple ways. This acceleration helps enable the solution of today's ``grand challenge'' problems. It is hoped that improved methods will lead to improved reactor designs and systems, and that the frontier of computational challenges will be moved forward.


\chapter{Re-weighting of the $W/Z$ boson $\pt$}
    \label{app:BosonPtReweight}

The comparison of the boson $\pt$ distribution in the control regions, in data and simulation, indicates that the Monte Carlo overestimates the tail of the measured distribution.
This is attributed to an inadequate modeling of the true boson $\pt$ that recoils the rest of the hadronic final state.
Therefore, a re-weighting procedure is applied to correct the generated boson $\pt$ distribution in the MC.

Weights are determined separately for $\wen$+jets, $\wmn$+jets and $\zmm$+jets control regions, and found to be compatible.
Nonetheless, the weights obtained from the $\wmn$+jets control region are adopted due to its higher statistical power.
These weights are shown in Figure \ref{fig:BosonPtWeights}.
The statistics in the region $\pt^{\text{boson}} > \unit[400]{GeV}$ is limited, and a single weight is therefore employed to correct the modeling of the boson $\pt$ in these events.

\begin{figure}[!ht]
  \begin{center}
    \mbox{
      \includegraphics[width=0.635\textwidth]{Appendix_BosonPtReweight/Figures/BosonPtWeights.eps}
    }
  \end{center}
  \caption{Weights applied to the generated boson $\pt$ in the $W/Z$+jets MC samples.}
  \label{fig:BosonPtWeights}
\end{figure}

Figure~\ref{fig:bosonptwcheck} shows the impact of the $\pt$ re-weighting in the boson $\pt$ and the $\met$ distributions for the signal region M1.
The distributions in this figure do not include the multijet contribution and only show the statistical uncertainties, but they remain sufficient to validate the re-weighting.
As expected, the shape of the distribution is adjusted to that of the data.

\begin{figure}
\begin{center}
\mbox{
  \includegraphics[width=0.49\textwidth]{Appendix_BosonPtReweight/Figures/plot_Stop_A6_SR_bosonVec_reco_pt_fitted_NoBosonPtReweight.eps}
  \includegraphics[width=0.49\textwidth]{Appendix_BosonPtReweight/Figures/plot_Stop_A6_SR_bosonVec_reco_pt_fitted_BosonPtReweightApplied.eps}
}
\mbox{
  \includegraphics[width=0.49\textwidth]{Appendix_BosonPtReweight/Figures/plot_Stop_A6_SR_met_fitted_NoBosonPtReweight.eps}
  \includegraphics[width=0.49\textwidth]{Appendix_BosonPtReweight/Figures/plot_Stop_A6_SR_met_fitted_BosonPtReweightApplied.eps}
}
\end{center}
\caption[Distributions for the boson $\pt$ and the $\met$ in the signal region M1.]{
Distributions for the boson $\pt$ (top) and the $\met$ (bottom) in the signal region M1. The distributions on the left do not include the boson $\pt$ re-weighting, whereas the distributions on the right do so. Only the statistical uncertainties are shown in the distributions.}
\label{fig:bosonptwcheck}
\end{figure}