\documentclass[11pt,twocolumn]{asaproc}

\usepackage{graphicx}
\graphicspath{{images/}}

\usepackage{amsmath}
\usepackage{url}
\usepackage{numberedblock}

%\usepackage{mathtime}

%%UNCOMMENT following line if you have package
\usepackage{times}

\title{Bayesian Analysis of COVID-19 cases in California}

\author{Gulzina Kuttubekova\thanks{UCSC Baskin School of Engineering, Department of Statistical Science}}
\begin{document}

\maketitle

\begin{abstract}
 Since mid-January of 2020, we have seen the first cases of coronavirus disease (COVID-19) in the US. Serious respiratory disease caused by the novel virus had taken the lives of approximately 300,000 people in the world and infected more than 4.3 million people worldwide. \url{https://www.worldometers.info/coronavirus/} provides various COVID-19 related statistics about the US as well.

% write what you will do here
We examine the dataset of COVID-19 cases in California, along with the death rate among those who were found infected by this novel virus. We also look for a possible correlation between the number of infected people and the population in each county. We employ two hierarchical models to examine the infection rate and assess models by posterior predictive checks.

% results
A modified hierarchical model on the joint distribution of the number of infections and the number of deaths showed a better fit than the hierarchical model which doesn't take into account the mortality rate and its distribution. In general, we saw systematic differences in the number of deaths and infections across counties. The assumption that the mean number of the number of infections is 20\% of the population had a high impact on the posterior distribution in the first model.

\begin{keywords}
Bayesian analysis, COVID-19, hierarchical models, MCMC
\end{keywords}
\end{abstract}

\section{Introduction\label{Introduction}}

% Intro - motivation
 The novel virus is still spreading across the world despite the precautions taken by governments of different countries. The numbers connected to this pandemic are changing very rapidly, not every day, but every hour.  There are only 13 countries left where there are no officially confirmed cases of COVID-19, according to \url{https://www.aljazeera.com/news/}. However, it's believed that it's only a matter of time while the novel virus reaches those countries. Since the first massive cases of COVID-19 and related death numbers started increasing in the Greater Seattle area in the US in mid-March, the epicenter of outbreak shifted quickly from West to East coast. At the current time, the Greater New York area became the epicenter of coronavirus battle. Although the state of California has as twice as the population as New York state, according to Worldometers number of reported infected people in the New York area is seven times greater than in California. Obviously there are many external factors that can explain this discrepancy.

%Problem
We want to know if there exists any discrepancy in the distribution of COVID-19 cases among the counties in California. If yes, which counties are more influential in explaining the number of death per reported number of infected people. We are also interested in the variation in the number of infected people by county. It's expected that as larger the population of the county as greater the number of infected people. However, it might be a false expectation, since there are other factors influencing the infection rate as population density, social culture, access to healthcare services and etc. We also integrate the new assumption that infection depends on mortality to answer the proposed research questions.

%What we employ/do?
We employ two hierarchical Poisson models constructed in different ways. We tackle the problem stated above using a fully Bayesian approach. Results from those models are reported and compared. We also use results from the previous study, which are available at \url{https://github.com/kgulzina/bayesian-analysis-of-COVID19-in-CA} for additional model checks.

\subsection{Data}

% Describe dataset
We analyze the dataset which contains information about incidence and mortality due to coronavirus in California. The number of reported cases and death is given per each county as of 04/13/2020. There are 58 observations, as there are 58 counties in the state in total. There are 4 variables: \{County, Total.cases, Deaths, Population\} s.t. population, the number of infected people, and number of people who died from COVID-19 are given. As we have count variables except for the county name, we treat the number of deaths $y_i$  out of $n_i$ infected people per $i^{th}$ county with population $c_i$ as discrete random variables. We also treat $y_i$ and $n_i$ as response variables.

\subsection{EDA}
The top three counties with the largest number of infected people are Los Angeles, San Diego, and Riverside.  Although San Francisco county has the largest population density $\sim18,553$, among all 58 counties in the state, it has 11 times less number of infected people compared to Los Angeles county with $\sim2,488$ population density. There are five counties that have zero reported cases of COVID-19. All five counties have population $\leq 32,000$. Interestingly four of those counties are in Northern California, and Mariposa county is relatively close to the highly populated county like Santa Clara. Up to date, $\sim 0.06\%$ of the population of California were infected with the virus. We would like to learn the trend in the number of deaths as this percentage increases. Moreover, throughout the study, we assume the mean number of coronavirus cases is equal to 20\% of the population.

% include plot1: case numbers
\begin{figure}[t]
\centering\includegraphics[scale=.30]{infected_per_county.jpeg}
\caption{Distribution of the total number of confirmed cases of COVID-19 across counties of California.}
\label{fig:totcases}
\end{figure}

% include plot2: case numbers by population
\begin{figure}[t]
\centering\includegraphics[scale=.31]{inf_vs_pop.jpeg}
\caption{Scatter plot number of confirmed cases of COVID-19 vs population for each county in California.}
\label{fig:infpop}
\end{figure}

% talk about infection rate vs population
According to the plot in Figure ~\ref{fig:totcases}, we see that number of confirmed cases is not evenly distributed across counties. Los Angeles has the most noticeable spike in cases. The top five counties with the total number of infected people $\geq$ 1,000 are Los Angeles, San Diego, Riverside, Santa Clara, and Orange counties. Apparently these high numbers are associated with the population in each county. Figure ~\ref{fig:infpop} shows that as population number increases the number of infected people increases as well. Scatter plot shows there is a potential outlier: Los Angeles county, which has the largest population and the highest number of infected people as well. As we stated earlier, there might be some other external factors affecting the non-homogenous distribution of cases across counties. For instance, infection numbers can be related to population density as we see in Figure ~\ref{fig:density},  the top 5 infected counties have population density of $\geq$ 1,000. Note that in the previous study we treated the number of confirmed cases $n_i$ as a known quantity, and now we treat it as a random variable of interest.

% include plot3: population density of California counties
\begin{figure}[t]
\centering\includegraphics[scale=.30]{density.png}
\caption{Population density of California (Hines 2019).}
\label{fig:density}
\end{figure}

% include plot4: death rate per infected
\begin{figure}[t]
\centering\includegraphics[scale=.30]{death_rate.jpeg}
\caption{Distribution of death rate due to COVID-19 across counties of California.}
\label{fig:deathrate}
\end{figure}

% talk about death rate per county explained by population:
Figure ~\ref{fig:deathrate} shows that the death rate, i.e. the proportion of deaths per confirmed coronavirus cases are far from being the same. For instance, Shasta, Sutter, and Kings counties have a greater mortality rate than in Los Angeles county. Even if we see this discrepancy, we can argue and treat the death rate to be the same across counties. The answer is obvious: since we don't have enough observations of confirmed cases in small populated counties like Sutter ($\leq 100,000$) empirical estimate of proportion may give misleading information about the true death rate. For instance, assume there is a county that has only 10 confirmed cases of COVID-19, by chance 2 people have serious underlying illnesses and those two patients die. Then the empirical estimate of proportion is 0.2! Which is greater than the sample death rate in any other county.

Our preliminary findings suggest that the distribution of total cases of coronavirus differ by counties. Although we assume that the death rate is the same across counties, we test it by employing different models. In the next section, we describe models in detail.

\section{Methods}

% state the variables by detail
As stated before, number of death $y_i$ out of $n_i$ infected people per county $i$ are independent observations from Binomial distribution conditioning on $\theta_i = P(death)$. Also, population of each county $c_i$ is considered as an explanatory variable for modeling distribution of number of infections $n_i$.

\subsection{Hierarchical Poisson model with Gamma priors}
First, we consider a model:
$$n_i \sim Poi(\lambda_i c_i/10^3)$$
$$\lambda_i \sim Ga(\alpha, \beta)$$
$$p(\alpha, \beta) = Ga(\alpha |a_{\alpha}, b_{\alpha})Ga(\beta | a_{\beta}, b_{\beta})$$

where prior and hyperprior distributions have $Ga(shape, scale)$ structure and all parameters are greater than zero.

We set the values of hyperparameters of the hyperprior distribution $p(\alpha, \beta)$ according to the assumption that, for a given county, the mean number of cases is equal to 20\% of the population. To include this assumption we do the following procedure:

\begin{enumerate}
\item For county $i$, $$E[n_i | \lambda_i] = \lambda_i c_i/10^3 = 0.2c_i$$ $$\lambda_i = 200$$
\item Assuming that distribution of $\lambda$ should be concentrated around 200, we set $$E[\lambda_i|\alpha, \beta] = \alpha\beta = 200$$
\item Then we let $$E[\alpha|a_{\alpha}, b_{\alpha}] = a_{\alpha}b_{\alpha} = 20 $$ $$E[\beta|a_{\beta}, b_{\beta}] = a_{\beta}b_{\beta} =10 $$
\item Consequently, we set $a_{\alpha} = 10, b_{\alpha} = 2, a_{\beta} = 20, b_{\beta} = 0.5$.
\end{enumerate}

We write the posterior distribution as $$p(\mathbf{\lambda}, \alpha, \beta| \mathbf{n}) = p(\mathbf{\lambda}|\alpha, \beta, \mathbf{n})p(\alpha, \beta| \mathbf{n})$$

We can also write $p(\mathbf{\lambda, \alpha, \beta}| \mathbf{n})$:
\begin{align*}
& \propto p(\alpha, \beta)p(\mathbf{\lambda}|\alpha, \beta)p(\mathbf{n}|\mathbf{\lambda}) \\
&\propto p(\alpha, \beta)\prod_{i=1}^{58}Ga({\lambda_i|\alpha, \beta})Poi(n_i|\frac{\lambda_i c_i}{10^3})
\end{align*}

Using this joint posterior we get the full conditional distribution of $\lambda: p(\lambda|-)$
$$p(\mathbf{\lambda}|\alpha, \beta, \mathbf{n}) = \prod_{i=1}^{58}p(\lambda_i|\alpha, \beta, n_i)$$
$$p(\lambda_i|\alpha, \beta, n_i) = Ga(\alpha + n_i, \big{(}\frac{1}{\beta} + \frac{c_i}{10^3}\big{)}^{-1})$$

Now we have to find $p(\alpha, \beta| \mathbf{n})$:
$$p(\alpha, \beta| \mathbf{n}) \propto p(\alpha, \beta)p(\mathbf{n}|\alpha, \beta)$$

where $p(\mathbf{n}|\alpha, \beta) = \prod_{i=1}^{58}p(n_i|\alpha, \beta)$ and

\begin{align*}
p(n_i|\alpha, \beta) & = \int p(n_i|\lambda_i)p(\lambda_i|\alpha, \beta) d\lambda_i \\
& = \int Poi(n_i|\frac{\lambda_ic_i }{10^3})Ga(\lambda_i | \alpha, \beta) d\lambda_i \\
& = \frac{\tilde{c}_i^{n_i}\Gamma(\alpha+n_i)}{n_i!\Gamma(\alpha)\beta^{\alpha}} (\frac{1}{\frac{1}{\beta} + \tilde{c}_i})^{(\alpha+n_i)}
\end{align*}

where $\tilde{c_i} = c_i/10^3$.

% Write the sampling procedure here:
We use this factorization to sample from the joint posterior distribution. First, we use \texttt{sample()} function in R and Griddy sampling approach to sample from $p(\alpha, \beta | \mathbf{n})$ and using those samples, we directly draw $\lambda_i$ from $Ga(\lambda_i|-)$ for each $i = 1, ..., 58$. Further, we get point and interval estimates of posterior mean and posterior variance using that sample for each parameter.

% How will you assess the model? Write model checking compared with results from previous study
We also obtain samples from the predictive posterior distribution of $\mathbf{n}$ in order to compare the predictive distribution of the number of deaths per 1000 habitants for the different counties, in combination with samples of $\mathbf{\theta} = Prob(death)$ from the previous study.

% How to obtain samples of the predictive distribution using those two above?
We can obtain posterior predictive simulations of $\tilde{n}$ from 58 counties:

\begin{itemize}
\item draw $(\alpha, \beta)$ from $p(\alpha, \beta | \mathbf{n})$
\item draw 58 parameters from $p(\tilde{\lambda}_i | \mu, \tau)$
\item for each $c_i$ in $i = 1, ..., 58$, draw $\tilde{n}_i$ from $\tilde{n}_i \sim p(\tilde{n}_i | \lambda_ic_i /10^3)$
\end{itemize}

\subsubsection{Predictive distribution of the number of deaths per 1000 habitants for different counties}

Consider the Beta-Binomial hierarchical model from the previous study:

$$y_i \sim Bin(n_i, \theta_i), \theta_i \sim Be(\mu\tau, (1-\mu)\tau)$$ $$p(\mu, \tau) = (\mu(1-\mu)(1+\tau)^2)^{-1}$$

We can write the posterior distribution of $\theta, \mu, \tau$ as follows: $$p(\theta, \mu, \tau | \mathbf{y}) = p(\theta|\mu, \tau,  \mathbf{y})p(\mu, \tau | \mathbf{y})$$

We draw samples from the posterior distribution using the factorization above. I.e. first we draw samples of $\mu, \tau$ from $p(\mu, \tau |  \mathbf{y})$, then using these samples we draw samples of $\theta$ from $p(\theta | \mu, \tau, \mathbf{y})$.

Full details of derivation of posterior distribution for parameters $\theta, \mu, \tau$ are given in Kuttubekova's (2020) report. We sample $\mu, \tau$ using \textit{rejection sampling} and $\theta_i$ using \textit{direct sampling}.

We estimate mean posterior death rate for each county $E[\theta_i |y_i]$ using samples from joint posterior, and use those estimates to obtain samples from posterior predictive distribution of the number of deaths. For each $c_i$ and $\tilde{n}_i = (\tilde{n}_{1,i}, \tilde{n}_{2,i}, ..., \tilde{n}_{n,i})$ s.t. $i \in {1, ..., 58}$ and $n$ is the sample size of $\tilde{n}_i$, we have $\tilde{y}_i = \tilde{n}_i \hat{E}[\theta_i |y_i] $.

\subsection{Hierarchical Poisson model with Gamma priors (Modified)}
Now we consider a modification of the previous model where infection rate depends on the mortality rate:

% whole model
\begin{align*}
f(y_i, n_i|\theta_i, \lambda) & = f(y_i|n_i, \theta_i, \lambda)f(n_i|\theta_i, \lambda) \\
& = Bin(y_i|n_i, \theta_i)Poi(n_i|\theta_i, \frac{\lambda c_i}{10^3})
\end{align*}

for $\theta_i \in (0,1)$ and $\lambda>0$.  We also assume that $\theta_i \sim Be(\theta_i| \mu\tau, (1-\mu)\tau)$ and we set priors:

$$p(\mu, \tau) = (\mu(1-\mu)(1+\tau)^2)^{-1}$$

where $\mu \in (0,1)$, $\tau>0$ and

$$\lambda \sim Ga(\lambda|a, b)$$

where $a$ and $b$ are fixed. We set values of $a = 20$ and $b = 10$ as in the previous case, according to the same assumption that the mean number of cases is equal to 20\% of the population. It's might not the best values for hyperparameters, however, we will check if the model is sensitive to our choice of $a$ and $b$ later in the report.

% How to obtain full conditionals of lambda and thetas
We can write joint posterior distribution of $\theta, \lambda, \mu$ and $\tau$ as follows:

\begin{align*}
p(\theta, \lambda, \mu, \tau |\mathbf{y}, \mathbf{n} ) & \propto  p(\theta, \lambda, \mu, \tau) \\
& \times p(\mathbf{y}, \mathbf{n} |\theta, \lambda , \mu, \tau) \\
& = p(\lambda, \mu, \tau) p(\theta | \lambda, \mu, \tau) \\
& \times p(\mathbf{n} | \theta, \lambda, \mu, \tau) \\
& \times p(\mathbf{y}| \mathbf{n}, \theta, \lambda, \mu, \tau) \\
& = p(\mu, \tau) p(\lambda) p(\theta|\mu, \tau) \\
& \times p(\mathbf{n}| \theta, \lambda) p(\mathbf{y} | \mathbf{n}, \theta, \lambda)
\end{align*}

Using this explicitly written joint posterior distribution, we can find full conditional distributions of each parameter up to a constant. Note that we are interested in posterior samples of parameters $\theta$ and $\lambda$. However, we use $\mu, \tau$ as latent variables, so it's easier to sample from the joint posterior.

% Full conditionals here
$$p(\lambda | -) = Ga(\lambda |a + \sum_{i=1}^{58}n_i, (\frac{1}{b} + \sum_{i=1}^{58}\frac{\theta_i c_i}{10^3})^{-1})$$

And for $i \in \{1, ..., 58\}$:
\begin{align*}
p(\theta_i| - ) & \propto exp(-\theta_i\frac{ c_i}{10^3}) \theta_i^{\mu\tau + n_i + y_i - 1}\\
& \times (1-\theta_i)^{(1-\mu)\tau + n_i - y_i - 1}
\end{align*}

Finally,

$$p(\mu, \tau | -) \propto p(\mu, \tau)\prod_{i=1}^{58}Be(\theta_i|\mu, \tau)$$

Note that the last full conditional is also proportional to

$$p(\mu, \tau)\prod_{i=1}^{58}Be(\theta_i|\mu, \tau)Bin(y_i|n_i, \theta_i)$$ which is proportional to the $p(\mu, \tau | \mathbf{y})$ from Beta-Binomial model if we integrate out $\theta_i$'s. For simplicity, we will use the latter kernel to obtain samples from posterior joint distribution.

% Explain MCMC structure
Since we were able to find conditional distributions of each parameter, we employ \textit{Gibbs sampling} to sample from posterior. Full conditionals are not given in the closed form, except the distribution for $\lambda$. So we employ \textit{Metropolis-Hastings} algorithm within the Gibbs algorithm to sample $\theta_i$'s. We set $N(\theta_i^{s}, 0.01)$ as a proposed distribution for each $p(\theta_i |-)$ inside Metropolis Hasting algorithm. Also, we use \textit{rejection sampling} within Gibbs algorithm to sample $\mu, \tau$. Note that the whole process of setting proposal distribution for rejection sampling is already given in Kuttubekova (2020).

% What will you do with results
After we obtain samples, we report point and interval estimates of the posterior mean and posterior variance for each parameter. We also compare the distributions of mortality rate i.e $\theta_i$ for each county.

% How to check model using predictive distribution of the number of cases
We also obtain samples of $\tilde{n}_i, \tilde{y}_i$ from posterior predictive distribution, and compare its distribution across counties. Note that we use the same samples to do posterior predictive checks in order to assess model fit.

\subsection{Model assessment}
% posterior predictive checks
Posterior predictive checks can be done in many different ways, but mainly all those methods try to evaluate if the phenomenon seen in observed data is also captured and reproduced by the model.

% test quantities
From EDA there are some unusual findings in observed data. For instance, San Francisco county with 884,363 population has almost the same number of infected people as those with a substantially higher population. Orange and San Diego counties have almost the same population densities, but San Diego has 1.5 times more infected people than Orange county. We want to use this phenomenon in order to assess model fit. We define test quantity

$$T(n_i, \lambda_i) = \frac{n_i}{\sum n_i}$$

= proportion of number of infected for county $i$ over the total number of infected people in California.

% DIC, Gelfand and Ghost (tell why AIC, BIC doesn't work here)
We also employ classical DIC and Gelfand \& Ghosh (1998) criteria to compare model fits. Note that the model is favorable if it has a smaller DIC and Gelfand \& Ghosh criterion. BIC and AIC cannot be used here, since they don't account for hierarchical models and they directly evaluate log-likelihood at MLE estimates of parameters, rather than at MAP estimates.

% How to perform sensitivity analysis
We are also interested in the effect of the hyperparameter values $a$ and $b$ on the overall model fit. To do sensitivity analysis, we fit the model over a grid of values for hyperparameters. Assessing criteria would be the posterior predictive distribution of the number of infected people in Orange county. (Orange county is chosen at random from the list of counties with some phenomenon).

\section{Analysis}

\subsection{Hierarchical Poisson model with Gamma priors}

% posterior of alpha, beta with their priors: PLOT
\begin{figure}[t]
\centering\includegraphics[scale=.30]{prior_post.jpeg}
\caption{Prior and posterior distribution of : $\alpha$ and $\beta$ parameters}
\label{fig:priorpost}
\end{figure}

% how many samples you draw; talk about the posterior distributions; give tables; posterior estimates and quantiles
We obtain 50,000 sample draws from the joint posterior distribution of $\lambda, \alpha$, and $\beta$. As we see in Figure ~\ref{fig:priorpost}, the posterior distribution of $\alpha$ is updated but still on the same range as its prior. However, the posterior distribution of $\beta$ is more concentrated around the left quarter tail of its prior distribution. Note that the model was given two hyperprior distributions for $\alpha$ and $\beta$ s.t. they account for the assumption that on average 20\% of the population in each county contracts coronavirus. We see that this assumption and the Poisson likelihood added their contribution in generating joint posterior and updating prior.

% table with estimates
\begin{table}
\caption{Posterior mean and variance estimated by Model 1.}
\label{tab:fmodelest}
\begin{center}
\begin{tabular}{ccccc}
\hline
\hline
\\[-5pt]
\multicolumn{1}{c}{Parameter} &
\multicolumn{1}{c}{2.5\%} &
\multicolumn{1}{c}{50\%} &
\multicolumn{1}{c}{97.5\%}\\
\hline
$\alpha$&     7.296&	16.252&  29.445\\
$\beta$&     6.272&    7.410&  7.950\\
\hline
\end{tabular}
\end{center}
\end{table}

% point estimates and the shrinkage
Point and interval estimates of $\alpha$ and $\beta$ are given in Table ~\ref{tab:fmodelest}. The same shrinkage of posterior distribution of $\beta$ can be seen from 95\% credible interval.

% box plots of all lambdas
\begin{figure}[t]
\centering\includegraphics[scale=.31]{lambdas.jpeg}
\caption{Posterior distribution of $\lambda_i$'s depicted by box plots.}
\label{fig:lambdas}
\end{figure}

% for posterior estimates use Gamma's expectation formula !!!
Now we analyze a sample from posterior distributions of each $\lambda_i$ for 58 counties. Figure ~\ref{fig:lambdas} shows the posterior distributions of each $\lambda_i$ for $i = 1, ..., 58$ depicted by box plots. Distribution for Los Angeles is one of the highest and it's also shrunk, comparative to the distribution for San Francisco county, which is of the same height but has more uncertainty. The sampling distribution of $n_i$ is $Poi(\lambda_i c_i / 10^3)$, so that $\lambda_i$ serves as coefficient parameter for each a scaled covariate $c_i/10^3$. For instance, Los Angeles county has population $c_i = 10,039,108$ and hence estimated $E[n_i|\lambda_i]$ is 9,546,425. It means assumption, that 20\% of the population will be infected, combined with observed data estimates infection of 95\% of the Los Angeles population. We also estimate the same rate $\sim 92\%$ for San Francisco county. These estimates are suspiciously high, it might be a lack of fit of data to a model, or we'll to see such high numbers of infection in those counties in the future.

% Are there any substantive differences between counties?
Overall, there is a noticeable difference in posterior distributions of $\lambda_i$'s across counties. Top five counties with the highest quartile 1 in the distribution of $\lambda_i$'s are Los Angeles, San Francisco, San Mateo, Santa Clara, and Riverside. Note that Orange and San Diego counties were among the top five counties with the highest number of infected people per county according to observed data.

% plot with checks
\begin{figure}[t]
\centering\includegraphics[scale=.30]{quantities.jpeg}
\caption{Distribution of test quantity on replicated data for different counties.}
\label{fig:quantities}
\end{figure}

% Model checks - draw samples from posterior predictive distribution
We draw 1,000 samples from the posterior predictive distribution of $n_i$ for each county. We included the samples of $\theta_i$'s from the previous study's hierarchical model to obtain samples of predictive distribution of the number of deaths. If the model fits the data well, it should be able to replicate data. We noticed in EDA that Los Angeles county has the largest proportion of infections across counties in California. We use this quantity as $T(\mathbf{n}, \lambda)$ and calculate the probabilities of observing more extreme values of it from replicated data for different counties. Counties in Figure \ref{fig:quantities} mostly have some discrepancies from the general trend which is confirmed by posterior probabilities in Table \ref{tab:postprobs}.

\subsubsection{Predictive distribution of the number of deaths per 1000 habitants for different counties}

% include plots: Boxplot of deaths
\begin{figure}[t]
\centering\includegraphics[scale=.30]{deaths1.jpeg}
\caption{Posterior predictive distribution of the number of deaths per 1000 habitants for 58 counties. }
\label{fig:deaths1}
\end{figure}

% intro
We obtain 50,000 samples of $\tilde{n}_i$ from $p(\tilde{n}_i | n)$ for each county. We also retrieved sample results from previous study: 250 samples of $\theta_i$ from $p(\theta, \mu, \tau | y)$. We combine those sampled to obtain 50,000 samples of the number of deaths per 1000 habitants for each county. Figure ~\ref{fig:deaths1} shows that the distribution of the number of deaths across counties is not homogenous. There are 4 counties with the median number of deaths per 1000 habitants $\geq 20$ and Los Angeles county with the median $\geq 30$. I.e, more than 30 people per 1000 die in Los Angeles due to a novel virus.

Table ~\ref{tab:deaths} shows that more than the number of deaths changed tremendously, as it was impacted by the assumption we gave in prior distribution.

\subsection{Hierarchical Poisson model with Gamma priors (Modified)}
We obtain 50,000 samples from the joint posterior distribution of $\theta, \lambda, \mu$ and $\tau$. Although we are interested in samples of $\theta$ and $\lambda$, we used $\mu, \tau$ as latent variables for an easier sampling procedure from the joint posterior distribution.

% lambda's plot
\begin{figure}[t]
\centering\includegraphics[scale=.30]{lambdass.jpeg}
\caption{Posterior distribution of $\lambda$}
\label{fig:lambdass}
\end{figure}

% report lambdas
Figure ~\ref{fig:lambdass} shows the prior and posterior distributions of $\lambda$ in model 2. The posterior distribution is highly skewed to the right, while prior is more concentrated. The posterior mean of $\lambda$ can be easily calculated since we know its full conditional distribution in a closed form. Hence, $E[\lambda |\mathbf{y, n}] = 154.75$ and 95\% credible interval is $[19.46, 492.97]$. Note that credible is quite wide, which is probably because of a heavy right tail.

% plot with mortality rates
\begin{figure}[t]
\centering\includegraphics[scale=.30]{postdeaths.jpeg}
\caption{Posterior distribution of mortality rate}
\label{fig:postdeaths}
\end{figure}

% compare mortality rates for each county as in previous study
According to the Figure \ref{fig:postdeaths} the distribution of mortality rate has been homogenized comparative to its distribution in Figure \ref{fig:deathrate}. We set non-informative $Beta(0,0)$ prior for $\mu$, where $\theta_i \sim Be(\theta_i | \mu, \tau)$. So posterior of $\theta_i$ is dominated by the Beta likelihood with combination with data likelihood, which also involve parameter $\theta_i$.

% talk about overall infections and death number -- DELETE if needed
%Remember, we mentioned that $~ 0.6\%$ of people California contracted the virus up to date, and in total, 725 people have died. The overall posterior mean of death rate = $0.0302$, which is multiplied by the total number of people infected in state equals to 733, which is close enough to the true value.

% distribution of number of deaths per 1000 habitants
\begin{figure}[t]
\centering\includegraphics[scale=.30]{deaths2.jpeg}
\caption{Posterior predictive distribution of number of deaths per 1000 habitants.}
\label{fig:postdeathss}
\end{figure}

% compare predictive distribution of the number of deaths per 1000 habitant
We obtain 50,000 samples of $\tilde{n}_i$ and $\tilde{y}_i$ for each county. Using those estimates, we obtain the posterior predictive distribution of the number of deaths. Figure \ref{fig:postdeathss} shows that the distribution of the number of deaths per 1000 habitants is far from being homogenous. We see that almost all distributions are highly skewed with the right heavy tails. We refer to Table \ref{tab:deaths} and see that surprisingly Los Angeles has fewer deaths per 1000 than other counties..

% infection per 1000
\begin{figure}[t]
\centering\includegraphics[scale=.30]{infection2.jpeg}
\caption{Posterior predictive distribution of number of infections per 1000 habitants.}
\label{fig:infections}
\end{figure}

% compare predictive distribution of the number of cases per 1000 habitant
We have seen in EDA that the distribution of the number of infections is heterogenous, i.e each county has significantly different (small or high) numbers. We would like to know if this trend is seen in predictions made by model 2. Figure \ref{fig:infections} shows that the distribution is uniform. However, some counties like Imperial, Solano, San Luis Obispo, which had a smaller number of infections tend to have slightly more than the average.

% table with sensitivity results
\begin{table}
\caption{Posterior probability of observing extreme T(n) for Orange county. Calculated by model 2}
\label{tab:orange}
\begin{center}
\begin{tabular}{ccccc}
\hline
\hline
\\[-5pt]
\multicolumn{1}{c}{a} &
\multicolumn{1}{c}{b} &
\multicolumn{1}{c}{T(n)}\\
\hline
20&	10&   0.1342\\
20&     5&  0.1265\\
5&	5& 0.4226\\
0.5&     0.5&  0.4927 \\
\hline
\end{tabular}
\end{center}
\end{table}

% Sensitivity analysis -- MODIFY
As you see from the table our quantity of interest is affected by a change in the values of $a$ and $b$. As their values decrease we see better fit (in terms fo Orange county only). So we have to be very careful in the choice of hyperparameter values. We can evaluate sensitivity w.r.t. other test quantities or aspects of the data.

\subsection{Comparing two models}
We fit five models in total to the COVID-19 data. The last three models are hierarchical, and the last two treat the total number of infected people $n_i$ in each county as a random variable rather than fixed. The last model also takes into account potential overdispersion in the number of deaths.

% report posterior distribution of deaths per 1000 habitants
\begin{table}
\caption{Distributions of the number of deaths per 1000 habitants for each county. Estimated by model 1 \& 2}
\label{tab:deaths}
\begin{center}
\begin{tabular}{ccccc}
\hline
\hline
\\[-5pt]
\multicolumn{1}{c}{County} &
\multicolumn{1}{c}{$E_{1}[\tilde{y}|y]$} &
\multicolumn{1}{c}{$E_{2}[\tilde{y}|y]$} &
\multicolumn{1}{c}{y}\\
\hline
Los Angeles&	31.35&   0.610 & 0.032\\
Santa Clara&     28.39&  0.402 &	 0.031\\
San Mateo&	23.64& 1.309	& 0.027\\
Riverside&     19.95&  1.270 & 0.021\\
Marin&     19.20& 0.824 & 0.038\\
\hline
\end{tabular}
\end{center}
\end{table}

% comment on the table above
We see in Table \ref{tab:deaths} the predictive distribution by model 1 was influenced by the assumption that the mean number of cases is equal to 20\% of the population. Model 2 has also overestimated the mean number of deaths comparative to a summary from observed data. On average, it's expected that the number of deaths per 100 habitants is between 5 and 14 (for counties in the table) by model 2, while the number of deaths per 1000 habitants is between 20 and 31 by model 1.

% table with probs from PPC
\begin{table}
\caption{Posterior probability of observing extreme T(n) calculated by model 1 \& 2}
\label{tab:postprobs}
\begin{center}
\begin{tabular}{ccccc}
\hline
\hline
\\[-5pt]
\multicolumn{1}{c}{County} &
\multicolumn{1}{c}{Model1} &
\multicolumn{1}{c}{Model 2}\\
\hline
 Los Angeles&	0.0000 &	0.0883\\
San Diego & 0.1368 &	0.3887\\
Orange & 0.1648 & 0.1342 \\
San Francisco & 0.0130  & 0.2225\\
San Mateo & 0.0107 & 0.1146\\
Santa Barbara & 0.0028 & 0.2313\\
Marin & 0.0001 & 0.0517\\
Yolo & 0.0009 & 0.1373\\
Santa Cruz & 0.0299 & 0.0009\\
\hline
\end{tabular}
\end{center}
\end{table}

% report predictive distribution checks
We draw samples from posterior predictive distribution of the number of infected people in each county to see if models were able to replicate the observed data. Initially, we observed that Los Angeles county had the highest number of infected people across all 58 counties, and we want to know if the same trend is the case for both models. Posterior probabilities of observing as extreme values as a test quantity are given in Table \ref{tab:postprobs} for both models. Note that we also test other counties, which have unusual behavior in the distribution of the number of infected people. Posterior probabilities calculated by model 2 are slightly better than in model 1. However, there is no major improvement except for counties like San Diego, San Francisco, and Santa Barbara.

% report comparisons table: DIC, Gelfand-Ghost
\begin{table}
\label{table:anan}
\caption{Model fit assessment}
\begin{center}
\begin{tabular}{ccccc}
\hline
\hline
\\[-5pt]
\multicolumn{1}{c}{Model} &
\multicolumn{1}{c}{DIC} &
\multicolumn{1}{c}{Gelfand-Ghost}\\
\hline
 Model 1&	294.13&	1.005e+14\\
Model 2&	-9182.21&	1.321e+10\\
\hline
\end{tabular}
\end{center}
\end{table}

% discuss table above
We employ two different methods to compare the models, and probably find a better fit. We also used two criteria to choose between models. Gelfand \& Ghosh criteria compare the predictive abilities of two models. Although the criteria values are very high for both models, the second modified model has substantively smaller criteria value. Also, we employed DIC, which is a generalization of AIC for hierarchical models. It's calculated by estimating the effective number of parameters in the model. We know that negative DIC values are legit and small values of DIC are preferred. Hence, by both criteria model 2 seems to be a better fit for the COVID-19 data than model 1.

% overall computational performance
There was a tremendous difference in the computation time of two models for obtaining samples from posterior. I.e model 2 required 3 times more time than model 1 for the same number of samples. Since we employed a combination of Metropolis-Hastings algorithm and rejection algorithms for model 2, it required more time to run 5 chains and obtain 50,000 samples from posterior. That time requirement triples since we obtain samples of $\tilde{n}_i$ and then samples of $\tilde{y}_i$ for each county.

% model complexity
Although the second model is more complex, it was easier to obtain full conditionals and construct a working sampling algorithm. We could use Gibbs sampler for the first model as well if we treated $\lambda_i$'s as a latent variable. Our approach was to use the Griddy approach to sample from $p(\alpha, \beta | mathbf{n})$ which created a lot of computational issues so we manually simplified the kernel of $p(\alpha, \beta | mathbf{n})$ and wrote many additional functions to evaluate density only.

\section{Discussion}

% intro
We employed two different constructions of the Poisson-Binomial models. We treated the number of infected people as a random variable along with the number of deaths per county. Our preliminary results showed that the infection rate and death rate across counties are not homogenous. It was also confirmed throughout by fitting the two models above.

% Talk about future: time when will we reach that 20%, how to slow the spread and find the patterns in different counties
We assumed that $20\%$ of people in California will contract the virus. Actually we used it as a piece of prior information in the first model and used it indirectly in the second model as well. We ignored the time feature of the spread of the virus, i.e we are not interested in estimating when will those $20\%$ be infected. So it can be the next step to include time in our model.

% Time-space varying parameter
As a next step, we can consider a model that accounts for the time and space varying nature of the spread of the virus. I.e we might want to employ Gaussian or Poisson processes that are suitable for time series as well as spatial data.

%Note:BibTeX also works

\begin{references}
\itemsep=0pt
{\footnotesize

\item
Aljazeera, News
\\\url{https://www.aljazeera.com/news/}

\item
Hines B 2019, Population Density of California
\\\url{https://storymaps.arcgis.com/stories/252b65cbe331420389c61453c61ea221}

\item
Kuttubekova G 2020, Bayesian Analysis of COVID-19 cases in California
\\\url{https://github.com/kgulzina/bayesian-analysis-of-COVID19-in-CA}

\item
Worldometer: Coronavirus,
\\\url{https://www.worldometers.info/coronavirus/}

% add BDA book
\item
Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., \& Rubin, D. B. (2013), $\textit{Bayesian Data Analysis} (3^{rd}$ ed.). CRC Press. Taylor \& Francis Group.

}
\end{references}

\end{document}


\section{Introduction}
\paragraph{}
This chapter starts with a mesh generated by the help of the octree based algorithm using STL file directly \citep{Liu2017}.
In this algorithm, the intersection is calculated between the edge of the element and the triangular surface which is an approximation of the exact geometry.
In order to achieve the geometric precision, a point projection method for 3D NURBS surface is presented.
It's computational efficiency is significantly improved by implementing a NURBS surface splitting and by utilizing the strong convex hull property of the NURBS.
The quick hull algorithm is also introduced to construct the convex hull from the control points in 3D.
Alternative method to retain the exact geometry is targeted as well.
The calculation of the intersection in \cite{Liu2017} is replaced by finding that between the edge of the element and the NURBS surface directly.

\paragraph{}
The advantage of the proposed method is that the exact geometry can be retained and hence improves the accuracy of the result.

\paragraph{}
This chapter will be organized as followed:
points projection of the NURBS surface is introduced at the beginning, together with the surface splitting and the convex hull construction.
After that, the calculation of the straight line with the NURBS surface is developed.
Furthermore, a brief introduction on SBFEM formulation in 3D elasticity is presented.
The accuracy and the convergence properties of the proposed method are demonstrated with benchmark problems in the context of linear elasticity.
Some other mesh examples from complex geometric input are also plotted at the end of this chapter.


\chapter{Privacy Preserving Algorithms}\label{c:pp-algorithms}

\section{Challenges in Privacy Preserving Algorithms}\label{s:challenges}
A private computation scheme allows the user to compute any arbitrary function, while revealing no information to any individual server about the identity of that function.
Computation over encrypted data can be utilized in a wide variety of use-cases.
For instance, the problem of how to use encrypted database fields in search queries, or the problem of testing for membership in a set.
Another very common use case is the private set intersection, \textit{i.e.} \textit{``Department of homeland security (DHS) wants to check its list of terrorist suspects against the passenger manifest of a flight operated by a foreign airline.
Neither party is willing to reveal its information, however, if there is a (non-empty) intersection, DHS will not give the flight permission to land".}

Despite the variety of applications that encrypted computation can assist, the translation of any function to its privacy preserving equivalent is not a trivial process.
As mentioned in section \ref{s:homomorphic-encryption}, termination problems are introduced and the avoidance of them is not straightforward.
First, one has to identify the corresponding termination channels that needs protection against side-channels and leakage; that is, any branch decision which is based on encrypted data (\textit{i.e.} \texttt{if ($Enc(X)$ is True) then statement}).
Consecutively, they have to translate the termination channels using oblivious computation in the encrypted domain.
Commonly, the \texttt{if} statement is replaced by a loop over all possible values and an oblivious selection of the desired value.
Early termination conditions, in general, eventually leak sensitive information, which is why the execution time should only depend on the length of the inputs, not their plaintext values (\textit{i.e.}, execution requires maximum iterations independent of the input).
Below we examine the privacy preserving equivalents of some simple and widely used algorithms in order to clarify the termination problems and the translation process.

\section{Branching Oracles}\label{s:bros}
Although we cannot take branch decisions that are based on encrypted values, recent research \cite{mazonka2016cryptoleq} has implemented special constructions -- dubbed \emph{BRanching} \emph{Oracles} (\emph{BROs}) -- which obliviously evaluate the branch outcomes.
\emph{BROs} are treated as decision-making black boxes and enable an alternative solution to branching on private information.
All the algorithms presented in the following sections suppose that the underlying encrypted architecture supports a privacy\hyp preserving \emph{BRO}.
More specifically, \emph{BROs} enable comparison of encrypted values and obliviously return an encrypted boolean variable (true, false).

For instance, the following example illustrates the usage of \emph{BROs}:

\texttt{if (x > 1) a = b; else a = c; }

Since \texttt{x} is encrypted, the host is unable to make the branch decision whether \texttt{a} should take the value of \texttt{b} or \texttt{c}.
Using a branching oracle, an encryption of true or false will emerge from the comparison \texttt{(x > 1)} (\textit{i.e.} \texttt{gt = (x > 1)}).
Then, one can compute obliviously the branch outcome using the result of the private comparison as follows:

\texttt{a = b * gt + c * (1-gt)}.

Using this technique a programmer can replace conditional statements on private data with such oblivious selection clauses.
Therefore, the control flow dependencies are converted into data dependencies.

\section{Notation}\label{s:notation}
Our notation for encrypted values and for operations between encrypted variables consists of a variation of the classical mathematical operations.
Namely, $\enc{X}$ corresponds to the encryption of number $X$, while $\hplus$ , $\hminus$ , $\hmul$ and $\hdiv$ represent homomorphic addition, subtraction, multiplication and division respectively, as shown in equation \ref{eq:operands}.
In a like manner, $\heq$ (eq. \ref{eq:equal}), $\hneq$ (eq. \ref{eq:not-equal}) $\hlt$ (eq. \ref{eq:lt}) and $\hgt$ (eq. \ref{eq:gt}) represent private equality and private comparison, returning an encryption of one (that is $\enc{1}$) if the operands map to equal plaintexts (or respectively the first is less than the second), or an encryption of zero ($\enc{0}$) otherwise.
All those aforementioned comparisons are utilizing the branching oracle of the underlying encrypted architecture.

\begin{equation}\label{eq:operands}
  \begin{aligned}
      Enc(X) \hplus Enc(Y) = Enc(X + Y)\\
      Enc(X) \hminus Enc(Y) = Enc(X - Y)\\
      Enc(X) \hmul Enc(Y) = Enc(X \cdot Y)\\
      Enc(X) \hdiv Enc(Y) = Enc(X \div Y)
  \end{aligned}
\end{equation}

\begin{equation}\label{eq:equal}
  Enc(X) \heq Enc(Y) :\begin{cases}
    \enc{1}, & \text{if $X = Y$}.\\
    \enc{0}, & \text{otherwise}.
  \end{cases}
\end{equation}

\begin{equation}\label{eq:not-equal}
  Enc(X) \hneq Enc(Y) :\begin{cases}
    \enc{1}, & \text{if $X \neq Y$}.\\
    \enc{0}, & \text{otherwise}.
  \end{cases}
\end{equation}

\begin{equation}\label{eq:lt}
  Enc(X) \hlt Enc(Y) :\begin{cases}
    \enc{1}, & \text{if $X < Y$}.\\
    \enc{0}, & \text{otherwise}.
  \end{cases}
\end{equation}

\begin{equation}\label{eq:gt}
  Enc(X) \hgt Enc(Y) :\begin{cases}
    \enc{1}, & \text{if $X > Y$}.\\
    \enc{0}, & \text{otherwise}.
  \end{cases}
\end{equation}

Also, in the algorithms described below, the variables in \red{red} correspond to private values, while the variables in black represent public data.

\section{Transforming Algorithms to their Privacy Preserving Equivalent}\label{s:simple-algorithms}
In this section we will examine three simple algorithms (namely {\fontfamily{lmss}\textsc{Sum}, \textsc{Max}, \textsc{PIR}}) and how to develop them in a way to preserve data privacy.
Firstly, algorithm \ref{a:sum} computes the sum of an array of numbers and it is quite similar to the private sum algorithm.
The latter obliviously updates the encrypted sum with each value of the array, utilizing the homomorphic properties of the encrypted variables.

\import{./}{algorithms/sum.tex}

On the other hand, algorithm \ref{a:max} finds and returns the maximum number of an array.
In the private {\fontfamily{lmss}\textsc{Max}} algorithm, the host is not able to determine for each value of the array if it is greater from the current value of max.
However, it can obliviously update the value of max, based on the encrypted comparison outcome (alg. \ref{a:max} line 13, 14).
In the cases that the value of the $array$ is less than the current $max$ the $gt$ flag will have the value $\enc{0}$, otherwise $\enc{1}$.
Thus the multiplexer operation in line 14 each time will obliviously ``keep'' the greatest value and set the variable $max$ to it.

\import{./}{algorithms/max.tex}

Both textbook algorithms' time complexity (algorithms. \ref{a:sum} and \ref{a:max}) is \bigo{n} and the transformation to their privacy preserving equivalents did not affect their complexity.
However, the privacy\hyp preserving algorithm is inevitably more computationally intensive, since the encrypted operations require many additional CPU cycles.

\import{./}{algorithms/pir.tex}

The third and final example is the Information Retrieval algorithm, where a user maintains a database and desires to retrieve some data based on a key.
The simplest textbook algorithm searches each consecutive position on the array to find a key match, and immediately returns the corresponding value.
In contrast, the privacy preserving algorithm (PIR) \cite{chor1995private} is not able to return before searching the whole database without revealing any information about the key and the result.
In this pair of examples the worst case complexity is also \bigo{n}, however in the average case, the former will search the half database.

The idea here is to brute\hyp force all possible table positions and obliviously find and keep the value for the requested key.
It has become obvious that it is not straightforward to make more sophisticated algorithms privacy preserving.
However, similar techniques are applied for simple and more elaborate algorithms, in order to transform them to their privacy preserving equivalents, sacrificing performance for privacy.

Motivated by privacy concerns and from the challenges that arise in securing any function, in this thesis we concentrate on developing an end-to-end infrastructure for privacy preserving analytics and incorporate some essential algorithms for aggregation statistics and classification.

\section{Algorithms for Two Types of Data: Categorical \& Numerical}\label{s:two-types-of-data}
Due to the limitless applications that our system can serve, the input data can have many different types.
We have separated the data in two broad categories -- categorical and continuous, therefore our algorithms are also logically separated for those two different kinds of data.
Categorical variables are those that can take on one of a limited, and usually fixed, number of possible values.
For instance, blood type, gender, the age-group and the country that a person lives in.
On the other hand, numerical\myslash continuous data can take any real number, rendering uncountable the number of different possible values.
Examples of the latter category include weight, price, profits, etc.

Medical data, of course, appear in both ways, depending on the ``nature" of the attributes.
One more reason that this distinction should be made is that even for the same attribute (for instance ``Age"), a doctor may either classify the exact age, or the birth date, or even classify the patient in one group (\textit{i.e.} ``child" or ``adult").

Due to these two different kinds of data that exist in medical datasets, we have separated our algorithms in two categories respectively, categorical and numerical.

\input{histograms}

\input{id3}

\input{c45}


\documentclass[10pt,twocolumn,letterpaper]{article}

\usepackage{cvpr}
\usepackage{times}
\usepackage{epsfig}
\usepackage{graphicx}
\usepackage{color}
\usepackage{transparent}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\newcommand{\norm}[1]{\left\lVert #1 \right\rVert}
\DeclarePairedDelimiter\abs{\lvert}{\rvert}

% Include other packages here, before hyperref.

% If you comment hyperref and then uncomment it, you should delete
% egpaper.aux before re-running latex.  (Or just hit 'q' on the first latex
% run, let it finish, and you should be clear).
%\usepackage[pagebackref=true,breaklinks=true,letterpaper=true,colorlinks,bookmarks=false]{hyperref}

\cvprfinalcopy % *** Uncomment this line for the final submission

\def\cvprPaperID{} % *** Enter the 3DV Paper ID here
\def\httilde{\mbox{\tt\raisebox{-.5ex}{\symbol{126}}}}

% Pages are numbered in submission mode, and unnumbered in camera-ready
%\ifcvprfinal\pagestyle{empty}\fi
\setcounter{page}{1}
\begin{document}

%%%%%%%%% TITLE
\title{Structure from Motion with RGB-D}

\author{Seonwook Park \\
ETH Zurich \\
{\tt\small spark@student.ethz.ch}
% For a paper whose authors are all at the same institution,
% omit the following lines up until the closing ``}''.
% Additional authors and addresses can be added with ``\and'',
% just like the second author.
% To save space, use either the email address or home page, not both
\and
Yifan Wang \\
ETH Zurich \\
{\tt\small yifan.wang@student.ethz.ch}
}

\maketitle
%\thispagestyle{empty}

%%%%%%%%% ABSTRACT
\begin{abstract}
   A structure from motion pipeline is constructed using C++, OpenCV, Ceres
   solver, and PCL. This includes the acquisition of data, extraction and
   matching of features, estimation of pairwise camera pose using depth data,
   transformation of estimates into a global coordinate frame, and performing
   bundle adjustment on all data. This pipeline incorporates depth data taken
   from a Microsoft Kinect in pairwise camera pose estimation to improve the
   final 3D reconstruction, which simplifies the conventional pipeline and improves the robustness of the result. Furthermore the inclusion of depth data in the bundle adjustment was attempted and compared with the result using a standard cost function.
\end{abstract}

%%%%%%%%% BODY TEXT
\section{Introduction}

Structure from Motion (SfM) concerns the recreation of a real environment
through the inferring of motion from images \cite{varga2008practical}. (1)
First, a 3D object or environment is imaged from multiple different
perspectives. (2) In each image, it is possible to find keypoints via feature
detection algorithms such as SIFT and SURF, and calculate descriptors with which
correspondences can be found in other images. (3) After finding corresponding
pairs, the relative pose of each camera can be inferred in various ways.

With early implementations of SfM, keypoints are projected into 3D space and
optimisation algorithms such as RANSAC and Levenberg-Marquardt are used to find
the homography or perspective transformation between two image planes. With the
advent of low-cost commercial depth sensing cameras, it is now possible to
incorporate depth information in hopes to improve or speed this procedure up. An
example is the Microsoft Kinect, which casts and reads an infra-red pattern
which is used to calculate depth values per pixel imaged \cite{zhang2012microsoft}.

For problems of corresponding 2D points (image plane) and 3D points (using depth
values), one can use PnP or Perspective-n-Point algorithms \cite{d2013p3p}. A P3P
algorithm uses a minimum of 3 data points to infer the relative pose between a
pair of cameras. There are optimised PnP algorithms available, which when
coupled with RANSAC can discard outliers and estimate relative camera pose
accurately \cite{lepetit2009epnp}.

Once an initial estimate for relative camera pose is made, this can be optimised
via bundle adjustment, an optimisation step operated on sparse keypoints from
the perspective of all cameras. To do so, (4) pairwise camera pose estimates
need to be transformed into a single coordinate system. This is achieved through
the construction of a minimum spanning tree with cameras as nodes and the
existence of corresponding pairs as edges. When camera pose estimates are
relative to a single coordinate system, it is possible to project keypoints into
the coordinate system from the perspective of each camera. Minimising this
reprojection is the (5) bundle adjustment step.

A final model can be constructed by projecting all known data into 3D space
using final camera pose estimates. In such a way a dense 3D reconstruction is
possible. The alignment of RGB-D data in the final reconstruction is
representative of the SfM pipeline's performance.

This study concerns the use of depth data in hopes to improve the accuracy of
the final 3D reconstruction output. This is done both in the pairwise camera
pose estimation step as well as the bundle adjustment step where depth data
is introduced into the minimisation problems.

%-------------------------------------------------------------------------

\section{Methodology}

\begin{figure*}[ht]
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/pipeline.pdf}
\end{center}
\caption{Overview of Structure from Motion pipeline}
\label{fig:pipeline}
\end{figure*}

As outlined in the previous sections, our structure from motion pipeline is
composed of 5 main steps as shown in figure \ref{fig:pipeline}.

\begin{enumerate}
\item Data acquisition
\item Feature detection and matching
\item Pairwise camera pose estimation
\item Transform to global coordinate system
\item Bundle adjustment
\end{enumerate}

It is important to note that the acquired images in this case is RGB-D and is
acquired using a Microsoft Kinect (first generation).

The pipeline is implemented in C++. Third party libraries used include OpenCV
3.0, Ceres Solver and PCL 1.8.

%-------------------------------------------------------------------------

\subsection{Data acquisition}
\begin{figure*}
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/BeersNMore_panorama.jpg}
\end{center}
\caption{Panorama of BeersNMore interior, displaying a high number of features}
\label{fig:panorama}
\end{figure*}
OpenNI and OpenCV are used to acquire RGB images and depth maps from a Kinect.
The two output images are stored with the same timestamps. The code used for
this step is provided by our supervisor, Bernhard Zeisl.

It is worth noting that the Kinect returns depth in mm in range $[0, 10000]$
which is stored as a 16-bit unsigned integer. Camera parameters are taken from
\cite{smisek20133d} and used in methods in the following steps.

While acquiring data, we attempt to find areas with sufficient potential
features and try to maximise the overlap between each shot to retain enough
correspondences.

This resulted in the BeersNMore dataset, taken at a craft beer shop on
Universit\"atstrasse. The interior of the shop (as seen in figure \ref{fig:panorama}) exhibits
numerous unique and repeating features in the form of labelled beer bottles
boxes, and crates. The dataset consists of 229 RGB and depth images.

The final reconstruction uses pixels with depth data in the range
$\left[0.4,8\right]$m. This is the range in which depth information
is reliable, as studied by Newcombe \etal \cite{newcombe2011kinectfusion}.

%-------------------------------------------------------------------------

\subsection{Feature detection and matching}\label{sec:extractfeature}

For each given RGB image, SIFT features are found, and 128-dimensional are
descriptors calculated. Standard OpenCV parameters are used for this step.
In each image, around 1000 to 2000 features are found.

A matching algorithm is then run between all potential image pairs. This is
an $\mathcal{O}(n^2)$ operation. The matcher computes the euclidian distances of the $i$-th descriptor in one image to all descriptors in the other image and returns the descriptor with minimum distance as the match to descriptor $i$. This produces numerous
incorrect matches, often visible by the violation of epipolar geometry.

The matching is therefore performed in a bi-directional manner, both from image
$i$ to $j$, and $j$ to $i$, so that only consistent matches are kept. This results in a better sample of matches. RANSAC
can be used in the camera registration step to further eliminate outliers.

%-------------------------------------------------------------------------

\subsection{Pairwise camera pose estimation}

The previous step yields a list of image pairs which have matching features.
As in equation \ref{eqn:backprojection}, given the camera matrix $K$, the discovered features $m_{i} = (u,v)^{T}$ from image $i$ can be projected into 3D space
using its depth map values $\rho$. These 3D points are then reprojected into image $j$ (see figure \ref{fig:reg}). The 3D points $P_{i}$ and their corresponding 2D matches $m^i_{j}$ in image $j$ are fed into OpenCV's EPnP solver, which computes the camera pose $i$ w.r.t camera $j$, $T_{ji} = \left(R_{ji}, t_{ji}\right)$ by minimising reprojection error $e$, as defined in equation \ref{eqn:reprojerror} in a RANSAC manner.

\begin{align}
P &= \rho K^{-1} \begin{pmatrix}
u\\v\\1
\end{pmatrix} \label{eqn:backprojection}\\
e &= \sqrt{\norm{\pi\left(P_{i};R_{ji},t_{ji}\right)-m^i_{j}}^2},\label{eqn:reprojerror}
\end{align}
where $\pi$ denotes the projection function \[
\rho
\begin{pmatrix}
\pi\left(P;R,t\right)\\1
\end{pmatrix} = KR\left(P+t\right).
\]

\begin{figure}[t]
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/registration.pdf}
\end{center}
\caption{Pairwise camera pose registration. Features of camera $i$ are unprojected into 3D space and the reprojection errors (red) are minimised.}
\label{fig:reg}
\end{figure}

The mentioned solver also identifies outlier matches via RANSAC. Only inliers
are retained to improve any further optimisations. To ensure that only good
image pairs are retained, we also filter these image pairs based on an absolute
minimum number of inliers of 30. If two images have less then 30 feature matches
which are inliers in the registration process, it is assumed that the image pair
is not good enough for subsequent steps.

Similar to the feature matching step, the PnP solver is run in both directions,
projecting 3D points from image $i$ into image $j$, and projecting 3D points
from image $j$ into image $i$. This allows for two things, (1) validation of registration result and (2) the averaging of
pose estimates to improve accuracy. In particular, the registration of a camera pair (i,j) is considered successful, only when the rotational vector $r_{ij}$ and $r_{ji}$ from the bi-directional registration are anti-parallel (equation \ref{eqn:check1}) and have similar magnitude (equation \ref{eqn:check2}).

\begin{align}
\norm{\frac{r_{ij}}{\norm{r_{ij}}} + \frac{r_{ji}}{\norm{r_{ji}}}} < 0.2
\label{eqn:check1}\\
\abs{\norm{r_{ij}} - \norm{r_{ji}}} < 0.2\label{eqn:check2}
\end{align}
The final output from this step is a list of camera pairs which are deemed to
have good matching features, and associated pairwise camera pose estimates.

%-------------------------------------------------------------------------

\subsection{Transform to global coordinate system}

The final goal of this SfM pipeline is to combine the data from all images
acquired. To do so, previously acquired pairwise camera pose estimates must
be transformed into a single coordinate frame.

\begin{figure}[t]
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/spanning_tree.eps}
\end{center}
\caption{An example of a minimum spanning tree in terms of this pipeline}
\label{fig:spanning}
\end{figure}

The 0th camera is selected to be the reference coordinate frame. A breadth-first
algorithm is used to construct a minimum spanning tree with cameras as nodes and
the existence of camera pose estimate as edges (whether an image pair exists).

An example of such a spanning tree structure can be seen in figure
\ref{fig:spanning}.
The spanning tree can be walked to calculate camera pose estimates relative to
the 0th camera as in equation \ref{eqn:globalpose}. Having the global poses, one can obtain the 3D keypoints in global frame using equation \ref{eqn:globalpoint}, where $P^{i}_{k}$ denotes the $i$-th keypoints in $k$-th camera and $P^{i}_{g,k}$ denotes this point transformed in global frame.

\begin{align}\label{eqn:globalpose}
R_{k} &= R_{kj}R_{ji}\ldots R_{0}\nonumber\\
t_{k} &= t_{kj}+t_{ji}+\ldots t_{0}\\
P^{i}_{g,k} &= R_{k}^T P^{i}_{k} - t_{k} \label{eqn:globalpoint}
\end{align}

\begin{figure}[t]
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/clusters.eps}
\end{center}
\caption{An example of a cluster of keypoint pose estimations}
\label{fig:clusters}
\end{figure}

The outcome of this step needs to be a cloud of keypoints and camera pose
estimates in global coordinate frame. However, each keypoint is observed by a
minimum of two cameras, resulting in a cluster of keypoint coordinate estimates.
This is illustrated in figure \ref{fig:clusters}.
The centre of mass (CoM) of this cluster is calculated by averaging the
coordinate estimations as in equation \ref{eqn:CoM} \cite{reckerdepth}. This results in a single keypoint coordinate estimate,
and consequently an initial point cloud of sparse features.
\begin{equation}\label{eqn:CoM}
P^{i}_{g} = \sum_{k}{P^{i}_{g,k}}
\end{equation}

%-------------------------------------------------------------------------

\subsection{Bundle adjustment}
With global keypoint and camera pose estimates, it is now possible to perform a global optimisation known as bundle adjustment. This step takes consideration of all estimated 3D points and camera pose into consideration and minimizes a predefined cost function, that reflects the reconstruction quality. Conventionally, this the total reprojection error (as in equation \ref{eqn:baNoD} is used as cost function.

\begin{equation}\label{eqn:baNoD}
\sum_{R_k,t_k}\sum_{P_i}{\norm{\pi\left(P_i;R_k,t_k\right)-m^i_{k}}}^2,
\end{equation}

To this end, having extra depth values of all 3D points, we altered the cost function to exploit this information in hope of an improvement in the reconstruction quality. As can in figure \ref{fig:baCost} and equation \ref{eqn:baD} ,relative error in depth value is incorporated as an extra term, where $d^i_k$ and $\rho^i_k$ denote the measured and estimated depth value respectively.

\begin{equation}\label{eqn:baD}
\sum_{R_k,t_k}\sum_{P_i}{\norm{\pi\left(P_i;R_k,t_k\right)-m^i_{k}}^2+\left\lvert\dfrac{d^i_k-\rho^i_k}{d^i_k}\right\rvert ^2}
\end{equation}

To improve the robustness of this optimization, cauchy loss function $\rho(s) = \log(1+s)$ is utilised which decrease the weight of outliers in the cost function, so that they do not overly influence the final solution.

The result and comparison of both cost functions is shown in section \ref{sec:result}.

\begin{figure}[t]
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/ba.pdf}
\end{center}
\caption{Visualization of error terms in cost function of bundle adjustment}
\label{fig:baCost}
\end{figure}

%-------------------------------------------------------------------------

\section{Results and discussion}\label{sec:result}

\begin{figure*}
\centering

   \begin{subfigure}[b]{1.0\columnwidth}
   \includegraphics[width=\textwidth]{figures/result_small_noBA.png}
   \caption{Before BA}
   \label{fig:beforeBA}
   \end{subfigure}
   \hfill
   \begin{subfigure}[b]{1.0\columnwidth}
   \includegraphics[width=\textwidth]{figures/result_small_BA_noD.png}
   \caption{After BA}
   \label{fig:afterBA}
   \end{subfigure}

\caption{3D reconstruction using a 37-image subset of data. An improvement in
reconstruction quality can be seen after the bundle adjustment step.}
\label{fig:3Dsmall}
\end{figure*}

The pipeline works very well on small datasets which cover an area well with
numerous overlapping shots. The resulting dense reconstruction is of high
quality and alignment is noticably improved with bundle adjustment. This can be
seen in figures \ref{fig:beforeBA} and \ref{fig:afterBA} where a rough initial
estimate is refined through bundle adjustment.

\begin{figure*}
\centering

   \begin{subfigure}[b]{1.0\columnwidth}
   \includegraphics[width=\textwidth]{figures/result_large_BA_noD.png}
   \caption{BA without depth}
   \label{fig:BA_noD}
   \end{subfigure}
   \hfill
   \begin{subfigure}[b]{1.0\columnwidth}
   \includegraphics[width=\textwidth]{figures/result_large_BA_D.png}
   \caption{BA with depth}
   \label{fig:BA_withD}
   \end{subfigure}

\caption{3D reconstruction using the full dataset (226 images). With the depth
term added to the bundle adjustment step, the reconstruction is no longer
successful. It can also be seen the the reconstruction quality drops for the
larger dataset.}
\label{fig:3Dlarge}
\end{figure*}

However, it performs less well when some correspondences are weak and thus
prevents the final spanning tree from spanning all cameras. This results in a
partial reconstruction of the scene. This is also evident when there are
sufficient correspondences for the spanning tree construction to propagate,
but not enough for a good estimate of camera pose to be made. This results in a
lower quality reconstruction (figure \ref{fig:3Dlarge}).

The lack of correspondences is not only due to issues in data acquisition, but
also due to varying lighting conditions. For example, the light bleeding from
the beer fridges and windows caused feature matches to succeed less well.

We attempt to use depth data in our bundle adjustment step. This was in hopes of
improving the final reconstruction accuracy. It can be seen however in figures
\ref{fig:BA_noD} and \ref{fig:BA_withD} that using depth data can sometimes
prevent the bundle adjustment step from executing successfully. This could be
due to the limited resolution of depth data compared to reprojection errors.

\begin{figure}
\begin{center}
   \includegraphics[width=0.9\linewidth]{figures/result_tiny_noBA.png}
\end{center}
\caption{3D reconstruction result using just 6 images}
\label{fig:3D_6images}
\end{figure}

Nonetheless, the pipeline works well in general, and does not require specific
parameters and is thus general purpose. Even with as few as 6 images, an
accurate dense reconstruction is possible (figure \ref{fig:3D_6images}) and this
can be extended to larger areas with more images provided sufficient
correspondences.

%-------------------------------------------------------------------------

\section{Conclusion}

A Structure from Motion pipeline taking advantage of both RGB and depth
information was successfully implemented. This involved all steps starting
from the acquisition of data to the visualisation of the final 3D model. A
good understanding of all parts of the pipeline as well as associated technology
was required.

One of the goals of this project was to investigate the improvements which can
be made by incorporating the depth information provided by the Kinect. We can
show that the camera registration step works quite well with depth data, as can
be seen in the final reconstructions. It cannot be said however that depth data
can improve the bundle adjustment step. Our initial assessment shows that the
low resolution of depth data ($2^{11}$ bits) can result in a degradation in
quality of the final model, but this may require more analysis.

It can be claimed that incorporating depth data into the pipeline makes it more
robust. This is because depth data is used in multiple steps such as camera
registration and the finding of centre-of-mass of keypoint pose clusters instead
of periodic bundle adjustment. A single bundle adjustment step is enough to
produce a high quality final model.

One assessment which may be helpful is the comparison of our 2D-3D registration
method against a more conventional 2D-2D registration as well as 3D-3D
registration using depth data from both images in a pair. This would further
inform us whether depth data is helpful in structure from motion and in which
steps. We propose this as future work in this area.

%-------------------------------------------------------------------------

\section*{Work distribution}

The authors worked together weekly and thus shared workload fairly.

Park focused more infrastructure, spanning tree, and visualisation, while Wang
focused more on camera registration, clusters finding and bundle adjustment.

Data acquisition was done with cooperation from Jonathan, the owner of craft
beer shop, BeersNMore.

We thank Pavol Vyhlidal, An-phi Nguyen, and Federico Danieli for their support
and enthusiasm.

%-------------------------------------------------------------------------

{\small
\bibliographystyle{ieeetr}
\bibliography{egbib}
}

\end{document}


\newcommand{\Harmonization}{
  \begin{figure}
    \includegraphics[width=5.5cm]{fig/piston.png}
    \hspace{1cm}
    \includegraphics[width=5.5cm]{fig/harmmt.png} \\
    \begin{flushleft}
    \begin{small}
     \hspace{2.1cm} V \ \ \ \ \ \ \ \,I \ \ \ \ \ \ \ \,IV \,VI \ \,III \ IV \ \ \,V
     \hspace{2.65cm} I \ \ V \ \ \ VI \ V \ \ IV \ I \ \ \ V  \,IV \ \ \ \ V
    \end{small}
    \end{flushleft}
    \caption{Harmonization by Piston (left) and Our Implementation (right)}
    \Description{Harmonization by Piston (left) and Our Implementation (right)}
    \label{fig:harmonization}
  \end{figure}
}

\newcommand{\HarmonizationF}{
  \begin{figure}
    \includegraphics[width=5.5cm]{fig/fharm.png}
    \begin{flushleft}
    \begin{small}
     \hspace{5.5cm} V \,III \ \ \ I \ \,III \ \,II \,IV \ \,III IV \ \ V
    \end{small}
    \end{flushleft}
    \caption{Harmonization by \fharm}
    \Description{Harmonization by \fharm}
    \label{fig:harmonizationF}
  \end{figure}
}

\section{Harmony}
\label{sec:harmony}

Harmony refers to simultaneously sounding tones (chords) and its study
is primarily concerned with the progression of chords throughout a
piece of music. It is in essence the dual of counterpoint. In this
section we formalize a small part of the classic text
\citet{piston-harmony}, in particular part of Chapter~9 which is
concerned with techniques for harmonizing a given melody.
The complete implementation can be found in \texttt{Harmony.agda}
and \texttt{Piston.agda} of the anonymized supplement.

\subsection{Harmonic Progression}
\label{sec:harmony:prog}

The primary Example~9.1 in Piston's text
(Figure~\ref{fig:harmonization} left) shows a
sample four part SATB harmonization in C~major. The melody is the
series of highest notes, i.e., the soprano (S) line, while the other three
parts alto (A), tenor (T), and bass (B) are the melodies below
it. Roman numerals denote the names of the triads at each vertical
slice. Note that even though the example consists of four voices there are only
three distinct pitches sounding at each vertical slice (where tones an
octave apart are considered to have the same relative pitch). As is
typical the root of the triad is always doubled. Also note that all
triads are in root position (i.e., the root is in the bass voice),
as Piston restricts to this case initially in the chapter (he does
relax it later). The goal of a harmonization is both to create an
aesthetically pleasing chord progression as well as fluid melodies in
each voice (called voice leading) and independence of each music
line (in other words good counterpoint). In this chapter
Piston gives advice on how to achieve these goals.

\Harmonization

We would like to formalize some of this advice and apply it to
automate harmonization of melodies. We start with selection of the
sequence of triads, in other words the harmonic progression. Earlier
in the text (on page~23) Piston presents a Table of Usual Root
Progressions, noting which triads usually follow a given one and
which are sometimes used or rare. It is easy to encode this in Agda
using the triad datatype explained in Section~\ref{sec:music}.

\begin{alltt}
record NextTriad : Set where
  constructor nextTriad
  field
    usual : List Triad; sometimes : List Triad; rare : List Triad

rootProgression : Triad \(\rightarrow\) NextTriad
rootProgression I   = nextTriad (IV :: V :: []) (VI :: []) (II  :: III : [])
rootProgression II  = nextTriad (V :: []) (IV :: VI :: []) (I :: III :: [])
rootProgression III = nextTriad (VI :: []) (IV :: []) (I :: II :: V :: [])
rootProgression IV  = nextTriad (V :: []) (I :: II :: []) (III :: VI :: [])
rootProgression V   = nextTriad (I :: []) (IV :: VI :: []) (II :: III :: [])
rootProgression VI  = nextTriad (II :: V :: []) (III :: IV :: []) (I :: [])
rootProgression VII = nextTriad (I  :: III :: []) (VI :: []) (II :: IV  :: V :: [])
\end{alltt}

It is easiest to create a harmonic progression in reverse, to
ensure that one ends a musical phrase with either a full (V--I) or half (V)
cadence, and work backward from there. So we would like to invert this
table to give a list of likely triads to precede a given triad. This
would normally be straightforward but here we see a disadvantage of
using Agda. As is typical with languages based on dependent type
theories, equality is a very subtle concept and even determining if a
term is a member of a list is nontrivial.

In Haskell this would be easy---there is an \texttt{Eq} type class and a
method for automatically generating equalities. In the Agda Standard
Library~\citep{agda-stdlib} list membership is a relation with
constructive evidence of membership, and requires far more effort than we
would like, given that we currently do not need to do
proofs. Alternatives include defining decidable equality for triads
(which requires writing out 49 cases), or mapping to natural numbers
and using already implemented decidable or boolean equality for
those. For now we simply write the inverse function manually,
excluding the rare cases.

\begin{alltt}
previousTriads : Triad \(\rightarrow\) List Triad
previousTriads I   = V :: IV :: VII :: []
previousTriads II  = VI :: IV :: []
previousTriads III = VI :: VII :: []
previousTriads IV  = I :: V :: II :: III :: []
previousTriads V   = I :: IV :: II :: VI :: []
previousTriads VI  = IV :: I :: II :: V :: VII :: []
previousTriads VII = []
\end{alltt}

\subsection{Completing the Harmonization}
\label{sec:harmony:complete}

As Piston does we require the harmonizing triad to include the melody
note. We use a function \texttt{harmonizations} to generate a list of
all candidate harmonizations for a given melody (see
\texttt{Harmony.agda} for its definition). Here again we encounter difficulties
with equality and membership. We use a bit vector to represent the
set of scale degrees in a triad, where each scale degree is mapped to
an element of \texttt{Fin 7}. However we are able to make use of
dependent types in the form of vectors to ensure the length of each
harmonization matches the melody, and continue to use vectors in the
following functions where appropriate.

We next restrict to progressions ending in a half cadence (V) to match
Piston's example. This results in 25
candidate progressions. For each pitch and triad pair we construct a
\texttt{harmonizingChord} in root position with the root doubled. The
function \texttt{voiceChord} spaces the notes nicely. All of this work
is done in the function \texttt{voicedHarmonizations}.

\begin{alltt}
voicedHarmonizations : \{n : N\} \(\rightarrow\) Vec Pitch n \(\rightarrow\) List (Vec (Vec Pitch 4) n)
voicedHarmonizations \{n\} ps =
  let ds = map pitchToDegreeCMajor ps
      hs : List (Vec Triad n)
      hs = filter halfCadence (harmonizations ds)
      in map (\(\lambda\) ts \(\rightarrow\) map (\(\lambda\) pt \(\rightarrow\)
             \(\text{proj}_1\) pt :: harmonizingChord (\(\text{proj}_1\) pt) (\(\text{proj}_2\) pt)) (zip ps ts)) hs
\end{alltt}

To determine the ``best'' of these harmonizations, we now make use of
\texttt{motionCheck} from the counterpoint work. We compare all six
pairs of melodies, count the number of motion errors present, and pick
the harmonization which minimizes these flaws. The function to
determine all errors is straightforward.

\begin{alltt}
motionErrors : \{n : N\} \(\rightarrow\) Vec (Vec Pitch 4) n \(\rightarrow\) List MotionError
motionErrors xs =
  let ss = map (head) xs
      as = map (head \(\circ\) tail) xs
      ts = map (head \(\circ\) tail \(\circ\) tail) xs
      bs = map (head \(\circ\) tail \(\circ\) tail \(\circ\) tail) xs

      sas = map pitchPairToPitchInterval (toList (zip as ss))
      sts = map pitchPairToPitchInterval (toList (zip ts ss))
      sbs = map pitchPairToPitchInterval (toList (zip bs ss))
      ats = map pitchPairToPitchInterval (toList (zip ts as))
      abs = map pitchPairToPitchInterval (toList (zip bs as))
      tbs = map pitchPairToPitchInterval (toList (zip bs ts))
  in concatMap checkMotion (sas :: sts :: sbs :: ats :: abs :: tbs :: [])
\end{alltt}

The harmonization produced is shown in Figure~\ref{fig:harmonization}
right.

\subsection{Comparison with Previous Work}
\label{sec:harmony:compare}

\HarmonizationF

Most of the Haskell-based work on
music~\citep{magalhaes-harmtrace,koops-fharm,magalhaes-fcomp} has
focused on harmony. In particular \fharm~\citep{koops-fharm} tackles
the problem of harmonizing a melody;
Figure~\ref{fig:harmonizationF} shows their harmonization of
Piston's example. Their method is quite different from
ours. Instead of using Piston's table of root progressions they use a
more sophisticated hierarchical grammar from previous
work~\citep{magalhaes-harmtrace}.
More specifically, they first require the melody notes to be members of
the harmonizing chords, then use their previous work to try to create
a harmonic analysis of every possible sequence of chords that arises,
choosing the one that most closely fits their grammar.

Although the result produced by \fharm is not bad, it clearly leaves
room for improvement. Noted in
the paper is that no attention is given to voicing (how each of the
four parts flows horizontally and how the lines interact---in other
words the counterpoint). They use inversions to ensure each melody line
does not move too much, but as the harmonizing triads are all in close
position separated from the melody, the result sounds like a soprano
melody with block chord accompaniment, rather than four independent
lines.

Among those issues not discussed in the paper, one is the choice of
notes to be doubled: \fharm always doubles the melody note, while
in four part harmony it is generally preferred to double the root
note as Piston does.
Another concern is that minimizing the number of parse errors as a means
to choose the best harmonization does not necessarily have a musical
meaning.

Our harmonization follows Piston's rules more closely and has arguably
better spacing of voices; readers can decide for themselves which is
more pleasing.


\chapter{{\tt MT} directory}
\label{chapter:MT}
\par
All methods that use multithreaded function calls
are found in this directory.
Three functionalities are presently supported:
matrix-matrix multiplies, sparse factorizations, and solves.
\par
The multithreaded methods to compute $Y := Y + \alpha A X$,
$Y := Y + \alpha A^T X$ and $Y := Y + \alpha A^H X$
are simple.
Their calling sequences are almost identical to their
serial counterparts: global data structures for $Y$,
$\alpha$, $A$ and $X$ are followed by the number of threads,
a message level and file.
Thread $q$ accesses part of $A$, part of $X$, and computes its own
$Y^q = \alpha A X$ using those entries of $A$ that it is
responsible for.
This work is done independently by all threads.
The global summation $Y := Y + \sum_q Y^q$ is done in serial mode
by the calling process.
\par
This approach is not scalable.
A better approach would be to explicitly partition $A$ into local
$A^q$ matrices, and use local $X^q$ and $Y^q$ to hold rows of $X$
and $Y$ that have support with $A^q$, as is done with the
distributed MPI matrix-matrix multiplies.
(With MPI there is added complexity since $X$ and $Y$ are distributed
among processors.)
\par
A matrix-matrix multiply does not exist in isolation.
For example, a block shifted eigensolver requires factorizations of
$A - \sigma B$ and multiplies using $A$ or $B$.
The data structure for the matrix that takes part in the multiply
needs to toggle back and for between its forms for the factor and
multiply.
Managing this in a distributed environment is actually easier than
a multithreaded environment, for $A$ and $B$ are already
distributed.
Our multithreaded factorization expects $A$ and $B$ in global form.
Insisting that $A$ and $B$ be partitioned as $A^q$ and $B^q$
matrices is too great a burden for the user that has no need for a
multithreaded matrix-matrix multiply.
Allowing the $A^q$ matrices to overlap or point into the global $A$
matrix in a persistent fashion is not cleanly possible, but
requires changes to the {\tt InpMtx} object.
\par
In the future we intend to provide a scalable multithreaded
matrix-matrix multiply.
It requires a more in-depth consideration of the issues involved
than we are able to give it at the present time.
\par
The multithreaded factorizations $A = LU$ and $A = QR$
are very similar to the serial factorizations,
in both the calling sequence visible to the user and
in the underlying code structure.
The only additional parameters in the calling sequence is a map
from the fronts to the threads that defines who does what
computation, and a {\it lookahead} parameter that allows some
ability to control and reduce the idle time during the factorization.
Inside the code, the deterministic post-order traversal of the
serial factorization is replaced by independent topological
traversals of the front tree.
It is the list and working storage data structures
(the {\tt ChvList}, {\tt ChvManager} and {\tt SubMtxManager} objects)
that have locks.
{\it What} is done is common code between the serial and
multithreaded environments, it is the choreography, i.e.,
{\it who} does what, that differs.
\par
Most of these same comments apply to the multithreaded solve methods.
The calling sequences between the serial and multithreaded solves
differs by one parameter, a {\tt SolveMap} object that maps the
submatrices of the factor matrix to the threads that will compute
with them.


\chapterimage{head2.png} % Chapter heading image
\chapter{EM: Condition Test}
\section{Four model systems}
\begin{figure}[h!]
\begin{center}
        \includegraphics[scale=0.25]{ch5/potential_traj.pdf}
        \caption{$S_{\rm eq}=-1.683$}
\end{center}
\end{figure}

\subsection{EM Results for model systems}
\begin{center}
        \includegraphics[scale=0.35]{ch5/em_model_systems.pdf}
\end{center}

\subsection{Scan of log-likelihood as change diffusity}
\begin{center}
        \includegraphics[scale=0.35]{ch5/model_systems_D_scan.pdf}
\end{center}

\section{Models for free diffusion}
\begin{center}
        \includegraphics[scale=0.24]{ch5/flat_potential_traj.pdf}
\end{center}
\subsection{Scan of log-likelihood as change diffusity}
\begin{center}
        \includegraphics[scale=0.35]{ch5/flat_D_scan_all.pdf}
\end{center}

\section{Photon Operator}
Ideally, the photon operator should be
\begin{align*}
        y(x) = \delta (x).
\end{align*}
but we use normal distribution to approximate it:
\begin{align*}
      y(x) =  N(x,\sigma^{2})
\end{align*}
\begin{center}
        \includegraphics[scale=0.3]{ch5/photon_operator_models.pdf}
\end{center}
We will used the following example to test
\begin{center}
        \includegraphics[scale=0.45]{ch5/photon_operator_test_model.pdf}
\end{center}

\subsection{Test for EM equilibrium probability}
\begin{center}
        \includegraphics[scale=0.4]{ch5/photon_operator_peq_em.pdf}
\end{center}

\subsection{Scan of log-likelihood as change diffusity}
\begin{center}
        \includegraphics[scale=0.35]{ch5/photon_operator_D_scan.pdf}
\end{center}

\subsection{The convergence and the diffusity}
\begin{center}
        \includegraphics[scale=0.35]{ch5/photon_operator_Dtest_for_peq.pdf}
\end{center}

\subsection{Comparison with KDE}
The cyan lines are Gaussian Kernel Density Estimation(KDE)
\begin{center}
        \includegraphics[scale=0.35]{ch5/photon_operator_compare_kde.pdf}
\end{center}


\documentclass[11pt]{article}
%============Macros==================%
% you don't need to change anything. start editing from main body.
\usepackage{amsmath,amsfonts,amssymb,amsthm}
\usepackage{qcircuit}
\usepackage[margin=1in]{geometry}
%--------------Cosmetic----------------%
\usepackage{mathtools}
\usepackage{hyperref}
\usepackage{fullpage}
\usepackage{microtype}
\usepackage{xspace}
\usepackage[svgnames]{xcolor}
\usepackage[sc]{mathpazo}
\usepackage{enumitem}
\usepackage{braket}
\setlist[enumerate]{itemsep=1pt,topsep=2pt}
\setlist[itemize]{itemsep=1pt,topsep=2pt}
%--------------Header------------------%
\def\course{CS 410/510 Introduction to Quantum Computing}
\def\term{Portland State U, Spring 2017}
\def\prof{Lecturer: Fang Song}
\newcommand{\handout}[5]{
   \renewcommand{\thepage}{\arabic{page}}
   \begin{center}
   \framebox{
      \vbox{
    \hbox to 5.78in { \hfill \large{\course} \hfill }
       \vspace{2mm}
       \hbox to 6in { {\Large \hfill #5  \hfill} }
       \vspace{2mm}
       \hbox to 6in { \term \hfill \emph{#2}}
       \hbox to 6in { {#3 \hfill \emph{#4}}}
      }
   }
   \end{center}
   \vspace*{4mm}
}
\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{#4}{{Lecture #1}}}

\newcommand\fang[1]{{\color{red}[Fang: #1]}}
%=============Main Doc=================%
\begin{document}
%-----Specs: change accordingly--------%
\def\lecdate{April 20, 2017} % put lecture date here
\def\scribe{Scribe: John Donahue} % put your name here
\def\lecnum{6} % change the lecture number

\lecture{\lecnum}{\lecdate}{\prof}{\scribe}%

\begin{center}
{\textsc{Version: \today}}
\end{center}

%==============================%
\section{Phase Estimation}
%==============================%
Remember, for this problem, our input is a quantum circuit for a unitary U, as well as the eigenvector $\ket{\psi}$. $U\ket{\psi}=e^{2\pi i \theta}\ket{\psi}$. Our goal is to approximate $\theta$.

\[
\Qcircuit @C=1em @R=1em {
& \lstick{\ket{0}}    & \qw & \gate{H} & \qw & \ctrl{1} & \qw & \multigate{3}{\ \mathcal{F}\ } & \qw & \meter \\
& \lstick{\ket{0}}    & \qw & \gate{H} & \qw & \ctrl{2} & \qw & \ghost{\ \mathcal{F}\ } & \qw & \meter \\
& ...\\
& \lstick{\ket{0}}    & \qw & \gate{H} & \qw & \ctrl{1} & \qw & \ghost{\ \mathcal{F}\ } & \qw & \meter \\
& \lstick{\ket{\psi}} & \qw & \qw      & \qw & \gate{U} & \qw
}
\]

%==============================%
\subsection{Special Case}
%==============================%

First, lets consider the special case $\theta =
\frac{j}{{2^m}}$. Remember, the state going into $F$ is
\[  \frac{1}{\sqrt[]{2^m}}\sum_{x \in \{0,1\}^m} e^{2 \pi
    i \theta x} \ket{x} \]

Let $\omega := e^{\frac{2\pi i}{2^m}}$, rewrite it as
$ \frac{1}{\sqrt[]{2^m}}\sum_{x \in \{0,1\}^m} \omega^{j x} \ket{x} =
: \ket{\phi_j}$. Therefore, computing $j$ amounts to distinguishing
each $\ket{\phi_j}$.

Now, consider $\{ \ket{\phi_j} : j \in \{ 0,1 \}^m
\}$. $\braket{\phi_j | \phi_{j'}'}$ will equal 1 if $j = j'$, and 0 if
otherwise. This forma an orthonormal basis for $m$-qubit states in
$\mathbb{C}$. There is also a standard basis
$\{ \ket{j} : j \in \{ 0,1 \}^m \}$. We know that there exists a
unitary F such that $\ket{\phi_j} \mapsto^F \ket{j}$. So, if we apply
$F$ to $\ket{\phi_j}$, we will see $j$.

But what is $F$? If $j = 0$, that means $F$ is the following
$2^m \times 2^m$ matrix.

\begin{align*}
F =
\begin{bmatrix}
1 & 1 & \dots & 1 & \dots & 1 \\
1 & \omega & \dots & \omega^j & \dots & \dots \\
\dots & \dots & \dots & \dots & \dots & \dots & \\
1 & \omega^{2^m - 1} & \dots & \omega^{j(2^m - 1)} & \dots & \omega^{(2^m - 1)(2^m - 1)}
\end{bmatrix}
\end{align*}

This is the discrete Fourier Transform.

%==============================%
\subsection{General Case}
%==============================%

Now, lets look at the general case, where $\theta \in [0,1)$. We will see the following.

\begin{align*}
F\ket{x} = \frac{1}{\sqrt[]{2^m}}\sum_{y \in \{0,1\}^m} \omega^{-y x} \ket{y}
\end{align*}
\begin{align*}
F\ket{\phi} = \frac{1}{\sqrt[]{2^m}}\sum_{x \in \{0,1\}^m} \omega^{j x} F\ket{x}
\end{align*}
\begin{align*}
F\ket{\phi} = \frac{1}{2^m}\sum_{x \in \{0,1\}^m} \omega^{j x} \sum_{y \in \{0,1\}^m} \omega^{-y x} \ket{y}
\end{align*}
\begin{align*}
F\ket{\phi} = \frac{1}{2^m}\sum_{y \in \{0,1\}^m} (\sum_{x \in \{0,1\}^m} e^{2 \pi i x (\theta - \frac{y}{2^m})} )\ket{y}
\end{align*}

For measuring, we will see this.

\begin{align*}
Pr[y] = \left|\frac{1}{2^m}\sum_{x \in \{0,1\}^m} e^{2 \pi i x (\theta - \frac{y}{2^m})}\right|^2
\end{align*}

Note that this looks exactly like a geometric series. That means we can rewrite it without the $\sum$.

\begin{align*}
Pr[y] = \frac{1}{2^{2m}}|\frac{e^{2 \pi i (M\theta - y)} - 1}{e^{2 \pi i (\theta - \frac{y}{M})} - 1}|^2
\end{align*}

\fang{add the details of analysis}

This means $\theta = \frac{j}{M} + \epsilon$, $|\epsilon| \leq \frac{1}{2^{m+1}}$. Essentially, this ends up showing $Pr[j] \geq \frac{4}{\pi^2} \geq 0.4$. The larger Pr[j] becomes, the more precise our approximation of $\theta$ will be. In short, the chance that we will have m bits corrext at least 40\% of the time.

%==============================%
\section{Quantum Fourier Transform}
%==============================%

\fang{More to add here}

The following shows a quantum circuit for computing Fourier Transform.

\begin{align*}
R_k =
\begin{bmatrix}
1 & 0 \\
0 & e^{\frac{2 \pi i}{2^k}}
\end{bmatrix}
\end{align*}

\[
\Qcircuit @C=1em @R=1em {
& \lstick{\ket{x_{m-1}}} & \qw & \gate{H} & \gate{R_2} & ... & \qw & \gate{R_{m-1}} & \gate{R_{m}} & \qw & \qw & \qw & \qw \\
& \lstick{\ket{x_{m-2}}} & \qw & \qw & \ctrl{-1} & ... \\
& ... \\
& \lstick{\ket{x_1}} & \qw & \qw & \qw & \qw & \qw & \ctrl{-3} & \qw & \gate{H} & \gate{R_2} & \qw & \qw \\
& \lstick{\ket{x_0}} & \qw & \qw & \qw & \qw & \qw & \qw & \ctrl{-4} & \qw & \ctrl{-1} & \gate{H} & \qw \\
}
\]

Lastly, the outputs are all reversed such that $\ket{x_0}$ is on the top.

\end{document}