/*  _______________________________________________________________________

    DAKOTA: Design Analysis Kit for Optimization and Terascale Applications
    Copyright 2014 Sandia Corporation.
    This software is distributed under the GNU Lesser General Public License.
    For more information, see the README file in the top Dakota directory.
    _______________________________________________________________________ */

//- Class:	     SensAnalysisGlobal
//- Description: Utility helper class which has correlations and VBD
//- Owner:       Laura Swiler, Brian Adams, Ahmad Rushdi
//- Checked by:
//- Version:

#include "SensAnalysisGlobal.hpp"
#include "ResultsManager.hpp"
#include "dakota_linear_algebra.hpp"
#include <algorithm>
#include <boost/iterator/counting_iterator.hpp>

static const char rcsId[]="@(#) $Id: SensAnalysisGlobal.cpp 6170 2009-10-06 22:42:15Z lpswile $";

namespace Dakota {

RealArray SensAnalysisGlobal::rawData = RealArray();

bool SensAnalysisGlobal::rank_sort(const int& x, const int& y)
{ return rawData[x]<rawData[y]; }

size_t SensAnalysisGlobal::
find_valid_samples(const IntResponseMap& resp_samples, BoolDeque& valid_sample)
{
  // TODO: later compute correlation on per-response basis to keep
  // partial faults
  using std::isfinite;

  size_t num_obs = resp_samples.size(), num_valid_samples = 0;
  IntRespMCIter it = resp_samples.begin();
  for (size_t j=0; j<num_obs; ++j, ++it) {
    valid_sample[j] = true;
    for (size_t k=0; k<numFns; ++k)
      if (!isfinite(it->second.function_value(k))) {
        valid_sample[j] = false;
        break;
      }
    if (valid_sample[j])
      ++num_valid_samples;
  }

  return num_valid_samples;
}

void SensAnalysisGlobal::
valid_sample_matrix(const VariablesArray& vars_samples,
                    const IntResponseMap& resp_samples,
                    const StringSetArray& dss_vals,
                    const BoolDeque is_valid_sample,
                    RealMatrix& valid_data)
{
  int num_obs = vars_samples.size(), num_corr = valid_data.numRows();
  IntRespMCIter it = resp_samples.begin();
  for (size_t j=0, s_cntr=0; j<num_obs; ++j, ++it)
    if (is_valid_sample[j]) {
      // get a view of the first numVars rows of the samples col
      RealVector td_col_vars(Teuchos::View, valid_data[s_cntr], (int)numVars);
      vars_samples[j].as_vector(dss_vals, td_col_vars);
      // get a view of the last numFns rows of the samples col
      RealVector td_col_resp(Teuchos::View, valid_data[s_cntr] + numVars,
                             (int)numFns);
      copy_data(it->second.function_values(), td_col_resp);
      ++s_cntr;
    }
}

void SensAnalysisGlobal::
valid_sample_matrix(const RealMatrix&     vars_samples,
                    const IntResponseMap& resp_samples,
                    const BoolDeque is_valid_sample,
                    RealMatrix& valid_data)
{
  int num_obs = vars_samples.numCols(), num_corr = valid_data.numRows();
  IntRespMCIter it = resp_samples.begin();
  for (int j=0, s_cntr=0; j<num_obs; ++j, ++it)
    if (is_valid_sample[j]) {
      for (int i=0; i<numVars; ++i)
        valid_data(i, s_cntr) = vars_samples(i, j);
      // get a view of the last numFns rows of the samples col
      RealVector td_col_resp(Teuchos::View, valid_data[s_cntr] + numVars,
                             (int)numFns);
      copy_data(it->second.function_values(), td_col_resp);
      ++s_cntr;
    }
}

/** When converting values to ranks, uses the average ranks of any tied values */
void SensAnalysisGlobal::values_to_ranks(RealMatrix& valid_data)
{
  int num_corr = valid_data.numRows(), num_valid_samples = valid_data.numCols();
  // for each var/resp
  for (int i=0; i<num_corr; ++i) {
    // create a multimap from value to array index (so it is sorted by value and
    // the ranks are given by the map order); don't need a stable sort as we are
    // replacing the tied values by their average rank
    RealIntMultiMap vals_inds;
    for (int j=0; j<num_valid_samples; ++j)
      vals_inds.insert(std::make_pair(valid_data(i,j), j));

    // iterate for each unique value and find tied values
    RealIntMultiMap::const_iterator vi_it = vals_inds.begin();
    RealIntMultiMap::const_iterator vi_end = vals_inds.end();
    for (int rank=0; vi_it != vi_end; ) {
      // find a range of tied values
      double value = vi_it->first;
      std::pair<RealIntMultiMap::const_iterator, RealIntMultiMap::const_iterator>
	tied_range = vals_inds.equal_range(value);
      int num_ties = std::distance(tied_range.first, tied_range.second);
      double avg_rank = (rank + rank+num_ties-1) / 2.0;
      // all tied values get assigned the average rank
      for ( ; tied_range.first != tied_range.second; ++tied_range.first)
	valid_data(i, tied_range.first->second) = avg_rank;
      // increment to the next unequal value
      vi_it = tied_range.second;
      rank += num_ties;
    }
  }
}

void SensAnalysisGlobal::center_rows(RealMatrix& data_matrix)
{
  int num_row = data_matrix.numRows(), num_col = data_matrix.numCols();
  for (int i=0; i<num_row; i++) {
    // normalize each row (input/output factor) by its mean across observations
    Real row_mean = 0.0;
    for (int j=0; j<num_col; j++)
      row_mean += data_matrix(i,j);
    row_mean /= (Real)num_col;
    for (int j=0; j<num_col; j++)
      data_matrix(i,j) -= row_mean;
  }
}

void SensAnalysisGlobal::correl_adjust(Real& corr_value)
{
  if (std::isfinite(corr_value) && std::abs(corr_value) > 1.0)
    corr_value = corr_value / std::abs(corr_value);
}

/** This version is used when full variables objects are being
    processed. Calculates simple correlation, partial correlation,
    simple rank correlation, and partial rank correlation
    coefficients. */
void SensAnalysisGlobal::
compute_correlations(const VariablesArray& vars_samples,
                     const IntResponseMap& resp_samples,
                     const StringSetArray& dss_vals)
{
  size_t num_obs = vars_samples.size();
  if (num_obs == 0) {
    Cerr << "Error: Number of samples must be nonzero in SensAnalysisGlobal::"
         << "compute_correlations()." << std::endl;
    abort_handler(-1);
  }
  if (resp_samples.size() != num_obs) {
    Cerr << "Error: Mismatch in array lengths in SensAnalysisGlobal::"
         << "compute_correlations()." << std::endl;
    abort_handler(-1);
  }

  numVars = vars_samples[0].cv() + vars_samples[0].div() +
    vars_samples[0].dsv() + vars_samples[0].drv();
  numFns  = resp_samples.begin()->second.num_functions();
  int num_corr = numVars + numFns;

  // determine which samples have valid responses
  BoolDeque is_valid_sample(num_obs);
  int num_valid_samples = find_valid_samples(resp_samples, is_valid_sample);

  // The following calls regenerate and destroy the valid_data matrix
  // to save memory

  // create a matrix containing only the valid sample data
  RealMatrix valid_data(num_corr, num_valid_samples);

  // calculate simple rank correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, dss_vals, is_valid_sample,
                      valid_data);
  simple_corr(valid_data, num_corr, simpleCorr);

  // calculate partial correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, dss_vals, is_valid_sample,
                      valid_data);
  partial_corr(valid_data, numVars, simpleCorr, partialCorr, numericalIssuesRaw);

  // calculate simple rank correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, dss_vals, is_valid_sample,
                      valid_data);
  values_to_ranks(valid_data);
  simple_corr(valid_data, num_corr, simpleRankCorr);

  // calculate partial rank correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, dss_vals, is_valid_sample,
                      valid_data);
  values_to_ranks(valid_data);
  partial_corr(valid_data, numVars, simpleRankCorr, partialRankCorr,
               numericalIssuesRank);

  corrComputed = true;
}

/** This version is used when compact samples matrix is being
    processed.  Calculates simple correlation, partial correlation,
    simple rank correlation, and partial rank correlation
    coefficients. */
void SensAnalysisGlobal::
compute_correlations(const RealMatrix&     vars_samples,
                     const IntResponseMap& resp_samples)
{
  int num_obs = vars_samples.numCols();
  if (!num_obs) {
    Cerr << "Error: Number of samples must be nonzero in SensAnalysisGlobal::"
         << "compute_correlations()." << std::endl;
    abort_handler(-1);
  }
  if (resp_samples.size() != num_obs) {
    Cerr << "Error: Mismatch in array lengths in SensAnalysisGlobal::"
         << "compute_correlations()." << std::endl;
    abort_handler(-1);
  }

  numVars = vars_samples.numRows();
  numFns  = resp_samples.begin()->second.num_functions();
  int num_corr = numVars + numFns;

  // determine which samples have valid responses
  BoolDeque is_valid_sample(num_obs);
  int num_valid_samples = find_valid_samples(resp_samples, is_valid_sample);

  // The following calls regenerate and destroy the valid_data matrix
  // to save memory

  // create a matrix containing only the valid sample data
  RealMatrix valid_data(num_corr, num_valid_samples);

  // calculate simple rank correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, is_valid_sample, valid_data);
  simple_corr(valid_data, num_corr, simpleCorr);

  // calculate partial correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, is_valid_sample, valid_data);
  partial_corr(valid_data, numVars, simpleCorr, partialCorr, numericalIssuesRaw);

  // calculate simple rank correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, is_valid_sample, valid_data);
  values_to_ranks(valid_data);
  simple_corr(valid_data, num_corr, simpleRankCorr);

  // calculate partial rank correlation coeff
  valid_sample_matrix(vars_samples, resp_samples, is_valid_sample, valid_data);
  values_to_ranks(valid_data);
  partial_corr(valid_data, numVars, simpleRankCorr, partialRankCorr,
               numericalIssuesRank);

  corrComputed = true;
}

/** Calculates simple correlation coefficients from a matrix of data
    (oriented factors x observations):
     - num_corr is number of rows of total data
     - num_in indicates whether only pairs of correlations should be
       calculated between pairs of columns (num_in
       vs. num_corr-num_in); if num_in = num_corr, correlations are
       calculated between all columns */
void SensAnalysisGlobal::
simple_corr(RealMatrix& total_data, const int& num_in, RealMatrix& corr_matrix)
{
  int num_corr = total_data.numRows(), num_obs = total_data.numCols();

  center_rows(total_data);

  for (int i=0; i<num_corr; i++) {
    // calculate sum of squares for each factor (row)
    Real row_sumsq = 0.0;
    for (int j=0; j<num_obs; j++)
      row_sumsq += total_data(i,j)*total_data(i,j);
    row_sumsq = std::sqrt(row_sumsq);
    // normalize the rows with the sumsquare term
    for (int j=0; j<num_obs; j++)
      total_data(i,j) /= row_sumsq;
  }

  // calculate matrix of simple correlation coefficients
  if (num_corr == num_in) {
    // all-to-all case
    corr_matrix.shape(num_corr, num_corr);
    if (num_obs <= 1)
      corr_matrix.putScalar(std::numeric_limits<double>::quiet_NaN());
    else {
      corr_matrix.multiply(Teuchos::NO_TRANS, Teuchos::TRANS, 1.0,
			   total_data, total_data, 0.0);
      for (int i=0; i<num_corr; ++i) {
	// set finite diagonal values to 1.0
	if (std::isfinite(corr_matrix(i,i)))
	  corr_matrix(i,i) = 1.0;
	// snap all finite values to [-1.0, 1.0]
	for (int j=0; j<i; ++j) {
	  correl_adjust(corr_matrix(i,j));
	  correl_adjust(corr_matrix(j,i));
	}
      }
    }
  }
  else {
    // input-to-output case
    int num_out = num_corr - num_in;
    corr_matrix.shape(num_in, num_out);
    if (num_obs <= 1)
      corr_matrix.putScalar(std::numeric_limits<double>::quiet_NaN());
    else {
      RealMatrix total_data_in(Teuchos::View, total_data, num_in, num_obs, 0, 0);
      RealMatrix total_data_out(Teuchos::View, total_data, num_out, num_obs,
				num_in, 0);
      corr_matrix.multiply(Teuchos::NO_TRANS, Teuchos::TRANS, 1.0,
			   total_data_in, total_data_out, 0.0);
      // snap all finite values to [-1.0, 1.0]
      for (int i=0; i<num_in; ++i)
	for (int j=0; j<num_out; ++j)
	  correl_adjust(corr_matrix(i,j));
    }
  }
}

/** Calculates partial correlation coefficients between num_in inputs
    and numRows() - num_in outputs. */
void SensAnalysisGlobal::
partial_corr(RealMatrix& total_data, const int num_in,
             const RealMatrix& simple_corr_mat,
             RealMatrix& corr_matrix, bool& numerical_issues)
{
  int num_obs = total_data.numCols(), num_out = total_data.numRows() - num_in;

  // initialize output data
  corr_matrix.reshape(num_in, num_out);
  numerical_issues = false;
  // TODO: return numerical issues per-input
  BoolDeque numerical_except(num_in, false);

  if (num_obs <= 1) {
    corr_matrix.putScalar(std::numeric_limits<double>::quiet_NaN());
    numerical_issues = true;
    return;
  }

  // For a single input factor, partial = simple (no controlling factors)
  if (num_in == 1) {
    for (int k=0; k<num_out; ++k)
      corr_matrix(0, k) = simple_corr_mat(0, k+1);
    return;
  }

  center_rows(total_data);

  // matrix of X = [Vi | R ]; the response cols don't change per variable
  RealMatrix correl_factors_X(num_obs, 1 + num_out);
  for (int j=0; j<num_obs; ++j)
    for (int k=0; k<num_out; ++k)
      correl_factors_X(j, 1+k) = total_data(num_in+k, j);
  // matrix of Z = [V~i]
  RealMatrix control_factors_Z(num_obs, num_in - 1);

  for (int i=0; i<num_in; ++i) {

    // partial correlation analysis for Vi, R, controlling for
    // V~i. Transpose everything to line up with more typical
    // convention Nobs x (Nvar + Nresp)
    for (int j=0; j<num_obs; ++j) {
      correl_factors_X(j, 0) = total_data(i, j); // Vi
      // (response columns are already populated above)
      // V~i
      for (int k=0; k<i; ++k)
        control_factors_Z(j, k) = total_data(k, j);
      for (int k=i+1; k<num_in; ++k)
        control_factors_Z(j, k-1) = total_data(k, j);
    }

    // form partial_cov = X'X - (X'Z)*inv(Z'Z)*(Z'X), preserving
    // symmetric and positive definite
    int mult_err = 0;

    // intialize to X'X
    RealMatrix partial_cov(1 + num_out, 1 + num_out);
    mult_err |= partial_cov.multiply(Teuchos::TRANS, Teuchos::NO_TRANS, 1.0,
				     correl_factors_X, correl_factors_X, 0.0);

    // Here we use truncated SVD to account for the case where there
    // are fewer data points than variables, including the degenerate
    // case of 2 observations.  Should be equivalent to minimum-norm
    // least squares in the regressions.

    RealMatrix Zinv_Zt_X;  // a bit misnamed, since Z is rectangular
    bool use_qr = false;
    if (use_qr) {
      Zinv_Zt_X.reshape(num_in - 1, 1 + num_out);
      // initialize to Z'X
      mult_err |= Zinv_Zt_X.multiply(Teuchos::TRANS, Teuchos::NO_TRANS, 1.0,
                                     control_factors_Z, correl_factors_X, 0.0);

      // factor Z = QR; in-place QR factorization destroys Z (this never errors)
      qr(control_factors_Z);

      // now backsolve to update to Rinv' * Z'X (a failed backsolve leaves junk)
      int qrs_info = qr_rsolve(control_factors_Z, true, Zinv_Zt_X);
      numerical_except[i] = numerical_except[i] || (qrs_info != 0);
    }
    else {
      // alternative with svd to numerical precision; some concern
      // remains that this might not yield nan/inf when it should

      // initialize to Z'X
      RealMatrix Zt_X(num_in - 1, 1 + num_out);
      mult_err |= Zt_X.multiply(Teuchos::TRANS, Teuchos::NO_TRANS, 1.0,
                                control_factors_Z, correl_factors_X, 0.0);
      RealVector sing_vals;
      RealMatrix v_trans;
      svd(control_factors_Z, sing_vals, v_trans);

      double tol =
        std::numeric_limits<double>::epsilon() * control_factors_Z.normInf();
      int sv_keep = 0;
      for ( ; sv_keep < sing_vals.length(); ++sv_keep)
        if (sing_vals[sv_keep] < tol)
          break;
      // TODO: Could opt for stricter check here:
      // if (sv_keep < std::min(num_obs, num_in-1))
      if (sv_keep == 0)
        numerical_except[i] = true;
      else {
        v_trans.reshape(sv_keep, num_in - 1);
        Zinv_Zt_X.reshape(sv_keep, 1 + num_out);
        Zinv_Zt_X.multiply(Teuchos::NO_TRANS, Teuchos::NO_TRANS, 1.0,
                           v_trans, Zt_X, 0.0);
        for (int j=0; j<sv_keep; ++j)
          for (int k=0; k < (1 + num_out); ++k)
            Zinv_Zt_X(j,k) /= sing_vals[j];
      }
    }

    if (!numerical_except[i])
      // X'X - (X'Z)*inv(Z'Z)*(Z'X) = X'X - (Zinv*Z'X)'(Zinv*Z'X)
      mult_err |= partial_cov.multiply(Teuchos::TRANS, Teuchos::NO_TRANS, -1.0,
				       Zinv_Zt_X, Zinv_Zt_X, 1.0);

    // This should never happen:
    if (mult_err != 0) {
      Cerr << "\nError (partial_corr): multiplying incompatible matrices.\n";
      abort_handler(-1);
    }
    numerical_issues = numerical_issues || numerical_except[i];

    for (int k=0; k<num_out; ++k)
      if (numerical_except[i])
        corr_matrix(i,k) = std::numeric_limits<Real>::quiet_NaN();
      else
        corr_matrix(i,k) = partial_cov(0, k+1) / std::sqrt(partial_cov(0,0)) /
          std::sqrt(partial_cov(k+1, k+1));
  }

  // snap all finite values to [-1.0, 1.0]
  for (int i=0; i<num_in; ++i)
    for (int j=0; j<num_out; ++j)
      correl_adjust(corr_matrix(i,j));
}

// Return true if any correlation coefficient is NaN or Inf, false otherwise
bool SensAnalysisGlobal::has_nan_or_inf(const RealMatrix &corr) const {
  int num_rows = corr.numRows(), num_cols = corr.numCols();
  for(int j = 0; j < num_cols; ++j)
    for(int i = 0; i < num_rows; ++i)
      if( ! std::isfinite(corr(i,j)))
        return true;
  return false;
}

// TODO: combine archive with print
void SensAnalysisGlobal::
archive_correlations(const StrStrSizet& run_identifier,
		     ResultsManager& iterator_results,
		     StringMultiArrayConstView cv_labels,
		     StringMultiArrayConstView div_labels,
		     StringMultiArrayConstView dsv_labels,
		     StringMultiArrayConstView drv_labels,
		     const StringArray& resp_labels) const
{
  if (!iterator_results.active())  return;

  int num_in_out = numVars + numFns;
  if (simpleCorr.numRows() == num_in_out &&
      simpleCorr.numCols() == num_in_out) {
    MetaDataType md;
    md["Row labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, resp_labels);
    md["Column labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, resp_labels);
    iterator_results.insert(run_identifier,
			    iterator_results.results_names.correl_simple_all,
			    simpleCorr, md);
  }
  else if (simpleCorr.numRows() == numVars &&
	   simpleCorr.numCols() == numFns) {
    MetaDataType md;
    md["Row labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, StringArray());
    md["Column labels"] = make_metadatavalue(resp_labels);
    iterator_results.insert(run_identifier,
			    iterator_results.results_names.correl_simple_io,
			    simpleCorr, md);
  }

  if (partialCorr.numRows() == numVars &&
      partialCorr.numCols() == numFns) {
    MetaDataType md;
    md["Row labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, StringArray());
    md["Column labels"] = make_metadatavalue(resp_labels);
    iterator_results.insert(run_identifier,
			    iterator_results.results_names.correl_partial_io,
			    partialCorr, md);
  }
  // TODO: metadata
  //  if (numericalIssuesRaw)

  if (simpleRankCorr.numRows() == num_in_out &&
      simpleRankCorr.numCols() == num_in_out) {
    MetaDataType md;
    md["Row labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, resp_labels);
    md["Column labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, resp_labels);
    iterator_results.insert(run_identifier,
			    iterator_results.results_names.correl_simple_rank_all,
			    simpleRankCorr, md);
  }
  else if (simpleRankCorr.numRows() == numVars &&
	   simpleRankCorr.numCols() == numFns) {
    MetaDataType md;
    md["Row labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, StringArray());
    md["Column labels"] = make_metadatavalue(resp_labels);
    iterator_results.insert(run_identifier,
			    iterator_results.results_names.correl_simple_rank_io,
			    simpleRankCorr, md);

  }

  if (partialRankCorr.numRows() == numVars &&
      partialRankCorr.numCols() == numFns) {
    MetaDataType md;
    md["Row labels"] =
      make_metadatavalue(cv_labels, div_labels, dsv_labels, drv_labels, StringArray());
    md["Column labels"] = make_metadatavalue(resp_labels);
    iterator_results.insert(run_identifier,
			    iterator_results.results_names.correl_partial_rank_io,
			    partialRankCorr, md);
  }
}

void SensAnalysisGlobal::
print_correlations(std::ostream& s, StringMultiArrayConstView cv_labels,
		   StringMultiArrayConstView div_labels,
		   StringMultiArrayConstView dsv_labels,
		   StringMultiArrayConstView drv_labels,
		   const StringArray& resp_labels) const
{
  // output correlation matrices

  if (!corrComputed) {
    Cout << "Correlation matrices not computed." << std::endl;
    return;
  }

  if( has_nan_or_inf(simpleCorr) ||
      has_nan_or_inf(partialCorr) ||
      has_nan_or_inf(simpleRankCorr) ||
      has_nan_or_inf(partialRankCorr) )
    s << "\n\nAt least one correlation coefficient is nan or inf. This " <<
      "commonly occurs when\ndiscrete variables (including histogram " <<
      "variables) are present, a response is\ncompletely insensitive to " <<
      "variables (response variance equal to 0), there are\nfewer samples " <<
      "than variables, or some samples are approximately collinear." <<
      std::endl;

  s << std::scientific << std::setprecision(5);

  if (resp_labels.size() != numFns) {
    Cerr << "Error: Number of response labels (" << resp_labels.size()
	 << ") passed to print_correlations not equal to number of output "
	 << "functions (" << numFns << ") in compute_correlations()."
	 << std::endl;
    abort_handler(-1);
  }

  size_t i, j, num_cv = cv_labels.size(), num_div = div_labels.size(),
    num_dsv = dsv_labels.size(), num_drv = drv_labels.size();
  if (num_cv+num_div+num_dsv+num_drv != numVars) {
    Cerr << "Error: Number of variable labels (" << num_cv+num_div+num_drv
	 << ") passed to print_correlations not equal to number of input "
	 << "variables (" << numVars << ") in compute_correlations()." << std::endl;
    abort_handler(-1);
  }

  int num_in_out = numVars + numFns;
  if (simpleCorr.numRows() == num_in_out &&
      simpleCorr.numCols() == num_in_out) {
    s << "\nSimple Correlation Matrix among all inputs and outputs:\n"
      << "             ";
    for (i=0; i<num_cv; ++i)
      s << std::setw(12) << cv_labels[i] << ' ';
    for (i=0; i<num_div; ++i)
      s << std::setw(12) << div_labels[i] << ' ';
    for (i=0; i<num_dsv; ++i)
      s << std::setw(12) << dsv_labels[i] << ' ';
    for (i=0; i<num_drv; ++i)
      s << std::setw(12) << drv_labels[i] << ' ';
    for (i=0; i<numFns; ++i)
      s << std::setw(12) << resp_labels[i] << ' ';
    s << '\n';
    for (i=0; i<num_in_out; ++i) {
      if (i<num_cv)
	s << std::setw(12) << cv_labels[i] << ' ';
      else if (i<num_cv+num_div)
	s << std::setw(12) << div_labels[i-num_cv] << ' ';
      else if (i<num_cv+num_div+num_dsv)
	s << std::setw(12) << dsv_labels[i-num_cv-num_div] << ' ';
      else if (i<numVars)
	s << std::setw(12) << drv_labels[i-num_cv-num_div-num_dsv] << ' ';
      else
	s << std::setw(12) << resp_labels[i-numVars] << ' ';
      for (j=0; j<=i; ++j)
	s << std::setw(12) << simpleCorr(i,j) << ' ';
      s << '\n';
    }
  }
  else if (simpleCorr.numRows() == numVars &&
	   simpleCorr.numCols() == numFns) {
    s << "\nSimple Correlation Matrix between input and output:\n"
      << "             ";
    for (j=0; j<numFns; ++j)
      s << std::setw(12) << resp_labels[j] << ' ';
    s << '\n';
    for (i=0; i<numVars; ++i) {
      if (i<num_cv)
	s << std::setw(12) << cv_labels[i] << ' ';
      else if (i<num_cv+num_div)
	s << std::setw(12) << div_labels[i-num_cv] << ' ';
      else if (i<num_cv+num_div+num_dsv)
	s << std::setw(12) << dsv_labels[i-num_cv-num_div] << ' ';
      else
	s << std::setw(12) << drv_labels[i-num_cv-num_div-num_dsv] << ' ';
      for (j=0; j<numFns; ++j)
	s << std::setw(12) << simpleCorr(i,j) << ' ';
      s << '\n';
    }
  }

  if (partialCorr.numRows() == numVars &&
      partialCorr.numCols() == numFns) {
    s << "\nPartial Correlation Matrix between input and output:\n"
      << "             ";
    for (j=0; j<numFns; ++j)
      s << std::setw(12) << resp_labels[j]<<' ';
    s << '\n';
    for (i=0; i<numVars; ++i) {
      if (i<num_cv)
	s << std::setw(12) << cv_labels[i] << ' ';
      else if (i<num_cv+num_div)
	s << std::setw(12) << div_labels[i-num_cv] << ' ';
      else if (i<num_cv+num_div+num_dsv)
	s << std::setw(12) << dsv_labels[i-num_cv-num_div] << ' ';
      else
	s << std::setw(12) << drv_labels[i-num_cv-num_div-num_dsv] << ' ';
      for (j=0; j<numFns; ++j)
	s << std::setw(12) << partialCorr(i,j) << ' ';
      s << '\n';
    }
  }
  //  This warning message has been supplanted by more generic tests for NaNs and
  //  Infs
  //if (numericalIssuesRaw)
  //  s << "\nThere may be some numerical issues associated with the calculation "
  //    << "of the \npartial correlation coefficients above.  This can be due to "
  //    << "very small \nnumbers of input samples, or to ill-conditioned matrices"
  //    << ", \nin situations where the partials are very close to zero, -1, or "
  //    << "+1.\n";

  if (simpleRankCorr.numRows() == num_in_out &&
      simpleRankCorr.numCols() == num_in_out) {
    s << "\nSimple Rank Correlation Matrix among all inputs and outputs:\n"
      << "             ";
    for (i=0; i<num_cv; ++i)
      s << std::setw(12) << cv_labels[i] << ' ';
    for (i=0; i<num_div; ++i)
      s << std::setw(12) << div_labels[i] << ' ';
    for (i=0; i<num_dsv; ++i)
      s << std::setw(12) << dsv_labels[i] << ' ';
    for (i=0; i<num_drv; ++i)
      s << std::setw(12) << drv_labels[i] << ' ';
    for (i=0; i<numFns; ++i)
      s << std::setw(12) << resp_labels[i] << ' ';
    s << '\n';
    for (i=0; i<num_in_out; ++i) {
      if (i<num_cv)
	s << std::setw(12) << cv_labels[i] << ' ';
      else if (i<num_cv+num_div)
	s << std::setw(12) << div_labels[i-num_cv] << ' ';
      else if (i<num_cv+num_div+num_dsv)
	s << std::setw(12) << dsv_labels[i-num_cv-num_div] << ' ';
      else if (i<numVars)
	s << std::setw(12) << drv_labels[i-num_cv-num_div-num_dsv] << ' ';
      else
	s << std::setw(12) << resp_labels[i-numVars] << ' ';
      for (j=0; j<=i; ++j)
	s << std::setw(12) << simpleRankCorr(i,j) << ' ';
      s << '\n';
    }
  }
  else if (simpleRankCorr.numRows() == numVars &&
	   simpleRankCorr.numCols() == numFns) {
    s << "\nSimple Rank Correlation Matrix between input and output:\n"
      << "             ";
    for (j=0; j<numFns; ++j)
      s << std::setw(12) << resp_labels[j] << ' ';
    s << '\n';
    for (i=0; i<numVars; ++i) {
      if (i<num_cv)
	s << std::setw(12) << cv_labels[i] << ' ';
      else if (i<num_cv+num_div)
	s << std::setw(12) << div_labels[i-num_cv] << ' ';
      else if (i<num_cv+num_div+num_dsv)
	s << std::setw(12) << dsv_labels[i-num_cv-num_div] << ' ';
      else
	s << std::setw(12) << drv_labels[i-num_cv-num_div-num_dsv] << ' ';
      for (j=0; j<numFns; ++j)
	s << std::setw(12) << simpleRankCorr(i,j) << ' ';
      s << '\n';
    }
  }

  if (partialRankCorr.numRows() == numVars &&
      partialRankCorr.numCols() == numFns) {
    s << "\nPartial Rank Correlation Matrix between input and output:\n"
      << "             ";
    for (j=0; j<numFns; ++j)
      s << std::setw(12) << resp_labels[j] << ' ';
    s << '\n';
    for (i=0; i<numVars; ++i) {
      if (i<num_cv)
	s << std::setw(12) << cv_labels[i] << ' ';
      else if (i<num_cv+num_div)
	s << std::setw(12) << div_labels[i-num_cv] << ' ';
      else if (i<num_cv+num_div+num_dsv)
	s << std::setw(12) << dsv_labels[i-num_cv-num_div] << ' ';
      else
	s << std::setw(12) << drv_labels[i-num_cv-num_div-num_dsv] << ' ';
      for (j=0; j<numFns; ++j)
	s << std::setw(12) << partialRankCorr(i,j) << ' ';
      s << '\n';
    }
  }

  //  This warning message has been supplanted by more generic tests for NaNs and
  //  Infs
  //if (numericalIssuesRank)
  //  s << "\nThere may be some numerical issues associated with the calculation "
  //    << "of the \npartial rank correlation coefficients above.  This can be "
  //    << "due to very small \nnumbers of input samples, or to ill-conditioned "
  //    << "matrices, \nin situations where the partials are very close to zero, "
  //    << "-1, or +1.\n";

  s << std::setprecision(write_precision)  // return to previous precision
    << std::endl;
}

} // namespace Dakota


// Copyright (c) 2015-2018 Daniel Cooke
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

#ifndef variational_bayes_mixture_model_hpp
#define variational_bayes_mixture_model_hpp

#include <array>
#include <vector>
#include <algorithm>
#include <numeric>
#include <iterator>
#include <cstddef>
#include <utility>
#include <cassert>
#include <limits>

#include <boost/optional.hpp>
#include <boost/math/special_functions/digamma.hpp>

#include "utils/maths.hpp"
#include "core/models/haplotype_likelihood_cache.hpp"

/**
 *
 * This file contains an implementation of the Variational Bayes mixture model
 * used by some genotype models. The notation follows the documentation.
 *
 */

namespace octopus { namespace model {

// Types needed for Variational Bayes model

struct VariationalBayesParameters
{
    double epsilon;
    unsigned max_iterations;
};

using ProbabilityVector    = std::vector<double>;
using LogProbabilityVector = std::vector<double>;

template <std::size_t K>
using VBAlpha = std::array<double, K>;
template <std::size_t K>
using VBAlphaVector = std::vector<VBAlpha<K>>;

class VBReadLikelihoodArray
{
public:
    using BaseType = HaplotypeLikelihoodCache::LikelihoodVector;

    VBReadLikelihoodArray() = default;

    explicit VBReadLikelihoodArray(const BaseType&);

    VBReadLikelihoodArray(const VBReadLikelihoodArray&)            = default;
    VBReadLikelihoodArray& operator=(const VBReadLikelihoodArray&) = default;
    VBReadLikelihoodArray(VBReadLikelihoodArray&&)                 = default;
    VBReadLikelihoodArray& operator=(VBReadLikelihoodArray&&)      = default;

    ~VBReadLikelihoodArray() = default;

    void operator=(const BaseType&);
    void operator=(std::reference_wrapper<const BaseType>);
    std::size_t size() const noexcept;
    BaseType::const_iterator begin() const noexcept;
    BaseType::const_iterator end() const noexcept;
    double operator[](const std::size_t n) const noexcept;

private:
    const BaseType* likelihoods;
};

template <std::size_t K>
using VBGenotype = std::array<VBReadLikelihoodArray, K>; // One element per haplotype in genotype (i.e. K)
template <std::size_t K>
using VBGenotypeVector = std::vector<VBGenotype<K>>; // Per element per genotype
template <std::size_t K>
using VBReadLikelihoodMatrix = std::vector<VBGenotypeVector<K>>; // One element per sample

template <std::size_t K>
using VBTau = std::array<double, K>; // One element per haplotype in genotype (i.e. K)
template <std::size_t K>
using VBResponsabilityVector = std::vector<VBTau<K>>; // One element per genotype
template <std::size_t K>
using VBResponsabilityMatrix = std::vector<VBResponsabilityVector<K>>; // One element per sample

template <std::size_t K>
struct VBLatents
{
    ProbabilityVector genotype_posteriors;
    LogProbabilityVector genotype_log_posteriors;
    VBAlphaVector<K> alphas;
    VBResponsabilityMatrix<K> responsabilities;
};

// Main VB method

template <std::size_t K>
std::pair<VBLatents<K>, double>
run_variational_bayes(const VBAlphaVector<K>& prior_alphas,
                      const LogProbabilityVector& genotype_log_priors,
                      const VBReadLikelihoodMatrix<K>& log_likelihoods,
                      const VariationalBayesParameters& params,
                      std::vector<LogProbabilityVector> seeds);

namespace detail {

using VBInverseLikelihood = std::vector<double>; // One element per genotype
using VBInverseGenotype = std::vector<VBInverseLikelihood>; // One element per read
template <std::size_t K>
using VBInverseGenotypeVector = std::array<VBInverseGenotype, K>; // One element per haplotype in genotype
template <std::size_t K>
using VBInverseReadLikelihoodMatrix = std::vector<VBInverseGenotypeVector<K>>; // One element per sample

template <std::size_t K>
auto invert(const VBGenotypeVector<K>& likelihoods)
{
    static_assert(K > 0, "K == 0");
    const auto num_genotypes = likelihoods.size();
    assert(num_genotypes > 0);
    const auto num_reads = likelihoods.front().front().size();
    VBInverseGenotypeVector<K> result {};
    for (std::size_t k {0}; k < K; ++k) {
        result[k] = VBInverseGenotype(num_reads, VBInverseLikelihood(num_genotypes));
        for (std::size_t n {0}; n < num_reads; ++n) {
            for (std::size_t g {0}; g < num_genotypes; ++g) {
                result[k][n][g] = likelihoods[g][k][n];
            }
        }
    }
    return result;
}

template <std::size_t K>
auto invert(const VBReadLikelihoodMatrix<K>& matrix)
{
    VBInverseReadLikelihoodMatrix<K> result {};
    result.reserve(matrix.size());
    std::transform(std::cbegin(matrix), std::cend(matrix), std::back_inserter(result),
                   [] (const auto& v) { return invert(v); });
    return result;
}

inline ProbabilityVector& exp(const LogProbabilityVector& log_probabilities, ProbabilityVector& result) noexcept
{
    std::transform(std::cbegin(log_probabilities), std::cend(log_probabilities), std::begin(result),
                   [] (const auto lp) noexcept { return std::exp(lp); });
    return result;
}

inline ProbabilityVector exp(const LogProbabilityVector& log_probabilities)
{
    ProbabilityVector result(log_probabilities.size());
    return exp(log_probabilities, result);
}

inline auto sum(const VBAlpha<2>& alpha) noexcept
{
    return alpha[0] + alpha[1];
}

inline auto sum(const VBAlpha<3>& alpha) noexcept
{
    return alpha[0] + alpha[1] + alpha[2];
}

template <std::size_t K>
auto sum(const VBAlpha<K>& alpha) noexcept
{
    return std::accumulate(std::cbegin(alpha), std::cend(alpha), 0.0);
}

template <typename T>
inline auto digamma_diff(const T a, const T b)
{
    using boost::math::digamma;
    return digamma(a) - digamma(b);
}

template <typename T>
inline T log_sum_exp(const std::array<T, 1>& logs)
{
    return logs[0];
}

template <typename T>
inline T log_sum_exp(const std::array<T, 2>& logs)
{
    return maths::log_sum_exp(logs[0], logs[1]);
}

template <typename T>
inline T log_sum_exp(const std::array<T, 3>& logs)
{
    return maths::log_sum_exp(logs[0], logs[1], logs[2]);
}

template <typename T, std::size_t K>
T log_sum_exp(const std::array<T, K>& logs)
{
    return maths::log_sum_exp(logs);
}

template <std::size_t K>
auto count_reads(const VBGenotypeVector<K>& likelihoods) noexcept
{
    return likelihoods[0][0].size();
}

template <std::size_t K>
auto count_reads(const VBInverseGenotypeVector<K>& likelihoods) noexcept
{
    return likelihoods[0].size();
}

template <std::size_t K>
auto marginalise(const ProbabilityVector& distribution, const VBGenotypeVector<K>& likelihoods,
                 const unsigned k, const std::size_t n) noexcept
{
    return std::inner_product(std::cbegin(distribution), std::cend(distribution),
                              std::cbegin(likelihoods), 0.0, std::plus<> {},
                              [k, n] (const auto p, const auto& haplotype_likelihoods) noexcept {
                                  return p * haplotype_likelihoods[k][n];
                              });
}

template <std::size_t K>
auto marginalise(const ProbabilityVector& distribution, const VBInverseGenotypeVector<K>& likelihoods,
                 const unsigned k, const std::size_t n) noexcept
{
    return std::inner_product(std::cbegin(distribution), std::cend(distribution), std::cbegin(likelihoods[k][n]), 0.0);
}

template <std::size_t K, typename VBLikelihoodVector_>
VBResponsabilityVector<K>
init_responsabilities(const VBAlpha<K>& prior_alphas,
                      const ProbabilityVector& genotype_probabilities,
                      const VBLikelihoodVector_& read_likelihoods)
{
    using T = typename VBAlpha<K>::value_type;
    std::array<T, K> al; // no need to keep recomputing this
    const auto a0 = sum(prior_alphas);
    for (unsigned k {0}; k < K; ++k) {
        al[k] = digamma_diff(prior_alphas[k], a0);
    }
    const auto N = count_reads(read_likelihoods);
    VBResponsabilityVector<K> result(N);
    std::array<T, K> ln_rho;
    for (std::size_t n {0}; n < N; ++n) {
        for (unsigned k {0}; k < K; ++k) {
            ln_rho[k] = al[k] + marginalise(genotype_probabilities, read_likelihoods, k, n);
        }
        const auto ln_rho_norm = log_sum_exp(ln_rho);
        for (unsigned k {0}; k < K; ++k) {
            result[n][k] = std::exp(ln_rho[k] - ln_rho_norm);
        }
    }
    return result;
}

template <std::size_t K, typename VBLikelihoodMatrix>
VBResponsabilityMatrix<K>
init_responsabilities(const VBAlphaVector<K>& prior_alphas,
                      const ProbabilityVector& genotype_probabilities,
                      const VBLikelihoodMatrix& read_likelihoods)
{
    const auto S = read_likelihoods.size(); // num samples
    VBResponsabilityMatrix<K> result{};
    result.reserve(S);
    for (std::size_t s {0}; s < S; ++s) {
        result.push_back(init_responsabilities(prior_alphas[s], genotype_probabilities, read_likelihoods[s]));
    }
    return result;
}

template <std::size_t K, typename VBLikelihoodVector_>
void update_responsabilities(VBResponsabilityVector<K>& result,
                             const VBAlpha<K>& posterior_alphas,
                             const ProbabilityVector& genotype_probabilities,
                             const VBLikelihoodVector_& read_likelihoods)
{
    using T = typename VBAlpha<K>::value_type;
    std::array<T, K> al;
    const auto a0 = sum(posterior_alphas);
    for (unsigned k {0}; k < K; ++k) {
        al[k] = digamma_diff(posterior_alphas[k], a0);
    }
    const auto N = count_reads(read_likelihoods);
    std::array<T, K> ln_rho;
    for (std::size_t n {0}; n < N; ++n) {
        for (unsigned k {0}; k < K; ++k) {
            ln_rho[k] = al[k] + marginalise(genotype_probabilities, read_likelihoods, k, n);
        }
        const auto ln_rho_norm = log_sum_exp(ln_rho);
        for (unsigned k {0}; k < K; ++k) {
            result[n][k] = std::exp(ln_rho[k] - ln_rho_norm);
        }
    }
}

// same as init_responsabilities but in-place
template <std::size_t K, typename VBLikelihoodMatrix>
void update_responsabilities(VBResponsabilityMatrix<K>& result,
                             const VBAlphaVector<K>& posterior_alphas,
                             const ProbabilityVector& genotype_probabilities,
                             const VBLikelihoodMatrix& read_likelihoods)
{
    const auto S = read_likelihoods.size();
    for (std::size_t s {0}; s < S; ++s) {
        update_responsabilities(result[s], posterior_alphas[s], genotype_probabilities, read_likelihoods[s]);
    }
}

template <std::size_t K>
auto sum(const VBResponsabilityVector<K>& taus, const unsigned k) noexcept
{
    return std::accumulate(std::cbegin(taus), std::cend(taus), 0.0,
                           [k] (const auto curr, const auto& tau) noexcept {
                               return curr + tau[k];
                           });
}

template <std::size_t K>
void update_alpha(VBAlpha<K>& alpha, const VBAlpha<K>& prior_alpha,
                  const VBResponsabilityVector<K>& taus) noexcept
{
    for (unsigned k {0}; k < K; ++k) {
        alpha[k] = prior_alpha[k] + sum(taus, k);
    }
}

template <std::size_t K>
void update_alphas(VBAlphaVector<K>& alphas, const VBAlphaVector<K>& prior_alphas,
                   const VBResponsabilityMatrix<K>& responsabilities) noexcept
{
    const auto S = alphas.size();
    assert(S == prior_alphas.size() && S == responsabilities.size());
    for (std::size_t s {0}; s < S; ++s) {
        update_alpha(alphas[s], prior_alphas[s], responsabilities[s]);
    }
}

template <std::size_t K>
auto marginalise(const VBResponsabilityVector<K>& responsabilities,
                 const VBGenotypeVector<K>& read_likelihoods,
                 const std::size_t g) noexcept
{
    double result {0};
    const auto N = read_likelihoods[0][0].size(); // num reads in sample s
    assert(responsabilities.size() == N);
    assert(responsabilities[0].size() == K && read_likelihoods[g].size() == K);
    for (unsigned k {0}; k < K; ++k) {
        const auto& k_likelihoods = read_likelihoods[g][k];
        for (std::size_t n {0}; n < N; ++n) {
            result += responsabilities[n][k] * k_likelihoods[n];
        }
    }
    return result;
}

template <std::size_t K>
auto marginalise(const VBResponsabilityMatrix<K>& responsabilities,
                 const VBReadLikelihoodMatrix<K>& read_likelihoods,
                 const std::size_t g) noexcept
{
    double result {0};
    const auto S = read_likelihoods.size(); // num samples
    assert(S == responsabilities.size());
    for (std::size_t s {0}; s < S; ++s) {
        result += marginalise(responsabilities[s], read_likelihoods[s], g);
    }
    return result;
}

template <std::size_t K>
void update_genotype_log_posteriors(LogProbabilityVector& result,
                                    const LogProbabilityVector& genotype_log_priors,
                                    const VBResponsabilityMatrix<K>& responsabilities,
                                    const VBReadLikelihoodMatrix<K>& read_likelihoods)
{
    const auto G = result.size();
    for (std::size_t g {0}; g < G; ++g) {
        result[g] = genotype_log_priors[g] + marginalise(responsabilities, read_likelihoods, g);
    }
    maths::normalise_logs(result);
}

inline auto max_change(const VBAlpha<2>& lhs, const VBAlpha<2>& rhs) noexcept
{
    return std::max(std::abs(lhs.front() - rhs.front()), std::abs(lhs.back() - rhs.back()));
}

inline auto max_change(const VBAlpha<3>& lhs, const VBAlpha<3>& rhs) noexcept
{
    return std::max({std::abs(lhs[0] - rhs[0]), std::abs(lhs[1] - rhs[1]), std::abs(lhs[2] - rhs[2])});
}

template <std::size_t K>
auto max_change(const VBAlpha<K>& lhs, const VBAlpha<K>& rhs) noexcept
{
    double result {0};
    for (std::size_t k {0}; k < K; ++k) {
        const auto curr = std::abs(lhs[k] - rhs[k]);
        if (curr > result) result = curr;
    }
    return result;
}

template <std::size_t K>
auto max_change(const VBAlphaVector<K>& prior_alphas, const VBAlphaVector<K>& posterior_alphas) noexcept
{
    const auto S = prior_alphas.size();
    assert(S == posterior_alphas.size());
    double result {0};
    for (std::size_t s {0}; s < S; ++s) {
        const auto curr = max_change(prior_alphas[s], posterior_alphas[s]);
        if (curr > result) result = curr;
    }
    return result;
}

template <std::size_t K>
std::pair<bool, double> check_convergence(const VBAlphaVector<K>& prior_alphas,
                                          const VBAlphaVector<K>& posterior_alphas,
                                          const double prev_max_change,
                                          const double epsilon) noexcept
{
    const auto new_max_change = max_change(prior_alphas, posterior_alphas);
    return std::make_pair(std::abs(new_max_change - prev_max_change) < epsilon, new_max_change);
}

template <std::size_t K>
auto entropy(const VBTau<K>& tau) noexcept
{
    return -std::accumulate(std::cbegin(tau), std::cend(tau), 0.0,
                            [] (const auto curr, const auto t) noexcept { return curr + (t * std::log(t)); });
}

// E [ln q(Z_s)]
template <std::size_t K>
auto sum_entropies(const VBResponsabilityVector<K>& taus) noexcept
{
    return std::accumulate(std::cbegin(taus), std::cend(taus), 0.0,
                           [] (const auto curr, const auto& tau) noexcept { return curr + entropy(tau); });
}

template <std::size_t K>
auto calculate_evidence_lower_bound(const VBAlphaVector<K>& prior_alphas,
                                    const VBAlphaVector<K>& posterior_alphas,
                                    const LogProbabilityVector& genotype_log_priors,
                                    const ProbabilityVector& genotype_posteriors,
                                    const LogProbabilityVector& genotype_log_posteriors,
                                    const VBResponsabilityMatrix<K>& taus,
                                    const VBReadLikelihoodMatrix<K>& log_likelihoods,
                                    const boost::optional<double> max_posterior_skip = boost::none)
{
    const auto G = genotype_log_priors.size();
    const auto S = log_likelihoods.size();
    double result {0};
    for (std::size_t g {0}; g < G; ++g) {
        if (!max_posterior_skip || genotype_posteriors[g] >= *max_posterior_skip) {
            auto w = genotype_log_priors[g] - genotype_log_posteriors[g];
            for (std::size_t s {0}; s < S; ++s) {
                const auto N_s = taus[s].size();
                for (std::size_t k {0}; k < K; ++k) {
                    for (std::size_t n {0}; n < N_s; ++n) {
                        w += taus[s][n][k] * log_likelihoods[s][g][k][n];
                    }
                }
            }
            result += genotype_posteriors[g] * w;
        }
    }
    for (std::size_t s {0}; s < S; ++s) {
        result += (maths::log_beta(posterior_alphas[s]) - maths::log_beta(prior_alphas[s]));
        result += sum_entropies(taus[s]);
    }
    return result;
}

// Main algorithm - single seed

// Starting iteration with given genotype_log_posteriors
template <std::size_t K, typename VBLikelihoodMatrix1, typename VBLikelihoodMatrix2>
VBLatents<K>
run_variational_bayes(const VBAlphaVector<K>& prior_alphas,
                      const LogProbabilityVector& genotype_log_priors,
                      const VBLikelihoodMatrix1& log_likelihoods1,
                      const VBLikelihoodMatrix2& log_likelihoods2,
                      LogProbabilityVector genotype_log_posteriors,
                      const VariationalBayesParameters& params)
{
    assert(!prior_alphas.empty());
    assert(!genotype_log_priors.empty());
    assert(!log_likelihoods1.empty());
    assert(log_likelihoods1.size() == log_likelihoods2.size());
    assert(prior_alphas.size() == log_likelihoods1.size()); // num samples
    assert(log_likelihoods1.front().size() == genotype_log_priors.size()); // num genotypes
    assert(params.max_iterations > 0);
    auto genotype_posteriors = exp(genotype_log_posteriors);
    auto posterior_alphas = prior_alphas;
    auto responsabilities = init_responsabilities<K>(posterior_alphas, genotype_posteriors, log_likelihoods2);
    assert(responsabilities.size() == log_likelihoods1.size()); // num samples
    auto prev_evidence = std::numeric_limits<double>::lowest();
    bool is_converged {};
    double max_change {};
    for (unsigned i {0}; i < params.max_iterations; ++i) {
        update_genotype_log_posteriors(genotype_log_posteriors, genotype_log_priors, responsabilities, log_likelihoods1);
        exp(genotype_log_posteriors, genotype_posteriors);
        update_alphas(posterior_alphas, prior_alphas, responsabilities);
        update_responsabilities(responsabilities, posterior_alphas, genotype_posteriors, log_likelihoods2);
        std::tie(is_converged, max_change) = check_convergence(prior_alphas, posterior_alphas, max_change, params.epsilon);
        if (is_converged) break;
        auto curr_evidence = calculate_evidence_lower_bound(prior_alphas, posterior_alphas, genotype_log_priors,
                                                            genotype_posteriors, genotype_log_posteriors, responsabilities,
                                                            log_likelihoods1, 1e-10);
        if (curr_evidence <= prev_evidence || (curr_evidence - prev_evidence) < params.epsilon) break;
        prev_evidence = curr_evidence;
    }
    return VBLatents<K> {
        std::move(genotype_posteriors), std::move(genotype_log_posteriors),
        std::move(posterior_alphas), std::move(responsabilities)
    };
}

// Not using inverted log likelihoods
template <std::size_t K>
VBLatents<K>
run_variational_bayes(const VBAlphaVector<K>& prior_alphas,
                      const LogProbabilityVector& genotype_log_priors,
                      const VBReadLikelihoodMatrix<K>& log_likelihoods,
                      LogProbabilityVector genotype_log_posteriors,
                      const VariationalBayesParameters& params)
{
    return run_variational_bayes(prior_alphas, genotype_log_posteriors, log_likelihoods,
                                 log_likelihoods, genotype_log_posteriors, params);
}

// Main algorithm - multiple seed

template <std::size_t K>
bool run_vb_with_matrix_inversion(const VBReadLikelihoodMatrix<K>& log_likelihoods,
                                  const VariationalBayesParameters& params,
                                  const std::vector<LogProbabilityVector>& seeds) noexcept
{
    return true;
}

template <std::size_t K>
std::vector<VBLatents<K>>
run_variational_bayes(const VBAlphaVector<K>& prior_alphas,
                      const LogProbabilityVector& genotype_log_priors,
                      const VBReadLikelihoodMatrix<K>& log_likelihoods,
                      const VariationalBayesParameters& params,
                      std::vector<LogProbabilityVector>&& seeds)
{
    std::vector<VBLatents<K>> result {};
    result.reserve(seeds.size());
    if (run_vb_with_matrix_inversion(log_likelihoods, params, seeds)) {
        const auto inverted_log_likelihoods = invert(log_likelihoods);
        for (auto& seed : seeds) {
            result.push_back(detail::run_variational_bayes(prior_alphas, genotype_log_priors,
                                                           log_likelihoods, inverted_log_likelihoods,
                                                           std::move(seed), params));
        }
    } else {
        for (auto& seed : seeds) {
            result.push_back(detail::run_variational_bayes(prior_alphas, genotype_log_priors,
                                                           log_likelihoods,
                                                           std::move(seed), params));
        }
    }
    return result;
}

// lower-bound calculation

template <std::size_t K>
auto calculate_evidence_lower_bound(const VBAlphaVector<K>& prior_alphas,
                                    const LogProbabilityVector& genotype_log_priors,
                                    const VBReadLikelihoodMatrix<K>& log_likelihoods,
                                    const VBLatents<K>& latents)
{
    return calculate_evidence_lower_bound(prior_alphas, latents.alphas, genotype_log_priors,
                                          latents.genotype_posteriors, latents.genotype_log_posteriors,
                                          latents.responsabilities, log_likelihoods);

}

template <std::size_t K>
std::pair<VBLatents<K>, double>
get_max_evidence_latents(const VBAlphaVector<K>& prior_alphas,
                         const LogProbabilityVector& genotype_log_priors,
                         const VBReadLikelihoodMatrix<K>& log_likelihoods,
                         std::vector<VBLatents<K>>&& latents)
{
    std::vector<double> seed_evidences(latents.size());
    std::transform(std::cbegin(latents), std::cend(latents), std::begin(seed_evidences),
                   [&] (const auto& seed_latents) {
                       return calculate_evidence_lower_bound(prior_alphas, genotype_log_priors, log_likelihoods, seed_latents);
                   });
    const auto max_itr = std::max_element(std::cbegin(seed_evidences), std::cend(seed_evidences));
    const auto max_idx = std::distance(std::cbegin(seed_evidences), max_itr);
    return std::make_pair(std::move(latents[max_idx]), *max_itr);
}

} // namespace detail

template <std::size_t K>
std::pair<VBLatents<K>, double>
run_variational_bayes(const VBAlphaVector<K>& prior_alphas,
                      const LogProbabilityVector& genotype_log_priors,
                      const VBReadLikelihoodMatrix<K>& log_likelihoods,
                      const VariationalBayesParameters& params,
                      std::vector<LogProbabilityVector> seeds)
{
    assert(!seeds.empty());
    auto latents = detail::run_variational_bayes(prior_alphas, genotype_log_priors, log_likelihoods, params, std::move(seeds));
    return detail::get_max_evidence_latents(prior_alphas, genotype_log_priors, log_likelihoods, std::move(latents));
}

inline VBReadLikelihoodArray::VBReadLikelihoodArray(const BaseType& underlying_likelihoods)
: likelihoods{std::addressof(underlying_likelihoods)} {}

inline void VBReadLikelihoodArray::operator=(const BaseType& other)
{
    likelihoods = std::addressof(other);
}

inline void VBReadLikelihoodArray::operator=(std::reference_wrapper<const BaseType> other)
{
    likelihoods = std::addressof(other.get());
}

inline std::size_t VBReadLikelihoodArray::size() const noexcept
{
    return likelihoods->size();
}

inline VBReadLikelihoodArray::BaseType::const_iterator VBReadLikelihoodArray::begin() const noexcept
{
    return likelihoods->begin();
}

inline VBReadLikelihoodArray::BaseType::const_iterator VBReadLikelihoodArray::end() const noexcept
{
    return likelihoods->end();
}

inline double VBReadLikelihoodArray::operator[](const std::size_t n) const noexcept
{
    return likelihoods->operator[](n);
}

} // namespace model
} // namespace octopus

#endif


//          Copyright Gavin Band 2008 - 2012.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <string>
#include <memory>
#include <set>
#include <map>
#include <utility>
#include <boost/bind.hpp>
#include <boost/function.hpp>
#include <boost/foreach.hpp>
#define foreach BOOST_FOREACH
#include <boost/regex.hpp>
#include <boost/math/distributions/hypergeometric.hpp>
#include "appcontext/CmdLineOptionProcessor.hpp"
#include "appcontext/ApplicationContext.hpp"
#include "appcontext/ProgramFlow.hpp"
#include "appcontext/get_current_time_as_string.hpp"
#include "genfile/FileUtils.hpp"
#include "genfile/utility.hpp"
#include "statfile/BuiltInTypeStatSource.hpp"
#include <Eigen/Core>

namespace globals {
	std::string const program_name = "overrep" ;
	std::string const program_version = "0.1" ;
}

struct OverrepOptions: public appcontext::CmdLineOptionProcessor {
	std::string get_program_name() const { return globals::program_name ; }

	void declare_options( appcontext::OptionProcessor& options ) {
		options.set_help_option( "-help" ) ;

		options.declare_group( "File handling options" ) ;
		options[ "-p" ]
			.set_description( "Specify the path of a file containing pathway definitions to load."
			 	" This file must have at least three columns; the first should be an identifier for the pathway (no whitespace),"
				" the second should be the pathway name, and the third should contain gene identifiers." )
			.set_takes_values_until_next_option()
			.set_minimum_multiplicity( 1 )
			.set_maximum_multiplicity( 100 )
		;

		options[ "-c" ]
			.set_description( "Specify the path of a file containing gene clusters to load." )
			.set_takes_single_value()
		;

		options[ "-u" ]
			.set_description( "Specify the path of a file containing a list of genes to treat as the gene \"universe\"." )
			.set_takes_single_value()
			.set_is_required()
		;

		options[ "-g" ]
			.set_description( "Specify the path of a file containing a list of genes to test with." )
			.set_takes_single_value()
		;

		options.declare_group( "Miscellaneous options" ) ;
		options[ "-P-value" ]
			.set_description( "Output lists of genes for all pathways getting this P-value or better." )
			.set_takes_single_value()
			.set_default_value( 0.000001 )
		;
		options[ "-analysis-name" ]
			.set_description( "Specify a name to label results from this analysis with.  (This applies to modules which store their results in a qcdb file.)" )
			.set_takes_single_value()
			.set_is_required() ;

		options[ "-intersect" ]
			.set_description( "Specify that the universe should only include genes that actuall occur in pathways." ) ;

		options[ "-log" ]
			.set_description( "Specify the path of the log file." )
			.set_takes_single_value()
			.set_default_value( "overrep.log" ) ;
	}
} ;

// Fisher's exact test between categories
//               red           black
//     good |     a      |       b        |
//      bad |     c      |       d        |
//
// Under the null that both rows are distributed the same, a is hypergeometrically
// distributed.  Fishers' p-value is then the mass under the hypergeometric distribution
// of all values of a greater than or equal to the one given that are consistent with
// the given margins of the table.
struct FishersExactTest {
	FishersExactTest( Eigen::Matrix2d const& matrix ):
		m_matrix( matrix ),
		m_distribution( m_matrix.col( 0 ).sum(), m_matrix.row( 0 ).sum(), m_matrix.sum() )
	{}

	double get_OR() const {
		return ( m_matrix(0,0) * m_matrix(1,1) ) / ( m_matrix(0,1)*m_matrix(1,0) ) ;
	}

	std::pair< double, double > get_confidence_interval() const {
		assert(0) ;
	}

	double get_pvalue() const {
		using boost::math::cdf ;
		using boost::math::complement ;

		if( m_matrix(0,0) == 0 ) {
			return 1 ;
		} else {
			return cdf( complement( m_distribution, m_matrix( 0, 0 ) - 1.0 )) ;
		}
	}

	private:
		Eigen::Matrix2d const m_matrix ;
		boost::math::hypergeometric m_distribution ;
} ;

void load_list_of_strings( std::string const& filename, boost::function< void ( std::string const& ) > setter ) {
	std::auto_ptr< std::istream > f = genfile::open_text_file_for_input( filename ) ;
	std::string element ;
	while( (*f) >> element ) {
		setter( element ) ;
	}
}

void load_pathways_list(
	statfile::BuiltInTypeStatSource& source,
	boost::function< void ( std::string const&, std::string const& ) > set_pathway_name,
	boost::function< void ( std::string const&, std::string const& ) > set_pathway_gene
) {
	std::string pathway_id ;
	std::string pathway_name ;
	std::string gene_id ;
	while( source >> pathway_id ) {
		source >> pathway_name >> gene_id >> statfile::ignore_all() ;
		set_pathway_name( pathway_id, pathway_name ) ;
		set_pathway_gene( pathway_id, gene_id ) ;
	}
}

void load_clusters(
	statfile::BuiltInTypeStatSource& source,
	boost::function< void ( std::string const&, std::string const& ) > set_cluster_member = boost::function< void ( std::string const&, std::string const& ) >(),
	boost::function< void ( std::string const&, std::string const& ) > set_cluster_comment = boost::function< void ( std::string const&, std::string const& ) >()
) {
	std::string cluster_id ;
	std::string cluster_comment ;
	std::string gene_regexp ;
	while( source >> cluster_id ) {
		source >> gene_regexp >> cluster_comment >> statfile::ignore_all() ;
		if( set_cluster_comment ) {
			set_cluster_comment( cluster_id, cluster_comment ) ;
		}
		if( set_cluster_member ) {
			set_cluster_member( cluster_id, gene_regexp ) ;
		}
	}
}

namespace impl {
	struct MapSetter {
		typedef std::string A ;
		typedef std::string B ;
		typedef std::map< A, B > Map ;
		MapSetter( Map& map ): m_map( &map ) {} ;
		MapSetter( MapSetter const& other ): m_map( other.m_map ) {} ;
		MapSetter& operator=( MapSetter const& other ) {
			m_map = other.m_map ;
			return *this ;
		}

		void operator()( std::string const& left, std::string const& right ) const {
			(*m_map)[ left ] = right ;
		}

	private:
		Map* m_map ;
	} ;

	struct MapSetSetter {
		typedef std::string A ;
		typedef std::set< std::string > B ;
		typedef std::map< A, B > Map ;
		MapSetSetter( Map& map ): m_map( &map ) {} ;
		MapSetSetter( MapSetSetter const& other ): m_map( other.m_map ) {} ;
		MapSetSetter& operator=( MapSetSetter const& other ) {
			m_map = other.m_map ;
			return *this ;
		}

		void operator()( std::string const& left, std::string const& right ) const {
			(*m_map)[ left ].insert( right ) ;
		}

	private:
		mutable Map* m_map ;
	} ;

	template< typename Container >
	void insert_into( typename Container::value_type const& value, Container* X ) {
		X->insert( value ) ;
	}

	template< typename Container >
	bool in( typename Container::value_type const& value, Container const& X ) {
		return X.find( value ) != X.end() ;
	}
}

struct OverrepApplication: public appcontext::ApplicationContext {
public:
	OverrepApplication( int argc, char **argv ):
		appcontext::ApplicationContext(
			globals::program_name,
			globals::program_version,
			std::auto_ptr< appcontext::OptionProcessor >( new OverrepOptions ),
			argc,
			argv,
			"-log"
		)
	{}

	void process() {
		unsafe_process() ;
	}

private:
	typedef std::set< std::string > StringSet ;
	typedef std::map< std::string, std::string > StringStringMap ;
	typedef std::map< std::string, std::set< std::string > > StringStringSetMap ;

	void unsafe_process() {
		load_data() ;
		m_cluster_mapping = get_cluster_mapping() ;
		map_genes_through_clusters( m_cluster_mapping ) ;

		summarise() ;

		run_tests() ;
	}

	void load_data() {
		StringStringMap pathway_names ;
		StringStringSetMap pathway_members ;
		StringStringSetMap clusters ;
		StringStringMap cluster_descriptions ;
		StringSet universe ;
		StringSet test_genes ;

		assert( options().check( "-u" ) ) ;
		if( options().check( "-u" )) {
			load_list_of_strings(
				options().get< std::string >( "-u" ),
				boost::bind(
					&impl::insert_into< StringSet >,
					_1,
					&universe
				)
			) ;

			get_ui_context().logger()
				<< "I loaded " << universe.size() << " universe genes.  First few are:\n" ;
			StringSet::const_iterator
				i = universe.begin(),
				end_i = universe.end() ;
			for( std::size_t count = 0; count < 5 && i != end_i; ++count, ++i ) {
				get_ui_context().logger() << "  " << *i << "\n" ;
			}
		}

		if( options().check( "-c" )) {
			statfile::BuiltInTypeStatSource::UniquePtr source = statfile::BuiltInTypeStatSource::open(
				options().get< std::string >( "-c" )
			) ;
			load_clusters(
				*source,
				impl::MapSetSetter( clusters ),
				impl::MapSetter( cluster_descriptions )
			) ;
			get_ui_context().logger() << "Loaded " << clusters.size() << " clusters:\n" ;
			foreach( StringStringSetMap::value_type const& elt, clusters ) {
				get_ui_context().logger()
					<< "  " << elt.first << ": " ;
				foreach( StringSet::value_type const& value, elt.second ) {
					get_ui_context().logger() << value << " " ;
				}
				get_ui_context().logger() << "\n" ;
			}
		}

		if( options().check( "-p" )) {
			std::vector< std::string > elts = options().get_values< std::string >( "-p" ) ;
			foreach( std::string const& elt, elts ) {
				statfile::BuiltInTypeStatSource::UniquePtr source = statfile::BuiltInTypeStatSource::open(
					genfile::wildcard::find_files_by_chromosome( elt )
				) ;
				load_pathways_list(
					*source,
					impl::MapSetter( pathway_names ),
					impl::MapSetSetter( pathway_members )
				) ;
			}

			typedef std::pair< std::string, std::set< std::string > > StringStringSet ;
			if( options().check( "-intersect" )) {
				StringSet pathway_universe ;
				foreach( StringStringSet const& entry, pathway_members ) {
					pathway_universe.insert( entry.second.begin(), entry.second.end() ) ;
				}
				get_ui_context().logger() << "Created pathway universe of " << universe.size() << " genes based on all genes in pathways.\n" ;
				std::size_t const N = universe.size() ;
				universe = genfile::utility::intersect( universe, pathway_universe ) ;
				get_ui_context().logger() << "Formed intersection with universe (" << universe.size() << " of " << N << " in original universe.\n" ;

			}

			std::map< std::string, StringSet > universe_pathway_members ;
			foreach( StringStringSet const& entry, pathway_members ) {
				universe_pathway_members[ entry.first ] = genfile::utility::intersect( entry.second, universe ) ;
			}

			get_ui_context().logger()
				<< "I loaded " << pathway_names.size() << " pathways.  First few are:\n" ;
			std::map< std::string, std::string >::const_iterator
				pathway_i = pathway_names.begin(),
				pathway_end = pathway_names.end() ;
			for( std::size_t count = 0; count < 5 && pathway_i != pathway_end; ++count, ++pathway_i ) {
				get_ui_context().logger() << "  " << pathway_i->first << ": " << pathway_i->second
					<< " (" << pathway_members[pathway_i->first].size()
					<< " genes of which "
					<< universe_pathway_members[pathway_i->first].size()
					<<" are in the universe)\n" ;
			}

			pathway_members = universe_pathway_members ;
		}

		if( options().check( "-g" )) {
			load_list_of_strings(
				options().get< std::string >( "-g" ),
				boost::bind(
					impl::insert_into< StringSet >,
					_1,
					&test_genes
				)
			) ;

			get_ui_context().logger()
				<< "I loaded " << test_genes.size() << " test genes " ;
			test_genes = genfile::utility::intersect( test_genes, universe ) ;
			get_ui_context().logger()
				<< "of which " << test_genes.size() << " are in the universe.  First few are:\n" ;
			StringSet::const_iterator
				i = test_genes.begin(),
				end_i = test_genes.end() ;
			for( std::size_t count = 0; count < 5 && i != end_i; ++count, ++i ) {
				get_ui_context().logger() << "  " << *i << "\n" ;
			}
		}

		m_pathway_names = pathway_names ;
		m_pathway_members = pathway_members ;
		m_clusters = clusters ;
		m_cluster_descriptions = cluster_descriptions ;
		m_universe = universe ;
		m_test_genes = test_genes ;
	}

	StringStringMap get_cluster_mapping() {
		get_ui_context().logger() << "Computing gene->cluster mapping...\n" ;
		StringStringMap result ;
		foreach( StringStringSetMap::value_type const& cluster, m_clusters ) {
			foreach( std::string const& regex_string, cluster.second ) {
				boost::regex regex( regex_string ) ;
				StringStringMap const& this_map = get_cluster_mapping( regex, cluster.first ) ;
				result.insert( this_map.begin(), this_map.end() ) ;
			}
		}

		return result ;
	}

	StringStringMap get_cluster_mapping( boost::regex const& regex, std::string const& replacement ) {
		StringStringMap result ;
		get_cluster_mapping( m_universe, regex, replacement, boost::bind( impl::insert_into< StringStringMap >, _1, &result )) ;
		get_cluster_mapping( m_test_genes, regex, replacement, boost::bind( impl::insert_into< StringStringMap >, _1, &result )) ;
		foreach( StringStringSetMap::value_type& value, m_pathway_members ) {
			get_cluster_mapping( value.second, regex, replacement, boost::bind( impl::insert_into< StringStringMap >, _1, &result )) ;
		}
		return result ;
	}

	void get_cluster_mapping(
		StringSet& values,
		boost::regex const& regex,
		std::string const& replacement,
		boost::function< void ( std::pair< std::string, std::string > const& ) > output
	) {
		StringSet::iterator i = values.begin(), end_i = values.end() ;
		for( ; i != end_i; ++i ) {
			std::string value = *i ;
			value = boost::regex_replace( value, regex, replacement ) ;
			if( value != *i ) {
				output( std::make_pair( *i, value ) ) ;
			}
		}
	}

	void map_genes_through_clusters( StringStringMap const& mapping ) {
		get_ui_context().logger() << "Mapping genes through clusters...\n" ;
		std::size_t n = m_universe.size() ;
		m_universe = map_genes_through_clusters( mapping, m_universe ) ;
		get_ui_context().logger() << "Mapped " << n << " genes to " << m_universe.size() << " in universe.\n" ;
		n = m_test_genes.size() ;
		m_test_genes = map_genes_through_clusters( mapping, m_test_genes ) ;
		get_ui_context().logger() << "Mapped " << n << " genes to " << m_test_genes.size() << " in test genes.\n" ;
		foreach( StringStringSetMap::value_type& value, m_pathway_members ) {
			value.second = map_genes_through_clusters( mapping, value.second ) ;
		}
	}

	StringSet map_genes_through_clusters( StringStringMap const& mapping, StringSet const& genes ) {
		StringSet result ;
		foreach( StringSet::value_type const& value, genes ) {
			StringStringMap::const_iterator where = mapping.find( value ) ;
			if( where != mapping.end() ) {
				result.insert( where->second ) ;
			} else {
				result.insert( value ) ;
			}
        }
		return result ;
	}

	void summarise() {
		get_ui_context().logger() << "\n-------------------------\n\n" ;
		if( m_clusters.size() > 0 ) {
			get_ui_context().logger() << std::setw( 12 ) << "Cluster mapping:\n" ;
            StringStringSetMap backMap ;
			foreach( StringStringMap::value_type const& key_value, m_cluster_mapping ) {
                backMap[ key_value.second ].insert( key_value.first ) ;
			}
            foreach( StringStringSetMap::value_type const& k, backMap ) {
                get_ui_context().logger() << std::setw( 12 ) << k.first << ":" ;
				std::size_t count = 0 ;
				for( StringSet::const_iterator i = k.second.begin(); i != k.second.end(); ++i, ++count ) {
                    get_ui_context().logger() << " " << *i ;
					if( count == 10 ) {
						get_ui_context().logger() << " (+ " << k.second.size() - 10 << " others)..." ;
						break ;
					}
                }
                get_ui_context().logger() << "\n" ;
            }
		}
#if 0
		if( m_clusters.size() > 0 ) {
			get_ui_context().logger() << std::setw( 12 ) << "Cluster mapping:\n" ;
			foreach( StringStringMap::value_type const& key_value, m_cluster_mapping ) {
				get_ui_context().logger() << std::setw( 12 ) << key_value.first << ": "  << key_value.second << "\n" ;
			}
		}
#endif
		get_ui_context().logger() << std::setw( 12 ) << "Test genes:" << "  " << m_test_genes.size() << " genes\n" ;
		get_ui_context().logger() << std::setw( 12 ) << "Pathways:" << "  " << m_pathway_names.size() << " pathways\n" ;
		get_ui_context().logger() << std::setw( 12 ) << "Universe:" << "  " << m_universe.size() << " genes\n" ;
	}

	void run_tests() {
		// table is
		//                 in pathway | not in pathway |
		//     test gene |     a      |       b        |
		// Not test gene |     c      |       d        |
		//
		Eigen::Matrix2d table ;

		using std::setw ;
		std::string const tab = "\t" ;
		std::cout
		//get_ui_context().logger()
			<< "analysis" << tab
			<< "pathway id" << tab
			<< "pathway name" << tab
			<< "hits in pathway" << tab
			<< "hits not in pathway" << tab
			<< "nonhits in pathway" << tab
			<< "nonhits not in pathway" << tab
			<< "total in pathway" << tab
			<< "total genes" << tab
			<< "sample odds ratio" << tab
			<< "fishers exact test p value" << tab
			<< "ids.of.hits.in.pathway"
			<< "\n" ;

		std::string const analysis = options().get< std::string >( "-analysis-name" ) ;
		double const pvalue_threshhold = options().get< double >( "-P-value" ) ;
		std::ostringstream hit_genes_in_pathway ;
		foreach( StringStringSetMap::value_type const& pathway, m_pathway_members ) {
			hit_genes_in_pathway.str( "" ) ;
			table = Eigen::Matrix2d::Zero() ;

			foreach( std::string const& value, m_universe ) {
				int i = 0, j = 0 ;
				if( impl::in( value, m_test_genes )) {
					i = 0 ;
				} else {
					i = 1 ;
				}

				if( impl::in( value, pathway.second ) ) {
					j = 0 ;
				} else {
					j = 1 ;
				}

				++table( i, j ) ;

				if( i == 0 && j == 0 ) {
					if( hit_genes_in_pathway.str().size() == 0 ) {
						hit_genes_in_pathway << value ;
					}
					else {
						hit_genes_in_pathway << "," << value ;
					}
				}
			}

			//get_ui_context().logger()
			std::cout
				<< analysis << tab
				<< pathway.first << tab
				<< "\"" << m_pathway_names[ pathway.first ] << "\"" << tab
				<< table( 0, 0 ) << tab
				<< table( 0, 1 ) << tab
				<< table( 1, 0 ) << tab
				<< table( 1, 1 ) << tab
				<< table.col( 0 ).sum() << tab
				<< table.sum() << tab ;
			try {
				FishersExactTest test( table ) ;
				//get_ui_context().logger()
				std::cout
					<< test.get_OR() << tab << test.get_pvalue() ;
				if( test.get_pvalue() <= pvalue_threshhold ) {
					std::cout << tab << hit_genes_in_pathway.str() ;
				} else {
					std::cout << tab << "NA" ;
				}
			}
			catch( std::exception const& ) {
				// get_ui_context().logger()
				std::cout
					<< "NA\tNA\t" ;
			}
			std::cout << "\n" ;
		}
	}

private:
	StringStringMap m_pathway_names ;
	StringStringSetMap m_pathway_members ;
	StringStringSetMap m_clusters ;
	StringStringMap m_cluster_descriptions ;
	StringStringMap m_cluster_mapping ;
	StringSet m_universe ;
	StringSet m_test_genes ;
} ;

int main( int argc, char **argv ) {
	try {
		OverrepApplication app( argc, argv ) ;
		app.process() ;
	}
	catch( appcontext::HaltProgramWithReturnCode const& e ) {
		return e.return_code() ;
	}
	return 0 ;
}


// Copyright (c) 2014-2018, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
//    conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
//    of conditions and the following disclaimer in the documentation and/or other
//    materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
//    used to endorse or promote products derived from this software without specific
//    prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// The LWMA-4 Difficulty algorithm as implemented below is under a separate license
// to the rest of this file.
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers

#include <boost/algorithm/clamp.hpp>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <vector>

#include "common/int-util.h"
#include "crypto/hash.h"
#include "cryptonote_config.h"
#include "difficulty.h"

#undef MONERO_DEFAULT_LOG_CATEGORY
#define MONERO_DEFAULT_LOG_CATEGORY "difficulty"

namespace cryptonote {

  using std::size_t;
  using std::uint64_t;
  using std::vector;

#if defined(__x86_64__)
  static inline void mul(uint64_t a, uint64_t b, uint64_t &low, uint64_t &high) {
    low = mul128(a, b, &high);
  }

#else

  static inline void mul(uint64_t a, uint64_t b, uint64_t &low, uint64_t &high) {
    // __int128 isn't part of the standard, so the previous function wasn't portable. mul128() in Windows is fine,
    // but this portable function should be used elsewhere. Credit for this function goes to latexi95.

    uint64_t aLow = a & 0xFFFFFFFF;
    uint64_t aHigh = a >> 32;
    uint64_t bLow = b & 0xFFFFFFFF;
    uint64_t bHigh = b >> 32;

    uint64_t res = aLow * bLow;
    uint64_t lowRes1 = res & 0xFFFFFFFF;
    uint64_t carry = res >> 32;

    res = aHigh * bLow + carry;
    uint64_t highResHigh1 = res >> 32;
    uint64_t highResLow1 = res & 0xFFFFFFFF;

    res = aLow * bHigh;
    uint64_t lowRes2 = res & 0xFFFFFFFF;
    carry = res >> 32;

    res = aHigh * bHigh + carry;
    uint64_t highResHigh2 = res >> 32;
    uint64_t highResLow2 = res & 0xFFFFFFFF;

    //Addition

    uint64_t r = highResLow1 + lowRes2;
    carry = r >> 32;
    low = (r << 32) | lowRes1;
    r = highResHigh1 + highResLow2 + carry;
    uint64_t d3 = r & 0xFFFFFFFF;
    carry = r >> 32;
    r = highResHigh2 + carry;
    high = d3 | (r << 32);
  }

#endif

  static inline bool cadd(uint64_t a, uint64_t b) {
    return a + b < a;
  }

  static inline bool cadc(uint64_t a, uint64_t b, bool c) {
    return a + b < a || (c && a + b == (uint64_t) -1);
  }

  bool check_hash(const crypto::hash &hash, difficulty_type difficulty) {
    uint64_t low, high, top, cur;
    // First check the highest word, this will most likely fail for a random hash.
    mul(swap64le(((const uint64_t *) &hash)[3]), difficulty, top, high);
    if (high != 0) {
      return false;
    }
    mul(swap64le(((const uint64_t *) &hash)[0]), difficulty, low, cur);
    mul(swap64le(((const uint64_t *) &hash)[1]), difficulty, low, high);
    bool carry = cadd(cur, low);
    cur = high;
    mul(swap64le(((const uint64_t *) &hash)[2]), difficulty, low, high);
    carry = cadc(cur, low, carry);
    carry = cadc(high, top, carry);
    return !carry;
  }

    // LWMA-4 difficulty algorithm
    // Copyright (c) 2017-2018 Zawy
    // MIT license
    //
    // Permission is hereby granted, free of charge, to any person obtaining a copy
    // of this software and associated documentation files (the "Software"), to
    // deal in the Software without restriction, including without limitation the
    // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    // sell copies of the Software, and to permit persons to whom the Software is
    // furnished to do so, subject to the following conditions:
    //
    // The above copyright notice and this permission notice shall be included in
    // all copies or substantial portions of the Software.
    //
    // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
    // WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
    // OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    //
    // https://github.com/zawy12/difficulty-algorithms/issues/3
    // See commented version for explanations & required config file changes. Fix FTL and MTP!
    difficulty_type next_difficulty(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) {
        uint64_t  T = target_seconds;
        uint64_t  N = DIFFICULTY_WINDOW; // N=45, 60, and 90 for T=600, 120, 60.
        uint64_t  L(0), ST(0), next_D, prev_D, avg_D, i;

        assert(timestamps.size() == cumulative_difficulties.size() && timestamps.size() <= N+1 );

        // If it's a new coin, do startup code. Do not remove in case other coins copy your code.
        uint64_t difficulty_guess = 600;
        if (timestamps.size() <= 12 ) {   return difficulty_guess;   }
        if ( timestamps.size()  < N +1 ) { N = timestamps.size()-1;  }

        // If hashrate/difficulty ratio after a fork is < 1/3 prior ratio, hardcode D for N+1 blocks after fork.
        // This will also cover up a very common type of backwards-incompatible fork.
        // difficulty_guess = 100000; //  Dev may change.  Guess low than anything expected.
        // if ( height <= UPGRADE_HEIGHT + 1 + N ) { return difficulty_guess;  }

        // Safely convert out-of-sequence timestamps into > 0 solvetimes.
        std::vector<uint64_t>TS(N+1);
        TS[0] = timestamps[0];
        for ( i = 1; i <= N; i++) {
            if ( timestamps[i]  > TS[i-1]  ) {   TS[i] = timestamps[i];  }
            else {  TS[i] = TS[i-1];   }
        }

        for ( i = 1; i <= N; i++) {
            // Temper long solvetime drops if they were preceded by 3 or 6 fast solves.
            if ( i > 4 && TS[i]-TS[i-1] > 5*T  && TS[i-1] - TS[i-4] < (14*T)/10 ) {   ST = 2*T; }
            else if ( i > 7 && TS[i]-TS[i-1] > 5*T  && TS[i-1] - TS[i-7] < 4*T ) {   ST = 2*T; }
            else { // Assume normal conditions, so get ST.
                // LWMA drops too much from long ST, so limit drops with a 5*T limit
                ST = std::min(5*T ,TS[i] - TS[i-1]);
            }
            L +=  ST * i ;
        }
        if (L < N*N*T/20 ) { L =  N*N*T/20; }
        avg_D = ( cumulative_difficulties[N] - cumulative_difficulties[0] )/ N;

        // Prevent round off error for small D and overflow for large D.
        if (avg_D > 2000000*N*N*T) {
            next_D = (avg_D/(200*L))*(N*(N+1)*T*97);
        }
        else {    next_D = (avg_D*N*(N+1)*T*97)/(200*L);    }

        prev_D =  cumulative_difficulties[N] - cumulative_difficulties[N-1] ;

        // Apply 10% jump rule.
        if (( TS[N] - TS[N-1] < (2*T)/10 ) ||
            ( TS[N] - TS[N-2] < (5*T)/10 ) ||
            ( TS[N] - TS[N-3] < (8*T)/10 ))
        {
            next_D = std::max( next_D, std::min( (prev_D*110)/100, (105*avg_D)/100 ) );
        }
        // Make all insignificant digits zero for easy reading.
        i = 1000000000;
        while (i > 1) {
            if ( next_D > i*100 ) { next_D = ((next_D+i/2)/i)*i; break; }
            else { i /= 10; }
        }
        return  next_D;
    }
}


/*
 Copyright (C) 2021 Quaternion Risk Management Ltd
 All rights reserved.

 This file is part of ORE, a free-software/open-source library
 for transparent pricing and risk analysis - http://opensourcerisk.org

 ORE is free software: you can redistribute it and/or modify it
 under the terms of the Modified BSD License.  You should have received a
 copy of the license along with this program.
 The license is also available online at <http://opensourcerisk.org>

 This program is distributed on the basis that it will form a useful
 contribution to risk analytics and model standardisation, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
*/

#include <qle/instruments/crossccyfixfloatmtmresetswap.hpp>
#include <qle/cashflows/floatingratefxlinkednotionalcoupon.hpp>
#include <qle/cashflows/fixedratefxlinkednotionalcoupon.hpp>

#include <boost/make_shared.hpp>
#include <ql/cashflows/fixedratecoupon.hpp>
#include <ql/cashflows/iborcoupon.hpp>
#include <ql/cashflows/simplecashflow.hpp>

using namespace QuantLib;

namespace QuantExt {

CrossCcyFixFloatMtMResetSwap::CrossCcyFixFloatMtMResetSwap(
    Real nominal, const Currency& fixedCurrency, const Schedule& fixedSchedule,
    Rate fixedRate, const DayCounter& fixedDayCount, const BusinessDayConvention& fixedPaymentBdc,
    Natural fixedPaymentLag, const Calendar& fixedPaymentCalendar, const Currency& floatCurrency,
    const Schedule& floatSchedule, const boost::shared_ptr<IborIndex>& floatIndex, Spread floatSpread,
    const BusinessDayConvention& floatPaymentBdc, Natural floatPaymentLag, const Calendar& floatPaymentCalendar,
    const boost::shared_ptr<FxIndex>& fxIdx, bool resetsOnFloatLeg, bool receiveFixed)
    : CrossCcySwap(3), nominal_(nominal), fixedCurrency_(fixedCurrency),
    fixedSchedule_(fixedSchedule), fixedRate_(fixedRate), fixedDayCount_(fixedDayCount),
    fixedPaymentBdc_(fixedPaymentBdc), fixedPaymentLag_(fixedPaymentLag), fixedPaymentCalendar_(fixedPaymentCalendar),
    floatCurrency_(floatCurrency), floatSchedule_(floatSchedule), floatIndex_(floatIndex),
    floatSpread_(floatSpread), floatPaymentBdc_(floatPaymentBdc),
    floatPaymentLag_(floatPaymentLag), floatPaymentCalendar_(floatPaymentCalendar),
    fxIndex_(fxIdx), resetsOnFloatLeg_(resetsOnFloatLeg), receiveFixed_(receiveFixed) {

    registerWith(floatIndex_);
    registerWith(fxIndex_);
    initialize();
}

void CrossCcyFixFloatMtMResetSwap::initialize() {

    // if resets on floating leg, set notional to zero, else the fixed
    Real floatNotional, fixedNotional;
    if (resetsOnFloatLeg_) {
        floatNotional = 0.0;
        fixedNotional = nominal_;
    } else {
        floatNotional = nominal_;
        fixedNotional = 0.0;
    }

    // Build the float leg
    Leg floatLeg = IborLeg(floatSchedule_, floatIndex_)
        .withNotionals(floatNotional)
        .withSpreads(floatSpread_)
        .withPaymentAdjustment(floatPaymentBdc_)
        .withPaymentLag(floatPaymentLag_)
        .withPaymentCalendar(floatPaymentCalendar_);

    // Register with each floating rate coupon
    for (Leg::const_iterator it = floatLeg.begin(); it < floatLeg.end(); ++it)
        registerWith(*it);

    // Build the fixed rate leg
    Leg fixedLeg = FixedRateLeg(fixedSchedule_)
        .withNotionals(fixedNotional)
        .withCouponRates(fixedRate_, fixedDayCount_)
        .withPaymentAdjustment(fixedPaymentBdc_)
        .withPaymentLag(fixedPaymentLag_)
        .withPaymentCalendar(fixedPaymentCalendar_);

    if (resetsOnFloatLeg_) {
        // Notional exchanges for fixed leg
        // Initial notional exchange
        Date aDate = fixedSchedule_.dates().front();
        aDate = fixedPaymentCalendar_.adjust(aDate, fixedPaymentBdc_);
        boost::shared_ptr<CashFlow> aCashflow = boost::make_shared<SimpleCashFlow>(-fixedNotional, aDate);
        fixedLeg.insert(fixedLeg.begin(), aCashflow);

        // Final notional exchange
        aDate = fixedLeg.back()->date();
        aCashflow = boost::make_shared<SimpleCashFlow>(fixedNotional, aDate);
        fixedLeg.push_back(aCashflow);

        // resetting floating leg
        for (Size j = 0; j < floatLeg.size(); ++j) {
            boost::shared_ptr<FloatingRateCoupon> coupon = boost::dynamic_pointer_cast<FloatingRateCoupon>(floatLeg[j]);
            Date fixingDate = fxIndex_->fixingCalendar().advance(coupon->accrualStartDate(),
                -static_cast<Integer>(fxIndex_->fixingDays()), Days);
            boost::shared_ptr<FloatingRateFXLinkedNotionalCoupon> fxLinkedCoupon =
                boost::make_shared<FloatingRateFXLinkedNotionalCoupon>(fixingDate, fixedNotional, fxIndex_, coupon);
            floatLeg[j] = fxLinkedCoupon;
        }

        // now build a separate leg to store the resetting notionals
        receiveFixed_ ? payer_[2] = -1.0 : payer_[2] = +1.0;
        currencies_[2] = floatCurrency_;
        for (Size j = 0; j < floatLeg.size(); j++) {
            boost::shared_ptr<Coupon> c = boost::dynamic_pointer_cast<Coupon>(floatLeg[j]);
            QL_REQUIRE(c, "Resetting XCCY - expected Coupon");
            // build a pair of notional flows, one at the start and one at the end of
            // the accrual period. Both with the same FX fixing date
            Date fixingDate = fxIndex_->fixingCalendar().advance(c->accrualStartDate(),
                -static_cast<Integer>(fxIndex_->fixingDays()), Days);
            legs_[2].push_back(boost::shared_ptr<CashFlow>(
                new FXLinkedCashFlow(c->accrualStartDate(), fixingDate, -fixedNotional, fxIndex_)));
            legs_[2].push_back(boost::shared_ptr<CashFlow>(
                new FXLinkedCashFlow(c->accrualEndDate(), fixingDate, fixedNotional, fxIndex_)));
        }
    } else {
        // Notional exchanges for floating leg
        // Initial notional exchange
        Date aDate = floatSchedule_.dates().front();
        aDate = floatPaymentCalendar_.adjust(aDate, floatPaymentBdc_);
        boost::shared_ptr<CashFlow> aCashflow = boost::make_shared<SimpleCashFlow>(-floatNotional, aDate);
        floatLeg.insert(fixedLeg.begin(), aCashflow);

        // Final notional exchange
        aDate = fixedLeg.back()->date();
        aCashflow = boost::make_shared<SimpleCashFlow>(floatNotional, aDate);
        floatLeg.push_back(aCashflow);

        // resetting fixed leg
        for (Size j = 0; j < fixedLeg.size(); ++j) {
            boost::shared_ptr<FixedRateCoupon> coupon = boost::dynamic_pointer_cast<FixedRateCoupon>(fixedLeg[j]);
            Date fixingDate = fxIndex_->fixingCalendar().advance(coupon->accrualStartDate(),
                -static_cast<Integer>(fxIndex_->fixingDays()), Days);
            boost::shared_ptr<FixedRateFXLinkedNotionalCoupon> fxLinkedCoupon =
                boost::make_shared<FixedRateFXLinkedNotionalCoupon>(fixingDate, floatNotional, fxIndex_, coupon);
            floatLeg[j] = fxLinkedCoupon;
        }

        // now build a separate leg to store the resetting notionals
        receiveFixed_ ? payer_[2] = -1.0 : payer_[2] = +1.0;
        currencies_[2] = fixedCurrency_;
        for (Size j = 0; j < fixedLeg.size(); j++) {
            boost::shared_ptr<Coupon> c = boost::dynamic_pointer_cast<Coupon>(fixedLeg[j]);
            QL_REQUIRE(c, "Resetting XCCY - expected Coupon");
            // build a pair of notional flows, one at the start and one at the end of
            // the accrual period. Both with the same FX fixing date
            Date fixingDate = fxIndex_->fixingCalendar().advance(c->accrualStartDate(),
                -static_cast<Integer>(fxIndex_->fixingDays()), Days);
            legs_[2].push_back(boost::shared_ptr<CashFlow>(
                new FXLinkedCashFlow(c->accrualStartDate(), fixingDate, -floatNotional, fxIndex_)));
            legs_[2].push_back(boost::shared_ptr<CashFlow>(
                new FXLinkedCashFlow(c->accrualEndDate(), fixingDate, floatNotional, fxIndex_)));
        }
    }

    // Deriving from cross currency swap where:
    //   First leg should hold the pay flows
    //   Second leg should hold the receive flows
    payer_[0] = -1.0;
    payer_[1] = 1.0;
    if (receiveFixed_) {
        legs_[1] = fixedLeg;
        currencies_[1] = fixedCurrency_;
        legs_[0] = floatLeg;
        currencies_[0] = floatCurrency_;
    } else {
        legs_[0] = fixedLeg;
        currencies_[0] = fixedCurrency_;
        legs_[1] = floatLeg;
        currencies_[1] = floatCurrency_;
    }

    // Register the instrument with all cashflows on each leg.
    for (Size legNo = 0; legNo < legs_.size(); legNo++) {
        Leg::iterator it;
        for (it = legs_[legNo].begin(); it != legs_[legNo].end(); ++it) {
            registerWith(*it);
        }
    }
}

void CrossCcyFixFloatMtMResetSwap::setupArguments(PricingEngine::arguments* a) const {

    CrossCcySwap::setupArguments(a);

    if (CrossCcyFixFloatMtMResetSwap::arguments* args = dynamic_cast<CrossCcyFixFloatMtMResetSwap::arguments*>(a)) {
        args->fixedRate = fixedRate_;
        args->spread = floatSpread_;
    }
}

void CrossCcyFixFloatMtMResetSwap::fetchResults(const PricingEngine::results* r) const {

    CrossCcySwap::fetchResults(r);

    // Depending on the pricing engine used, we may have CrossCcyFixFloatSwap::results
    if (const CrossCcyFixFloatMtMResetSwap::results* res = dynamic_cast<const CrossCcyFixFloatMtMResetSwap::results*>(r)) {
        // If we have CrossCcyFixFloatSwap::results from the pricing engine
        fairFixedRate_ = res->fairFixedRate;
        fairSpread_ = res->fairSpread;
    } else {
        // If not, set them to Null to indicate a calculation is needed below
        fairFixedRate_ = Null<Rate>();
        fairSpread_ = Null<Spread>();
    }

    // Calculate fair rate and spread if they are still Null here
    static Spread basisPoint = 1.0e-4;

    Size idxFixed = receiveFixed_ ? 1 : 0;
    if (fairFixedRate_ == Null<Rate>() && legBPS_[idxFixed] != Null<Real>())
        fairFixedRate_ = fixedRate_ - NPV_ / (legBPS_[idxFixed] / basisPoint);

    Size idxFloat = receiveFixed_ ? 0 : 1;
    if (fairSpread_ == Null<Spread>() && legBPS_[idxFloat] != Null<Real>())
        fairSpread_ = floatSpread_ - NPV_ / (legBPS_[idxFloat] / basisPoint);
}

void CrossCcyFixFloatMtMResetSwap::setupExpired() const {
    CrossCcySwap::setupExpired();
    fairFixedRate_ = Null<Rate>();
    fairSpread_ = Null<Spread>();
}

void CrossCcyFixFloatMtMResetSwap::arguments::validate() const {
    CrossCcySwap::arguments::validate();
    QL_REQUIRE(fixedRate != Null<Rate>(), "Fixed rate cannot be null");
    QL_REQUIRE(spread != Null<Spread>(), "Spread cannot be null");
}

void CrossCcyFixFloatMtMResetSwap::results::reset() {
    CrossCcySwap::results::reset();
    fairFixedRate = Null<Rate>();
    fairSpread = Null<Spread>();
}
} // namespace QuantExt


// Distributed under the MIT License.
// See LICENSE.txt for details.

#pragma once

#include <array>
#include <boost/functional/hash.hpp>
#include <cstddef>
#include <utility>

#include "DataStructures/FixedHashMap.hpp"
#include "DataStructures/Index.hpp"
#include "DataStructures/Variables.hpp"
#include "Domain/Structure/Direction.hpp"
#include "Domain/Structure/DirectionMap.hpp"
#include "Domain/Structure/Element.hpp"
#include "Domain/Structure/ElementId.hpp"
#include "Domain/Structure/MaxNumberOfNeighbors.hpp"
#include "Evolution/DgSubcell/NeighborData.hpp"
#include "Evolution/Systems/Burgers/Tags.hpp"
#include "NumericalAlgorithms/Spectral/Mesh.hpp"
#include "Utilities/ErrorHandling/Assert.hpp"
#include "Utilities/Gsl.hpp"
#include "Utilities/TMPL.hpp"

namespace Burgers::fd {
template <typename TagsList, typename Reconstructor>
void reconstruct_work(
    const gsl::not_null<std::array<Variables<TagsList>, 1>*> vars_on_lower_face,
    const gsl::not_null<std::array<Variables<TagsList>, 1>*> vars_on_upper_face,
    const Reconstructor& reconstruct,
    const Variables<tmpl::list<Tags::U>> volume_vars, const Element<1>& element,
    const FixedHashMap<maximum_number_of_neighbors(1) + 1,
                       std::pair<Direction<1>, ElementId<1>>,
                       evolution::dg::subcell::NeighborData,
                       boost::hash<std::pair<Direction<1>, ElementId<1>>>>
        neighbor_data,
    const Mesh<1>& subcell_mesh, const size_t ghost_zone_size) {
  const size_t volume_num_pts = subcell_mesh.number_of_grid_points();
  const size_t reconstructed_num_pts = volume_num_pts + 1;

  // create views/spans into the face data, which will be filled by the
  // reconstructor
  std::array<gsl::span<double>, 1> upper_face_vars{};
  std::array<gsl::span<double>, 1> lower_face_vars{};
  for (size_t i = 0; i < 1; i++) {
    gsl::at(upper_face_vars, i) =
        gsl::make_span(get<Tags::U>(gsl::at(*vars_on_upper_face, i))[0].data(),
                       reconstructed_num_pts);
    gsl::at(lower_face_vars, i) =
        gsl::make_span(get<Tags::U>(gsl::at(*vars_on_lower_face, i))[0].data(),
                       reconstructed_num_pts);
  }

  const size_t neighbor_num_ghost_fd_points = ghost_zone_size;

  // make span of ghost cell variables for each direction
  DirectionMap<1, gsl::span<const double>> ghost_cell_vars{};

  // for all Directions, make the pointers in ghost_cell_vars span to point
  // the received neighbor data
  for (const auto& direction : Direction<1>::all_directions()) {
    const auto& neighbors_in_direction = element.neighbors().at(direction);

    ASSERT(neighbors_in_direction.size() == 1,
           "Currently only support one neighbor in each direction, but "
           "got "
               << neighbors_in_direction.size() << " in direction "
               << direction);

    ghost_cell_vars[direction] = gsl::make_span(
        &neighbor_data.at(std::pair{direction, *neighbors_in_direction.begin()})
             .data_for_reconstruction[0],
        neighbor_num_ghost_fd_points);
  }

  // make span of volume variables
  auto& volume_tensor = get<Tags::U>(volume_vars);
  const gsl::span<const double> volume_vars_span =
      gsl::make_span((volume_tensor)[0].data(), volume_num_pts);

  // perform reconstruction
  reconstruct(make_not_null(&upper_face_vars), make_not_null(&lower_face_vars),
              volume_vars_span, ghost_cell_vars, subcell_mesh.extents(), 1);
}

template <typename TagsList, typename ReconstructLower,
          typename ReconstructUpper>
void reconstruct_fd_neighbor_work(
    const gsl::not_null<Variables<TagsList>*> vars_on_face,
    const ReconstructLower& reconstruct_lower_neighbor,
    const ReconstructUpper& reconstruct_upper_neighbor,
    const Variables<tmpl::list<Tags::U>>& subcell_volume_vars,
    const Element<1>& element,
    const FixedHashMap<maximum_number_of_neighbors(1) + 1,
                       std::pair<Direction<1>, ElementId<1>>,
                       evolution::dg::subcell::NeighborData,
                       boost::hash<std::pair<Direction<1>, ElementId<1>>>>
        neighbor_data,
    const Mesh<1>& subcell_mesh, const Direction<1>& direction_to_reconstruct,
    const size_t ghost_zone_size) {
  const std::pair mortar_id{
      direction_to_reconstruct,
      *element.neighbors().at(direction_to_reconstruct).begin()};

  Index<1> ghost_data_extents = subcell_mesh.extents();
  ghost_data_extents[direction_to_reconstruct.dimension()] = ghost_zone_size;

  // allocate Variable for storing data from neighbor
  Variables<tmpl::list<Tags::U>> neighbor_vars{ghost_data_extents.product()};

  {
    ASSERT(neighbor_data.contains(mortar_id),
           "The neighbor data does not contain the mortar: ("
               << mortar_id.first << ',' << mortar_id.second << ")");

    const auto& neighbor_data_in_direction = neighbor_data.at(mortar_id);
    std::copy(
        neighbor_data_in_direction.data_for_reconstruction.begin(),
        std::next(neighbor_data_in_direction.data_for_reconstruction.begin(),
                  static_cast<std::ptrdiff_t>(ghost_data_extents.product())),
        neighbor_vars.data());
  }

  const auto& tensor_volume = get<Tags::U>(subcell_volume_vars);
  const auto& tensor_neighbor = get<Tags::U>(neighbor_vars);
  auto& tensor_on_face = get<Tags::U>(*vars_on_face);

  if (direction_to_reconstruct.side() == Side::Upper) {
    for (size_t tensor_index = 0; tensor_index < tensor_on_face.size();
         ++tensor_index) {
      reconstruct_upper_neighbor(
          make_not_null(&tensor_on_face[tensor_index]),
          tensor_volume[tensor_index], tensor_neighbor[tensor_index],
          subcell_mesh.extents(), ghost_data_extents, direction_to_reconstruct);
    }
  } else {
    for (size_t tensor_index = 0; tensor_index < tensor_on_face.size();
         ++tensor_index) {
      reconstruct_lower_neighbor(
          make_not_null(&tensor_on_face[tensor_index]),
          tensor_volume[tensor_index], tensor_neighbor[tensor_index],
          subcell_mesh.extents(), ghost_data_extents, direction_to_reconstruct);
    }
  }
}
}  // namespace Burgers::fd