/*
 * (C) Copyright 2018-2019 UCAR
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 */

#include <Eigen/Core>
#include <cmath>
#include <fstream>
#include <memory>
#include <random>
#include <set>

#include "ufo/ObsBiasCovariance.h"

#include "ioda/distribution/Accumulator.h"
#include "ioda/Engines/Factory.h"
#include "ioda/Engines/HH.h"
#include "ioda/Layout.h"
#include "ioda/ObsGroup.h"
#include "ioda/ObsSpace.h"
#include "ioda/ObsVector.h"

#include "oops/util/IntSetParser.h"
#include "oops/util/Logger.h"
#include "oops/util/Random.h"

#include "ufo/ObsBias.h"
#include "ufo/ObsBiasIncrement.h"
#include "ufo/ObsBiasPreconditioner.h"
#include "ufo/predictors/PredictorBase.h"
#include "ufo/utils/IodaGroupIndices.h"

namespace ufo {

// -----------------------------------------------------------------------------

ObsBiasCovariance::ObsBiasCovariance(ioda::ObsSpace & odb,
                                     const Parameters_ & params)
  : odb_(odb), prednames_(0), vars_(odb.obsvariables()), variances_(),
    preconditioner_(0),
    ht_rinv_h_(0), obs_num_(0), analysis_variances_(0), minimal_required_obs_number_(0),
    rank_(odb.distribution()->rank()) {
  oops::Log::trace() << "ObsBiasCovariance::Constructor starting" << std::endl;

  // Predictor factory
  for (const PredictorParametersWrapper &wrapper :
       params.variationalBC.value().predictors.value()) {
    std::shared_ptr<PredictorBase> pred(PredictorFactory::create(wrapper.predictorParameters,
                                                                 vars_));
    prednames_.push_back(pred->name());
  }

  if (prednames_.size()*vars_.size() > 0) {
    if (params.covariance.value() == boost::none)
      throw eckit::UserError("obs bias.covariance section missing from the YAML file");
    const ObsBiasCovarianceParameters &biasCovParams = *params.covariance.value();

    // Get the minimal required filtered obs number
    minimal_required_obs_number_ = biasCovParams.minimalRequiredObsNumber;

    // Override the variance range if provided
    {
      const std::vector<double> &range = biasCovParams.varianceRange.value();
      ASSERT(range.size() == 2);
      smallest_variance_ = range[0];
      largest_variance_ = range[1];
    }

    // Override the preconditioning step size if provided
    step_size_ = biasCovParams.stepSize;

    // Override the largest analysis variance if provided
    largest_analysis_variance_ = biasCovParams.largestAnalysisVariance;

    // Initialize the variances to upper limit
    variances_ = Eigen::VectorXd::Constant(prednames_.size()*vars_.size(), largest_variance_);

    // Initialize the hessian contribution to zero
    ht_rinv_h_.resize(prednames_.size() * vars_.size());
    std::fill(ht_rinv_h_.begin(), ht_rinv_h_.end(), 0.0);

    // Initialize the preconditioner to default step size
    preconditioner_.resize(prednames_.size() * vars_.size());
    std::fill(preconditioner_.begin(), preconditioner_.end(), step_size_);

    // Initialize obs_num_ to ZERO
    obs_num_.resize(vars_.size());
    std::fill(obs_num_.begin(), obs_num_.end(), 0);

    // Initialize analysis error variances to the upper limit
    analysis_variances_.resize(prednames_.size() * vars_.size());
    std::fill(analysis_variances_.begin(), analysis_variances_.end(), largest_variance_);

    // Initializes from given prior
    if (biasCovParams.prior.value() != boost::none) {
      const ObsBiasCovariancePriorParameters &priorParams = *biasCovParams.prior.value();

      // Get default inflation ratio
      const double inflation_ratio = priorParams.inflation.value().ratio;

      // Check the large inflation ratio when obs number < minimal_required_obs_number
      const double large_inflation_ratio = priorParams.inflation.value().ratioForSmallDataset;

      // read in Variances prior (analysis_variances_) and number of obs. (obs_num_)
      // from previous cycle
      this->read(priorParams);

      // set variances for bias predictor coeff. based on diagonal info
      // of previous analysis error variance
      std::size_t ii;
      for (std::size_t j = 0; j < vars_.size(); ++j) {
        const double inflation = (obs_num_[j] <= minimal_required_obs_number_) ?
                                 large_inflation_ratio : inflation_ratio;
        for (std::size_t p = 0; p < prednames_.size(); ++p) {
          ii = j*prednames_.size() + p;
          if (inflation > inflation_ratio)
            analysis_variances_[ii] = inflation * analysis_variances_[ii] + smallest_variance_;
          variances_[ii] = inflation * analysis_variances_[ii] + smallest_variance_;
          if (variances_[ii] > largest_variance_) variances_[ii] = largest_variance_;
          if (analysis_variances_[ii] > largest_analysis_variance_)
            analysis_variances_[ii] = largest_analysis_variance_;
        }
      }
    }
  }

  oops::Log::trace() << "ObsBiasCovariance::Constructor is done" << std::endl;
}

// -----------------------------------------------------------------------------

void ObsBiasCovariance::read(const ObsBiasCovariancePriorParameters & params) {
  oops::Log::trace() << "ObsBiasCovariance::read from file " << std::endl;

  if (params.inputFile.value() != boost::none) {
    // Open an hdf5 file, read only
    ioda::Engines::BackendNames  backendName = ioda::Engines::BackendNames::Hdf5File;
    ioda::Engines::BackendCreationParameters backendParams;
    backendParams.fileName = *params.inputFile.value();
    backendParams.action   = ioda::Engines::BackendFileActions::Open;
    backendParams.openMode = ioda::Engines::BackendOpenModes::Read_Only;

    // Create the backend and attach it to an ObsGroup
    // Use the None DataLyoutPolicy for now to accommodate the current file format
    ioda::Group backend = constructBackend(backendName, backendParams);
    ioda::ObsGroup obsgroup = ioda::ObsGroup(backend,
                   ioda::detail::DataLayoutPolicy::generate(
                         ioda::detail::DataLayoutPolicy::Policies::None));

    // Read coefficients error variances into the Eigen array
    ioda::Variable bcerrvar = obsgroup.vars["bias_coeff_errors"];
    Eigen::ArrayXXf allbcerrors;
    bcerrvar.readWithEigenRegular(allbcerrors);

    // Read nobs into Eigen array
    ioda::Variable nobsvar = obsgroup.vars["number_obs_assimilated"];
    Eigen::ArrayXf nobsassim;
    nobsvar.readWithEigenRegular(nobsassim);

    // Find indices of predictors and variables/channels that we need in the data read from the file
    const std::vector<int> pred_idx = getRequiredVariableIndices(obsgroup, "predictors",
                                              prednames_.begin(), prednames_.end());
    const std::vector<int> var_idx = getRequiredVarOrChannelIndices(obsgroup, vars_);

    // Filter predictors and channels that we need
    // FIXME: may be possible by indexing allbcerrors(pred_idx, chan_idx) when Eigen 3.4
    // is available
    for (size_t jvar = 0; jvar < var_idx.size(); ++jvar) {
      obs_num_[jvar] = nobsassim(var_idx[jvar]);
      for (size_t jpred = 0; jpred < pred_idx.size(); ++jpred) {
        analysis_variances_[jvar*pred_idx.size()+jpred] =
             allbcerrors(pred_idx[jpred], var_idx[jvar]);
      }
    }
  }
  oops::Log::trace() << "ObsBiasCovariance::read is done " << std::endl;
}

// -----------------------------------------------------------------------------

void ObsBiasCovariance::write(const Parameters_ & params) {
  // only write files out on the task with MPI rank 0
  if (rank_ != 0) return;

  oops::Log::trace() << "ObsBiasCovariance::write to file " << std::endl;
  const ObsBiasCovarianceParameters &biasCovParams = *params.covariance.value();

  if (biasCovParams.outputFile.value() != boost::none) {
    // FIXME: only implemented for channels currently
    if (vars_.channels().size() == 0) {
      throw eckit::NotImplemented("ObsBiasCovariance::write not implemented for without channels",
                                  Here());
    }
    // Create a file, overwrite if exists
    const std::string output_filename = *biasCovParams.outputFile.value();
    ioda::Group group = ioda::Engines::HH::createFile(output_filename,
                        ioda::Engines::BackendCreateModes::Truncate_If_Exists);

    // put only variable bias predictors into the predictors vector
    std::vector<std::string> predictors(prednames_.begin(), prednames_.end());
    // map coefficients to 2D for saving
    Eigen::Map<const Eigen::MatrixXd>
        allbcerrors(variances_.data(), prednames_.size(), vars_.size());
    const std::vector<int> channels = vars_.channels();
    std::vector<int> obs_assimilated(obs_num_.begin(), obs_num_.end());

    // dimensions
    ioda::NewDimensionScales_t dims {
        ioda::NewDimensionScale<int>("npredictors", predictors.size()),
        ioda::NewDimensionScale<int>("nchannels", channels.size())
    };
    // new ObsGroup
    ioda::ObsGroup ogrp = ioda::ObsGroup::generate(group, dims);

    // save the predictors
    ioda::Variable predsVar = ogrp.vars.createWithScales<std::string>(
                              "predictors", {ogrp.vars["npredictors"]});
    predsVar.write(predictors);
    // and the variables
    ioda::Variable chansVar = ogrp.vars.createWithScales<int>(
                              "channels", {ogrp.vars["nchannels"]});
    chansVar.write(channels);

    // and the number_obs_assimilated
    ioda::Variable nobsVar = ogrp.vars.createWithScales<int>(
                             "number_obs_assimilated", {ogrp.vars["nchannels"]});
    nobsVar.write(obs_assimilated);

    // Set up the creation parameters for the bias covariance coefficients variable
    ioda::VariableCreationParameters float_params;
    float_params.chunk = true;               // allow chunking
    float_params.compressWithGZIP();         // compress using gzip
    float missing_value = util::missingValue(missing_value);
    float_params.setFillValue<float>(missing_value);

    // Create a variable for bias covariance coefficients,
    // save bias covariance coeffs to the variable
    ioda::Variable anvarVar = ogrp.vars.createWithScales<float>("bias_coeff_errors",
                       {ogrp.vars["npredictors"], ogrp.vars["nchannels"]}, float_params);
    anvarVar.writeWithEigenRegular(allbcerrors);
  }
  oops::Log::trace() << "ObsBiasCovariance::write is done " << std::endl;
}

// -----------------------------------------------------------------------------

void ObsBiasCovariance::linearize(const ObsBias & bias, const eckit::Configuration & innerConf) {
  oops::Log::trace() << "ObsBiasCovariance::linearize starts" << std::endl;
  if (bias) {
    // Retrieve the QC flags and do statistics from second outer loop
    const int jouter = innerConf.getInt("iteration");
    if (jouter >= 1) {
      std::unique_ptr<ioda::Accumulator<std::vector<size_t>>> obs_num_accumulator =
          odb_.distribution()->createAccumulator<size_t>(obs_num_.size());

      // Retrieve the QC flags of previous outer loop and recalculate the number of effective obs.
      const std::string qc_group_name = "EffectiveQC" + std::to_string(jouter-1);
      const std::vector<std::string> vars = odb_.obsvariables().variables();
      std::vector<int> qc_flags(odb_.nlocs(), 999);
      for (std::size_t jvar = 0; jvar < vars.size(); ++jvar) {
        if (odb_.has(qc_group_name, vars[jvar])) {
          odb_.get_db(qc_group_name, vars[jvar], qc_flags);
          for (std::size_t jloc = 0; jloc < qc_flags.size(); ++jloc)
            if (qc_flags[jloc] == 0)
              obs_num_accumulator->addTerm(jloc, jvar, 1);
        } else {
          throw eckit::UserError("Unable to find QC flags : " + vars[jvar] + "@" + qc_group_name);
        }
      }

      // Sum across the processors
      obs_num_ = obs_num_accumulator->computeResult();

      const float missing = util::missingValue(missing);

      // compute the hessian contribution from Jo bias terms channel by channel
      // retrieve the effective error (after QC) for this channel
      const std::string err_group_name = "EffectiveError" + std::to_string(jouter-1);
      ioda::ObsVector r_inv(odb_, err_group_name);

      // compute \mathrm{R}^{-1}
      std::size_t nvars = r_inv.nvars();
      for (size_t vv = 0; vv < nvars; ++vv) {
        for (size_t ii = 0; ii < r_inv.nlocs(); ++ii) {
          if (r_inv[ii*nvars + vv] != missing) {
            r_inv[ii*nvars + vv] = 1.0f / pow(r_inv[ii*nvars + vv], 2);
          } else {
            r_inv[ii*nvars + vv] = 0.0f;
          }
        }
      }

      // compute \mathrm{H}_\beta^\intercal \mathrm{R}^{-1} \mathrm{H}_\beta
      // -----------------------------------------
      std::unique_ptr<ioda::Accumulator<std::vector<double>>> ht_rinv_h_accumulator =
          odb_.distribution()->createAccumulator<double>(ht_rinv_h_.size());
      for (std::size_t p = 0; p < prednames_.size(); ++p) {
        // retrieve the predictors
        const ioda::ObsVector predx(odb_, prednames_[p] + "Predictor");

        // for each variable
        ASSERT(r_inv.nlocs() == predx.nlocs());
        std::size_t nvars = predx.nvars();
        // only keep the diagnoal
        for (size_t vv = 0; vv < nvars; ++vv) {
          for (size_t ii = 0; ii < predx.nlocs(); ++ii)
            ht_rinv_h_accumulator->addTerm(ii,
                                           vv*prednames_.size() + p,
                                           pow(predx[ii*nvars + vv], 2) * r_inv[ii*nvars + vv]);
        }
      }

      // Sum the hessian contributions across the tasks
      ht_rinv_h_ = ht_rinv_h_accumulator->computeResult();
    }

    // reset variances for bias predictor coeff. based on current data count
    for (std::size_t j = 0; j < obs_num_.size(); ++j) {
      if (obs_num_[j] <= minimal_required_obs_number_) {
        for (std::size_t p = 0; p < prednames_.size(); ++p)
          variances_[j*prednames_.size() + p] = smallest_variance_;
      }
    }

    // set a coeff. factor for variances of control variables
    for (std::size_t j = 0; j < vars_.size(); ++j) {
      for (std::size_t p = 0; p < prednames_.size(); ++p) {
        const std::size_t index = j*prednames_.size() + p;
        preconditioner_[index] = step_size_;
        // L = \mathrm{A}^{-1}
        if (obs_num_[j] > 0)
          preconditioner_[index] = 1.0 / (1.0 / variances_[index] + ht_rinv_h_[index]);
        if (obs_num_[j] > minimal_required_obs_number_) {
          if (ht_rinv_h_[index] > 0.0) {
            analysis_variances_[index] = 1.0 / (1.0 / variances_[index] + ht_rinv_h_[index]);
          } else {
            analysis_variances_[index] = largest_analysis_variance_;
          }
        }
      }
    }
  }
  oops::Log::trace() << "ObsBiasCovariance::linearize is done" << std::endl;
}

// -----------------------------------------------------------------------------

void ObsBiasCovariance::multiply(const ObsBiasIncrement & dx1,
                                 ObsBiasIncrement & dx2) const {
  oops::Log::trace() << "ObsBiasCovariance::multiply starts" << std::endl;

  dx2.data().array() = dx1.data().array() * variances_.array();

  oops::Log::trace() << "ObsBiasCovariance::multiply is done" << std::endl;
}

// -----------------------------------------------------------------------------

void ObsBiasCovariance::inverseMultiply(const ObsBiasIncrement & dx1,
                                        ObsBiasIncrement & dx2) const {
  oops::Log::trace() << "ObsBiasCovariance::inverseMultiply starts" << std::endl;

  dx2.data().array() = dx1.data().array() / variances_.array();

  oops::Log::trace() << "ObsBiasCovariance::inverseMultiply is done" << std::endl;
}

// -----------------------------------------------------------------------------

void ObsBiasCovariance::randomize(ObsBiasIncrement & dx) const {
  oops::Log::trace() << "ObsBiasCovariance::randomize starts" << std::endl;
  if (dx) {
    static util::NormalDistribution<double> dist(variances_.size());
    for (std::size_t jj = 0; jj < variances_.size(); ++jj) {
      dx.data()[jj] = dist[jj] * std::sqrt(variances_[jj]);
    }
  }
  oops::Log::trace() << "ObsBiasCovariance::randomize is done" << std::endl;
}
// -----------------------------------------------------------------------------

std::unique_ptr<ObsBiasPreconditioner> ObsBiasCovariance::preconditioner() const {
    return std::make_unique<ObsBiasPreconditioner> (preconditioner_);
}

// -----------------------------------------------------------------------------

}  // namespace ufo


/*
 * _build_in.cpp
 *
 *  Created on: 2013-2-14
 *      Author: fasiondog
 */

#include <boost/python.hpp>
#include <hikyuu/indicator/build_in.h>

using namespace hku;
using namespace boost::python;

Indicator (*PRICELIST2)(const PriceList&, int) = PRICELIST;
Indicator (*PRICELIST3)(const Indicator&, int) = PRICELIST;
Indicator (*PRICELIST4)(int) = PRICELIST;

Indicator (*KDATA1)(const KData&) = KDATA;
Indicator (*KDATA2)(const Indicator&) = KDATA;
Indicator (*KDATA3)() = KDATA;

Indicator (*OPEN1)(const KData&) = OPEN;
Indicator (*OPEN2)(const Indicator&) = OPEN;
Indicator (*OPEN3)() = OPEN;

Indicator (*HIGH1)(const KData&) = HIGH;
Indicator (*HIGH2)(const Indicator&) = HIGH;
Indicator (*HIGH3)() = HIGH;

Indicator (*LOW1)(const KData&) = LOW;
Indicator (*LOW2)(const Indicator&) = LOW;
Indicator (*LOW3)() = LOW;

Indicator (*CLOSE1)(const KData&) = CLOSE;
Indicator (*CLOSE2)(const Indicator&) = CLOSE;
Indicator (*CLOSE3)() = CLOSE;

Indicator (*AMO1)(const KData&) = AMO;
Indicator (*AMO2)(const Indicator&) = AMO;
Indicator (*AMO3)() = AMO;

Indicator (*VOL1)(const KData&) = VOL;
Indicator (*VOL2)(const Indicator&) = VOL;
Indicator (*VOL3)() = VOL;

Indicator (*KDATA_PART1)(const KData& kdata, const string& part) = KDATA_PART;
Indicator (*KDATA_PART2)(const Indicator& kdata, const string& part) = KDATA_PART;
Indicator (*KDATA_PART3)(const string& part) = KDATA_PART;

Indicator (*AMA_1)(int, int, int) = AMA;
Indicator (*AMA_2)(const Indicator&, int, int, int) = AMA;

Indicator (*ATR_1)(int) = ATR;
Indicator (*ATR_2)(const Indicator&, int) = ATR;

Indicator (*DIFF_1)() = DIFF;
Indicator (*DIFF_2)(const Indicator&) = DIFF;

Indicator (*MA_1)(int, const string&) = MA;
Indicator (*MA_2)(const Indicator&, int, const string&) = MA;
//BOOST_PYTHON_FUNCTION_OVERLOADS(MA_1_overload, MA, 1, 2);
//BOOST_PYTHON_FUNCTION_OVERLOADS(MA_2_overload, MA, 2, 3);

Indicator (*SMA_1)(int) = SMA;
Indicator (*SMA_2)(const Indicator&, int) = SMA;

Indicator (*EMA_1)(int) = EMA;
Indicator (*EMA_2)(const Indicator&, int) = EMA;

Indicator (*MACD_1)(int, int, int) = MACD;
Indicator (*MACD_2)(const Indicator&, int, int, int) = MACD;
//BOOST_PYTHON_FUNCTION_OVERLOADS(MACD_1_overload, MACD, 0, 3);
//BOOST_PYTHON_FUNCTION_OVERLOADS(MACD_2_overload, MACD, 1, 4);

Indicator (*REF_1)(int) = REF;
Indicator (*REF_2)(const Indicator&, int) = REF;

Indicator (*SAFTYLOSS_1)(int n1, int n2, double p) = SAFTYLOSS;
Indicator (*SAFTYLOSS_2)(const Indicator&, int n1, int n2, double p) = SAFTYLOSS;
//BOOST_PYTHON_FUNCTION_OVERLOADS(SAFTYLOSS_1_overload, SAFTYLOSS, 0, 3);
//BOOST_PYTHON_FUNCTION_OVERLOADS(SAFTYLOSS_2_overload, SAFTYLOSS, 1, 4);

Indicator (*STDEV_1)(int) = STDEV;
Indicator (*STDEV_2)(const Indicator&, int) = STDEV;

Indicator (*HHV_1)(int) = HHV;
Indicator (*HHV_2)(const Indicator&, int) = HHV;

Indicator (*LLV_1)(int) = LLV;
Indicator (*LLV_2)(const Indicator&, int) = LLV;

Indicator (*VIGOR_1)(const KData&, int) = VIGOR;
Indicator (*VIGOR_2)(int) = VIGOR;
Indicator (*VIGOR_3)(const Indicator&, int) = VIGOR;

Indicator (*WEAVE_1)() = WEAVE;
Indicator (*WEAVE_2)(const Indicator&) = WEAVE;

Indicator (*CVAL_1)(double, size_t, size_t) = CVAL;
Indicator (*CVAL_2)(const Indicator&, double) = CVAL;

Indicator (*IND_AND1)(const Indicator&, const Indicator&) = IND_AND;
Indicator (*IND_AND2)(const Indicator&, price_t) = IND_AND;
Indicator (*IND_AND3)(price_t, const Indicator&) = IND_AND;

Indicator (*IND_OR1)(const Indicator&, const Indicator&) = IND_OR;
Indicator (*IND_OR2)(const Indicator&, price_t) = IND_OR;
Indicator (*IND_OR3)(price_t, const Indicator&) = IND_OR;

void export_Indicator_build_in() {
    docstring_options doc_options(false);

    def("KDATA", KDATA1);
    def("KDATA", KDATA2);
    def("KDATA", KDATA3);

    def("CLOSE", CLOSE1);
    def("CLOSE", CLOSE2);
    def("CLOSE", CLOSE3);

    def("OPEN", OPEN1);
    def("OPEN", OPEN2);
    def("OPEN", OPEN3);

    def("HIGH", HIGH1);
    def("HIGH", HIGH2);
    def("HIGH", HIGH3);

    def("LOW", LOW1);
    def("LOW", LOW2);
    def("LOW", LOW3);

    def("AMO", AMO1);
    def("AMO", AMO2);
    def("AMO", AMO3);

    def("VOL", VOL1);
    def("VOL", VOL2);
    def("VOL", VOL3);

    def("KDATA_PART", KDATA_PART1, (arg("data"), arg("kpart")));
    def("KDATA_PART", KDATA_PART2, (arg("data"), arg("kpart")));
    def("KDATA_PART", KDATA_PART3, (arg("kpart")));

    //无法直接使用Null<size_t>()，会导致异常退出
    size_t null_size = Null<size_t>();
    def("PRICELIST", PRICELIST2, (arg("data"), arg("discard")=0));
    def("PRICELIST", PRICELIST3, (arg("data"), arg("result_index")=0));
    def("PRICELIST", PRICELIST4, (arg("result_index")=0));

    def("SMA", SMA_1, (arg("n")=22));
    def("SMA", SMA_2, (arg("data"), arg("n")=22));
    def("EMA", EMA_1, (arg("n")=22));
    def("EMA", EMA_2, (arg("data"), arg("n")=22));

    def("MA", MA_1, (arg("n")=22, arg("type")="SMA"),"type='SMA|EMA|AMA'");
    def("MA", MA_2, (arg("data"), arg("n")=22, arg("type")="SMA"), "type='SMA|EMA|AMA'");

    def("AMA", AMA_1, (arg("n")=10, arg("fast_n")=2, arg("slow_n")=30));
    def("AMA", AMA_2, (arg("data"), arg("n")=10, arg("fast_n")=2, arg("slow_n")=30));

    def("ATR", ATR_1, (arg("n")=14));
    def("ATR", ATR_2, (arg("data"), arg("n")=14));

    def("MACD", MACD_1, (arg("n1")=12, arg("n2")=26, arg("n3")=9));
    def("MACD", MACD_2, (arg("data"), arg("n1")=12, arg("n2")=26, arg("n3")=9));

    def("VIGOR", VIGOR_1, (arg("kdata"), arg("n")=2));
    def("VIGOR", VIGOR_2, (arg("n")=2));
    def("VIGOR", VIGOR_3, (arg("ind"), arg("n")=2));

    def("SAFTYLOSS", SAFTYLOSS_1, (arg("n1")=10, arg("n2")=3, arg("p")=2.0));
    def("SAFTYLOSS", SAFTYLOSS_2, (arg("data"), arg("n1")=10, arg("n2")=3, arg("p")=2.0));
    def("DIFF", DIFF_1);
    def("DIFF", DIFF_2);

    def("REF", REF_1, (arg("n")));
    def("REF", REF_2, (arg("data"), arg("n")));

    def("STDEV", STDEV_1, (arg("n")=10));
    def("STDEV", STDEV_2, (arg("data"), arg("n")=10));

    def("POS", POS, (arg("block"), arg("query"), arg("sg")));

    def("HHV", HHV_1, (arg("n")=20));
    def("HHV", HHV_2, (arg("data"), arg("n")=20));

    def("LLV", LLV_1, (arg("n")=20));
    def("LLV", LLV_2, (arg("data"), arg("n")=20));

    def("WEAVE", WEAVE_1);
    def("WEAVE", WEAVE_2);

    def("CVAL", CVAL_1, (arg("value")=0.0, arg("len")=0, arg("discard")=0));
    def("CVAL", CVAL_2, (arg("data"), arg("value")=0.0));

    def("IND_AND", IND_AND1);
    def("IND_AND", IND_AND2);
    def("IND_AND", IND_AND3);

    def("IND_OR", IND_OR1);
    def("IND_OR", IND_OR2);
    def("IND_OR", IND_OR3);

}


///////////////////////////////////////////////////////////////////////////////
/// \file scaled_series.hpp
/// A scaled series that satisfies the \c InfiniteRangeRunStorage concept
//
//  Copyright 2006 Eric Niebler. Distributed under the Boost
//  Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_TIME_SERIES_SCALED_SERIES_EAN_06_16_2006
#define BOOST_TIME_SERIES_SCALED_SERIES_EAN_06_16_2006

#include <utility>
#include <algorithm>
#include <boost/time_series/concepts.hpp>
#include <boost/time_series/traits/is_time_series.hpp>
#include <boost/time_series/traits/offset_type.hpp>
#include <boost/time_series/traits/storage_category.hpp>
#include <boost/time_series/traits/discretization_type.hpp>
#include <boost/time_series/utility/time_series_base.hpp>
#include <boost/time_series/storage/detail/scaled_elements.hpp>

// A view of an array that has been scaled by a constant factor.
namespace boost { namespace time_series
{
    // The scaled_series
    template<typename Series, typename Factor>
    struct scaled_series
      : time_series_base<scaled_series<Series, Factor> >
    {
    public:
        typedef Series storage_type;
        typedef Factor factor_type;
        typedef concepts::TimeSeries<Series> series_concept;

        typedef typename series_concept::value_type value_type;
        typedef typename series_concept::cursor cursor;
        typedef typename series_concept::elements elements;
        typedef typename series_concept::runs runs;
        typedef typename series_concept::run_type run_type;
        typedef typename series_concept::offset_type offset_type;
        typedef typename series_concept::discretization_type discretization_type;

        typedef value_type const reference;

        scaled_series(Series &series, factor_type factor)
          : data_(series)
          , factor_(factor)
        {}

        discretization_type discretization() const
        {
            return this->data().discretization();
        }

        reference operator [](offset_type n) const
        {
            return range_run_storage::get_at(this->data(), n) * this->factor();
        }

        // TODO make these private
    //private:
        factor_type factor() const
        {
            return this->factor_;
        }

        storage_type &data() const
        {
            return this->data_;
        }

    private:
        Series &data_;
        factor_type factor_;
    };

    namespace traits
    {
        /// INTERNAL ONLY
        template<typename Series, typename Factor>
        struct discretization_type<scaled_series<Series, Factor> >
          : discretization_type<Series>
        {};

        /// INTERNAL ONLY
        template<typename Series, typename Factor>
        struct offset_type<scaled_series<Series, Factor> >
          : offset_type<Series>
        {};

        /// INTERNAL ONLY
        template<typename Series, typename Factor>
        struct storage_category<scaled_series<Series, Factor> >
          : storage_category<Series>
        {};
    }
}} // namespace boost::time_series

namespace boost
{
    namespace sequence { namespace impl
    {
        /// INTERNAL ONLY
        struct scaled_series_tag;

        /// INTERNAL ONLY
        template<typename Series, typename Factor>
        struct tag<time_series::scaled_series<Series, Factor> >
        {
            typedef scaled_series_tag type;
        };

        /// INTERNAL ONLY
        template<typename S>
        struct elements<S, scaled_series_tag>
        {
            typedef typename S::factor_type factor_type;
            typedef typename S::elements elements_type;
            typedef time_series::storage::scaled_elements<elements_type, factor_type> result_type;

            result_type operator ()(S &s) const
            {
                return result_type(sequence::elements(s.data()), s.factor());
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct begin<S, scaled_series_tag>
        {
            typedef typename S::cursor result_type;

            result_type operator ()(S &s) const
            {
                return sequence::begin(s.data());
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct end<S, scaled_series_tag>
        {
            typedef typename S::cursor result_type;

            result_type operator ()(S &s) const
            {
                return sequence::end(s.data());
            }
        };
    }}

    namespace range_run_storage { namespace impl
    {
        /// INTERNAL ONLY
        template<typename S>
        struct runs<S, sequence::impl::scaled_series_tag>
        {
            typedef typename S::runs result_type;

            result_type operator ()(S &s) const
            {
                return range_run_storage::runs(s.data());
            }
        };

        /// INTERNAL ONLY
        template<typename S, typename I>
        struct get_at<S, I, sequence::impl::scaled_series_tag>
        {
            typedef typename S::reference result_type;

            result_type operator ()(S &s, I &i) const
            {
                return s[i];
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct zero<S, sequence::impl::scaled_series_tag>
        {
            typedef typename S::storage_type storage_type;
            typedef typename result_of<op::zero(storage_type &)>::type result_type;

            result_type operator ()(S &s) const
            {
                return range_run_storage::zero(s.data());
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct pre_run<S, sequence::impl::scaled_series_tag>
        {
            typedef typename S::storage_type storage_type;
            typedef typename result_of<op::pre_run(storage_type &)>::type result_type;

            result_type operator ()(S &s) const
            {
                return range_run_storage::pre_run(s.data());
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct pre_value<S, sequence::impl::scaled_series_tag>
        {
            typedef typename S::reference result_type;

            result_type operator ()(S &s) const
            {
                return range_run_storage::pre_value(s.data()) * s.factor();
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct post_run<S, sequence::impl::scaled_series_tag>
        {
            typedef typename S::storage_type storage_type;
            typedef typename result_of<op::post_run(storage_type &)>::type result_type;

            result_type operator ()(S &s) const
            {
                return range_run_storage::post_run(s.data());
            }
        };

        /// INTERNAL ONLY
        template<typename S>
        struct post_value<S, sequence::impl::scaled_series_tag>
        {
            typedef typename S::reference result_type;

            result_type operator ()(S &s) const
            {
                return range_run_storage::post_value(s.data()) * s.factor();
            }
        };

    }}
}

#endif


#include "db/timeline.hpp"

#include <exception>
#include <fstream>
#include <functional>
#include <boost/filesystem.hpp>

namespace fs = boost::filesystem;

namespace henhouse::db
{
    const offset_type ADD_BUCKET_BACK_LIMIT = 60;

    /**
     * This is the main function to compute the partial sums given previous bucket.
     * It turns the current non-summed bucket into a summed bucket.
     *
     * current.value is assumed to be set to the count in that bucket.
     *
     * It computes partial sum(X) and partial sum(X^2) up the current bucket.
     */
    void propogate(data_item prev, data_item& current)
    {
        const auto v = current.value;
        current.integral = prev.integral + v;
        current.second_integral = prev.second_integral + (v  * v);
    }

    //Adds a count c to the current bucket and updates the partial sum
    //values of the current bucket.
    void update_current(data_item prev, data_item& current, count_type c)
    {
        current.value += c;
        propogate(prev, current);
    }

    /**
     *
     * Mean is computing as the (running sum of x) / N.
     * In addition, variance requires maintaining the running sum of x^2
     *
     * mean = sum(x) / N
     * mean of squared x = sum(x^2) / N
     *
     * variance = (sum(x^2) / N) - (sum(x) / N )^2
     *          = (sum(x^2) / N) - mean^2
     *          = (mean of squared x) - mean^2
     *          = (mean of squared x) - (mean squared)
     */
    diff_result diff_buckets(
            const time_type ta,          //time from
            const time_type tb,          //time to
            const time_type resolution,  //resolution of time buckets
            const offset_type index_offset,
            const data_item a,           //bucket from
            const data_item b,           //bucket to
            const count_type n)          //number of buckets
    {
        REQUIRE_GREATER(resolution, 0);
        REQUIRE_GREATER(n, 0);

        //Sum here is the values added within
        const auto sum = b.integral - a.integral;
        const auto second_sum = b.second_integral - a.second_integral;
        const auto mean = static_cast<mean_type>(sum) / n;
        const auto mean_squared = mean * mean;
        const auto second_mean = static_cast<mean_type>(second_sum) / n;
        const auto variance = second_mean - mean_squared;

        return diff_result
        {
            ta,
            tb,
            resolution,
            index_offset,
            sum,
            mean,
            variance,
            n,
            a,
            b,
        };
    }

    bool timeline::put(time_type t, count_type c)
    {
        //We already have data, let's add and index new point.
        if(index.size() > 0)
        {
            const auto last_range = index.cend() - 1;

            //don't add if time is before last range
            if(t >= last_range->time)
            {
                //get last position only because we want to keep
                //a specific performance profile. This is a deliberate limitation.
                auto p = index.find_pos_from_range(t, last_range, index.cend());
                const auto pos = p.pos + p.offset;

                //bucket is current or in the past, no need to index.
                if(pos < data.size())
                {
                    //if we are too far back in the range, skip it,
                    //otherwise propogate the values up.
                    //This limitation is to keep performance predictable for
                    //inserts while providing a buffer for slow inserters
                    //to catch up.
                    if(data.size() - pos < ADD_BUCKET_BACK_LIMIT)
                    {
                        const auto prev = pos > 0 ? data[pos - 1] : data_item{0, 0, 0};
                        update_current(prev, data[pos], c);
                        for(auto p = pos + 1; p < data.size(); p++)
                            propogate(data[p-1], data[p]);
                    }
                    else return false;
                }
                //if we move beyond end, append data
                else
                {
                    const auto last_pos = data.size() - 1;
                    const auto prev = data[last_pos];

                    //don't compute integral and second_integral
                    //because propogate will overwrite
                    data_item current{c, 0, 0};
                    propogate(prev, current);
                    data.push_back(current);

                    //skip if we have no gaps, otherwise index.
                    auto new_pos = last_pos + 1;
                    if(pos == new_pos) return true;

                    //index position
                    const auto resolution = index.meta().resolution;
                    CHECK_GREATER(resolution, 0);

                    const auto aliased_time = p.time + (p.offset * resolution);
                    index_item index_entry = {aliased_time, new_pos};

                    CHECK_LESS_EQUAL(aliased_time, t);
                    index.push_back(index_entry);
                }
            }
            else return false;
        }
        //We have an empty timeline, let's add initial data point and index it.
        else
        {
            CHECK_EQUAL(data.size(), 0);

            data_item v{c, c, c * c};
            data.push_back(v);

            index_item i = {t, 0};
            index.push_back(i);
        }

        return true;
    }

    summary_result timeline::summary() const
    {
        const auto resolution = index.meta().resolution;
        CHECK_GREATER(resolution, 0);

        if(index.empty()) return summary_result{0,0,resolution, 0,0,0,0};
        REQUIRE(!data.empty());

        const auto front = index.front();
        const auto back = index.back();

        //time of first bucket
        const auto from = front.time;

        //compute time of last bucket
        CHECK_GREATER(data.size(), back.pos);
        auto last_buckets = data.size() - back.pos;
        auto to = back.time + (last_buckets * resolution);

        CHECK_GREATER(to, from);

        count_type n = (to - from) /  resolution;

        //if we have one bucket then first is empty data item
        auto first_bucket = data_item{0,0,0};
        auto last_bucket = data.back();

        //diff the two buckets
        auto diff = diff_buckets(from, to, resolution, 0, first_bucket, last_bucket, n);
        return summary_result
        {
            from,
            to,
            resolution,
            diff.sum,
            diff.mean,
            diff.variance,
            n
        };
    }

    void clamp(pos_result& r, std::size_t size)
    {
        REQUIRE_LESS(r.pos, size);

        const auto pos = r.pos + r.offset;
        if(pos < size) return;
        r.offset = size - r.pos - 1;

        ENSURE_RANGE(r.pos + r.offset, 0, size);
    }

    get_result timeline::get(time_type t, const offset_type index_offset) const
    {
        auto p = index.find_pos(t, index_offset);

        clamp(p, data.size());

        // zero out data before beginning of collection
        const bool before_beginning =  t < p.time;
        const auto dat = before_beginning ? data_item{0,0,0} : data[p.pos + p.offset];

        return get_result
        {
            p.index_offset,
                t,
                p.time,
                p.pos,
                p.offset,
                dat
        };
    }

    diff_result timeline::diff(time_type a, time_type b, const offset_type index_offset) const
    {
        const auto resolution = index.meta().resolution;
        CHECK_GREATER(resolution, 0);

        if(a > b) std::swap(a,b);
        if(data.size() == 0) return diff_result{ a, b, resolution, 0, 0, 0, 0, 0, {0}, {0}};

        auto ar = get(a, index_offset);
        auto br = get(b, index_offset);

        b = std::max(br.query_time, br.range_time);
        a = std::min(ar.query_time, b);

        const auto time_diff = b - a;
        auto n = time_diff / resolution;

        if(n == 0) return diff_result{ a, b, resolution, 0, 0, 0, 0, 0, ar.value, br.value};

        CHECK_GREATER(n , 0);
        CHECK_LESS_EQUAL(ar.index_offset, br.index_offset);
        return diff_buckets(a, b, resolution, ar.index_offset, ar.value, br.value, n);
    }

    timeline from_directory(const std::string& path, const time_type resolution)
    {
        REQUIRE(!path.empty());
        REQUIRE_GREATER(resolution, 0);

        fs::create_directory(path);
        if(!fs::is_directory(path))
            throw std::runtime_error{"path " + path + " is not a directory"};

        fs::path root = path;

        timeline t;

        fs::path idx_data = root / "_.i";
        t.index = std::move(index_type{idx_data, resolution});

        fs::path cdata = root / "_.d";
        t.data = std::move(data_type{cdata, DATA_SIZE});

        return t;
    }
}


///////////////////////////////////////////////////////////////////////////////
// statistics::model::wrap::unary::covariate.hpp                             //
//                                                                           //
//  Copyright 2009 Erwann Rogard. Distributed under the Boost                //
//  Software License, Version 1.0. (See accompanying file                    //
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)         //
///////////////////////////////////////////////////////////////////////////////
#ifndef BOOST_STATISTICS_MODEL_WRAP_UNARY_COVARIATE_HPP_ER_2009
#define BOOST_STATISTICS_MODEL_WRAP_UNARY_COVARIATE_HPP_ER_2009
#include <boost/statistics/model/wrap/unary/detail/base.hpp>

namespace boost{
namespace statistics{
namespace model{

    template<typename X>
    class covariate_wrapper : detail::base_<X>{
        typedef detail::base_<X> base_;
    public:
        typedef X                       covariate_type;
        typedef covariate_wrapper<X>    covariate_wrapper_type;

        // Construction
        covariate_wrapper();
        covariate_wrapper(const X& x);
        covariate_wrapper(const covariate_wrapper&);

        // Access
        const X& covariate()const;
    };

    // Implementation //

    template<typename X>
    covariate_wrapper<X>::covariate_wrapper() : base_(){}

    template<typename X>
    covariate_wrapper<X>::covariate_wrapper( const X& x ) : base_(x){}

    template<typename X>
    covariate_wrapper<X>::covariate_wrapper(const covariate_wrapper& that)
    :base_(that){}

    template<typename X>
    const X&
     covariate_wrapper<X>::covariate()const{ return (this->value); }

    // Free functions

    template<typename X>
    covariate_wrapper<X>
    make_covariate_wrapper(const X& x){
        typedef covariate_wrapper<X> result_type;
        return result_type(x);
    }

}// model
}// statistics
}// boost

#endif


//
// $Id: IPrecursorMaskCodec.hpp 10370 2017-01-16 22:33:53Z chambm $
//
//
// Original author: Jarrett Egertson <jegertso .@. uw.edu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#ifndef _IPRECURSORMASKCODEC_HPP
#define _IPRECURSORMASKCODEC_HPP

#include "DemuxTypes.hpp"
#include <boost/format.hpp>

namespace pwiz{
namespace analysis{

    inline double precursor_upper_offset(const msdata::Precursor& p)
    {
        auto upperOffsetParam = p.isolationWindow.cvParam(cv::MS_isolation_window_upper_offset);
        if (upperOffsetParam.value.empty())
            throw std::runtime_error("precursor_upper_offset() No isolation window upper offset m/z specified");
        double upperOffset = upperOffsetParam.valueAs<double>();
        if (upperOffset <= 0.0)
            throw std::runtime_error("precursor_upper_offset() Positive values expected for isolation window m/z offsets");
        return upperOffset;
    }

    inline double precursor_lower_offset(const msdata::Precursor& p)
    {
        auto lowerOffsetParam = p.isolationWindow.cvParam(cv::MS_isolation_window_lower_offset);
        if (lowerOffsetParam.value.empty())
            throw std::runtime_error("precursor_lower_offset() No isolation window lower offset m/z specified");
        double lowerOffset = lowerOffsetParam.valueAs<double>();
        if (lowerOffset <= 0.0)
            throw std::runtime_error("precursor_lower_offset() Positive values expected for isolation window m/z offsets");
        return lowerOffset;
    }

    inline double precursor_target(const msdata::Precursor& p)
    {
        auto targetParam = p.isolationWindow.cvParam(cv::MS_isolation_window_target_m_z);
        if (targetParam.value.empty())
            throw std::runtime_error("precursor_target() No isolation window target m/z specified");
        return targetParam.valueAs<double>();
    }

    inline double precursor_mz_low(const msdata::Precursor& p)
    {
        return precursor_target(p) - precursor_lower_offset(p);
    }

    inline double precursor_mz_high(const msdata::Precursor& p)
    {
        return precursor_target(p) + precursor_upper_offset(p);
    }

    inline double precursor_iso_center(const msdata::Precursor& p)
    {
        double target = precursor_target(p);
        double mzLow = target - precursor_lower_offset(p);
        double mzHigh = target + precursor_upper_offset(p);
        return (mzLow + mzHigh) / 2.0;
    }

    inline double precursor_iso_width(const msdata::Precursor& p)
    {
        return precursor_lower_offset(p) + precursor_upper_offset(p);
    }

    inline std::string prec_to_string(const msdata::Precursor& p)
    {
        return str(boost::format("%.2f") % precursor_iso_center(p));
    }

    inline bool stringToFloatCompare(std::string i, std::string j){ return stof(i) < stof(j); }

    // TODO Rework the MZHash system to be strongly typed. Implicit conversion of uint to float and vice-versa is possible.
    typedef uint64_t MZHash;

    /// A method of hashing an isolation window to a unique long value
    /// mz is and m/z of a unique point in the isolation window, such as
    /// the lower bound, upper bound, or center. This value is multiplied
    /// by 100000000 and rounded to convert the isolation m/z to an integer that
    /// is used as the hash. This creates an effective fuzzy window of
    /// +/- 5e-8 m/z. For example: a window with m/z 500.49 would
    /// be hashed to 50049000000.
    class IsoWindowHasher
    {
    public:

        /// Hash a floating-point m/z value to an integer
        static MZHash Hash(double mz)
        {
            auto mult = mz * 100000000.0;
            auto rounded = llround(mult);
            return static_cast<MZHash>(rounded);
        }

        /// Unhash an integer to a floating-point m/z value
        static double UnHash(MZHash hashed)
        {
            return hashed / 100000000.0;
        }
    };

    /// A container for describing the isolation windows that are dedicated to columns of the design matrix for demultiplexing.
    ///
    /// Ideally, a ProteoWizard Precursor container could be used instead with manipulation of its internal data since a demultiplexed spectrum
    /// is in many ways able to be thought of as an isolated spectrum with narrower isolation boundaries. However, this is used as a slimmer
    /// container in favor of the existing ProteoWizard Precursor because in the case of overlapping spectra the Precursor object would have
    /// to be copied and manipulated to split the isolation ranges.
    struct DemuxWindow
    {
        MZHash mzLow; ///< Start m/z of the window range

        MZHash mzHigh; ///< End m/z of the window range

        /// Constructs a DemuxWindow from a Precursor by using its isolation window.
        explicit DemuxWindow(const msdata::Precursor& p)
        {
            double target = precursor_target(p);
            mzLow = IsoWindowHasher::Hash(target - precursor_lower_offset(p));
            mzHigh = IsoWindowHasher::Hash(target + precursor_upper_offset(p));
        }

        /// Constructs a DemuxWindow for a given mass range.
        explicit DemuxWindow(MZHash mzLow, MZHash mzHigh) :    mzLow(mzLow), mzHigh(mzHigh)
        {
        }

        /// Isolation windows are sorted by their start value
        bool operator<(const DemuxWindow& rhs) const { return this->mzLow < rhs.mzLow; }

        /// Can be used to find whether the mass range of another DemuxWindow is a subset of this one.
        bool Contains(const DemuxWindow& inner) const
        {
            return inner.mzLow >= this->mzLow && inner.mzHigh <= this->mzHigh;
        }

        /// Used to find whether a window's center is contained within this window
        bool ContainsCenter(const DemuxWindow& inner) const
        {
            auto center = static_cast<size_t>(llround(inner.mzLow + (inner.mzHigh - inner.mzLow) / 2.0));
            return center >= this->mzLow && center <= this->mzHigh;
        }

        /// Can be used to find whether two windows are identical within the error of the hash
        bool operator==(const DemuxWindow& rhs) const { return rhs.Contains(*this) && this->Contains(rhs); }

        /// Can be used to find whether two windows are identical within the error of the hash
        bool operator!=(const DemuxWindow& rhs) const { return !(*this == rhs); }
    };

    /// A container that wraps DemuxWindow to preserve the full precision window boundaries
    struct IsolationWindow
    {

        /// Constructs an IsolationWindow from a Precursor
        explicit IsolationWindow(const msdata::Precursor& p) :
            lowMz(precursor_mz_low(p)), highMz(precursor_mz_high(p)), demuxWindow(p) {}

        /// Constructs an IsolationWindow from a given mass range
        IsolationWindow(double mzLow, double mzHigh) :
            lowMz(mzLow), highMz(mzHigh), demuxWindow(IsoWindowHasher::Hash(mzLow), IsoWindowHasher::Hash(mzHigh)) {}

        double lowMz; ///< Full precision lower m/z bound

        double highMz; ///< Full precision upper m/z bound

        /// Set of isolation window boundaries that provides useful operations for sorting and comparing
        /// different isolation windows.
        DemuxWindow demuxWindow;

        /// Isolation windows are sorted by their start value
        bool operator<(const IsolationWindow& rhs) const { return this->demuxWindow < rhs.demuxWindow; }
    };

    /// Interface for generating and accessing precursor masks for a demultiplexing scheme.
    class IPrecursorMaskCodec
    {
    public:

        /// Shared pointer definition
        typedef boost::shared_ptr<IPrecursorMaskCodec> ptr;

        /// Constant shared pointer definition
        typedef boost::shared_ptr<const IPrecursorMaskCodec> const_ptr;

        /// Generates a design matrix row describing which precursor isolation windows are present in the given spectrum. This row can be weighted by
        /// a given scalar.
        /// @param[in] sPtr Multiplexed spectrum from which to extract precursor windows.
        /// @param[in] weight Scalar value by which to weight the resulting design matrix vector.
        ///                   This weighting is a simple scalar multiplication of the vector.
        /// @return Design matrix row describing which precursor isolation windows are present in the given spectrum.
        virtual Eigen::VectorXd GetMask(msdata::Spectrum_const_ptr sPtr, double weight = 1.0) const = 0;

        /// Generates a design matrix row describing which precursor isolation windows are present in the given spectrum and places it into the specified
        /// row of the user-provided matrix.
        /// @param[in] sPtr Multiplexed spectrum from which to extract precursor windows.
        /// @param[out] m Matrix in which to place the design vector.
        /// @param[in] rowNum Row of the matrix in which to place the design vector corresponding to the given spectrum.
        /// @param[in] weight Scalar value by which to weight the resulting design matrix vector.
        ///                   This weighting is a simple scalar multiplication of the vector.
        /// \pre Out array must be same size as GetDemuxBlockSize()
        virtual void GetMask(msdata::Spectrum_const_ptr sPtr, DemuxTypes::MatrixType& m, size_t rowNum, double weight = 1.0) const = 0;

        /// Identifies the precursor windows within a spectrum and returns the indices to the design matrix columns corresponding to those windows
        /// @param[in] sPtr Multiplexed spectrum from which to extract precursor windows.
        /// @param[out] indices Indices of the design matrix columns that correspond to the precursor windows in the given spectrum.
        virtual void SpectrumToIndices(msdata::Spectrum_const_ptr sPtr, std::vector<size_t>& indices) const = 0;

        /// Returns the precursor window for a given index.
        /// @param[in] i Index of the column of the design matrix corresponding to the precursor window.
        /// @return A DemuxWindow describing the m/z range of the precursor window.
        virtual IsolationWindow GetIsolationWindow(size_t i) const = 0;

        /// Returns the total number of demux'd precursor windows. This is the number of possible indices returned by SpectrumToIndices().
        virtual size_t GetNumDemuxWindows() const = 0;

        /// Returns the number of spectra required to cover all precursor isolation windows
        virtual int GetSpectraPerCycle() const = 0;

        /// Returns the number of precursor isolations per spectrum. This is verified to be constant for all spectra.
        virtual int GetPrecursorsPerSpectrum() const = 0;

        /// Returns the number of overlap repeats per cycle. So for no overlap, this returns 1. For an overlap that splits each precursor in two, this returns 2. Etc.
        virtual int GetOverlapsPerCycle() const = 0;

        /// Returns the number of windows required to demultiplex
        virtual size_t GetDemuxBlockSize() const = 0;

        /// Returns a descriptor of the processing done by this PrecursorMaskCodec.
        /// WARNING: It is important that this gives a string containing "Demultiplexing" in order for SpectrumWorkerThreads.cpp to handle demultiplexing properly.
        /// @return The processing method performed by this PrecursorMaskCodec
        virtual msdata::ProcessingMethod GetProcessingMethod() const = 0;

        virtual ~IPrecursorMaskCodec() {}
    };

} // namespace analysis
} // namespace pwiz

#endif // _IPRECURSORMASKCODEC_HPP


//
// Created by przemek on 27.03.2020.
//

#include <AST/expression/additive_expr.h>
#include <AST/expression/base_math_expr.h>
#include <boost/test/unit_test.hpp>
#include <error/exeception.h>

using namespace vecc;
using namespace vecc::error;
using namespace vecc::ast;

using std::make_unique;
using std::move;
using std::unique_ptr;

BOOST_AUTO_TEST_SUITE(AST_Test_Suite)

BOOST_AUTO_TEST_SUITE(Add_Expr_Test_Suite)

BOOST_AUTO_TEST_CASE(GivenMathExpr_ValueIsCorrect) {
  Variable var = Variable({1, 2, 3});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));

  BOOST_CHECK_EQUAL(expr.calculate(), var);
}

BOOST_AUTO_TEST_CASE(GivenAdd_ValueIsCorrect) {
  Variable var = Variable({1});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({4}))),
                  AdditiveExpr::OperatorType::Add);

  BOOST_CHECK_EQUAL(expr.calculate(), Variable({5}));
}

BOOST_AUTO_TEST_CASE(GivenSubstract_ValueIsCorrect) {
  Variable var = Variable({1});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({3}))),
                  AdditiveExpr::OperatorType::Substract);

  BOOST_CHECK_EQUAL(expr.calculate(), Variable({-2}));
}

BOOST_AUTO_TEST_CASE(GivenVec2Add_ValueIsCorrect) {
  Variable var = Variable({1, 2});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({-2, 2}))),
                  AdditiveExpr::OperatorType::Add);

  BOOST_CHECK_EQUAL(expr.calculate(), Variable({-1, 4}));
}

BOOST_AUTO_TEST_CASE(GivenVec2Substract_ValueIsCorrect) {
  Variable var = Variable({1, 2});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({-1, 2}))),
                  AdditiveExpr::OperatorType::Substract);

  BOOST_CHECK_EQUAL(expr.calculate(), Variable({2, 0}));
}

BOOST_AUTO_TEST_CASE(GivenVal_ToStringCorrect) {
  Variable var = Variable({1, 2});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));

  BOOST_CHECK_EQUAL(expr.toString(), "vec(1, 2)");
}

BOOST_AUTO_TEST_CASE(GivenAdd_ToStringCorrect) {
  Variable var = Variable({1, 2});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({-2, 2}))),
                  AdditiveExpr::OperatorType::Add);

  BOOST_CHECK_EQUAL(expr.toString(), "(vec(1, 2)+vec(-2, 2))");
}

BOOST_AUTO_TEST_CASE(GivenMultiple_ToStringCorrect) {
  Variable var = Variable({1, 2});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({-1, 2}))),
                  AdditiveExpr::OperatorType::Substract);
  expr.addOperand(make_unique<BaseMathExpr>((Variable({-2, 2}))),
                  AdditiveExpr::OperatorType::Add);

  BOOST_CHECK_EQUAL(expr.toString(), "(vec(1, 2)-vec(-1, 2)+vec(-2, 2))");
}

BOOST_AUTO_TEST_CASE(GivenVec3Add_ValueIsCorrect) {
  Variable var = Variable({1, 2, 3});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({2, 2, 2}))),
                  AdditiveExpr::OperatorType::Add);

  BOOST_CHECK_EQUAL(expr.calculate(), Variable({3, 4, 5}));
}

BOOST_AUTO_TEST_CASE(Givenvec3Substract_ValueIsCorrect) {
  Variable var = Variable({1, 2, 3});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({3, 2, 1}))),
                  AdditiveExpr::OperatorType::Substract);

  BOOST_CHECK_EQUAL(expr.calculate(), Variable({-2, 0, 2}));
}

BOOST_AUTO_TEST_CASE(GivenWrongDimAdd_Throws) {
  Variable var = Variable({1, 2, 3});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({2, 2}))),
                  AdditiveExpr::OperatorType::Add);
  Variable temp;
  BOOST_CHECK_THROW(temp = expr.calculate(), MathException);
}

BOOST_AUTO_TEST_CASE(GivenWrongDimSubstract_Throws) {
  Variable var = Variable({1, 2, 3});

  AdditiveExpr expr(make_unique<BaseMathExpr>(var));
  expr.addOperand(make_unique<BaseMathExpr>((Variable({2, 2}))),
                  AdditiveExpr::OperatorType::Substract);
  Variable temp;
  BOOST_CHECK_THROW(temp = expr.calculate(), MathException);
}

BOOST_AUTO_TEST_SUITE_END()

BOOST_AUTO_TEST_SUITE_END()


//          Copyright Carl Philipp Reh 2006 - 2019.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <sge/opengl/texture/check_dim.hpp>
#include <sge/opengl/texture/pp_dims.hpp>
#include <sge/opengl/texture/warn_min.hpp>
#include <sge/opengl/texture/warn_pow2.hpp>
#include <sge/renderer/basic_dim.hpp>
#include <sge/renderer/size_type.hpp>
#include <fcppt/string.hpp>
#include <fcppt/log/object_fwd.hpp>
#include <fcppt/math/size_type.hpp>
#include <fcppt/config/external_begin.hpp>
#include <boost/preprocessor/seq/for_each.hpp>
#include <fcppt/config/external_end.hpp>

template <fcppt::math::size_type Size>
void sge::opengl::texture::check_dim(
    fcppt::log::object &_log,
    sge::renderer::basic_dim<Size> const &_dim,
    sge::renderer::size_type const _min_value,
    fcppt::string const &_what)
{
  sge::opengl::texture::warn_min(_log, _dim, _min_value, _what);

  sge::opengl::texture::warn_pow2(_log, _dim, _what);
}

#define SGE_OPENGL_TEXTURE_INSTANTIATE_CHECK_DIM(seq, _, dimension) \
  template void sge::opengl::texture::check_dim<dimension>( \
      fcppt::log::object &, \
      sge::renderer::basic_dim<dimension> const &, \
      sge::renderer::size_type, \
      fcppt::string const &);

BOOST_PP_SEQ_FOR_EACH(SGE_OPENGL_TEXTURE_INSTANTIATE_CHECK_DIM, _, SGE_OPENGL_TEXTURE_PP_DIMS)


/*!
  \file gpp_random.hpp
  \rst
  This file specifies two classes for abstracting/handling psuedo-random number generation.  Currently, we have:

  1. UniformRandomGenerator (container for a PRNG "engine")
  2. NormalRNG (functor for N(0, 1)-distributed PRNs, uses UniformRandomGenerator)

  It additionally contains two methods for randomly generating points in a tensor-product domain:
  ``[x_0_min, x_0_max] X [x_1_min, x_1_max] X ... X [x_d_min, x_d_max]``
  This file supports a naive sampling and a latin hypercube sampling.

  Lastly, this file contains a method for randomly generating points in a unit simplex domain.

  These classes exist to ease the use of PRNGs--in particular, they remember the most recent seed so that "rollbacks"
  are easy; and they make it easy to generate unique seeds in multi-threaded environments.  The seeding allows users to
  specify a seed or have the class generate one using a combination of time-of-day and thread id.  They also provide a method to
  print the underlying PRNG's current state.

  UniformRandomGenerator is currently a wrapper for the ``boost::mt19937`` engine (although any boost, C++11, etc. engine
  will work), which is the mersenne twister using a common set of parameters.  (The same functionality is also available
  through ``C++11``'s ``<random>``.)  Unlike NormalRNG, this class is not a functor since the range and even type can change
  frequently in usage.  Thus the actual distribution should be constructed as needed (e.g., see ComputeLatinHypercubePointsInDomain).

  NormalRNG is a functor for generating ``mean = 0, variance = 1``, normally distributed (pseudo) random numbers.  In addition
  to UniformRandomGenerator, NormalRNG also implements operator() to draw from the aforementioned distribution.  N(0, 1) is
  a common choice (and the only one used in gpp_* so far), so NormalRNG wraps the entire number generation process.
\endrst*/

#ifndef MOE_OPTIMAL_LEARNING_CPP_GPP_RANDOM_HPP_
#define MOE_OPTIMAL_LEARNING_CPP_GPP_RANDOM_HPP_

#include <iosfwd>
#include <vector>

#include <boost/random/mersenne_twister.hpp>  // NOLINT(build/include_order)
#include <boost/random/normal_distribution.hpp>  // NOLINT(build/include_order)
#include <boost/random/uniform_real.hpp>  // NOLINT(build/include_order)
#include <boost/random/variate_generator.hpp>  // NOLINT(build/include_order)

#include "gpp_common.hpp"
#include "gpp_exception.hpp"
#include "gpp_geometry.hpp"

namespace optimal_learning {

/*!\rst
  Abstract class for a functor for generating random numbers distributed ~ N(0, 1) (i.e., standard normal).
  This interface currently does not specify many facilities for seeding the underlying RNG as these
  (particularly variable width) can vary by implementation.

  This class *only* has pure virtual functions.
\endrst*/
class NormalRNGInterface {
 public:
  /*!\rst
    Generate a random number from standard normal distribution.

    \return
      a random number from a standard (``N(0, 1)``) normal distribution
  \endrst*/
  virtual double operator()() = 0;

  /*!\rst
    Reseeds the generator with its most recently specified seed value.
    Useful for testing--e.g., can conduct multiple runs with the same initial conditions
  \endrst*/
  virtual void ResetToMostRecentSeed() noexcept = 0;

  virtual ~NormalRNGInterface() = default;
};

/*!\rst
  Container for an uniform random generator (e.g., mersenne twister).  Member functions are for easy manipulation
  of seeds and have signatures matching corresponding members of NormalRNG.

  .. Note:: seed values take type ``EngineType::result_type``. Do not pass in a wider integer type!

  .. WARNING:: this class is NOT THREAD-SAFE. You must construct one object per thread (and
  ensure that the seeds are different for practical computations).
\endrst*/
struct UniformRandomGenerator final {
  using EngineType = boost::mt19937;

  //! Default seed value to make reproducing test results simple.
  static constexpr EngineType::result_type kDefaultSeed = 314;

  /*!\rst
    Default-constructs a UniformRandomGenerator, seeding with kDefaultSeed.
  \endrst*/
  UniformRandomGenerator() noexcept;

  /*!\rst
    Construct a UniformRandomGenerator, seeding with the specified seed.
    See UniformRandomGenerator::SetExplicitSeed for details.

    \param
      :seed: new seed to set
  \endrst*/
  explicit UniformRandomGenerator(EngineType::result_type seed) noexcept;

  /*!\rst
    Construct a UniformRandomGenerator, seeding with an automatically selected seed based on time, thread_id, etc.
    See UniformRandomGenerator::SetRandomizedSeed for details.

    \param
      :base_seed: base value for the new seed
      :thread_id: id of the thread using this object
  \endrst*/
  UniformRandomGenerator(EngineType::result_type seed, int thread_id) noexcept;

  /*!\rst
    Get a reference to the RNG engine used by this class.

    Not necessary for this class since ``engine`` is public but we expose this to maintain a uniform
    interface with NormalRNG.

    \return
      reference to the underlying RNG engine
  \endrst*/
  EngineType& GetEngine() noexcept OL_PURE_FUNCTION OL_WARN_UNUSED_RESULT {
    return engine;
  }

  EngineType::result_type last_seed() const noexcept OL_PURE_FUNCTION OL_WARN_UNUSED_RESULT {
    return last_seed_;
  }

  /*!\rst
    Seed the random number generator with the input value.
    The main purpose of this function is for testing--to allow seeding the RNG with
    a known value for repeatability.

    \param
      :seed: new seed to set
  \endrst*/
  void SetExplicitSeed(EngineType::result_type seed) noexcept;

  /*!\rst
    Set a new seed for the random number generator.  A "random" seed is selected based on
    the input seed value, the current time, and the thread_id.

    This function is meant to initialize unique UniformRandomGenerator objects for
    each computation thread::

      std::vector<UniformRandomGenerator> uniform_generator_vector(num_threads);
      for (int i = 0; i < num_threads; ++i) {
        uniform_generator_vector.SetRandomizedSeed(base_seed, i);
      }

    This function is meant to generate seeds so that:

    1. this function can be called multiple times successively (e.g., in the above loop)
       with different thread_ids to initialize RNGs for multiple threads
    2. multiple runs of this code are unlikely to generate the same seed values

    Item 2. is important for minimizing the probability that we run EI computations
    (see gpp_math.hpp) with the "same" randomness.

    \param
      :base_seed: base value for the new seed
      :thread_id: id of the thread using this object
  \endrst*/
  void SetRandomizedSeed(EngineType::result_type base_seed, int thread_id) noexcept;

  /*!\rst
    Reseeds the generator with its most recently specified seed value.
    Useful for testing--e.g., can conduct multiple runs with the same initial conditions
  \endrst*/
  void ResetToMostRecentSeed() noexcept;

  /*!\rst
    Prints the state of the generator to specified ostream.  For testing.

    \param
      :out_stream[1]: a ``std::ostream`` object ready for `operator<<`` use
    \output
      :out_stream[1]: ``std::ostream`` with the engine state "written" to its ``operator<<``
  \endrst*/
  void PrintState(std::ostream * out_stream) const OL_NONNULL_POINTERS;

  bool operator==(const UniformRandomGenerator& other) const;

  bool operator!=(const UniformRandomGenerator& other) const;

  //! An (boost) PRNG engine that can be passed to a ``<boost/random>`` distribution, e.g., ``uniform_real<>``.
  EngineType engine;

 private:
  //! The last seed value that was written to ``engine``. Useful for testing.
  EngineType::result_type last_seed_;
};

/*!\rst
  Functor for computing normally distributed (N(0, 1)) random numbers.
  Uses/maintains an uniform RNG (currently UniformRandomGenerator) and transforms the output to be
  distributed ~ N(0, 1).

  .. Note:: seed values take type ``EngineType::result_type``. Do not pass in a wider integer type!

  .. WARNING:: this class is NOT THREAD-SAFE. You must construct one object per thread (and
    ensure that the seeds are different for practical computations).
\endrst*/
class NormalRNG final : public NormalRNGInterface {
 public:
  using UniformGeneratorType = UniformRandomGenerator;
  using EngineType = UniformRandomGenerator::EngineType;

  //! Default seed value to make reproducing test results simple.
  static constexpr EngineType::result_type kDefaultSeed = 314;

  /*!\rst
    Default-constructs a NormalRNG, seeding with kDefaultSeed.
  \endrst*/
  NormalRNG() noexcept;

  /*!\rst
    Construct a NormalRNG, seeding with the specified seed.
    See NormalRNG::SetExplicitSeed for details.

    \param
      :seed: new seed to set
  \endrst*/
  explicit NormalRNG(EngineType::result_type seed) noexcept;

  /*!\rst
    Construct a NormalRNG, seeding with an automatically selected seed based on time, thread_id, etc.
    See NormalRNG::SetRandomizedSeed for details.

    \param
      :base_seed: base value for the new seed
      :thread_id: id of the thread using this object
  \endrst*/
  NormalRNG(EngineType::result_type seed, int thread_id) noexcept;

  /*!\rst
    Get a reference to the RNG engine used by this class.

    \return
      reference to the underlying RNG engine
  \endrst*/
  EngineType& GetEngine() noexcept OL_PURE_FUNCTION OL_WARN_UNUSED_RESULT {
    return uniform_generator.engine;
  }

  virtual double operator()();

  EngineType::result_type last_seed() const noexcept OL_PURE_FUNCTION OL_WARN_UNUSED_RESULT {
    return uniform_generator.last_seed();
  }

  /*!\rst
    Clears state of ``normal_distribution_`` so that future uses do not depend on any previous actions.
    This is important: the underlying normal distribution likely generates numbers \emph{two} at a time.
    So re-seeding the engine WITHOUT resetting can lead to surprising behavior.
  \endrst*/
  void ResetGenerator() noexcept;

  /*!\rst
    Seed the random number generator with the input value.
    See UniformRandomGenerator::SetExplicitSeed() for more information.

    \param
      :seed: new seed to set
  \endrst*/
  void SetExplicitSeed(EngineType::result_type seed) noexcept;

  /*!\rst
    Set a new seed for the random number generator.  A "random" seed is selected based on
    the input seed value, the current time, and the thread_id.
    See UniformRandomGenerator::SetExplicitSeed() for more information.

    \param
      :seed: base value for the new seed
      :thread_id: id of the thread using this object
  \endrst*/
  void SetRandomizedSeed(EngineType::result_type seed, int thread_id) noexcept;

  /*!\rst
    Reseeds the generator with its most recently specified seed value.
    Useful for testing--e.g., can conduct multiple runs with the same initial conditions
  \endrst*/
  virtual void ResetToMostRecentSeed() noexcept;

  /*!\rst
    Prints the state of the generator to specified ostream.  For testing.

    \param
      :out_stream[1]: a std::ostream object ready for operator<< use
    \output
      :out_stream[1]: std::ostream with engine state <<'d to it
  \endrst*/
  void PrintState(std::ostream * out_stream) const OL_NONNULL_POINTERS;

  //! The underlying generator providing uniform PRNGs for this object to transform to N(0, 1).
  UniformGeneratorType uniform_generator;

 private:
  //! Object for ransforming from uniform to N(0, 1); may carry internal state (e.g., normal random numbers generated 2 at a time).
  boost::normal_distribution<double> normal_distribution_;
  //! Object (for convenience) providing operator() that returns a value distributed ~ N(0, 1).
  boost::variate_generator<EngineType&, boost::normal_distribution<double> > normal_random_variable_;
};

/*!\rst
  RNG that generates normally distributed (N(0,1)) random numbers simply by reading random numbers stored in
  its "random_number_table", a data member in this class.

  .. Note:: this class is used in unit test only, and you have to be careful to ensure the total number of random numbers
    generated from last reset must be smaller than size of "random_number_table", otherwise exception will be thrown.

  .. Warning:: this class is NOT THREAD-SAFE. You must construct one object per thread.
\endrst*/
class NormalRNGSimulator final : public NormalRNGInterface {
 public:
  /*!\rst
    Construct a NormalRNGSimulator by providing table storing random numbers, and size of this random table.

    \param
      :random_number_table_in: pointer to the table storing random numbers
      :size_of_table_in: size of the random table
  \endrst*/
  explicit NormalRNGSimulator(const std::vector<double>& random_number_table_in);

  virtual double operator()();

  virtual void ResetToMostRecentSeed() noexcept;

  int index() const {
    return index_;
  }

  OL_DISALLOW_DEFAULT_AND_COPY_AND_ASSIGN(NormalRNGSimulator);

 private:
  //! Table that stores all random numbers.
  std::vector<double> random_number_table_;
  //! Index of the random number in the table to return when the generator is called.
  int index_;
};

/*!\rst
  Computes a set of random points inside some domain that lie in a latin hypercube.  In 2D, a latin hypercube is a latin
  square--a checkerboard--such that there is exactly one sample in each row and each column.  This notion is generalized
  for higher dimensions where each dimensional "slice" has precisely one sample.

  See wikipedia: http://en.wikipedia.org/wiki/Latin_hypercube_sampling
  for more details on the latin hypercube sampling process.

  \param
    :domain[dim]: array of ClosedInterval specifying the boundaries of a dim-dimensional tensor-product domain.
    :dim: the number of spatial dimensions
    :num_samples: number of random points desired
    :uniform_generator[1]: a UniformRandomGenerator object providing the random engine for uniform random numbers
  \output
    :uniform_generator[1]: UniformRandomGenerator object will have changed state due to random draws
    :random_points[num_samples][dim]: array containing random points inside the domain
\endrst*/
OL_NONNULL_POINTERS void ComputeLatinHypercubePointsInDomain(ClosedInterval const * restrict domain, int dim, int num_samples, UniformRandomGenerator * uniform_generator, double * restrict random_points);

/*!\rst
  Computes a set of random points that lie inside a dim-dimensional simplex or d-simplex.
  The points are uniformly-distributed by volume.

  \param
    :dim: the number of spatial dimensions
    :num_samples: number of random points desired
    :uniform_generator[1]: a UniformRandomGenerator object providing the random engine for uniform random numbers
  \output
    :uniform_generator[1]: UniformRandomGenerator object will have changed state due to random draws
    :random_points[num_samples][dim]: array containing random points inside the domain
\endrst*/
OL_NONNULL_POINTERS void ComputeUniformPointsInUnitSimplex(int dim, int num_samples, UniformRandomGenerator * uniform_generator, double * restrict random_points);

/*!\rst
  Computes a "random" point guaranteed to be within the specified domain boundaries (inclusive).
  No inputs may be nullptr.

  "random": This code simply draws a uniform random coordinate from this direction.  Each coordinate is drawn
  independently and multiple calls to the function are independent.  As indicated under "random sampling" here,
  http://en.wikipedia.org/wiki/Latin_hypercube_sampling
  the we have no guarantees on the actual distribution of the resulting point set since we make no effort to control it.

  WARNING: this function is NOT THREAD-SAFE.

  \param
    :domain[dim]: array of ClosedInterval specifying the boundaries of a dim-dimensional tensor-product domain.
    :dim: the number of spatial dimensions
    :uniform_generator[1]: a UniformRandomGenerator object providing the random engine for uniform random numbers
  \output
    :uniform_generator[1]: UniformRandomGenerator object will have its state changed due to random draws
    :random_point[dim]: array containing a random point inside the domain
\endrst*/
inline OL_NONNULL_POINTERS void ComputeRandomPointInDomain(ClosedInterval const * restrict domain, int dim, UniformRandomGenerator * uniform_generator, double * restrict random_point) noexcept {
  for (int i = 0; i < dim; ++i) {
    boost::uniform_real<double> uniform_double(domain[i].min, domain[i].max);
    random_point[i] = uniform_double(uniform_generator->engine);
  }
}

}  // end namespace optimal_learning

#endif  // MOE_OPTIMAL_LEARNING_CPP_GPP_RANDOM_HPP_


//######################################################################
//#   GT Writter Module
//#
//#   Copyright (C) 2020 Siemens AG
//#   SPDX-License-Identifier: MIT
//#   Author 2020: This module has been developed by
//#                Roman Kaskman under supervision of Slobodan Ilic
//#######################################################################

#include <Eigen/Core>
#include "util.h"
#include <nlohmann/json.hpp>
#include "io_utils.hpp"
#include <filesystem>
#include "sequence_composer.h"
#include "output_writer.h"

using json = nlohmann::json;
namespace fs = std::filesystem;

using namespace std;
using namespace Eigen;

#define CHECK_PARAM_EXISTS(parser, param_name) \
               if (!parser.has(param_name)) {\
                  cerr << "Missing parameter: " << param_name << endl;\
                  return false;\
               }

#define CHECK_VALID_DIR(dir) \
      if (!fs::is_directory(dir)) {\
        cerr << "Invalid directory: " << dir << endl;\
        return false;\
      }

#define CHECK_VALID_FILE(f) \
     if (!fs::is_regular_file(f)) {\
       cerr << "Invalid file: " << f << endl;\
       return false;\
     }

bool parse_scenes_dirs(const string &scenes_dirs_string, vector<string> &scenes_dirs)
{
	stringstream scenes_ss(scenes_dirs_string);
	try
	{
		while (scenes_ss.good())
		{
			string scene_dir;
			getline(scenes_ss, scene_dir, ',');
			CHECK_VALID_DIR(scene_dir)
			scenes_dirs.push_back(scene_dir);
		}
	}
	catch (const std::exception&)
	{
		cerr << "Invalid scenes dirs: " << scenes_dirs_string << endl;
		return false;
	}

	if (scenes_dirs.empty())
	{
		cerr << "No scene directories provided" << endl;
		return false;
	}

	return true;

}

bool get_image_size(const Configuration &configuration, int &width, int &height)
{
	const vector<string>& scene_dirs = configuration.get_scenes_dirs();

	string first_dir = scene_dirs[0];
	fs::path rgb_dir = (fs::path(first_dir) / "images" / "rgb");

	//pick the first file
	string filename = get_image_filename_for_idx(0);
	string first_file = (rgb_dir / filename).string();
	CHECK_VALID_FILE(first_file)

	cv::Mat first_image = cv::imread(first_file);

	height = first_image.rows;
	width = first_image.cols;

	return height != 0 && width != 0;
}

bool parse_configuration(const cv::CommandLineParser &parser, Configuration &configuration)
{
	CHECK_PARAM_EXISTS(parser, "intrinsics_file")
	string intrinsics_file = parser.get<string>("intrinsics_file");
	CHECK_VALID_FILE(intrinsics_file)
	Matrix3f intrinsics = read_intrinsics_from_file<float>(intrinsics_file);

	CHECK_PARAM_EXISTS(parser, "reference_models_dir")
	string reference_models_dir = parser.get<string>("reference_models_dir");
	CHECK_VALID_DIR(reference_models_dir)

	CHECK_PARAM_EXISTS(parser, "out_dir")
	string out_dir = parser.get<string>("out_dir");

	CHECK_PARAM_EXISTS(parser, "scenes_dirs")
	string scenes_dirs_string = parser.get<string>("scenes_dirs");

	vector<string> scenes_dirs;
	if (!parse_scenes_dirs(scenes_dirs_string, scenes_dirs)) return false;

	configuration.set_scenes_dirs(scenes_dirs);

	bool copy_images = parser.has("copy_images");

	int image_width, image_height;

	if (!get_image_size(configuration, image_width, image_height)) return false;

	configuration.set_intrinsics(intrinsics);
	configuration.set_reference_models_dir(reference_models_dir);
	configuration.set_out_dir(out_dir);
	configuration.set_focal_length_scale(0.5f);
	configuration.set_copy_images(copy_images);
	configuration.set_scenes_dirs(scenes_dirs);
	configuration.set_image_height(image_height);
	configuration.set_image_width(image_width);

	return true;
}

int main(int argc, char* argv[])
{
	const cv::String arg_keys =
		"{help h |      | help message   }"
		"{intrinsics_file       |     | intrinsics file         }"
		"{reference_models_dir           |     | directory with reference models         }"
		"{scenes_dirs           |     | comma separated directories to be joined         }"
		"{out_dir           |     | output directory         }"
		"{copy_images           |     | should copy and reindex images         }";

	cv::CommandLineParser parser(argc, argv, arg_keys);

	if (parser.has("help")) {
		parser.printMessage();
		return EXIT_SUCCESS;
	}

	Configuration configuration;
	if (!parse_configuration(parser, configuration)) return EXIT_FAILURE;

	vector<Frame> joined_sequences;
	if (!compose_joined_sequence(configuration, joined_sequences)) return EXIT_FAILURE;

	write_output(configuration, joined_sequences);

	return 0;
}


#include <iostream>
#include <stdint.h>
#include <vector>

#include <boost/filesystem.hpp>

#include <opencv2/core/core.hpp>
#include <opencv2/imgcodecs.hpp>

#include <aks/AksKernelBase.h>
#include <aks/AksDataDescriptor.h>
#include <aks/AksNodeParams.h>

class OpticalFlowPostProcess : public AKS::KernelBase
{
  public:
    int id =0;
    int exec_async (
           std::vector<AKS::DataDescriptor*> &in,
           std::vector<AKS::DataDescriptor*> &out,
           AKS::NodeParams* nodeParams,
           AKS::DynamicParamValues* dynParams);
};

void boundPixels(AKS::DataDescriptor* src, cv::Mat& dst, int bound) {
  auto shape = src->getShape();
  int rows = shape[0];
  int cols = shape[1];
  float* srcPtr = static_cast<float*>(src->data());
  for (int i=0; i<rows; ++i) {
    for (int j=0; j<cols; ++j) {
      float x = srcPtr[i*cols+j];
      x = (x + bound) * (255.0 / (2 * bound));
      int y = cvRound(x);
      dst.at<uint8_t>(i, j) = (y>255 ? 255 : (y<0 ? 0 : y));
    }
  }
}

void mat2DD(const cv::Mat &src, int8_t* dst) {
  // Gray scale cv::Mat --> Gray scale DataDescriptor
  int channels = src.channels();
  int rows = src.rows;
  int cols = src.cols;
  for (int k=0; k<channels; k++) {
    for (int i=0; i<rows; i++) {
      for (int j=0; j<cols; j++) {
        dst[(k*rows*cols) + (i*cols) + j] = src.at<cv::Vec<uint8_t, 2>>(i,j)[k] - 128;
      }
    }
  }
}

extern "C" { // Add this to make this available for python bindings and

AKS::KernelBase* getKernel (AKS::NodeParams *params)
{
  return new OpticalFlowPostProcess();
}

int OpticalFlowPostProcess::exec_async (
                      std::vector<AKS::DataDescriptor*> &in,
                      std::vector<AKS::DataDescriptor*> &out,
                      AKS::NodeParams* nodeParams,
                      AKS::DynamicParamValues* dynParams)
{
    // in[0] contains flowx data
    // in[1] contains flowy data

    // std::cout << "[DBG] OpticalFlowPostProcess: running now ... " << std::endl;
    auto shape = in[0]->getShape();
    int rows = shape[0];
    int cols = shape[1];

    int bound = nodeParams->_intParams["bound"];
    cv::Mat boundedFlowX(rows, cols, CV_8UC1);
    cv::Mat boundedFlowY(rows, cols, CV_8UC1);

    float* flowxData = static_cast<float*>(in[0]->data());
    float* flowyData = static_cast<float*>(in[1]->data());

    boundPixels(in[0], boundedFlowX, bound);
    boundPixels(in[1], boundedFlowY, bound);

    cv::Mat flow[2] = { boundedFlowX, boundedFlowY };
    cv::Mat bounded(rows, cols, CV_8UC(2));
    cv::merge(flow, 2, bounded);

    AKS::DataDescriptor *flowDD = new AKS::DataDescriptor(
        { 1, 2, rows, cols }, AKS::DataType::INT8);
    int8_t* flowData = static_cast<int8_t*>(flowDD->data());
    mat2DD(bounded, flowData);
    out.push_back(flowDD);

    std::string output_folder = \
      nodeParams->_stringParams.find("visualize") == nodeParams->_stringParams.end() ?
        "" : nodeParams->_stringParams["visualize"];
    if (!output_folder.empty()) {
        boost::filesystem::path p(dynParams->imagePaths.front());
        std::string filename = p.filename().string();
        std::string folder = p.parent_path().filename().string();
        std::string output_path = output_folder + "/" + folder;
        boost::filesystem::create_directories(output_path);
        std::string tmp = "_" + filename;
        cv::imwrite(output_path + "/flow_x" + tmp, boundedFlowX);
        cv::imwrite(output_path + "/flow_y" + tmp, boundedFlowY);
    }

    // std::cout << "[DBG] OpticalFlowPostProcess: Done!" << std::endl << std::endl;
    return -1; // No wait
}

} //extern "C"


//==================================================================================================
/*!

  Copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#include <boost/simd/constant/expx2c2.hpp>
#include <boost/simd/as.hpp>
#include <stf.hpp>

STF_CASE_TPL( "Check expx2c2 behavior for integral types"
            , (std::uint32_t)(std::uint64_t)
              (std::int32_t)(std::int64_t)
            )
{
  using boost::simd::as;
  using boost::simd::detail::expx2c2;
  using boost::simd::Expx2c2;
  T ref = T(0);
  STF_TYPE_IS(decltype(Expx2c2<T>()), T);
  STF_EQUAL(Expx2c2<T>(), ref);
  STF_EQUAL(expx2c2( as(T{}) ),ref);
}

STF_CASE_TPL( "Check expx2c2 behavior for double"
            , (double)
            )
{
  using boost::simd::as;
  using boost::simd::detail::expx2c2;
  using boost::simd::Expx2c2;
  T ref = T(1.0/128);

  STF_TYPE_IS(decltype(Expx2c2<T>()), T);
  STF_IEEE_EQUAL(Expx2c2<T>(), ref);
  STF_IEEE_EQUAL(expx2c2( as(T{}) ), ref);
}

STF_CASE_TPL( "Check expx2c2 behavior for float"
            , (float)
            )
{
  using boost::simd::as;
  using boost::simd::detail::expx2c2;
  using boost::simd::Expx2c2;
  T ref = 1.f/32;
  STF_TYPE_IS(decltype(Expx2c2<T>()), T);
  STF_IEEE_EQUAL(Expx2c2<T>(), ref);
  STF_IEEE_EQUAL(expx2c2( as(T{}) ), ref);
}


// Boost.Units - A C++ library for zero-overhead dimensional analysis and
// unit/quantity manipulation and conversion
//
// Copyright (C) 2003-2008 Matthias Christian Schabel
// Copyright (C) 2008 Steven Watanabe
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

/**
\file

\brief temperature.cpp

\details
Conversions between Fahrenheit and Kelvin for absolute temperatures and
temperature differences.

Output:
@verbatim

//[ temperature_output_1
{ 32 } F
{ 273.15 } K
{ 273.15 } K
[ 32 ] F
[ 17.7778 ] K
[ 17.7778 ] K
//]

@endverbatim
**/

#include <iomanip>
#include <iostream>

#include <boost/units/absolute.hpp>
#include <boost/units/get_system.hpp>
#include <boost/units/io.hpp>
#include <boost/units/unit.hpp>
#include <boost/units/quantity.hpp>
#include <boost/units/systems/si/temperature.hpp>
#include <boost/units/detail/utility.hpp>

#include <boost/units/base_units/temperature/fahrenheit.hpp>

using namespace boost::units;

namespace boost {

namespace units {

namespace fahrenheit {

//[temperature_snippet_1
typedef temperature::fahrenheit_base_unit::unit_type    temperature;
typedef get_system<temperature>::type                   system;

BOOST_UNITS_STATIC_CONSTANT(degree,temperature);
BOOST_UNITS_STATIC_CONSTANT(degrees,temperature);
//]

} // fahrenheit

} // namespace units

} // namespace boost

int main()
{
    //[temperature_snippet_3
    quantity<absolute<fahrenheit::temperature> >    T1p(
        32.0*absolute<fahrenheit::temperature>());
    quantity<fahrenheit::temperature>               T1v(
        32.0*fahrenheit::degrees);

    quantity<absolute<si::temperature> >            T2p(T1p);
    quantity<si::temperature>                       T2v(T1v);
    //]

    typedef conversion_helper<
        quantity<absolute<fahrenheit::temperature> >,
        quantity<absolute<si::temperature> > >          absolute_conv_type;
    typedef conversion_helper<
        quantity<fahrenheit::temperature>,
        quantity<si::temperature> >                     relative_conv_type;

    std::cout << T1p << std::endl
              << absolute_conv_type::convert(T1p) << std::endl
              << T2p << std::endl
              << T1v << std::endl
              << relative_conv_type::convert(T1v) << std::endl
              << T2v << std::endl
              << std::endl;

    return 0;
}