/** svd.cc
    Jeremy Barnes, 16 December 2014
    Copyright (c) 2014 mldb.ai inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

    Implementation of an SVD algorithm for embedding of a dataset.
*/

#include "svd.h"
#include "mldb/builtin/matrix.h"
#include "mldb/core/mldb_engine.h"
#include "mldb/core/dataset.h"
#include "mldb/utils/distribution.h"
#include <boost/multi_array.hpp>
#include "mldb/base/scope.h"
#include "mldb/base/parallel.h"
#include "mldb/utils/pair_utils.h"
#include "mldb/arch/timers.h"
#include "mldb/arch/simd_vector.h"
#include "mldb/builtin/intersection_utils.h"
#include "mldb/utils/vector_utils.h"
#include "mldb/ext/svdlibc/svdlib.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/types/distribution_description.h"
#include "mldb/types/optional_description.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/builtin/sql_config_validator.h"
#include "mldb/vfs/fs_utils.h"
#include "mldb/types/map_description.h"
#include "mldb/types/any_impl.h"
#include "mldb/types/annotated_exception.h"
#include "mldb/types/hash_wrapper_description.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/utils/progress.h"
#include "mldb/utils/log.h"
#include <sstream>

using namespace std;

namespace MLDB {

// jsonDecode implementation for any type which:
// 1) has a default description;
template<typename T>
T jsonDecodeFile(const std::string & filename, T * = 0)
{
    T result;

    MLDB::filter_istream stream(filename);

    static auto desc = MLDB::getDefaultDescriptionSharedT<T>();
    MLDB::StreamingJsonParsingContext context(filename, stream);
    desc->parseJson(&result, context);
    return result;
}

DEFINE_STRUCTURE_DESCRIPTION(SvdConfig);

SvdConfigDescription::
SvdConfigDescription()
{
    Optional<PolyConfigT<Dataset> > optionalOutputDataset;
    optionalOutputDataset.emplace(PolyConfigT<Dataset>().
                                  withType(SvdConfig::defaultOutputDatasetType));

    addField("trainingData", &SvdConfig::trainingData,
             "Specification of the data for input to the SVD Procedure.  This should be "
             "organized as an embedding, with each selected row containing the same "
             "set of columns with numeric values to be used as coordinates.  The select statement "
             "does not support groupby and having clauses.");
    addField("columnOutputDataset", &SvdConfig::columnOutput,
             "Output dataset for embedding (column singular vectors go here)",
             optionalOutputDataset);
    addField("rowOutputDataset", &SvdConfig::rowOutput,
             "Output dataset for embedding (row singular vectors go here)",
             optionalOutputDataset);
    addField("numSingularValues", &SvdConfig::numSingularValues,
             "Maximum number of singular values to work with.  "
             "If there are not enough "
             "degrees of freedom in the dataset (it is rank-deficient), then "
             "less than this number may be used", 100);
    addField("numDenseBasisVectors", &SvdConfig::numDenseBasisVectors,
             "Maximum number of dense basis vectors to use for the SVD.  "
             "This parameter gives the number of dimensions into which the "
             "projection is made.  Higher values may allow the SVD to model "
             "slightly more diverse behaviour.  "
             "The runtime goes up with the square of this parameter, "
             "in other words 10 times as many is 100 times as long to run.",
             2000);
    addField("outputColumn", &SvdConfig::outputColumn,
             "Base name of the column that will be written by the SVD.  "
             "It will be an embedding with numSingularValues elements.",
             PathElement("embedding"));
    addField("modelFileUrl", &SvdConfig::modelFileUrl,
             "URL where the model file (with extension '.svd') should be saved. "
             "This file can be loaded by the ![](%%doclink svd.embedRow function). "
             "This parameter is optional unless the `functionName` parameter is used.");
    addField("functionName", &SvdConfig::functionName,
             "If specified, an instance of the ![](%%doclink svd.embedRow function) of this name will be created using "
             "the trained model. Note that to use this parameter, the `modelFileUrl` must "
             "also be provided.");
    addParent<ProcedureConfig>();

    onPostValidate = chain(validateQuery(&SvdConfig::trainingData,
                                         NoGroupByHaving(),
                                         MustContainFrom()),
                           validateFunction<SvdConfig>());
}

DEFINE_STRUCTURE_DESCRIPTION(SimpleIntersectionEntry);

SimpleIntersectionEntryDescription::
SimpleIntersectionEntryDescription()
{
    addParent<ColumnSpec>();
    addField("singularVector", &SimpleIntersectionEntry::singularVector,
             "Singular vector for this column");
}

DEFINE_STRUCTURE_DESCRIPTION(SvdColumnIndexEntry);

SvdColumnIndexEntryDescription::
SvdColumnIndexEntryDescription()
{
    addField("columnName", &SvdColumnIndexEntry::columnName,
             "Name of the column");
    addField("values", &SvdColumnIndexEntry::values,
             "Values of fields for this column");
}

/** Given the other column, project it onto the basis. */
distribution<float>
SvdBasis::
rightSingularVector(const ColumnIndexEntries & basisColumns,
                    const ColumnIndexEntry & column,
                    shared_ptr<spdlog::logger> logger) const
{
    // For each basis vector, calculate the overlap
    distribution<float> result(singularValues.size());

    for (unsigned i = 0;  i < columns.size();  ++i) {
        double overlap = column.correlation(basisColumns.at(i));
        if (overlap == 0.0)
            continue;
        result += overlap * columns[i].singularVector;
    }

    result /= singularValues * singularValues;

    if (logger->should_log(spdlog::level::trace) && (result == 0.0).all()) {
        logger->trace() <<  "all zero projection"
                        << "column " << column.columnName;

        for (unsigned i = 0;  i < columns.size();  ++i) {
            double overlap = column.correlation(basisColumns.at(i));
            logger->trace() << "overlap with " << columns[i].columnName
                            << " is " << overlap;
        }

    }

    return result;
}

distribution<float>
SvdBasis::
rightSingularVectorForColumn(ColumnHash col, const CellValue & value,
                             int maxValues,
                             bool acceptUnknownValues,
                             shared_ptr<spdlog::logger> logger) const
{
    if (maxValues < 0 || maxValues > singularValues.size())
        maxValues = singularValues.size();

    // 1.  Find the columns involved with the index
    auto it = columnIndex.find(col);
    if (it == columnIndex.end()) {
        TRACE_MSG(logger) << "column " << col << " not found in "
                          << columnIndex.size() << " entries";
        return distribution<float>();
    }

    // 2.  Look up the value of the cell
    const SvdColumnIndexEntry & columnEntry = it->second;

    auto it2 = columnEntry.values.find(value);
    if (it2 == columnEntry.values.end()) {
        // Extract the cell value...
        if (value.isNumeric()) {
            double d = value.toDouble();

            TRACE_MSG(logger) << "looking into " << columnEntry.values.size() << " values"
                              << " for column " << it->second.columnName;

            for (auto & e: columnEntry.values) {
                if (logger->should_log(spdlog::level::trace)) {
                    logger->trace() << "e.first = " << jsonEncodeStr(e.first);
                    logger->trace() << "e.first.cellType() = " << jsonEncodeStr(e.first.cellType());
                    logger->trace() << "e.second = " << e.second;
                }

                if (!e.first.empty() && e.first != value)
                    continue;
                int columnNum = e.second;
                auto & col = columns[columnNum];
                distribution<float> result = col.singularVector;
                result.resize(maxValues);

                double oldd = d;

                d += col.offset;
                d *= col.scale;

                TRACE_MSG(logger) << "value " << oldd << " transformed to " << d;

                result *= d;
                return result;
            }

            if (columnEntry.values.size() == 1) {
                // MLDB-687
                // We saw only a single value in training, which makes it
                // essentially a "has value" or "doesn't have value"
                // feature.  Take the output for the one and only value
                // seen in training.

                auto & col = columns[columnEntry.values.begin()->second];

                distribution<float> result = col.singularVector;
                result.resize(maxValues);

                double oldd = d;

                d += col.offset;
                d *= col.scale;

                TRACE_MSG(logger) << "value " << oldd << " transformed to " << d;

                result *= d;
                return result;
            }

        }

        if (acceptUnknownValues) {
            DEBUG_MSG(logger) << "Numeric value not found";
            return distribution<float>();
        }

        Json::Value details;
        details["columnName"] = jsonEncode(it->second.columnName);
        details["columnValueNotFound"] = jsonEncode(value);
        details["columnValueNotFoundType"] = jsonEncode(value.cellType());

        auto & vals = details["firstTenknownColumnValues"];

        bool expectedNumber = false;
        bool onlyNumber = true;
        for (auto & v: it->second.values) {
            if (vals.size() >= 10)
                break;

            auto & col = columns.at(v.second);

            Json::Value thisVal;
            if (col.op == COL_VALUE) {
                thisVal = "<<any number>>";
                expectedNumber = true;
            }
            else {
                thisVal = jsonEncode(v.first);
                onlyNumber = false;
            }

            vals.append(thisVal);
        }

        Utf8String message;
        if (value.isNumeric()) {
            message = "Column '" + it->second.columnName.toUtf8String()
                + "' was a string in training but has numeric value "
                + jsonEncodeUtf8(value) + " when passed to SVD";
        }
        else if (expectedNumber && onlyNumber) {
            message = "Column '" + it->second.columnName.toUtf8String()
                + "' passed as a string value " + jsonEncodeUtf8(value)
                + " but only numbers were seen in training";
        }
        else {
            message = "Column '" + it->second.columnName.toUtf8String()
                + "' passed a value " + jsonEncodeUtf8(value)
                + " that was never seen in training when passed to SVD";
        }

        DEBUG_MSG(logger) << details;

        throw AnnotatedException(400, message, details);
    }

    distribution<float> result = columns[it2->second].singularVector;
    result.resize(maxValues);
    return result;
}

std::pair<distribution<float>, Date>
SvdBasis::
leftSingularVector(const std::vector<std::tuple<ColumnPath, CellValue, Date> > & row,
                   int maxValues,
                   bool acceptUnknownValues,
                   shared_ptr<spdlog::logger> logger) const
{
    return doLeftSingularVector(row, maxValues, acceptUnknownValues, logger);
}

std::pair<distribution<float>, Date>
SvdBasis::
leftSingularVector(const std::vector<std::tuple<ColumnHash, CellValue, Date> > & row,
                   int maxValues,
                   bool acceptUnknownValues,
                   shared_ptr<spdlog::logger> logger) const
{
    return doLeftSingularVector(row, maxValues, acceptUnknownValues, logger);
}

template<typename Tuple>
std::pair<distribution<float>, Date>
SvdBasis::
doLeftSingularVector(const std::vector<Tuple> & row,
                     int maxValues,
                     bool acceptUnknownValues,
                     shared_ptr<spdlog::logger> logger) const
{
    if (maxValues < 0 || maxValues > singularValues.size())
        maxValues = singularValues.size();

    Date ts = modelTs;
    distribution<float> result(maxValues);

    for (auto & v: row) {
        ColumnHash column;
        CellValue value;
        Date columnTs;

        std::tie(column, value, columnTs) = v;

        const distribution<float> & rsv
            = rightSingularVectorForColumn(column, value, maxValues, acceptUnknownValues, logger);

        // If it was excluded, it will have an empty vector calculated
        if (rsv.empty())
            continue;
        result += rsv;
        ts.setMax(columnTs);
    }

    for (unsigned i = 0;  i < result.size();  ++i)
        result[i] /= singularValues[i];

    return make_pair(std::move(result), ts);
}

void
SvdBasis::
validate()
{
    for (unsigned i = 0;  i < columns.size();  ++i) {
        auto & c = columns[i];
        ExcAssert(columnIndex.count(c.columnName));
        ExcAssert(columnIndex[c.columnName].values.count(c.cellValue));
        ExcAssertEqual(columnIndex[c.columnName].columnName, c.columnName);
        ExcAssertEqual(columnIndex[c.columnName].values[c.cellValue], i);
        if (c.op == COL_VALUE)
            ExcAssertEqual(c.cellValue, CellValue());
    }
}

DEFINE_STRUCTURE_DESCRIPTION(SvdBasis);

SvdBasisDescription::
SvdBasisDescription()
{
    addField("columns", &SvdBasis::columns, "Columns of SVD");
    addField("singularValues", &SvdBasis::singularValues,
             "Singular values of SVD");
    addField("columnIndex", &SvdBasis::columnIndex, "Index of columns");
    addField("modelTs", &SvdBasis::modelTs, "Timestamp of latest information incorporated into model");
}

struct SvdTrainer {
    static SvdBasis calcSvdBasis(const ColumnCorrelations & correlations,
                                 int numSingularValues,
                                 shared_ptr<spdlog::logger> logger);

    static SvdBasis calcRightSingular(const ClassifiedColumns & columns,
                                      const ColumnIndexEntries & columnIndex,
                                      const SvdBasis & svd,
                                      shared_ptr<spdlog::logger> logger);
};

SvdBasis
SvdTrainer::
calcSvdBasis(const ColumnCorrelations & correlations,
             int numSingularValues,
             shared_ptr<spdlog::logger> logger)
{
#if 0
    static int n = 0;
    {
        INFO_MSG(logger) << "saving correlations " << n;
        filter_ostream stream(MLDB::format("correlations-%d.json", n++));
        stream << jsonEncode(correlations.columns);
        for (unsigned i = 0;  i < correlations.correlations.shape()[0];  ++i) {
            for (unsigned j = 0;  j < correlations.correlations.shape()[1];  ++j) {
                stream << i << " " << j << " " << correlations.correlations[i][j]
                       << endl;
            }
        }
        INFO_MSG(logger) << "done saving correlations ";
    }
#endif

    int ndims = correlations.columnCount();

    Timer timer;

    if (logger->should_log(spdlog::level::trace)) {
        for (unsigned i = 0;  i < ndims;  ++i) {
            logger->trace() << "correlation between " << 0 << " and "
                            << i << " is " << correlations.correlations[0][i];
        }
    }

    /**************************************************************
     * multiplication of matrix B by vector x, where B = A'A,     *
     * and A is nrow by ncol (nrow >> ncol). Hence, B is of order *
     * n = ncol (y stores product vector).		              *
     **************************************************************/

    auto opb_fn = [&] (const double * x, double * y)
    {
        for (unsigned i = 0; i != ndims; i++) {
            y[i] = MLDB::SIMD::vec_dotprod_dp(&correlations.correlations[i][0], x, ndims);
        }
    };

    SVDParams params;
    params.opb = opb_fn;
    params.ierr = 0;
    params.nrows = ndims;
    params.ncols = ndims;
    params.nvals = 0;
    params.doU = false;
    params.calcPrecision(params.ncols);

    svdrec * svdResult = svdLAS2A(numSingularValues, params);
    Scope_Exit(svdFreeSVDRec(svdResult));

    INFO_MSG(logger) << "done SVD " << timer.elapsed();

    // It doesn't clean up the ones that didn't converge properly... do it ourselves
    // We go until we get a NaN or one with too small a ratio.
    // Eg, seen in the wild:
    // svalues = { 3.06081 2.01797 1.91045 1.39165 1.20556 1.0859 1.01295 0.973041 0.96686 0.795663 0.787847 0.753074 0.663018 0.58732 0.566861 0.53674 0.507972 0.481893 0.476135 0.451054 0.434212 0.428739 0.406749 0.396502 0.388368 0.383147 0.381553 0.34724 0.322744 0.311273 0.297784 0.285271 0.275972 0.272025 0.271609 0.265779 0.254749 0.244108 0.234286 0.229235 0.21586 0.208849 0.207129 0.194427 0.186311 0.184302 0.18284 0.170876 0.1612 0.153722 0.145908 0.145039 0.139881 0.136478 0.134853 0.131319 0.124427 0.112027 0.0839514 0.0766772 0.0687135 0.0484199 0.0354719 0.034498 9.62614e-05 7.98612e-05 7.48308e-05 6.6479e-05 5.5881e-05 5.00391e-05 4.59796e-05 4.33525e-05 3.0214e-05 2.67698e-05 2.66379e-05 1.749e-05 1.64916e-05 1.20429e-05 5.02268e-08 -nan -nan -nan -nan 2.46486e-09 -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan 1.61711e-08 }

    unsigned realD = 0;
    while (realD < svdResult->d
           && isfinite(svdResult->S[realD])
           && svdResult->S[realD] / svdResult->S[0] > 1e-9)
        ++realD;

    INFO_MSG(logger) << "skipped " << svdResult->d - realD << " bad singular values";
    ExcAssertLessEqual(realD, svdResult->d);
    ExcAssertLessEqual(realD, numSingularValues);
    svdResult->d = realD;

    INFO_MSG(logger) << "got " << svdResult->d << " singular values";

    numSingularValues = svdResult->d;

    TRACE_MSG(logger) << "Vt rows " << svdResult->Vt->rows;
    TRACE_MSG(logger) << "Vt cols " << svdResult->Vt->cols;

    SvdBasis result;
    result.modelTs = correlations.modelTs;
    result.singularValues.resize(numSingularValues);
    std::copy(svdResult->S, svdResult->S + numSingularValues,
              result.singularValues.begin());

    INFO_MSG(logger) << "svalues = " << result.singularValues;

    result.columns.resize(ndims);
    std::copy(correlations.columns.begin(), correlations.columns.end(),
              result.columns.begin());

#if 1
    // Extract the singular vectors for the dense behaviours
    for (unsigned i = 0;  i < ndims;  ++i) {

        distribution<float> & d = result.columns[i].singularVector;
        d.resize(numSingularValues);
        for (unsigned j = 0;  j < numSingularValues;  ++j)
            d[j] = svdResult->Vt->value[j][i];

        ColumnPath columnName = result.columns[i].columnName;
        CellValue cellValue = result.columns[i].cellValue;

        result.columnIndex[columnName].values[cellValue] = i;
        result.columnIndex[columnName].columnName = columnName;

        //std::copy(svdResult->Vt->value[i],
        //          svdResult->Vt->value[i] + numSingularValues,
        //          d.begin());
    }
#endif

    TRACE_MSG(logger) << "result.columnIndex.size() = " << result.columnIndex.size();
    TRACE_MSG(logger) << "ndims = " << ndims;

    for (auto & i: result.columnIndex)
        ExcAssertNotEqual(i.second.columnName, ColumnPath());

    if (logger->should_log(spdlog::level::trace)) {
        logger->trace() << "Testing the orthonormal-ness of the singular vectors";

        for (unsigned i = 0;  i < ndims;  ++i) {
            const auto & v = result.columns[i].singularVector;
              logger->trace() << "dim " << i << ": two_norm = " << v.two_norm() << " min " << v.min()
                              << " max " << v.max() << " one_norm " << v.total();
        }

        for (unsigned i = 0;  i < ndims;  ++i) {
              logger->trace() << "dim 0 with dim " << i << ": two_norm = "
                              << result.columns[i].singularVector.dotprod(result.columns[0].singularVector);
        }
    }

    return result;
}

SvdBasis
SvdTrainer::
calcRightSingular(const ClassifiedColumns & columns,
                  const ColumnIndexEntries & columnIndex,
                  const SvdBasis & svd,
                  shared_ptr<spdlog::logger> logger)

{
    DEBUG_MSG(logger) << "project extra " << columns.continuousColumns.size() - svd.columns.size()
                      << " continuous columns onto basis";
    DEBUG_MSG(logger) << "projecting " << columns.sparseColumns.size()
                      << " discrete columns onto basis";

    Timer timer;

    std::atomic<int> numDone(0);
    double lastSeconds = 0.0;

    size_t totalColumns = columns.continuousColumns.size() + columns.sparseColumns.size();

    SvdBasis result(svd);
    result.modelTs = svd.modelTs;
    result.columns.resize(totalColumns);

    auto outputFirstRightSingular = [&svd, logger](const ColumnIndexEntries & columnIndex) {
        stringstream output;

        // Calc the right singular of the first continuous column
        auto vec0 = svd.rightSingularVector(columnIndex, columnIndex[0], logger);
        auto svec = svd.columns[0].singularVector;

        vec0.resize(10);
        svec.resize(10);

        output << "vec0 = " << vec0 << " svec = " << svec;
        return output.str();
    };

    if (logger->should_log(spdlog::level::debug) && svd.columns.size() > 0)
        logger->debug() << outputFirstRightSingular(columnIndex);

    auto calcRightSingular = [&] (int i)
        {
            int done = numDone.fetch_add(1);

            // If we've already done it, then skip
            if (!result.columns[i].singularVector.empty())
                return;

            auto vec = svd.rightSingularVector(columnIndex, columnIndex[i], logger);

            SimpleIntersectionEntry column;
            column = columnIndex[i];
            column.singularVector = std::move(vec);

            result.columns[i] = std::move(column);

            if (done % 1000 == 0) {
                double seconds = timer.elapsed_wall();
                INFO_MSG(logger) << "done " << done << " of " << totalColumns
                     << " in " << timer.elapsed();
                INFO_MSG(logger) << "Average " << done / seconds << " per second; inst "
                     << 1000 / (seconds - lastSeconds) << " per second";
                lastSeconds = seconds;
            }
        };

    parallelMap(0, totalColumns, calcRightSingular);

    for (unsigned i = 0;  i < totalColumns;  ++i) {
        ColumnPath columnName = result.columns[i].columnName;
        CellValue cellValue = result.columns[i].cellValue;
        if (result.columns[i].op == COL_VALUE)
            ExcAssertEqual(cellValue, CellValue());

        TRACE_MSG(logger) << "column " << i << " name " << columnName
                          << " value " << jsonEncodeStr(cellValue) << " op "
                          << result.columns[i].op;

        result.columnIndex[columnName].values[cellValue] = i;
        result.columnIndex[columnName].columnName = columnName;
    }

    for (auto & i: result.columnIndex) {
        ExcAssertNotEqual(i.second.columnName, ColumnPath());
        for (auto & v: i.second.values) {
            auto col = result.columns.at(v.second);
            auto val = v.first;

            ExcAssertEqual(val, col.cellValue);
            ExcAssertEqual(i.second.columnName, col.columnName);

            if (col.op == COL_VALUE)
                ExcAssertEqual(val, CellValue());
        }
    }

    ExcAssertLessEqual(result.columnIndex.size(), result.columns.size());

    for (auto & c: result.columns) {
        ExcAssertNotEqual(c.columnName, ColumnPath());
    }

    return result;
}

/*****************************************************************************/
/* SVD PROCEDURE                                                              */
/*****************************************************************************/

SvdProcedure::
SvdProcedure(MldbEngine * owner,
            PolyConfig config,
            const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    this->svdConfig = config.params.convert<SvdConfig>();
}

Any
SvdProcedure::
getStatus() const
{
    return Any();
}

RunOutput
SvdProcedure::
run(const ProcedureRunConfig & run,
      const std::function<bool (const Json::Value &)> & onProgress) const
{
    auto runProcConf = applyRunConfOverProcConf(svdConfig, run);

    if (runProcConf.outputColumn.null()) {
        throw AnnotatedException
            (400, "SVD training procedure requires a non-empty output column name",
             "config", runProcConf);
    }

    if (!runProcConf.modelFileUrl.empty()) {
        checkWritability(runProcConf.modelFileUrl.toDecodedString(), "modelFileUrl");
    }

    int numBasisVectors = runProcConf.numDenseBasisVectors;

    SqlExpressionMldbScope context(engine);

    ConvertProgressToJson convertProgressToJson(onProgress);
    auto dataset = runProcConf.trainingData.stm->from->bind(context, convertProgressToJson).dataset;

    Progress svdProgress;
    std::shared_ptr<Step> classificationStep = svdProgress.steps({
            make_pair("classifying columns", "percentile"),
            make_pair("extracting features", "percentile"),
            make_pair("inverting features", "percentile"),
            make_pair("calculating correlations", "percentile")
            });

    ClassifiedColumns columns = classifyColumns(runProcConf.trainingData.stm->select,
                                                *dataset,
                                                runProcConf.trainingData.stm->when,
                                                *runProcConf.trainingData.stm->where,
                                                runProcConf.trainingData.stm->orderBy,
                                                runProcConf.trainingData.stm->offset,
                                                runProcConf.trainingData.stm->limit,
                                                logger,
                                                convertProgressToJson);

    auto outputColumns = [](const ClassifiedColumns & columns) {
        stringstream output;
        output << "continuous columns are:" << endl;
        for (auto & c: columns.continuousColumns) {
            output << "name " << c.columnName << " val " << c.cellValue << " op " << c.op
                   << " rowCount " << c.rowCount << endl;;
        }
        output << "sparse columns are:" << endl;
        for (auto & c: columns.sparseColumns) {
            output << "name " << c.columnName << " val " << c.cellValue << " op " << c.op
                   << " rowCount " << c.rowCount << endl;
        }
        return output.str();
    };

    DEBUG_MSG(logger) <<  "classified columns: " << columns.continuousColumns.size()
                      << " continuous columns, " << columns.sparseColumns.size()
                      << " sparse columns";
    DEBUG_MSG(logger) << outputColumns(columns);

    auto extractionStep = classificationStep->nextStep(1);

    FeatureBuckets extractedFeatures = extractFeaturesFromRows(runProcConf.trainingData.stm->select,
                                                               *dataset,
                                                               runProcConf.trainingData.stm->when,
                                                               runProcConf.trainingData.stm->where,
                                                               runProcConf.trainingData.stm->orderBy,
                                                               runProcConf.trainingData.stm->offset,
                                                               runProcConf.trainingData.stm->limit,
                                                               columns,
                                                               logger,
                                                               convertProgressToJson);

    auto inversionStep = extractionStep->nextStep(1);

    ColumnIndexEntries columnIndex = invertFeatures(columns, extractedFeatures, logger, convertProgressToJson);

    ColumnCorrelations correlations = calculateCorrelations(columnIndex, numBasisVectors, logger);
    SvdBasis svd = SvdTrainer::calcSvdBasis(correlations,
                                            runProcConf.numSingularValues,
                                            logger);

    auto outputSvdColumns = [](const SvdBasis & basis) {
        stringstream output;
         for (auto & c: basis.columns) {
             output << "name " << c.columnName << " val " << c.cellValue << " op " << c.op << endl;
         }
        return output.str();
    };

    DEBUG_MSG(logger) << "----------- SVD columns";
    DEBUG_MSG(logger) << outputSvdColumns(svd);

    SvdBasis allSvd = SvdTrainer::calcRightSingular(columns, columnIndex, svd, logger);

    DEBUG_MSG(logger) << "----------- ALL SVD columns";
    DEBUG_MSG(logger) << outputSvdColumns(allSvd);

    if (!runProcConf.modelFileUrl.empty()) {
        makeUriDirectory(runProcConf.modelFileUrl.toDecodedString());
        filter_ostream stream(runProcConf.modelFileUrl);
        jsonEncodeToStream(allSvd, stream);
    }

    Date ts;  // TODO: actually fill this in...

    // Save the column embedding to a dataset if we ask for it
    if (runProcConf.columnOutput) {

        PolyConfigT<Dataset> columnOutput = *runProcConf.columnOutput;
        if (columnOutput.type.empty())
            columnOutput.type = SvdConfig::defaultOutputDatasetType;

        auto onProgress2 = [&] (const Json::Value & progress) {
            Json::Value value;
            value["dataset"] = progress;
            return onProgress(value);
        };

        auto output = createDataset(engine, columnOutput, onProgress2, true /*overwrite*/);

        auto doColumn = [&] (size_t i)
            {
                auto & col = allSvd.columns[i];

                if (i % 10000 == 0)
                    INFO_MSG(logger) << "saving column " << i << " of " << allSvd.columns.size();

                ColumnPath outputName = col.columnName;

                if (col.op == COL_EQUAL) {
                    if (col.cellValue.empty()) {
                        outputName = outputName + PathElement("isNull");
                    }
                    else if (col.cellValue.isNumber()) {
                        outputName = outputName + PathElement("numberEquals")
                            + col.cellValue.toUtf8String();
                    }
                    else if (col.cellValue.isString()) {
                        if (col.cellValue.toUtf8String().empty()) {
                            outputName = outputName + PathElement("stringEmpty");
                        }
                        else {
                            outputName = outputName + PathElement("stringEquals")
                                + col.cellValue.toUtf8String();
                        }
                    }
#if 0
                    else if (col.cellValue.isTimestamp()) {
                    }
                    else if (col.cellValue.isTimeinterval()) {
                    }
                    else if (col.cellValue.isBlob()) {
                    }
#endif
                    else {
                        throw AnnotatedException
                            (400,"Can't apply an SVD to a column that's not "
                             "numeric or categorical (string)",
                             "columnValue", col.cellValue,
                             "columnName", col.columnName);
                    }
                }
                else if (col.op == COL_VALUE) {
                    outputName = outputName + PathElement("numericValue");
                }
                try {
                    StructValue cols;
                    cols.emplace_back(runProcConf.outputColumn,
                                      ExpressionValue(col.singularVector, ts));
                    output->recordRowExpr(outputName, std::move(cols));
                } catch (const std::exception & exc) {
                    rethrowException(-1, "Error adding SVD column '" + outputName.toUtf8String() + "' to output: "
                                         + exc.what(),
                                         "columnName", outputName);
                }
            };

        parallelMap(0, allSvd.columns.size(), doColumn);

        output->commit();
    }

    // Save the row embedding to a dataset.  This is optional (it's not actually
    // needed to apply the SVD).

    if (runProcConf.rowOutput) {

        PolyConfigT<Dataset> rowOutput = *runProcConf.rowOutput;
        if (rowOutput.type.empty())
            rowOutput.type = SvdConfig::defaultOutputDatasetType;

        auto onProgress2 = [&] (const Json::Value & progress) {
            Json::Value value;
            value["dataset"] = progress;
            return onProgress(value);
        };
        auto output = createDataset(engine, rowOutput, onProgress2, true /*overwrite*/);

        // getRowPaths can return row names in an arbitrary order as long as it is deterministic.
        auto rows = dataset->getMatrixView()->getRowPaths(0, -1);

        DEBUG_MSG(logger) << "writing embeddings for " << rows.size() << " rows to dataset "
                          << runProcConf.rowOutput->id;

        int numSingularValues = allSvd.numSingularValues();

        auto doRow = [&] (int rowNum)
            {
                if (rowNum % 10000 == 0)
                    INFO_MSG(logger) << "saving row " << rowNum << " of " << rows.size();

                auto row = dataset->getMatrixView()->getRow(rows[rowNum]);

                distribution<float> embedding;
                Date ts;

                std::tie(embedding, ts)
                = allSvd.leftSingularVector(row.columns, numSingularValues,
                                            false /* acceptUnknownValues*/,
                                            logger);

                StructValue cols;
                cols.emplace_back(runProcConf.outputColumn,
                                  ExpressionValue(std::move(embedding), ts));
                output->recordRowExpr(row.rowName, std::move(cols));
            };

        parallelMap(0, rows.size(), doRow);

        output->commit();
    }

    if(!runProcConf.functionName.empty()) {
        PolyConfig svdFuncPC;
        svdFuncPC.type = "svd.embedRow";
        svdFuncPC.id = runProcConf.functionName;
        svdFuncPC.params = SvdEmbedConfig(runProcConf.modelFileUrl);

        createFunction(engine, svdFuncPC, onProgress, true);
    }

    return Any();
}

/*****************************************************************************/
/* SVD EMBED ROW                                                             */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(SvdEmbedConfig);

SvdEmbedConfigDescription::
SvdEmbedConfigDescription()
{
    addField("modelFileUrl", &SvdEmbedConfig::modelFileUrl,
             "URL of the model file (with extension '.svd') to load. "
             "This file is created by the ![](%%doclink svd.train procedure).");
    addField("maxSingularValues", &SvdEmbedConfig::maxSingularValues,
             "Maximum number of singular values to use (-1 = all)", -1);
    addField("acceptUnknownValues", &SvdEmbedConfig::acceptUnknownValues,
             "This parameter (which defaults to false) tells us whether or "
             "not unknown values should be accepted by the SVD.  An unknown "
             "value occurs when a column that was always a number in training "
             "is presented with a string value, or vice versa, or when a "
             "string valued column is presented with a value unknown in "
             "training.  If its value is true, an unknown value will be "
             "silently ignored.  If its value is false, an unknown value "
             "will return an error when the function is applied.",
             false);
}

DEFINE_STRUCTURE_DESCRIPTION(SvdInput);

SvdInputDescription::
SvdInputDescription()
{
    addField("row", &SvdInput::row,
             "Row to apply SVD to.  The embedding will be the average of "
             "the SVD of each of the keys of the row.");
}

DEFINE_STRUCTURE_DESCRIPTION(SvdOutput);

SvdOutputDescription::
SvdOutputDescription()
{
    addField("embedding", &SvdOutput::embedding,
             "Embedding of the row into the vector space defined by the "
             "SVD.  There will be a number of coordinates equal to the "
             "`maxSingluarValues` value of the configuration, or if not "
             "set, the number of SVD coordinates available in the SVD model "
             "file.  PathElementinates will have simple numerical names.");
}

SvdEmbedRow::
SvdEmbedRow(MldbEngine * owner,
            PolyConfig config,
            const std::function<bool (const Json::Value &)> & onProgress)
    : BaseT(owner, config)
{
    functionConfig = config.params.convert<SvdEmbedConfig>();
    svd = jsonDecodeFile<SvdBasis>(functionConfig.modelFileUrl.toDecodedString());

    std::map<ColumnHash, SvdColumnIndexEntry> columnIndex2;

    // Deal with the older version of the file
    for (auto & c: svd.columnIndex) {
        columnIndex2[c.second.columnName] = std::move(c.second);
    }

    svd.columnIndex = std::move(columnIndex2);

    svd.validate();

    nsv = functionConfig.maxSingularValues;
    if (nsv < 0 || nsv > svd.numSingularValues())
        nsv = svd.numSingularValues();
}

SvdOutput
SvdEmbedRow::
call(SvdInput input) const
{
    RowValue row;
    input.row.mergeToRowDestructive(row);

    distribution<float> embedding;
    Date ts;

    std::tie(embedding, ts)
        = svd.leftSingularVector(row, nsv,
                                 functionConfig.acceptUnknownValues,
                                 logger);

    DEBUG_MSG(logger) << "nsv = " << nsv;
    DEBUG_MSG(logger) << "embedding = " << embedding;

    // TODO: what to do when no embedding is returned?
    if (embedding.empty())
        embedding.resize(nsv);

    SvdOutput result;
    result.embedding = ExpressionValue(std::move((vector<float> &)embedding), ts);

    return result;
}

namespace {

RegisterProcedureType<SvdProcedure, SvdConfig>
regSvd(builtinPackage(),
       "Train a SVD to convert rows or columns to embedding coordinates",
       "procedures/Svd.md.html");

RegisterFunctionType<SvdEmbedRow, SvdEmbedConfig>
regSvdEmbedRow(builtinPackage(),
               "svd.embedRow",
               "Apply a trained SVD to embed a row into a coordinate space",
               "functions/SvdEmbedRow.md.html");

struct InitSvdParallelMap {
    InitSvdParallelMap()
    {
        svdParallelMap = &MLDB::parallelMap;
    }

} initSvdParallelMap;

} // file scope

} // namespace MLDB


#ifndef CUFFT_DOUBLE_PRECSION
#define CUFFT_DOUBLE_PRECISION 1
#endif
#include <cudatbx/cufft/cufft.hpp>

#include <boost/python/def.hpp>
#include <boost/python/args.hpp>
#include <scitbx/array_family/boost_python/utils.h>
#include <scitbx/array_family/versa.h>
#include <scitbx/array_family/accessors/c_grid.h>

namespace cudatbx { namespace cufft {

  void wrap_cufft_double_precision ()
  {
    using namespace boost::python;
    def("real_to_complex_3d_in_place_dp", real_to_complex_3d_in_place, (
      arg("data")));
    def("complex_to_complex_3d_in_place_dp", complex_to_complex_3d_in_place, (
      arg("data"),
      arg("direction")));
    def("complex_to_real_3d_in_place_dp", complex_to_real_3d_in_place, (
      arg("data"),
      arg("n")));
  }

}} // namespace cudatbx::cufft


//==============================================================================
//         Copyright 2003 - 2011   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef BOOST_SIMD_TOOLBOX_PREDICATES_FUNCTIONS_OPTIMIZE_IS_NOT_GREATER_EQUAL_HPP_INCLUDED
#define BOOST_SIMD_TOOLBOX_PREDICATES_FUNCTIONS_OPTIMIZE_IS_NOT_GREATER_EQUAL_HPP_INCLUDED

#include <boost/simd/toolbox/predicates/functions/is_not_greater_equal.hpp>
#include <boost/simd/toolbox/operator/functions/is_greater_equal.hpp>
#include <boost/simd/toolbox/operator/functions/logical_not.hpp>
#include <boost/dispatch/dsl/category.hpp>
#include <boost/dispatch/functor/preprocessor/call.hpp>

namespace boost { namespace simd { namespace ext
{
  BOOST_SIMD_FUNCTOR_IMPLEMENTATION( boost::simd::tag::logical_not_
                                   , tag::formal_, (D)(A0)(Arity)
                                   , ((node_<A0, boost::simd::tag::is_greater_equal_, Arity, D>))
                                   )
  {
    BOOST_DISPATCH_RETURNS(1, (A0 const& a0),
      is_not_greater_equal(boost::proto::child_c<0>(a0))
    )
  };
} } }

#endif


// Boost.Geometry

// Copyright (c) 2007-2012 Barend Gehrels, Amsterdam, the Netherlands.

// This file was modified by Oracle on 2015-2017.
// Modifications copyright (c) 2015-2017 Oracle and/or its affiliates.

// Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_STRATEGIES_GEOGRAPHIC_THOMAS_HPP
#define BOOST_GEOMETRY_STRATEGIES_GEOGRAPHIC_THOMAS_HPP

#include <boost/geometry/strategies/geographic/distance.hpp>
#include <boost/geometry/strategies/geographic/parameters.hpp>

namespace boost { namespace geometry
{

namespace strategy { namespace distance
{

/*!
\brief The solution of the inverse problem of geodesics on latlong coordinates,
       Forsyth-Andoyer-Lambert type approximation with second order terms.
\ingroup distance
\tparam Spheroid The reference spheroid model
\tparam CalculationType \tparam_calculation
\author See
    - Technical Report: PAUL D. THOMAS, MATHEMATICAL MODELS FOR NAVIGATION SYSTEMS, 1965
      http://www.dtic.mil/docs/citations/AD0627893
    - Technical Report: PAUL D. THOMAS, SPHEROIDAL GEODESICS, REFERENCE SYSTEMS, AND LOCAL GEOMETRY, 1970
      http://www.dtic.mil/docs/citations/AD703541
*/
template
<
    typename Spheroid = srs::spheroid<double>,
    typename CalculationType = void
>
class thomas
    : public strategy::distance::geographic
        <
            strategy::thomas, Spheroid, CalculationType
        >
{
    typedef strategy::distance::geographic
        <
            strategy::thomas, Spheroid, CalculationType
        > base_type;

public :
    inline thomas()
        : base_type()
    {}

    explicit inline thomas(Spheroid const& spheroid)
        : base_type(spheroid)
    {}
};

#ifndef DOXYGEN_NO_STRATEGY_SPECIALIZATIONS
namespace services
{

template <typename Spheroid, typename CalculationType>
struct tag<thomas<Spheroid, CalculationType> >
{
    typedef strategy_tag_distance_point_point type;
};

template <typename Spheroid, typename CalculationType, typename P1, typename P2>
struct return_type<thomas<Spheroid, CalculationType>, P1, P2>
    : thomas<Spheroid, CalculationType>::template calculation_type<P1, P2>
{};

template <typename Spheroid, typename CalculationType>
struct comparable_type<thomas<Spheroid, CalculationType> >
{
    typedef thomas<Spheroid, CalculationType> type;
};

template <typename Spheroid, typename CalculationType>
struct get_comparable<thomas<Spheroid, CalculationType> >
{
    static inline thomas<Spheroid, CalculationType> apply(thomas<Spheroid, CalculationType> const& input)
    {
        return input;
    }
};

template <typename Spheroid, typename CalculationType, typename P1, typename P2>
struct result_from_distance<thomas<Spheroid, CalculationType>, P1, P2 >
{
    template <typename T>
    static inline typename return_type<thomas<Spheroid, CalculationType>, P1, P2>::type
        apply(thomas<Spheroid, CalculationType> const& , T const& value)
    {
        return value;
    }
};

} // namespace services
#endif // DOXYGEN_NO_STRATEGY_SPECIALIZATIONS

}} // namespace strategy::distance

}} // namespace boost::geometry

#endif // BOOST_GEOMETRY_STRATEGIES_GEOGRAPHIC_THOMAS_HPP


#ifndef LM_NEURAL_WORDVECS_H
#define LM_NEURAL_WORDVECS_H

#include "util/scoped.hh"
#include "lm/vocab.hh"

#include <Eigen/Dense>

namespace util { class FilePiece; }

namespace lm {
namespace neural {

class WordVecs {
  public:
    // Columns of the matrix are word vectors.  The column index is the word.
    typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor> Storage;

    /* The file should begin with a line stating the number of word vectors and
     * the length of the vectors.  Then it's followed by lines containing a
     * word followed by floating-point values.
     */
    explicit WordVecs(util::FilePiece &in);

    const Storage &Vectors() const { return vecs_; }

    WordIndex Index(StringPiece str) const { return vocab_.Index(str); }

  private:
    util::scoped_malloc vocab_backing_;
    ngram::ProbingVocabulary vocab_;

    Storage vecs_;
};

}} // namespaces

#endif // LM_NEURAL_WORDVECS_H


/*=============================================================================
Copyright 2018 Pranam Lashkari <plashkari628@gmail.com>

Distributed under the Boost Software License, Version 1.0. (See accompanying
file License.txt or copy at https://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/

#ifndef BOOST_ASTRONOMY_COORDINATE_ALT_AZ_HPP
#define BOOST_ASTRONOMY_COORDINATE_ALT_AZ_HPP

#include <type_traits>
#include <tuple>

#include <boost/geometry/core/cs.hpp>
#include <boost/geometry/geometries/point.hpp>
#include <boost/geometry/algorithms/transform.hpp>
#include <boost/geometry/algorithms/equals.hpp>
#include <boost/units/quantity.hpp>
#include <boost/units/systems/si/base.hpp>
#include <boost/units/systems/si/pressure.hpp>
#include <boost/units/systems/temperature/celsius.hpp>
#include <boost/units/systems/angle/degrees.hpp>
#include <boost/units/systems/si/volume.hpp>
#include <boost/units/systems/si/mass.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/static_assert.hpp>

#include <boost/astronomy/detail/is_base_template_of.hpp>
#include <boost/astronomy/coordinate/ref_frame/base_frame.hpp>
#include <boost/astronomy/coordinate/rep/representation.hpp>
#include <boost/astronomy/coordinate/diff/differential.hpp>

namespace boost { namespace astronomy { namespace coordinate {

template <typename Representation, typename Differential>
struct alt_az : public base_frame<Representation, Differential>
{
    ///@cond INTERNAL
    BOOST_STATIC_ASSERT_MSG((std::is_base_of
        <spherical_representation<typename Representation::type,
        typename Representation::quantity1, typename Representation::quantity2,
        typename Representation::quantity3>, Representation>::value),
        "argument type is expected to be a spherical_representation class");
    BOOST_STATIC_ASSERT_MSG((std::is_base_of
        <spherical_coslat_differential<typename Differential::type,
        typename Differential::quantity1, typename Differential::quantity2,
        typename Differential::quantity3>, Differential>::value),
        "argument type is expected to be a spherical_coslat_differential class");
    ///@endcond

protected:
    //frame parameters
    spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        > earth_location;

    bu::quantity<bu::si::pressure> pressure = 0.0 * bu::si::pascals;
    bu::quantity<bu::celsius::temperature> temperature = 0.0 * bu::celsius::degrees;
    boost::posix_time::ptime obs_time;
    bu::quantity<bu::si::dimensionless> relative_humidity = 0.0;

public:
    alt_az() {}

    template <typename OtherRepresentation>
    alt_az(OtherRepresentation const& representation_data)
    {
        BOOST_STATIC_ASSERT_MSG((boost::astronomy::detail::is_base_template_of
         <boost::astronomy::coordinate::base_representation, OtherRepresentation>::value),
         "Invalid representation class");

        auto temp = make_spherical_representation
            <
            typename Representation::type,
            typename Representation::quantity1,
            typename Representation::quantity2,
            typename Representation::quantity3,
            typename OtherRepresentation::type,
            typename OtherRepresentation::quantity1,
            typename OtherRepresentation::quantity2,
            typename OtherRepresentation::quantity3
            >(representation_data);

        this->data = temp;
    }

    alt_az
    (
        typename Representation::quantity1 const& alt,
        typename Representation::quantity2 const& az,
        typename Representation::quantity3 const& distance
    )
    {
        this->data.set_lat_lon_dist(alt, az, distance);
    }

    alt_az
    (
        typename Representation::quantity1 const& alt,
        typename Representation::quantity2 const& az,
        typename Representation::quantity3 const& distance,
        typename Differential::quantity1 const& pm_alt,
        typename Differential::quantity2 const& pm_az_cosalt,
        typename Differential::quantity3 const& radial_velocity) :
        alt_az(alt, az, distance)
    {
        this->motion.set_dlat_dlon_coslat_ddist(pm_alt, pm_az_cosalt, radial_velocity);
    }

    template <typename OtherRepresentation, typename OtherDifferential>
    alt_az
    (
        OtherRepresentation const& representation_data,
        OtherDifferential const& differential_data
    )
    {
        BOOST_STATIC_ASSERT_MSG((boost::astronomy::detail::is_base_template_of
         <boost::astronomy::coordinate::base_representation, OtherRepresentation>::value),
         "argument type is expected to be a representation class");

        BOOST_STATIC_ASSERT_MSG((boost::astronomy::detail::is_base_template_of
          <boost::astronomy::coordinate::base_differential, OtherDifferential>::value),
            "argument type is expected to be a differential class");

        auto rep_temp = make_spherical_representation
            <
            typename Representation::type,
            typename Representation::quantity1,
            typename Representation::quantity2,
            typename Representation::quantity3,
            typename OtherRepresentation::type,
            typename OtherRepresentation::quantity1,
            typename OtherRepresentation::quantity2,
            typename OtherRepresentation::quantity3
            >(representation_data);
        this->data = rep_temp;

        auto dif_temp = make_spherical_coslat_differential
            <
            typename Differential::type,
            typename Differential::quantity1,
            typename Differential::quantity2,
            typename Differential::quantity3,
            typename OtherDifferential::type,
            typename OtherDifferential::quantity1,
            typename OtherDifferential::quantity2,
            typename OtherDifferential::quantity3
            >(differential_data);
        this->motion = dif_temp;
    }

    alt_az(alt_az<Representation, Differential> const& other)
    {
        this->data = other.get_data();
        this->motion = other.get_differential();
        this->earth_location = other.get_location();
        this->obs_time = other.get_obs_time();
        this->pressure = other.get_pressure();
        this->temperature = other.get_temprature();
        this->relative_humidity = other.get_relative_humidity();
    }

    //!returns altitude component of the coordinate
    typename Representation::quantity1 get_alt() const
    {
        return this->data.get_lat();
    }

    //!returns azimuth component of the coordinate
    typename Representation::quantity2 get_az() const
    {
        return this->data.get_lon();
    }

    //!returns distance component of the coordinate
    typename Representation::quantity3 get_distance() const
    {
        return this->data.get_dist();
    }

    //!returns the (alt, az, dist) in the form of tuple
    std::tuple
    <
        typename Representation::quantity1,
        typename Representation::quantity2,
        typename Representation::quantity3
    > get_alt_az_dist() const
    {
        return this->data.get_lat_lon_dist();
    }

    //!returns proper motion in altitude
    typename Differential::quantity1 get_pm_alt() const
    {
        return this->motion.get_dlat();
    }

    //!returns proper motion in azimuth including cos(alt)
    typename Differential::quantity2 get_pm_az_cosalt() const
    {
        return this->motion.get_dlon_coslat();
    }

    //!returns radial_velocity
    typename Differential::quantity3 get_radial_velocity() const
    {
        return this->motion.get_ddist();
    }

    //!returns the proper motion in tuple form
    std::tuple
    <
        typename Differential::quantity1,
        typename Differential::quantity2,
        typename Differential::quantity3
    > get_pm_alt_az_radial() const
    {
        return this->motion.get_dlat_dlon_coslat_ddist();
    }

    //!sets value of altitude component of the coordinate
    void set_alt(typename Representation::quantity1 const& alt)
    {
        this->data.set_lat(alt);
    }

    //!sets value of azimuth component of the coordinate
    void set_az(typename Representation::quantity3 const& az)
    {
        this->data.set_lon(az);
    }

    //!sets value of distance component of the coordinate
    void set_distance(typename Representation::quantity3 const& distance)
    {
        this->data.set_dist(distance);
    }

    //!sets value of all component of the coordinate including cos(alt)
    void set_alt_az_dist
    (
        typename Representation::quantity1 const& alt,
        typename Representation::quantity2 const& az,
        typename Representation::quantity3 const& dist)
    {
        this->data.set_lat_lon_dist(alt, az, dist);
    }

    //!sets the proper motion in altitude
    void set_pm_alt(typename Differential::quantity1 const& pm_alt)
    {
        this->motion.set_dlat(pm_alt);
    }

    //!sets the proper motion in azimuth including cos(alt)
    void set_pm_az_cosalt(typename Differential::quantity2 const& pm_az_cosalt)
    {
        this->motion.set_dlon_coslat(pm_az_cosalt);
    }

    //!sets the radial_velocity
    void set_radial_velocity(typename Differential::quantity3 const& radial_velocity)
    {
        this->motion.set_ddist(radial_velocity);
    }

    //!set value of motion
    void set_pm_alt_az_radial
    (
        typename Differential::quantity1 const& pm_alt,
        typename Differential::quantity2 const& pm_az_cosalt,
        typename Differential::quantity3 const& radial_velocity
    )
    {
        this->motion.set_dlat_dlon_coslat_ddist(pm_alt, pm_az_cosalt, radial_velocity);
    }

    //!set all the perameters of current frame object
    void set_frame_parameters
    (
        spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        > const& location,
        bu::quantity<bu::si::pressure> const& pressure,
        bu::quantity<bu::celsius::temperature> const& temperature,
        boost::posix_time::ptime const& obs_time,
        bu::quantity<bu::si::dimensionless> const& relative_humidity
    )
    {
        this->earth_location = location;
        this->pressure = pressure;
        this->temperature = temperature;
        this->obs_time = obs_time;
        this->relative_humidity = relative_humidity;
    }

    //!get all the perameters of current frame object
    std::tuple<
        spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        >,
        bu::quantity<bu::si::pressure>,
        bu::quantity<bu::celsius::temperature>,
        boost::posix_time::ptime,
        bu::quantity<bu::si::dimensionless>
    > get_frame_parameters() const
    {
        return std::make_tuple
        (
            this->earth_location,
            this->pressure,
            this->temperature,
            this->obs_time,
            this->relative_humidity
        );
    }

    //!get earth location of the current object
    spherical_equatorial_representation
    <
        double,
        bu::quantity<bu::degree::plane_angle>,
        bu::quantity<bu::degree::plane_angle>
    > get_location() const
    {
        return this->earth_location;
    }

    //!set earth location of the current object
    void set_location
    (
        spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        > const& location
    )
    {
        this->earth_location = location;
    }

    //!get atmospheric pressure
    bu::quantity<bu::si::pressure> get_pressure() const
    {
        return this->pressure;
    }

    //!set atmospheric pressure
    void set_pressure(bu::quantity<bu::si::pressure> const& pressure)
    {
        this->pressure = pressure;
    }

    //!get temperature of the location
    bu::quantity<bu::celsius::temperature> get_temprature() const
    {
        return this->temperature;
    }

    //!set temperature of the location
    void set_temprature(bu::quantity<bu::celsius::temperature> const& temperature)
    {
        this->temperature = temperature;
    }

    //!get observation time
    boost::posix_time::ptime get_obs_time() const
    {
        return this->obs_time;
    }

    //!set observation time
    void set_obs_time(boost::posix_time::ptime const& time)
    {
        this->obs_time = time;
    }

    //!get relative humidity
    bu::quantity<bu::si::dimensionless> get_relative_humidity() const
    {
        return this->relative_humidity;
    }

    //!set relative humidity
    void set_relative_humidity(bu::quantity<bu::si::dimensionless> const& humidity)
    {
        this->relative_humidity = humidity;
    }
};
}}} //namespace boost::astronomy::cordinate
#endif // !BOOST_ASTRONOMY_COORDINATE_ALT_AZ_HPP


/* decision_tree_xor_test.cc
   Jeremy Barnes, 25 February 2008
   Copyright (c) 2008 Jeremy Barnes.  All rights reserved.

   Test of the decision tree class.
*/

#define BOOST_TEST_MAIN
#define BOOST_TEST_DYN_LINK

#include <boost/test/unit_test.hpp>
#include <boost/thread.hpp>
#include <boost/thread/barrier.hpp>
#include <boost/bind.hpp>
#include <vector>
#include <stdint.h>
#include <iostream>

#include "jml/boosting/decision_tree_generator.h"
#include "jml/boosting/training_data.h"
#include "jml/boosting/dense_features.h"
#include "jml/boosting/feature_info.h"
#include "jml/utils/smart_ptr_utils.h"
#include "jml/utils/vector_utils.h"

using namespace ML;
using namespace std;

using boost::unit_test::test_suite;

static const char * config_options = "\
trace=0\n\
";

BOOST_AUTO_TEST_CASE( test_decision_tree_multithreaded_binary )
{
    /* Create the dataset */

    Dense_Feature_Space fs;
    fs.add_feature("LABEL", Feature_Info(BOOLEAN, false, true));
    fs.add_feature("feature1", REAL);
    fs.add_feature("feature2", REAL);
    fs.add_feature("feature3", REAL);

    std::shared_ptr<Dense_Feature_Space> fsp(make_unowned_sp(fs));

    Training_Data data(fsp);

    int nfv = 10000;

    for (unsigned i = 0;  i < nfv;  ++i) {
        distribution<float> features;
        features.push_back(i % 2);
        features.push_back(i);
        features.push_back(i);
        features.push_back(i);

        //features.push_back(random());
        //features.push_back(random());
        //features.push_back(random());

        std::shared_ptr<Feature_Set> fset
            = fs.encode(features);

        data.add_example(fset);
    }

    /* Create the decision tree generator */
    Configuration config;
    config.parse_string(config_options, "inbuilt config file");

    Decision_Tree_Generator generator;
    generator.configure(config);
    generator.init(fsp, fs.features()[0]);

    distribution<float> training_weights(nfv, 1);

    vector<Feature> features = fs.features();
    features.erase(features.begin(), features.begin() + 1);

    Thread_Context context;

    generator.generate(context, data, training_weights, features);
}

BOOST_AUTO_TEST_CASE( test_decision_tree_multithreaded_regression )
{
    /* Create the dataset */

    Dense_Feature_Space fs;
    fs.add_feature("LABEL", Feature_Info(REAL, false, true));
    fs.add_feature("feature1", REAL);
    fs.add_feature("feature2", REAL);
    fs.add_feature("feature3", REAL);

    std::shared_ptr<Dense_Feature_Space> fsp(make_unowned_sp(fs));

    Training_Data data(fsp);

    int nfv = 10000;

    for (unsigned i = 0;  i < nfv;  ++i) {
        distribution<float> features;
        features.push_back(i % 2);
        features.push_back(i);
        features.push_back(i);
        features.push_back(i);

        //features.push_back(random());
        //features.push_back(random());
        //features.push_back(random());

        std::shared_ptr<Feature_Set> fset
            = fs.encode(features);

        data.add_example(fset);
    }

    /* Create the decision tree generator */
    Configuration config;
    config.parse_string(config_options, "inbuilt config file");

    Decision_Tree_Generator generator;
    generator.configure(config);
    generator.init(fsp, fs.features()[0]);

    distribution<float> training_weights(nfv, 1);

    vector<Feature> features = fs.features();
    features.erase(features.begin(), features.begin() + 1);

    Thread_Context context;

    generator.generate(context, data, training_weights, features);
}


#include <cstdio>
#include <cstdlib>
#include <fstream>
/// RGBD_RTK
#include <config_loader.h>
#include <marker_finder.h>
#include <reconstruction_visualizer.h>
/// ROS
#include "ros/ros.h"
#include "std_msgs/String.h"
#include <actionlib/client/simple_action_client.h>
#include <cv_bridge/cv_bridge.h>
#include <image_transport/image_transport.h>
#include <message_filters/subscriber.h>
#include <message_filters/sync_policies/approximate_time.h>
#include <message_filters/time_synchronizer.h>
#include <move_base_msgs/MoveBaseAction.h>
#include <ros/ros.h>
#include <sensor_msgs/Image.h>
#include <tf/transform_broadcaster.h>
/// Opencv
#include <Eigen/Dense>
#include <Eigen/Geometry>
#include <opencv2/highgui/highgui.hpp>
// Aruco
#include <aruco/aruco.h>
#include <aruco/cvdrawingutils.h>
// Autonomous Robot
#include <goal.h>
#include <handleFiles.h>
#include <structures.h>

using namespace std;
using namespace cv;
using namespace aruco;

// ROS Variables
typedef actionlib::SimpleActionClient<move_base_msgs::MoveBaseAction> MoveBaseClient;
tf::TransformBroadcaster *br;

MarkerDetector marker_detector;
MarkerFinder marker_finder;

float aruco_max_distance, aruco_offset_distance;
string aruco_poses_file;
vector<Pose> all_markers;
cv::Mat rgb;

Goal goal;
HandleFiles handleFiles;

void imageCallback(const sensor_msgs::ImageConstPtr &msg);
// void markerGetCloser(int marker_id);
void listenKeyboardGoal(const std_msgs::String::ConstPtr &msg);
bool moveToGoal(Eigen::Quaternionf orientation, Eigen::Affine3f pose);
void loadMarkers(string aruco_poses_file);
void initRos(int argc, char **argv, string rgb_topic);
void loadParams();
void publishArucoTF();

int main(int argc, char **argv) {
    string camera_calibration_file, aruco_dic, rgb_topic;
    float aruco_marker_size;

    // Loading config variables
    ConfigLoader param_loader("../../../src/ros_autonomous_robot/config_files/ConfigFile.yaml");
    param_loader.checkAndGetString("camera_calibration_file", camera_calibration_file);
    param_loader.checkAndGetString("aruco_dic", aruco_dic);
    param_loader.checkAndGetString("rgb_topic", rgb_topic);
    param_loader.checkAndGetString("aruco_poses_file", aruco_poses_file);
    param_loader.checkAndGetFloat("aruco_offset_distance", aruco_offset_distance);
    param_loader.checkAndGetFloat("aruco_marker_size", aruco_marker_size);
    param_loader.checkAndGetFloat("aruco_max_distance", aruco_max_distance);

    marker_finder.markerParam(camera_calibration_file, aruco_marker_size, aruco_dic);

    initRos(argc, argv, rgb_topic);

    return 0;
}
/**
 * Ros Listener to rgb topic
 * @params reads a rgb message type ImageConstPtr
 */
void imageCallback(const sensor_msgs::ImageConstPtr &msgRGB) {

    cv_bridge::CvImageConstPtr cv_ptrRGB;
    try {
        cv_ptrRGB = cv_bridge::toCvShare(msgRGB);
    } catch (cv_bridge::Exception &e) {
        ROS_ERROR("cv_bridge exception: %s", e.what());
        return;
    }
    rgb = cv_ptrRGB->image;
    publishArucoTF();

    // Detect and get pose of all aruco markers
    marker_finder.detectMarkersPoses(rgb, Eigen::Affine3f::Identity(), aruco_max_distance);

    cv::imshow("OPENCV_WINDOW", rgb); // showing rgb image
    cv::waitKey(1);
}

/**
 * Listen to a marker number and send a goal to the robot goes to marker position
 * @Params Keeps listen to any string message sent in ROS
 */
void sendGoal(const std_msgs::String::ConstPtr &msg) {
    string listen_id;
    int marker_id_asked;

    handleFiles.loadPoses(aruco_poses_file, aruco_offset_distance); // loading markers
    listen_id = msg->data.c_str();
    string::size_type sz;

    try {
        marker_id_asked = stoi(listen_id, &sz); // converting string to int
    } catch (std::invalid_argument &e) {
        cout << listen_id << " is not a number\n" << endl;
    }
    for (auto pose : all_markers) {
        // validing a marker(a marker is valid if it was detected in any frame)
        if (pose.id == marker_id_asked) {
            ROS_INFO("[%s] is a valid marker", msg->data.c_str());
            goal.send2dGoal(pose.x, pose.y, Eigen::Quaterniond(1, 0, 0, 0));

        } else
            ROS_INFO("[%s] is not a valid marker", msg->data.c_str());
    }
}

/**
 * This function publishes the aruco markers tf related to 0,0
 */
void publishArucoTF() {
    br = new tf::TransformBroadcaster();
    tf::Transform transform;
    for (auto pose : all_markers) {
        // set the xyz and rotation pose
        transform.setOrigin(tf::Vector3(pose.x, pose.y, pose.z));
        transform.setRotation(
            tf::Quaternion(pose.w_rotation, pose.x_rotation, pose.y_rotation, pose.z_rotation));
        string aruco_tf = "aruco" + to_string(pose.id); // set aruco name
        // broadcasting to tf related to odom
        br->sendTransform(tf::StampedTransform(transform, ros::Time::now(), "odom", aruco_tf));
    }
}

/**
 * Initialize ROS
 */
void initRos(int argc, char **argv, string rgb_topic) {
    ros::init(argc, argv, "autonomous_robot");
    ros::start();

    ros::NodeHandle n;
    ros::Subscriber sub = n.subscribe("marker_goal", 1000, sendGoal);

    ros::NodeHandle nh;
    image_transport::ImageTransport it(nh);
    image_transport::Subscriber rgb_sub = it.subscribe(rgb_topic, 1, imageCallback);

    ros::spin(); //"while true"
}


#ifndef PROFILE_PROPERTIES_HPP
#define PROFILE_PROPERTIES_HPP

#include <boost/numeric/conversion/cast.hpp>

namespace osrm
{
namespace extractor
{

const constexpr auto DEFAULT_MAX_SPEED = 180 / 3.6; // 180kmph -> m/s

struct ProfileProperties
{
    ProfileProperties()
        : traffic_signal_penalty(0), u_turn_penalty(0),
          max_speed_for_map_matching(DEFAULT_MAX_SPEED), continue_straight_at_waypoint(true),
          use_turn_restrictions(false), left_hand_driving(false)
    {
    }

    double GetUturnPenalty() const { return u_turn_penalty / 10.; }

    void SetUturnPenalty(const double u_turn_penalty_)
    {
        u_turn_penalty = boost::numeric_cast<int>(u_turn_penalty_ * 10.);
    }

    double GetTrafficSignalPenalty() const { return traffic_signal_penalty / 10.; }

    void SetTrafficSignalPenalty(const double traffic_signal_penalty_)
    {
        traffic_signal_penalty = boost::numeric_cast<int>(traffic_signal_penalty_ * 10.);
    }

    double GetMaxSpeedForMapMatching() const { return max_speed_for_map_matching; }

    void SetMaxSpeedForMapMatching(const double max_speed_for_map_matching_)
    {
        max_speed_for_map_matching = max_speed_for_map_matching_;
    }

    //! penalty to cross a traffic light in deci-seconds
    int traffic_signal_penalty;
    //! penalty to do a uturn in deci-seconds
    int u_turn_penalty;
    double max_speed_for_map_matching;
    bool continue_straight_at_waypoint;
    bool use_turn_restrictions;
    bool left_hand_driving;
};
}
}

#endif


// Copyright (c) 2015-2018 Daniel Cooke
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

#include "downsampler.hpp"

#include <vector>
#include <deque>
#include <functional>
#include <iterator>
#include <algorithm>
#include <numeric>
#include <random>
#include <cassert>

#include <boost/random/uniform_int_distribution.hpp>
#include <boost/random/discrete_distribution.hpp>

#include "basics/mappable_reference_wrapper.hpp"
#include "concepts/mappable_range.hpp"
#include "utils/mappable_algorithms.hpp"
#include "utils/read_algorithms.hpp"
#include "utils/append.hpp"

// Use boost distributions as std distributions not guaranteed to be deterministic across compilers.
// Generators are however.
// https://stackoverflow.com/questions/48730363/if-we-seed-c11-mt19937-as-the-same-on-different-machines-will-we-get-the-same.

namespace octopus { namespace readpipe {

Downsampler::Report::DownsampleRegion::DownsampleRegion(GenomicRegion region, std::size_t num_reads)
: region_ {std::move(region)}
, num_reads_ {num_reads}
{}

namespace {

using ReadWrapper = MappableReferenceWrapper<const AlignedRead>;

using PositionCoverages = std::vector<unsigned>;

bool has_minimum_coverage(const PositionCoverages& required_coverage)
{
    return std::all_of(std::cbegin(required_coverage), std::cend(required_coverage),
                       [] (const auto coverage) noexcept { return coverage == 0; });
}

template <typename ForwardIt>
auto calculate_minimum_coverages(const ForwardIt first, const ForwardIt last,
                                 const GenomicRegion& region, const unsigned min_coverage)
{
    auto result = calculate_positional_coverage(first, last, region);
    std::transform(std::cbegin(result), std::cend(result), std::begin(result),
                   [min_coverage] (const auto coverage) noexcept {
                       return std::min(coverage, min_coverage);
                   });
    return result;
}

auto sample(const PositionCoverages& required_coverage)
{
    static std::mt19937 generator {42};
    // TODO: Do we really need to keep regenerating this distribution?
    boost::random::discrete_distribution<std::size_t> dist {std::cbegin(required_coverage), std::cend(required_coverage)};
    return dist(generator);
}

template <typename BidirIt>
BidirIt random_sample(const BidirIt first, const BidirIt last)
{
    static std::mt19937 generator {42};
    boost::random::uniform_int_distribution<std::size_t> dist(0, std::distance(first, last) - 1);
    return std::next(first, dist(generator));
}

template <typename T>
auto random_sample(const OverlapRange<T>& range)
{
    return random_sample(std::begin(range), std::end(range)).base();
}

template <typename BidirIt>
auto pick_sample(BidirIt first_unsampled, BidirIt last_unsampled,
                 const std::vector<GenomicRegion>& positions,
                 const PositionCoverages& required_coverage,
                 const AlignedRead::MappingDomain::Size max_read_size)
{
    assert(first_unsampled < last_unsampled);
    const auto candidates = overlap_range(first_unsampled, last_unsampled,
                                          positions[sample(required_coverage)],
                                          max_read_size);
    assert(!candidates.empty());
    return random_sample(candidates);
}

void reduce(PositionCoverages& coverages, const ReadWrapper& read, const GenomicRegion& region)
{
    assert(!begins_before(read, region));
    const auto read_offset = begin_distance(region, read);
    const auto first = std::next(std::begin(coverages), read_offset);
    auto last = std::next(first, region_size(read));
    if (last > std::end(coverages)) last = std::end(coverages);
    std::transform(first, last, first, [] (const auto x) noexcept { return (x > 0) ? x - 1 : 0; });
}

template <typename ForwardIt>
ForwardIt shift_left(ForwardIt first, ForwardIt last)
{
    return std::rotate(first, std::next(first), last);
}

template <typename BidirIt>
BidirIt shift_right(BidirIt first, BidirIt last)
{
    const auto rfirst = std::make_reverse_iterator(last);
    const auto rlast  = std::make_reverse_iterator(first);
    return std::rotate(rfirst, std::next(rfirst), rlast).base();
}

template <typename BidirIt>
void remove_sample(BidirIt& first, BidirIt sample, BidirIt& last)
{
    assert(first <= sample && sample < last);
    // rotating is linear time so it's cheaper to shift the sample into the closer of the
    // two sampled groups.
    if (std::distance(first, sample) < std::distance(sample, last)) {
        if (sample != first) {
            shift_right(first, std::next(sample));
        }
        ++first;
    } else {
        if (sample != std::prev(last)) {
            shift_left(sample, last);
        }
        --last;
    }
}

auto extract_sampled(std::vector<ReadWrapper>& reads,
                     std::vector<ReadWrapper>::iterator first_unsampled,
                     std::vector<ReadWrapper>::iterator last_unsampled)
{
    reads.erase(first_unsampled, last_unsampled);
    std::sort(std::begin(reads), std::end(reads));
    std::vector<AlignedRead> result {};
    result.reserve(reads.size());
    std::transform(std::begin(reads), std::end(reads), std::back_inserter(result),
                   [] (const auto& wrapper) -> AlignedRead { return wrapper.get(); });
    return result;
}

} // namespace

template <typename InputIt>
auto sample(const InputIt first_read, const InputIt last_read, const GenomicRegion& region,
            const unsigned target_coverage)
{
    if (first_read == last_read) return std::vector<AlignedRead> {};
    const auto positions = decompose(region);
    auto required_coverage = calculate_minimum_coverages(first_read, last_read, region, target_coverage);
    assert(positions.size() == required_coverage.size());
    std::vector<ReadWrapper> reads {first_read, last_read};
    const auto max_read_size = size(largest_region(reads)); // for efficient overlap detection
    // The reads are partitioned into three groups: sampled | unsampled | sampled
    // which allows a sampled read to be optimally moved out of the unsampled partition
    auto first_unsampled_itr = std::begin(reads);
    auto last_unsampled_itr  = std::end(reads);
    while (!has_minimum_coverage(required_coverage)) {
        const auto sampled_itr = pick_sample(first_unsampled_itr, last_unsampled_itr,
                                             positions, required_coverage, max_read_size);
        reduce(required_coverage, *sampled_itr, region);
        remove_sample(first_unsampled_itr, sampled_itr, last_unsampled_itr);
    }
    return extract_sampled(reads, first_unsampled_itr, last_unsampled_itr);
}

namespace {

// Look for regions with coverage above max_coverage interconnected by positions
// with coverage abouve min_coverage. The idea being samples are better taken from the
// larger joined regions.
auto find_target_regions(const ReadContainer& reads, const unsigned max_coverage, const unsigned min_coverage)
{
    const auto above_max_coverage_regions = find_high_coverage_regions(reads, max_coverage);
    std::vector<GenomicRegion> result {};
    if (above_max_coverage_regions.empty()) return result;
    result.reserve(above_max_coverage_regions.size());
    auto above_min_coverage_regions = find_high_coverage_regions(reads, min_coverage);
    std::copy_if(std::make_move_iterator(std::begin(above_min_coverage_regions)),
                 std::make_move_iterator(std::end(above_min_coverage_regions)),
                 std::back_inserter(result),
                 [&above_max_coverage_regions] (const auto& region) {
                     return has_contained(std::cbegin(above_max_coverage_regions), std::cend(above_max_coverage_regions), region);
                 });
    return result;
}

} // namespace

Downsampler::Report sample(ReadContainer& reads, const unsigned trigger_coverage, const unsigned target_coverage)
{
    using std::begin; using std::end; using std::make_move_iterator;

    Downsampler::Report result {};

    if (reads.empty()) return result;
    const auto targets = find_target_regions(reads, trigger_coverage, target_coverage);
    if (targets.empty()) return result;

    // We avoid using ReadContainers member methods for inserting sampled reads as they are order
    //  N log(size() + N) + N * size(). This is because they assume the input range is unsorted,
    // and must therefore to a binary search for every read inserted. We can avoid this as
    // we know the reads are originally sorted, and we sample reads in non-overlapping blocks.
    // The blocks are themselves sorted, so no comparisons are required.
    std::vector<std::deque<AlignedRead>> sampled_read_blocks {};
    sampled_read_blocks.reserve(targets.size());
    std::vector<std::vector<AlignedRead>> unsampled_read_blocks {};
    unsampled_read_blocks.reserve(targets.size());
    std::size_t num_reads {0};

    // Downsample in reverse order because erasing near back of MappableFlatMultiSet is much
    // cheaper than erasing near front.
    std::for_each(std::crbegin(targets), std::crend(targets), [&] (const auto& region) {
        const auto contained = bases(contained_range(begin(reads), end(reads), region));
        num_reads += std::distance(end(contained), end(reads));
        unsampled_read_blocks.emplace_back(make_move_iterator(end(contained)), make_move_iterator(end(reads)));
        auto sampled_reads = sample(begin(contained), end(contained), region, target_coverage);
        num_reads += sampled_reads.size();
        const auto num_reads_in_target = size(contained);
        assert(num_reads_in_target >= sampled_reads.size());
        const auto num_reads_removed = num_reads_in_target - sampled_reads.size();
        result.downsampled_regions.emplace(region, num_reads_removed);
        sampled_read_blocks.emplace_back(make_move_iterator(begin(sampled_reads)), make_move_iterator(end(sampled_reads)));
        reads.erase(begin(contained), end(reads));
        reads.shrink_to_fit();
    });

    num_reads += reads.size();
    std::vector<AlignedRead> buffer {};
    buffer.reserve(num_reads);
    buffer.assign(make_move_iterator(begin(reads)), make_move_iterator(end(reads)));
    reads.clear();
    reads.shrink_to_fit();
    for (auto i = static_cast<int>(targets.size()) - 1; i >= 0; --i) {
        utils::append(std::move(sampled_read_blocks[i]), buffer);
        utils::append(std::move(unsampled_read_blocks[i]), buffer);
    }
    reads = ReadContainer {make_move_iterator(begin(buffer)), make_move_iterator(end(buffer))};
    return result;
}

// Downsampler

Downsampler::Downsampler(const unsigned trigger_coverage, const unsigned target_coverage)
: trigger_coverage_ {trigger_coverage}
, target_coverage_ {target_coverage}
{
    if (target_coverage > trigger_coverage) {
        target_coverage_ = trigger_coverage;
    }
}

Downsampler::Report Downsampler::downsample(ReadContainer& reads) const
{
    return sample(reads, trigger_coverage_, target_coverage_);
}

std::size_t count_downsampled_reads(const DownsamplerReportMap& reports)
{
    return std::accumulate(std::cbegin(reports), std::cend(reports), std::size_t {0},
                           [] (auto curr, const auto& p) {
                               return curr + std::accumulate(std::cbegin(p.second.downsampled_regions),
                                                             std::cend(p.second.downsampled_regions), std::size_t {0},
                                                             [] (auto curr, const auto& target) {
                                   return curr + target.num_reads(); });
                           });
}

std::size_t count_downsampled_reads(const DownsamplerReportMap& reports, const GenomicRegion& region)
{
    return std::accumulate(std::cbegin(reports), std::cend(reports), std::size_t {0},
                           [&region] (auto curr, const auto& p) {
                               const auto overlapped = overlap_range(p.second.downsampled_regions, region);
                               return curr + std::accumulate(std::cbegin(overlapped), std::cend(overlapped), std::size_t {0},
                                                             [] (auto curr, const auto& target) {
                                                                 return curr + target.num_reads(); });
                           });
}

} // namespace readpipe
} // namespace octopus