//
// Created by senbaikang on 14.05.21.
//

#include <cfloat>
#include <iostream>
#include <regex>

#include <dlib/global_optimization.h>
#include <boost/algorithm/string.hpp>
#include <boost/math/distributions/chi_squared.hpp>

#include "process_data.h"
#include "probabilities.h"

void process_data(Config &config)
{
  // Build a thread pool.
  ThreadPool threadPool(config.getThreads());

  // Get pointers to data files.
  auto filePtrs(std::move(getReadCountsFiles(config)));

  // Object storing output data.
  Data data;

  // Mutex for data synchronization.
  std::mutex dataMutex;

  // Read the cellular information.
  std::vector<u_int32_t> tumorCellPos;
  std::vector<u_int32_t> normalCellPos;
  u_int32_t tumorBulkPos;
  u_int32_t normalBulkPos;
  readCellInformation(
      tumorCellPos,
      normalCellPos,
      tumorBulkPos,
      normalBulkPos,
      config,
      data
      );

  // Get the number of tumor cells.
  const u_int32_t tumorCellNum = tumorCellPos.size();
  config.setTumorCellNum(tumorCellNum);

  // Get sites to skip.
  std::set<std::pair<ChromosomeLabel, u_int32_t>> exMap;
  if (!config.getExcludedSitesFileName().empty())
    readExclusionList(config.getExcludedSitesFileName(), exMap);

  // Get mutated sites to skip during the error rate estimation.
  std::set<std::pair<ChromosomeLabel, u_int32_t>> exMuMap;
  if (!config.getExcludedMutatedSitesFileName().empty())
    readExclusionList(config.getExcludedMutatedSitesFileName(), exMuMap);

  // Get sites to keep.
  std::set<std::tuple<ChromosomeLabel, u_int32_t, char, std::vector<char>>> incMap;
  if (!config.getIncludedSitesFileName().empty())
    readInclusionVCF(config.getIncludedSitesFileName(), incMap);

  // Estimate effective error rate if necessary using multithreading.
  if (config.isEstimateErrRate())
  {
    // <error, cov>
    u_int32_t _error = 0;
    u_int32_t _cov = 0;
    std::vector<std::future<std::pair<u_int32_t, u_int32_t>>> collection{};

    for (auto &i : filePtrs)
      collection.emplace_back(
          threadPool.addTask(
              estimateSeqErrorRate,
              std::ref(config),
              std::ref(i),
              std::ref(exMap),
              std::ref(exMuMap),
              std::ref(tumorCellPos),
              std::ref(normalCellPos)
              )
          );

    for (auto &&i : collection) {
      const auto &j = i.get();

      _error += std::get<0>(j);
      _cov += std::get<1>(j);
    }

    config.setEffectiveSeqErrRate(
        static_cast<double>(_error) / static_cast<double>(_cov)
        );
  }

  // Multithreading.
  for (auto &i : filePtrs)
    threadPool.addTask(
        process_single_file,
        std::ref(config),
        std::ref(i),
        std::ref(tumorCellNum),
        std::ref(tumorCellPos),
        std::ref(normalCellPos),
        std::ref(tumorBulkPos),
        std::ref(normalBulkPos),
        std::ref(exMap),
        std::ref(incMap),
        std::ref(dataMutex),
        std::ref(data)
    );

  std::cout << threadPool.waitUntilFinished() << " files processed." << std::endl;

  // Write to output file.
  data.createNoiseCounts();
  data.sortEntries();

  try
  {
    writeAltNucInfo(data, config);
  }
  catch (const std::exception &e)
  {
    std::cerr << e.what() << std::endl;
  }

  closeReadCountsFile(filePtrs);
}

/**
 * Get indices of different cells / cell groups.
 *
 * @param config        configuration object
 * @param data          object storing output data
 */
void readCellInformation(
    std::vector<u_int32_t> &tumorCellPos,
    std::vector<u_int32_t> &normalCellPos,
    u_int32_t &tumorBulkPos,
    u_int32_t &normalBulkPos,
    const Config &config,
    Data &data
    )
{
  std::cout << "> Reading cell names... ";

  readCellNames(config, data);

  tumorBulkPos = UINT_MAX;
  normalBulkPos = UINT_MAX;

  std::ifstream inputStream(config.getCellNamesFileName());

  std::vector<std::string> splitVec;
  u_int32_t counter = 0;
  std::string currLine;

  while(getline(inputStream, currLine))
  {
    if (!currLine.empty())
    {
      boost::split(splitVec, currLine, boost::is_any_of("\t"));

      if (splitVec[1] == "BN")
      {
        if (normalBulkPos != UINT_MAX)
          std::cout << "WARNING: Multiple bulk control files provided. Only using the first one!";
        else
          normalBulkPos = counter;
      }

      // TODO:
      if (splitVec[1] == "BT")
      {
      }

      if (splitVec[1] == "CN")
        normalCellPos.emplace_back(counter);

      if (splitVec[1] == "CT")
        tumorCellPos.emplace_back(counter);

      ++counter;
    }
  }

  std::cout << "Done." << std::endl;
}

/**
 * Get names of cancer cells.
 *
 * @param config configuration object
 * @param data   object storing output data
 */
void readCellNames(const Config & config, Data & data)
{
  std::ifstream inputStream(config.getCellNamesFileName());
  std::vector<std::string> splitVec;
  std::vector<std::string> splitVecEntry;

  std::string currLine;

  while(getline(inputStream, currLine))
  {
    if (!currLine.empty())
    {
      boost::split(splitVec, currLine, boost::is_any_of("\t"));

      if (splitVec[1] == "CT")
      {
        boost::split(splitVecEntry, splitVec[0], boost::is_any_of("/"));

        std::string name = splitVecEntry.back();
        std::regex suf(config.getCellNameSuffix());

        if (std::regex_search(name, suf))
          name = std::regex_replace(name, suf, "");

        data.addCellName(std::move(name));
      }
    }
  }
}

/**
 * Get sites which will be excluded.
 */
void readExclusionList(
    const std::string &v,
    std::set<std::pair<ChromosomeLabel, u_int32_t>> &_map
    )
{
  std::cout << "> Reading sites to be excluded... ";

  std::ifstream inputStream(v);
  if (!inputStream)
    throw std::runtime_error("Error: Could not open " + v);

  std::string currLine;
  std::vector<std::string> splitVec;

  while (std::getline(inputStream, currLine))
  {
    if (currLine[0] != '#')
    {
      boost::split(splitVec, currLine, boost::is_any_of("\t"));

      _map.emplace(
          std::make_pair(
              std::move(ChromosomeLabel(std::move(splitVec[0]))),
              std::stoi(splitVec[1])
              )
          );
    }
  }

  std::cout << "Done. " << _map.size() << " sites will be excluded." << std::endl;
}

/**
 * Get sites which will be included.
 */
void readInclusionVCF(
    const std::string &v,
    std::set<std::tuple<ChromosomeLabel, u_int32_t, char, std::vector<char>>> &_map
)
{
  std::cout << "> Reading sites to be included... ";

  std::ifstream inputStream(v);
  if (!inputStream)
    throw std::runtime_error("Error: Could not open " + v);

  std::string currLine;
  std::vector<std::string> splitVec;

  while (std::getline(inputStream, currLine))
  {
    if (currLine[0] != '#')
    {
      boost::split(splitVec, currLine, boost::is_any_of("\t"));

      if (splitVec[3].size() != 1)
        throw std::runtime_error("Error: Illegal format of reference nucleotide: " + splitVec[3]);

      _map.emplace(
          std::make_tuple(
              std::move(ChromosomeLabel(std::move(splitVec[0]))),
              std::stoi(splitVec[1]),
              splitVec[3][0],
              std::move(std::vector<char>(splitVec[4].begin(), splitVec[4].end()))
              )
          );
    }
  }

  std::cout << "Done. " << _map.size() << " sites will be included." << std::endl;
}

std::vector<std::unique_ptr<ReadCountsFile>> getReadCountsFiles(const Config &config)
{
  const std::regex name(GZ_FILE_PATTERN);

  std::vector<std::unique_ptr<ReadCountsFile>> ret;

  for (const std::string &i : config.getInputFileNames())
  {
    if (std::regex_match(i, name))
      ret.emplace_back(std::unique_ptr<ReadCountsFile>(new GZFile(i, config.getReadLength())));
    else
      ret.emplace_back(std::unique_ptr<ReadCountsFile>(new MPileupFile(i)));
  }

  return ret;
}

void closeReadCountsFile(std::vector<std::unique_ptr<ReadCountsFile>> &files)
{
  for (auto &i : files)
    i.reset();
}

// This function is used to collect the sequencing information of all cells.
void extractSeqInformation(
    std::vector<std::array<u_int32_t, 5>> &counts,
    const std::vector<std::string> &splitVec,
    const std::vector<u_int32_t> &positions
)
{
  // loop over the cells
  for (size_t i = 0; i < positions.size(); ++i)
    extractSeqInformation(counts[i], splitVec, positions[i]);
}

// This functions is used to extract information about
// sequencing errors. Only if a single cell shows a mutation
// the error rates will be collected.
void updateSeqErrorStats(
    u_int32_t &seqErrors,
    u_int32_t &seqErrorsCombCov,
    const std::vector<std::array<u_int32_t, 5>> &counts
    )
{
  for (size_t j = 0; j < 4; ++j) // loop over the nucleotides
    for (const auto & count : counts) // loop over all cells
      seqErrors += count[j];

  for (const auto & count : counts) // update the coverage
    seqErrorsCombCov += count[4];
}

std::pair<u_int32_t, u_int32_t> estimateSeqErrorRate(
    const Config &config,
    std::unique_ptr<ReadCountsFile> &filePtr,
    const std::set<std::pair<ChromosomeLabel, u_int32_t>> &exMap,
    const std::set<std::pair<ChromosomeLabel, u_int32_t>> &errExMap,
    const std::vector<u_int32_t> &tumorCellPos,
    const std::vector<u_int32_t> &normalCellPos
    )
{
  u_int32_t maxEstLoop = config.getErrRateEstLoops();

  std::string currLine;
  std::vector<std::string> splitVec;
  std::vector<std::array<u_int32_t, 5>> tumor_counts(tumorCellPos.size(), {{0,0,0,0,0}});
  std::vector<std::array<u_int32_t, 5>> normal_counts(normalCellPos.size(), {{0,0,0,0,0}});

  u_int32_t seqErrors = 0;
  u_int32_t seqErrorsCombCov = 0;

  for (size_t lineNumber = 0; lineNumber < maxEstLoop && filePtr->getLine(currLine); lineNumber++)
  {
    if (currLine.empty())
    {
      lineNumber--;
      continue;
    }

    boost::split(splitVec, currLine, boost::is_any_of("\t"));

    if (!isRefKnown(splitVec[2]))
    {
      lineNumber--;
      continue;
    }

    const auto &tmp = std::make_pair(
        std::move(ChromosomeLabel(splitVec[0])),
        std::stoi(splitVec[1])
        );
    auto itEx = exMap.find(tmp);
    auto itErrEx = errExMap.find(tmp);

    if (itEx == exMap.end() && itErrEx == errExMap.end())
    {
      extractSeqInformation(tumor_counts, splitVec, tumorCellPos);
      updateSeqErrorStats(seqErrors, seqErrorsCombCov, tumor_counts);

      extractSeqInformation(normal_counts, splitVec, normalCellPos);
      updateSeqErrorStats(seqErrors, seqErrorsCombCov, normal_counts);
    }
    else
      lineNumber--;
  }

  // add pseudo counts
  seqErrors++;

  filePtr->rewind();

  return std::make_pair(seqErrors, seqErrorsCombCov);
}

// Check if the reference base is known.
bool isRefKnown(const std::string & n)
{
  if (n[0] == 'n' || n[0] == 'N')
    return false;

  return true;
}

// This function is used to collect the sequencing information of all cells.
void extractSeqInformation(
    std::array<u_int32_t, 5> &count,
    const std::vector<std::string> &splitVec,
    u_int32_t position
    )
{
  count = {{0,0,0,0,0}}; // (a,c,g,t, coverage)
  count[4] = std::stoi(splitVec[position * 3 + 3]);
  extractCellNucCountInformation(count, splitVec[position * 3 + 4]);
}

void extractCellNucCountInformation(
    std::array<u_int32_t, 5> &counts,
    const std::string &nucleotides
    )
{
  //counts = {{0, 0, 0, 0, 0}}; // (a,c,g,t, coverage)
  for (size_t j = 0; j < nucleotides.size(); ++j) // loop over the nucleotides of a cell in the pileup
  {
    u_int16_t index = charToIndex(nucleotides[j]);
    if (index < 4)
      ++counts[index];
    else if (index == 9)
      --counts[4];
    else if (index == 4)
      j = skipIndels(nucleotides, j);
    else if (index == 5)
      ++j;
    else if (index == 6)
      continue;
  }
}

u_int16_t charToIndex(char c)
{
  switch (std::toupper(c))
  {
  case('A'):
    return 0;
  case('C'):
    return 1;
  case('G'):
    return 2;
  case('T'):
    return 3;
  case('-'):
  case('+'):
    return 4;
  case('^'):
    return 5;
  case('$'):
    return 6;
  case('.'):
  case(','):
    return 7;
  case('*'):
    return 8;
  }

  if (('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z'))
  {
    return 9;
  }

  std::cerr << "Unknown character \"" << c << "\" in pileup sequence!" << std::endl;
  std::exit(EXIT_FAILURE);

  return 10;
}

// This function is used to skip indels in the pileup.
u_int32_t skipIndels(
    const std::string &nucleotides,
    u_int32_t currentPos
    )
{
  if(nucleotides[currentPos] != '-' && nucleotides[currentPos] != '+')
    return currentPos;

  u_int32_t numIndels = 0;
  u_int32_t i = currentPos + 1; // skip the '-' or '+'

  for (; i < nucleotides.size(); ++i)
  {
    if (nucleotides[i] >= '0' && nucleotides[i] <= '9')
    {
      numIndels *= 10;
      numIndels += static_cast<int32_t>(nucleotides[i]) - 48;
    }
    else
      break;
  }
  return i + (numIndels - 1);
}

bool passNormalFilter(
    const std::array<u_int32_t , 5> &normalCounts,
    const Config &config
    )
{
  if (normalCounts[4] >= config.getMinCovInBulk())
  {
    for (size_t i = 0; i < 4; ++i)
      if (normalCounts[i] >= config.getMaxSupInBulk())
        return false;

    return true;
  }

  return false;
}

bool passNormalCellCoverage(
    const std::vector<std::array<u_int32_t, 5>> &normalCellCounts,
    const Config &config
    )
{
  u_int32_t maxCov = 0;

  for (const auto &normalCellCount : normalCellCounts)
    if (normalCellCount[4] > maxCov)
      maxCov = normalCellCount[4];

  if(maxCov < config.getMinCovNormalCell())
    return false;

  return true;
}

void applyCoverageFilterPerCell(
    std::vector<std::array<u_int32_t , 5>> &counts,
    const Config &config
    )
{
  for (auto &count : counts)
    if(count[4] < config.getMinCov())
      for (size_t j = 0; j < 5; ++j)
        count[j] = 0;
}

bool applyFilterAcrossCells(
    const std::vector<std::array<u_int32_t, 5>> &counts,
    const Config &config,
    u_int32_t nucId
    )
{
  u_int32_t numCellsAboveThresh = 0;
  for (auto &count : counts)
  {
    if (passCovFilter(count[4], config.getMinCov()) &&
        passFreqFilter(count[nucId],count[4], config.getMinFreq()) &&
        passSuppFilter(count[nucId], config.getMinSup()))
    {
      ++numCellsAboveThresh;

      if(numCellsAboveThresh >= config.getMinTumorCellsToPass())
        return true;
    }
  }

  return false;
}

bool passSuppFilter(
    u_int32_t altCount,
    u_int32_t minSupport
    )
{
  if (altCount >= minSupport)
    return true;

  return false;
}

bool passFreqFilter(
    double altCount,
    double coverage,
    double minFreq
    )
{
  if (altCount/coverage >= minFreq)
    return true;

  return false;
}

bool passCovFilter(
    u_int32_t coverage,
    u_int32_t minCov
    )
{
  if (coverage >= minCov)
    return true;

  return false;
}

bool passNormalCellFilter(
    const std::vector<std::array<u_int32_t, 5>> &normalCellCounts,
    u_int16_t j,
    const Config &config
    )
{
  u_int32_t numMutated = 0;

  for (const auto &normalCellCount : normalCellCounts)
  {
    const auto homoRef = computeRawWildLogScore(config, normalCellCount[j], normalCellCount[4]);
    const auto heteroMu = computeRawHeteroMutLogScore(config, normalCellCount[j], normalCellCount[4]);
    const auto homoMu = computeRawHomoMutLogScore(config, normalCellCount[4] - normalCellCount[j], normalCellCount[4]);

    if (homoRef < heteroMu || homoRef < homoMu)
      ++numMutated;
  }

  if (numMutated > config.getMaxNormalCellsToMutate())
    return false;

  return true;
}

double updateLogH1Temp(
    double logH1Temp,
    u_int32_t numCells,
    u_int32_t numMuts,
    bool tumorCells,
    double dropOut
    )
{
  if (tumorCells)
    return logH1Temp + 2 * logNChooseK(numCells, numMuts) - std::log(2*numMuts - 1) - logNChooseK(2*numCells, 2*numMuts);

  return logH1Temp + logNChooseK(numCells, numMuts) + std::log(std::pow(1.0 - dropOut, numMuts)) + std::log(std::pow(dropOut, numCells - numMuts)) - (1.0 - std::pow(dropOut, numCells));
}

bool mustH0Win(
    double &logH1Max,
    double logH1Temp,
    double logNumCells,
    double logH0
    )
{
  if (logH1Temp >= logH1Max)
  {
    logH1Max = logH1Temp;
    return false;
  }
  else
  {
    if (logH1Max + logNumCells < logH0)
      return true;
  }

  return false;
}

bool computeProbCellsAreMutated(
    const Config &config,
    std::vector<long double> &logProbs,
    std::vector<long double> &tempLogProbs,
    std::vector<double> &logProbTempValues,
    std::vector<std::array<u_int32_t, 5>> &filteredCounts,
    std::vector<double> &cellsNotMutated,
    std::vector<double> &cellsMutated,
    u_int32_t currentChar,
    bool tumorCells
    )
{
  u_int32_t numCells = filteredCounts.size();
  double logNumCells = log(numCells);

  tempLogProbs[0] = 0.0;
  for (size_t i = 1; i < filteredCounts.size() + 1; ++i)
  {
    cellsNotMutated[i - 1] = computeRawWildLogScore(config, filteredCounts[i - 1][currentChar], filteredCounts[i - 1][4]);
    cellsMutated[i - 1] = logSumExp(computeRawHeteroMutLogScore(config, filteredCounts[i - 1][currentChar], filteredCounts[i - 1][4]), computeRawHomoMutLogScore(config, filteredCounts[i - 1][4] - filteredCounts[i - 1][currentChar], filteredCounts[i - 1][4]));
//    cellsMutated[i - 1] = computeRawHeteroMutLogScore(config, filteredCounts[i - 1][currentChar], filteredCounts[i - 1][4]);
    tempLogProbs[i] = tempLogProbs[i - 1] + cellsNotMutated[i - 1];
  }

  swap(logProbs, tempLogProbs);

  const double prior = tumorCells ? config.getMuRatePrior() : config.getGermlineRatePrior();

  const double logH0 = logProbs.back() + log(1.0 - prior);

  logProbTempValues[0] = logH0;

  // compute the probabilitues of observing 1, 2, 3, ... mutations
  double logH1Max = -DBL_MAX; // the current best alternative score
  long double logNOverK = 0;  // helper to efficiently compute nChooseK
  size_t numMut = 1;          // number of mutations currently computet

  for (; numMut <= numCells; ++numMut)
  {
    double logProbAllPrevCellsMutated = logProbs[numMut - 1];
    double currentCellMutated = cellsMutated[numMut - 1];
    tempLogProbs[numMut] = logProbAllPrevCellsMutated + currentCellMutated;
    for (size_t i = numMut + 1; i < filteredCounts.size() + 1; ++i)
    {
      double previousCellNotMutated = logProbs[i - 1];
      currentCellMutated = cellsMutated[i - 1];
      double previousCellMutated = tempLogProbs[i -1];
      double currentCellNotMutated = cellsNotMutated[i - 1];
      tempLogProbs[i] = addLogProb(
          previousCellNotMutated + currentCellMutated,
          previousCellMutated + currentCellNotMutated
          );

    }
    swap(logProbs, tempLogProbs);
    logNOverK = logNChooseK(numCells, numMut, logNOverK);
    double logH1Temp = logProbs.back() + log(prior) - logNOverK;
    logH1Temp = updateLogH1Temp(logH1Temp, numCells, numMut, tumorCells, config.getAdoRatePrior() / 2.0);

    // check whether the alternative hypothesis can win
    bool h0Wins = mustH0Win(logH1Max, logH1Temp, logNumCells, logH0);
    logProbTempValues[numMut] = logH1Temp;
    if (h0Wins)
      return true;
  }

  return false;
}

double sumValuesInLogSpace(
    std::vector<double>::const_iterator itBegin,
    std::vector<double>::const_iterator itEnd
    )
{
  auto it = itBegin;
  double maxLogValue = *it;
  ++it;

  for (;it != itEnd; ++it)
    if (maxLogValue < *it)
      maxLogValue = *it;

  it = itBegin;
  double h1 = 0;

  for (;it != itEnd; ++it)
    h1 += exp(*it - maxLogValue);

  return log(h1) + maxLogValue;
}

char indexToChar(u_int16_t index)
{
  switch (index)
  {
  case(0):
    return 'A';
  case(1):
    return 'C';
  case(2):
    return 'G';
  case(3):
    return 'T';
  default:
    return 'N';
  }
}

void writeAltNucInfo(
    const Data &data,
    const Config &config
) {
  if (config.getOutputFileName().has_parent_path()) {
    std::string mkdir =
        "mkdir -p " +
        static_cast<std::string>(config.getOutputFileName().parent_path());
    std::system(mkdir.c_str());
  }

  std::ofstream out;
  out.open(config.getOutputFileName());

  out << "=numSamples=" << "\n";
  out << config.getTumorCellNum() << "\n";

  out << "=numCandidateMutatedSites=" << "\n";
  out << data.getCandidateMutatedSitesNum() << "\n";

  if (data.getBackgroundSitesNum() % static_cast<u_int64_t>(config.getTumorCellNum()) != 0)
    std::cerr << "WARNING! Read counts data of some cells for background sites are missing. Make sure your mpileups are generated correctly.\n";

  out << "=numBackgroundSites=" << "\n";
  out << std::fixed;
  out.precision(0);
  out << std::floor(data.getBackgroundSitesNum() / static_cast<u_int64_t>(config.getTumorCellNum())) << "\n";

  out << "=mutations=" << "\n";
  for (const auto & i : data.getCandidateMutatedReadCounts()) {
    // chrom
    out << std::get<0>(std::get<0>(i)) << "\t";

    // pos
    out << std::get<1>(std::get<0>(i)) << "\t";

    // ref nuc
    out << std::get<2>(std::get<0>(i)) << "\t";

    // significant alt nucs
    if (std::get<3>(std::get<0>(i)).empty())
      out << "N";
    else
    {
      for (size_t j = 0; j < std::get<3>(std::get<0>(i)).size(); j++)
      {
        out << std::get<3>(std::get<0>(i))[j];

        if (j < std::get<3>(std::get<0>(i)).size() - 1)
          out << ',';
      }
    }

    // each cell
    for (size_t j = 0 ; j < config.getTumorCellNum(); j++)
      out << "\t" << std::get<1>(i)[j];

    out << "\n";
  }

  out << "=background=" << "\n";
  out << data.getBackgroundSitesReadCounts();

  out.close();
}

void process_single_file(
    const Config &config,
    std::unique_ptr<ReadCountsFile> &filePtr,
    const u_int32_t &tumorCellNum,
    const std::vector<u_int32_t> &tumorCellPos,
    const std::vector<u_int32_t> &normalCellPos,
    const u_int32_t &tumorBulkPos,
    const u_int32_t &normalBulkPos,
    const std::set<std::pair<ChromosomeLabel, u_int32_t>> &exMap,
    const std::set<std::tuple<ChromosomeLabel, u_int32_t, char, std::vector<char>>> &incMap,
    std::mutex &_mutex,
    Data &data
    )
{
  const std::thread::id _id(std::this_thread::get_id());
  std::cout << "> [" << _id << "] Processing " + filePtr->getFileName() + "... " << std::endl;

  TData _data;
  std::vector<CellReadCounts> cellReadCounts{};
  cellReadCounts.reserve(tumorCellNum);
  SignificantAltNucs significantAltNucs{};
  std::array<u_int16_t, 3> altNucs{};
  u_int16_t altNucIdx{};
  ContinuousNoiseCounts continuousNoiseCounts{};
  bool hasSignificantAltNucs;

  std::array<u_int32_t, 5> normalBulkCounts{};

  // vector to hold the nucleotide information {a,c,g,t,coverage}
  std::vector<std::array<u_int32_t, 5>> counts(tumorCellNum, {{0,0,0,0,0}});

  // vector to hold the nucleotide information {a,c,g,t,coverage}
  std::vector<std::array<u_int32_t, 5>> countsNormal(normalCellPos.size(), {{0,0,0,0,0}});

  // vector to hold the filtered nucleotide information {a,c,g,t,coverage}
  std::vector<std::array<u_int32_t, 5>> filteredCounts(tumorCellNum, {{0,0,0,0,0}});

  // probabilities of observing 0, 1, 2, 3, 4 ... mutations
  std::vector<long double> logProbsNormal(normalCellPos.size() + 1, 0);

  // helper array for probabilities of observing 0, 1, 2, 3, 4 ... mutations
  std::vector<long double> tempLogProbsNormal(normalCellPos.size() + 1, 0);

  std::vector<double> logProbTempValuesNormal(normalCellPos.size() + 1);

  // probabilities of observing 0, 1, 2, 3, 4 ... mutations
  std::vector<long double> logProbs(tumorCellNum + 1, 0);

  std::vector<double> logProbTempValues(tumorCellNum + 1);

  // helper array for probabilities of observing 0, 1, 2, 3, 4 ... mutations
  std::vector<long double> tempLogProbs(tumorCellNum + 1, 0);

  std::vector<double> cellsNotMutated(tumorCellNum);
  std::vector<double> cellsNotMutatedNormal(normalCellPos.size());
  std::vector<double> cellsMutated(tumorCellNum);
  std::vector<double> cellsMutatedNormal(normalCellPos.size());

  std::string currentLine;
  std::vector<std::string> splitVec;

  filePtr->setClearCache(true);

  while (filePtr->getLine(currentLine)) {
    if (currentLine.empty())
      continue;

    significantAltNucs.resetSigAltNucs();
    altNucIdx = 0;
    hasSignificantAltNucs = false;
    cellReadCounts.clear();

    // split the current line into easily accessible chunks
    boost::split(splitVec, currentLine, boost::is_any_of("\t"));

    if (!isRefKnown(splitVec[2]))
      continue;

    // check if the current pos is to be excluded
    auto it = exMap.find(
        std::make_pair(
            std::move(ChromosomeLabel(splitVec[0])),
            std::stoi(splitVec[1])
            )
        );

    if (it == exMap.end())
    {
      if (normalBulkPos != UINT_MAX)
      {
        extractSeqInformation(normalBulkCounts, splitVec, normalBulkPos);

        if (!passNormalFilter(normalBulkCounts, config))
          continue;
      }

      if (!normalCellPos.empty())
      {
        extractSeqInformation(countsNormal, splitVec, normalCellPos);
        if (!passNormalCellCoverage(countsNormal, config))
          continue;
      }

      extractSeqInformation(counts, splitVec, tumorCellPos);

      filteredCounts = counts;
      applyCoverageFilterPerCell(filteredCounts, config);

      bool positionMutated = false;
      for (size_t altAlleleIdx = 0; altAlleleIdx < 4; ++altAlleleIdx) {
        if (altAlleleIdx == charToIndex(splitVec[2][0]))
          continue;

        altNucs[altNucIdx++] = altAlleleIdx;
        bool h0Wins = !applyFilterAcrossCells(filteredCounts, config, altAlleleIdx);

        if (!normalCellPos.empty())
        {
          // use the normal cell filter
          if (config.getNormalCellFilterMode() == 1)
          {
            if (!passNormalCellFilter(countsNormal, altAlleleIdx, config))
            {
              positionMutated = true;
              h0Wins = true;
              continue;
            }
          }
          else if (config.getNormalCellFilterMode() == 2)
          {
            bool h0WinsNormal = computeProbCellsAreMutated(
                config,
                logProbsNormal,
                tempLogProbsNormal,
                logProbTempValuesNormal,
                countsNormal,
                cellsNotMutatedNormal,
                cellsMutatedNormal,
                altAlleleIdx,
                false
                );

            if (!h0WinsNormal)
            {
              double logH0Normal = sumValuesInLogSpace(
                  logProbTempValuesNormal.begin(),
                  logProbTempValuesNormal.begin() +
                      config.getMaxNormalCellsToMutate() + 1);

              double logH1Normal = sumValuesInLogSpace(
                  logProbTempValuesNormal.begin() +
                      config.getMaxNormalCellsToMutate() + 1,
                  logProbTempValuesNormal.end());

              if (logH0Normal < logH1Normal)
              {
                positionMutated = true;
                h0Wins = true;
                continue;
              }
            }
          }
        }

        double logH0 = -DBL_MAX;
        double logH1 = -DBL_MAX;
        if (!h0Wins)
          h0Wins = computeProbCellsAreMutated(
              config, logProbs, tempLogProbs, logProbTempValues,
              filteredCounts, cellsNotMutated, cellsMutated, altAlleleIdx,
              true);

        if (h0Wins)
        {
          logH1 = -DBL_MAX;
          logH0 = DBL_MAX;
        } else {
          logH0 =
              sumValuesInLogSpace(
                  logProbTempValues.begin(),
                  logProbTempValues.begin() + config.getMinTumorCellsToCallMu()
                  );
          logH1 = sumValuesInLogSpace(
              logProbTempValues.begin() + config.getMinTumorCellsToCallMu(),
              logProbTempValues.end()
              );
        }

        if (logH1 > logH0 ||
            incMap.count(
                std::make_tuple(
                    std::move(ChromosomeLabel(splitVec[0])),
                    std::stoi(splitVec[1]), splitVec[2][0],
                    std::move(std::vector<char>(indexToChar(altAlleleIdx)))
                    )
                ) != 0
            )
        {
          positionMutated = true;

          std::vector<std::pair<u_int32_t, u_int32_t>> testCounts;
          for (size_t cell = 0; cell < counts.size(); ++cell)
            if (cellsNotMutated[cell] < cellsMutated[cell])
              testCounts.emplace_back(counts[cell][altAlleleIdx], counts[cell][4]);

          dlib::matrix<double, 0, 1> startingPointMeanOverDis = {0.05, 5.0};
          dlib::matrix<double, 0, 1> dLibMinMeanOverDis = {0.001, 0.1};
          dlib::matrix<double, 0, 1> dLibMaxMeanOverDis = {0.999, 10000.0};
          OptimizeBetaBinMeanOverDis optBetaBinMeanOverDis(testCounts);
          OptimizeBetaBinMeanOverDisDerivates optBetaBinDerMeanOverDis(
              testCounts);
          double resultMeanOverDis = dlib::find_max_box_constrained(
              dlib::bfgs_search_strategy(), // Use BFGS search algorithm
              dlib::objective_delta_stop_strategy(
                  1e-7), // Stop when the change in rosen() is less than 1e-7
              optBetaBinMeanOverDis, optBetaBinDerMeanOverDis,
              startingPointMeanOverDis, dLibMinMeanOverDis,
              dLibMaxMeanOverDis);

          dlib::matrix<double, 0, 1> startingPointOverDis = {2.0};
          dlib::matrix<double, 0, 1> dLibMinOverDis = {0.1};
          dlib::matrix<double, 0, 1> dLibMaxOverDis = {10000.0};
          OptimizeBetaBinOverDis optBetaBinOverDis(
              testCounts, config.getMeanFreqSite());
          OptimizeBetaBinOverDisDerivates optBetaBinDerOverDis(
              testCounts, config.getMeanFreqSite());
          double resultOverDis = dlib::find_max_box_constrained(
              dlib::bfgs_search_strategy(), // Use BFGS search algorithm
              dlib::objective_delta_stop_strategy(
                  1e-7), // Stop when the change in rosen() is less than 1e-7
              optBetaBinOverDis, optBetaBinDerOverDis, startingPointOverDis,
              dLibMinOverDis, dLibMaxOverDis);

          double pValue;
          if (resultOverDis >
              resultMeanOverDis) // if true the results are within the optimization limit
            pValue = 1;
          else
          {
            double testStat = -2 * (resultOverDis - resultMeanOverDis);
            boost::math::chi_squared mydist(1);
            pValue = 1 - boost::math::cdf(mydist, testStat);
          }

          if (pValue > 0.05 ||
              startingPointMeanOverDis(0) >= config.getMeanFreqSite() ||
              incMap.count(
                  std::make_tuple(std::move(ChromosomeLabel(splitVec[0])),
                                  std::stoi(splitVec[1]), splitVec[2][0],
                                  std::move(std::vector<char>(
                                      indexToChar(altAlleleIdx))))) != 0)
          {
            hasSignificantAltNucs = true;
            significantAltNucs.addSigAltNuc(SignificantAltNuc(
                altAlleleIdx, startingPointMeanOverDis(0), pValue));
          }
        }
      }

      if (!positionMutated)
        for (size_t cell = 0; cell < tumorCellNum; cell++)
          continuousNoiseCounts.add(std::array<u_int32_t, 4>{
              counts[cell][altNucs[0]], counts[cell][altNucs[1]],
              counts[cell][altNucs[2]], counts[cell][4]});

      // only treat the site as a candidate snv if it contains significant alternative nucleotides
      if (hasSignificantAltNucs)
      {
        // sort significant alternative nucleotides
        significantAltNucs.getOrderedSigAltNucs();

        // collect read counts for each alternative nucleotide in each cell
        for (size_t cell = 0; cell < tumorCellNum; cell++)
        {
          cellReadCounts.emplace_back(CellReadCounts(
              AltNucReadCount(altNucs[0], indexToChar(altNucs[0]),
                              counts[cell][altNucs[0]], significantAltNucs),
              AltNucReadCount(altNucs[1], indexToChar(altNucs[1]),
                              counts[cell][altNucs[1]], significantAltNucs),
              AltNucReadCount(altNucs[2], indexToChar(altNucs[2]),
                              counts[cell][altNucs[2]], significantAltNucs),
              counts[cell][4]));

          cellReadCounts[cell].sortAltNucReadCounts();
        }

        // save the data
        _data.emplace_back(std::move(TDataEntry(
            std::move(
                TPosInfo(std::move(ChromosomeLabel(std::move(splitVec[0]))),
                         std::stoi(splitVec[1]), splitVec[2][0],
                         std::move(significantAltNucs.convertAltNucsType(
                             indexToChar)))),
            std::move(cellReadCounts))));
      }
    }
  }

  {
    std::lock_guard<std::mutex> lock(_mutex);
    data.addReadCounts(std::move(_data));
    data.addContinuousNoiseCounts(continuousNoiseCounts);
  }

  std::cout << "> [" << _id << "] Done." << std::endl;
}


/** accuracy.cc
    Jeremy Barnes, 16 December 2014
    Copyright (c) 2014 mldb.ai inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

    Implementation of an ACCURACY algorithm for embedding of a dataset.
*/

#include "accuracy.h"
#include "mldb/builtin/matrix.h"
#include "mldb/core/mldb_engine.h"
#include "mldb/core/dataset.h"
#include "mldb/core/bound_queries.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/utils/distribution.h"
#include <boost/multi_array.hpp>
#include "mldb/base/scope.h"
#include "mldb/base/parallel.h"
#include "mldb/utils/pair_utils.h"
#include "mldb/arch/timers.h"
#include "mldb/arch/simd_vector.h"
#include "mldb/utils/vector_utils.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/plugins/jml/value_descriptions.h"
#include "mldb/plugins/jml/separation_stats.h"
#include "mldb/core/analytics.h"
#include "mldb/types/any_impl.h"
#include "mldb/base/per_thread_accumulator.h"
#include "mldb/types/optional_description.h"
#include "mldb/types/annotated_exception.h"
#include "mldb/builtin/sql_config_validator.h"
#include "mldb/builtin/sql_expression_extractors.h"
#include "mldb/base/parallel_merge_sort.h"
#include "mldb/utils/log.h"
#include "mldb/utils/possibly_dynamic_buffer.h"

#define NO_DATA_ERR_MSG "Cannot run classifier.test procedure on empty test set"

using namespace std;

namespace MLDB {

typedef std::vector<std::pair<RowPath, std::vector<std::tuple<ColumnPath, CellValue, Date> > > > Rows;

DEFINE_STRUCTURE_DESCRIPTION(AccuracyConfig);

AccuracyConfigDescription::
AccuracyConfigDescription()
{
    Optional<PolyConfigT<Dataset> > optionalOutputDataset;
    optionalOutputDataset.emplace(PolyConfigT<Dataset>().
                                  withType(AccuracyConfig::defaultOutputDatasetType));

    addField("mode", &AccuracyConfig::mode,
             "Model mode: `boolean`, `regression` or `categorical`. "
             "Controls how the label is interpreted and what is the output of the classifier. "
             "This must match what was used during training.", CM_BOOLEAN);
    addField("testingData", &AccuracyConfig::testingData,
             "SQL query which specifies the scores, labels and optional weights for evaluation. "
             "The query is usually of the form: "
             "`select classifier_function({features: {f1, f2}})[score] as score, x as label from ds`.\n\n"
             "The select expression must contain these two columns: \n\n"
             "  * `score`: one scalar expression which evaluates to the score a classifier "
             "has assigned to the given row, and \n"
             "  * `label`: one scalar expression to identify the row's label, and whose type "
             "must match that of the classifier mode. Rows with null labels will be ignored. \n"
             "     * `boolean` mode: a boolean (0 or 1)\n"
             "     * `regression` mode: a real number\n"
             "     * `categorical` mode: any combination of numbers and strings for\n\n"
             "The select expression can contain an optional `weight` column. The weight "
             "allows the relative importance of examples to be set. It must "
             "be a real number. If the `weight` is not specified each row will have "
             "a weight of 1. Rows with a null weight will cause a training error. \n\n"
             "The query must not contain `GROUP BY` or `HAVING` clauses. ");
    addField("outputDataset", &AccuracyConfig::outputDataset,
             "Output dataset for scored examples. The score for the test "
             "example will be written to this dataset. Examples get grouped when "
              "they have the same score when `mode` is `boolean`. Specifying a "
             "dataset is optional.", optionalOutputDataset);
    addField("uniqueScoresOnly", &AccuracyConfig::uniqueScoresOnly,
              "If `outputDataset` is set and `mode` is set to `boolean`, setting this parameter "
              "to `true` will output a single row per unique score. This is useful if the "
              "test set is very large and aggregate statistics for each unique score is "
              "sufficient, for instance to generate a ROC curve. This has no effect "
              "for other values of `mode`.", false);
    addField("recallOverN", &AccuracyConfig::accuracyOverN,
              "Calculate a recall score over the top scoring labels."
              "Does not apply to boolean or regression modes.");

    addParent<ProcedureConfig>();

    onPostValidate = validateQuery(&AccuracyConfig::testingData,
                                   NoGroupByHaving(),
                                   PlainColumnSelect(),
                                   ScoreLabelSelect(),
                                   MustContainFrom());

}

/*****************************************************************************/
/* ACCURACY PROCEDURE                                                         */
/*****************************************************************************/

AccuracyProcedure::
AccuracyProcedure(MldbEngine * owner,
                 PolyConfig config,
                 const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    this->accuracyConfig = config.params.convert<AccuracyConfig>();
    if (!accuracyConfig.testingData.stm)
        throw AnnotatedException(400, "Classifier testing procedure requires 'testingData' to be set",
                                  "config", this->accuracyConfig);
}

Any
AccuracyProcedure::
getStatus() const
{
    return Any();
}

RunOutput
runBoolean(AccuracyConfig & runAccuracyConf,
           BoundSelectQuery & selectQuery,
           std::shared_ptr<Dataset> output)
{

    PerThreadAccumulator<ScoredStats> accum;
    auto logger = MLDB::getMldbLog<AccuracyProcedure>();

    auto processor = [&] (NamedRowValue & row,
                          const std::vector<ExpressionValue> & scoreLabelWeight)
        {
            double score = scoreLabelWeight[0].toDouble();
            bool label = scoreLabelWeight[1].asBool();
            double weight = scoreLabelWeight[2].toDouble();

            TRACE_MSG(logger) << "score=" << score << "; label=" << label << "; weight=" << weight;

            accum.get().update(label, score, weight, row.rowName);

            return true;
        };

    selectQuery.execute({processor,true/*processInParallel*/}, runAccuracyConf.testingData.stm->offset,
             runAccuracyConf.testingData.stm->limit,
             nullptr /* progress */);

    // Now merge out stats together
    ScoredStats stats;
    bool gotStuff = false;

    accum.forEach([&] (ScoredStats * thrStats)
                  {
                      gotStuff = true;
                      thrStats->sort();
                      stats.add(*thrStats);
                  });

    if (!gotStuff) {
        throw MLDB::Exception(NO_DATA_ERR_MSG);
    }

    //stats.sort();
    stats.calculate();
    if(output) {
        const Date recordDate = Date::now();

        int prevIncludedPop = 0;

        std::vector<std::pair<RowPath, std::vector<std::tuple<ColumnPath, CellValue, Date> > > > rows;

        auto recordRow = [&] (unsigned i, const BinaryStats & bstats, ScoredStats::ScoredEntry & entry)
        {
            std::vector<std::tuple<RowPath, CellValue, Date> > row;

            row.emplace_back(ColumnPath("index"), i, recordDate);
            row.emplace_back(ColumnPath("label"), entry.label, recordDate);
            row.emplace_back(ColumnPath("score"), entry.score, recordDate);
            row.emplace_back(ColumnPath("weight"), entry.weight, recordDate);
            row.emplace_back(ColumnPath("truePositives"), bstats.truePositives(), recordDate);
            row.emplace_back(ColumnPath("falsePositives"), bstats.falsePositives(), recordDate);
            row.emplace_back(ColumnPath("trueNegatives"), bstats.trueNegatives(), recordDate);
            row.emplace_back(ColumnPath("falseNegatives"), bstats.falseNegatives(), recordDate);
            row.emplace_back(ColumnPath("accuracy"), bstats.accuracy(), recordDate);
            row.emplace_back(ColumnPath("precision"), bstats.precision(), recordDate);
            row.emplace_back(ColumnPath("recall"), bstats.recall(), recordDate);
            row.emplace_back(ColumnPath("truePositiveRate"), bstats.truePositiveRate(), recordDate);
            row.emplace_back(ColumnPath("falsePositiveRate"), bstats.falsePositiveRate(), recordDate);

            rows.emplace_back(std::any_cast<RowPath>(entry.key), std::move(row));
            if (rows.size() > 10000) {
                output->recordRows(rows);
                rows.clear();
            }
        };

        for (unsigned i = 1, j = 0;  i < stats.stats.size();  ++i) {
            auto & bstats = stats.stats[i];
            auto & entry = stats.entries[j];

            // the difference between included population of the current versus
            // last stats.stats represents the number of exemples included in the stats.
            // examples get grouped when they have the same score. use the unweighted
            // scores because we care about the actual number of examples, whatever
            // what their training weight was
            unsigned next_j = j + (bstats.includedPopulation(false) - prevIncludedPop);
            prevIncludedPop = bstats.includedPopulation(false);

            if(runAccuracyConf.uniqueScoresOnly) {
                j = next_j;
                ExcAssertEqual(bstats.threshold, entry.score);
                recordRow(i, bstats, entry);
            }
            else {
                for(; j<next_j; j++) {
                    entry = stats.entries[j];
                    ExcAssertEqual(bstats.threshold, entry.score);
                    recordRow(i, bstats, entry);
                }
            }
        }

        output->recordRows(rows);

        output->commit();
    }

    DEBUG_MSG(logger) << "stats are ";

    DEBUG_MSG(logger) << stats.toJson();

    DEBUG_MSG(logger) << stats.atPercentile(0.50).toJson();
    DEBUG_MSG(logger) << stats.atPercentile(0.20).toJson();
    DEBUG_MSG(logger) << stats.atPercentile(0.10).toJson();
    DEBUG_MSG(logger) << stats.atPercentile(0.05).toJson();
    DEBUG_MSG(logger) << stats.atPercentile(0.01).toJson();

    return Any(stats.toJson());
}

RunOutput
runCategorical(AccuracyConfig & runAccuracyConf,
               BoundSelectQuery & selectQuery,
               std::shared_ptr<Dataset> output,
               const std::vector<size_t>& accuracyOverN)
{
    typedef vector<std::tuple<CellValue, CellValue, double>> AccumBucket;
    typedef vector<std::tuple<CellValue, CellValue, std::vector<CellValue>, std::vector<double>,  double>> AccumBucketWithBestLabels;

    //We use one or the other
    PerThreadAccumulator<AccumBucket> accum;
    PerThreadAccumulator<AccumBucketWithBestLabels> accumWithBestLabels;

    PerThreadAccumulator<Rows> rowsAccum;
    Date recordDate = Date::now();
    bool computeTopN = accuracyOverN.size() > 0;
    size_t maxTopN = 1;
    for (auto& v : accuracyOverN) {
        maxTopN = std::max(maxTopN, v);
    }

    auto processor = [&] (NamedRowValue & row,
                           const std::vector<ExpressionValue> & scoreLabelWeight)
        {
            CellValue maxLabel;
            double maxLabelScore = -INFINITY;

            std::vector<std::tuple<RowPath, CellValue, Date> > outputRow;

            static const ColumnPath score("score");

            PossiblyDynamicBuffer<std::pair<double, CellValue>> bestlabelsCandidates(scoreLabelWeight[0].getAtomCount());
            size_t labelCount = 0;
            auto onAtom = [&] (const Path & columnName,
                               const Path & prefix,
                               const CellValue & val,
                               Date ts)
                {
                    auto v = val.toDouble();

                    if (std::isnan(v))
                        throw MLDB::Exception(MLDB::format("Classifier returned a NaN score ",
                        columnName.toSimpleName().rawString().c_str()));

                    if(v > maxLabelScore) {
                        maxLabelScore = v;
                        maxLabel = jsonDecodeStr<CellValue>(columnName.toSimpleName());
                    }

                    if(output) {
                        outputRow.emplace_back(score + columnName, v, recordDate);
                    }

                    if (computeTopN) {
                        bestlabelsCandidates[labelCount] = std::pair<double, CellValue>(v, jsonDecodeStr<CellValue>(columnName.toSimpleName()));
                        ++labelCount;
                    }

                    return true;
                };
            scoreLabelWeight[0].forEachAtom(onAtom);

            if (computeTopN) {
                //We cannot do a partial sort in case there are ties
                std::sort(bestlabelsCandidates.data(),
                          bestlabelsCandidates.data() + bestlabelsCandidates.size(),
                          std::greater<std::pair<double, CellValue>>());
            }

            size_t numBest = std::min(maxTopN, bestlabelsCandidates.size());

            //check for ties
            while (numBest > 0 && numBest < bestlabelsCandidates.size()) {
                if (bestlabelsCandidates[numBest - 1].second != bestlabelsCandidates[numBest].second)
                    break;

                numBest++;
            }

            PossiblyDynamicBuffer<CellValue> bestlabels(numBest);
            PossiblyDynamicBuffer<double> bestscores(numBest);
            for (size_t i = 0; i < numBest; ++i) {
                bestlabels[i] = bestlabelsCandidates[i].second;
                bestscores[i] = bestlabelsCandidates[i].first;
            }

            CellValue label = scoreLabelWeight[1].getAtom();
            double weight = scoreLabelWeight[2].toDouble();

            if (computeTopN) {
                accumWithBestLabels.get().emplace_back(label,
                                                       maxLabel,
                                                       std::vector<CellValue>(bestlabels.data(), bestlabels.data()+numBest),
                                                       std::vector<double>(bestscores.data(), bestscores.data()+numBest),
                                                       weight);
            }
            else
                accum.get().emplace_back(label, maxLabel, weight);

            if(output) {
                outputRow.emplace_back(ColumnPath("maxLabel"), maxLabel, recordDate);
                outputRow.emplace_back(ColumnPath("label"), label, recordDate);
                outputRow.emplace_back(ColumnPath("weight"), weight, recordDate);

                rowsAccum.get().emplace_back(row.rowName, std::move(outputRow));
                if(rowsAccum.get().size() > 1000) {
                    output->recordRows(rowsAccum.get());
                    rowsAccum.get().clear();
                }
            }

            return true;
        };

    selectQuery.execute({processor,true/*processInParallel*/},
            runAccuracyConf.testingData.stm->offset,
            runAccuracyConf.testingData.stm->limit,
            nullptr /* progress */);

    if(output) {
        rowsAccum.forEach([&] (Rows * thrRow)
            {
                output->recordRows(*thrRow);
            });
        output->commit();
    }

    // Create confusion matrix (actual / predicted)
    std::string recallString = "recallOverTopN";
    map<CellValue, map<CellValue, double>> confusion_matrix;
    map<CellValue, double> predicted_sums;
    map<CellValue, double> real_sums;
    map<std::pair<CellValue, int>, double> predicted_topn_sums;
    double conf_mat_sum = 0;
    bool gotStuff = false;
    if (computeTopN) {
        accumWithBestLabels.forEach([&] (AccumBucketWithBestLabels * thrBucket)
        {
            gotStuff = true;
            for(auto & elem : *thrBucket) {
                const CellValue & label = std::get<0>(elem);
                const CellValue & predicted = std::get<1>(elem);
                const std::vector<CellValue>& topPredicted = std::get<2>(elem);
                const std::vector<double>& topPredictedScores = std::get<3>(elem);
                const double & weight = std::get<4>(elem);
                confusion_matrix[label][predicted] += weight;
                real_sums[label] += weight;
                predicted_sums[predicted] += weight;
                conf_mat_sum += weight;
                auto it = std::find(topPredicted.begin(), topPredicted.end(), label);
                if (it != topPredicted.end()){

                    size_t rank = it - topPredicted.begin();
                    double score = topPredictedScores[rank];
                    size_t earliestRank = std::find(topPredictedScores.begin(), topPredictedScores.end(), score) - topPredictedScores.begin();
                    size_t ties = std::count(topPredictedScores.begin(), topPredictedScores.end(), score);
                    ExcAssert(ties > 0);

                    for (int i = 0; i < accuracyOverN.size(); ++i) {
                        auto limit = accuracyOverN[i];

                        if (earliestRank < limit)
                        {
                            //do we share last rank?
                            size_t lastRank = earliestRank + (ties-1);
                            if (ties > 1 && lastRank >= limit) {
                                //how many "last positions" are there?
                                size_t numPos = (limit - earliestRank);
                                ExcAssert(numPos < ties);
                                double correctedWeight = (weight * numPos) / ties;
                                predicted_topn_sums[{label, i}] += correctedWeight;
                            }
                            else {
                                predicted_topn_sums[{label, i}] += weight;
                            }
                        }
                    }
                }
            }
        });
    }
    else {
        accum.forEach([&] (AccumBucket * thrBucket)
        {
            gotStuff = true;
            for(auto & elem : *thrBucket) {
                const CellValue & label = std::get<0>(elem);
                const CellValue & predicted = std::get<1>(elem);
                const double & weight = std::get<2>(elem);
                confusion_matrix[label][predicted] += weight;
                real_sums[label] += weight;
                predicted_sums[predicted] += weight;
                conf_mat_sum += weight;
            }
        });
    }

    if (!gotStuff) {
        throw MLDB::Exception(NO_DATA_ERR_MSG);
    }
    // Create per-class statistics
    Json::Value results;
    results["labelStatistics"] = Json::Value();
    if (computeTopN)
        results["recallOverN"] = jsonEncode(accuracyOverN);

    double total_accuracy = 0;
    double total_precision = 0;
    std::vector<double> total_recall_topn(accuracyOverN.size(), 0);
    double total_recall = 0; // i'll be back!
    double total_f1 = 0;
    double total_support = 0;

    int nb_predicted_no_label = 0;

    auto logger = MLDB::getMldbLog<AccuracyProcedure>();
    results["confusionMatrix"] = Json::Value(Json::arrayValue);
    for(const auto & actual_it : confusion_matrix) {

        for(const auto & predicted_it : actual_it.second) {
            Json::Value conf_mat_elem;
            conf_mat_elem["predicted"] = jsonEncode(predicted_it.first);
            conf_mat_elem["actual"] = jsonEncode(actual_it.first);
            conf_mat_elem["count"] = predicted_it.second;
            results["confusionMatrix"].append(conf_mat_elem);
        }

        double tp = confusion_matrix[actual_it.first][actual_it.first];
        double fp = predicted_sums[actual_it.first] - tp;
        double fn = real_sums[actual_it.first] - tp;
        double tn = conf_mat_sum - fn - fp - tp;

        DEBUG_MSG(logger) << "label: " << actual_it.first;
        DEBUG_MSG(logger) << "TP: " << tp;
        DEBUG_MSG(logger) << "FP: " << fp;
        DEBUG_MSG(logger) << "TN: " << tn;
        DEBUG_MSG(logger) << "FN: " << fn;

        // if our class was (wrongfully) predicted (fp) but was never actually
        // there (tp + fn == 0), then this is strange
        if (tp + fn == 0 && fp > 0) {
            nb_predicted_no_label++;
            DEBUG_MSG(logger)
                << "WARNING!! Label '" << actual_it.first
                << "' was predicted but not in known labels!";
        }

        Json::Value class_stats;

        double accuracy = MLDB::xdiv(tp + tn, conf_mat_sum);
        double precision = MLDB::xdiv(tp, tp + fp);
        double support = tp + fn;
        double recall = MLDB::xdiv(tp, support);
        class_stats["accuracy"] = accuracy;
        class_stats["precision"] = precision;
        class_stats["recall"] = recall;
        class_stats["f1Score"] = 2 * MLDB::xdiv(precision * recall,
                                        precision + recall);
        class_stats["support"] = support;
        for (int i = 0; i < accuracyOverN.size(); ++i) {
            auto tptopn = predicted_topn_sums[{actual_it.first,i}];
            total_recall_topn[i] += tptopn;
            class_stats[recallString][i] = MLDB::xdiv(tptopn, real_sums[actual_it.first]);
        }

        results["labelStatistics"][actual_it.first.toUtf8String()] = class_stats;

        total_accuracy += accuracy * support;
        total_precision += precision * support;
        total_recall += recall * support;
        total_f1 += class_stats["f1Score"].asDouble() * support;
        total_support += support;
    }

    // Create weighted statistics
    Json::Value weighted_stats;
    weighted_stats["accuracy"] = total_accuracy / total_support;
    weighted_stats["precision"] = total_precision / total_support;
    weighted_stats["recall"] = total_recall / total_support;
    weighted_stats["f1Score"] = total_f1 / total_support;
    weighted_stats["support"] = total_support;

    for (int i = 0; i < accuracyOverN.size(); ++i) {
        weighted_stats[recallString][i] = total_recall_topn[i] / total_support;
    }

    results["weightedStatistics"] = weighted_stats;

    // TODO maybe this should always return an error? The problem is it is not impossible that because
    // of the way the dataset is split, it is a normal situation. But it can also point to
    // misalignment in the way columns are named

    // throw if precision is zero and at least one predicted value wasn't even
    // in the labels
    if (weighted_stats["precision"].asDouble() == 0
        && nb_predicted_no_label > 0) {
        throw MLDB::Exception(MLDB::format("Weighted precision is 0 and %i"
                "labels were predicted but not in true labels! "
                "Are the columns of the predicted labels named properly?",
                nb_predicted_no_label));
    }

    // cout << results.toStyledString() << endl;

    return Any(results);
}

RunOutput
runMultilabel(AccuracyConfig & runAccuracyConf,
               BoundSelectQuery & selectQuery,
               std::shared_ptr<Dataset> output,
               const std::vector<size_t>& accuracyOverN)
{
    //labels, bestlabels, weight
    typedef vector<std::tuple<std::vector<CellValue>, std::vector<CellValue>, std::vector<double>, double>> AccumBucket;
    PerThreadAccumulator<AccumBucket> accum;

    PerThreadAccumulator<Rows> rowsAccum;
    Date recordDate = Date::now();

    auto processor = [&] (NamedRowValue & row,
                           const std::vector<ExpressionValue> & scoreLabelWeight)
        {
            std::vector<CellValue> bestlabels;
            std::vector<double> bestlabelsScores;
            std::vector<std::pair<double, CellValue>> bestlabelsCandidates;

            bestlabels.reserve(scoreLabelWeight[0].getAtomCount());
            bestlabelsScores.reserve(scoreLabelWeight[0].getAtomCount());

            std::vector<std::tuple<RowPath, CellValue, Date> > outputRow;

            static const ColumnPath score("score");

            auto onAtom = [&] (const Path & columnName,
                               const Path & prefix,
                               const CellValue & val,
                               Date ts)
                {
                    auto v = val.toDouble();

                    if (std::isnan(v))
                        throw MLDB::Exception(MLDB::format("Classifier returned a NaN score ",
                        columnName.toSimpleName().rawString().c_str()));

                    CellValue scoreLabel = jsonDecodeStr<CellValue>(columnName.toSimpleName());
                    bestlabelsCandidates.push_back({v,scoreLabel});

                    if(output) {
                        outputRow.emplace_back(score + columnName, v, recordDate);
                    }

                    return true;
                };

                scoreLabelWeight[0].forEachAtom(onAtom);

                std::sort(bestlabelsCandidates.begin(),
                          bestlabelsCandidates.end(),
                          std::greater<std::pair<double, CellValue>>());

            for (const auto& pair : bestlabelsCandidates) {
                bestlabels.push_back(pair.second);
                bestlabelsScores.push_back(pair.first);
            }

            std::vector<CellValue> labels;
            std::function<bool (const PathElement & columnName,
                                const ExpressionValue & val)> randomStrategy = [&] (const PathElement & columnName,
                                                                              const ExpressionValue & val) ->bool
                {
                    if (!val.empty()) {
                        labels.push_back(columnName.toUtf8String());
                    }

                    return true;
                };

            scoreLabelWeight[1].forEachColumn(randomStrategy);

            if (labels.size() == 0)
                return true;

            double weight = scoreLabelWeight[2].toDouble();
            accum.get().emplace_back(labels, bestlabels, bestlabelsScores, weight);

            if(output) {
                for (int i = 0; i < labels.size(); ++i) {
                    outputRow.emplace_back(ColumnPath("label") + ColumnPath(i), labels[i], recordDate);
                }

                outputRow.emplace_back(ColumnPath("weight"), weight, recordDate);

                rowsAccum.get().emplace_back(row.rowName, std::move(outputRow));
                if(rowsAccum.get().size() > 1000) {
                    output->recordRows(rowsAccum.get());
                    rowsAccum.get().clear();
                }
            }

            return true;
        };

    selectQuery.execute({processor,true/*processInParallel*/},
            runAccuracyConf.testingData.stm->offset,
            runAccuracyConf.testingData.stm->limit,
            nullptr /* progress */);

    if(output) {
        rowsAccum.forEach([&] (Rows * thrRow)
            {
                output->recordRows(*thrRow);
            });
        output->commit();
    }

    map<std::pair<CellValue, int>, double> recallsums;
    map<CellValue, double> labelsums;

    double totalWeight = 0;
    std::vector<double> totalAccurateWeight(accuracyOverN.size(),0);
    double totalCoverageError = 0;

    accum.forEach([&] (AccumBucket * thrBucket)
            {
               // gotStuff = true;
                for(auto & elem : *thrBucket) {

                    const std::vector<CellValue> & labels = std::get<0>(elem);
                    const std::vector<CellValue> & topPredicted = std::get<1>(elem);
                    const std::vector<double> & topPredictedScores = std::get<2>(elem);
                    const double & weight = std::get<3>(elem);

                    if (weight == 0)
                        continue;

                    double maxRank = 0;
                    double totalExampleWeight = 0;

                    for (auto& label : labels) {
                        auto labelScoreIt = std::find(topPredicted.begin(), topPredicted.end(), label);
                        size_t rank = topPredicted.size();
                        double averageRank = (double)rank;
                        if ( labelScoreIt != topPredicted.end()) {
                            rank = labelScoreIt - topPredicted.begin();
                            double score = topPredictedScores[rank];
                            size_t earliestRank = std::find(topPredictedScores.begin(), topPredictedScores.end(), score) - topPredictedScores.begin();
                            size_t ties = std::count(topPredictedScores.begin(), topPredictedScores.end(), score);
                            ExcAssert(ties > 0);
                            averageRank = (2.f * earliestRank + (ties - 1)) / 2.0f;

                            for (int i = 0; i < accuracyOverN.size(); ++i) {

                                size_t limit = accuracyOverN[i];

                                if (earliestRank < limit) {

                                    //do we share last rank?
                                    size_t lastRank = earliestRank + (ties-1);
                                    if (ties > 1 && lastRank >= limit) {

                                        //how many "last positions" are there?
                                        size_t numPos = (limit - earliestRank);
                                        ExcAssert(numPos < ties);
                                        double correctedWeight = (weight * numPos) / ties;
                                        recallsums[{label, i}] += correctedWeight;
                                        totalAccurateWeight[i] += correctedWeight;
                                    }
                                    else {
                                        recallsums[{label, i}] += weight;
                                        totalAccurateWeight[i] += weight;
                                    }
                                }

                            }
                        }

                        maxRank = std::max(maxRank, averageRank);
                        labelsums[label] += weight;
                        totalWeight += weight;
                        totalExampleWeight += weight;
                    }

                    totalCoverageError += (1+maxRank)*totalExampleWeight;
                }
            });

    // Create per-class statistics
    Json::Value results;
    results["labelStatistics"] = Json::Value();
    results["recallOverN"] = jsonEncode(accuracyOverN);

    std::string recallString = "recallOverTopN";

    for(const auto & actual_it : labelsums) {
        Json::Value class_stats;
        for (int i = 0; i < accuracyOverN.size(); ++i) {
            double recallSum = 0;
            auto recall_it = recallsums.find({actual_it.first, i});
            if (recall_it != recallsums.end())
                recallSum = recall_it->second;

            class_stats[recallString][i] = recallSum / actual_it.second;
        }

        results["labelStatistics"][actual_it.first.toUtf8String()] = class_stats;
    }

    // Create weighted statistics
    Json::Value weighted_stats;
    for (int i = 0; i < accuracyOverN.size(); ++i)
        weighted_stats[recallString][i] = totalAccurateWeight[i] / totalWeight;
    weighted_stats["coverageError"] = totalCoverageError / totalWeight;
    results["weightedStatistics"] = weighted_stats;

    return Any(results);
}

RunOutput
runRegression(AccuracyConfig & runAccuracyConf,
               BoundSelectQuery & selectQuery,
               std::shared_ptr<Dataset> output)
{
    /* Calculate the r-squared. */
    struct ThreadStats {
        ThreadStats() :
            mse_sum(0), totalWeight(0)
        {}

        void increment(double v, double l, double w) {
            if (!std::isfinite(v)) return;

            mse_sum += pow(v-l, 2) * w;
            absolute_percentage.push_back(abs( (v-l)/l ));

            labelsWeights.emplace_back(l,w);
            totalWeight += w;
        }

        static void merge(ThreadStats & t1, ThreadStats & t2)
        {
            size_t split = t1.absolute_percentage.size();

            t1.absolute_percentage.insert(t1.absolute_percentage.end(),
                          std::make_move_iterator(t2.absolute_percentage.begin()),
                          std::make_move_iterator(t2.absolute_percentage.end()));
            t2.absolute_percentage.clear();

            std::inplace_merge(t1.absolute_percentage.begin(),
                               t1.absolute_percentage.begin() + split,
                               t1.absolute_percentage.end());
        }

        double mse_sum;
        double totalWeight;
        distribution<double> absolute_percentage;
        vector<pair<double, double>> labelsWeights;
    };

    PerThreadAccumulator<ThreadStats> accum;

    PerThreadAccumulator<Rows> rowsAccum;
    Date recordDate = Date::now();

    auto processor = [&] (NamedRowValue & row,
                          const std::vector<ExpressionValue> & scoreLabelWeight)
        {
            double score = scoreLabelWeight[0].toDouble();
            double label = scoreLabelWeight[1].toDouble();
            double weight = scoreLabelWeight[2].toDouble();

            accum.get().increment(score, label, weight);

            if(output) {
                std::vector<std::tuple<RowPath, CellValue, Date> > outputRow;

                outputRow.emplace_back(ColumnPath("score"), score, recordDate);
                outputRow.emplace_back(ColumnPath("label"), label, recordDate);
                outputRow.emplace_back(ColumnPath("weight"), weight, recordDate);

                rowsAccum.get().emplace_back(row.rowName, outputRow);
                if(rowsAccum.get().size() > 1000) {
                    output->recordRows(rowsAccum.get());
                    rowsAccum.get().clear();
                }
            }

            return true;
        };

    selectQuery.execute({processor,true/*processInParallel*/},
             runAccuracyConf.testingData.stm->offset,
             runAccuracyConf.testingData.stm->limit,
             nullptr /* progress */);

    if(output) {
        rowsAccum.forEach([&] (Rows * thrRow)
            {
                output->recordRows(*thrRow);
            });
        output->commit();
    }

    double totalWeight = 0, mse_sum = 0;
    vector<vector<pair<double,double>> > allThreadLabelsWeights;
    accum.forEach([&] (ThreadStats * thrStats)
                  {
                        totalWeight += thrStats->totalWeight;
                        mse_sum += thrStats->mse_sum;
                        allThreadLabelsWeights.emplace_back(std::move(thrStats->labelsWeights));
                  });

    if(totalWeight == 0) {
        throw MLDB::Exception(NO_DATA_ERR_MSG);
    }

    std::mutex mergeAccumsLock;

    double meanOfLabel = 0;
    auto doThreadMeanLbl = [&] (int threadNum) -> bool
    {
        double averageAccum = 0;
        for(auto & labelWeight : allThreadLabelsWeights[threadNum])
            averageAccum += labelWeight.first * labelWeight.second / totalWeight;

        std::unique_lock<std::mutex> guard(mergeAccumsLock);
        meanOfLabel += averageAccum;
        return true;
    };
    parallelMap(0, allThreadLabelsWeights.size(), doThreadMeanLbl);

    double totalSumSquares = 0;
    auto doThreadSS = [&] (int threadNum) -> bool
    {
        double ssAccum = 0;
        for(auto & labelWeight : allThreadLabelsWeights[threadNum])
            ssAccum += pow(labelWeight.first - meanOfLabel, 2) * labelWeight.second;

        std::unique_lock<std::mutex> guard(mergeAccumsLock);
        totalSumSquares += ssAccum;
        return true;
    };
    parallelMap(0, allThreadLabelsWeights.size(), doThreadSS);

    // calculate the r2 while catching the edge cases
    double r_squared;
    if      (mse_sum == 0)          r_squared = 1;
    else if (totalSumSquares == 0)  r_squared = 0;
    else                            r_squared = 1 - (mse_sum / totalSumSquares);

    // prepare absolute_percentage distribution
    distribution<double> absolute_percentage;

    parallelMergeSortRecursive(accum.threads, 0, accum.threads.size(),
                               [] (const std::shared_ptr<ThreadStats> & t)
                               {
                                   std::sort(t->absolute_percentage.begin(),
                                             t->absolute_percentage.end());
                               },
                               [] (const std::shared_ptr<ThreadStats> & t1,
                                   const std::shared_ptr<ThreadStats> & t2)
                               {
                                   ThreadStats::merge(*t1, *t2);
                               },
                               [] (const std::shared_ptr<ThreadStats> & t)
                               {
                                   return t->absolute_percentage.size();
                               },
                               10000 /* thread threshold */);
    if (!accum.threads.empty()) {
        absolute_percentage = std::move(accum.threads[0]->absolute_percentage);
    }

    // create return object
    Json::Value results;
    results["r2"] = r_squared;
//     results["b"] = b;
//     results["bd"] = bd;
    results["mse"] = mse_sum / totalWeight;

    Json::Value quantile_errors;
    if(absolute_percentage.size() > 0) {
        size_t size = absolute_percentage.size() - 1;
        quantile_errors["0.25"] = absolute_percentage[(int)(size*0.25)];
        quantile_errors["0.5"] = absolute_percentage[(int)(size*0.5)];
        quantile_errors["0.75"] = absolute_percentage[(int)(size*0.75)];
        quantile_errors["0.9"] = absolute_percentage[(int)(size*0.9)];
    }
    results["quantileErrors"] = quantile_errors;

    return Any(results);
}

RunOutput
AccuracyProcedure::
run(const ProcedureRunConfig & run,
    const std::function<bool (const Json::Value &)> & onProgress) const
{
    auto runAccuracyConf = applyRunConfOverProcConf(accuracyConfig, run);

    // 1.  Get the input dataset
    SqlExpressionMldbScope context(engine);

    ConvertProgressToJson convertProgressToJson(onProgress);
    auto dataset = runAccuracyConf.testingData.stm->from->bind(context, convertProgressToJson).dataset;

    // prepare output dataset
    std::shared_ptr<Dataset> output;
    if(runAccuracyConf.outputDataset) {
        PolyConfigT<Dataset> outputDataset = *runAccuracyConf.outputDataset;
        if (outputDataset.type.empty())
            outputDataset.type = AccuracyConfig::defaultOutputDatasetType;

        output = createDataset(engine, outputDataset, nullptr, true /*overwrite*/);
    }

    // 5.  Run it
    auto score = extractNamedSubSelect("score", runAccuracyConf.testingData.stm->select)->expression;
    auto label = extractNamedSubSelect("label", runAccuracyConf.testingData.stm->select)->expression;
    auto weightSubSelect = extractNamedSubSelect("weight", runAccuracyConf.testingData.stm->select);
    shared_ptr<SqlExpression> weight = weightSubSelect ? weightSubSelect->expression : SqlExpression::ONE;

    std::vector<std::shared_ptr<SqlExpression> > calc = {
        score,
        label,
        weight
    };

    auto boundQuery =
        BoundSelectQuery({} /* select */, *dataset, "" /* table alias */,
                     runAccuracyConf.testingData.stm->when,
                     *runAccuracyConf.testingData.stm->where,
                     runAccuracyConf.testingData.stm->orderBy,
                     calc);

    if(runAccuracyConf.mode == CM_BOOLEAN)
        return runBoolean(runAccuracyConf, boundQuery, output);
    if(runAccuracyConf.mode == CM_CATEGORICAL)
        return runCategorical(runAccuracyConf, boundQuery, output, runAccuracyConf.accuracyOverN);
    if(runAccuracyConf.mode == CM_REGRESSION)
        return runRegression(runAccuracyConf, boundQuery, output);
    if(runAccuracyConf.mode == CM_MULTILABEL)
        return runMultilabel(runAccuracyConf, boundQuery, output, runAccuracyConf.accuracyOverN);

    throw MLDB::Exception("Classification mode '%d' not implemented", runAccuracyConf.mode);
}

namespace {

RegisterProcedureType<AccuracyProcedure, AccuracyConfig>
regAccuracy(builtinPackage(),
            "Calculate the accuracy of a classifier on held-out data",
            "procedures/Accuracy.md.html");

} // file scope

} // namespace MLDB


// This file is part of snark, a generic and flexible library for robotics research
// Copyright (c) 2011 The University of Sydney
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. Neither the name of the University of Sydney nor the
//    names of its contributors may be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
// GRANTED BY THIS LICENSE.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

/// @author vsevolod vlaskine

#ifdef WIN32
#include <stdio.h>
#include <fcntl.h>
#include <io.h>
#endif

#include <comma/application/signal_flag.h>
#include <comma/base/exception.h>
#include <comma/csv/stream.h>
#include <comma/csv/impl/program_options.h>
#include <comma/math/compare.h>
#include <comma/visiting/traits.h>
#include <snark/visiting/eigen.h>

// #include <boost/random/mersenne_twister.hpp>
// #include <boost/random/uniform_real.hpp>
// #include <boost/random/variate_generator.hpp>
// #include <Eigen/Geometry>
// boost::mt19937 generator;
// boost::uniform_real< double > distribution( 0, 1 );
// boost::variate_generator< boost::mt19937&, boost::uniform_real< double > > r( generator, distribution );

int main( int argc, char** argv )
{
//     for( unsigned int i = 0; i < boost::lexical_cast< unsigned int >( argv[1] ); ++i )
//     {
//         double x = r() * 20 - 10;
//         double y = r() * 20 - 10;
//         double z = r() * 20 - 10;
//         std::cout << x << ',' << y << ',' << z << ',' << int( std::max( std::abs( x ), std::max( std::abs( y ), std::abs( z ) ) ) ) << std::endl;
//     }
//     return 0;

    try
    {
        std::string normal_string;
        std::string points_string;
        std::string point_outside;
        boost::program_options::options_description description( "options" );
        description.add_options()
            ( "help,h", "display help message" )
            ( "points,p", boost::program_options::value< std::string >( &points_string )->default_value( "0,0,0" ), "point(s) belonging to the plane, either 3 points, or 1 point, if --normal defined" )
            ( "point-outside", boost::program_options::value< std::string >( &point_outside ), "point on the side of the plane where the normal would point, a convenience option; 3 points are enough" )
            ( "normal,n", boost::program_options::value< std::string >( &normal_string ), "normal to the plane" )
            ( "intersections", "assume the input represents a trajectory, find all its intersections with the plane")
            ( "threshold", boost::program_options::value< double >(), "if --intersections present, any separation between contiguous points of trajectory greater than threshold will be treated as a gap in the trajectory (no intersections will lie in the gaps)");
        description.add( comma::csv::program_options::description( "x,y,z" ) );
        boost::program_options::variables_map vm;
        boost::program_options::store( boost::program_options::parse_command_line( argc, argv, description), vm );
        boost::program_options::notify( vm );
        if ( vm.count( "help" ) )
        {
            std::cerr << std::endl;
            std::cerr << "take points on stdin, append distance from a given plane" << std::endl;
            std::cerr << std::endl;
            std::cerr << "if --intersections is specified, assume the input represents a trajectory, find its intersections with the plane," << std::endl;
            std::cerr << "for each intersection, output adjacent points between which it occurs, the intersection point, and the direction of intersection (-1,0,+1)," << std::endl;
            std::cerr << "where 0 indicates that both adjacent points are in the plane" << std::endl;
            std::cerr << std::endl;
            std::cerr << "usage: " << std::endl;
            std::cerr << "    cat points.csv | points-slice [options] > points_with_distance.csv" << std::endl;
            std::cerr << "    cat trajectory.csv | points-slice [options] --intersections > intersections.csv" << std::endl;
            std::cerr << std::endl;
            std::cerr << description << std::endl;
            std::cerr << std::endl;
            std::cerr << "defining the plane" << std::endl;
            std::cerr << "    --points x1,y1,z1,x2,y2,z2,x3,y3,x3" << std::endl;
            std::cerr << "    --points x1,y1,z1,x2,y2,z2,x3,y3,x3 --point-outside x4,y4,z4" << std::endl;
            std::cerr << "    --points x,y,z --normal n1,n2,n3" << std::endl;
            std::cerr << "    --normal n1,n2,n3    (the plane passes through 0,0,0)" << std::endl;
            std::cerr << std::endl;
            std::cerr << "output" << std::endl;
            std::cerr << "    default: " << std::endl;
            std::cerr << "        x,y,z,distance, where distance is signed distance to the plane" << std::endl;
            std::cerr << std::endl;
            std::cerr << "    if --intersections is specified:" << std::endl;
            std::cerr << "        x1,y1,z1,x2,y2,z2,p1,p2,p3,i, where x1,y1,z1,x2,y2,z2 are the adjacent points, p1,p2,p3 is the intersection, and i is the direction" << std::endl;
            std::cerr << std::endl;
            std::cerr << "examples:" << std::endl;
            std::cerr << "   echo -e \"0,0,-1\\n0,0,0\\n0,0,1\" | points-slice --points 0,0,0,0,1,0,1,0,0" << std::endl;
            std::cerr << "   echo -e \"0,0,-1\\n0,0,0\\n0,0,1\" | points-slice --points 0,0,0,0,1,0,1,0,0 --point-outside 0,0,1" << std::endl;
            std::cerr << "   echo -e \"0,0,-1\\n0,0,0\\n0,0,1\" | points-slice --points 0,0,0 --normal 0,0,1" << std::endl;
            std::cerr << "   echo -e \"0,0,-1\\n0,0,0\\n0,0,1\" | points-slice --normal 0,0,1" << std::endl;
            std::cerr << "   echo -e \"0,0,-1\\n0,0,0\\n0,0,1\" | points-slice --normal 0,0,1 --intersections" << std::endl;
            std::cerr << "   echo -e \"0,0,-1\\n0,0,-0.5\\n0,0,0\\n0,0,1\\n0,0,1.5\" | points-slice --normal 0,0,1 --intersections --threshold=0.5" << std::endl;
            std::cerr << std::endl;
            return 1;
        }
        if( vm.count( "points" ) == 0 ) { std::cerr << "points-slice: please specify --points" << std::endl; return 1; }
        comma::csv::options csv = comma::csv::program_options::get( vm );
        Eigen::Vector3d normal;
        Eigen::Vector3d point;
        if( vm.count( "normal" ) )
        {
            normal = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", ',' ).get( normal_string );
            point = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", ',' ).get( points_string );
        }
        else
        {
            boost::array< Eigen::Vector3d, 3 > points; // quick and dirty
            std::vector< std::string > v = comma::split( points_string, ',' );
            if( v.size() != 9 ) { std::cerr << "points-slice: expected 3 points, got: \"" << points_string << "\"" << std::endl; return 1; }
            point = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", ',' ).get( v[0] + ',' + v[1] + ',' + v[2] ); // quick and dirty
            Eigen::Vector3d a = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", ',' ).get( v[3] + ',' + v[4] + ',' + v[5] ); // quick and dirty
            Eigen::Vector3d b = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", ',' ).get( v[6] + ',' + v[7] + ',' + v[8] ); // quick and dirty
            a -= point;
            b -= point;
            if( comma::math::equal( std::abs( a.dot( b ) ), a.norm() * b.norm() ) ) { std::cerr << "points-slice: given points are not corners or a triangle: \"" << points_string << "\"" << std::endl; }
            normal = a.cross( b );
            if( vm.count( "point-outside" ) )
            {
                Eigen::Vector3d p = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", ',' ).get( point_outside );
                if( comma::math::equal( ( p - point ).dot( normal ), 0 ) ) { std::cerr << "points-slice: expected a point outside of the plane, got: " << point_outside << ", which belongs to the plane" << std::endl; return 1; }
                normal *= normal.dot( p - point ) > 0 ? 1 : -1;
            }
        }
        normal.normalize();
        #ifdef WIN32
            _setmode( _fileno( stdout ), _O_BINARY ); /// @todo move to a library
        #endif
        comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
        comma::signal_flag is_shutdown;
        comma::csv::ascii< Eigen::Vector3d > ascii( "x,y,z", csv.delimiter );
        comma::csv::binary< Eigen::Vector3d > binary( "3d", "x,y,z" );
        if( vm.count("intersections") )
        {
            Eigen::Hyperplane< double, 3 > plane( normal, point );
            boost::optional< Eigen::Vector3d > last;
            double d_last = 0;
            boost::optional< double > threshold;
            if( vm.count("threshold") ) { threshold.reset( vm["threshold"].as< double >() ); }
            while( !is_shutdown && ( istream.ready() || ( !std::cin.eof() && std::cin.good() ) ) )
            {
                const Eigen::Vector3d* p = istream.read();
                if( !p ) { break; }
                double d = ( *p - point ).dot( normal );
                bool valid_intersection = false;
                if( last )
                {
                    bool intersects = d * d_last <= 0;
                    bool interval_within_threshold = !threshold || ( threshold && ( *p - *last ).norm() <= *threshold );
                    valid_intersection = intersects && interval_within_threshold;
                }
                if( valid_intersection )
                {
                    Eigen::Vector3d intersection_point;
                    BOOST_STATIC_ASSERT( sizeof( Eigen::Vector3d ) == sizeof( double ) * 3 );
                    bool lies_on_plane = ( d == 0 && d_last == 0 );
                    if( lies_on_plane )
                    {
                        intersection_point = *last;
                    }
                    else
                    {
                        Eigen::ParametrizedLine< double, 3 > line = Eigen::ParametrizedLine< double, 3 >::Through( *last, *p );
                        intersection_point = line.intersectionPoint( plane );
                    }
                    comma::int32 direction;
                    if( d_last != 0 ) { direction = ( d_last > 0 ) ? 1 : -1; }
                    else if( d != 0 ) { direction = ( d < 0 ) ? 1 : -1; }
                    else { direction = 0; }
                    if( csv.binary() )
                    {
                        std::cout.write( reinterpret_cast< const char* >( &( *last ) ), sizeof( double ) * 3 );
                        std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
                        std::cout.write( reinterpret_cast< const char* >( &intersection_point ), sizeof( double ) * 3 );
                        std::cout.write( reinterpret_cast< const char* >( &direction ), sizeof( comma::int32 ) );
                    }
                    else
                    {
                        std::cout << ascii.put( *last ) << csv.delimiter << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter
                                    << ascii.put( intersection_point ) << csv.delimiter << direction << std::endl;
                    }
                }
                last.reset( *p );
                d_last = d;
            }
        }
        else
        {
            while( !is_shutdown && ( istream.ready() || ( !std::cin.eof() && std::cin.good() ) ) )
            {
                const Eigen::Vector3d* p = istream.read();
                if( !p ) { break; }
                double d = ( *p - point ).dot( normal );
                if( csv.binary() )
                {
                    std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
                    std::cout.write( reinterpret_cast< const char* >( &d ), sizeof( double ) );
                }
                else
                {
                    std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter << d << std::endl;
                }
            }
        }
        return 0;
    }
    catch( std::exception& ex )
    {
        std::cerr << "points-slice: " << ex.what() << std::endl;
    }
    catch( ... )
    {
        std::cerr << "points-slice: unknown exception" << std::endl;
    }
    return 1;
}