/* Copyright (C) 2019-2020 IBM Corp.
 * This program is Licensed under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *   http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. See accompanying LICENSE file.
 */

#include <helib/PolyMod.h>
#include <helib/exceptions.h>
#include <NTL/ZZX.h>
#include <NTL/ZZ_p.h>
#include <vector>
#include <helib/NumbTh.h>

#include "io.h"

namespace helib {

PolyMod::PolyMod() : ringDescriptor(nullptr) {}
PolyMod::PolyMod(const std::shared_ptr<PolyModRing>& ringDescriptor) :
    PolyMod(NTL::ZZX(0), ringDescriptor)
{}
PolyMod::PolyMod(long input,
                 const std::shared_ptr<PolyModRing>& ringDescriptor) :
    PolyMod(NTL::ZZX(input), ringDescriptor)
{}
PolyMod::PolyMod(const std::vector<long>& input,
                 const std::shared_ptr<PolyModRing>& ringDescriptor) :
    PolyMod(ringDescriptor)
{
  *this = input;
}
PolyMod::PolyMod(const NTL::ZZX& input,
                 const std::shared_ptr<PolyModRing>& ringDescriptor) :
    ringDescriptor(ringDescriptor), data(input)
{
  this->modularReduce();
}

PolyMod& PolyMod::operator=(long input)
{
  assertValidity(*this);
  this->data = NTL::ZZX(input);
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator=(const std::vector<long>& input)
{
  assertValidity(*this);
  NTL::clear(data); // Make sure higher-degree terms don't remain
  for (std::size_t i = 0; i < input.size(); ++i)
    NTL::SetCoeff(data, i, input[i]);
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator=(const std::initializer_list<long>& input)
{
  assertValidity(*this);
  *this = std::vector<long>(input);
  return *this;
}

PolyMod& PolyMod::operator=(const NTL::ZZX& input)
{
  assertValidity(*this);
  this->data = input;
  this->modularReduce();
  return *this;
}

PolyMod::operator long() const
{
  assertValidity(*this);
  long ret;
  NTL::conv(ret, NTL::ConstTerm(this->data));
  return ret;
}

PolyMod::operator std::vector<long>() const
{
  assertValidity(*this);
  std::vector<long> ret(NTL::deg(ringDescriptor->G));
  for (std::size_t i = 0; i < ret.size(); ++i)
    NTL::conv(ret[i], NTL::coeff(data, i));
  return ret;
}

PolyMod::operator NTL::ZZX() const
{
  assertValidity(*this);
  return getData();
}

bool PolyMod::isValid() const { return ringDescriptor != nullptr; }

long PolyMod::getp2r() const { return ringDescriptor->p2r; }

NTL::ZZX PolyMod::getG() const { return ringDescriptor->G; }

const NTL::ZZX& PolyMod::getData() const
{
  assertValidity(*this);
  return this->data;
}

bool PolyMod::operator==(const PolyMod& rhs) const
{
  if (!isValid() && !rhs.isValid())
    return true;
  else
    return isValid() && rhs.isValid() &&
           *ringDescriptor == *(rhs.ringDescriptor) && data == rhs.data;
}

bool PolyMod::operator==(long rhs) const { return *this == NTL::ZZX(rhs); }

bool PolyMod::operator==(const std::vector<long>& rhs) const
{
  if (!this->isValid()) {
    return false;
  } else {
    PolyMod other(rhs, ringDescriptor);
    return *this == other;
  }
}

bool PolyMod::operator==(const NTL::ZZX& rhs) const
{
  if (!this->isValid()) {
    return false;
  } else {
    PolyMod copy(*this);
    // Using subtraction to ensure modularReduce is called.
    // We are checking for divisibility of difference by G in Z_p rather than
    // direct equality
    copy -= rhs;
    return copy.data == NTL::ZZX(0);
  }
}

PolyMod& PolyMod::negate()
{
  assertValidity(*this);
  *this *= -1;
  return *this;
}

PolyMod PolyMod::operator-() const
{
  assertValidity(*this);
  PolyMod poly(*this);
  poly.negate();
  return poly;
}

PolyMod PolyMod::operator*(const PolyMod& rhs) const
{
  assertInterop(*this, rhs);
  PolyMod result = *this;
  result *= rhs;
  return result;
}

PolyMod PolyMod::operator*(long rhs) const { return operator*(NTL::ZZX{rhs}); }

PolyMod PolyMod::operator*(const NTL::ZZX& rhs) const
{
  PolyMod result(*this);
  PolyMod multiplier(*this);
  multiplier = rhs;
  return result * multiplier;
}

PolyMod PolyMod::operator+(const PolyMod& rhs) const
{
  assertInterop(*this, rhs);
  PolyMod result(*this);
  result += rhs;
  return result;
}

PolyMod PolyMod::operator+(long rhs) const { return operator+(NTL::ZZX{rhs}); }

PolyMod PolyMod::operator+(const NTL::ZZX& rhs) const
{
  PolyMod result(*this);
  PolyMod addend(result);
  addend = rhs;
  return result + addend;
}

PolyMod PolyMod::operator-(const PolyMod& rhs) const
{
  assertInterop(*this, rhs);
  PolyMod result = *this;
  result -= rhs;
  return result;
}

PolyMod PolyMod::operator-(long rhs) const { return operator-(NTL::ZZX{rhs}); }

PolyMod PolyMod::operator-(const NTL::ZZX& rhs) const
{
  PolyMod result(*this);
  PolyMod subtrahend(result);
  subtrahend = rhs;
  return result - subtrahend;
}

PolyMod& PolyMod::operator*=(const PolyMod& otherPoly)
{
  assertInterop(*this, otherPoly);
  this->data *= otherPoly.data;
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator*=(long scalar)
{
  assertValidity(*this);
  this->data *= NTL::ZZX(scalar);
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator*=(const NTL::ZZX& otherPoly)
{
  assertValidity(*this);
  this->data *= otherPoly;
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator+=(const PolyMod& otherPoly)
{
  assertInterop(*this, otherPoly);
  this->data += otherPoly.data;
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator+=(long scalar)
{
  assertValidity(*this);
  this->data += NTL::ZZX(scalar);
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator+=(const NTL::ZZX& otherPoly)
{
  assertValidity(*this);
  this->data += otherPoly;
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator-=(const PolyMod& otherPoly)
{
  assertInterop(*this, otherPoly);
  this->data -= otherPoly.data;
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator-=(long scalar)
{
  assertValidity(*this);
  this->data -= NTL::ZZX(scalar);
  this->modularReduce();
  return *this;
}

PolyMod& PolyMod::operator-=(const NTL::ZZX& otherPoly)
{
  assertValidity(*this);
  this->data -= otherPoly;
  this->modularReduce();
  return *this;
}

void PolyMod::writeToJSON(std::ostream& os) const
{
  PolyMod::assertValidity(*this);
  executeRedirectJsonError<void>([&]() { os << writeToJSON(); });
}

JsonWrapper PolyMod::writeToJSON() const
{
  PolyMod::assertValidity(*this);

  return executeRedirectJsonError<JsonWrapper>(
      [&]() { return wrap(this->data); });
}

PolyMod PolyMod::readFromJSON(
    std::istream& is,
    const std::shared_ptr<PolyModRing>& ringDescriptor)
{
  PolyMod poly(ringDescriptor);
  poly.readJSON(is);
  return poly;
}

PolyMod PolyMod::readFromJSON(
    const JsonWrapper& jw,
    const std::shared_ptr<PolyModRing>& ringDescriptor)
{
  PolyMod poly(ringDescriptor);
  poly.readJSON(jw);
  return poly;
}

void PolyMod::readJSON(std::istream& is)
{
  executeRedirectJsonError<void>([&]() {
    json j;
    is >> j;
    this->readJSON(wrap(j));
  });
}

void PolyMod::readJSON(const JsonWrapper& jw)
{
  auto body = [&]() {
    PolyMod::assertValidity(*this);

    NTL::ZZX poly = unwrap(jw);

    long g_degree = NTL::deg(this->ringDescriptor->G);
    if (deg(poly) >= g_degree) {
      // Too many elements. Raising an error.
      std::stringstream err_msg;
      err_msg << "Cannot deserialize to PolyMod: Degree is too small.  "
              << "Trying to deserialize " << deg(poly) + 1 << " coefficients.  "
              << "Slot modulus degree is " << g_degree << ".";
      throw IOError(err_msg.str());
    }

    NTL::clear(this->data); // Make sure higher-degree terms don't remain
    this->data = poly;

    // Normalization (removal of leading zeros) is done by modularReduce.
    this->modularReduce();
  };

  executeRedirectJsonError<void>(body);
}

std::istream& operator>>(std::istream& is, PolyMod& poly)
{
  PolyMod::assertValidity(poly);

  poly.readJSON(is);
  return is;
}

std::ostream& operator<<(std::ostream& os, const PolyMod& poly)
{
  PolyMod::assertValidity(poly);

  poly.writeToJSON(os);
  return os;
}

void PolyMod::modularReduce()
{
  NTL::ZZ_pContext pContext;
  pContext.save();
  NTL::ZZ_p::init(NTL::ZZ(ringDescriptor->p2r));
  NTL::ZZ_pX poly_mod_p2r;
  NTL::conv(poly_mod_p2r, this->data);
  NTL::ZZ_pX G_mod_p2r;
  NTL::conv(G_mod_p2r, ringDescriptor->G);
  poly_mod_p2r %= G_mod_p2r;
  NTL::conv(this->data, poly_mod_p2r);
  pContext.restore();
  this->data.normalize();
}

void PolyMod::assertValidity(const PolyMod& poly)
{
  if (!poly.isValid()) {
    throw LogicError("Cannot operate on invalid (default constructed) PolyMod");
  }
}

void PolyMod::assertInterop(const PolyMod& lhs, const PolyMod& rhs)
{
  assertValidity(lhs);
  assertValidity(rhs);
  if (*(lhs.ringDescriptor) != *(rhs.ringDescriptor))
    throw LogicError("Ring descriptors are not equal between PolyMod objects");
}

} // namespace helib


//==================================================================================================
/*!
  @file

  @copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#ifndef BOOST_SIMD_FUNCTION_NEXT_HPP_INCLUDED
#define BOOST_SIMD_FUNCTION_NEXT_HPP_INCLUDED

#if defined(DOXYGEN_ONLY)
namespace boost { namespace simd
{

 /*!

    @ingroup group-ieee
    Function object implementing next capabilities

    Returns if it exists the smallest representable value  strictly greater than the parameter

    @par Semantic:

    @code
    auto r = next(x);
    @endcode

    @par Note

    - for entries of floating types

       - next(Valmax) is Inf

       - next(-Inf) is Valmin

       - next(Inf) is Inf

       - next(Nan) is Nan

    - for entries of integral type

      - next(Valmax) is Valmin

    @see nextafter, prev, successor, predecessor

  **/
  Value next(Value const & x);
} }
#endif

#include <boost/simd/function/scalar/next.hpp>
#include <boost/simd/function/simd/next.hpp>

#endif


#include <cstdio>
#include <cmath>
#include <cfloat>
#include <cstdarg>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <map>
#include <Eigen/Dense>
#include <unsupported/Eigen/CXX11/Tensor>
#include <boost/filesystem.hpp>
#include <omp.h>
#include <H5Cpp.h>

#include "timer/timer.h"

/** \brief Read all files in a directory matching the given extension.
 * \param[in] directory path to directory
 * \param[out] files read file paths
 * \param[in] extension extension to filter for
 */
void read_directory(const boost::filesystem::path directory, std::map<int, boost::filesystem::path>& files, const std::string extension = ".off") {
  files.clear();
  boost::filesystem::directory_iterator end;

  for (boost::filesystem::directory_iterator it(directory); it != end; ++it) {
    if (it->path().extension().string() == extension) {
      if (!boost::filesystem::is_empty(it->path()) && !it->path().empty() && it->path().filename().string() != "") {
        int number = std::stoi(it->path().filename().string());
        files.insert(std::pair<int, boost::filesystem::path>(number, it->path()));
      }
    }
  }
}

/** \brief Just encapsulating vertices and faces. */
class Mesh {
public:
  /** \brief Empty constructor. */
  Mesh() {

  }

  /** \brief Add a vertex.
   * \param[in] vertex vertex to add
   */
  void add_vertex(Eigen::Vector3f& vertex) {
    this->vertices.push_back(vertex);
  }

  /** \brief Get the number of vertices.
   * \return number of vertices
   */
  int num_vertices() const {
    return static_cast<int>(this->vertices.size());
  }

  Eigen::Vector3f vertex(int v) {
    assert(v >= 0 && v < this->vertices.size());
    return this->vertices[v];
  }

  /** \brief Add a face.
   * \param[in] face face to add
   */
  void add_face(Eigen::Vector3i& face) {
    this->faces.push_back(face);
  }

  /** \brief Get the number of faces.
   * \return number of faces
   */
  int num_faces() const {
    return static_cast<int>(this->faces.size());
  }

  Eigen::Vector3i face(int f) {
    assert(f >= 0 && f < this->num_faces());
    return this->faces[f];
  }

  /** \brief Rotate the point cloud around the origin.
   * \param[in] rotation rotation matrix
   */
  void rotate(const Eigen::Matrix3f &rotation) {
    for (int v = 0; v < this->num_vertices(); ++v) {
      this->vertices[v] = rotation*this->vertices[v];
    }
  }

  /** \brief Translate the mesh.
   * \param[in] translation translation vector
   */
  void translate(const Eigen::Vector3f& translation) {
    for (int v = 0; v < this->num_vertices(); ++v) {
      for (int i = 0; i < 3; ++i) {
        this->vertices[v](i) += translation(i);
      }
    }
  }

  /** \brief Scale the mesh.
   * \param[in] scale scale vector
   */
  void scale(const Eigen::Vector3f& scale) {
    for (int v = 0; v < this->num_vertices(); ++v) {
      for (int i = 0; i < 3; ++i) {
        this->vertices[v](i) *= scale(i);
      }
    }
  }

  /** \brief Sample points from the mesh
   * \param[in] mesh mesh to sample from
   * \param[in] n batch index in points
   * \param[in] points pre-initialized tensor holding points
   */
  bool sample(const int N, const int k, Eigen::Tensor<float, 3, Eigen::RowMajor> &points) {

    // Stores the areas of faces.
    std::vector<float> areas(this->num_faces());
    float sum = 0;

    // Build a probability distribution over faces.
    for (int f = 0; f < this->num_faces(); f++) {
      Eigen::Vector3f a = this->vertices[this->faces[f][0]];
      Eigen::Vector3f b = this->vertices[this->faces[f][1]];
      Eigen::Vector3f c = this->vertices[this->faces[f][2]];

      // Angle between a->b and a->c.
      Eigen::Vector3f ab = b - a;
      Eigen::Vector3f ac = c - a;
      float cos_angle = ab.dot(ac)/(ab.norm()*ac.norm());
      float angle = std::acos(cos_angle);

      // Compute triangle area.
      float area = std::max(0., 0.5*ab.norm()*ac.norm()*std::sin(angle));
      //std::cout << area << " " << std::pow(area, 1./4.) << " " << angle << " " << ab.norm() << " " << ac.norm() << " " << std::sin(angle) << std::endl;

      // Accumulate.
      //area = std::sqrt(area);
      areas[f] = area;
      sum += area;
      //areas.push_back(1);
      //sum += 1;
    }

    //std::cout << sum << std::endl;
    if (sum < 1e-6) {
      std::cout << "[Error] face area sum of " << sum << std::endl;
      return false;
    }

    for (int f = 0; f < this->num_faces(); f++) {
      //std::cout << areas[f] << " ";
      areas[f] /= sum;
      //std::cout << areas[f] << std::endl;
    }

    std::vector<float> cum_areas(areas.size());
    cum_areas[0] = areas[0];

    for (int f = 1; f < this->num_faces(); f++) {
      cum_areas[f] = areas[f] + cum_areas[f - 1];
    }

    int j = 0;
    for (int f = 0; f < this->num_faces(); f++) {
      int n = std::max(static_cast<int>(areas[f]*N), 1);

      for (int i = 0; i < n; i++) {
        float r1 = 0;
        float r2 = 0;
        do {
          r1 = static_cast<float>(std::rand())/static_cast<float>(RAND_MAX);
          r2 = static_cast<float>(std::rand())/static_cast<float>(RAND_MAX);
        }
        while (r1 + r2 > 1.f);

        int s = std::rand()%3;
        //std::cout << face << " " << areas[face] << std::endl;

        Eigen::Vector3f a = this->vertices[this->faces[f](s)];
        Eigen::Vector3f b = this->vertices[this->faces[f]((s + 1)%3)];
        Eigen::Vector3f c = this->vertices[this->faces[f]((s + 2)%3)];

        Eigen::Vector3f ab = b - a;
        Eigen::Vector3f ac = c - a;

        Eigen::Vector3f point = a + r1*ab + r2*ac;
        points(k, j, 0) = point(0);
        points(k, j, 1) = point(1);
        points(k, j, 2) = point(2);
        j++;
      }
    }

    return true;
  }

  /** \brief Reading an off file and returning the vertices x, y, z coordinates and the
   * face indices.
   * \param[in] filepath path to the OFF file
   * \param[out] mesh read mesh with vertices and faces
   * \return success
   */
  static bool from_off(const std::string filepath, Mesh& mesh) {

    std::ifstream* file = new std::ifstream(filepath.c_str());
    std::string line;
    std::stringstream ss;
    int line_nb = 0;

    std::getline(*file, line);
    ++line_nb;

    if (line != "off" && line != "OFF") {
      std::cout << "[Error] Invalid header: \"" << line << "\", " << filepath << std::endl;
      return false;
    }

    size_t n_edges;
    std::getline(*file, line);
    ++line_nb;

    int n_vertices;
    int n_faces;
    ss << line;
    ss >> n_vertices;
    ss >> n_faces;
    ss >> n_edges;

    for (size_t v = 0; v < n_vertices; ++v) {
      std::getline(*file, line);
      ++line_nb;

      ss.clear();
      ss.str("");

      Eigen::Vector3f vertex;
      ss << line;
      ss >> vertex(0);
      ss >> vertex(1);
      ss >> vertex(2);

      mesh.add_vertex(vertex);
    }

    size_t n;
    for (size_t f = 0; f < n_faces; ++f) {
      std::getline(*file, line);
      ++line_nb;

      ss.clear();
      ss.str("");

      size_t n;
      ss << line;
      ss >> n;

      if(n != 3) {
        std::cout << "[Error] Not a triangle (" << n << " points) at " << (line_nb - 1) << std::endl;
        return false;
      }

      Eigen::Vector3i face;
      ss >> face(0);
      ss >> face(1);
      ss >> face(2);

      mesh.add_face(face);
    }

    if (n_vertices != mesh.num_vertices()) {
      std::cout << "[Error] Number of vertices in header differs from actual number of vertices." << std::endl;
      return false;
    }

    if (n_faces != mesh.num_faces()) {
      std::cout << "[Error] Number of faces in header differs from actual number of faces." << std::endl;
      return false;
    }

    file->close();
    delete file;

    return true;
  }

  /** \brief Write mesh to OFF file.
   * \param[in] filepath path to OFF file to write
   * \return success
   */
  bool to_off(const std::string filepath) {
    std::ofstream* out = new std::ofstream(filepath, std::ofstream::out);
    if (!static_cast<bool>(*out)) {
      return false;
    }

    (*out) << "OFF" << std::endl;
    (*out) << this->vertices.size() << " " << this->num_faces() << " 0" << std::endl;

    for (unsigned int v = 0; v < this->vertices.size(); v++) {
      (*out) << this->vertices[v](0) << " " << this->vertices[v](1) << " " << this->vertices[v](2) << std::endl;
    }

    for (unsigned int f = 0; f < this->num_faces(); f++) {
      (*out) << "3 " << this->faces[f](0) << " " << this->faces[f](1) << " " << this->faces[f](2) << std::endl;
    }

    out->close();
    delete out;

    return true;
  }

  /** \brief Write mesh to obj file.
   * \param[in] filepath
   * \param[in] mtl_lib
   * \param[in] materials
   * \return success
   */
  bool to_obj(const std::string filepath) {

    std::ofstream* out = new std::ofstream(filepath, std::ofstream::out);
    if (!static_cast<bool>(out)) {
      return false;
    }

    for (unsigned int v = 0; v < this->vertices.size(); v++) {
      (*out) << "v " << this->vertices[v](0) << " " << this->vertices[v](1) << " " << this->vertices[v](2) << std::endl;
    }

    for (unsigned int f = 0; f < this->num_faces(); f++) {
      (*out) << "f " << this->faces[f](0) + 1 << " " << this->faces[f](1) + 1 << " " << this->faces[f](2) + 1 << std::endl;
    }

    out->close();
    delete out;

    return true;
  }

private:

  /** \brief Vertices as (x,y,z)-vectors. */
  std::vector<Eigen::Vector3f> vertices;

  /** \brief Faces as list of vertex indices. */
  std::vector<Eigen::Vector3i> faces;
};

/** \brief Write the given set of volumes to h5 file.
 * \param[in] filepath h5 file to write
 * \param[in] n number of volumes
 * \param[in] height height of volumes
 * \param[in] width width of volumes
 * \param[in] depth depth of volumes
 * \param[in] dense volume data
 */
bool write_hdf5(const std::string filepath, Eigen::Tensor<float, 3, Eigen::RowMajor>& dense) {

  try {

    /*
     * Turn off the auto-printing when failure occurs so that we can
     * handle the errors appropriately
     */
    H5::Exception::dontPrint();

    /*
     * Create a new file using H5F_ACC_TRUNC access,
     * default file creation properties, and default file
     * access properties.
     */
    H5::H5File file(filepath, H5F_ACC_TRUNC);

    /*
     * Define the size of the array and create the data space for fixed
     * size dataset.
     */
    hsize_t rank = 3;
    hsize_t dimsf[rank];
    dimsf[0] = dense.dimension(0);
    dimsf[1] = dense.dimension(1);
    dimsf[2] = dense.dimension(2);
    H5::DataSpace dataspace(rank, dimsf);

    /*
     * Define datatype for the data in the file.
     * We will store little endian INT numbers.
     */
    H5::IntType datatype(H5::PredType::NATIVE_FLOAT);
    datatype.setOrder(H5T_ORDER_LE);

    /*
     * Create a new dataset within the file using defined dataspace and
     * datatype and default dataset creation properties.
     */
    H5::DataSet dataset = file.createDataSet("tensor", datatype, dataspace);

    /*
     * Write the data to the dataset using default memory space, file
     * space, and transfer properties.
     */
    float* data = static_cast<float*>(dense.data());
    dataset.write(data, H5::PredType::NATIVE_FLOAT);
  }  // end of try block

  // catch failure caused by the H5File operations
  catch(H5::FileIException error) {
    error.printError();
    return false;
  }

  // catch failure caused by the DataSet operations
  catch(H5::DataSetIException error) {
    error.printError();
    return false;
  }

  // catch failure caused by the DataSpace operations
  catch(H5::DataSpaceIException error) {
    error.printError();
    return false;
  }

  // catch failure caused by the DataSpace operations
  catch(H5::DataTypeIException error) {
    error.printError();
    return false;
  }

  return true;
}

/** \brief Main entrance point of script.
 * Expects one argument, the path to the config file.
 */
int main(int argc, char** argv) {
  if (argc < 2) {
    std::cout << "[Error] Usage: sample off_directory h5_file" << std::endl;
    exit(1);
  }

  boost::filesystem::path off_directory(argv[1]);
  boost::filesystem::path h5_file(argv[2]);

  if (!boost::filesystem::is_directory(off_directory)) {
    std::cout << "[Error] directory " << off_directory << " not found" << std::endl;
    exit(1);
  }

  std::map<int, boost::filesystem::path> off_files;
  read_directory(off_directory, off_files);
  std::cout << "[ICP] found " << off_files.size() << " files" << std::endl;

  std::vector<int> indices;
  for (std::map<int, boost::filesystem::path>::iterator it = off_files.begin(); it != off_files.end(); it++) {
    indices.push_back(it->first);
  }

  int N_points = 1000000;
  Eigen::Tensor<float, 3, Eigen::RowMajor> points(indices.size(), N_points, 3);
  points.setZero();

  float total = 0;

  omp_set_num_threads(16);
  #pragma omp parallel
  {
    #pragma omp for
    for (unsigned int i = 0; i < indices.size(); i++) {
      int n = indices[i];
      std::string off_file = off_files[n].string();

      Mesh mesh;
      Mesh::from_off(off_file, mesh);

      Timer timer;
      timer.start();
      mesh.sample(N_points, i, points);
      timer.stop();

      float elapsed = timer.getElapsedTimeInMilliSec();
      std::cout << "[Sample] sampled " << off_file << " (" << elapsed << "ms)" << std::endl;

      #pragma omp critical
      {
        total += elapsed;
      }
    }
  }

  write_hdf5(h5_file.string(), points);
  std::cout << "[Sample] wrote " << h5_file.string() << std::endl;
  std::cout << "[Sample] took  on average " << total/indices.size() << "ms" << std::endl;
  exit(0);
}


//  (C) Copyright Steve Cleary & John Maddock 2000.
//  Use, modification and distribution are subject to the
//  Boost Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

//  See http://www.boost.org for most recent version including documentation.

#include <climits>
#include <cwchar>
#include <boost/static_assert.hpp>

#if !defined(WCHAR_MIN)
#define WCHAR_MIN 0
#endif

namespace boost{

namespace my_conditions {
BOOST_STATIC_ASSERT(sizeof(int) * CHAR_BIT >= 32);
BOOST_STATIC_ASSERT(WCHAR_MIN >= 0);

} // namespace my_conditions

} // namespace boost

int main()
{
   return 0;
}


#include <boost/python/class.hpp>
#include <boost/python/return_internal_reference.hpp>
#include <boost/python/args.hpp>
#include <boost/python/return_arg.hpp>
#include <boost/python/operators.hpp>
#include <boost/python/tuple.hpp>
#include <boost/python/str.hpp>
#include <boost/python/slice.hpp>
#include <boost/python/extract.hpp>
#include <boost/python/make_constructor.hpp>

#include <boost_adaptbx/convertible.h>

#include <scitbx/array_family/accessors/flex_grid.h>
#include <scitbx/sparse/matrix.h>
#include <scitbx/sparse/io.h>
#include <scitbx/sparse/boost_python/vector.h>

namespace scitbx { namespace sparse { namespace boost_python {

template<typename T>
struct matrix_wrapper
{
  typedef matrix<T> wt;
  typedef typename wt::column_type column_type;
  typedef typename wt::index_type index_type;
  typedef typename wt::value_type value_type;

  static wt *from_list_of_dict(index_type m, index_type n,
                               boost::python::list cols)
  {
    using namespace boost::python;
    SCITBX_ASSERT(len(cols) == n);
    typedef vector_from_dict<T, copy_semantic_vector_container>
            vector_from_dict_t;
    wt *result = new wt(m, n);
    for (index_type j=0; j<n; ++j) {
      dict col = extract<dict>(cols[j]);
      result->col(j) = vector_from_dict_t::make_on_stack(m, col);
    }
    return result;
  }

  static boost::python::object setitem(wt& self,
                                       boost::python::tuple ij,
                                       boost::python::object x)
  {
    using namespace boost::python;
    object none;
    extract<index_type> p_i(ij[0]);
    extract<index_type> p_j(ij[1]);
    if (p_j.check()) {
      if (p_i.check()) {
        self(p_i(), p_j()) = extract<T>(x);
        return x;
      }
      else {
        extract<slice> p_i(ij[0]);
        if (p_i.check()) {
          slice i = p_i();
          if (i.start() == none && i.stop() == none) {
            self.col(p_j()) = extract<column_type>(x);
            return x;
          }
        }
      }
    }
    throw scitbx::error("Only self[i,j] = float() "
                        "and self[:,j] = sparse.vector() are supported.");
  }

  static boost::python::object getitem(wt& self, boost::python::tuple ij) {
    using namespace boost::python;
    object none;
    extract<index_type> p_i(ij[0]);
    extract<index_type> p_j(ij[1]);
    if (p_j.check()) {
      if (p_i.check()) {
        T x = self(p_i(), p_j());
        return object(x);
      }
      else {
        extract<slice> p_i(ij[0]);
        if (p_i.check()) {
          slice i = p_i();
          if (i.start() == none && i.stop() == none) {
            return object(self.col(p_j()));
          }
        }
      }
    }
    throw scitbx::error("Only self[i,j] and self[:,j] are supported.");
  }

  static boost::python::str str_(wt const &m) {
    std::stringstream o(std::ios_base::out);
    o << dense_display(m);
    return boost::python::str(o.str().c_str());
  }

  static boost::python::str repr(wt const &m) {
    std::stringstream o(std::ios_base::out);
    std::string start("sparse.matrix(");
    o << start << "rows=" << m.n_rows() << ", columns=" << m.n_cols() << ",\n";
    std::string elts("elements_by_columns=[ ");
    o << std::setw(start.size()) << "" << elts;
    for (index_type j=0; j<m.n_cols(); ++j) {
      if (j > 0) o << std::setw(start.size() + elts.size()) << "";
      o << compressed_display(m.col(j)) << ",";
      if (j+1 < m.n_cols()) o << "\n";
    }
    o << " ])";
    return boost::python::str(o.str().c_str());
  }

  static wt &permute_rows(wt &m, af::const_ref<index_type> const &p) {
    return m.permute_rows(p);
  }

  static void wrap() {
    using namespace boost::python;
    return_internal_reference<> rir;
    boost_adaptbx::convertible_type_to_python<
      matrix_times_dense_vector<T>,
      af::shared<T>,
      af::versa<value_type, af::flex_grid<> > >();
    class_<wt>("matrix", no_init)
      .def(init<typename wt::index_type,
                typename wt::index_type>
           ((arg("rows"), arg("columns"))))
      .def("__init__",
           make_constructor(from_list_of_dict,
                            default_call_policies(),
                            (arg("rows"), arg("columns"),
                             arg("elements_by_columns"))))
      .add_property("n_cols", &wt::n_cols)
      .add_property("n_rows", &wt::n_rows)
      .def("col",
           static_cast<column_type& (wt::*)(index_type)>(&wt::col),
           return_internal_reference<1>())
      .def("__getitem__", getitem)
      .def("__setitem__", setitem)
      .def("assign_block",
           (void (wt::*)(af::const_ref<double, af::mat_grid> const &,
                         int, int))&wt::assign_block)
      .def("assign_block",
           (void (wt::*)(wt const &, int, int))&wt::assign_block)
      .def("is_structural_zero", &wt::is_structural_zero)
      .def("compact", &wt::compact)
      .def("is_upper_triangular", &wt::is_upper_triangular)
      .def("is_unit_lower_triangular", &wt::is_unit_lower_triangular)
      .add_property("non_zeroes", &wt::non_zeroes)
      .def("clone", &wt::clone)
      .def("transpose", &wt::transpose)
      .def("permute_rows", permute_rows, arg("permutation"), rir)
      .def("select_columns", &wt::select_columns, arg("indices"))
      .def(self == self)
      .def(self*vector<T>(0))
      .def(self * typename wt::dense_vector_const_ref())
      .def(typename wt::dense_vector_const_ref() * self)
      .def(self*self)
      .def("self_transpose_times_symmetric_times_self",
           &wt::this_transpose_times_symmetric_times_this)
      .def("self_times_symmetric_times_self_transpose",
           &wt::this_times_symmetric_times_this_transpose)
      .def("self_transpose_times_self", &wt::this_transpose_times_this)
      .def("self_transpose_times_diagonal_times_self",
           &wt::this_transpose_times_diagonal_times_this)
      .def("__str__", str_)
      .def("__repr__", repr)
    ;
  }

};

void wrap_matrix() {
  matrix_wrapper<double>::wrap();
}

}}}


#include "kernel_poly_viscosity.hpp"

#include "kernel_test.hpp"

#include <boost/test/unit_test.hpp>

using namespace GooBalls;
using namespace d2;
using namespace Physics;

namespace utf = boost::unit_test;

using KKernel = PolyViscosity;

BOOST_AUTO_TEST_CASE(poly_viscosity_zero_border_1d, *utf::tolerance(0.0001)){
    KKernel k;
    testZeroBorder1d(k);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_normalization_1d, *utf::tolerance(0.0001)){
    KKernel k;
    testNormalization1d(k, 100000);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_zero_border_gradient_1d, *utf::tolerance(0.0001)){
    KKernel k;
    testZeroBorderGradient1d(k);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_grad_finite_diff_1d, *utf::tolerance(0.001)){
    KKernel k;
    testGradientFiniteDifference1d(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_lap_grad_finite_diff_1d, *utf::tolerance(0.0001)){
    KKernel k;
    testLaplacianFromGradientFiniteDifferences1d(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_lap_finite_diff_1d, *utf::tolerance(0.00001)){
    KKernel k;
    testLaplacianFiniteDifference1d(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_zero_border, *utf::tolerance(0.0001)){
    KKernel k;
    testZeroBorder(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_zero_border_gradient, *utf::tolerance(0.0001)){
    KKernel k;
    testZeroBorderGradient(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_normalization, *utf::tolerance(0.00001)){
    KKernel k;
    testNormalization(k, 100000);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_monotonicity, *utf::tolerance(0.0001)){
    KKernel k;
    testMonotonicity(k, 10);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_radial_symmetric){
    KKernel k;
    testRadialSymmetry(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_grad_finite_diff, *utf::tolerance(0.0005)){
    KKernel k;
    testGradientFiniteDifference(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_lap_grad_finite_diff, *utf::tolerance(0.001)){
    KKernel k;
    testLaplacianFromGradientFiniteDifferences(k, 100);
}

BOOST_AUTO_TEST_CASE(poly_viscosity_lap_finite_diff, *utf::tolerance(0.001)){
    KKernel k;
    testLaplacianFiniteDifference(k, 100);
}


// Copyright (c) 2015-2020 Daniel Cooke
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

#include "mapping_quality_divergence.hpp"

#include <algorithm>
#include <iterator>
#include <cmath>
#include <limits>
#include <numeric>

#include <boost/optional.hpp>
#include <boost/variant.hpp>

#include "basics/aligned_read.hpp"
#include "core/types/allele.hpp"
#include "io/variant/vcf_record.hpp"
#include "utils/maths.hpp"
#include "../facets/samples.hpp"
#include "../facets/alleles.hpp"
#include "../facets/read_assignments.hpp"

namespace octopus { namespace csr {

const std::string MappingQualityDivergence::name_ = "MQD";

std::unique_ptr<Measure> MappingQualityDivergence::do_clone() const
{
    return std::make_unique<MappingQualityDivergence>(*this);
}

Measure::ValueType MappingQualityDivergence::get_value_type() const
{
    return int {};
}

namespace {

using MappingQualityVector = std::vector<AlignedRead::MappingQuality>;

auto extract_mapping_qualities(const ReadRefSupportSet& reads)
{
    MappingQualityVector result(reads.size());
    std::transform(std::cbegin(reads), std::cend(reads), std::begin(result),
                   [](const AlignedRead& read) { return read.mapping_quality(); });
    return result;
}

auto extract_mapping_qualities(const std::vector<Allele>& alleles, const AlleleSupportMap& support, const bool drop_empty = true)
{
    std::vector<MappingQualityVector> result {};
    result.reserve(alleles.size());
    for (const auto& allele : alleles) {
        const auto support_set_itr = support.find(allele);
        if (support_set_itr != std::cend(support) && (!support_set_itr->second.empty() || !drop_empty)) {
            result.push_back(extract_mapping_qualities(support_set_itr->second));
        }
    }
    return result;
}

using MappingQualityHistogram = std::vector<unsigned>;

auto make_histogram(const MappingQualityVector& mapping_qualities, const unsigned prior = 1)
{
    using Q = AlignedRead::MappingQuality;
    static constexpr unsigned max_qualities {std::numeric_limits<Q>::max() - std::numeric_limits<Q>::min()};
    MappingQualityHistogram result(max_qualities, prior);
    for (auto quality : mapping_qualities) {
        ++result[quality];
    }
    return result;
}

auto convert_to_probabilities(const MappingQualityHistogram& counts)
{
    const auto norm = std::accumulate(std::cbegin(counts), std::cend(counts), 0.0);
    std::vector<double> result(counts.size());
    std::transform(std::cbegin(counts), std::cend(counts), std::begin(result),
                   [norm] (const auto c) { return static_cast<double>(c) / norm; });
    return result;
}

auto kl_divergence(const std::vector<double>& p, const std::vector<double>& q) noexcept
{
    return std::inner_product(std::cbegin(p), std::cend(p), std::cbegin(q), 0.0,
                              std::plus<> {}, [] (const auto a, const auto b) {
        return a * std::log(a / b);
    });
}

auto symmetric_kl_divergence(const std::vector<double>& p, const std::vector<double>& q) noexcept
{
    return kl_divergence(p, q) + kl_divergence(q, p);
}

double calculate_max_pairwise_kl_divergence(const std::vector<std::vector<double>>& mq_distributions)
{
    double result {0};
    const auto num_distributions = mq_distributions.size();
    if (num_distributions < 2) return result;
    for (std::size_t i {0}; i < num_distributions - 1; ++i) {
        for (auto j = i + 1; j < num_distributions; ++j) {
            result = std::max(result, symmetric_kl_divergence(mq_distributions[i], mq_distributions[j]));
        }
    }
    return result;
}

double calculate_max_pairwise_kl_divergence(const std::vector<MappingQualityVector>& mapping_qualities)
{
    std::vector<std::vector<double>> mq_distributions {};
    mq_distributions.reserve(mapping_qualities.size());
    for (const auto& mqs : mapping_qualities) {
        mq_distributions.push_back(convert_to_probabilities(make_histogram(mqs)));
    }
    return calculate_max_pairwise_kl_divergence(mq_distributions);
}

auto calculate_median_mapping_qualities(const std::vector<MappingQualityVector>& mapping_qualities)
{
    std::vector<AlignedRead::MappingQuality> result(mapping_qualities.size());
    std::transform(std::cbegin(mapping_qualities), std::cend(mapping_qualities), std::begin(result),
                   [] (const auto& mqs) { return maths::median(mqs); });
    return result;
}

auto calculate_max_pairwise_difference(const std::vector<AlignedRead::MappingQuality>& mapping_qualities)
{
    assert(!mapping_qualities.empty());
    const auto p = std::minmax_element(std::cbegin(mapping_qualities), std::cend(mapping_qualities));
    return *p.second - *p.first;
}

auto max_pairwise_median_mapping_quality_difference(const std::vector<MappingQualityVector>& mapping_qualities)
{
    const auto median_mapping_qualities = calculate_median_mapping_qualities(mapping_qualities);
    return calculate_max_pairwise_difference(median_mapping_qualities);
}

} // namespace

Measure::ResultType MappingQualityDivergence::do_evaluate(const VcfRecord& call, const FacetMap& facets) const
{
    const auto& samples = get_value<Samples>(facets.at("Samples"));
    const auto& alleles = get_value<Alleles>(facets.at("Alleles"));
    const auto& assignments = get_value<ReadAssignments>(facets.at("ReadAssignments")).alleles;
    Array<Optional<ValueType>> result(samples.size());
    for (std::size_t s {0}; s < samples.size(); ++s) {
        const auto& sample = samples[s];
        if (call.is_heterozygous(sample)) {
            const auto mapping_qualities = extract_mapping_qualities(get_called(alleles, call, sample), assignments.at(sample));
            if (!mapping_qualities.empty()) {
                    result[s] = max_pairwise_median_mapping_quality_difference(mapping_qualities);
            }
        }
    }
    return result;
}

Measure::ResultCardinality MappingQualityDivergence::do_cardinality() const noexcept
{
    return ResultCardinality::samples;
}

const std::string& MappingQualityDivergence::do_name() const
{
    return name_;
}

std::string MappingQualityDivergence::do_describe() const
{
    return "Maximum pairwise difference in median mapping qualities of reads supporting each haplotype";
}

std::vector<std::string> MappingQualityDivergence::do_requirements() const
{
    return {"Samples", "Alleles", "ReadAssignments"};
}

} // namespace csr
} // namespace octopus


// MIT License
//
// Copyright (c) 2019 Lennart Braun
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#include "analysis.h"

#include <cmath>
#include <sstream>

#include <fmt/format.h>
#include <boost/accumulators/statistics/variance.hpp>
#include <boost/core/alloc_construct.hpp>
#include <boost/json.hpp>

#include "communication/transport.h"
#include "utility/runtime_info.h"
#include "utility/version.h"

namespace json = boost::json;

namespace MOTION {
namespace Statistics {

static double compute_duration(const RunTimeStats::time_point_pair& tpp) {
  std::chrono::duration<double, AccumulatedRunTimeStats::resolution> d = tpp.second - tpp.first;
  return d.count();
}

void AccumulatedRunTimeStats::add(const RunTimeStats& stats) {
  for (std::size_t i = 0; i <= static_cast<std::size_t>(RunTimeStats::StatID::MAX); ++i) {
    accumulators_[i](compute_duration(stats.data_[i]));
  }
  ++count_;
}

template <typename C>
static typename C::value_type at(const C& container, RunTimeStats::StatID id) {
  return container.at(static_cast<std::size_t>(id));
}

using StatID = RunTimeStats::StatID;

static std::string format_line(std::string name, std::string unit,
                               AccumulatedRunTimeStats::accumulator_type accumulator,
                               std::size_t field_width) {
  std::stringstream ss;
  ss << fmt::format("{:19s} ", name);
  ss << fmt::format("{:{}.3f} {:s} ", boost::accumulators::mean(accumulator), field_width, unit);
  ss << fmt::format("{:{}.3f} {:s} ", boost::accumulators::median(accumulator), field_width, unit);
  // uncorrected standard deviation
  ss << fmt::format("{:{}.3f} {:s}", std::sqrt(boost::accumulators::variance(accumulator)),
                    field_width, unit);
  ss << "\n";
  return ss.str();
}

std::string AccumulatedRunTimeStats::print_human_readable() const {
  std::size_t field_width = 10;
  std::stringstream ss;
  std::string unit = "ms";

  ss << fmt::format("Run time statistics over {} iterations\n", count_)
     << "---------------------------------------------------------------------------\n"
     << fmt::format("                    {:>{}s}    {:>{}s}    {:>{}s}\n", "mean", field_width,
                    "median", field_width, "stddev", field_width)
     << "---------------------------------------------------------------------------\n"
     << format_line("MT Presetup", unit, at(accumulators_, StatID::mt_presetup), field_width)
     << format_line("MT Setup", unit, at(accumulators_, StatID::mt_setup), field_width)
     << format_line("SP Presetup", unit, at(accumulators_, StatID::sp_presetup), field_width)
     << format_line("SP Setup", unit, at(accumulators_, StatID::sp_setup), field_width)
     << format_line("SB Presetup", unit, at(accumulators_, StatID::sb_presetup), field_width)
     << format_line("SB Setup", unit, at(accumulators_, StatID::sb_setup), field_width)
     << format_line("Base OTs", unit, at(accumulators_, StatID::base_ots), field_width)
     << format_line("OT Extension Setup", unit, at(accumulators_, StatID::ot_extension_setup),
                    field_width)
     << "---------------------------------------------------------------------------\n"
     << format_line("Preprocessing Total", unit, at(accumulators_, StatID::preprocessing),
                    field_width)
     << format_line("Gates Setup", unit, at(accumulators_, StatID::gates_setup), field_width)
     << format_line("Gates Online", unit, at(accumulators_, StatID::gates_online), field_width)
     << "---------------------------------------------------------------------------\n"
     << format_line("Circuit Evaluation", unit, at(accumulators_, StatID::evaluate), field_width);

  return ss.str();
}

json::object AccumulatedRunTimeStats::to_json() const {
  const auto mk_triple = [this](const auto& stat_id) {
    const auto& acc = at(accumulators_, stat_id);
    return json::object({{"mean", boost::accumulators::mean(acc)},
                         {"median", boost::accumulators::median(acc)},
                         // uncorrected standard deviation
                         {"stddev", std::sqrt(boost::accumulators::variance(acc))}});
  };
  return {{"repetitions", count_},
          {"mt_setup", mk_triple(StatID::mt_setup)},
          {"sp_setup", mk_triple(StatID::sp_setup)},
          {"sb_setup", mk_triple(StatID::sb_setup)},
          {"linalgtriple_setup", mk_triple(StatID::linalgtriple_setup)},
          {"base_ots", mk_triple(StatID::base_ots)},
          {"ot_extension_setup", mk_triple(StatID::ot_extension_setup)},
          {"preprocessing", mk_triple(StatID::preprocessing)},
          {"gates_setup", mk_triple(StatID::gates_setup)},
          {"gates_online", mk_triple(StatID::gates_online)},
          {"evaluate", mk_triple(StatID::evaluate)}};
}

void AccumulatedCommunicationStats::add(const Communication::TransportStatistics& stats) {
  accumulators_[idx_num_messages_sent](stats.num_messages_sent);
  accumulators_[idx_num_messages_received](stats.num_messages_received);
  accumulators_[idx_num_bytes_sent](stats.num_bytes_sent);
  accumulators_[idx_num_bytes_received](stats.num_bytes_received);
  ++count_;
}

void AccumulatedCommunicationStats::add(
    const std::vector<Communication::TransportStatistics>& stats) {
  for (const auto& s : stats) {
    add(s);
  }
}

std::string AccumulatedCommunicationStats::print_human_readable() const {
  std::stringstream ss;
  ss << "Communication with each other party:\n"
     << fmt::format("Sent: {:0.3f} MiB in {:d} messages\n",
                    boost::accumulators::mean(accumulators_[idx_num_bytes_sent]) / 1048576,
                    static_cast<std::size_t>(
                        boost::accumulators::mean(accumulators_[idx_num_messages_sent])))
     << fmt::format("Received: {:0.3f} MiB in {:d} messages\n",
                    boost::accumulators::mean(accumulators_[idx_num_bytes_received]) / 1048576,
                    static_cast<std::size_t>(
                        boost::accumulators::mean(accumulators_[idx_num_messages_received])));
  return ss.str();
}

json::object AccumulatedCommunicationStats::to_json() const {
  return {
      {"bytes_sent",
       static_cast<std::size_t>(boost::accumulators::mean(accumulators_[idx_num_bytes_sent]))},
      {"num_messages_sent",
       static_cast<std::size_t>(boost::accumulators::mean(accumulators_[idx_num_messages_sent]))},
      {"bytes_received",
       static_cast<std::size_t>(boost::accumulators::mean(accumulators_[idx_num_bytes_received]))},
      {"num_messages_received", static_cast<std::size_t>(boost::accumulators::mean(
                                    accumulators_[idx_num_messages_received]))}};
}

std::string print_motion_info() {
  std::stringstream ss;
  ss << fmt::format("MOTION version: {} @ {}\n", get_git_version(), get_git_branch())
     << fmt::format("invocation: {}\n", get_cmdline())
     << fmt::format("by {}@{}, PID {}\n", get_username(), get_hostname(), get_pid());
  return ss.str();
}

std::string print_stats(const std::string& experiment_name,
                        const AccumulatedRunTimeStats& exec_stats,
                        const AccumulatedCommunicationStats& comm_stats) {
  std::stringstream ss;
  ss << "===========================================================================\n"
     << experiment_name << "\n"
     << "===========================================================================\n"
     << print_motion_info()
     << "===========================================================================\n"
     << exec_stats.print_human_readable()
     << "===========================================================================\n"
     << comm_stats.print_human_readable()
     << "===========================================================================\n";
  return ss.str();
}

json::object to_json(const std::string& experiment_name, const AccumulatedRunTimeStats& exec_stats,
                     const AccumulatedCommunicationStats& comm_stats) {
  json::object obj({{"experiment", experiment_name},
                    {"meta",
                     {{"invocation", get_cmdline()},
                      {"timestamp", get_timestamp()},
                      {"user", get_username()},
                      {"hostname", get_hostname()},
                      {"pid", get_pid()},
                      {"build-type", get_build_type()},
                      {"git-branch", get_git_branch()},
                      {"git-commit", get_git_commit()},
                      {"git-version", get_git_version()}}}});
  obj.emplace("runtime", exec_stats.to_json());
  obj.emplace("communication", comm_stats.to_json());
  return obj;
}

}  // namespace Statistics
}  // namespace MOTION


#pragma once

#include <boost/numeric/mtl/mtl.hpp>

// TODO: find replacement for this explicit typedefs
namespace AMDiS
{
  namespace MTLTypes
  {
    typedef double value_type;
    typedef std::size_t size_type;
    typedef mtl::matrix::parameters<mtl::row_major, mtl::index::c_index, mtl::non_fixed::dimensions, false, size_type> para;
    typedef mtl::matrix::compressed2D<value_type, para> MTLMatrix;
    typedef mtl::vector::dense_vector<value_type> MTLVector;

#if defined(HAVE_PARALLEL_MTL4)
    typedef mtl::matrix::distributed<MTLMatrix> PMTLMatrix;
    typedef mtl::vector::distributed<MTLVector> PMTLVector;
#endif
  }

} // endnamespace AMDiS


#ifndef TMC_UNIT_TESTS
#define BOOST_TEST_MODULE lib_test_layer_partitioner
#endif
#include <boost/test/included/unit_test.hpp>
#include <boost/graph/adjacency_list.hpp>

#include "sample_creator.h"
#include "graph_creator.h"
#include "layer_partitioner.h"

BOOST_AUTO_TEST_SUITE( layer_partitioner )

BOOST_AUTO_TEST_CASE( tmc_9 ) {
    const std::string names_file("data/tmc_paper_9/tmc_paper.names");
    const std::string data_file("data/tmc_paper_9/tmc_paper.data");
    BOOST_TEST_MESSAGE("Creating sample from file: " << data_file);

    std::shared_ptr<FeatureNames> pFN = std::make_shared<FeatureNames>(names_file);

    SampleCreator sample_creator;
    sample_creator.set_feature_names(pFN);

    std::shared_ptr<Sample> pSample = sample_creator.from_file(data_file);

    BOOST_TEST_MESSAGE("Resulting sample: " << *pSample);
    BOOST_CHECK_EQUAL(pSample->dim(), 2);
    BOOST_CHECK_EQUAL(pSample->size(), 4);

    typedef boost::adjacency_list_traits<boost::vecS, boost::vecS, boost::directedS> Traits;
    typedef boost::adjacency_list<boost::listS, boost::vecS, boost::directedS,
            boost::no_property,
            boost::property<boost::edge_capacity_t, double,
                    boost::property<boost::edge_residual_capacity_t, double,
                            boost::property<boost::edge_reverse_t, Traits::edge_descriptor> > > > SampleGraphType;
    typedef boost::adjacency_list<boost::setS, boost::vecS, boost::directedS> AuxTrGraphType;

    auto pGraphCreator = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pSample);

    BOOST_CHECK_EQUAL(pSample->size(), pGraphCreator->size());
    //BOOST_CHECK_EQUAL(pGraphCreator->num_edges(), 5);
    BOOST_TEST_MESSAGE("Full induced graph:");
    pGraphCreator->print();
    //pGraphCreator->do_transitive_reduction();
    BOOST_CHECK_EQUAL(pGraphCreator->num_edges(), 4);
    BOOST_TEST_MESSAGE("TR-Reduced graph:");
    pGraphCreator->print();

    auto pLayerPartitioner = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner->set_layer(pSample);
    pLayerPartitioner->set_graph(pGraphCreator->get_graph());

    boost::dynamic_bitset<> mask;
    bool decomposable;
    std::tie(mask, decomposable) = pLayerPartitioner->compute();
    BOOST_CHECK(decomposable);
    BOOST_CHECK_EQUAL(mask, boost::dynamic_bitset<>(std::string("1010")));
    BOOST_CHECK_EQUAL(pGraphCreator->num_edges(), 4);

}

BOOST_AUTO_TEST_CASE( thirtysix_points ) {
    const std::string names_file("data/4layers_36points/4layers_36points.names");
    const std::string data_file("data/4layers_36points/4layers_36points.data");
    BOOST_TEST_MESSAGE("Creating sample from file: " << data_file);

    std::shared_ptr<FeatureNames> pFN = std::make_shared<FeatureNames>(names_file);

    SampleCreator sample_creator;
    sample_creator.set_feature_names(pFN);

    std::shared_ptr<Sample> pSample = sample_creator.from_file(data_file);

    BOOST_TEST_MESSAGE("Resulting sample: " << *pSample);
    BOOST_CHECK_EQUAL(pSample->dim(), 2);
    BOOST_CHECK_EQUAL(pSample->size(), 36);

    typedef boost::adjacency_list_traits<boost::vecS, boost::vecS, boost::directedS> Traits;
    typedef boost::adjacency_list<boost::listS, boost::vecS, boost::directedS,
            boost::property<boost::vertex_name_t, std::string>,
            boost::property<boost::edge_capacity_t, double,
                    boost::property<boost::edge_residual_capacity_t, double,
                            boost::property<boost::edge_reverse_t, Traits::edge_descriptor> > > > SampleGraphType;
    typedef boost::adjacency_list<boost::setS, boost::vecS, boost::directedS> AuxTrGraphType;

    auto pGraphCreator = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pSample);

    BOOST_CHECK_EQUAL(pSample->size(), pGraphCreator->size());
    //BOOST_CHECK_EQUAL(pGraphCreator->num_edges(), 197);
    BOOST_TEST_MESSAGE("Full induced graph:");
    pGraphCreator->print();
    //pGraphCreator->do_transitive_reduction();
    BOOST_CHECK_EQUAL(pGraphCreator->num_edges(), 69);
    BOOST_TEST_MESSAGE("TR-Reduced graph:");
    pGraphCreator->print();

    auto pLayerPartitioner = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner->set_layer(pSample);
    pLayerPartitioner->set_graph(pGraphCreator->get_graph());

    boost::dynamic_bitset<> mask;
    bool decomposable;
    std::tie(mask, decomposable) = pLayerPartitioner->compute();
    BOOST_CHECK(decomposable);
    BOOST_CHECK_EQUAL(mask, boost::dynamic_bitset<>(std::string("111111111111111110000000000000000000")));
    BOOST_CHECK_EQUAL(pGraphCreator->num_edges(), 69);

    std::shared_ptr<Layer> pLayer0;
    std::shared_ptr<Layer> pLayer1;
    std::tie(pLayer0,pLayer1) = sample_creator.split_sample(pSample, mask);
    BOOST_CHECK_EQUAL(pLayer0->size(), 19);
    BOOST_CHECK_EQUAL(pLayer1->size(), 17);

    auto pGraphCreator0 = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pLayer0);
    BOOST_CHECK_EQUAL(pLayer0->size(), pGraphCreator0->size());
    //pGraphCreator0->do_transitive_reduction();
    auto pLayerPartitioner0 = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner0->set_layer(pLayer0);
    pLayerPartitioner0->set_graph(pGraphCreator0->get_graph());
    boost::dynamic_bitset<> mask0;
    bool decomposable0;
    std::tie(mask0, decomposable0) = pLayerPartitioner0->compute();
    BOOST_CHECK(decomposable0);
    BOOST_CHECK_EQUAL(mask0, boost::dynamic_bitset<>(std::string("1111111000000000000")));
    std::shared_ptr<Layer> pLayer00;
    std::shared_ptr<Layer> pLayer01;
    std::tie(pLayer00,pLayer01) = sample_creator.split_sample(pLayer0, mask0);

    auto pGraphCreator1 = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pLayer1);
    BOOST_CHECK_EQUAL(pLayer1->size(), pGraphCreator1->size());
    //pGraphCreator1->do_transitive_reduction();
    auto pLayerPartitioner1 = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner1->set_layer(pLayer1);
    pLayerPartitioner1->set_graph(pGraphCreator1->get_graph());
    boost::dynamic_bitset<> mask1;
    bool decomposable1;
    std::tie(mask1, decomposable1) = pLayerPartitioner1->compute();
    BOOST_CHECK(decomposable1);
    BOOST_CHECK_EQUAL(mask1, boost::dynamic_bitset<>(std::string("11111111100000000")));
    std::shared_ptr<Layer> pLayer10;
    std::shared_ptr<Layer> pLayer11;
    std::tie(pLayer10,pLayer11) = sample_creator.split_sample(pLayer1, mask1);

    auto pGraphCreator00 = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pLayer00);
    BOOST_CHECK_EQUAL(pLayer00->size(), pGraphCreator00->size());
    //pGraphCreator00->do_transitive_reduction();
    auto pLayerPartitioner00 = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner00->set_layer(pLayer00);
    pLayerPartitioner00->set_graph(pGraphCreator00->get_graph());
    boost::dynamic_bitset<> mask00;
    bool decomposable00;
    std::tie(mask00, decomposable00) = pLayerPartitioner00->compute();
    BOOST_CHECK(!decomposable00);
    BOOST_CHECK_EQUAL(mask00, boost::dynamic_bitset<>(std::string("000000000000")));

    auto pGraphCreator01 = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pLayer01);
    BOOST_CHECK_EQUAL(pLayer01->size(), pGraphCreator01->size());
    //pGraphCreator01->do_transitive_reduction();
    auto pLayerPartitioner01 = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner01->set_layer(pLayer01);
    pLayerPartitioner01->set_graph(pGraphCreator01->get_graph());
    boost::dynamic_bitset<> mask01;
    bool decomposable01;
    std::tie(mask01, decomposable01) = pLayerPartitioner01->compute();
    BOOST_CHECK(!decomposable01);
    BOOST_CHECK_EQUAL(mask01, boost::dynamic_bitset<>(std::string("0000000")));

    auto pGraphCreator10 = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pLayer10);
    BOOST_CHECK_EQUAL(pLayer10->size(), pGraphCreator10->size());
    //pGraphCreator10->do_transitive_reduction();
    auto pLayerPartitioner10 = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner10->set_layer(pLayer10);
    pLayerPartitioner10->set_graph(pGraphCreator10->get_graph());
    boost::dynamic_bitset<> mask10;
    bool decomposable10;
    std::tie(mask10, decomposable10) = pLayerPartitioner10->compute();
    BOOST_CHECK(!decomposable10);
    BOOST_CHECK_EQUAL(mask10, boost::dynamic_bitset<>(std::string("00000000")));

    auto pGraphCreator11 = std::make_shared<GraphCreator<SampleGraphType, AuxTrGraphType> >(pLayer11);
    BOOST_CHECK_EQUAL(pLayer11->size(), pGraphCreator11->size());
    //pGraphCreator11->do_transitive_reduction();
    auto pLayerPartitioner11 = std::make_shared<LayerPartitioner<SampleGraphType> >();
    pLayerPartitioner11->set_layer(pLayer11);
    pLayerPartitioner11->set_graph(pGraphCreator11->get_graph());
    boost::dynamic_bitset<> mask11;
    bool decomposable11;
    std::tie(mask11, decomposable11) = pLayerPartitioner11->compute();
    BOOST_CHECK(!decomposable11);
    BOOST_CHECK_EQUAL(mask11, boost::dynamic_bitset<>(std::string("000000000")));
}

BOOST_AUTO_TEST_SUITE_END()


// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
//
// gDLS*: Generalized Pose-and-Scale Estimation Given Scale and Gravity Priors
//
// Victor Fragoso, Joseph DeGol, Gang Hua.
// Proc. of the IEEE/CVF Conf. on Computer Vision and Pattern Recognition 2020.
//
// Please contact the author of this library if you have any questions.
// Author: Victor Fragoso (victor.fragoso@microsoft.com)

#include "gdls_star/util.h"

#include <vector>
#include <Eigen/Core>
#include "gdls_star/camera_feature_correspondence_2d_3d.h"
#include "gdls_star/gdls_star.h"

namespace msft {

GdlsStar::Input ComputeInputDatum(
    const std::vector<CameraFeatureCorrespondence2D3D>& correspondences) {
  GdlsStar::Input input;
  input.ray_origins.resize(correspondences.size());
  input.ray_directions.resize(correspondences.size());
  input.world_points.resize(correspondences.size());
  for (int i = 0; i < correspondences.size(); ++i) {
    // Keep 3D point.
    input.world_points[i] = correspondences[i].point;
    // Compute ray direction.
    const Eigen::Vector2d& pixel = correspondences[i].observation;
    input.ray_directions[i] = correspondences[i].camera.PixelToUnitRay(pixel);
    // Compute ray origins.
    input.ray_origins[i] = correspondences[i].camera.GetPosition();
  }
  return input;
}

}  // namespace msft


//
// Created by Alex Beccaro on 25/02/18.
//

#define BOOST_TEST_DYN_LINK
#include <boost/test/unit_test.hpp>
#include "../../src/problems/51-100/84/problem84.hpp"

BOOST_AUTO_TEST_SUITE( Problem84 )

    BOOST_AUTO_TEST_CASE( Example ) {
        auto res = problems::problem84::solve(6);
        BOOST_CHECK_EQUAL(res, "102400");
    }

    BOOST_AUTO_TEST_CASE( Solution ) {
        auto res = problems::problem84::solve();
        BOOST_CHECK_EQUAL(res, "101524");
    }

BOOST_AUTO_TEST_SUITE_END()


//==============================================================================
//         Copyright 2003 - 2012   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2012   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef BOOST_SIMD_ARITHMETIC_FUNCTIONS_ICEIL_HPP_INCLUDED
#define BOOST_SIMD_ARITHMETIC_FUNCTIONS_ICEIL_HPP_INCLUDED
#include <boost/simd/include/functor.hpp>
#include <boost/dispatch/include/functor.hpp>

namespace boost { namespace simd { namespace tag
  {
    /*!
      @brief  iceil generic tag

      Represents the iceil function in generic contexts.

      @par Models:
      Hierarchy
    **/
    struct iceil_ : ext::elementwise_<iceil_>
    {
      /// @brief Parent hierarchy
      typedef ext::elementwise_<iceil_> parent;
      template<class... Args>
      static BOOST_FORCEINLINE BOOST_AUTO_DECLTYPE dispatch(Args&&... args)
      BOOST_AUTO_DECLTYPE_BODY( dispatching_iceil_( ext::adl_helper(), static_cast<Args&&>(args)... ) )
    };
  }
  namespace ext
  {
    template<class Site, class... Ts>
    BOOST_FORCEINLINE generic_dispatcher<tag::iceil_, Site> dispatching_iceil_(adl_helper, boost::dispatch::meta::unknown_<Site>, boost::dispatch::meta::unknown_<Ts>...)
    {
      return generic_dispatcher<tag::iceil_, Site>();
    }
    template<class... Args>
    struct impl_iceil_;
  }
  /*!
    Computes the integer conversion of the ceil of its parameter.

    @par semantic:
    For any given value @c x of type @c T:

    @code
    as_integer<T> r = iceil(x);
    @endcode

    is equivalent to:

    @code
    as_integer<T> r = toints(ceil(x));
    @endcode

    @par Note:
    This operation is properly saturated

    @see funcref{fast_iceil}
    @param  a0

    @return an integral value of the integral type associated to the input.

  **/
  BOOST_DISPATCH_FUNCTION_IMPLEMENTATION(tag::iceil_, iceil, 1)
} }

#endif


#include <mi4/PriorityQueue.hpp>
#include <iostream>
#include <Eigen/Dense>
int main ( int argc,  char** argv )
{
        mi4::PriorityQueue< Eigen::Vector2d > pq;
        pq.push(Eigen::Vector2d(10, 10), 3);
        pq.push(Eigen::Vector2d(30, 10), 2);
        pq.push(Eigen::Vector2d(10, 10), 0);
        pq.push(Eigen::Vector2d(40, 10), 1);
        pq.push(Eigen::Vector2d(10, 30), 4);

        while ( !pq.empty() ) {
                std::cerr << "cost" << pq.getTopCost() << std::endl;
                pq.pop();
        }

        return 0;
}


//          Copyright Tunc Bahcecioglu 2012 - 2013.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <iostream>
#include <array>

#include <boost/test/unit_test.hpp>
#include "StaticEnvironment.h"

#include <shiva\collective.h>
#include <shiva\distribution.h>

BOOST_AUTO_TEST_SUITE(ScatterTest)

BOOST_AUTO_TEST_CASE(Scatter)
{
	shiva::environment& env = StaticEnvironment::mpi_environment;
	shiva::communicator comm;
	int mpi_size = comm.size();
	int mpi_rank = comm.rank();

	std::vector<double> recv_data(2);

	if (mpi_rank == 0) {
		std::vector<double> send_data(mpi_size * 2);
		for (int i = 0; i < mpi_size; ++i) {
			send_data[i*2]     = (double) (i * 2);
			send_data[i*2 + 1] = (double) (i * 2 + 1);
		}
		shiva::scatter(0, comm, shiva::make_message(send_data), shiva::make_message(recv_data) );
	} else {
		shiva::scatter(0, comm, shiva::make_message(recv_data) );
	}

	BOOST_CHECK_EQUAL(recv_data[0], (double)(mpi_rank * 2));
	BOOST_CHECK_EQUAL(recv_data[1], (double)(mpi_rank * 2 + 1));
}

BOOST_AUTO_TEST_CASE(Scatterv)
{
	shiva::environment& env = StaticEnvironment::mpi_environment;
	shiva::communicator comm;

	int mpi_size = comm.size();
	int mpi_rank = comm.rank();

	const int n = 14;
	auto dist = shiva::make_equal_distribution(comm, n);
	const int local_size = std::get<0>(dist)[mpi_rank];
	std::vector<int> recv_data(local_size);

	if (mpi_rank == 0) {
		std::vector<int> send_data((size_t)n);
		for (int i = 0; i < n; ++i) {
			send_data[(size_t)i] = i;
		}
		shiva::scatterv(0, comm, shiva::make_message(send_data), dist, shiva::make_message(recv_data) );
	} else {
		shiva::scatterv(0, comm, dist, shiva::make_message(recv_data.data(), local_size) );
	}

	const int local_disp = std::get<1>(dist)[mpi_rank];
	for (int i = 0; i < local_size; ++i) {
		BOOST_CHECK_EQUAL(recv_data[i], local_disp + i);
	}
}

BOOST_AUTO_TEST_SUITE_END()


// Copyright (c) 2020 Kent Hu

/// @author kent hu

#include <algorithm>
#include <cmath>
#include <cstdlib>
#include <exception>
#include <deque>
#include <functional>
#include <iostream>
#include <limits>
#include <random>
#include <string>
#include <thread>
#include <utility>
#include <vector>

#include <boost/lexical_cast.hpp>
#include <boost/optional/optional.hpp>
#include <comma/application/command_line_options.h>
#include <comma/base/exception.h>
#include <comma/base/types.h>
#include <comma/csv/ascii.h>
#include <comma/csv/options.h>
#include <comma/csv/stream.h>
#include <comma/string/split.h>
#include <tbb/blocked_range.h>

#define TBB_PREVIEW_GLOBAL_CONTROL 1 // required to #include <tbb/global_control.h>

#include <tbb/global_control.h>
#include <tbb/parallel_for.h>
#include <tbb/parallel_reduce.h>
#include "../../visiting/traits.h"

#ifdef SNARK_USE_CUDA

#include "math-k-means/device.h"
#include "math-k-means/math_k_means.h"

#endif

void usage( const bool verbose )
{
    std::cerr << std::endl;
    std::cerr << "run k-means on input data" << std::endl;
    std::cerr << std::endl;
    std::cerr << "usage: cat sample.csv | math-k-means [<options>]" << std::endl;
    std::cerr << std::endl;
    std::cerr << "fields" << std::endl;
    std::cerr << "    block: block number; output k-means centroid id for each" << std::endl;
    std::cerr << "           contiguous block of samples with the same block id" << std::endl;
    std::cerr << "    data: vector of size <size>" << std::endl;
    std::cerr << "    default: data" << std::endl;
    std::cerr << "output" << std::endl;
    std::cerr << "    appended fields: centroid/id,centroid/data" << std::endl;
    std::cerr << "    centroids only: if block field present: block,centroid/id,centroid/data" << std::endl;
    std::cerr << "                    if no block field present: centroid/id,centroid/data" << std::endl;
    std::cerr << "    binary format: 64-bit floating-point for centroid/data, 32-bit unsigned integer for centroid/id" << std::endl;
    std::cerr << "options" << std::endl;
#ifdef SNARK_USE_CUDA
    std::cerr << "    --cuda: use gpu; ATTENTION: only 32-bit float precision currently implemented; if you need 64-bit float precision, run without --cuda" << std::endl;
    std::cerr << "    --cuda-use-pitched,--use-pitched,--pitched: use pitched memory for 2d array (padded 2d array for memory coalescing; faster but uses more memory)" << std::endl;
#endif
    std::cerr << "    --help,-h: show this help; --help --verbose: more help" << std::endl;
    std::cerr << "    --ignore-tolerance: ignore tolerance value used for early exit" << std::endl;
    std::cerr << "    --max-iterations,--iterations=<n>: number of iterations for Lloyd's algorithm; default: 300" << std::endl;
    std::cerr << "    --max-threads,--threads=<n>: maximum number of threads to run, if 0, set to number of cores in system (" << std::thread::hardware_concurrency() << ')' << "; default: 0" << std::endl;
    std::cerr << "    --number-of-clusters,--clusters=<n>: number of k-means cluster per block/all" << std::endl;
    std::cerr << "    --number-of-runs,--runs=<n>: number of times to run k-means, best run is one with lowest inertia (sum of distance of cluster points to cluster centroid); default: 10" << std::endl;
    std::cerr << "    --output-centroids,--centroids: output centroids only" << std::endl;
    std::cerr << "    --seed=[<seed>]: use seed for random initialization of centroids" << std::endl;
    std::cerr << "    --size=[<n>]: a hint of number of elements in the data vector; ignored, if data indices specified, e.g. data[0],data[1],data[2]" << std::endl;
    std::cerr << "    --tolerance=<distance>: difference between two consecutive iteration centroid l2 norms to declare convergence and stop iterating in LLoyd's algorithm; default: 1.0e-4" << std::endl;
    if( verbose ) { std::cerr << std::endl << "csv options" << std::endl << comma::csv::options::usage() << std::endl; }
    std::cerr << std::endl;
    std::cerr << "examples" << std::endl;
    std::cerr << "    cat sample.csv | math-k-means --number-of-clusters=3" << std::endl;
    std::cerr << "    cat sample.csv | math-k-means --number-of-clusters=3 --fields=data --size=3" << std::endl;
    std::cerr << "    cat sample.csv | math-k-means --number-of-clusters=3 --fields=,,,data[0],,,,data[1],,data[2]" << std::endl;
    std::cerr << "    cat sample.bin | math-k-means --number-of-clusters=3 --fields=block,data --size=3 --binary=ui,3d" << std::endl;
    std::cerr << std::endl;
    exit( 0 );
}

static comma::csv::options csv;
static boost::optional< unsigned int > size = boost::none;
static bool output_centroids = false;
static boost::optional< std::mt19937::result_type > seed = boost::none;
static bool use_block = false;
#ifdef SNARK_USE_CUDA
static bool use_pitched = false;
#endif
static bool verbose = false;

namespace snark {

#ifdef SNARK_USE_CUDA
namespace cuda { namespace k_means {

struct input_t
{
    std::vector< float > data;
    comma::uint32 block = 0;

    input_t() : data( *size ), block( 0 ) {}
};

class k_means
{
public:
    k_means( float tolerance, unsigned int max_iterations, unsigned int number_of_runs,
             comma::uint32 number_of_clusters, unsigned int size ) noexcept : tolerance_( tolerance ),
                                                                              max_iterations_( max_iterations ),
                                                                              number_of_runs_( number_of_runs ),
                                                                              number_of_clusters_( number_of_clusters ),
                                                                              ncols_( size ) {}

    template < template < typename > class Matrix >
    std::tuple< std::vector< float >, std::vector< comma::uint32 > >
    run_on_block( const std::vector< float >& h_matrix ) const
    {
        const size_t nrows = h_matrix.size() / ncols_;

        // device arrays/matrices
        Matrix< float > d_matrix( nrows, ncols_, h_matrix );
        Matrix< float > d_means( number_of_clusters_, ncols_ );
        Matrix< float > d_sums( number_of_clusters_, ncols_ );
        CUDA_CHECK_ERRORS( d_sums.fill( 0.0f ) )

        device::array< comma::uint32 > d_assignments( nrows );
        device::array< unsigned int > d_counts( number_of_clusters_ );
        CUDA_CHECK_ERRORS( d_counts.fill( 0 ) )

        // host vectors
        std::vector< float > h_means( number_of_clusters_* ncols_ );
        std::vector< comma::uint32 > h_assignments( nrows );

        // save runs to choose best centroid
        std::vector< float > all_scores;
        all_scores.reserve( number_of_runs_ );
        std::vector< decltype( h_means ) > all_centroids;
        all_centroids.reserve( number_of_runs_ );
        std::vector< decltype( h_assignments ) > all_centroid_assignments;
        all_centroid_assignments.reserve( number_of_runs_ );

        cudaDeviceProp prop{};
        CUDA_CHECK_ERRORS( cudaGetDeviceProperties( &prop, 0 ) )
        for( unsigned int run = 0; run < number_of_runs_; ++run )
        {
            CUDA_CHECK_ERRORS( d_means.to_device( initialize_centroids( h_matrix ) ) )
            float difference = std::numeric_limits< float >::max();
            for( unsigned int iteration = 0; iteration < max_iterations_; ++iteration )
            {
                assign_centroids( d_matrix, d_means, d_assignments, d_sums, d_counts, number_of_clusters_, prop.maxThreadsPerBlock );
                update_centroids( d_means, d_sums, d_counts, number_of_clusters_ );
                if( tolerance_ < 0 ) { continue; }
                std::vector< float > new_means( number_of_clusters_* ncols_ ); // temp buffer for calculating tolerance difference
                CUDA_CHECK_ERRORS( d_means.to_host( new_means ) )
                difference = tbb::parallel_reduce( tbb::blocked_range< comma::uint32 >( 0, number_of_clusters_ ), 0.0f,
                                                   [&]( const tbb::blocked_range< comma::uint32 > chunk, float difference ) -> float
                                                   {
                                                       for( comma::uint32 i = chunk.begin(); i < chunk.end(); ++i )
                                                       {
                                                           difference += squared_euclidean_distance( ncols_, &h_means[i * ncols_], &new_means[i * ncols_] );
                                                       }
                                                       return difference;
                                                   },
                                                   std::plus< float >()
                );
                h_means = std::move( new_means );
                if( difference < tolerance_ * tolerance_ ) { break; }
            }
            CUDA_CHECK_ERRORS( d_assignments.to_host( h_assignments ) )
            if( tolerance_ < 0 || difference > tolerance_ * tolerance_ ) { CUDA_CHECK_ERRORS( d_means.to_host( h_means ) ) }
            const float score = tbb::parallel_reduce( tbb::blocked_range< size_t >( 0, nrows ), 0.0f,
                                                      [&]( const tbb::blocked_range< size_t > chunk, float score ) -> float
                                                      {
                                                          for( size_t point = chunk.begin(); point < chunk.end(); ++point )
                                                          {
                                                              const auto cluster_assignment = h_assignments[point];
                                                              score += std::sqrt( squared_euclidean_distance( ncols_, &h_matrix[point * ncols_], &h_means[cluster_assignment * ncols_] ) );
                                                          }
                                                          return score;
                                                      },
                                                      std::plus< float >()
            );
            all_scores.emplace_back( score );
            all_centroids.emplace_back( h_means );
            all_centroid_assignments.emplace_back( h_assignments );
        }
        const auto& best_index = std::distance( all_scores.begin(), std::min_element( all_scores.begin(), all_scores.end() ) );
        return std::make_tuple( all_centroids[best_index], all_centroid_assignments[best_index] );
    }

private:
    std::vector< float > initialize_centroids( const std::vector< float >& h_matrix ) const
    {
        static std::mt19937 generator( seed.is_initialized() ? *seed : std::random_device{}() );
        static std::vector< float > init_centroids( number_of_clusters_* ncols_ );
        static std::uniform_int_distribution< size_t > indices( 0, h_matrix.size() / ncols_ - 1 );
        for( size_t cluster = 0; cluster < number_of_clusters_; ++cluster )
        {
            for( size_t col = 0; col < ncols_; ++col )
            {
                init_centroids[cluster * ncols_ + col] = h_matrix[indices( generator ) * ncols_ + col];
            }
        }
        return init_centroids;
    }

    const float tolerance_;
    const unsigned int max_iterations_;
    const unsigned int number_of_runs_;
    const unsigned int number_of_clusters_;
    const unsigned int ncols_;
};

} } // namespace cuda { namespace k_means {

#endif

namespace k_means {

static double square( const double value ) { return value * value; }

static double squared_euclidean_distance( const std::vector< double >& first, const std::vector< double >& second )
{
    double ret = 0.0;
    for( size_t i = 0; i < first.size(); ++i ) { ret += square( first[i] - second[i] ); }
    return ret;
}

struct input_t
{
    std::vector< double > data;
    comma::uint32 block;

    input_t() : data( *size ), block( 0 ) {}
};

struct k_means
{
    const double tolerance;
    const unsigned int max_iterations;
    const unsigned int number_of_runs;
    const comma::uint32 number_of_clusters;

    k_means( double tolerance, unsigned int max_iterations, unsigned int number_of_runs, comma::uint32 number_of_clusters ) noexcept :
            tolerance( tolerance ),
            max_iterations( max_iterations ),
            number_of_runs( number_of_runs ),
            number_of_clusters( number_of_clusters ) {}

    std::vector< std::vector< double > > initialize_centroids( const std::deque< std::vector< double > >& dataframe ) const
    {
        static std::mt19937 generator( seed.is_initialized() ? *seed : std::random_device{}() );
        std::vector< std::vector< double > > centroids( number_of_clusters );
        std::uniform_int_distribution< size_t > indices( 0, dataframe.size() - 1 );
        std::generate( centroids.begin(), centroids.end(), [&]() { return dataframe[indices( generator )]; } );
        return centroids;
    }

    std::vector< comma::uint32 >
    assign_centroids( const std::vector< std::vector< double > >& centroids, const std::deque< std::vector< double > >& dataframe ) const
    {
        std::vector< comma::uint32 > centroid_assignments( dataframe.size() );
        tbb::parallel_for( tbb::blocked_range< size_t >( 0, centroid_assignments.size() ), [&]( const tbb::blocked_range< size_t >& chunk )
        {
            for( size_t point = chunk.begin(); point < chunk.end(); ++point )
            {
                auto best_distance = std::numeric_limits< double >::max();
                comma::uint32 best_centroid = 0;
                for( comma::uint32 centroid_i = 0; centroid_i < number_of_clusters; ++centroid_i )
                {
                    const double distance = squared_euclidean_distance( dataframe[point], centroids[centroid_i] );
                    if( distance < best_distance ) { best_distance = distance; best_centroid = centroid_i; }
                }
                centroid_assignments[point] = best_centroid;
            }
        } );
        return centroid_assignments;
    }

    std::vector< std::vector< double > >
    update_centroids( const std::vector< comma::uint32 >& centroid_assignments, const std::deque< std::vector< double > >& dataframe ) const
    {
        // for each cluster, calculate sum of data vectors and number of data vectors
        std::vector< size_t > size_of_centroid( number_of_clusters );
        std::vector< std::vector< double > > sum_of_points_in_centroid( number_of_clusters, std::vector< double >( *size, 0.0 ) );
        for( size_t point = 0; point < dataframe.size(); ++point )
        {
            const auto centroid_i = centroid_assignments[point];
            for( size_t point_i = 0;
                 point_i < dataframe[point].size(); ++point_i ) { sum_of_points_in_centroid[centroid_i][point_i] += dataframe[point][point_i]; }
            ++size_of_centroid[centroid_i];
        }
        // calculate new centroid means
        std::vector< std::vector< double > > new_centroids( number_of_clusters );
        tbb::parallel_for( tbb::blocked_range< comma::uint32 >( 0, number_of_clusters ), [&]( const tbb::blocked_range< comma::uint32 >& chunk )
        {
            for( comma::uint32 centroid_i = chunk.begin(); centroid_i < chunk.end(); ++centroid_i )
            {
                new_centroids[centroid_i].reserve( *size );
                const auto centroid_size = std::max< size_t >( 1, size_of_centroid[centroid_i] ); // minimum size is at least 1
                for( size_t point_i = 0; point_i < sum_of_points_in_centroid[centroid_i].size(); ++point_i )
                {
                    new_centroids[centroid_i].emplace_back( sum_of_points_in_centroid[centroid_i][point_i] / centroid_size );
                }
            }
        } );
        return new_centroids;
    }

    // refer to http://www.goldsborough.me/c++/python/cuda/2017/09/10/20-32-46-exploring_k-means_in_python,_c++_and_cuda/
    std::tuple< std::vector< std::vector< double > >, std::vector< comma::uint32 > > run_on_block( const std::deque< std::vector< double > >& dataframe ) const
    {
        std::vector< double > all_scores( number_of_runs );
        std::vector< std::vector< std::vector< double > > > all_centroids( number_of_runs );
        std::vector< std::vector< comma::uint32 > > all_centroid_assignments( number_of_runs );
        tbb::parallel_for( tbb::blocked_range< unsigned int >( 0, number_of_runs ), [&]( const tbb::blocked_range< unsigned int >& chunk )
        {
            for( unsigned int run = chunk.begin(); run < chunk.end(); ++run )
            {
                std::vector< comma::uint32 > centroid_assignments;
                std::vector< std::vector< double > > run_centroids = initialize_centroids( dataframe );
                for( unsigned int iteration = 0; iteration < max_iterations; ++iteration )
                {
                    centroid_assignments = assign_centroids( run_centroids, dataframe );
                    std::vector< std::vector< double > > new_centroids = update_centroids( centroid_assignments, dataframe );
                    if( tolerance < 0 ) { run_centroids = std::move( new_centroids ); continue; }
                    double summed_difference = tbb::parallel_reduce( tbb::blocked_range< comma::uint32 >( 0, number_of_clusters ), 0.0,
                            [&]( const tbb::blocked_range< comma::uint32 > chunk, double difference ) -> double
                            {
                                for( comma::uint32 centroid_i = chunk.begin(); centroid_i < chunk.end(); ++centroid_i )
                                {
                                    difference += squared_euclidean_distance( run_centroids[centroid_i], new_centroids[centroid_i] );
                                }
                                return difference;
                            },
                            std::plus< double >()
                    );
                    run_centroids = std::move( new_centroids );
                    if( summed_difference < tolerance * tolerance ) { break; }
                }
                double run_score = tbb::parallel_reduce( tbb::blocked_range< size_t >( 0, dataframe.size() ), 0.0,
                                                         [&]( const tbb::blocked_range< size_t > chunk, double score ) -> double
                                                         {
                                                             for( size_t point = chunk.begin(); point < chunk.end(); ++point )
                                                             {
                                                                 score += std::sqrt( squared_euclidean_distance( dataframe[point], run_centroids[centroid_assignments[point]] ) );
                                                             }
                                                             return score;
                                                         },
                                                         std::plus< double >()
                );
                all_scores[run] = run_score;
                all_centroids[run] = std::move( run_centroids );
                all_centroid_assignments[run] = std::move( centroid_assignments );
            }
        } );
        const std::vector< double >::iterator& min_element = std::min_element( all_scores.begin(), all_scores.end() );
        const auto best_index = std::distance( all_scores.begin(), min_element );
        return std::make_pair( all_centroids[best_index], all_centroid_assignments[best_index] );
    }
};

} // namespace k_means {

} // namespace snark {

static std::string get_size()
{
    std::string first;
    const std::vector< std::string >& fields = comma::split( csv.fields, ',' );
    if( csv.has_field( "data" ) )
    {
        unsigned int count;
        if( csv.binary() )
        {
            count = csv.format().count();
        }
        else
        {
            while( std::cin.good() && first.empty() ) { std::getline( std::cin, first ); }
            count = comma::split( first, csv.delimiter ).size(); // quick and dirty, wasteful
        }
        size = count - fields.size() + 1;
    }
    else
    {
        unsigned int max = 0;
        for( const auto& field : fields )
        {
            if( field.substr( 0, 5 ) == "data[" && *field.rbegin() == ']' )
            {
                const auto k = boost::lexical_cast< unsigned int >( field.substr( 5, field.size() - 6 ) ) + 1;
                if( k > max ) { max = k; }
            }
        }
        if( max == 0 ) { COMMA_THROW( comma::exception, "please specify valid data fields" ) }
        size = max;
    }
    return first;
}

#ifdef SNARK_USE_CUDA

static int run_cuda_( const float tolerance, const unsigned int max_iterations, const unsigned int number_of_runs, const comma::uint32 number_of_clusters )
{
    using namespace snark::cuda::k_means;
    comma::uint32 block = 0;
    std::vector< float > dataframe;
    std::vector< std::string > input_lines;
    auto set_input_values_ = [&]( const input_t& p, const std::string& line )
    {
        block = p.block;
        input_lines.emplace_back( line );
        dataframe.insert( end( dataframe ), begin( p.data ), end( p.data ) );
    };
    if( !size )
    {
        const auto& first_line = get_size();
        if( !first_line.empty() ) { set_input_values_( comma::csv::ascii< input_t >( csv ).get( first_line ), first_line ); }
    }
    auto write_centroids_only_ = [number_of_clusters]( const std::vector< float >& centroids, const comma::uint32 block )
    {
        for( comma::uint32 i = 0; i < number_of_clusters; ++i )
        {
            if( csv.binary() )
            {
                if( use_block ) { std::cout.write( reinterpret_cast< const char* >( &block ), sizeof( block ) ); }
                std::cout.write( reinterpret_cast< const char* >( &i ), sizeof( i ) );
                for( size_t col = 0; col < *size; ++col )
                {
                    const auto point = static_cast< double >( centroids[i * *size + col] );
                    std::cout.write( reinterpret_cast< const char* >( &point ), sizeof( point ) );
                }
            }
            else
            {
                if( use_block ) { std::cout << block << csv.delimiter; }
                std::cout << i;
                for( size_t col = 0; col < *size; ++col ) { std::cout << csv.delimiter << static_cast< double >( centroids[i * *size + col] ); }
                std::cout << std::endl;
            }
        }
    };
    auto write_lines_ = []( const std::vector< std::string >& input_lines, const std::vector< float >& centroids,
                            const std::vector< comma::uint32 >& centroid_assignments )
    {
        for( size_t i = 0; i < centroid_assignments.size(); ++i )
        {
            const comma::uint32 centroid_assignment = centroid_assignments[i];
            std::cout << input_lines[i];
            if( csv.binary() )
            {
                std::cout.write( reinterpret_cast< const char* >( &centroid_assignment ), sizeof( centroid_assignment ) );
                for( size_t col = 0; col < *size; ++col )
                {
                    const auto point = static_cast< double >( centroids[centroid_assignment * *size + col] );
                    std::cout.write( reinterpret_cast< const char* >( &point ), sizeof( point ) );
                }
            }
            else
            {
                std::cout << csv.delimiter << centroid_assignment;
                for( size_t col = 0; col < *size; ++col ) { std::cout << csv.delimiter << static_cast< double >( centroids[centroid_assignment * *size + col] ); }
                std::cout << std::endl;
            }
        }
    };
    std::vector< float > centroids;
    std::vector< comma::uint32 > centroid_assignments;
    k_means operation{ tolerance, max_iterations, number_of_runs, number_of_clusters, *size };
    comma::csv::input_stream< input_t > istream( std::cin, csv );
    while( istream.ready() || std::cin.good() )
    {
        const input_t* p = istream.read();
        if( !dataframe.empty() && ( !p || block != p->block ) )
        {
            std::tie( centroids, centroid_assignments ) = use_pitched ? operation.run_on_block< snark::cuda::device::pitched_matrix >( dataframe ) : operation.run_on_block< snark::cuda::device::matrix >( dataframe );
            output_centroids ? write_centroids_only_( centroids, block ) : write_lines_( input_lines, centroids, centroid_assignments );
            if( csv.flush ) { std::cout.flush(); }
            dataframe.clear();
            input_lines.clear();
        }
        if( !p ) { break; }
        set_input_values_( *p, istream.last() );
    }
    return 0;
}

#endif

static int run_( const double tolerance, const unsigned int max_iterations, const unsigned int number_of_runs, const comma::uint32 number_of_clusters )
{
    using namespace snark::k_means;
    std::deque< std::vector< double > > dataframe;
    std::deque< std::string > input_lines;
    comma::uint32 block = 0;
    auto set_input_values_ = [&]( const input_t& p, const std::string& line )
    {
        block = p.block;
        input_lines.emplace_back( line );
        dataframe.emplace_back( p.data );
    };
    if( !size )
    {
        const auto& first_line = get_size();
        if( !first_line.empty()) { set_input_values_( comma::csv::ascii< input_t >( csv ).get( first_line ), first_line ); }
    }
    auto write_centroids_only_ = [number_of_clusters]( const std::vector< std::vector< double > >& centroids, const comma::uint32 block )
    {
        for( comma::uint32 i = 0; i < number_of_clusters; ++i )
        {
            if( csv.binary() )
            {
                if( use_block ) { std::cout.write( reinterpret_cast< const char* >( &block ), sizeof( block ) ); }
                std::cout.write( reinterpret_cast< const char* >( &i ), sizeof( i ) );
                for( const auto point : centroids[i] ) { std::cout.write( reinterpret_cast< const char* >( &point ), sizeof( point ) ); }
            }
            else
            {
                if( use_block ) { std::cout << block << csv.delimiter; }
                std::cout << i;
                for( const auto point : centroids[i] ) { std::cout << csv.delimiter << point; }
                std::cout << std::endl;
            }
        }
    };
    auto write_lines_ = []( const std::deque< std::string >& input_lines,
                            const std::vector< std::vector< double > >& centroids,
                            const std::vector< comma::uint32 >& centroid_assignments )
    {
        for( size_t i = 0; i < centroid_assignments.size(); ++i )
        {
            std::cout << input_lines[i];
            const auto centroid_assignment = centroid_assignments[i];
            const std::vector< double >& centroid = centroids[centroid_assignment];
            if( csv.binary() )
            {
                std::cout.write( reinterpret_cast< const char* >( &centroid_assignment ), sizeof( centroid_assignment ) );
                for( const auto point : centroid ) { std::cout.write( reinterpret_cast< const char* >( &point ), sizeof( point ) ); }
            }
            else
            {
                std::cout << csv.delimiter << centroid_assignment;
                for( const auto point : centroid ) { std::cout << csv.delimiter << point; }
                std::cout << std::endl;
            }
        }
    };
    std::vector< std::vector< double > > centroids;
    std::vector< comma::uint32 > centroid_assignments;
    k_means operation{ tolerance, max_iterations, number_of_runs, number_of_clusters };
    comma::csv::input_stream< input_t > istream( std::cin, csv );
    while( istream.ready() || std::cin.good() )
    {
        const input_t* p = istream.read();
        if( !dataframe.empty() && ( !p || block != p->block ) )
        {
            std::tie( centroids, centroid_assignments ) = operation.run_on_block( dataframe );
            if( output_centroids ) { write_centroids_only_( centroids, block ); }
            else { write_lines_( input_lines, centroids, centroid_assignments ); }
            if( csv.flush ) { std::cout.flush(); }
            dataframe.clear();
            input_lines.clear();
        }
        if( !p ) { break; }
        set_input_values_( *p, istream.last() );
    }
    return 0;
}

static int run( const comma::command_line_options& options )
{
    output_centroids = options.exists( "--output-centroids,--centroids" );
    csv = comma::csv::options( options, "data" );
    std::cout.precision( csv.precision );
    use_block = csv.has_field( "block" );
    seed = options.optional< std::mt19937::result_type >( "--seed" );
    const auto max_iterations = options.value< unsigned int >( "--max-iterations,--iterations", 300 );
    if( max_iterations == 0 ) { std::cerr << "math-k-means: got --max-iterations=0, --max-iterations should be at least 1" << std::endl; return 1; }
    const auto number_of_clusters = options.value< comma::uint32 >( "--number-of-clusters,--clusters" );
    if( number_of_clusters == 0 ) { std::cerr << "math-k-means: got --number-of-clusters=0, --number-of-clusters should be at least 1" << std::endl; return 1; }
    const auto number_of_runs = options.value< unsigned int >( "--number-of-runs,--runs", 10 );
    if( number_of_runs == 0 ) { std::cerr << "math-k-means: got --number-of-runs=0, --number-of-runs should be at least 1" << std::endl; return 1; }
    size = options.optional< unsigned int >( "--size" );
    auto tolerance = options.value< double >( "--tolerance", 1.0e-4 );
    if( tolerance <= 0 ) { std::cerr << "math-k-means: got --tolerance=" << tolerance << ", --tolerance should be greater than 0" << std::endl; return 1; }
    if( options.exists( "--ignore-tolerance" ) ) { tolerance = -1.0; }
#ifdef SNARK_USE_CUDA
    use_pitched = options.exists( "--cuda-use-pitched,--use-pitched,--pitched" );
    return options.exists( "--cuda" ) ? run_cuda_( static_cast< float >( tolerance ), max_iterations, number_of_runs, number_of_clusters ) : run_( tolerance, max_iterations, number_of_runs, number_of_clusters );
#else
    if( options.exists( "--cuda" ) ) { std::cerr << "math-k-means: given --cuda, but built without cuda support; run anyway, but may be slower than you expect" << std::endl; }
    return run_( tolerance, max_iterations, number_of_runs, number_of_clusters );
#endif
}

namespace comma { namespace visiting {

template <>
struct traits< snark::k_means::input_t >
{
    template < typename K, typename V >
    static void visit( const K&, snark::k_means::input_t& p, V& v )
    {
        v.apply( "data", p.data );
        v.apply( "block", p.block );
    }

    template < typename K, typename V >
    static void visit( const K&, const snark::k_means::input_t& p, V& v )
    {
        v.apply( "data", p.data );
        v.apply( "block", p.block );
    }
};

#ifdef SNARK_USE_CUDA
template <>
struct traits< snark::cuda::k_means::input_t >
{
    template < typename K, typename V >
    static void visit( const K&, snark::cuda::k_means::input_t& p, V& v )
    {
        v.apply( "data", p.data );
        v.apply( "block", p.block );
    }

    template < typename K, typename V >
    static void visit( const K&, const snark::cuda::k_means::input_t& p, V& v )
    {
        v.apply( "data", p.data );
        v.apply( "block", p.block );
    }
};
#endif

} } // namespace comma { namespace visiting {

int main( int argc, char** argv )
{
    try
    {
        comma::command_line_options options( argc, argv, usage );
        if( options.exists( "--input-fields" ) ) { std::cout << "data" << std::endl; return 0; }
        if( options.exists( "--output-fields" ) ) { std::cout << "data,centroid/id,centroid/data" << std::endl; return 0; }
        verbose = options.exists( "--verbose,-v" );
        auto max_threads = options.value< unsigned int >( "--max-threads,--threads", std::thread::hardware_concurrency() );
        if( max_threads <= 1 )
        {
            std::cerr << "math-k-means: warning: you set or std::thread::hardware_concurrency returned " << max_threads << "; will not explicitly call tbb::global_control::max_allowed_parallelism(); everything still will work, but multithreading may not be set to what you expected" << std::endl;
            return run( options );
        }
        tbb::global_control gc( tbb::global_control::max_allowed_parallelism, max_threads );
        return run( options );
    }
    catch( std::exception& ex ) { std::cerr << "math-k-means: " << ex.what() << std::endl; }
    catch( ... ) { std::cerr << "math-k-means: unknown exception" << std::endl; }
    return 1;
}