#include <boost/test/unit_test.hpp>

#include "defines.hpp"

#include <scorum/protocol/asset.hpp>
#include <scorum/rewards_math/formulas.hpp>

using namespace scorum::rewards_math;
using namespace scorum::protocol;

using scorum::protocol::curve_id;

namespace database_fixture {
struct rewards_math_voting_power_fixture
{
    const fc::time_point_sec last_voted = fc::time_point_sec(TEST_GENESIS_TIMESTAMP);
    const fc::time_point_sec now = last_voted + SCORUM_VOTE_REGENERATION_SECONDS.to_seconds() / 2;
    const vote_weight_type vote_weight = SCORUM_PERCENT(50);
    percent_type voting_power = SCORUM_100_PERCENT;
};
}

using namespace database_fixture;

BOOST_FIXTURE_TEST_SUITE(rewards_math_voting_power_tests, rewards_math_voting_power_fixture)

BOOST_AUTO_TEST_CASE(calculate_restoring_power_invalid_params)
{
    SCORUM_REQUIRE_THROW(calculate_restoring_power(voting_power, now, last_voted, fc::microseconds(0)), fc::exception);

    SCORUM_REQUIRE_THROW(calculate_restoring_power(voting_power, last_voted, now, SCORUM_VOTE_REGENERATION_SECONDS),
                         fc::exception);
}

BOOST_AUTO_TEST_CASE(calculate_restoring_power_check_voting_power_is_max)
{
    BOOST_CHECK_EQUAL(calculate_restoring_power(voting_power, now, last_voted, SCORUM_VOTE_REGENERATION_SECONDS),
                      voting_power);
}

BOOST_AUTO_TEST_CASE(calculate_used_power_positive)
{
    BOOST_CHECK_GT(calculate_used_power(voting_power, vote_weight, SCORUM_VOTING_POWER_DECAY_PERCENT), 0u);

    BOOST_CHECK_LE(calculate_used_power(voting_power, vote_weight, SCORUM_VOTING_POWER_DECAY_PERCENT), voting_power);
}

BOOST_AUTO_TEST_CASE(decreasing_and_restoring_voting_power_check)
{
    const int max_steps = 1000;
    int ci = 0;
    // decrease power with limiting by result percent or iterations (if decreasing too slow)
    while (voting_power > SCORUM_PERCENT(75) && ci++ < max_steps)
    {
        uint16_t used_power = calculate_used_power(voting_power, vote_weight, SCORUM_VOTING_POWER_DECAY_PERCENT);
        voting_power -= used_power;
    }

    // check if power decays too slow
    BOOST_CHECK_LT(ci, max_steps);

    const fc::time_point_sec last_voted(TEST_GENESIS_TIMESTAMP);
    const fc::time_point_sec now(last_voted + SCORUM_VOTE_REGENERATION_SECONDS.to_seconds() / 2);

    BOOST_CHECK_EQUAL(calculate_restoring_power(voting_power, now, last_voted, SCORUM_VOTE_REGENERATION_SECONDS),
                      SCORUM_100_PERCENT);
}

BOOST_AUTO_TEST_SUITE_END()


#ifndef HASHED_ARRAY_TREE_H
#define HASHED_ARRAY_TREE_H

#include <algorithm>
#include <bitset>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <functional>
#include <iomanip>
#include <iostream>
#include <iterator>
#include <stdexcept>
#include <boost/container/allocator.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <typeinfo>
#include <exception>
#include <memory>
#include <utility>
#include "wmath_forward.hpp"
#include "wmath_bits.hpp"
#include "wmath_hash.hpp"
#include "wmath_math.hpp"

namespace wmath{
  template<
    typename T,
    class alloc = std::allocator<T>
  >
  class hashed_array_tree{
    public:
      typedef typename alloc::size_type size_type;
      typedef alloc allocator_type;
    private:
      size_type n = 0;
      T** data = nullptr;
      allocator_type allocator;
      size_type log2rootsize_n;
      size_type static constexpr log2rootsize(const size_type& n) {
        return (digits<size_type>()-clz(n)+1)/2;
      }
      size_type static constexpr rootsize(const size_type& n) {
        return n?(size_type(1)<<((digits<size_type>()-clz(n)+1)/2)):0;
      }
      size_type static constexpr rootsize_from_log2(
          const size_type& log2rootsize_n){
        return size_type(1)<<log2rootsize_n;
      }
      tuple<size_type,size_type> constexpr resolve(
          const size_type& i,
          const size_type& n,
          const size_type& log2rootsize_n
          ) const {
        assert(n);
        assert(i<n);
        const size_type m = (size_type(1)<<log2rootsize_n)-1;
        return {i>>log2rootsize_n,i&m};
      }
      tuple<size_type,size_type> constexpr resolve(
          const size_type& i,
          const size_type& n) const {
        const size_type log2roosize_n = log2rootsize(n);
        return resolve(i,n,log2rootsize_n);
      }
    public:
      ~hashed_array_tree() {
        if (data!=nullptr) {
          for (size_type i=0;i!=rootsize(n);++i) {
            allocator.deallocate(data[i],rootsize(n));
          }
          delete[] data;
        }
      }
      inline const T& operator[](const size_type& i) const {
        const auto r = resolve(i,n,log2rootsize_n);
        //cerr << i << " " << n << " " << get<0>(r) << " " << get<1>(r) << endl;
        assert(data!=nullptr);
        assert(get<0>(r)<rootsize(n));
        assert(data[get<0>(r)]!=nullptr);
        assert(get<1>(r)<rootsize(n));
        return data[get<0>(r)][get<1>(r)];
      }
      inline T& operator[](const size_type& i) {
        const auto r = resolve(i,n,log2rootsize_n);
        //cerr << i << " " << n << " " << get<0>(r) << " " << get<1>(r) << endl;
        assert(data!=nullptr);
        assert(get<0>(r)<rootsize(n));
        assert(data[get<0>(r)]!=nullptr);
        assert(get<1>(r)<rootsize(n));
        return data[get<0>(r)][get<1>(r)];
      }
      template<bool is_const>
      class const_noconst_iterator {
        friend class hashed_array_tree;
        private:
          typename conditional<is_const,const T**,T**>::type data;
          size_type position;
          const size_type log2rootsize;
        public:
          typedef typename alloc::difference_type difference_type;
          typedef typename alloc::value_type value_type;
          typedef typename
            conditional<is_const,
                        const typename alloc::reference,
                              typename alloc::reference
                       >::type
            reference;
          typedef typename
            conditional<is_const,
                        const typename alloc::pointer,
                              typename alloc::pointer
                       >::type
            pointer;
          typedef std::random_access_iterator_tag iterator_category;
          const_noconst_iterator(
            typename conditional<is_const,
                                 const hashed_array_tree*,
                                       hashed_array_tree*
                                >::type tree
                                )
            :data(tree->data),
             position(0),
             log2rootsize(hashed_array_tree::log2rootsize(tree->n))
          {}
          const_noconst_iterator(
              typename conditional<is_const,
                                 const hashed_array_tree*,
                                       hashed_array_tree*
                                >::type tree,
              const size_type& i
                                )
            :data(tree->data),
             position(i),
             log2rootsize(hashed_array_tree::log2rootsize(tree->n))
          {}
          template<bool is_const_other>
          bool operator==(
              const const_noconst_iterator<is_const_other>& o) const {
            if (position!=o.position) return false;
            if (data!=o.data) return false;
            return true;
          }
          template<bool is_const_other>
          bool operator!=(
              const const_noconst_iterator<is_const_other>& o) const {
            return !((*this)==o);
          }
          template<bool is_const_other>
          bool operator< (
              const const_noconst_iterator<is_const_other>& o) const {
            if (position < o.position) return true;
            else return false;
          }
          template<bool is_const_other>
          bool operator> (
              const const_noconst_iterator<is_const_other>& o) const {
            if (position > o.position) return true;
            else return false;
          }
          template<bool is_const_other>
          bool operator<=(
              const const_noconst_iterator<is_const_other>& o) const {
            if (position <=o.position) return true;
            else return false;
          }
          template<bool is_const_other>
          bool operator>=(
              const const_noconst_iterator<is_const_other>& o) const {
            if (position >=o.position) return true;
            else return false;
          }
          const_noconst_iterator<is_const>& operator++(){   // prefix
            ++position;
            return *this;
          }
          const_noconst_iterator<is_const> operator++(int){ // postfix
            iterator pre(*this);
            operator++();
            return pre;
          }
          const_noconst_iterator<is_const>& operator--(){   // prefix
            --position;
            return *this;
          }
          const_noconst_iterator<is_const> operator--(int){ // postfix
            iterator pre(*this);
            operator--();
            return pre;
          }
          const_noconst_iterator<is_const>& operator+=(const size_type& n){
            position+=n;
            return *this;
          }
          const_noconst_iterator<is_const> operator+(const size_type& n) const {
            return (const_noconst_iterator<is_const>(*this)+=n);
          }
          friend const_noconst_iterator<is_const> operator+(
              const size_type& n,
              const const_noconst_iterator<is_const>& it){
            return (const_noconst_iterator<is_const>(it)+=n);
          }
          const_noconst_iterator<is_const>& operator-=(const size_type& n){
            position-=n;
            return *this;
          }
          const_noconst_iterator<is_const> operator-(const size_type& n) const{
            return (const_noconst_iterator<is_const>(*this)-=n);
          }
          reference operator*() {
            const size_type m = (size_type(1)<<log2rootsize)-1;
            return data[position>>log2rootsize][position&m];
          }
          pointer operator->() {
            const size_type m = (size_type(1)<<log2rootsize)-1;
            return data[position>>log2rootsize]+(position&m);
          }
          reference operator*() const {
            const size_type m = (size_type(1)<<log2rootsize)-1;
            return data[position>>log2rootsize][position&m];
          }
          pointer operator->() const {
            const size_type m = (size_type(1)<<log2rootsize)-1;
            return data[position>>log2rootsize]+(position&m);
          }
      };
      typedef const_noconst_iterator<false> iterator;
      typedef const_noconst_iterator<true>  const_iterator;
      iterator begin(){
        return iterator(this);
      }
      const_iterator begin() const {
        return const_iterator(this);
      }
      const_iterator cbegin() const {
        return const_iterator(this);
      }
      iterator end() {
        return iterator(this,n);
      }
      const_iterator end() const {
        return const_iterator(this,n);
      }
      const_iterator cend() const {
        return const_iterator(this,n);
      }
      size_type constexpr max_size() const {
        return numeric_limits<size_type>::max();
      }
      const size_type size() const {
        return n;
      }
      size_type constexpr static next_size(const size_type& n) {
        return (1+(n-1)/rootsize(n))*rootsize(n);
      }
      void inline resize(const size_type& m) {
        //cerr << "resize hashed_array_tree from " << n << " to " << m << endl;
        if (m==n) return;
        const size_type rootsize_n = rootsize_from_log2(log2rootsize_n);
        const size_type log2rootsize_m = log2rootsize(m);
        const size_type rootsize_m = rootsize(m);
        if (m==0) {
          for (size_type i=0;i!=rootsize_n;++i){
            allocator.deallocate(data[i],rootsize_n);
          }
          delete[] data;
          data = nullptr;
          n = m;
          log2rootsize_n = log2rootsize(n);
          return;
        }
        //cout << rootsize_n << " " << rootsize_m << endl;
        if (rootsize_m!=rootsize_n) {
          //cerr << "new index needed" << endl;
          T** new_data = new T*[rootsize_m];
          //cout << "first new[]" << endl;
          //cout << "new rootsize = " << rootsize_m << endl;
          for (size_type i=0;i!=rootsize_m;++i) new_data[i] = nullptr;
          size_type old_leaf = 0;
          size_type old_leafposition = 0;
          size_type new_leaf = 0;
          size_type new_leafposition = 0;
          for (size_type i=0;;++i) {
            if (new_leafposition == 0) {
              new_data[new_leaf] = allocator.allocate(rootsize_m);
              //cout << "new leaf" << endl;
            }
            if (i==min(m,n)) break;
            //cerr << "copying values" << endl;
            new_data[new_leaf][new_leafposition] =
              std::move(data[old_leaf][old_leafposition]);
            if (i+1==min(m,n)) break;
            if (++new_leafposition==rootsize_m) {
              ++new_leaf;
              new_leafposition = 0;
            }
            if (++old_leafposition==rootsize_n) {
              allocator.deallocate(data[old_leaf],rootsize_n);
              ++old_leaf;
              old_leafposition = 0;
            }
          }
          //cout << new_leaf << " " << (((m-1)>>log2rootsize_m)+1) << endl;
          for (++new_leaf;
              new_leaf!=(((m-1)>>log2rootsize_m)+1);
               ++new_leaf) {
            new_data[new_leaf] = allocator.allocate(rootsize_m);
            //cout << "new leaf" << endl;
          }
          //cout << "this were supposed to be all the leaves" << endl;
          if (data!=nullptr) {
            for (;old_leaf!=(((n-1)>>log2rootsize_n)+1);++old_leaf) {
              allocator.deallocate(data[old_leaf],rootsize_n);
            }
            delete[] data;
          }
          data = new_data;
          n = m;
          log2rootsize_n = log2rootsize(n);
          //cout << "I think I am done oO " << endl;
        } else {
          //cout << "only new leaves needed" << endl;
          if (m>n) {
            //cout << ((n-1)>>log2rootsize_n)+1 << " "
            //     << ((m-1)>>log2rootsize_n)+1 << endl;
            for (size_type i = n?((n-1)>>log2rootsize_n)+1:0;
                           i!=(m?((m-1)>>log2rootsize_n)+1:0);
                         ++i) {
              //cout << i << " " << rootsize_n << endl;
              assert(i<rootsize_n);
              data[i] = allocator.allocate(rootsize_n);
            }
          } else {
            for (size_type i = m?((m-1)>>log2rootsize_n)+1:0;
                           i!=(n?((n-1)>>log2rootsize_n)+1:0);
                         ++i) {
              assert(i<rootsize_n);
              allocator.deallocate(data[i],rootsize_n);
            }
          }
          n = m;
          log2rootsize_n = log2rootsize(n);
        }
      }
  };
}
#endif // HASHED_ARRAY_TREE_H


// Distributed under the MIT License.
// See LICENSE.txt for details.

#include "Domain/FunctionsOfTime/QuaternionFunctionOfTime.hpp"

// BoostMultiArray is used internally in odeint, so BoostMultiArray MUST be
// included before odeint
#include "DataStructures/BoostMultiArray.hpp"

#include <boost/numeric/odeint.hpp>
#include <pup_stl.h>

#include "Domain/FunctionsOfTime/QuaternionHelpers.hpp"
#include "NumericalAlgorithms/OdeIntegration/OdeIntegration.hpp"
#include "Utilities/ErrorHandling/Assert.hpp"
#include "Utilities/ErrorHandling/Error.hpp"
#include "Utilities/GenerateInstantiations.hpp"
#include "Utilities/MakeArray.hpp"
#include "Utilities/StdHelpers.hpp"

namespace domain::FunctionsOfTime {
template <size_t MaxDeriv>
QuaternionFunctionOfTime<MaxDeriv>::QuaternionFunctionOfTime(
    const double t, std::array<DataVector, 1> initial_quat_func,
    std::array<DataVector, MaxDeriv + 1> initial_omega_func,
    const double expiration_time)
    : stored_quaternions_and_times_{{t, std::move(initial_quat_func)}},
      omega_f_of_t_(t, std::move(initial_omega_func), expiration_time) {}

template <size_t MaxDeriv>
std::unique_ptr<FunctionOfTime> QuaternionFunctionOfTime<MaxDeriv>::get_clone()
    const {
  return std::make_unique<QuaternionFunctionOfTime>(*this);
}

template <size_t MaxDeriv>
void QuaternionFunctionOfTime<MaxDeriv>::pup(PUP::er& p) {
  FunctionOfTime::pup(p);
  p | stored_quaternions_and_times_;
  p | omega_f_of_t_;
}

template <size_t MaxDeriv>
void QuaternionFunctionOfTime<MaxDeriv>::update(
    const double time_of_update, DataVector updated_max_deriv,
    const double next_expiration_time) {
  omega_f_of_t_.update(time_of_update, std::move(updated_max_deriv),
                       next_expiration_time);
  update_stored_info();
}

template <size_t MaxDeriv>
void QuaternionFunctionOfTime<MaxDeriv>::update_stored_info() {
  const auto& omega_deriv_info = omega_f_of_t_.get_deriv_info();

  ASSERT(
      omega_deriv_info.size() == stored_quaternions_and_times_.size() + 1,
      "The number of stored quaternions is not one less than the number of "
      "stored omegas (which it should be after an update). Currently there are "
          << omega_deriv_info.size() << " stored omegas and "
          << stored_quaternions_and_times_.size() << " stored quaternions.");

  const size_t last_index = stored_quaternions_and_times_.size();
  // Final time, initial time, and quaternion
  const double t = omega_deriv_info[last_index].time;
  const double t0 = omega_deriv_info[last_index - 1].time;
  boost::math::quaternion<double> quaternion_to_integrate =
      datavector_to_quaternion(
          stored_quaternions_and_times_[last_index - 1].stored_quantities[0]);

  solve_quaternion_ode(make_not_null(&quaternion_to_integrate), t0, t);

  // normalize quaternion
  normalize_quaternion(make_not_null(&quaternion_to_integrate));

  stored_quaternions_and_times_.emplace_back(
      t, std::array<DataVector, 1>{
             quaternion_to_datavector(quaternion_to_integrate)});

  ASSERT(omega_deriv_info.size() == stored_quaternions_and_times_.size(),
         "The number of stored omegas must be the same as the number of stored "
         "quaternions after updating the missing quaternion. Now there are "
             << omega_deriv_info.size() << " stored omegas and "
             << stored_quaternions_and_times_.size() << " stored quaternions.");
}

template <size_t MaxDeriv>
void QuaternionFunctionOfTime<MaxDeriv>::solve_quaternion_ode(
    const gsl::not_null<boost::math::quaternion<double>*>
        quaternion_to_integrate,
    const double t0, const double t) const {
  // lambda that stores the internals of the ode
  const auto quaternion_ode_system =
      [this](const boost::math::quaternion<double>& state,
             boost::math::quaternion<double>& dt_state, const double time) {
        const boost::math::quaternion<double> omega =
            datavector_to_quaternion(omega_f_of_t_.func(time)[0]);
        dt_state = 0.5 * state * omega;
      };

  // Dense stepper
  auto dense_stepper = boost::numeric::odeint::make_dense_output(
      1.0e-15, 1.0e-14,
      boost::numeric::odeint::runge_kutta_dopri5<
          boost::math::quaternion<double>, double,
          boost::math::quaternion<double>, double,
          boost::numeric::odeint::vector_space_algebra>{});

  // Integrate from t0 to t, storing result in quaternion_to_integrate
  boost::numeric::odeint::integrate_adaptive(
      dense_stepper, quaternion_ode_system, *quaternion_to_integrate, t0, t,
      1e-4);
}

template <size_t MaxDeriv>
boost::math::quaternion<double> QuaternionFunctionOfTime<MaxDeriv>::setup_func(
    const double t) const {
  // Get quaternion and time at closest time before t
  const auto& stored_info_at_t0 =
      stored_info_from_upper_bound(t, stored_quaternions_and_times_);
  boost::math::quaternion<double> quat_to_integrate =
      datavector_to_quaternion(stored_info_at_t0.stored_quantities[0]);

  // Solve the ode and store the result in quat_to_integrate
  solve_quaternion_ode(make_not_null(&quat_to_integrate),
                       stored_info_at_t0.time, t);

  // Make unit quaternion
  normalize_quaternion(make_not_null(&quat_to_integrate));

  return quat_to_integrate;
}

template <size_t MaxDeriv>
std::array<DataVector, 1> QuaternionFunctionOfTime<MaxDeriv>::quat_func(
    const double t) const {
  return std::array<DataVector, 1>{quaternion_to_datavector(setup_func(t))};
}

template <size_t MaxDeriv>
std::array<DataVector, 2>
QuaternionFunctionOfTime<MaxDeriv>::quat_func_and_deriv(const double t) const {
  boost::math::quaternion<double> quat = setup_func(t);

  // Get omega and however many derivatives we need
  std::array<DataVector, 1> omega_func = omega_f_of_t_.func(t);

  boost::math::quaternion<double> omega =
      datavector_to_quaternion(omega_func[0]);

  boost::math::quaternion<double> dtquat = 0.5 * quat * omega;

  return std::array<DataVector, 2>{quaternion_to_datavector(quat),
                                   quaternion_to_datavector(dtquat)};
}

template <size_t MaxDeriv>
std::array<DataVector, 3>
QuaternionFunctionOfTime<MaxDeriv>::quat_func_and_2_derivs(
    const double t) const {
  boost::math::quaternion<double> quat = setup_func(t);

  // Get omega and however many derivatives we need
  std::array<DataVector, 2> omega_func_and_deriv =
      omega_f_of_t_.func_and_deriv(t);

  boost::math::quaternion<double> omega =
      datavector_to_quaternion(omega_func_and_deriv[0]);
  boost::math::quaternion<double> dtomega =
      datavector_to_quaternion(omega_func_and_deriv[1]);

  boost::math::quaternion<double> dtquat = 0.5 * quat * omega;
  boost::math::quaternion<double> dt2quat =
      0.5 * (dtquat * omega + quat * dtomega);

  return std::array<DataVector, 3>{quaternion_to_datavector(quat),
                                   quaternion_to_datavector(dtquat),
                                   quaternion_to_datavector(dt2quat)};
}

template <size_t MaxDeriv>
bool operator==(const QuaternionFunctionOfTime<MaxDeriv>& lhs,
                const QuaternionFunctionOfTime<MaxDeriv>& rhs) {
  return lhs.stored_quaternions_and_times_ ==
             rhs.stored_quaternions_and_times_ and
         lhs.omega_f_of_t_ == rhs.omega_f_of_t_;
}

template <size_t MaxDeriv>
bool operator!=(const QuaternionFunctionOfTime<MaxDeriv>& lhs,
                const QuaternionFunctionOfTime<MaxDeriv>& rhs) {
  return not(lhs == rhs);
}

// do explicit instantiation of MaxDeriv = {2,3,4}
#define DIM(data) BOOST_PP_TUPLE_ELEM(0, data)

#define INSTANTIATE(_, data)                                   \
  template class QuaternionFunctionOfTime<DIM(data)>;          \
  template bool operator==                                     \
      <DIM(data)>(const QuaternionFunctionOfTime<DIM(data)>&,  \
                  const QuaternionFunctionOfTime<DIM(data)>&); \
  template bool operator!=                                     \
      <DIM(data)>(const QuaternionFunctionOfTime<DIM(data)>&,  \
                  const QuaternionFunctionOfTime<DIM(data)>&);

GENERATE_INSTANTIATIONS(INSTANTIATE, (2, 3, 4))

#undef DIM
#undef INSTANTIATE
}  // namespace domain::FunctionsOfTime


/* sort word by frequency, sharding while counting version.

  1. read input file, do counting, if counts > 10M keys, write counts to 10 shard files:
       word \t count
  2. assume each shard file fits in memory, read each shard file, accumulate counts, and write to 10 count files:
       count \t word
  3. merge 10 count files using heap.

Limits: each shard must fit in memory.
*/
#include <boost/noncopyable.hpp>
#include <boost/ptr_container/ptr_vector.hpp>

#include <fstream>
#include <iostream>
#include <unordered_map>

#ifdef STD_STRING
#warning "STD STRING"
#include <string>
using std::string;
#else
#include <ext/vstring.h>
typedef __gnu_cxx::__sso_string string;
#endif

const size_t kMaxSize = 10 * 1000 * 1000;

class Sharder : boost::noncopyable
{
 public:
  explicit Sharder(int nbuckets)
    : buckets_(nbuckets)
  {
    for (int i = 0; i < nbuckets; ++i)
    {
      char buf[256];
      snprintf(buf, sizeof buf, "shard-%05d-of-%05d", i, nbuckets);
      buckets_.push_back(new std::ofstream(buf));
    }
    assert(buckets_.size() == static_cast<size_t>(nbuckets));
  }

  void output(const string& word, int64_t count)
  {
    size_t idx = std::hash<string>()(word) % buckets_.size();
    buckets_[idx] << word << '\t' << count << '\n';
  }

 protected:
  boost::ptr_vector<std::ofstream> buckets_;
};

void shard(int nbuckets, int argc, char* argv[])
{
  Sharder sharder(nbuckets);
  for (int i = 1; i < argc; ++i)
  {
    std::cout << "  processing input file " << argv[i] << std::endl;
    std::unordered_map<string, int64_t> counts;
    std::ifstream in(argv[i]);
    while (in && !in.eof())
    {
      counts.clear();
      string word;
      while (in >> word)
      {
        counts[word]++;
        if (counts.size() > kMaxSize)
        {
          std::cout << "    split" << std::endl;
          break;
        }
      }

      for (const auto& kv : counts)
      {
        sharder.output(kv.first, kv.second);
      }
    }
  }
  std::cout << "shuffling done" << std::endl;
}

// ======= sort_shards =======

std::unordered_map<string, int64_t> read_shard(int idx, int nbuckets)
{
  std::unordered_map<string, int64_t> counts;

  char buf[256];
  snprintf(buf, sizeof buf, "shard-%05d-of-%05d", idx, nbuckets);
  std::cout << "  reading " << buf << std::endl;
  {
    std::ifstream in(buf);
    string line;

    while (getline(in, line))
    {
      size_t tab = line.find('\t');
      if (tab != string::npos)
      {
        int64_t count = strtol(line.c_str() + tab, NULL, 10);
        if (count > 0)
        {
          counts[line.substr(0, tab)] += count;
        }
      }
    }
  }

  ::unlink(buf);
  return counts;
}

void sort_shards(const int nbuckets)
{
  for (int i = 0; i < nbuckets; ++i)
  {
    // std::cout << "  sorting " << std::endl;
    std::vector<std::pair<int64_t, string>> counts;
    for (const auto& entry : read_shard(i, nbuckets))
    {
      counts.push_back(make_pair(entry.second, entry.first));
    }
    std::sort(counts.begin(), counts.end());

    char buf[256];
    snprintf(buf, sizeof buf, "count-%05d-of-%05d", i, nbuckets);
    std::ofstream out(buf);
    std::cout << "  writing " << buf << std::endl;
    for (auto it = counts.rbegin(); it != counts.rend(); ++it)
    {
      out << it->first << '\t' << it->second << '\n';
    }
  }

  std::cout << "reducing done" << std::endl;
}

// ======= merge =======

class Source  // copyable
{
 public:
  explicit Source(std::istream* in)
    : in_(in),
      count_(0),
      word_()
  {
  }

  bool next()
  {
    string line;
    if (getline(*in_, line))
    {
      size_t tab = line.find('\t');
      if (tab != string::npos)
      {
        count_ = strtol(line.c_str(), NULL, 10);
        if (count_ > 0)
        {
          word_ = line.substr(tab+1);
          return true;
        }
      }
    }
    return false;
  }

  bool operator<(const Source& rhs) const
  {
    return count_ < rhs.count_;
  }

  void outputTo(std::ostream& out) const
  {
    out << count_ << '\t' << word_ << '\n';
  }

 private:
  std::istream* in_;
  int64_t count_;
  string word_;
};

void merge(const int nbuckets)
{
  boost::ptr_vector<std::ifstream> inputs;
  std::vector<Source> keys;

  for (int i = 0; i < nbuckets; ++i)
  {
    char buf[256];
    snprintf(buf, sizeof buf, "count-%05d-of-%05d", i, nbuckets);
    inputs.push_back(new std::ifstream(buf));
    Source rec(&inputs.back());
    if (rec.next())
    {
      keys.push_back(rec);
    }
    ::unlink(buf);
  }

  std::ofstream out("output");
  std::make_heap(keys.begin(), keys.end());
  while (!keys.empty())
  {
    std::pop_heap(keys.begin(), keys.end());
    keys.back().outputTo(out);

    if (keys.back().next())
    {
      std::push_heap(keys.begin(), keys.end());
    }
    else
    {
      keys.pop_back();
    }
  }
  std::cout << "merging done\n";
}

int main(int argc, char* argv[])
{
  int nbuckets = 10;
  shard(nbuckets, argc, argv);
  sort_shards(nbuckets);
  merge(nbuckets);
}


# define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
# define BOOST_MPL_LIMIT_LIST_SIZE 40
# define BOOST_MPL_LIMIT_VECTOR_SIZE 40
# define BOOST_MATH_ASSERT_UNDEFINED_POLICY false

#include <boost/mpl/vector/vector30.hpp>

# include <cstdint>
# include <boost/random.hpp>
# include <bml/random/random.hpp>
# include <bml/math/math.hpp>
# include <kdb/kdb.hpp>
# include <boost/bind.hpp>
# include <boost/math/distributions.hpp>

namespace qtype = kx::qtype;

# define Random_generator_																		\
													 (kreutzer1986)								\
													 (taus88)									\
													 (hellekalek1995)							\
													 (mt11213b)									\
													 (mt19937)									\
													 (mt19937_64)								\
													 (lagged_fibonacci607)						\
													 (lagged_fibonacci1279)						\
													 (lagged_fibonacci2281)						\
													 (lagged_fibonacci3217)						\
													 (lagged_fibonacci4423)						\
													 (lagged_fibonacci9689)						\
													 (lagged_fibonacci19937)					\
													 (lagged_fibonacci23209)					\
													 (lagged_fibonacci44497)					\
													 (ranlux3)									\
													 (ranlux4)									\
													 (ranlux64_3)								\
													 (ranlux64_4)								\
													 (ranlux3_01)								\
													 (ranlux4_01)								\
													 (ranlux64_3_01)							\
													 (ranlux64_4_01)							\
													 (ranlux24)									\
													 (ranlux48)									\

# define Random_generator(r,data,elem)															\
kx::K elem(kx::K seed_)																			\
{																								\
	kx::result_of::value<qtype::long_>::type seed = kx::value<qtype::long_>(seed_);				\
	boost::random::elem* gst = new boost::random::elem(seed);									\
	std::uintptr_t ptr = reinterpret_cast<long>(gst);											\
	bml::random::generator_map[ptr] = gst;														\
	return kx::kj(ptr);																			\
}																								\

BOOST_PP_SEQ_FOR_EACH(Random_generator, _, Random_generator_)

# define DIST_table_dim 3

# define DIST_table																				\
((runiform_smallint,uniform_smallint<kx::J>,(long_)(long_)))									\
((runiform_int_distribution,uniform_int_distribution<kx::J>,(long_)(long_)))					\
((runiform_real_distribution,uniform_real_distribution<>,(float_)(float_)))						\
((rbernoulli_distribution,bernoulli_distribution<>,(float_)))									\
((rbinomial_distribution,binomial_distribution<kx::J>,(long_)(float_)))							\
((rgeometric_distribution,geometric_distribution<kx::J>,(float_)))								\
((rnegative_binomial_distribution,negative_binomial_distribution<kx::J>,(long_)(float_)))		\
((rpoisson_distribution,poisson_distribution<kx::J>,(float_)))									\
((rexponential_distribution,exponential_distribution<>,(float_)))								\
((rgamma_distribution,gamma_distribution<>,(float_)(float_)))									\
((rweibull_distribution,weibull_distribution<>,(float_)(float_)))								\
((rextreme_value_distribution,extreme_value_distribution<>,(float_)(float_)))					\
((rnormal_distribution,normal_distribution<kx::F>,(float_)(float_)))							\
((rlognormal_distribution,lognormal_distribution<>,(float_)(float_)))							\
((rchi_squared_distribution,chi_squared_distribution<>,(float_)))								\
((rcauchy_distribution,cauchy_distribution<>,(float_)(float_)))									\
((rfisher_f_distribution,fisher_f_distribution<>,(float_)(float_)))								\
((rstudent_t_distribution,student_t_distribution<>,(float_)))									\

// ((discrete_distribution,discrete_distribution<kx::J>,(long_)(float_)))			\
// ((chi_squared_distribution,chi_squared_distribution<>,(float_)))			\

# define DIST_create_argument(z,n,data) kx::K BOOST_PP_CAT(k,n)
# define DIST_create_convert(r,data,n,elem) kx::result_of::value<qtype::elem>::type BOOST_PP_CAT(l,n) = kx::value<qtype::elem>(BOOST_PP_CAT(k,n));

# define DIST_generator(r,data,tuple)																																									\
	kx::K BOOST_PP_TUPLE_ELEM(DIST_table_dim,0,tuple)(BOOST_PP_ENUM(BOOST_PP_SEQ_SIZE(BOOST_PP_TUPLE_ELEM(DIST_table_dim,2,tuple)),DIST_create_argument, _ ))												\
{																																																		\
	BOOST_PP_SEQ_FOR_EACH_I(DIST_create_convert, _ , BOOST_PP_TUPLE_ELEM(DIST_table_dim,2,tuple))																										\
	boost::random::BOOST_PP_TUPLE_ELEM(kdb_dim,1,tuple) * dst;																																			\
	dst = new boost::random::BOOST_PP_TUPLE_ELEM(kdb_dim,1,tuple) ( BOOST_PP_ENUM_PARAMS(BOOST_PP_SEQ_SIZE(BOOST_PP_TUPLE_ELEM(DIST_table_dim,2,tuple)),l) );											\
	std::uintptr_t ptr = reinterpret_cast<long>(dst);																																					\
	bml::random::distribution_map[ptr] = dst;																																							\
	return kx::kj(ptr);																																													\
};																																																		\

BOOST_PP_SEQ_FOR_EACH(DIST_generator,~,DIST_table)

kx::K uniform_01()
{
	boost::random::uniform_01<>* dst = new boost::random::uniform_01<>();
	std::uintptr_t ptr = reinterpret_cast<long>(dst);
	bml::random::distribution_map[ptr] = dst;
	return kx::kj(ptr);
}

kx::K delete_(kx::K ptr_)
{
	kx::result_of::value<qtype::long_>::type ptr = kx::value<qtype::long_>(ptr_);
	bml::random::generator_map_::iterator git = bml::random::generator_map.find(ptr);
	if(git!=bml::random::generator_map.end()){
		bml::random::rdelete_(git->second);
		bml::random::generator_map.erase(git);
		return kx::kb(1);
	}

	bml::random::distribution_map_::iterator dit = bml::random::distribution_map.find(ptr);
	if(dit!=bml::random::distribution_map.end()){
		bml::random::rdelete_(dit->second);
		bml::random::distribution_map.erase(dit);
		return kx::kb(1);
	}

	return kx::kb(0);
}

kx::K generate(kx::K g__,kx::K d__,kx::K num_)
{
	kx::result_of::value<qtype::long_>::type g_ = kx::value<qtype::long_>(g__);
	bml::random::generator_map_::iterator g = bml::random::generator_map.find(g_);
	if(bml::random::generator_map.end()==g){return kx::kb(0);}

	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::random::distribution_map_::iterator d = bml::random::distribution_map.find(d_);
	if(bml::random::distribution_map.end()==d){return kx::kb(0);}

	kx::result_of::value<qtype::long_>::type num = kx::value<qtype::long_>(num_);
	return bml::random::generate_(g->second,d->second,num);
}

# define FDIST_table_dim 3

# define FDIST_table																				\
((dbernoulli_distribution,bernoulli_distribution<>,(float_)))										\
((dbeta_distribution,beta_distribution<>,(float_)(float_)))											\
((dbinomial_distribution,binomial_distribution<>,(float_)(float_)))									\
((dcauchy_distribution,cauchy_distribution<>,(float_)(float_)))										\
((dchi_squared_distribution,chi_squared_distribution<>,(float_)))									\
((dexponential_distribution,exponential_distribution<>,(float_)))									\
((dextreme_value_distribution,extreme_value_distribution<>,(float_)(float_)))						\
((dfisher_f_distribution,fisher_f_distribution<>,(float_)(float_)))									\
((dgamma_distribution,gamma_distribution<>,(float_)(float_)))										\
((dgeometric_distribution,geometric_distribution<>,(float_)))										\
((dhypergeometric_distribution,hypergeometric_distribution<>,(long_)(long_)(long_)))				\
((dlaplace_distribution,laplace_distribution<>,(float_)(float_)))									\
((dlogistic_distribution,logistic_distribution<>,(float_)(float_)))									\
((dlognormal_distribution,lognormal_distribution<>,(float_)(float_)))								\
((dnegative_binomial_distribution,negative_binomial_distribution<>,(float_)(float_)))				\
((dnon_central_t_distribution,non_central_t_distribution<>,(float_)(float_)))						\
((dnormal_distribution,normal_distribution<>,(float_)(float_)))										\
((dpareto_distribution,pareto_distribution<>,(float_)(float_)))										\
((dpoisson_distribution,poisson_distribution<>,(float_)))											\
((drayleigh_distribution,rayleigh_distribution<>,(float_)))											\
((dskew_normal_distribution,skew_normal_distribution<>,(float_)(float_)(float_)))                   \
((dstudents_t_distribution,students_t_distribution<>,(float_)))										\
((dinverse_gamma_distribution,inverse_gamma_distribution<>,(float_)(float_)))						\
((dweibull_distribution,weibull_distribution<>,(float_)(float_)))									\

//((dnon_central_beta_distribution,non_central_beta_distribution<>,(float_)(float_)(float_)))			\
//((dnon_central_chi_squared_distribution,non_central_chi_squared_distribution<>,(float_)(float_)))	\
//((dnon_central_f_distribution,non_central_f_distribution<>,(float_)(float_)(float_)))				\
//((dinverse_chi_squared_distribution,inverse_chi_squared_distribution<>,(float_)(float_)))			\
//((dinverse_gamma_distribution,inverse_gamma_distribution<>,(float_)(float_)))						\
//((dinverse_gaussian_distribution,inverse_gaussian_distribution<>,(float_)(float_)))					\
// ((dtriangular_distribution,triangular_distribution<>,(float_)(float_)(float_)))						\

# define FDIST_create_argument(z,n,data) kx::K BOOST_PP_CAT(k,n)
# define FDIST_create_convert(r,data,n,elem) kx::result_of::value<qtype::elem>::type BOOST_PP_CAT(l,n) = kx::value<qtype::elem>(BOOST_PP_CAT(k,n));

# define FDIST_generator(r,data,tuple)																																									\
	kx::K BOOST_PP_TUPLE_ELEM(FDIST_table_dim,0,tuple)(BOOST_PP_ENUM(BOOST_PP_SEQ_SIZE(BOOST_PP_TUPLE_ELEM(FDIST_table_dim,2,tuple)),FDIST_create_argument, _ ))												\
{																																																		\
	BOOST_PP_SEQ_FOR_EACH_I(DIST_create_convert, _ , BOOST_PP_TUPLE_ELEM(DIST_table_dim,2,tuple))																										\
	boost::math::BOOST_PP_TUPLE_ELEM(kdb_dim,1,tuple) * dst;																																			\
	dst = new boost::math::BOOST_PP_TUPLE_ELEM(kdb_dim,1,tuple) ( BOOST_PP_ENUM_PARAMS(BOOST_PP_SEQ_SIZE(BOOST_PP_TUPLE_ELEM(FDIST_table_dim,2,tuple)),l) );											\
	std::uintptr_t ptr = reinterpret_cast<long>(dst);																																					\
	bml::math::distribution_map[ptr] = dst;																																							\
	return kx::kj(ptr);																																													\
};																																																		\

BOOST_PP_SEQ_FOR_EACH(FDIST_generator,~,FDIST_table)

//kx::K dnormal_distribution(kx::K k0,kx::K k1)
//{
//	kx::result_of::value<qtype::float_>::type l0 = kx::value<qtype::float_>(k0);
//	kx::result_of::value<qtype::float_>::type l1 = kx::value<qtype::float_>(k1);
//	boost::math::normal_distribution<>* dst = new boost::math::normal_distribution<>(l0,l1);
//	std::uintptr_t ptr = reinterpret_cast<long>(dst);
//	bml::math::distribution_map[ptr] = dst;
//	return kx::kj(ptr);
//}

kx::K cdf(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::cdf_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K cdf_comp(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::cdf_comp_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K pdf(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::pdf_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K quantile(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::quantile_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K quantile_comp(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::quantile_comp_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K hazard(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::hazard_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K chf(kx::K d__,kx::K x_)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	kx::vector<qtype::float_> x(x_);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::chf_visitor<kx::vector<qtype::float_> > cv(x);
	return boost::apply_visitor(cv,d->second);
}

kx::K mean_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::mean_visitor cv;
	return boost::apply_visitor(cv,d->second);
}

kx::K median_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::median_visitor cv;
	return boost::apply_visitor(cv,d->second);
}

kx::K mode_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::mode_visitor cv;
	return boost::apply_visitor(cv,d->second);
}

kx::K standard_deviation_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::standard_deviation_visitor cv;
	return boost::apply_visitor(cv,d->second);
}

kx::K variance_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::variance_visitor cv;
	return boost::apply_visitor(cv,d->second);
}

kx::K skewness_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::skewness_visitor cv;
	return boost::apply_visitor(cv,d->second);
}

kx::K kurtosis_(kx::K d__)
{
	kx::result_of::value<qtype::long_>::type d_ = kx::value<qtype::long_>(d__);
	bml::math::distribution_map_::iterator d = bml::math::distribution_map.find(d_);
	if(bml::math::distribution_map.end()==d){return kx::kb(0);}

	bml::math::kurtosis_visitor cv;
	return boost::apply_visitor(cv,d->second);
}