#include "utils.h"
#include "ps/ps.h"
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>
#include <thread>
#include <boost/random/uniform_int.hpp>
#include <boost/random/uniform_real.hpp>
#include <boost/random/normal_distribution.hpp>
#include <boost/random/variate_generator.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/timer/timer.hpp>
#include <boost/generator_iterator.hpp>
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <limits>
#include <sstream>
#include <string>
#include <iostream>
#include <unistd.h>
#include <bitset>
#include <Eigen/Dense>
#include <Eigen/SparseCore>
#include "mf/update.h"
#include "mf/data.h"
#include "mf/io.h"

using namespace ps;
using namespace std;

typedef double ValT;
typedef DefaultColoServerHandle<ValT> HandleT;
typedef ColoKVServer<ValT, HandleT> ServerT;
typedef ColoKVWorker<ValT, HandleT> WorkerT;

typedef Eigen::VectorXd vect;
typedef Eigen::SparseMatrix<ValT,Eigen::RowMajor> SpMatrixRM;
typedef Eigen::SparseMatrix<ValT,Eigen::ColMajor> SpMatrixCM;
typedef Eigen::Matrix<ValT, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> DeMatrixRM;
typedef Eigen::Matrix<ValT, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor> DeMatrixCM;

// Matrix factorization config
string dataset;
uint mf_rank;
uint param_len;
int epochs;
double initial_step_size;
double lambda;
double increase_step_factor;
double decrease_step_factor;
bool use_wor_block_schedule;
bool use_wor_point_schedule;
bool compute_loss;
bool bold_driver;

// System config
uint num_workers;
int  num_threads;
long num_cols;
long num_rows;
Alg algorithm;
bool   signal_intent_rows;
size_t signal_intent_cols;
int init_parameters;
bool enforce_random_keys;
unsigned long early_stop;
bool sync_push;
long max_runtime;
bool prevent_full_model_pull;
std::string write_generated_factors;
bool enforce_full_replication;

// set by system
Key first_col_key;
Key num_keys;
std::vector<Key> key_assignment;
unsigned model_seed;

// Shared data
std::vector<Key> full_h_keys;
std::vector<ValT> full_h;
std::vector<ValT> local_w_shared {};

inline Key row_key(size_t i) {
  return enforce_random_keys ? key_assignment[i] : i;
}
inline Key col_key(size_t j) {
  return enforce_random_keys ? key_assignment[first_col_key+j] : first_col_key+j;
}

#include "mf/loss.h"

// Calculate loss on training and test data
double calculate_loss(const int epoch, std::vector<Key>& local_w_keys,
                      mf::DataPart& train_data, mf::DataPart& test_data,
                      WorkerT& kv, const int worker_id, const int num_workers, const int customer_id) {

  // key and vector for aggregation (in PS)
  std::vector<Key> loss_key { num_keys-1 };
  std::vector<ValT> agg_vec (param_len);
  assert(param_len >= 3); // need two elements to aggregate train and test loss

  // reset aggregation
  if (worker_id == 0) {
    kv.Wait(kv.Pull(loss_key, &agg_vec));
    for(size_t i=0; i!=agg_vec.size(); ++i) {
      agg_vec[i] = -agg_vec[i];
    }
    kv.Wait(kv.Push(loss_key, agg_vec));
  }
  kv.WaitSync();
  kv.Barrier();

  // wait for sync
  kv.WaitSync();

  // in columnwise SGD, multiple threads hold the same part of w
  // we calculate l2 for this block only in the first of these threads
  bool add_w_l2 = (algorithm == Alg::columnwise && customer_id != 0 ? false : true);

  // calculate local loss
  double local_train_loss = 0;
  double local_test_loss = 0;
  double local_train_reg = 0;
  if (!prevent_full_model_pull) {
    if (worker_id == 0) ADLOG("Pull model for loss calc");

    // pull local part of W
    std::vector<ValT> local_w_worker {};
    std::vector<ValT>* local_w;
    if (algorithm != Alg::columnwise) {
      // normally, each worker has its dedicated local part of w (i.e., no sharing possible)
      local_w_worker.resize(local_w_keys.size() * param_len);
      kv.Wait(kv.Pull(local_w_keys, &local_w_worker)); // fetch local part of w
      local_w = &local_w_worker;
    } else {
      // in the columnwise alg, the workers of one process work on the same part
      // of W, so we share W within a process to save memory and bandwidth
      if (customer_id == 0) {
        local_w_shared.resize(local_w_keys.size() * param_len);
        kv.Wait(kv.Pull(local_w_keys, &local_w_shared));
      }
      kv.Barrier();
      local_w = &local_w_shared;
    }

    // pull (full) H (once per process)
    if (customer_id == 0) { // fetch full h once per process
      full_h.resize(num_cols * param_len);
      kv.Wait(kv.Pull(full_h_keys, &full_h));
    }
    kv.Barrier();

    // calculate loss (with a copy of the model)
    std::tie(local_train_loss, local_train_reg) = loss_Nzsl_L2(train_data, *local_w, full_h, lambda, worker_id, add_w_l2);
    std::tie(local_test_loss, std::ignore)      = loss_Nzsl_L2(test_data,  *local_w, full_h, 0.0,    worker_id, add_w_l2);
  } else {
    // calculate loss (pulling necessary factors on demand)
    if (worker_id == 0) ADLOG("No full model pull for eval. Pull factors on demand.");
    std::tie(local_train_loss, local_train_reg) = loss_Nzsl_L2_pull(train_data, kv, local_w_keys, lambda, worker_id, add_w_l2);
    std::tie(local_test_loss, std::ignore)      = loss_Nzsl_L2_pull(test_data,  kv, local_w_keys, 0.0,    worker_id, add_w_l2);
  }
  // ADLOG("Local training loss (worker " << worker_id << "): " << local_train_loss);
  // ADLOG("Local test loss (worker " << worker_id << "): " << local_test_loss);

  // aggregate local losses to global loss (using the PS)
  agg_vec[0] = local_train_loss;
  agg_vec[1] = local_train_reg;
  agg_vec[2] = local_test_loss;
  kv.Wait(kv.Push(loss_key, agg_vec));
  kv.Barrier();
  kv.Wait(kv.Pull(loss_key, &agg_vec));

  auto train_loss = (agg_vec[0] + agg_vec[1]) / train_data.total_nnz();
  auto test_loss = (agg_vec[2]) / test_data.total_nnz();

  if (worker_id == 0) {
    ADLOG("(Ep." << epoch << ") Train loss: " << train_loss << " (" << agg_vec[0] / train_data.total_nnz() << " + " << agg_vec[1] / train_data.total_nnz() << ")");
    ADLOG("(Ep." << epoch << ") Test loss:  " << test_loss);
  }

  return train_loss;
}

void RunWorker(int customer_id, ServerT* server=nullptr) {
  WorkerT kv(customer_id, *server);

  boost::random::mt19937 rng(static_cast<unsigned int>(std::time(0)));
  std::unordered_map<std::string, util::Stopwatch> sw {};
  util::Trace trace {"./measurements/mf/mf_trace.csv"};

  UpdateNsqlL2Adagrad update_fun {mf_rank, param_len, initial_step_size, lambda};
  int worker_id = ps::MyRank()*num_threads+customer_id; // a unique id for this worker thread
  Eigen::MatrixXi block_schedule = mf::initBlockSchedule(num_workers);

  /* Load data, factors, and allocate memory */

  // Load train data
  sw["read_data"].start();
  mf::DataPart data =      read_sparse_matrix_part(dataset + string("train.mmc"), true, num_workers, worker_id, Postoffice::Get()->num_servers(), ps::MyRank(), num_rows, num_cols, customer_id, algorithm, kv);
  mf::DataPart data_test = read_sparse_matrix_part(dataset + string("test.mmc"),  true, num_workers, worker_id, Postoffice::Get()->num_servers(), ps::MyRank(), num_rows, num_cols, customer_id, algorithm, kv);
  sw["read_data"].stop();
  ALOG("Finished reading train data in worker " << worker_id << " (" << sw["read_data"] << ")");

  // allocate memory for factors in training loop
  std::vector<Key> factors_keys ( 2 );
  std::vector<ValT> factors ( factors_keys.size() * param_len );
  std::vector<ValT> factors_update ( factors.size() );

  // allocate memory for factors in loss computation (once per process)
  if (customer_id == 0) {
    full_h_keys.resize(num_cols);
    for(size_t j=0; j!=full_h_keys.size(); ++j) {
      full_h_keys[j] = col_key(j);
    }
  }

  std::vector<Key> local_h_keys ( data.num_cols_per_block() );
  std::vector<Key> local_w_keys ( data.num_rows_per_block() );
  int my_block = (algorithm == Alg::columnwise ? ps::MyRank() : worker_id); // per-process blocks in columnwise
  for(size_t z=0; z!=data.num_rows_per_block(); ++z) {
    local_w_keys[z] = row_key(data.num_rows_per_block() * my_block + z);
  }

  boost::random::normal_distribution<> factor_generator (0, 1/sqrt(sqrt(mf_rank)));

  // push factors into the parameter servers
  kv.BeginSetup();
  if (worker_id == 0 && init_parameters > 0) {
    // read and send H
    sw["read_factors"].start();
    std::vector<ValT> init_h {};
    if (init_parameters == 1) {
      // read factors from file
      init_h = read_dense_matrix<std::vector<ValT>>(dataset + "H.mma", false);
    } else if (init_parameters == 2) {
      // init model randomly
      ALOG("Init H randomly (seed " << model_seed << ")");
      init_h.resize(num_cols * param_len);
      boost::random::mt19937 rng_H (model_seed+133273);
      for (long col=0; col!=num_cols ; ++col) {
          for (long r=0; r!=mf_rank; ++r) {
            init_h[col*param_len+r] = factor_generator(rng_H);
          }
          // leave 0's for AdaGrad values (pos mf_rank..param_len)
      }
    }
    kv.Wait(kv.StaggeredPush(full_h_keys, init_h));
    sw["read_factors"].stop();

    // optional: write out initial random factors
    if (!write_generated_factors.empty()) {
      std::ofstream out (write_generated_factors + "H.mma");
      if (!out.is_open()) {
        ALOG("Cannot open file to write initial factors: " << write_generated_factors << "H.mma");
        abort();
      }

      out << "%%MatrixMarket matrix array real general" << std::endl;
      out << "% First line: ROWS COLUMNS" << std::endl;
      out << "% Subsequent lines: entries in column-major order" << std::endl;
      out << mf_rank << " " << num_cols << std::endl;
      for (size_t z=0; z!=init_h.size(); ++z) {
        out << init_h[z] << "\n";
      }
      out.close();
    }

    ALOG("Initialized H (" << sw["read_factors"] << "): " << init_h[0] << " " << init_h[1] << " " << init_h[2] << " ..");
  }

  if (init_parameters > 0 && static_cast<uint>(worker_id) == num_workers-1) {
    // read and send W
    sw["read_factors"].start();
    ValT first, second;
    if (init_parameters == 1) {
      // read factors from files
      vector<Key> full_w_keys (num_rows);
      for(size_t i = 0; i!=full_w_keys.size(); ++i) full_w_keys[i] = row_key(i);
      std::vector<ValT> full_w;
      full_w = read_dense_matrix<std::vector<ValT>>(dataset + "W.mma", true);
      kv.Wait(kv.StaggeredPush(full_w_keys, full_w));
      first = full_w[0];
      second = full_w[1];
    } else if (init_parameters == 2) {
      // init model randomly
      ALOG("Init W randomly (seed " << model_seed << ")");
      boost::random::mt19937 rng_W (model_seed);

      // optional: write out initial random factors
      std::ofstream out {};
      if (!write_generated_factors.empty()) {
        out.open(write_generated_factors + "W.mma");
        if (!out.is_open()) {
          ALOG("Cannot open file to write initial factors: " << write_generated_factors << "W.mma");
          abort();
        }

        // write header
        out << "%%MatrixMarket matrix array real general" << std::endl;
        out << "% First line: ROWS COLUMNS" << std::endl;
        out << "% Subsequent lines: entries in row-major order" << std::endl;
        out << num_rows << " " << mf_rank << std::endl;
      }

      // use (more complicated) batched implementation so we can run large models on a single machine
      size_t batch_size = 10000;
      vector<Key> partial_w_keys {};
      std::vector<ValT> partial_w {};
      vector<int> tss {};
      size_t gen = 0;
      for (int i=0; i!=num_rows; ++i) {
        partial_w_keys.push_back(row_key(i));

        // generate
        for (size_t z=0; z!=mf_rank; ++z) {
          auto r = factor_generator(rng_W);
          if (gen == 0) first = r;
          if (gen == 1) second = r;
          ++gen;
          partial_w.push_back(r);

          if (!write_generated_factors.empty()) {
            out << r << "\n";
          }
        }

        // initialize AdaGrad values with 0
        for (size_t z=mf_rank; z!=param_len; ++z) {
          partial_w.push_back(0);
        }

        // push
        if (i % batch_size == batch_size-1) {
          tss.push_back(kv.Push(partial_w_keys, partial_w));
          partial_w_keys.resize(0);
          partial_w.resize(0);
        }
      }
      tss.push_back(kv.Push(partial_w_keys, partial_w));
      for (auto ts : tss) kv.Wait(ts);

      if (!write_generated_factors.empty()) {
        out.close();
      }
    }
    sw["read_factors"].stop();
    ALOG("Initialized W (" << sw["read_factors"] << "): " << first << " " << second << " ..");
  }
  kv.EndSetup(); // waits for sync and does barrier

  // replicate all keys on all nodes throughout training
  // (sensible only in ablation experiments)
  if (enforce_full_replication && customer_id == 0) {
  	std::vector<Key> keys (num_keys);
    std::iota(keys.begin(), keys.end(), 0);
    kv.Intent(keys, 0, CLOCK_MAX);
  }
  kv.WaitSync();
  kv.Barrier();
  kv.WaitSync();

  // Signal intent for the parameters of the (statically partitioned) rows
  if (signal_intent_rows) {
    kv.Intent(local_w_keys, 0, CLOCK_MAX);
    kv.WaitSync();
  }

  // prep for column-wise access
  std::vector<size_t> my_columns {};
  if (algorithm == Alg::columnwise) {
    for (long j=0; j!=num_cols; ++j) {
      if (data.block_has_nnz(j)) {
        my_columns.push_back(j);
      }
    }

    srand(worker_id ^ 13 * 17); // make sure workers have unique column permutations
    if (use_wor_block_schedule) std::random_shuffle(my_columns.begin(), my_columns.end());
  }

  // Compute initial loss
  ValT previous_train_loss=0;
  if (compute_loss) {
    auto train_loss = calculate_loss(0, local_w_keys, data, data_test,
                                     kv, worker_id, num_workers, customer_id);
    previous_train_loss = train_loss;
  }

  kv.Barrier(); // make sure all workers start training at the same time

  /* Training loop */
  for(int epoch = 1; epoch != epochs+1; ++epoch) {
    if (worker_id == 0) ADLOG("(Ep." << epoch << ") Starting epoch " << epoch);
    sw["epoch.worker"].start();
    sw["epoch"].start();
    sw["runtime"].resume();

    // create WOR block schedule for this epoch, if desired (WOR = random, without replacement)
    if (use_wor_block_schedule) mf::newWorSchedule(block_schedule, epoch, rng);

    long updates = 0;

    if (algorithm == Alg::dsgd) { // use DSGD to create locality in parameter accesses
      for(uint subepoch = 0; subepoch != num_workers; ++subepoch) {
        if (worker_id == 0) ADLOG("Subepoch " << epoch << "." << subepoch);

        // get the block this workers works on in this epoch
        sw["comm"].resume();
        int h_block = block_schedule(subepoch, worker_id);
        if (signal_intent_cols != 0) {
          for (size_t z = 0; z!=local_h_keys.size(); ++z) {
            local_h_keys[z] = col_key(h_block * data.num_cols_per_block() + z);
          }
          kv.Intent(local_h_keys, kv.currentClock());
        }
        sw["comm"].stop();

        // permute training points (WOR training schedule)
        sw["perm"].resume();
        if (use_wor_point_schedule) data.permuteBlock(h_block);
        sw["perm"].stop();

        // train on this block

        // iterate over the nonzeros in this block
        sw["comp"].resume();
        for (unsigned long z=data.block_start(h_block); z!=data.block_end(h_block); ++z) {
          const mf::DataPoint& dp = data.data()[data.permutation()[z]];

          // get factors
          factors_keys[0] = row_key(dp.i);
          factors_keys[1] = col_key(dp.j);
          kv.Wait(kv.Pull(factors_keys, &factors));

          // run update
          update_fun(dp.i, dp.j, factors, factors_update, dp.x, data.row_nnz(dp.i), data.col_nnz(dp.j));
          ++updates;

          // send factor updates
          auto ts = kv.Push(factors_keys, factors_update);
          if (sync_push) kv.Wait(ts);
        }

        sw["comp"].stop();

        kv.advanceClock();

        // wait for all workers to finish the subepoch
        sw["barrier"].resume();
        kv.Barrier();
        sw["barrier"].stop();

        // ADLOG("Subepoch " << epoch << "." << subepoch << " took " << sw["subepoch"]);
      }
    } else if (algorithm == Alg::columnwise) { // columnwise
      // permute columns
      sw["perm"].resume();
      ADLOG("Permute columns");
      if(use_wor_block_schedule) std::random_shuffle(my_columns.begin(), my_columns.end());
      sw["perm"].stop();

      // iterate over the nonzeros that are partitioned to this worker
      sw["comp"].resume();
      long col_future=0;
      long z_epoch=0;
      long z_future=0;
      for (unsigned long col=0; col!=my_columns.size(); ++col) {
        auto j = my_columns[col];

        // permute data points of this column
        if (use_wor_point_schedule) data.permuteBlock(j);

        // signal intent for column parameters
        if (signal_intent_cols != 0) {
          while (col_future < static_cast<long>(col + signal_intent_cols) &&
                 col_future < static_cast<long>(my_columns.size())) {
            Clock futureClock = kv.currentClock() + z_future - z_epoch;
            auto col_datapoints = data.block_size(my_columns[col_future]);
            kv.Intent(col_key(my_columns[col_future]), futureClock, futureClock+col_datapoints);
            z_future += col_datapoints;
            ++col_future;
          }
        }

        for (unsigned long z=data.block_start(j); z!=data.block_end(j); ++z, ++z_epoch) {
          const mf::DataPoint& dp = data.data()[data.permutation()[z]];

          // get factors
          factors_keys[0] = row_key(dp.i);
          factors_keys[1] = col_key(dp.j);
          kv.Wait(kv.Pull(factors_keys, &factors));

          // run update
          update_fun(dp.i, dp.j, factors, factors_update, dp.x, data.row_nnz(dp.i), data.col_nnz(dp.j));
          ++updates;

          // send factor updates
          auto ts = kv.Push(factors_keys, factors_update);
          if (sync_push) kv.Wait(ts);

          kv.advanceClock();

          // early stopping (for debugging)
          if(early_stop != 0 && z == early_stop) {
            ALOG("Worker " << worker_id << " stops the epoch early after " << early_stop << " of " << data.num_nnz() << " data points");
            goto endepoch;
          }
        }
      }
    endepoch:
      sw["comp"].stop();
      sw["epoch.worker"].stop();
      ALOG("Worker " << worker_id << " is through its " << data.num_nnz() << " data points (" << sw["epoch.worker"] << ")");

      // wait for all workers to finish the epoch
      sw["barrier"].resume();
      kv.Barrier();
      sw["barrier"].stop();

    } else { // plain SGD: access the data points in fully random order (i.e., no blocks and no columnwise access)

      // permute training data points (WOR training schedule)
      sw["perm"].resume();
      if (worker_id == 0) ADLOG("Permute ...");
      if (use_wor_point_schedule) data.permuteData();
      sw["perm"].stop();

      // iterate over the nonzeros that are partitioned to this worker
      sw["comp"].resume();
      if (worker_id == 0) ADLOG("Compute ...");
      unsigned long z_future=0;
      for (unsigned long z=0; z!=data.num_nnz(); ++z) {
        const mf::DataPoint& dp = data.data()[data.permutation()[z]];

        // early stopping (for debugging)
        if(early_stop != 0 && z == early_stop) {
          ALOG("Worker " << worker_id << " stops the epoch early after " << early_stop << " of " << data.num_nnz() << " data points");
          break;
        }

        // signal intent for column parameters
        if (signal_intent_cols != 0) {
          while (z_future < z + signal_intent_cols &&
                 z_future < data.num_nnz()) {
            Clock futureClock = kv.currentClock() + z_future - z;
            const mf::DataPoint& dp_future = data.data()[data.permutation()[z_future]];
            kv.Intent(col_key(dp_future.j), futureClock);
            ++z_future;
          }
        }

        // get factors
        factors_keys[0] = row_key(dp.i);
        factors_keys[1] = col_key(dp.j);
        kv.Wait(kv.Pull(factors_keys, &factors));

        // run update
        update_fun(dp.i, dp.j, factors, factors_update, dp.x, data.row_nnz(dp.i), data.col_nnz(dp.j));
        ++updates;

        // send factor updates
        auto ts = kv.Push(factors_keys, factors_update);
        if (sync_push) kv.Wait(ts);

        kv.advanceClock();
      }
      sw["comp"].stop();
      sw["epoch.worker"].stop();
      ALOG("Worker " << worker_id << " is through its " << data.num_nnz() << " data points (" << sw["epoch.worker"] << ")");

      // wait for all workers to finish the epoch
      sw["barrier"].resume();
      kv.Barrier();
      sw["barrier"].stop();

    }
    sw["epoch"].stop();
    sw["runtime"].stop();
    ADLOG("(Ep." << epoch << ") Worker " << worker_id << " finished epoch " << epoch <<" (" << sw["epoch"] << ": " << sw["comm"] << " comm, "  << sw["comp"] << " comp, " << sw["barrier"] << " barrier, updates: " << updates <<")");
    if (worker_id == 0) {
      ALOG("All workers finished epoch " << epoch << " (epoch: " << sw["epoch"] << ", total: " << sw["runtime"] << ").");
    }
    sw["comm"].reset(); sw["comp"].reset(); sw["perm"].reset(); sw["barrier"].reset();

    if (compute_loss) {
      auto train_loss = calculate_loss(epoch, local_w_keys, data, data_test,
                                       kv, worker_id, num_workers, customer_id);

      // bold driver: adapt step size
      if (bold_driver) {
        if (train_loss < previous_train_loss) {
          update_fun.update_step_size(increase_step_factor); // good epoch. increase step size slightly
          if (worker_id == 0) {
            ADLOG("Bold driver: Increase step size to " << update_fun.current_step_size() << " (Train loss: " << previous_train_loss << " -> " << train_loss << ")");
          }
        } else {
          update_fun.update_step_size(decrease_step_factor); // bad epoch. reduce step size
          if (worker_id == 0) {
            ADLOG("Bold driver: Decrease step size to " << update_fun.current_step_size() << " (Train loss: " << previous_train_loss << " -> " << train_loss << ")");
          }
        }
      }

      previous_train_loss = train_loss;
    }

    // maximum time
    if (sw["runtime"].elapsed_s() > max_runtime ||
        sw["runtime"].elapsed_s() + sw["epoch"].elapsed_s() > max_runtime * 1.05) {
      ADLOG("Worker " << worker_id << " stops after epoch " << epoch << " because max. time is reached: " << sw["runtime"].elapsed_s() << "s (+1 epoch) > " << max_runtime << "s (epoch: " << sw["epoch"].elapsed_s() << "s)");
      break;
    }
  }

  // make sure all workers finished
  LLOG("Worker " << worker_id << " done.");
  kv.Finalize();
}

int process_program_options(const int argc, const char *const argv[]) {
  namespace po = boost::program_options;
  po::options_description desc("Allowed options");
  desc.add_options()
    ("help,h", "produce help message")
    ("dataset,d", po::value<std::string>(&dataset), "Dataset to train from")
    ("num_rows", po::value<long>(&num_rows), "number of rows in the dataset")
    ("num_cols", po::value<long>(&num_cols), "number of columns in the dataset")
    ("rank,r", po::value<uint>(&mf_rank), "Rank of matrix factorization")
    ("epochs,e", po::value<int>(&epochs), "Number of epochs to run")
    ("lambda,l", po::value<double>(&lambda)->default_value(0.05), "Regularization parameter lambda")
    ("eps", po::value<double>(&initial_step_size)->default_value(0.001), "Initial step size")
    ("bold_driver", po::value<bool>(&bold_driver)->default_value(true), "Use bold driver for step size selection")
    ("increase_step_factor", po::value<double>(&increase_step_factor)->default_value(1.05), "Factor to increase step size after successful epoch")
    ("decrease_step_factor", po::value<double>(&decrease_step_factor)->default_value(0.5), "Factor to decrease step size after unsuccessful epoch")
    ("algorithm", po::value<Alg>(&algorithm)->default_value(Alg::dsgd), "which algorithm to use. options: (1) dsgd = DSGD, (2) columnwise = access data points by column, (3) plain_sgd")
    ("signal_intent_rows", po::value<bool>(&signal_intent_rows)->default_value(true), "whether to signal intent for row parameters (default: on, i.e., do localize row parameters)")
    ("signal_intent_cols", po::value<size_t>(&signal_intent_cols)->default_value(1000), "whether to signal intent for column parameters (0: no signal, N>0 for dsgd: signal at block start, N>0 for colwise+plain: signal N data points ahead of time)")
    ("num_threads,t", po::value<int>(&num_threads)->default_value(1), "number of worker threads to run")
    ("wor_blocks", po::value<bool>(&use_wor_block_schedule)->default_value(true), "use WOR schedule for blocks")
    ("wor_points", po::value<bool>(&use_wor_point_schedule)->default_value(true), "use WOR schedule for data points")
    ("sync_push", po::value<bool>(&sync_push)->default_value(false), "synchronous or asynchronous (default) pushes")
    ("compute_loss", po::value<bool>(&compute_loss)->default_value(true), "compute loss")
    ("init_parameters", po::value<int>(&init_parameters)->default_value(2), "how to initialize parameters. 0: no init, 1: read factors from files, 2: draw random factors")
    ("model_seed", po::value<unsigned>(&model_seed)->default_value(134827), "seed for model generation")
    ("enforce_random_keys", po::value<bool>(&enforce_random_keys)->default_value(false), "enforce that keys are assigned randomly")
    ("enforce_full_replication", po::value<bool>(&enforce_full_replication)->default_value(false), "manually enforce full model replication")
    ("early_stop", po::value<unsigned long>(&early_stop)->default_value(0), "stop an epoch early after N data points (for debugging)")
    ("max_runtime", po::value<long>(&max_runtime)->default_value(std::numeric_limits<long>::max()), "set a maximum run tim, after which the job will be terminated (in seconds)")
    ("prevent_full_model_pull", po::value<bool>(&prevent_full_model_pull)->default_value(false), "prevent a full model pull (slower, but makes it possible to run large models in memory constrained settings)")
    ("write_generated_factors", po::value<std::string>(&write_generated_factors)->default_value(""), "Whether and to which path to write generated initial factors (default: empty, i.e., don't write factors)")
    ;

  // add system options
  ServerT::AddSystemOptions(desc);

  po::variables_map vm;
  po::store(po::parse_command_line(argc, argv, desc), vm);
  po::notify(vm);

  if (vm.count("help")) {
    cout << desc << "\n";
    return 1;
  }

  if (!vm.count("dataset") || !vm.count("rank") || !vm.count("epochs")) {
    cout << "Either the dataset, the MF rank, or the number of epochs to run was not specified. Usage:\n\n";
    cout << desc << "\n";
    return 1;
  }

  return 0;
}

int main(int argc, char *argv[]) {
  // Read cmd arguments
  int po_error = process_program_options(argc, argv);
  if(po_error) return 1;
  num_keys = num_cols + num_rows + 50000;

  Setup(num_keys, num_threads);

  std::string role = std::string(getenv("DMLC_ROLE"));
  ALOG("mf. " << role << ": " << epochs << " epochs on " << dataset <<
       "\nsignal intent rows " << signal_intent_rows <<
       ", cols " << signal_intent_cols);

  // calculate H key offset
  num_workers = atoi(Environment::Get()->find("DMLC_NUM_SERVER")) * num_threads;
  first_col_key = round(ceil(1.0*num_rows/num_workers)) * num_workers;

  // we store parameters and AdaGrad values in the same vector.
  // 0..mf_rank: parameters; mf_rank..param_len: AdaGrad values
  param_len = mf_rank * 2;

  // enforce random parameter allocation
  if (enforce_random_keys) {
    key_assignment.resize(num_keys-1); // exclude the loss key
    iota(key_assignment.begin(), key_assignment.end(), 0);
    srand(2); // enforce same seed among different ranks
    random_shuffle(key_assignment.begin(), key_assignment.end());
  }

  if (role.compare("scheduler") == 0) {
    Scheduler();
  } else if (role.compare("server") == 0) { // worker+server

    // Start the server system
    auto server = new ServerT(param_len);
    RegisterExitCallback([server](){ delete server; });

    // make sure all servers are set up
    server->Barrier();

    // run worker(s)
    std::vector<std::thread> workers {};
    for (int i=0; i!=num_threads; ++i) {
      workers.push_back(std::thread(RunWorker, i, server));
      std::string name = std::to_string(ps::MyRank())+"-worker-"+std::to_string(ps::MyRank()*num_threads + i);
      SET_THREAD_NAME((&workers[workers.size()-1]), name.c_str());
    }

    // wait for the workers to finish
    for (auto & w : workers)
      w.join();

    // stop the server
    server->shutdown();
  }
}


/**
 * Copyright (c) 2012 Andrew Prock. All rights reserved.
 * $Id: CardSet.cpp 2649 2012-06-30 04:53:24Z prock $
 */
#include <cstdio>
#include <cassert>
#include <algorithm>
#include <iostream>
#include <limits>
#include <vector>
#include <set>
#include <boost/array.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/math/special_functions/binomial.hpp>
#include <pokerstove/util/combinations.h>
#include "Rank.h"
#include "Suit.h"
#include "Card.h"
#include "CardSet.h"
#include "PokerEvaluation.h"
#include "PokerEvaluationTables.h"

using namespace std;
using namespace boost;
using namespace pokerstove;
using namespace pokerstove;

// some suit mask macros
#define SMASK(n)   (static_cast<int>(_cardmask >> (n)*Rank::NUM_RANK) & 0x1FFF)

#if 1
#define C()        (static_cast<int>(_cardmask >> (0)*Rank::NUM_RANK) & 0x1FFF)
#define D()        (static_cast<int>(_cardmask >> (1)*Rank::NUM_RANK) & 0x1FFF)
#define H()        (static_cast<int>(_cardmask >> (2)*Rank::NUM_RANK) & 0x1FFF)
#define S()        (static_cast<int>(_cardmask >> (3)*Rank::NUM_RANK) & 0x1FFF)
#else
#define C()        SMASK(0)
#define D()        SMASK(1)
#define H()        SMASK(2)
#define S()        SMASK(3)
#endif

#define RMASK()    (C() | D() | H() | S())

#define LOWBALL_ROTATE_RANKS(ranks)             \
  ((((ranks) & ~(1 << Rank::AceVal())) << 1)    \
   | (((ranks) >> Rank::AceVal()) & 0x01))

/**
 * ctors
 */

CardSet::CardSet()
    : _cardmask(0)
{}

CardSet::CardSet(const CardSet& cs)
    : _cardmask(cs._cardmask)
{}

CardSet::CardSet(const Card& c)
    : _cardmask(ONE64 << c._card)
{}

CardSet::CardSet(const string& c)
    : _cardmask(0)
{
    fromString(c);
}

string CardSet::str() const
{
    string out = "";
    uint64_t v = _cardmask;
    while (v) {
        Card card(lastbit64(v));
        out += card.str();
        v &= v - 1; // clear the least significant bit set
    }
    return out;
}

// TODO: use same loop as in ::str()
string CardSet::rankstr() const
{
    string out = "";
    for (size_t i=0; i<STANDARD_DECK_SIZE; i++)
    {
        uint64_t mask = ONE64 << i;
        if (_cardmask & mask)
        {
            Card c(i);
            out += c.rank().str();
        }
    }
    sort(out.begin(), out.end());
    return out;
}

string CardSet::toRankBitString() const
{
    string out = "";
    for (size_t i=0; i<STANDARD_DECK_SIZE; i++)
    {
        if (_cardmask & ONE64 << i)
            out += "1";
        else
            out += ".";
        if (((i+1)%Rank::NUM_RANK) == 0)
            out += " ";
    }
    return out;
}

size_t CardSet::countSuits() const
{
    int ns = 0;
    for (size_t i=0; i<Suit::NUM_SUIT; i++)
        if (SMASK(i) > 0)
            ns++;
    return ns;
}

size_t CardSet::countMaxSuit() const
{
    vector<size_t> suit(Suit::NUM_SUIT);
    suit[0] = (nRanksTable[SMASK(0)]);
    suit[1] = (nRanksTable[SMASK(1)]);
    suit[2] = (nRanksTable[SMASK(2)]);
    suit[3] = (nRanksTable[SMASK(3)]);

    return *std::max_element(suit.begin(),suit.end());
}

size_t CardSet::size() const
{
    uint64_t v = _cardmask;
    size_t c;
    for (c = 0; v; c++)
    {
        v &= v - 1; // clear the least significant bit set
    }
    return c;
}

void CardSet::fromString(const string& instr)
{
	clear ();

	for (size_t i=0; i<instr.size(); i+=2) {
        // skip whitespace
        if (instr[i] == ' ') {
            i -= 1;
            continue;
        }
        int code = Rank::rank_code(instr[i]) + Suit::suit_code(instr[i+1]) * Rank::NUM_RANK;
        uint64_t mask = (ONE64 << code);
		if (_cardmask & mask) {
			clear();  // card duplication is an error, no hand parsed
			return;
		}
        _cardmask |= mask;
    }
}

CardSet& CardSet::insert(const CardSet& c) {
  _cardmask |= c._cardmask;
  return *this;
}

CardSet CardSet::rotateSuits(int c, int d, int h, int s) const {
  return CardSet(
          static_cast<uint64_t>(suitMask(Suit::Clubs())) << Rank::NUM_RANK * c |
          static_cast<uint64_t>(suitMask(Suit::Diamonds())) << Rank::NUM_RANK * d |
          static_cast<uint64_t>(suitMask(Suit::Hearts())) << Rank::NUM_RANK * h |
          static_cast<uint64_t>(suitMask(Suit::Spades())) << Rank::NUM_RANK * s);
}

void CardSet::flipSuits() {
  *this = rotateSuits(3, 2, 1, 0);
}

CardSet CardSet::canonize() const {
  int smasks[Suit::NUM_SUIT];
  int i = 0;
  for (Suit s = Suit::begin(); s < Suit::end(); ++s)
    smasks[i++] = suitMask(s);

  sort(smasks, smasks + Suit::NUM_SUIT);

  return CardSet(
          static_cast<uint64_t>(smasks[3]) |
          static_cast<uint64_t>(smasks[2]) << Rank::NUM_RANK |
          static_cast<uint64_t>(smasks[1]) << Rank::NUM_RANK * 2 |
          static_cast<uint64_t>(smasks[0]) << Rank::NUM_RANK * 3);
}

CardSet CardSet::canonize(const CardSet& other) const
{
    CardSet cother = other.canonize();
    vector<int> perms = findSuitPermutation(other, cother);
    CardSet hand = *this;
    CardSet chand = hand.rotateSuits(perms[0],perms[1],perms[2],perms[3]);
    return chand;
}

// This is super slow, make it faster
bool CardSet::insertRanks(const CardSet& rset)
{
    // try the fastest way first
    if ((_cardmask & rset._cardmask) == 0)
    {
        _cardmask |= rset._cardmask;
        return true;
    }

    // now try shifting by flipping around suits
    for (size_t s=1; s< Suit::NUM_SUIT; s++)
    {
        CardSet flipped(rset);
        flipped.flipSuits();
        if ((_cardmask & flipped._cardmask) == 0)
        {
            _cardmask |= flipped._cardmask;
            return true;
        }
    }

    // ok, just do it the slow way
    for (Rank r=Rank::Two(); r<=Rank::Ace(); ++r)
    {
        size_t nrin = rset.count(r);
        if (nrin == 0)
            continue;

        if (count(r) + nrin > Suit::NUM_SUIT)
            return false;

        for (Suit s=Suit::Clubs(); s<=Suit::Spades(); ++s)
            //for (Suit s=Suit::Spades(); s>=Suit::Spades(); --s)
        {
            Card c(r,s);
            if (!contains(c))
            {
                insert(c);
                nrin--;
            }
            if (nrin == 0)
                break;
        }
    }
    return true;
}

CardSet CardSet::canonizeRanks() const {
    // this is very slow, optimize if it winds up in an inner loop
    CardSet ret;
    string ranks = rankstr();
    for (size_t i = 0; i < ranks.size(); i++) {
        for (Suit s = Suit::Clubs(); s <= Suit::Spades(); ++s) {
            Card candidate(Rank(ranks.substr(i, 1)), s);
            if (!ret.contains(candidate)) {
                ret.insert(candidate);
                break;
            }
        }
    }
    return ret;
}

CardSet& CardSet::insert(const Card& c)
{
    _cardmask |= (ONE64 << c.code());
    return *this;
}

vector<Card> CardSet::cards() const
{
    vector<Card> out(size());
    size_t n = 0;
    for (size_t i=0; i<STANDARD_DECK_SIZE; i++)
        if (_cardmask & (ONE64 << i))
            out[n++] = Card(i);
    return out;
}

vector<CardSet> CardSet::cardSets() const
{
    vector<CardSet> out(size());
    size_t n = 0;
    for (size_t i=0; i<STANDARD_DECK_SIZE; i++)
        if (_cardmask & (ONE64 << i))
            out[n++] = CardSet(ONE64<<i);
    return out;
}

CardSet& CardSet::remove(const CardSet& c)
{
    _cardmask ^= c._cardmask;
    return *this;
}

CardSet& CardSet::remove(const Card& c)
{
    _cardmask ^= ONE64<<c.code();
    return *this;
}

bool CardSet::contains(const CardSet& c) const
{
    return (c._cardmask == (_cardmask & c._cardmask));
}

bool CardSet::contains(const Card& c) const
{
    uint64_t bit = ONE64 << c.code();
    return ((bit & _cardmask) != 0);
}

bool CardSet::contains(const Rank& r) const
{
    return (RMASK() & r.rankBit()) > 0;
}

Card CardSet::find(const Rank& r) const
{
    for (uint8_t i=0; i<Suit::NUM_SUIT; i++)
    {
        if (SMASK(i) & r.rankBit())
            return Card(r, Suit(i));
    }
    return Card();
}

bool CardSet::contains(const Suit& s) const
{
    if (SMASK(s.code()) > 0)
        return true;
    return false;
}

int CardSet::rankMask() const
{
    return RMASK();
}

size_t CardSet::countRanks() const
{
    return nRanksTable[C() | D() | H() | S()];
}

int CardSet::suitMask(const Suit& s) const
{
    return SMASK(s.code());
}

size_t CardSet::count(const Rank& r) const
{
#if 1
    // this version is faster, if less obvious
    int c = C();
    int d = D();
    int h = H();
    int s = S();

    int four_mask = c & d & h & s;
    if (four_mask & r.rankBit())
        return 4;

    int three_mask =
        ((c&d)|(h&s)) &
        ((c&h)|(d&s));
    if (three_mask & r.rankBit())
        return 3;

    int rankmask = c | d | h | s;
    int two_mask = rankmask ^(c ^ d ^ h ^ s);
    if (two_mask & r.rankBit())
        return 2;

    if (rankmask & r.rankBit())
        return 1;

    return 0;
#else
    size_t ret = 0;
    for (size_t i=0; i<Suit::NUM_SUIT; i++)
        if (r.rankBit() & SMASK(i))
            ret++;
    return ret;
#endif
}

bool CardSet::hasStraight() const
{
    if (straightTable[RMASK()] > 0)
        return true;
    return false;
}

// this is a rough draft of the evaluateHigh routine, the idea is that
// it should be AFAP.  However, this is a general 1-7 card evaluator, so
// it cannot be too fast.  We could unwrap the method calls here for the
// PokerHand, maybe it would speed things up.
//
// note, there are no function calls in this function
PokerEvaluation CardSet::evaluateHigh() const
{
    // first the easy stuff
    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;

    if (nRanksTable[rankmask] >= 5)
    {
        int sranks = 0;
        int suitindex=-1;
        if (nRanksTable[c] >= 5)
        {
            suitindex = Suit::ClubVal();
            sranks = c;
        }
        else if (nRanksTable[d] >= 5)
        {
            suitindex = Suit::DiamondVal();
            sranks = d;
        }
        else if (nRanksTable[h] >= 5)
        {
            suitindex = Suit::HeartVal();
            sranks = h;
        }
        else if (nRanksTable[s] >= 5)
        {
            suitindex = Suit::SpadeVal();
            sranks = s;
        }
        if (suitindex >= 0)
        {
            int strval = straightTable[sranks];
            if (strval > 0)
                return PokerEvaluation((STRAIGHT_FLUSH<<VSHIFT) ^ strval<<MAJOR_SHIFT);
            else
                return PokerEvaluation((FLUSH<<VSHIFT) ^ topFiveRanksTable[sranks]);
        }
        int strval = straightTable[rankmask];
        if (strval > 0)
            return PokerEvaluation((STRAIGHT<<VSHIFT) ^(strval<<MAJOR_SHIFT));
    }

    int ncards = nRanksTable[c] + nRanksTable[d] + nRanksTable[h] + nRanksTable[s];
    int ndups = ncards - nRanksTable[rankmask];

    switch (ndups)
    {
        case 0:     // no pair
        {
            return PokerEvaluation((NO_PAIR<<VSHIFT) ^ topFiveRanksTable[rankmask]);
        }
        break;

        case 1:     // one pair
        {
            int two_mask = rankmask ^(c ^ d ^ h ^ s);
            int topind = topRankTable[two_mask];
            int kickers = topThreeRanksTable[rankmask ^(0x01<<topind)];
            return PokerEvaluation((ONE_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^ kickers);
        }
        break;

        case 2:
        {
            int two_mask = rankmask ^(c ^ d ^ h ^ s);

            if (two_mask)   // two pair
            {
                int topind = topRankTable[two_mask];
                int botind = botRankTable[two_mask];
                int kicker = topRankTable[rankmask ^ two_mask];
                if (kicker >= 0)
                    return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT) ^(0x01<<kicker));
                else
                    return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
            }
            else
            {
                int three_mask =
                    ((c&d)|(h&s)) &
                    ((c&h)|(d&s));
                int topind = topRankTable[three_mask];
                int kickers = rankmask ^(0x01<<topind);
                int kbits = 0;
                if (kickers > 0)
                    kbits   = 0x01<<topRankTable[kickers];
                if (kbits >= 0 && ((kickers^kbits) > 0))
                    kbits      ^= 0x01<<topRankTable[kickers^kbits];
                return PokerEvaluation((THREE_OF_A_KIND<<VSHIFT) ^(topind << MAJOR_SHIFT) ^ kbits);
            }
        }
        break;

        default:
        {
            int four_mask = c & d & h & s;
            if (four_mask)
            {
                int topind = topRankTable[four_mask];
                int kicker = rankmask;
                kicker ^= (0x01<<topind);
                kicker  = topRankTable[kicker];
                if (kicker >= 0)
                    return PokerEvaluation((FOUR_OF_A_KIND<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(0x01<<kicker));
                else
                    return PokerEvaluation((FOUR_OF_A_KIND<<VSHIFT) ^(topind<<MAJOR_SHIFT));
            }

            int two_mask = rankmask ^(c ^ d ^ h ^ s);
            if (nRanksTable[two_mask] != ndups)
            {
                int three_mask =
                    ((c&d)|(h&s)) &
                    ((c&h)|(d&s));
                int topind = topRankTable[three_mask];
                if (two_mask > 0)
                {
                    int botind = topRankTable[two_mask];
                    return PokerEvaluation((FULL_HOUSE<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
                }
                else
                {
                    int botind = topRankTable[three_mask ^ 0x01<<topind];
                    return PokerEvaluation((FULL_HOUSE<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
                }
            }

            int topind = topRankTable[two_mask];
            int botind = topRankTable[two_mask ^ 0x01<<topind];
            int kicker = rankmask ^ 0x01<<topind ^ 0x01<<botind;
            kicker  = topRankTable[kicker];
            if (kicker >= 0)
                return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT) ^(0x01<<kicker));
            else
                return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
        }
        break;
    }

    cerr << "oops\n";
    return PokerEvaluation(0);
}

// same as evaluateHigh, but with the non-flush logic
// elided
PokerEvaluation CardSet::evaluateHighFlush() const
{
    // first the easy stuff
    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;

    if (nRanksTable[rankmask] >= 5)
    {
        int sranks = 0;
        int suitindex=-1;
        if (nRanksTable[c] >= 5)
        {
            suitindex = Suit::ClubVal();
            sranks = c;
        }
        else if (nRanksTable[d] >= 5)
        {
            suitindex = Suit::DiamondVal();
            sranks = d;
        }
        else if (nRanksTable[h] >= 5)
        {
            suitindex = Suit::HeartVal();
            sranks = h;
        }
        else if (nRanksTable[s] >= 5)
        {
            suitindex = Suit::SpadeVal();
            sranks = s;
        }
        if (suitindex >= 0)
        {
            int strval = straightTable[sranks];
            if (strval > 0)
                return PokerEvaluation((STRAIGHT_FLUSH<<VSHIFT) ^ strval<<MAJOR_SHIFT);
            else
                return PokerEvaluation((FLUSH<<VSHIFT) ^ topFiveRanksTable[sranks]);
        }
    }
    return PokerEvaluation(0);
}

// evaluteHighRanks, same as evaluateHigh, but with the flush logic
// elided
PokerEvaluation CardSet::evaluateHighRanks() const
{
    // first the easy stuff
    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;

    if (nRanksTable[rankmask] >= 5)
    {
        int strval = straightTable[rankmask];
        if (strval > 0)
            return PokerEvaluation((STRAIGHT<<VSHIFT) ^(strval<<MAJOR_SHIFT));
    }

    int ncards = nRanksTable[c] + nRanksTable[d] + nRanksTable[h] + nRanksTable[s];
    int ndups = ncards - nRanksTable[rankmask];

    switch (ndups)
    {
        case 0:     // no pair
        {
            return PokerEvaluation((NO_PAIR<<VSHIFT) ^ topFiveRanksTable[rankmask]);
        }
        break;

        case 1:     // one pair
        {
            int two_mask = rankmask ^(c ^ d ^ h ^ s);
            int topind = topRankTable[two_mask];
            int kickers = topThreeRanksTable[rankmask ^(0x01<<topind)];
            return PokerEvaluation((ONE_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^ kickers);
        }
        break;

        case 2:
        {
            int two_mask = rankmask ^(c ^ d ^ h ^ s);

            if (two_mask)   // two pair
            {
                int topind = topRankTable[two_mask];
                int botind = botRankTable[two_mask];
                int kicker = topRankTable[rankmask ^ two_mask];
                if (kicker >= 0)
                    return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT) ^(0x01<<kicker));
                else
                    return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
            }
            else
            {
                int three_mask =
                    ((c&d)|(h&s)) &
                    ((c&h)|(d&s));
                int topind = topRankTable[three_mask];
                int kickers = rankmask ^(0x01<<topind);
                int kbits = 0;
                if (kickers > 0)
                    kbits   = 0x01<<topRankTable[kickers];
                if (kbits >= 0 && ((kickers^kbits) > 0))
                    kbits      ^= 0x01<<topRankTable[kickers^kbits];
                return PokerEvaluation((THREE_OF_A_KIND<<VSHIFT) ^(topind << MAJOR_SHIFT) ^ kbits);
            }
        }
        break;

        default:
        {
            int four_mask = c & d & h & s;
            if (four_mask)
            {
                int topind = topRankTable[four_mask];
                int kicker = rankmask;
                kicker ^= (0x01<<topind);
                kicker  = topRankTable[kicker];
                if (kicker >= 0)
                    return PokerEvaluation((FOUR_OF_A_KIND<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(0x01<<kicker));
                else
                    return PokerEvaluation((FOUR_OF_A_KIND<<VSHIFT) ^(topind<<MAJOR_SHIFT));
            }

            int two_mask = rankmask ^(c ^ d ^ h ^ s);
            if (nRanksTable[two_mask] != ndups)
            {
                int three_mask =
                    ((c&d)|(h&s)) &
                    ((c&h)|(d&s));
                int topind = topRankTable[three_mask];
                if (two_mask > 0)
                {
                    int botind = topRankTable[two_mask];
                    return PokerEvaluation((FULL_HOUSE<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
                }
                else
                {
                    int botind = topRankTable[three_mask ^ 0x01<<topind];
                    return PokerEvaluation((FULL_HOUSE<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
                }
            }

            int topind = topRankTable[two_mask];
            int botind = topRankTable[two_mask ^ 0x01<<topind];
            int kicker = rankmask ^ 0x01<<topind ^ 0x01<<botind;
            kicker  = topRankTable[kicker];
            if (kicker >= 0)
                return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT) ^(0x01<<kicker));
            else
                return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
        }
        break;
    }

    cerr << "oops\n";
    return PokerEvaluation(0);
}

// this is just evaluateHigh without the straight or flush code
PokerEvaluation CardSet::evaluatePairing() const
{
    // first the easy stuff
    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;
    int ncards = nRanksTable[c] + nRanksTable[d] + nRanksTable[h] + nRanksTable[s];
    int ndups = ncards - nRanksTable[rankmask];

    switch (ndups)
    {
        case 0:     // no pair
        {
            return PokerEvaluation((NO_PAIR<<VSHIFT) ^ topFiveRanksTable[rankmask]);
        }
        break;

        case 1:     // one pair
        {
            int two_mask = rankmask ^(c ^ d ^ h ^ s);
            int topind = topRankTable[two_mask];
            int kickers = topThreeRanksTable[rankmask ^(0x01<<topind)];
            return PokerEvaluation((ONE_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^ kickers);
        }
        break;

        case 2:
        {
            int two_mask = rankmask ^(c ^ d ^ h ^ s);

            if (two_mask)   // two pair
            {
                int topind = topRankTable[two_mask];
                int botind = botRankTable[two_mask];
                int kicker = topRankTable[rankmask ^ two_mask];
                if (kicker >= 0)
                    return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT) ^(0x01<<kicker));
                else
                    return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
            }
            else
            {
                int three_mask =
                    ((c&d)|(h&s)) &
                    ((c&h)|(d&s));
                int topind = topRankTable[three_mask];
                int kickers = rankmask ^(0x01<<topind);
                int kbits = 0;
                if (kickers > 0)
                    kbits   = 0x01<<topRankTable[kickers];
                if (kbits >= 0 && ((kickers^kbits) > 0))
                    kbits      ^= 0x01<<topRankTable[kickers^kbits];
                return PokerEvaluation((THREE_OF_A_KIND<<VSHIFT) ^(topind << MAJOR_SHIFT) ^ kbits);
            }
        }
        break;

        default:
        {
            int four_mask = c & d & h & s;
            if (four_mask)
            {
                int topind = topRankTable[four_mask];
                int kicker = rankmask;
                kicker ^= (0x01<<topind);
                kicker  = topRankTable[kicker];
                if (kicker >= 0)
                    return PokerEvaluation((FOUR_OF_A_KIND<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(0x01<<kicker));
                else
                    return PokerEvaluation((FOUR_OF_A_KIND<<VSHIFT) ^(topind<<MAJOR_SHIFT));
            }

            int two_mask = rankmask ^(c ^ d ^ h ^ s);
            if (nRanksTable[two_mask] != ndups)
            {
                int three_mask =
                    ((c&d)|(h&s)) &
                    ((c&h)|(d&s));
                int topind = topRankTable[three_mask];
                if (two_mask > 0)
                {
                    int botind = topRankTable[two_mask];
                    return PokerEvaluation((FULL_HOUSE<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
                }
                else
                {
                    int botind = topRankTable[three_mask ^ 0x01<<topind];
                    return PokerEvaluation((FULL_HOUSE<<VSHIFT) ^(topind<<MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
                }
            }

            int topind = topRankTable[two_mask];
            int botind = topRankTable[two_mask ^ 0x01<<topind];
            int kicker = rankmask ^ 0x01<<topind ^ 0x01<<botind;
            kicker  = topRankTable[kicker];
            if (kicker >= 0)
                return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT) ^(0x01<<kicker));
            else
                return PokerEvaluation((TWO_PAIR<<VSHIFT) ^(topind << MAJOR_SHIFT) ^(botind << MINOR_SHIFT));
        }
        break;
    }

    cerr << "oops\n";
    return PokerEvaluation(0);
}

// This one is not fully optimized
PokerEvaluation CardSet::evaluateLow2to7() const
{
    PokerEvaluation high;

    // if there are five or fewer cards, we just evaluate the high,
    // fix the wheel, and take the complement
    switch (size())
    {
        case 0:
        case 1:
        case 2:
        case 3:
        case 4:
            high = evaluateHigh();
            break;

        case 5:
            high = evaluateHigh();
            high.fixWheel2to7(rankMask());
            break;

        default:
            // this is a slow way to handle the general case.
            // TODO: specialize the code for the 6 and 7 card cases.
            vector<Card> cards = this->cards();
            combinations combo(size(), FULL_HAND_SIZE);
            PokerEvaluation best;
            do
            {
                CardSet candidate;
                for (size_t i=0; i<static_cast<size_t>(FULL_HAND_SIZE); i++)
                    candidate.insert(cards[combo[i]]);
                PokerEvaluation e = candidate.evaluateLow2to7();
                if (e > best)
                    best = e;
            }
            while (combo.next());
            return best;

    }

    high.flip();
    return high;
}

// This one is not fully optimized
PokerEvaluation CardSet::evaluateRanksLow2to7() const
{
    PokerEvaluation high;
    PokerEvaluation h;

    // if there are five or fewer cards, we just evaluate the high,
    // fix the wheel, and take the complement
    switch (size())
    {
        case 0:
        case 1:
        case 2:
        case 3:
        case 4:
            high = evaluateHighRanks();
            break;

        case 5:
            high = evaluateHighRanks();
            high.fixWheel2to7(rankMask());
            break;

        default:
            // this is a slow way to handle the general case.
            // TODO: specialize the code for the 6 and 7 card cases.
            vector<Card> cards = this->cards();
            combinations combo(size(), FULL_HAND_SIZE);
            PokerEvaluation best;
            do
            {
                CardSet candidate;
                for (size_t i=0; i<static_cast<size_t>(FULL_HAND_SIZE); i++)
                    candidate.insert(cards[combo[i]]);
                PokerEvaluation e = candidate.evaluateRanksLow2to7();
                if (e > best)
                    best = e;
            }
            while (combo.next());
            return best;

    }

    high.flip();
    return high;
}

// This one is not fully optimized
PokerEvaluation CardSet::evaluateSuitsLow2to7() const
{
    PokerEvaluation high;

    // if there are five or fewer cards, we just evaluate the high,
    // fix the wheel, and take the complement
    switch (size())
    {
        case 0:
        case 1:
        case 2:
        case 3:
        case 4:
            high = evaluateHighFlush();
            break;

        case 5:
            high = evaluateHighFlush();
            high.fixWheel2to7(rankMask());
            break;

        default:
            // this is a slow way to handle the general case.
            // TODO: specialize the code for the 6 and 7 card cases.
            vector<Card> cards = this->cards();
            combinations combo(size(), FULL_HAND_SIZE);
            PokerEvaluation best;
            do
            {
                CardSet candidate;
                for (size_t i=0; i<static_cast<size_t>(FULL_HAND_SIZE); i++)
                    candidate.insert(cards[combo[i]]);
                PokerEvaluation e = candidate.evaluateSuitsLow2to7();
                if (e > best)
                    best = e;
            }
            while (combo.next());
            return best;

    }

    high.flip();
    return high;
}

// This one is not fully optimized
PokerEvaluation CardSet::evaluateLowA5() const
{
    //PokerEvaluation::generateLowballLookupA5();

    // this is a rank only evaluator, so we just get
    // the rank masks and then fill accordingly.  We
    // also have to shift the rank mask according to the
    // the fact that the ace swings low
    // this could probably be much faster

    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;

    int ncards = nRanksTable[c] + nRanksTable[d] + nRanksTable[h] + nRanksTable[s];
    int ndups = ncards - nRanksTable[rankmask];
    int nranks = nRanksTable[rankmask];

    // first the easy cases
    // 1) no duplicate ranks
    // 2) 5 or more ranks
    if (ndups == 0 || nranks >= FULL_HAND_SIZE)
    {
        PokerEvaluation ret((NO_PAIR<<VSHIFT) ^ lowballA5Ranks[rankmask] ^ ACE_LOW_BIT);
        ret.flip();
        return ret;
    }
    // another easy case
    // *) we have five or fewer cards
    else if (ncards <= FULL_HAND_SIZE)
    {
        PokerEvaluation ret = evaluatePairing();
        ret.playAceLow();
        ret.flip();
        return ret;
    }
    else
    {
        // now the general case
        // the algorithm is as follows:
        // cycle through the "pairing list" till a total of five
        // cards are found
        uint64_t chash = 0;// = rankmask;
        int rset[4];

        rset[0]  = rankmask;
        rset[1]  = rankmask ^(c ^ d ^ h ^ s);             // mask of cards with 2/4 ranks
        rset[2]  = ((c&d)|(h&s)) & ((c&h)|(d&s));         // mask of cards with 3 ranks
        rset[3]  =  c & d & h & s;                        // fourmask
        rset[1] |= rset[2];       // add the threes back to teh twos

        rset[0] = LOWBALL_ROTATE_RANKS(rset[0]);
        rset[1] = LOWBALL_ROTATE_RANKS(rset[1]);
        rset[2] = LOWBALL_ROTATE_RANKS(rset[2]);
        rset[3] = LOWBALL_ROTATE_RANKS(rset[3]);

        // we add pairs first, then trips, then quads
        int nc=0;
        for (int i=0; i<4; i++)
        {
            // do the ace first
            if (rset[i] & (0x01<<Rank::AceVal()))
            {
                chash |= (ONE64<<(Rank::AceVal()+Rank::NUM_RANK*i));
                nc++;
            }
            if (nc == FULL_HAND_SIZE)
            {
                break;
            }

            // now the rest of the ranks
            for (int r=0; r<11; r++)
            {
                if (rset[i] & (0x01<<r))
                {
                    chash |= (ONE64<<(r+Rank::NUM_RANK*i));
                    nc++;
                }
                if (nc == FULL_HAND_SIZE)
                {
                    break;
                }
            }
            if (nc == FULL_HAND_SIZE)
            {
                break;
            }
        }

        CardSet paird(chash);
        PokerEvaluation ret(paird.evaluatePairing().code() | ACE_LOW_BIT);
        ret.flip();
        return ret;
    }
}

PokerEvaluation CardSet::evaluate8LowA5() const
{
    // bits 2-8+A are set here
    const int LOW_MASK = 0x107F;

    int c = C() & LOW_MASK;
    int d = D() & LOW_MASK;
    int h = H() & LOW_MASK;
    int s = S() & LOW_MASK;

    int rankmask = c | d | h | s;
    int nranks = nRanksTable[rankmask];

    if (nranks >= FULL_HAND_SIZE)
    {
        PokerEvaluation ret(((NO_PAIR<<VSHIFT) ^ lowballA5Ranks[rankmask]) | ACE_LOW_BIT);
        ret.flip();
        return ret;
    }

    PokerEvaluation e;
    return e;
}

PokerEvaluation CardSet::evaluate3CP() const
{
    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;

    int ncards = nRanksTable[c] + nRanksTable[d] + nRanksTable[h] + nRanksTable[s];
    if (ncards > 3)
        throw std::invalid_argument("3CP evaluator only works on three cards");

    // check for three flush
    bool threeflush = countSuits() == 1 ? true : false;

    // then three straight
    const int THREE_WHEEL = 0x01<<Rank::AceVal() | 0x01<<Rank::TwoVal() | 0x01<<Rank::ThreeVal();
    int  topr = topRankTable[rankmask];
    int  botr = botRankTable[rankmask];
    int  strr = -1;
    bool threestr = false;
    if (topr - botr == 2 && nRanksTable[rankmask] == 3)
    {
        threestr = true;
        strr = topr;
    }
    else if (rankmask == THREE_WHEEL)
    {
        threestr = true;
        strr = Rank::ThreeVal();
    }

    if (threeflush)
    {
        if (threestr)
            return PokerEvaluation((THREE_STRAIGHT_FLUSH<<VSHIFT) ^ strr<<MAJOR_SHIFT);
        else
            return PokerEvaluation((THREE_FLUSH<<VSHIFT) ^ rankmask);
    }
    else if (threestr)
    {
        return PokerEvaluation((THREE_STRAIGHT<<VSHIFT) ^(strr<<MAJOR_SHIFT));
    }
    else
    {
        return evaluatePairing();
    }
}

// three card badugis are better than two card badugis
static inline bool badugiless(int c1, int c2)
{
    int n1 = nRanksTable[c1];
    int n2 = nRanksTable[c2];
    if (n1 == n2)
        return c1 < c2;
    return n1 > n2;
}

/**
 * Badugi hand evaluator. To make this faster, I need a
 * bottomRankMask.
 *
 * To do a four card evaluation it suffices to pass through the suits
 * twice, once in each direction.  To do a general evaluation, all
 * suits must be traversed in all orders so that 4*3*2*1=24 suit
 * traversals will be done.  Suits of size zero can be ruled out and
 * those with size one can be ruled in.
 */
PokerEvaluation CardSet::evaluateBadugi() const
{
    // get our ranks orgainzed in lowball order by suit
    boost::array<int,4> suits =
    {
        {
            LOWBALL_ROTATE_RANKS(C()),
            LOWBALL_ROTATE_RANKS(D()),
            LOWBALL_ROTATE_RANKS(H()),
            LOWBALL_ROTATE_RANKS(S())
        }
    };

    // We try to save some time by being smart about which suits we loop
    // over.  Empty suits are ignored, and suits with one rank are used
    // as is.  At the end we sort to make next_permutation to work properly.
    boost::array<int,4> ind;
    int bmust = 0;
    size_t k=0;
    for (size_t i=0; i<suits.size(); i++)
    {
        switch (nRanksTable[suits[i]])
        {
            case 1:
                bmust |= suits[i];
                // fall through
            case 0:
                break;
            default:
                ind[k++] = suits[i];
        };
    }
    sort(ind.begin(),ind.begin()+k);

    // now enter into the fray to find the best possible badugi.  suit
    // order in this loop matters, so we cycle through the permutations
    // of suit order.
    //
    // zeroing minbadugi works because it has no ranks and a one rank
    // badugi is less than a zero rank badugi.
    int minbadugi = 0;
    do
    {
        // explanation of bit ops, b=current badugi, s=suit being considered
        // (b|s)-b                    set of ranks in suit s not in the badugi yet
        // b |= botRankMask[(b|s)-b]  for each relevant suit do this:
        int branks = bmust;
        for (size_t i=0; i<k; i++)
            branks |= bottomRankMask[(branks|ind[i])-branks];

        if (badugiless(branks, minbadugi))
            minbadugi = branks;
    }
    while (next_permutation(ind.begin(),ind.begin()+k));

    // encode the badugi, it's a low hand where aces plays low.  we set
    // the number of missing suits in the badugi as the "major rank". the
    // value is chosen so that more cards maps to a lower value
    PokerEvaluation ret(((NO_PAIR<<VSHIFT) ^ minbadugi) | ACE_LOW_BIT);
    ret.setMajorRank(Suit::NUM_SUIT - nRanksTable[minbadugi]);
    ret.flip();
    return ret;
}

bool CardSet::isPaired() const  // returns true if *any* two cards match rank
{
    int c = C();
    int d = D();
    int h = H();
    int s = S();
    int rankmask = c | d | h | s;

    int two_mask = rankmask ^(c ^ d ^ h ^ s);
    if (two_mask > 0)
        return true;
    return false;
}

size_t CardSet::countMaxRank() const         // returns true if trips or quads
{
    int c = C();
    int d = D();
    int h = H();
    int s = S();

    int four_mask = c & d & h & s;
    if (four_mask)
        return 4;

    int three_mask =
        ((c&d)|(h&s)) &
        ((c&h)|(d&s));
    if (three_mask)
        return 3;

    int rankmask = c | d | h | s;
    int two_mask = rankmask ^(c ^ d ^ h ^ s);
    if (two_mask)
        return 2;

    if (rankmask > 0)
        return 1;

    return 0;
}

bool CardSet::isTripped() const         // returns true if trips or quads
{
    int c = C();
    int d = D();
    int h = H();
    int s = S();

    int three_mask =
        ((c&d)|(h&s)) &
        ((c&h)|(d&s));

    if (three_mask > 0)
        return true;
    return false;
}

size_t CardSet::count(const Suit& s) const
{
    return nRanksTable[SMASK(s.code())];
}

Rank CardSet::flushRank(const Suit& s) const
{
    return Rank(topRankTable[SMASK(s.code())]);
}

Rank CardSet::topRank() const
{
    return Rank(topRankTable[RMASK()]);
}

Rank CardSet::bottomRank() const
{
    return Rank(botRankTable[RMASK()]);
}

double clampedChoose(int n, int m)
{
    if (n < m)
        return 0;
    return static_cast<size_t>(boost::math::binomial_coefficient<double>(n,m));
}

int CardSet::evaluateStraightOuts() const
{
    int sval = straightTable[RMASK()];
    if (sval > 0)         // straight on board, all cards make straight
        return STANDARD_DECK_SIZE;
    if (sval == -2)       // open-ended, eight outs
        return 8;
    if (sval == -1)       // gut-shot, four outs
        return 4;
    if (sval == -3)       // runner-runner, "1" out
        return 1;
    return 0;
}

// original version in r2488
size_t CardSet::rankColex() const
{
    size_t ret  = 0;
    int slot = 0;
    int sz   = 1;
    int c    = C();
    int d    = D();
    int h    = H();
    int s    = S();
    int rbit = 0x01;

    for (int i=0; i<13; i++)
    {
        if (rbit&c) ret += clampedChoose(slot++,sz++);
        if (rbit&d) ret += clampedChoose(slot++,sz++);
        if (rbit&h) ret += clampedChoose(slot++,sz++);
        if (rbit&s) ret += clampedChoose(slot++,sz++);
        slot++;
        rbit <<= 1;
    }
    return ret;
}

std::ostream& operator<<(std::ostream& sout, const pokerstove::CardSet& e)
{
    sout << e.str();
    return sout;
}

vector<int> pokerstove::findSuitPermutation(const CardSet& source, const CardSet& dest) {
    vector<int> rot(4, -1);
    vector<int> taken(4, 0);

    int i = 0;
    for (Suit s = Suit::begin(); s < Suit::end(); ++s, ++i) {
        int j = 0;
        for (Suit t = Suit::begin(); t < Suit::end(); ++t, ++j) {
            if (source.suitMask(s) == dest.suitMask(t)) {
                if (taken[j] == 0) {
                    rot[i] = j;
                    taken[j] = 1;
                    break;
                }
            }
        }
    }
    return rot;
}

CardSet pokerstove::canonizeToBoard(const CardSet& board, const CardSet& hand)
{
    CardSet cboard = board.canonize();
    vector<int> perms = findSuitPermutation(board, cboard);
    CardSet chand = hand.rotateSuits(perms[0],perms[1],perms[2],perms[3]);
    return chand;
}

// some suit mask macros
#undef C
#undef D
#undef H
#undef S
#undef RMASK
#undef SUITMASK

size_t CardSet::colex() const {
    vector<Card> cards = this->cards();
    size_t value = 0;
    for (size_t i = 0; i < cards.size(); i++) {
        size_t code = cards[i].code();
#ifdef __AP_MATH__
        value += tchoose(code, i+1);
#else
        if (code >= i + 1)
            value += static_cast<size_t>(boost::math::binomial_coefficient<double>(code, i + 1));
#endif
    }
    return value;
}


#ifndef DETAIL_VALUE_WITH_ERROR_HPP
#define DETAIL_VALUE_WITH_ERROR_HPP

#include <cmath>
#include <iostream>
#include <iomanip>
#include <boost/math/special_functions/digamma.hpp>
#include <boost/math/special_functions/hypot.hpp>
#include <boost/math/tools/promotion.hpp>
#include <boost/type_traits/has_less.hpp>

#include "fwd.hpp" // for DEFAULT_POLICY_CLASS define

#if __cplusplus >= 201103L

  #include <type_traits>
  #include <initializer_list>

  // CONST is being mapped to constexpr (C++11) or const (C++98)
  #ifdef BOOST_NO_CXX11_CONSTEXPR
    #define CONSTEXPR
    #define CONST const
  #else
    #define CONST constexpr
    #define CONSTEXPR constexpr
  #endif
#else
  #include <boost/type_traits.hpp>
  #define CONST const
  #define CONSTEXPR
#endif // __cplusplus >= 201103L

namespace error_propagation {
  namespace detail {

    using boost::math::tools::promote_args;

    typedef boost::math::policies::policy<
                                          boost::math::policies::domain_error<boost::math::policies::errno_on_error>,
                                          boost::math::policies::pole_error<boost::math::policies::errno_on_error>,
                                          boost::math::policies::overflow_error<boost::math::policies::errno_on_error>,
                                          boost::math::policies::evaluation_error<boost::math::policies::errno_on_error>
                                         > policy_errno_on_error;

    // Only use hypot for types which support std::less
    template<typename T, typename U, bool has_less_op>
    struct hypot_impl;

    template<typename T, typename U>
    struct hypot_impl<T, U, false>
    {
      CONSTEXPR typename promote_args<T, U>::type
      operator()(const T& a, const U& b) const
      {
        using std::sqrt;
        return sqrt(a*a + b*b);
      }
    };

    template<typename T, typename U>
    struct hypot_impl<T, U, true>
    {
      CONSTEXPR typename promote_args<T, U>::type
      operator()(const T& a, const U& b) const
      {
        // we don't use std::hypot here as that does not know how to work with boost::multiprecision
        return boost::math::hypot(a,b,policy_errno_on_error());
      }
    };

    template<typename T, typename U>
    CONSTEXPR typename promote_args<T,U>::type
    hypot(const T& a, const U& b)
    {
      return hypot_impl<T, U, boost::has_less<T,U,bool>::value>()(a,b);
    }

    template<typename T>
    CONSTEXPR T digamma(const T& a)
    {
      return boost::math::digamma(a,policy_errno_on_error());
    }

  } // namespace detail
} // namespace error_propagation

#endif // DETAIL_VALUE_WITH_ERROR_HPP


/**
 *  biascorrection.cpp
 *  MetaQuant
 *
 *  Kevin McLoughlin
 *  Based on code from eXpress, created by Adam Roberts in 2011.
 *  Copyright 2014 Kevin McLoughlin, Adam Roberts. All rights reserved.
 *
 */

#include <algorithm>
#include <boost/assign.hpp>
#include <cassert>

#include "biascorrection.h"

#include "fragments.h"
#include "frequencymatrix.h"
#include "main.h"
#include "sequence.h"
#include "targets.h"

using namespace std;

// size of bias window
const int WINDOW = 21;
// center position in window
const int CENTER = 11;
// number of bases on either side of center
const int SURROUND = 10;

SeqWeightTable::SeqWeightTable(size_t window_size, size_t order, double alpha)
    : _order(order),
      _observed(order, window_size, window_size, alpha),
      _expected(order, window_size, order+1, EPSILON) {
}

SeqWeightTable::SeqWeightTable(size_t window_size, size_t order,
                               string param_file_name, string identifier)
    : _order(order),
      _observed(order, window_size, window_size, 0),
      _expected(order, window_size, order+1, 0) {

  //TODO: Allow for orders and window sizes to be read from param file.

  ifstream infile (param_file_name.c_str());
  const size_t BUFF_SIZE = 99999;
  char line_buff[BUFF_SIZE];
  if (!infile.is_open()) {
    logger.severe("Unable to open parameter file '%s'.",
                  param_file_name.c_str());
  }

  do {
    infile.getline (line_buff, BUFF_SIZE, '\n');
  } while (strncmp(line_buff, identifier.c_str(), identifier.size()));
  infile.getline (line_buff, BUFF_SIZE, '\n');

  do {
    infile.getline (line_buff, BUFF_SIZE, '\n');
  } while (strcmp(line_buff, "\tObserved Conditional Probabilities"));
  infile.getline (line_buff, BUFF_SIZE, '\n');

  //FG
  for (size_t k = 0; k < (size_t)WINDOW; ++k) {
    infile.getline (line_buff, BUFF_SIZE, '\n');
    char *p = strtok(line_buff, "\t");
    for (size_t i=0; i < pow(4.0, (double)min(order, k)); ++i) {
      for (size_t j=0; j < NUM_NUCS; ++j) {
        p = strtok(NULL, "\t");
        _observed.update(k, i, j, log(strtod(p,NULL)));
      }
    }
  }

  infile.getline (line_buff, BUFF_SIZE, '\n');
  infile.getline (line_buff, BUFF_SIZE, '\n');
  //BG
  for (size_t k = 0; k < order+1; ++k) {
    infile.getline (line_buff, BUFF_SIZE, '\n');
    char *p = strtok(line_buff, "\t");
    for (size_t i=0; i < pow(4.0, (double)min(order, k)); ++i) {
      for (size_t j=0; j < NUM_NUCS; ++j) {
        p = strtok(NULL, "\t");
        _expected.update(k, i, j, log(strtod(p,NULL)));
      }
    }
  }
}

void SeqWeightTable::copy_observed(const SeqWeightTable& other) {
  _observed = other._observed;
}

void SeqWeightTable::copy_expected(const SeqWeightTable& other) {
  _expected = other._expected;
}

void SeqWeightTable::increment_expected(const Sequence& seq, double mass,
                                        const vector<double>& fl_cdf) {
  _expected.fast_learn(seq, mass, fl_cdf);
}

void SeqWeightTable::normalize_expected() {
  _expected.calc_marginals();
}

void SeqWeightTable::increment_observed(const Sequence& seq, size_t i,
                                        double mass) {
  int left = (int)i - SURROUND;
  _observed.update(seq, left, mass);
}

double SeqWeightTable::get_weight(const Sequence& seq, size_t i) const {
  int left = (int)i - SURROUND;
  return _observed.seq_prob(seq, left) - _expected.seq_prob(seq, left);
}

void SeqWeightTable::append_output(ofstream& outfile) const {
  char buff[200];
  string header = "";
  for (int i = 0; i < WINDOW; i++) {
    sprintf(buff, "\t%d", i-CENTER+1);
    header += buff;
  }
  header += '\n';

  outfile << "\tObserved Marginal Distribution\n" << header;

  for (size_t j = 0; j < NUM_NUCS; j++) {
    outfile << NUCS[j] << ":\t";
    for (int i = 0; i < WINDOW; i++) {
      outfile << scientific << sexp(_observed.marginal_prob(i,j)) << "\t";
    }
    outfile<<endl;
  }

  outfile << "\tObserved Conditional Probabilities\nPosition\t";

  for (size_t j = 0; j < pow((double)NUM_NUCS, (double)_order+1); j++) {
    string s = "->";
    s += NUCS[j & 3];
    size_t cond = j >> 2;
    for (size_t k = 0; k < _order; ++k) {
      s = NUCS[cond & 3] + s;
      cond = cond >> 2;
    }
    outfile << s << '\t';
  }
  outfile << endl;

  for (int i = 0; i < WINDOW; i++) {
    outfile << i-SURROUND << ":\t";
    for (size_t j = 0; j < pow((double)NUM_NUCS, (double)_order+1); j++) {
      outfile << scientific << sexp(_observed.transition_prob(i,j>>2,j&3))
              << "\t";
    }
    outfile<<endl;
  }

  outfile << "\tBackground Conditional Probabilities\nPosition\t";

  for (size_t j = 0; j < pow((double)NUM_NUCS, (double)_order+1); j++) {
    string s = "->";
    s += NUCS[j & 3];
    size_t cond = j >> 2;
    for (size_t k = 0; k < _order; ++k) {
      s = NUCS[cond & 3] + s;
      cond = cond >> 2;
    }
    outfile << s << '\t';
  }
  outfile << endl;

  for (size_t i = 0; i < _order+1; i++) {
    outfile << i+1 << ":\t";
    for (size_t j = 0; j <  pow((double)NUM_NUCS, (double)_order+1); j++) {
      outfile << scientific << sexp(_expected.transition_prob(i,j>>2,j&3))
              << "\t";
    }
    outfile<<endl;
  }
}

BiasBoss::BiasBoss(size_t order, double alpha)
    : _order(order),
      _5_seq_bias(WINDOW, order, alpha),
      _3_seq_bias(WINDOW, order, alpha){
}

BiasBoss::BiasBoss(size_t order, string param_file_name)
    : _order(order),
      _5_seq_bias(WINDOW, order, param_file_name, ">5"),
      _3_seq_bias(WINDOW, order, param_file_name, ">3") {
}

void BiasBoss::copy_observations(const BiasBoss& other) {
  _5_seq_bias.copy_observed(other._5_seq_bias);
  _3_seq_bias.copy_observed(other._3_seq_bias);
}

void BiasBoss::copy_expectations(const BiasBoss& other) {
  _5_seq_bias.copy_expected(other._5_seq_bias);
  _3_seq_bias.copy_expected(other._3_seq_bias);
}

void BiasBoss::update_expectations(const Target& targ, double mass,
                                   const vector<double>& fl_cdf) {
  if ((mass == LOG_0) || (mass < LOG_MIN_TARG_RHO)) {
    return;
  }

  if (direction != R) {
    const Sequence& seq_fwd = targ.seq(0);
    _5_seq_bias.increment_expected(seq_fwd, mass, fl_cdf);
  }
  if (direction != F) {
    const Sequence& seq_rev = targ.seq(1);
    _3_seq_bias.increment_expected(seq_rev, mass, fl_cdf);
  }
}

void BiasBoss::normalize_expectations() {
  _5_seq_bias.normalize_expected();
  _3_seq_bias.normalize_expected();
}

void BiasBoss::update_observed(const FragHit& hit, double normalized_mass)
{
  assert (hit.pair_status() != PAIRED || (int)hit.length() > WINDOW);

  const Sequence& t_seq_fwd = hit.target()->seq(0);
  const Sequence& t_seq_rev = hit.target()->seq(1);

  if (hit.pair_status() != RIGHT_ONLY) {
    _5_seq_bias.increment_observed(t_seq_fwd, hit.left(), normalized_mass);
  }
  if (hit.pair_status() != LEFT_ONLY) {
    _3_seq_bias.increment_observed(t_seq_rev, t_seq_rev.length()-hit.right(),
                                   normalized_mass);
  }
}

double BiasBoss::get_target_start_bias(const Target& targ, size_t pos) const {

  const Sequence& t_seq_fwd = targ.seq(0);
  return _5_seq_bias.get_weight(t_seq_fwd, pos);
}

double BiasBoss::get_target_end_bias(const Target& targ, size_t pos) const {

  const Sequence& t_seq_rev = targ.seq(1);
  return _3_seq_bias.get_weight(t_seq_rev, targ.length()-pos-1);
}

double BiasBoss::get_average_target_bias(std::map<const size_t, float>& start_bias,
                                         std::map<const size_t, float>& end_bias,
                                         const Target& targ) const {
  double tot_start = LOG_0;
  double tot_end = LOG_0;

  for (size_t i = 0; i < targ.length(); ++i) {
    double start_bias_i;
    double end_bias_i;
    try {
      start_bias_i = start_bias.at(i);
    } catch (out_of_range) {
      start_bias_i = get_target_start_bias(targ, i);
    }
    try {
      end_bias_i = end_bias.at(i);
    } catch (out_of_range) {
      end_bias_i = get_target_end_bias(targ, i);
    }
    tot_start = log_add(tot_start, start_bias_i);
    tot_end = log_add(tot_end, end_bias_i);
  }

  //double avg_start_bias = (tot_start ) - (log((double)targ.length()));
  //double avg_end_bias = (tot_end) - (log((double)targ.length()));
  double avg_bias = (tot_start + tot_end) - (2*log((double)targ.length()));
  //logger.info("Target %s avg biases: start = %f, end = %f, total = %f", targ.name().c_str(),
  //   avg_start_bias, avg_end_bias, avg_bias);
  assert(!isnan(avg_bias));
  return avg_bias;
}

void BiasBoss::update_target_bias(std::map<const size_t, float>& start_bias,
                                 std::map<const size_t, float>& end_bias,
                                 const Target& targ) const {

  const Sequence& t_seq_fwd = targ.seq(0);
  const Sequence& t_seq_rev = targ.seq(1);

  for (std::map<const size_t, float>::iterator it = start_bias.begin();
       it != start_bias.end(); ++it) {
    //=-=TODO: Shouldn't this be it->first???
    size_t pos = it->second;
    start_bias[pos] = _5_seq_bias.get_weight(t_seq_fwd, pos);
  }
  for (std::map<const size_t, float>::iterator it = end_bias.begin();
       it != end_bias.end(); ++it) {
    size_t pos = it->second;
    end_bias[targ.length()-pos-1] = _3_seq_bias.get_weight(t_seq_rev, pos);
  }
}

void BiasBoss::append_output(ofstream& outfile) const {
  outfile << ">5' Sequence-Specific Bias\n";
  _5_seq_bias.append_output(outfile);
  outfile << ">3' Sequence-Specific Bias\n";
  _3_seq_bias.append_output(outfile);
}


// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <Eigen/Core>
#include <Eigen/Jacobi>

#define NOTR    0
#define TR      1
#define ADJ     2

#define LEFT    0
#define RIGHT   1

#define UP      0
#define LO      1

#define NUNIT   0
#define UNIT    1

#define INVALID 0xff

#define OP(X)   (   ((X)=='N' || (X)=='n') ? NOTR   \
                  : ((X)=='T' || (X)=='t') ? TR     \
                  : ((X)=='C' || (X)=='c') ? ADJ    \
                  : INVALID)

#define SIDE(X) (   ((X)=='L' || (X)=='l') ? LEFT   \
                  : ((X)=='R' || (X)=='r') ? RIGHT  \
                  : INVALID)

#define UPLO(X) (   ((X)=='U' || (X)=='u') ? UP     \
                  : ((X)=='L' || (X)=='l') ? LO     \
                  : INVALID)

#define DIAG(X) (   ((X)=='N' || (X)=='n') ? NUNIT  \
                  : ((X)=='U' || (X)=='u') ? UNIT   \
                  : INVALID)

inline bool check_op(const char* op)
{
  return OP(*op)!=0xff;
}

inline bool check_side(const char* side)
{
  return SIDE(*side)!=0xff;
}

inline bool check_uplo(const char* uplo)
{
  return UPLO(*uplo)!=0xff;
}

using namespace Eigen;

typedef float Scalar;
typedef NumTraits<Scalar>::Real RealScalar;
typedef std::complex<RealScalar> Complex;

enum
{
  IsComplex = Eigen::NumTraits<float>::IsComplex,
  Conj = IsComplex
};

typedef Matrix<Scalar,Dynamic,Dynamic,ColMajor> PlainMatrixType;
typedef Map<Matrix<Scalar,Dynamic,Dynamic,ColMajor>, 0, OuterStride<> > MatrixType;
typedef Map<Matrix<Scalar,Dynamic,1>, 0, InnerStride<Dynamic> > StridedVectorType;
typedef Map<Matrix<Scalar,Dynamic,1> > CompactVectorType;

template<typename T>
Map<Matrix<T,Dynamic,Dynamic,ColMajor>, 0, OuterStride<> >
matrix(T* data, int rows, int cols, int stride)
{
  return Map<Matrix<T,Dynamic,Dynamic,ColMajor>, 0, OuterStride<> >(data, rows, cols, OuterStride<>(stride));
}

template<typename T>
Map<Matrix<T,Dynamic,1>, 0, InnerStride<Dynamic> > vector(T* data, int size, int incr)
{
  return Map<Matrix<T,Dynamic,1>, 0, InnerStride<Dynamic> >(data, size, InnerStride<Dynamic>(incr));
}

template<typename T>
Map<Matrix<T,Dynamic,1> > vector(T* data, int size)
{
  return Map<Matrix<T,Dynamic,1> >(data, size);
}

template<typename T>
T* get_compact_vector(T* x, int n, int incx)
{
  if(incx==1)
    return x;

  T* ret = new Scalar[n];
  if(incx<0) vector(ret,n) = vector(x,n,-incx).reverse();
  else       vector(ret,n) = vector(x,n, incx);
  return ret;
}

template<typename T>
T* copy_back(T* x_cpy, T* x, int n, int incx)
{
  if(x_cpy==x)
    return 0;

  if(incx<0) vector(x,n,-incx).reverse() = vector(x_cpy,n);
  else       vector(x,n, incx)           = vector(x_cpy,n);
  return x_cpy;
}

#define EIGEN_BLAS_FUNC(X) EIGEN_CAT(eigen_,X)

static inline int EIGEN_BLAS_FUNC(gemm)(char *opa, char *opb, int *m, int *n, int *k, RealScalar *palpha, RealScalar *pa, int *lda, RealScalar *pb, int *ldb, RealScalar *pbeta, RealScalar *pc, int *ldc)
{
//   std::cerr << "in gemm " << *opa << " " << *opb << " " << *m << " " << *n << " " << *k << " " << *lda << " " << *ldb << " " << *ldc << " " << *palpha << " " << *pbeta << "\n";
  typedef void (*functype)(DenseIndex, DenseIndex, DenseIndex, const Scalar *, DenseIndex, const Scalar *, DenseIndex, Scalar *, DenseIndex, Scalar, internal::level3_blocking<Scalar,Scalar>&, Eigen::internal::GemmParallelInfo<DenseIndex>*);
  static functype func[12];

  static bool init = false;
  if(!init)
  {
    for(int kk=0; kk<12; ++kk)
      func[kk] = 0;
    func[NOTR  | (NOTR << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,ColMajor,false,Scalar,ColMajor,false,ColMajor>::run);
    func[TR    | (NOTR << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,RowMajor,false,Scalar,ColMajor,false,ColMajor>::run);
    func[ADJ   | (NOTR << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,RowMajor,Conj, Scalar,ColMajor,false,ColMajor>::run);
    func[NOTR  | (TR   << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,ColMajor,false,Scalar,RowMajor,false,ColMajor>::run);
    func[TR    | (TR   << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,RowMajor,false,Scalar,RowMajor,false,ColMajor>::run);
    func[ADJ   | (TR   << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,RowMajor,Conj, Scalar,RowMajor,false,ColMajor>::run);
    func[NOTR  | (ADJ  << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,ColMajor,false,Scalar,RowMajor,Conj, ColMajor>::run);
    func[TR    | (ADJ  << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,RowMajor,false,Scalar,RowMajor,Conj, ColMajor>::run);
    func[ADJ   | (ADJ  << 2)] = (internal::general_matrix_matrix_product<DenseIndex,Scalar,RowMajor,Conj, Scalar,RowMajor,Conj, ColMajor>::run);
    init = true;
  }

  Scalar* a = reinterpret_cast<Scalar*>(pa);
  Scalar* b = reinterpret_cast<Scalar*>(pb);
  Scalar* c = reinterpret_cast<Scalar*>(pc);
  Scalar alpha  = *reinterpret_cast<Scalar*>(palpha);
  Scalar beta   = *reinterpret_cast<Scalar*>(pbeta);

  if(beta!=Scalar(1))
  {
    if(beta==Scalar(0)) matrix(c, *m, *n, *ldc).setZero();
    else                matrix(c, *m, *n, *ldc) *= beta;
  }

  internal::gemm_blocking_space<ColMajor,Scalar,Scalar,Dynamic,Dynamic,Dynamic> blocking(*m,*n,*k, 1, false);

  int code = OP(*opa) | (OP(*opb) << 2);
  func[code](*m, *n, *k, a, *lda, b, *ldb, c, *ldc, alpha, blocking, 0);
  return 0;
}