/// \file   price_setter.cpp
///
/// \brief
///
/// \authors    Maarten P. Scholl
/// \date       2018-07-19
/// \copyright  Copyright 2017-2019 The Institute for New Economic Thinking,
///             Oxford Martin School, University of Oxford
///
///             Licensed under the Apache License, Version 2.0 (the "License");
///             you may not use this file except in compliance with the License.
///             You may obtain a copy of the License at
///
///                 http://www.apache.org/licenses/LICENSE-2.0
///
///             Unless required by applicable law or agreed to in writing,
///             software distributed under the License is distributed on an "AS
///             IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
///             express or implied. See the License for the specific language
///             governing permissions and limitations under the License.
///
///             You may obtain instructions to fulfill the attribution
///             requirements in CITATION.cff
///
#include <esl/economics/markets/walras/price_setter.hpp>

#include <algorithm>
#include <iostream>
#include <map>
#include <type_traits>
using std::map;

#include <esl/economics/markets/walras/quote_message.hpp>
#include <esl/economics/markets/walras/tatonnement.hpp>
using esl::law::property;
using esl::economics::finance::securities_lending_contract;

#include <esl/economics/finance/securities_lending_contract.hpp>

template<class... T>
struct always_false : std::false_type
{ };

template<>
struct always_false<> : std::true_type
{ };

namespace esl::economics::markets::walras {
    price_setter::price_setter()
        : price_setter(identity<price_setter>())
    {

    }

    ///
    /// \brief
    ///
    /// \details    Initialises the differentiable variable context to 1.0 times
    ///             the initial quotes. In essence, the
    ///             solver starts at 1.0 times the initial quote
    /// \param i
    /// \param traded_assets
    price_setter::price_setter(const identity<price_setter> &i,
                               law::property_map<quote>
                               traded_properties)
    : agent(i)
    , market(i)
    , state(sending_quotes)
    , traded_properties(traded_properties)
    {

        output_clearing_prices_ =
            create_output<std::vector<price>>("clearing_prices");

        output_volumes_ =
            create_output<std::vector<uint64_t>>("volumes");

        register_callback<walras::differentiable_order_message>(
            [this](auto msg, simulation::time_interval ti,
                   std::seed_seq &seed) {
                (void)msg;
                (void)seed;
                return ti.upper;
            });
    }

    ///
    /// \brief
    ///
    /// \param step
    /// \param seed
    /// \return
    simulation::time_point price_setter::act(simulation::time_interval step, std::seed_seq &seed)
    {
        (void)seed;
        simulation::time_point next_ = step.upper;
        std::vector<quote> quotes_;

        if(state == sending_quotes){
            // send out initial quotes and wait for orders,
            // scheduled in the same time step
            next_ = step.lower;
            for(const auto &[k, v]: traded_properties) {
                (void)k;
                quotes_.push_back(v);
            }
        }else{
            std::unordered_map<
                identity<agent>,
                std::shared_ptr<walras::differentiable_order_message>>
                orders_;
            for(const auto &[k, message_]: inbox) {
                (void)k;
                if(walras::differentiable_order_message::code == message_->type) {
                    auto order_ = std::dynamic_pointer_cast<
                        walras::differentiable_order_message>(message_);

                    if(message_->sent < step.lower) {
                        next_ = step.lower;
                        break;
                    }
                    orders_.emplace(order_->sender, order_);
                }
            }

            if(!orders_.empty()) {
                // there is at least one order so we clear the market
                auto before_ = std::chrono::high_resolution_clock::now();
                auto scalars_ = clear_market(orders_, step);
                LOG(notice) << "clearing market took " << (double((std::chrono::high_resolution_clock::now()-before_).count()) / 1e+6) <<  " milliseconds" << std::endl;

                std::vector<price> prices_;
                for(auto &[k, v]: traded_properties){
                    (void)v;
                    // TODO: generalise to all types of quotes
                    prices_.emplace_back(std::get<price>(v.type));
                    quotes_.emplace_back(quote(v));
                }

                LOG(notice) << "quotes: " << quotes_ << std::endl;

                output_clearing_prices_->put(step.lower, prices_);

            }else{  // restore previous prices
                for(const auto &[k, v]: traded_properties){
                    (void)k;
                    quotes_.push_back(v);
                }
                if(step.lower > 1){
                    next_ = step.lower;
                }
            }
        }

        law::property_map<quote> quote_map_;
        {
            size_t sequence_ = 0;
            for(const auto &[k, v]: traded_properties) {
                quote_map_.insert({k, quotes_[sequence_]});
                ++sequence_;
            }
        }
        LOG(trace) << describe() << " " << identifier << " time " << step.lower <<  " clearing prices " << quote_map_ << std::endl;

        for(const auto &p : participants) {
            auto m = this->template create_message<walras::quote_message>(
                p, step.lower + 1, identifier, p, quote_map_);
        }
        state = clearing_market;
        return next_;
    }

    ///
    /// \brief  Applies a solution of quotes to the currently stored traded_properties
    ///
    /// \param result_
    /// \param traded_properties
    /// \return
    std::map<identity<law::property>, std::tuple<quote, double>> apply_results(
        std::map<identity<law::property>, double> result_
        , law::property_map<quote> &traded_properties
    )
    {
        std::map<identity<law::property>, std::tuple<quote, double>> solution_;
        for(auto [p, q]: traded_properties) {
            double scalar_ = result_.find(p->identifier)->second;
            std::tuple<quote, double> part_ = std::make_tuple(q, scalar_);
            solution_.emplace(p->identifier, part_);

            std::visit(
                [&, result_, p = p](auto &quote) {
                    using type_ = std::decay_t<decltype(quote)>;
                    if constexpr(std::is_same_v<type_, price>) {
                        auto value_ = int64_t(quote.value
                                              * result_.find(p->identifier)->second);
                        if(0 == value_){//} && POSITIVE){
                            value_ = 1;
                        }
                        std::get<price>(traded_properties[p].type).value =value_
                            ;

                    } else if constexpr(std::is_same_v<type_, exchange_rate>) {
                        quote = exchange_rate(
                            uint64_t(quote.numerator()
                                     * result_.find(p->identifier)->second),
                            quote.denominator());
                        traded_properties[p].type = quote;
                    } else {
                        static_assert(always_false<type_>::value,
                                      "non-exhaustive handling of quote types");
                    }
                },
                q.type);
        }
        return solution_;
    }

    ///
    /// \brief  Computes the transfers from and to the market agent
    ///
    /// \param traded_properties
    /// \param volumes_
    /// \param orders_
    /// \return
    map<identity<property>, map<identity<agent>, int64_t>> price_setter::compute_transfers
        ( const law::property_map<quote> &traded_properties
            , const map<identity<property>, double> &volumes_
            , const map<identity<property>, map<identity<agent>, std::tuple<double, quantity, quantity>>> &orders_
        )
    {
        map<identity<property>, map<identity<agent>, int64_t>> transfers_;
        for(const auto &[property_, data_]: traded_properties) {
            auto i = volumes_.find(property_->identifier);
            if(volumes_.end() == i) {
                continue;  // no agents expressed interest
            }
            int64_t error_ = 0;
            std::vector<std::tuple<identity<agent>, int64_t>> allocations_;
            for(auto [p, a]: orders_.find(property_->identifier)->second) {
                auto alloc_ = int64_t(std::get<0>(a));
                error_ += alloc_;
                allocations_.emplace_back(p, alloc_);
            }
            std::sort(allocations_.begin(), allocations_.end(),
                      [](const auto &a, const auto &b) -> bool {
                          return std::abs(std::get<1>(a))
                                 < std::abs(std::get<1>(b));
                      });

            if(error_ < 0){  // assigned too many
                if(allocations_.size() < uint64_t(-error_)) {
                    LOG(notice) << "clearing price beyond rounding error" << std::endl;
                }

                for(size_t ii = 0; ii < uint64_t(-error_); ++ii) {
                    std::get<1>(allocations_[ii % allocations_.size()]) += 1;
                }
            }else{
                if(allocations_.size() < uint64_t(error_)) {
                    LOG(notice) << "clearing price beyond rounding error" << std::endl;
                }

                for(size_t ii = 0; ii < uint64_t(error_); ++ii) {
                    std::get<1>(allocations_[ii % allocations_.size()]) -= 1;
                }
            }
            auto pair_ = transfers_.emplace(property_->identifier, map<identity<agent>, int64_t>());

            for(auto [p, a]: allocations_) {
                pair_.first->second.emplace(p, a);
            }
        }
        //if(transfers_.size()){
        //    LOG(trace) << transfers_ << std::endl;
        //}
        return transfers_;
    }

    ///
    /// \brief  Clear market using tatonnement. It is assumed the price_setter
    ///         has received at least one excess demand function for each
    ///         property from market participants, otherwise
    ///         it will return the previous price for the property with no
    ///         excess demand curve.
    ///
    ///
    std::map<identity<law::property>, double> price_setter::clear_market(
        const std::unordered_map<
            identity<agent>,
            std::shared_ptr<walras::differentiable_order_message>> &orders,
        const simulation::time_interval &step)
    {
        law::property_map<quote> old_quotes_ = traded_properties;

        tatonnement::excess_demand_model model_(traded_properties);
        for(auto [key, function_] : orders) {
            (void)key;
            model_.excess_demand_functions_.push_back(function_);
        }
        auto result1_ = model_.compute_clearing_quotes();

        // if finding a price failed, return previous price vector
        if(!result1_.has_value()){
            std::map<identity<law::property>, double> previous_;
            for(const auto &[k, v] : traded_properties) {
                (void)k;
                previous_.insert({k->identifier, 1.0 * double(v)});
            }
            return previous_;
        }

        auto solution_ = apply_results(result1_.value(), traded_properties);

        ////////////////////////////////////////////////////////////////////////
        map<identity<property>, double> volumes_;
        map<identity<property>, map<identity<agent>, std::tuple<double, quantity, quantity>>> orders_;

        //  this is to normalize positive demand, so that we dont create
        //  naked short positions by inventing new property
        map<identity<property>, double> scales_;

        map<identity<property>, double> existingdemand_;

        // total supply is the maximum number of properties
        map<identity<property>, double> total_supply_;

        for(const auto &[participant, order_] : orders) {
            auto demand_ = order_->excess_demand(solution_);
            for(const auto &[property_, excess_] : demand_) {
                auto i = scales_.emplace(property_, 0.).first;
                auto j = total_supply_.emplace(property_, 0.).first;
                auto h = existingdemand_.emplace(property_, 0.).first;

                // TODO: the minimum transfer amount should be derived from the quote
                constexpr double minimum_transfer_amount = 0.00001;
                if(excess_ > -minimum_transfer_amount && excess_ < minimum_transfer_amount) {
                    continue;
                }
                auto quote_ = solution_.find(property_)->second;
                auto units_ = excess_ / (double(std::get<0>(quote_)) * std::get<1>(quote_));
                auto units_with_existing_ = units_;

                auto k = order_->supply.find(property_);
                if(order_->supply.end() != k){
                    // TODO: there is the excess, and this is the reserve
                    units_with_existing_ -= double(std::get<0>(k->second));
                    units_with_existing_ -= double(std::get<1>(k->second));

                    j->second += double(std::get<0>(k->second)) - double(std::get<1>(k->second)) ;
                }

                if(units_with_existing_ > 0.) {
                    h->second += units_with_existing_;
                }

                if(units_ > 0.){
                    i->second += units_;
                }
            }
        }

        // TODO: check to make sure we use supply properly
        //std::cout << " total_supply_ = " << total_supply_ << std::endl;
        //std::cout << " existing_demand_ = " << existingdemand_ << std::endl;
        //std::cout << " demand(scale) = " << scales_ << std::endl;

        for(const auto &[participant, order_]: orders) {
            auto demand_ = order_->excess_demand(solution_);
            for(const auto &[property_, excess_]: demand_) {
                if(excess_ >= -0.00001 && excess_ <= 0.00001) {
                    continue;
                }

                auto quote_ = solution_.find(property_)->second;

                auto units_ = (excess_ ) / (double(std::get<0>(quote_)) * std::get<1>(quote_));

                auto i = volumes_.find(property_);
                if(volumes_.end() == i){
                    i = volumes_.emplace(property_, 0).first;
                    orders_.emplace( property_
                        , map<identity<agent>, std::tuple<double, quantity, quantity>>()
                    );
                }
                i->second += abs(units_);

                auto j = order_->supply.find(property_);
                if(order_->supply.end() == j){
                    orders_.find(property_)->second.emplace( participant, std::make_tuple(units_, 0, 0));

                    //LOG(trace) << participant << " demands {" << property_ << ", "
                    //           << units_ << "}" << std::endl;
                }else{
                    //LOG(trace) << participant << " demands {" << property_ << ", "
                    //           << std::setprecision(5) << units_
                    //           << "}" << std::endl;

                    orders_.find(property_)->second.emplace(participant, std::make_tuple(units_, std::get<0>(j->second), std::get<1>(j->second)));
                }
            }
        }

        ////////////////////////////////////////////////////////////////////////
        auto transfers_ = compute_transfers(traded_properties, volumes_, orders_);
        output_volumes_->put(step.lower, { std::uint64_t(volumes_.begin()->second )} );

        //  send_: we, the market maker, send items to participant
        //  receive_: we, the market maker, receive items from participant
        map<identity<agent>, accounting::inventory_filter<law::property>> send_, receive_;

        auto usd_ = std::make_shared<cash>(currencies::USD);

        for(const auto &[property_, data_] : traded_properties) {
            // if no agents expressed interest, transfers_ might be empty
            if(transfers_.end() == transfers_.find(*property_)) {
                continue;
            }
            for(auto &[p, v] : transfers_.find(*property_)->second) {
                if(v == 0) {
                    continue;
                }

                auto i = orders.find(p)->second->supply.find(*property_);
                if(orders.find(p)->second->supply.end() == i) {
                    auto [ii,b] = orders.find(p)->second->supply.insert({property_->identifier
                                                                            , std::make_tuple(quantity(0)
                            ,quantity(0))});
                    i = ii;
                }
                uint64_t &long_  = std::get<0>(i->second).amount;
                uint64_t &short_ = std::get<1>(i->second).amount;

                if(v > 0) {  // send properties to this agent, or receive shorts
                    if(short_ > 0) {  // there was a short position
                        assert(long_ == 0);

                        uint64_t cancel_ = std::min(uint64_t(v), short_);
                        //LOG(trace) << "cancel the short position of " << p <<" by " << cancel_ << std::endl;

                        auto short_contract_ = std::make_shared<securities_lending_contract>(identifier, p, property_->identifier, quantity(1));
                        auto r = receive_.emplace(p, accounting::inventory_filter<law::property>()).first;
                        r->second.insert(short_contract_, quantity(cancel_));

                        // ???
                        // auto s = send_.emplace(p, accounting::inventory_filter<law::property>()).first;
                        //
                        auto s = receive_.emplace(p, accounting::inventory_filter<law::property>()).first;
                        auto o = old_quotes_.find(property_);
                        auto collateral_ = usd_->amount(cancel_ * double(std::get<price>(o->second.type)) / o->second.lot);

                        s->second.insert(usd_, collateral_);
                        v -= cancel_;
                    }

                    if(uint64_t(v) > 0){
                        //LOG(trace) << p << " purchased additional " << v << std::endl;
                        accounting::inventory_filter<law::property> purchased_;
                        purchased_.insert(property_, quantity(uint64_t(v)));
                        send_.emplace(p, purchased_);

                        // they should send cash for the purchase
                        auto r = receive_.emplace(p, accounting::inventory_filter<law::property>()).first;
                        r->second.insert(usd_, usd_->amount(v * double(std::get<price>(data_.type)) / data_.lot));
                    }
                }else{  // v < 0 so participant wants to sell/short

                    if(long_ > 0){

                        uint64_t cancel_ = std::min(uint64_t(-v), long_);
                        //LOG(trace) << "cancel the long position of " << p <<" by " << cancel_ << std::endl;

                        auto r = receive_.emplace( p, accounting::inventory_filter<law::property>()).first;
                        r->second.insert(property_, quantity(cancel_));

                        auto s = send_.emplace(p, accounting::inventory_filter<law::property>()).first;
                        auto proceeds_ = usd_->amount(cancel_ * double(std::get<price>(data_.type)) / data_.lot);
                        s->second.insert(usd_, proceeds_);
                        v += cancel_;
                    }

                    if(v <  0){ // is still smaller than zero
                        // extend the short position
                        auto amount_to_extend = uint64_t(-v);
                        //LOG(trace) << "extend the short position of " << p <<" by " << amount_to_extend << std::endl;

                        auto short_contract_ = std::make_shared<securities_lending_contract>(identifier, p, property_->identifier, quantity(1));
                        auto s = send_.emplace( p, accounting::inventory_filter<law::property>()).first;

                        auto sale_ = usd_->amount(amount_to_extend * double(std::get<price>(data_.type)) / data_.lot);

                        s->second.insert(usd_, sale_);
                        s->second.insert(short_contract_, quantity(amount_to_extend));
                    }
                }
            }
        }

        // apply netting
        for(auto &[p, i_r] : receive_) {
            auto offset_ = send_.find(p);
            if(send_.end() == offset_) {
                continue;
            }

            for(auto &[k, v] : i_r.items) {
                if(0 == v.amount){
                    //i_r.items.erase(k);
                    continue;
                }

                auto property_ = offset_->second.items.find(k);
                if(offset_->second.items.end() == property_) {
                    continue;
                }
                // receiving more than sending
                if(v > property_->second) {
                    auto diff_ = v - property_->second;
                    v          = diff_;
                    offset_->second.items.erase(property_);
                } else if(v < property_->second) {  // sending more than recv
                    auto diff_        = property_->second - v;
                    property_->second = diff_;
                    //i_r.items.erase(k);
                } else {  // sending and rececing same amount
                    //i_r.items.erase(k);
                    offset_->second.items.erase(property_);
                }
            }
            if(i_r.items.empty()){
                receive_.erase(p);
            }
        }

        for(auto [p, i] : send_) {
            LOG(trace) << "market sends to " << p << " items " << i
                       << std::endl;
            this->template create_message<interaction::transfer>(
                p, step.lower, identifier, p,
                reinterpret_identity_cast<law::owner<law::property>>(
                    identifier),
                reinterpret_identity_cast<law::owner<law::property>>(p), i);
        }

        for(auto [p, i]: receive_) {
            LOG(trace) << "market receives from " << p << " items " << i
                       << std::endl;
            this->template create_message<interaction::transfer>(
                p, step.lower, p, identifier,
                reinterpret_identity_cast<law::owner<law::property>>(p),
                reinterpret_identity_cast<law::owner<law::property>>(
                    identifier),
                i);
        }

        return result1_.value();
    }

}  // namespace esl::economics::markets::walras

#include <boost/serialization/export.hpp>

#include <esl/data/serialization.hpp>

BOOST_CLASS_EXPORT(std::vector<esl::economics::price>);
typedef std::tuple<esl::simulation::time_point,
    std::vector<esl::economics::price>>
    tuple_time_point_price_vector;
BOOST_CLASS_EXPORT(tuple_time_point_price_vector);
typedef std::vector<
    std::tuple<esl::simulation::time_point, std::vector<esl::economics::price>>>
    time_series_price_vector;
BOOST_CLASS_EXPORT(time_series_price_vector);
BOOST_CLASS_EXPORT(esl::data::output<std::vector<esl::economics::price>>);


#include <algorithm>
#include <chrono>
#include <functional>
#include <map>
#include <memory>
#include <regex>
#include <set>
#include <sstream>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#include <unordered_set>
#include <utility>
#include <vector>

#include <boost/multiprecision/cpp_int.hpp>

#include <nlohmann/json.hpp>
#include <pybind11/operators.h>
#include <pybind11/pybind11.h>
#include <pybind11/pytypes.h>
#include <pybind11/stl.h>

#include "arch_graph.hpp"
#include "arch_graph_automorphisms.hpp"
#include "arch_graph_cluster.hpp"
#include "arch_graph_system.hpp"
#include "arch_uniform_super_graph.hpp"
#include "nauty_graph.hpp"
#include "parse.hpp"
#include "perm.hpp"
#include "perm_group.hpp"
#include "perm_set.hpp"
#include "task_mapping.hpp"
#include "task_mapping_orbit.hpp"
#include "timeout.hpp"
#include "util.hpp"

namespace mp = boost::multiprecision;

namespace py = pybind11;

using namespace py::literals;

using json = nlohmann::json;

using mpsym::ArchGraph;
using mpsym::ArchGraphCluster;
using mpsym::ArchGraphSystem;
using mpsym::ArchUniformSuperGraph;
using mpsym::ReprOptions;
using mpsym::TaskMapping;
using mpsym::TMO;
using mpsym::TMORs;

using mpsym::internal::ArchGraphAutomorphisms;
using mpsym::internal::BSGS;
using mpsym::internal::NautyGraph;
using mpsym::internal::Perm;
using mpsym::internal::PermGroup;
using mpsym::internal::PermSet;

using mpsym::util::IteratorAdaptor;
using mpsym::util::parse_perm;
using mpsym::util::stream;

using mpsym::internal::timeout::flag;
using mpsym::internal::timeout::run_abortable_with_timeout;
using mpsym::internal::timeout::TimeoutError;

namespace
{

template<typename T = unsigned>
using Sequence = std::vector<T>;

template<typename T>
using contained_type =
  typename std::remove_reference<decltype(*std::declval<T>().begin())>::type;

template<typename T>
Sequence<contained_type<T>> to_sequence(T const &obj)
{ return Sequence<contained_type<T>>(obj.begin(), obj.end()); }

template<typename T = unsigned>
Sequence<T> to_sequence(py::iterator it)
{
  Sequence<T> seq;

  while (it != py::iterator::sentinel()) {
    try {
      seq.push_back(it->cast<T>());
    } catch (py::cast_error const &) {
      throw std::runtime_error("failed to convert iterable to matching sequence");
    }

    ++it;
  }

  return seq;
}

template<typename T>
py::tuple sequence_to_tuple(Sequence<T> const &seq)
{
  py::tuple t = py::cast(seq);
  return t;
}

template<typename T>
py::tuple to_tuple(T const &obj)
{ return sequence_to_tuple(to_sequence(obj)); }

ReprOptions str_to_repr_options(std::string const &method)
{
  ReprOptions options;

  if (method == "auto") {
    options.method = ReprOptions::Method::AUTO;
  } else if (method == "iterate") {
    options.method = ReprOptions::Method::ITERATE;
  } else if (method == "orbit") {
    options.method = ReprOptions::Method::ORBITS;
  } else if (method == "local_search_bfs") {
    options.method = ReprOptions::Method::LOCAL_SEARCH;
    options.variant = ReprOptions::Variant::LOCAL_SEARCH_BFS;
  } else if (method == "local_search_dfs") {
    options.method = ReprOptions::Method::LOCAL_SEARCH;
    options.variant = ReprOptions::Variant::LOCAL_SEARCH_DFS;
  } else {
    throw std::invalid_argument("invalid 'method'");
  }

  return options;
}

Perm str_to_perm(unsigned degree, std::string cycles)
{
  static std::regex re_perm(R"((\(\)|(\(( *\d+,)+ *\d+ *\))+))");

  cycles.erase(std::remove(cycles.begin(), cycles.end(), ' ' ), cycles.end());

  if (!std::regex_match(cycles, re_perm))
    throw std::invalid_argument("invalid permutation string");

  return parse_perm(degree, cycles);
}

template<typename T>
T arch_graph_json_cast(ArchGraphSystem const &self, std::string const &key)
{
  auto j(json::parse(self.to_json()));
  T ret = j["graph"][key];
  return ret;
}

template<typename FUNC, typename ...ARGS>
typename std::result_of<FUNC(ArchGraphSystem *, ARGS..., flag)>::type
arch_graph_timeout(std::string const &what,
                   double timeout,
                   ArchGraphSystem &self,
                   FUNC f,
                   ARGS &&...args)
{
  return run_abortable_with_timeout(
    what,
    std::chrono::duration<double>(timeout),
    [&](flag aborted)
    { return (self.*f)(std::forward<ARGS>(args)..., aborted); });
}

} // anonymous namespace

namespace pybind11
{

namespace detail
{

template <>
struct type_caster<mp::cpp_int> {
  PYBIND11_TYPE_CASTER(mp::cpp_int, _("cpp_int"));

  static handle cast(mp::cpp_int const &src, return_value_policy, handle)
  { return PyLong_FromString(stream(src).c_str(), nullptr, 10); }
};

} // namespace detail

} // namespace pybind11

#define PYBIND11_MODULE_(name, m) PYBIND11_MODULE(name, m)

PYBIND11_MODULE_(PYTHON_MODULE_INTERNAL, m)
{
  m.attr("__version__") = PYTHON_VERSION;
  m.doc() = PYTHON_DESCRIPTION;

  // ArchGraphSystem
  py::class_<ArchGraphSystem,
             std::shared_ptr<ArchGraphSystem>>(m, "ArchGraphSystem")
    .def("__repr__", &ArchGraphSystem::to_json)
    .def_static("from_lua", &ArchGraphSystem::from_lua,
                "lua"_a, "args"_a = std::vector<std::string>())
    .def_static("from_lua_file", &ArchGraphSystem::from_lua_file,
                "lua_file"_a, "args"_a = std::vector<std::string>())
    .def_static("from_nauty",
                [](int vertices,
                   std::map<int, std::vector<int>> const &adjacencies,
                   int vertices_reduced,
                   bool directed,
                   std::vector<std::set<int>> const &coloring)
                {
                  // validate number of vertices
                  if (vertices <= 0)
                    throw std::invalid_argument("number of vertices must be non-negative");

                  // validate adjacencies
                  std::set<std::pair<int, int>> edges;
                  std::set<std::pair<int, int>> edge_parities;

                  for (auto const &p : adjacencies) {
                    int from = p.first;

                    if (from >= vertices)
                      throw std::invalid_argument("vertex index out of range");

                    for (int to : p.second) {
                      if (to >= vertices)
                        throw std::invalid_argument("vertex index out of range");

                      auto edge(std::make_pair(from, to));
                      auto reverse_edge(std::make_pair(to, from));

                      if (!edges.insert(edge).second)
                        throw std::invalid_argument("duplicate edges");
                    }
                  }

                  // validate coloring
                  if (!coloring.empty()) {
                    std::set<int> tmp;
                    for (auto const &p : coloring)
                      tmp.insert(p.begin(), p.end());

                    if (static_cast<int>(tmp.size()) != vertices
                        || *tmp.begin() != 0
                        || *tmp.rbegin() != vertices - 1) {

                      throw std::invalid_argument("invalid coloring");
                    }
                  }

                  // construct graph
                  if (vertices_reduced == 0)
                    vertices_reduced = vertices;

                  NautyGraph g{vertices, directed};

                  g.add_edges(adjacencies);

                  if (!coloring.empty()) {
                    std::vector<std::vector<int>> coloring_;
                    for (auto const &c : coloring)
                      coloring_.emplace_back(c.begin(), c.end());

                    g.set_partition(coloring_);
                  }

                  // extract automorphisms
                  return std::make_shared<ArchGraphAutomorphisms>(
                    PermGroup(vertices_reduced, g.automorphism_generators()));
                },
                "vertices"_a,
                "adjacencies"_a,
                "vertices_reduced"_a = 0,
                "directed"_a = true,
                "coloring"_a = std::vector<int>())
    .def_static("from_json", &ArchGraphSystem::from_json,
                "json"_a)
    .def_static("from_json_file", &ArchGraphSystem::from_json_file,
                "json_file"_a)
    .def("to_json", &ArchGraphSystem::to_json)
    .def("processor_types",
         [](ArchGraphSystem const &self)
         {
           using T = std::vector<std::string>;
           return arch_graph_json_cast<T>(self, "processor_types");
         })
    .def("channel_types",
         [](ArchGraphSystem const &self)
         {
           using T = std::vector<std::string>;
           return arch_graph_json_cast<T>(self, "channel_types");
         })
    .def("processors",
         [](ArchGraphSystem const &self)
         {
           using T = std::pair<unsigned, std::string>;
           using U = std::vector<T>;
           return arch_graph_json_cast<U>(self, "processors");
         })
    .def("channels",
         [](ArchGraphSystem const &self)
         {
           using T = std::pair<unsigned, std::string>;
           using U = std::map<unsigned, std::vector<T>>;
           return arch_graph_json_cast<U>(self, "channels");
         })
    .def("num_processors", &ArchGraphSystem::num_processors)
    .def("num_channels", &ArchGraphSystem::num_channels)
    .def("initialize",
         [&](ArchGraphSystem &self, double timeout)
         {
           arch_graph_timeout("initialize",
                              timeout,
                              self,
                              &ArchGraphSystem::init_repr,
                              nullptr);
         },
         "timeout"_a = 0.0)
    .def("num_automorphisms",
         [](ArchGraphSystem &self, double timeout)
         {
           return arch_graph_timeout("num_automorphisms",
                                     timeout,
                                     self,
                                     &ArchGraphSystem::num_automorphisms,
                                     nullptr);
         },
         "timeout"_a = 0.0)
    .def("automorphisms_generators",
         [](ArchGraphSystem &self, double timeout)
         {
           auto generators(arch_graph_timeout(
             "automorphisms_generators",
             timeout,
             self,
             &ArchGraphSystem::automorphisms_generators,
             nullptr));

           return Sequence<Perm>(generators.begin(), generators.end());
         },
         "timeout"_a = 0.0)
    .def("automorphisms",
         [](ArchGraphSystem &self, double timeout)
         {
           return arch_graph_timeout("automorphisms",
                                     timeout,
                                     self,
                                     &ArchGraphSystem::automorphisms,
                                     nullptr);
         },
         "timeout"_a = 0.0)
    .def("expand_automorphisms", &ArchGraphSystem::expand_automorphisms)
    .def("orbit",
         [&](ArchGraphSystem &self,
             Sequence<> const &mapping,
             double timeout)
         {
           for (unsigned task : mapping) {
             if (task >= self.automorphisms_degree())
               throw std::invalid_argument("task index out of range");
           }

           return arch_graph_timeout("orbit",
                                     timeout,
                                     self,
                                     &ArchGraphSystem::automorphisms_orbit,
                                     mapping,
                                     nullptr);
         },
         "mapping"_a, "timeout"_a = 0.0)
    .def("representative",
         [&](ArchGraphSystem &self,
             Sequence<> const &mapping,
             std::string const &method,
             double timeout)
         {
           using T = TaskMapping(ArchGraphSystem::*)(TaskMapping const &,
                                                     ReprOptions const *,
                                                     flag);

           auto options(str_to_repr_options(method));

           auto repr(arch_graph_timeout("representative",
                                        timeout,
                                        self,
                                        (T)&ArchGraphSystem::repr,
                                        mapping,
                                        &options));

           return to_tuple(repr);
         },
         "mapping"_a, "method"_a = "auto", "timeout"_a = 0.0)
    .def("representative",
         [&](ArchGraphSystem &self,
             Sequence<> const &mapping,
             TMORs &representatives,
             std::string const &method,
             double timeout)
         {
           using T = std::tuple<TaskMapping, bool, unsigned>
                     (ArchGraphSystem::*)(TaskMapping const &,
                                          TMORs &,
                                          ReprOptions const *,
                                          flag);

           auto options(str_to_repr_options(method));

           TaskMapping repr;
           bool orbit_new;
           unsigned orbit_index;

           std::tie(repr, orbit_new, orbit_index) =
             arch_graph_timeout("representative",
                                timeout,
                                self,
                                (T)&ArchGraphSystem::repr,
                                mapping,
                                representatives,
                                &options);

           return std::make_tuple(to_tuple(repr),
                                  orbit_new,
                                  orbit_index);
         },
         "mapping"_a, "representatives"_a, "method"_a = "auto", "timeout"_a = 0.0);

  // ArchGraphAutomorphisms
  py::class_<ArchGraphAutomorphisms,
             ArchGraphSystem,
             std::shared_ptr<ArchGraphAutomorphisms>>(m, "ArchGraphAutomorphisms")
    .def(py::init<PermGroup>(), "automorphisms"_a)
    .def(py::pickle(
        [](ArchGraphAutomorphisms &self)
        { return self.to_json(); },
        [](std::string const &json)
        {
          return std::dynamic_pointer_cast<ArchGraphAutomorphisms>(
            ArchGraphSystem::from_json(json));
        }));

  // ArchGraph
  py::class_<ArchGraph,
             ArchGraphSystem,
             std::shared_ptr<ArchGraph>>(m, "ArchGraph")
    .def(py::init<bool>(), "directed"_a = true)
    .def(py::pickle(
        [](ArchGraph &self)
        { return self.to_json(); },
        [](std::string const &json)
        {
          return std::dynamic_pointer_cast<ArchGraph>(
            ArchGraphSystem::from_json(json));
        }))
    .def("directed", &ArchGraph::directed)
    .def("add_processor",
         (unsigned(ArchGraph::*)(std::string const &))
         &ArchGraph::add_processor,
         "pl"_a)
    .def("add_processors",
         (unsigned(ArchGraph::*)(unsigned, std::string const &))
         &ArchGraph::add_processors,
         "n"_a, "pl"_a)
    .def("add_channel",
         [](ArchGraph &self, unsigned pe1, unsigned pe2, std::string const &cl)
         {
           if (pe1 >= self.num_processors() || pe2 >= self.num_processors())
             throw std::out_of_range("processor index out of range");

           self.add_channel(pe1, pe2, cl);
         },
         "pe1"_a, "pe2"_a, "cl"_a)
    .def("add_channels",
         [](ArchGraph &self, ArchGraph::ChannelDict<std::string> const &cm)
         {
           for (auto const &tmp : cm) {
             unsigned pe1 = tmp.first;
             if (pe1 >= self.num_processors())
               throw std::out_of_range("processor index out of range");

             for (auto const &edge : tmp.second) {
               unsigned pe2 = edge.first;
               if (pe2 >= self.num_processors())
                 throw std::out_of_range("processor index out of range");
             }
           }

           self.add_channels<std::string>(cm);
         },
         "channels"_a)
    .def("add_channels",
         [](ArchGraph &self,
            ArchGraph::ChannelDict<> const &cm,
            std::string const &ct)
         {
           for (auto const &tmp : cm) {
             unsigned pe1 = tmp.first;
             if (pe1 >= self.num_processors())
               throw std::out_of_range("processor index out of range");

             for (unsigned pe2 : tmp.second) {
               if (pe2 >= self.num_processors())
                 throw std::out_of_range("processor index out of range");
             }
           }

           self.add_channels(cm, ct);
         },
         "channels"_a, "ct"_a)
    .def("fully_connect",
         (void(ArchGraph::*)(std::string const &))
         &ArchGraph::fully_connect,
         "cl"_a)
    .def("fully_connect",
         (void(ArchGraph::*)(std::string const &, std::string const &))
         &ArchGraph::fully_connect,
         "pl"_a, "cl"_a)
    .def("fully_connect",
         [](ArchGraph &self, py::iterable it, std::string const &cl)
         {
           auto processors(to_sequence<>(py::make_iterator(it)));

           for (unsigned pe1 : processors) {
             for (unsigned pe2 : processors) {
               if (pe1 != pe2)
                 self.add_channel(pe1, pe2, cl);
             }
           }
         },
         "processors"_a, "cl"_a)
    .def("self_connect",
         (void(ArchGraph::*)(std::string const &))
         &ArchGraph::self_connect,
         "cl"_a)
    .def("self_connect",
         (void(ArchGraph::*)(std::string const &, std::string const &))
         &ArchGraph::self_connect,
         "pl"_a, "cl"_a)
    .def("self_connect",
         [](ArchGraph &self, py::iterator it, std::string const &cl)
         {
           auto processors(to_sequence<>(it));

           for (unsigned pe : processors)
             self.add_channel(pe, pe, cl);
         },
         "processors"_a, "cl"_a);

  // ArchGraphCluster
  py::class_<ArchGraphCluster,
             ArchGraphSystem,
             std::shared_ptr<ArchGraphCluster>>(m, "ArchGraphCluster")
    .def(py::init<>())
    .def(py::pickle(
        [](ArchGraphCluster &self)
        { return self.to_json(); },
        [](std::string const &json)
        {
          return std::dynamic_pointer_cast<ArchGraphCluster>(
            ArchGraphSystem::from_json(json));
        }))
    .def("add_subsystem", &ArchGraphCluster::add_subsystem, "subsystem"_a)
    .def("num_subsystems", &ArchGraphCluster::num_subsystems);

  // ArchUniformSuperGraph
  py::class_<ArchUniformSuperGraph,
             ArchGraphSystem,
             std::shared_ptr<ArchUniformSuperGraph>>(m, "ArchUniformSuperGraph")
    .def(py::init<std::shared_ptr<ArchGraphSystem>,
                  std::shared_ptr<ArchGraphSystem>>(),
         "super_graph"_a, "proto"_a)
    .def(py::pickle(
        [](ArchUniformSuperGraph &self)
        { return self.to_json(); },
        [](std::string const &json)
        {
          return std::dynamic_pointer_cast<ArchUniformSuperGraph>(
            ArchGraphSystem::from_json(json));
        }));

  // TMO
  py::class_<TMO>(m, "Orbit")
    .def("__iter__",
         [](TMO &orbit)
         {
           IteratorAdaptor<TMO, py::tuple> adaptor(orbit, to_tuple<TaskMapping>);

           return py::make_iterator<py::return_value_policy::copy>(adaptor.begin(),
                                                                   adaptor.end());
         }, py::keep_alive<0, 1>());

  // TMORs
  py::class_<TMORs>(m, "Representatives")
    .def(py::init<>())
    .def(py::self == py::self)
    .def(py::self != py::self)
    .def("__len__", &TMORs::num_orbits)
    .def("__iter__",
         [](TMORs const &orbits)
         { return py::make_iterator(orbits.begin(), orbits.end()); },
         py::keep_alive<0, 1>())
    .def("__contains__",
         [](TMORs const &orbits, Sequence<> const &mapping)
         { return orbits.is_repr(mapping); },
         "mapping"_a);

  // Perm
  py::class_<Perm>(m, "Perm")
    .def(py::init<unsigned>(), "degree"_a = 1)
    .def(py::init(
           [](Sequence<> const &v)
           {
             if (v.empty())
                throw std::invalid_argument("invalid permutation");

             auto max = *std::max_element(v.begin(), v.end());
             if (v.size() != max + 1)
               throw std::invalid_argument("invalid permutation");

             std::set<unsigned> s(v.begin(), v.end());
             if (s.size() != max + 1 || *s.begin() != 0)
               throw std::invalid_argument("invalid permutation");

             return Perm(v);
           }),
         "perm"_a)
    .def(py::init(
           [](unsigned degree, Sequence<Sequence<>> const &cycles)
           {
             std::unordered_set<unsigned> cycles_flattened;

             for (auto const &cycle : cycles) {
                for (unsigned x : cycle) {
                  if (x > degree || !cycles_flattened.insert(x).second)
                    throw std::invalid_argument("invalid permutation");
                }
             }

             return Perm(degree, cycles);
           }),
           "degree"_a, "cycles"_a)
    .def(py::init(
           [](unsigned degree, std::string cycles)
           { return str_to_perm(degree, cycles); }),
           "degree"_a, "cycles"_a)
    .def("__eq__",
         [](Perm const &self, Perm const &other)
         {
           if (self.degree() != other.degree())
             throw std::invalid_argument("can only compare permutations of equal degree");

           return self == other;
         })
    .def("__ne__",
         [](Perm const &self, Perm const &other)
         {
           if (self.degree() != other.degree())
             throw std::invalid_argument("can only compare permutations of equal degree");

           return self != other;
         })
    .def(hash(py::self))
    .def("__getitem__",
         [](Perm const &self, unsigned x)
         {
           if (x > self.degree() - 1)
             throw std::out_of_range("not in domain");

           return self[x];
         },
         "x"_a)
    .def("__invert__", &Perm::operator~)
    .def("__mul__",
         [](Perm const &self, Perm const &other)
         {
           if (self.degree() != other.degree())
             throw std::invalid_argument("permutation degrees do not match");

           return self * other;
         },
         "other"_a)
    .def("__rmul__",
         [](Perm const &self, Perm const &other)
         {
           if (self.degree() != other.degree())
             throw std::invalid_argument("permutation degrees do not match");

           return other * self;
         },
         "other"_a)
    .def("__bool__",
         [](Perm const &self)
         { return !self.id(); })
    .def("__repr__",
         [](Perm const &self)
         { return stream(self); })
    .def("degree", &Perm::degree);

  py::implicitly_convertible<Sequence<>, Perm>();

  // PermGroup
  py::class_<PermGroup>(m, "PermGroup")
    .def(py::init<unsigned>(), "degree"_a = 1)
    .def(py::init(
           [](Sequence<Perm> const &generators_)
           {
             if (generators_.empty())
               throw std::invalid_argument("generating set must not be empty");

             unsigned degree = generators_[0].degree();
             for (std::size_t i = 1; i < generators_.size(); ++i) {
                if (generators_[i].degree() != degree)
                  throw std::invalid_argument("mismatched generator degrees");
             }

             PermSet generators(generators_.begin(), generators_.end());

             return PermGroup(degree, generators);
           }),
         "generators"_a)
    .def(py::init(
           [](unsigned degree, Sequence<std::string> const &generators_)
           {
             PermSet generators;
             for (auto const &gen : generators_)
               generators.insert(str_to_perm(degree, gen));

             return PermGroup(degree, generators);
           }),
         "degree"_a, "generators"_a)
    .def_static("symmetric", PermGroup::symmetric, "degree"_a)
    .def_static("cyclic", PermGroup::cyclic, "degree"_a)
    .def_static("dihedral", PermGroup::dihedral, "degree"_a)
    .def_static("direct_product",
                [](py::iterable it)
                {
                  auto groups(to_sequence<PermGroup>(py::make_iterator(it)));

                  return PermGroup::direct_product(groups.begin(), groups.end());
                },
                "groups"_a)
    .def_static("wreath_product",
                [](PermGroup const &lhs, PermGroup const &rhs)
                { return PermGroup::wreath_product(lhs, rhs); },
                "lhs"_a, "rhs"_a)
    .def("__eq__",
         [](PermGroup const &self, PermGroup const &other)
         {
           if (self.degree() != other.degree())
             throw std::invalid_argument("can only compare permutation groups of equal degree");

           return self == other;
         })
    .def("__ne__",
         [](PermGroup const &self, PermGroup const &other)
         {
           if (self.degree() != other.degree())
             throw std::invalid_argument("can only compare permutation groups of equal degree");

           return self != other;
         })
    .def("__len__", &PermGroup::order)
    .def("__iter__",
         [](PermGroup const &self)
         { return py::make_iterator<py::return_value_policy::copy>(self.begin(), self.end()); },
         py::keep_alive<0, 1>())
    .def("__contains__",
         [](PermGroup const &self, Perm const &p)
         {
           if (p.degree() != self.degree())
             throw std::invalid_argument("mismatched degrees");

           return self.contains_element(p);
         },
         "perm"_a)
    .def("__contains__",
         [](PermGroup const &self, std::string const &p)
         { return self.contains_element(str_to_perm(self.degree(), p)); },
         "perm"_a)
    .def("__bool__",
         [](PermGroup const &self)
         { return !self.is_trivial(); })
    .def("__repr__",
         [](PermGroup const &self)
         { return stream(self.generators()); })
    .def("degree", &PermGroup::degree)
    .def("bsgs",
         [&](PermGroup const &self)
         {
           auto bsgs(self.bsgs());

           return std::make_pair<BSGS::Base, Sequence<Perm>>(
             bsgs.base(),
             to_sequence(bsgs.strong_generators()));
         })
    .def("generators",
         [&](PermGroup const &self, bool sorted)
         {
           auto generators(self.generators());

           if (sorted)
             std::sort(generators.begin(), generators.end());

           return to_sequence(generators);
         },
         "sorted"_a = true)
    .def("is_symmetric", &PermGroup::is_symmetric)
    .def("is_transitive", &PermGroup::is_transitive);

  py::implicitly_convertible<Sequence<Perm>, PermGroup>();
}


/*
 * modules/cmoea/cmoea_nsga2_mpi.hpp
 *
 *  Created on: Mar 12, 2015
 *      Author: Joost Huizinga
 */

#ifndef MODULES_CMOEA_CMOEA_NSGA2_MPI_HPP_
#define MODULES_CMOEA_CMOEA_NSGA2_MPI_HPP_

// Standard includes
#include <algorithm>
#include <limits>

// Boost includes
#include <boost/foreach.hpp>
#include <boost/multi_array.hpp>
#include <boost/array.hpp>
#include <boost/fusion/algorithm/iteration/for_each.hpp>
#include <boost/fusion/include/for_each.hpp>
#include <boost/spirit/include/karma.hpp>
#include <boost/mpi.hpp>
#include <boost/mpi/environment.hpp>
#include <boost/mpi/detail/point_to_point.hpp>

// Sferes includes
#include <sferes/stc.hpp>
#include <sferes/ea/ea.hpp>
#include <sferes/fit/fitness.hpp>
#include <sferes/dbg/dbg.hpp>
#include <sferes/ea/dom_sort_basic.hpp>
#include <sferes/ea/common.hpp>
#include <sferes/ea/crowd.hpp>

// Module includes
#include <modules/datatools/common_compare.hpp>
#include <modules/debugext/dbgext.hpp>

// Local includes
#include "cmoea_nsga2.hpp"
#include "mpi_util.hpp"
#include "nsga_util.hpp"

// Debug defines
#define DBOW dbg::out(dbg::info, "mpi") << "Worker " << _world->rank()
#define DBOM dbg::out(dbg::info, "mpi") << "Master " << this->eval().world()->rank()
#define DBOE dbg::out(dbg::info, "ea")

namespace karma = boost::spirit::karma;

namespace sferes
{
namespace ea
{

template<typename Phen, typename FitModifier, typename Params>                                                       \
class ArchiveTask{
public:
    //Params
    typedef ArchiveTask<Phen, FitModifier, Params> this_t;

    //The type of Pareto domination sort to use.
    //Currently available types are:
    // - sferes::ea::dom_sort_basic_f  (defined in sferes/ea/dom_sort_basic.hpp)
    //   Sorts according to pareto dominance and will add individuals from the
    //   highest to the lowest layer, with crowding as a tie-breaker in the last
    //   layer to be added.
    // - sferes::ea::dom_sort_no_duplicates_f
    //   (defined in sferes/ea/dom_sort_no_duplicates.hpp)
    //   Same as dom_sort_basic, accept that, for each front, only one
    //   individual per pareto location is added. Other individuals at the same
    //   location will be bumped to the next layer.
    typedef typename Params::ea::dom_sort_f dom_sort_f;

    //The type non dominated comparator to use
    //Currently available types are:
    // - sferes::ea::_dom_sort_basic::non_dominated_f
    //   (defined in sferes/ea/dom_sort_basic.hpp)
    //   Regular comparisons based on dominance
    // - sferes::ea::cmoea_nsga::prob_dom_f<Params>
    //   Comparisons based on probabilistic sorting, where some objectives can
    //   be stronger than others.
    typedef typename Params::cmoea_nsga::non_dom_f non_dom_f;

    //The index used to temporarily store the category
    //This index should hold a dummy value, as it will be overwritten
    //constantly.
    //The default would be 0.
    static const size_t obj_index = Params::cmoea::obj_index;

    //The number of objectives (bins) used in the map
    size_t nr_of_bins;

    //The size of each bin
    static const size_t bin_size = Params::cmoea::bin_size;

    //The number of individuals initially generated to fill the archive
    //If equal to the bin_size, every initially generated individual is added
    //to every bin of every category.
    //The init_size has to be greater than or equal to the bin_size
    static const size_t init_size = Params::pop::init_size;

    //Not actually the size of the standing population
    //(which is bin_size*nr_of_bins)
    //but the number of individuals that are generated each epoch.
    //Because individuals are always produced in pairs,
    //pop_size has to be divisible by 2.
    static const size_t indiv_per_gen = Params::pop::select_size;

    //Modifier for calculating distance
    typedef typename boost::mpl::if_<boost::fusion::traits::is_sequence<FitModifier>,
                     FitModifier,
                     boost::fusion::vector<FitModifier> >::type modifier_t;

    typedef Phen phen_t;
    typedef crowd::Indiv<phen_t> crowd_t;
    typedef boost::shared_ptr<crowd_t> indiv_t;
    typedef typename std::vector<indiv_t> pop_t;
    typedef typename std::vector<boost::shared_ptr<phen_t> > ea_pop_t;
    typedef typename std::vector<std::vector<indiv_t> > front_t;

    modifier_t _fit_modifier;
    ea_pop_t _pop;

    ArchiveTask(){
    	nr_of_bins = Params::cmoea::nb_of_bins;
    }

    void run(boost::shared_ptr<boost::mpi::communicator> _world,
    		boost::mpi::status s,
			boost::shared_ptr<boost::mpi::environment> env)
    {
        dbg::trace trace("ea", DBG_HERE);
        pop_t pop;
        pop_t archive;
        pop_t new_bin;
        indiv_t temp;

        DBOW << " receiving broadcast from world: " << _world << std::endl;
        better_broadcast(_world, pop);
        DBOW << " received pop of size: " << pop.size() << std::endl;

        while(true){
        	DBOW << " waiting for message in cmoea_nsga2_mpi.hpp" << std::endl;
            s = _world->probe();
            DBOW << " receveived message in cmoea_nsga2_mpi.hpp tag: " <<
            		s.tag() << " source: " << s.source()  << std::endl;
            if (s.tag() == env->max_tag()){
                break;
            }

            DBOW << " receiving archive." << std::endl;
            _world->recv(0, s.tag(), archive);
            DBOW << " archive received." << std::endl;

            for(size_t j = 0; j < pop.size(); ++j){
                archive.push_back(pop[j]);
            }
            DBOW << " applying modifier" << std::endl;
            _apply_modifier(archive);
            _cat_to_obj(archive, s.tag());
            DBOW << " sorting" << std::endl;
            _fill_nondominated_sort(archive, new_bin);
            DBOW << " sending bin: " << s.tag() <<
            		" size: " << new_bin.size() << std::endl;
            _world->send(0, s.tag(), new_bin);
        }
    }

    // modifiers
   void apply_modifier()
   { boost::fusion::for_each(_fit_modifier, ApplyModifier_f<this_t>(*this)); }

   const ea_pop_t& pop() const { return _pop; };
   ea_pop_t& pop() { return _pop; };

protected:
    /**
     * Converts a population from array individuals to regular individuals.
     */
    void _convert_pop(const pop_t& pop1, ea_pop_t& pop2){
        dbg::trace trace("ea", DBG_HERE);
        pop2.resize(pop1.size());
        for (size_t i = 0; i < pop1.size(); ++i){
            pop2[i] = pop1[i];
        }
    }

    /**
     * Converts a population from regular individuals to crowd individuals.
     */
    void _convert_pop(const ea_pop_t& pop1, pop_t& pop2){
        dbg::trace trace("ea", DBG_HERE);
        pop2.resize(pop1.size());
        for (size_t i = 0; i < pop1.size(); ++i){
            pop2[i] = boost::shared_ptr<crowd_t>(new crowd_t(*pop1[i]));
        }
    }

    /**
     * Does not actually convert anything, merely copies the content from one
     * pop to the other pop.
     */
    void _convert_pop(const pop_t& pop1, pop_t& pop2){
        dbg::trace trace("ea", DBG_HERE);
        pop2.resize(pop1.size());
        for (size_t i = 0; i < pop1.size(); ++i){
            pop2[i] = pop1[i];
        }
    }

    /**
     * Applies the modifier to the supplied population (vector of individuals).
     *
     * Note that this overwrites the this->_pop population.
     */
    void _apply_modifier(pop_t pop){
        dbg::trace trace("ea", DBG_HERE);
        _convert_pop(pop, this->_pop);
        apply_modifier();
    }

    /**
     * Selects a random individual from the supplied population.
     */
    indiv_t _selection(const pop_t& pop){
        dbg::trace trace("ea", DBG_HERE);
        int x1 = misc::rand< int > (0, pop.size());
        dbg::check_bounds(dbg::error, 0, x1, pop.size(), DBG_HERE);
        return pop[x1];
    }

    /**
     * Takes a mixed population, sorts it according to Pareto dominance, and
     * generates a new population depending on the bin size.
     *
     * @Param mixed_pop The mixed population from which to select.
     *                  The mixed population must be larger than the bin_size
     *                  for selection to occur.
     * @Param new_pop   Output parameter. After execution, should contain a
     * 					number of individuals equal to the bin_size, selected
     * 					based on Pareto dominance first, crowding second.
     */
    void _fill_nondominated_sort(pop_t& mixed_pop, pop_t& new_pop)
    {
        dbg::trace trace("ea", DBG_HERE);
        dbg::out(dbg::info, "ea") << "Mixed pop size: " << mixed_pop.size() <<
        		" bin size: " << bin_size << std::endl;
        dbg::assertion(DBG_ASSERTION(mixed_pop.size()));
        dbg::assertion(DBG_ASSERTION(mixed_pop.size() >= bin_size));

        //Rank the population according to Pareto fronts
        front_t fronts;
        _rank_crowd(mixed_pop, fronts);

        //Add Pareto layers to the new population until the current layer no
        //longer fits
        new_pop.clear();
        size_t front_index = 0;
        while(fronts[front_index].size() + new_pop.size() < bin_size){
            new_pop.insert(new_pop.end(), fronts[front_index].begin(),
            		fronts[front_index].end());
            ++front_index;
        }

        // sort the last layer
        size_t size_remaining = bin_size - new_pop.size();
        if (size_remaining > 0){
            dbg::assertion(DBG_ASSERTION(front_index < fronts.size()));
            std::sort(fronts[front_index].begin(), fronts[front_index].end(),
            		crowd::compare_crowd());
            for (size_t k = 0; k < size_remaining; ++k){
                new_pop.push_back(fronts[front_index][k]);
            }
        }
        dbg::assertion(DBG_ASSERTION(new_pop.size() == bin_size));
    }

    // --- rank & crowd ---

    /**
     * Ranks and crowds a population.
     *
     * Takes a population and divides it based on objectives.
     *
     * @param pop    The population to be ranked.
     * @param fronts The resulting Pareto fronts will be stored here.
     */
    void _rank_crowd(pop_t& pop, front_t& fronts)
    {
        dbg::trace trace("ea", DBG_HERE);
        //Execute ranking based on dominance
        std::vector<size_t> ranks;
        dom_sort_f()(pop, fronts, non_dom_f(), ranks);

        //Why are we assigning a crowd score to every individual?
        parallel::p_for(parallel::range_t(0, fronts.size()),
        		crowd::assign_crowd<indiv_t >(fronts));
    }

    /**
     * For the specified category and array, copies the category score to the
     * obj_index (usually 1).
     */
    void _cat_to_obj(pop_t& bin, size_t bin_i){
        dbg::trace trace("ea", DBG_HERE);
        for(size_t i=0; i<bin.size(); ++i){
        	float fit = bin[i]->fit().getBinFitness(bin_i);
            bin[i]->fit().set_obj(obj_index, fit);
        }
    }

    /**
     * Copies the stored diversity back to the relevant objective.
     *
     * Does nothing when DIV is not defined
     */
    void _div_to_obj(pop_t& bin, size_t category){
        dbg::trace trace("ea", DBG_HERE);
#if defined(DIV)
        for(size_t i=0; i<bin.size(); ++i){
            size_t div_index = bin[i]->fit().objs().size() - 1;
            float div = bin[i]->fit().getBinDiversity(category);
            bin[i]->fit().set_obj(div_index, div);
        }
#endif
    }

    /**
     * Copies the calculated diversity to the individuals diversity array.
     *
     * Does nothing when DIV is not defined
     */
    void _obj_to_div(pop_t& bin, size_t category){
        dbg::trace trace("ea", DBG_HERE);
#if defined(DIV)
        for(size_t j = 0; j < bin.size(); ++j){
            bin[j]->fit().initDiv();
            size_t div_index = bin[j]->fit().objs().size() - 1;
            bin[j]->fit().setDiv(category, bin[j]->fit().obj(div_index));
        }
#endif
    }
};

// Main class
SFERES_EA(CmoeaNsga2Mpi, CmoeaNsga2){
public:
    //Params

    //The type of Pareto domination sort to use.
    //Currently available types are:
    // - sferes::ea::dom_sort_basic_f
	//   (defined in sferes/ea/dom_sort_basic.hpp)
    //   Sorts according to pareto dominance and will add individuals from the
	//   highest to the lowest layer, with crowding as a tie-breaker in the last
	//   layer to be added.
    // - sferes::ea::dom_sort_no_duplicates_f
	//   (defined in sferes/ea/dom_sort_no_duplicates.hpp)
    //   Same as dom_sort_basic, accept that, for each front, only one
	//   individual per pareto location is added. Other individuals at the same
	//   location will be bumped to the next layer.
    typedef typename Params::ea::dom_sort_f dom_sort_f;

    //The type non dominated comparator to use
    //Currently available types are:
    // - sferes::ea::_dom_sort_basic::non_dominated_f
    //   (defined in sferes/ea/dom_sort_basic.hpp)
    //   Regular comparisons based on dominance
    // - sferes::ea::cmoea_nsga::prob_dom_f<Params>
    //   Comparisons based on probabilistic sorting, where some objectives can
    //   be stronger than others.
    typedef typename Params::cmoea_nsga::non_dom_f non_dom_f;

    //The index used to temporarily store the category
    //This index should hold a dummy value, as it will be overwritten
    //constantly. The default would be 0.
    static const size_t obj_index = Params::cmoea::obj_index;

    //The number of objectives (bins) used in the map
    size_t nr_of_bins;

    //The size of each bin
    static const size_t bin_size = Params::cmoea::bin_size;

    //The number of individuals initially generated to fill the archive
    //If equal to the bin_size, every initially generated individual is added
    //to every bin of every category.
    //The init_size has to be greater than or equal to the bin_size
    static const size_t init_size = Params::pop::init_size;

    // Very large initial populations may cause CMOEA to run out of memory.
    // To avoid this, you can add the initial populations in init_batch batches
    // of init_size.
//    static const size_t init_batch = Params::pop::init_batch;

    //Not actually the size of the standing population
    //(which is bin_size*nr_of_bins)
    //but the number of individuals that are generated each epoch.
    //Because individuals are always produced in pairs,
    //pop_size has to be divisible by 2.
    static const size_t indiv_per_gen = Params::pop::select_size;

    typedef Phen phen_t;
    typedef crowd::Indiv<phen_t> crowd_t;
    typedef boost::shared_ptr<crowd_t> indiv_t;
    typedef std::vector<indiv_t> pop_t;
    typedef boost::shared_ptr<phen_t> raw_indiv_t;
    typedef std::vector<raw_indiv_t> raw_pop_t;
    typedef typename std::vector<pop_t> front_t;
    typedef std::vector<pop_t> array_t;
    SFERES_EA_FRIEND(CmoeaNsga2Mpi);

    CmoeaNsga2Mpi()
    {
        dbg::trace trace("ea", DBG_HERE);
        //dbg::compile_assertion<pop_size%2 == 0>("Population size has to be
        //divisible by 2.");
        DBOE << "Objectives: " << nr_of_bins << std::endl;
        nr_of_bins = Params::cmoea::nb_of_bins;
        this->_array.resize(nr_of_bins);
    }

//    void random_pop()
//    {
//        dbg::trace trace("ea", DBG_HERE);
//        DBOE << "Generating random pop" << std::endl;
//        //Create and evaluate the initial random population
//        parallel::init();
//        for(unsigned j=0; j<init_batch; ++j){
//            pop_t pop;
//            pop.resize(init_size);
//            int i = 0;
//            BOOST_FOREACH(indiv_t& indiv, pop)
//            {
//            	DBOE << "Creating random individual: " << i++ << std::endl;
//                indiv = indiv_t(new crowd_t());
//                indiv->random();
//            }
//            DBOE << "Evaluating population" << std::endl;
//            this->_eval.eval(pop, 0, pop.size(), this->_fit_proto);
//
//            DBOE << "Applying modifier" << std::endl;
//            _apply_modifier(pop);
//
//            DBOE << "Adding to archive" << std::endl;
//            _add_to_archive(pop);
//        }
//        DBOE << "Generating random pop done" << std::endl;
//
//        DBG_CONDITIONAL(dbg::info, "archive", this->_init_debug_array());
//    }
//
//    void epoch(){
//        dbg::trace trace("ea", DBG_HERE);
//
//        //We are creating and selecting a number of individuals equal to the
//        //population size. A simpler variant would only select and mutate one
//        //individual
//        pop_t ptmp;
//        for(size_t i=0; i<_array.size(); ++i){
//            if(_array[i].size() != bin_size){
//                std::cout << "Before reproduction: bin " << i <<
//                		" contains only " << _array[i].size() <<
//						" indiv." <<std::endl;
//            }
//        }
//        for (size_t i = 0; i < (indiv_per_gen/2); ++i)
//        {
//        	DBOE << "Creating individual: " << i <<std::endl;
//            indiv_t p1 = _selection(_array);
//            indiv_t p2 = _selection(_array);
//            indiv_t i1, i2;
//
//            p1->cross(p2, i1, i2);
//            i1->mutate();
//            i2->mutate();
//            ptmp.push_back(i1);
//            ptmp.push_back(i2);
//        }
//        this->_eval.eval(ptmp, 0, ptmp.size(), this->_fit_proto);
//        _add_to_archive(ptmp);
//
//        for(size_t i=0; i<_array.size(); ++i){
//            if(_array[i].size() != bin_size){
//                std::cout << "After reproduction: bin " << i <<
//                		" contains only " << _array[i].size() <<
//						" indiv." <<std::endl;
//            }
//        }
//
//        //For writing statistics only from the first bin:
//        _convert_pop(_array, this->_pop);
//
//        DBG_CONDITIONAL(dbg::info, "archive", this->_print_archive());
//    }
//
//    const array_t& archive() const { return _array; }

    /**
     * Adds the new `population' (vector of individuals) to the archive by
     * adding every individual to every bin, and then running NSGA 2 (or pNSGA)
     * selection on every bin.
     */
    void add_to_archive(pop_t& pop){
        dbg::trace trace("ea", DBG_HERE);
        //Add everyone to the archive

        //Set new task
        for (size_t i = 1; i < this->eval().world()->size(); ++i){
            this->eval().world()->send(i, this->eval().env()->max_tag(), 1);
        }

        //Broadcast the new individuals to all workers
        DBOM << "Broadcasting population of size: " <<
        		pop.size() << " to world: " << this->eval().world() <<
				std::endl;
        better_broadcast(this->eval().world(), pop);

        size_t current = 0;
        std::vector<bool> done(nr_of_bins);
        std::fill(done.begin(), done.end(), false); // @suppress("Ambiguous problem")
//        std::fill<std::vector<bool>::iterator>(done.begin(), done.end(), false);

        // first round
        size_t world_size = this->eval().world()->size();
        for (size_t i = 1; i < world_size && current < nr_of_bins; ++i) {
        	DBOM << "Sending bin: " << current  <<
        			" to worker " << i << std::endl;
            this->eval().world()->send(i, current, this->_array[current]);
            ++current;
        }

        // Subsequent rounds
        while (current < nr_of_bins){
            boost::mpi::status s = this->eval().world()->probe();
            DBOM << "Receiving bin: " << s.tag() << std::endl;
            this->eval().world()->recv(s.source(), s.tag(), this->_array[s.tag()]);
            DBOM << "Received bin: " << s.tag() <<
            		" size: " << this->_array[s.tag()].size()  << std::endl;
            done[s.tag()] = true;
            DBOM << "Sending bin: " << current << std::endl;
            this->eval().world()->send(s.source(), current, this->_array[current]);
            ++current;
        }

        // Join
        bool all_done = true;
        do{
        	DBOM << "joining..."<<std::endl;
            all_done = true;
            for (size_t i = 0; i < nr_of_bins; ++i){
                if (!done[i]){
                    boost::mpi::status s = this->eval().world()->probe();
                    DBOM << "Receiving bin: " << s.tag() << std::endl;
                    this->eval().world()->recv(s.source(), s.tag(),
                    		this->_array[s.tag()]);
                    DBOM << "Received bin: " << s.tag() <<
                    		" size: " << this->_array[s.tag()].size()  << std::endl;
                    done[s.tag()] = true;
                    all_done = false;
                }
            }
        }
        while (!all_done);
    }

protected:
//    array_t _array;
//
//    /**
//     * Converts a population from array individuals to regular individuals.
//     */
//    void _convert_pop(const pop_t& pop1, raw_pop_t& pop2){
//        dbg::trace trace("ea", DBG_HERE);
//        pop2.resize(pop1.size());
//        for (size_t i = 0; i < pop1.size(); ++i){
//            pop2[i] = pop1[i];
//        }
//    }
//
//    /**
//     * Converts the entire array of individuals to regular individuals.
//     */
//    void _convert_pop(const array_t& array, raw_pop_t& pop2){
//        dbg::trace trace("ea", DBG_HERE);
//        pop2.resize(array.size() * bin_size);
//        size_t k=0;
//        for (size_t i = 0; i < array.size(); ++i){
//          for (size_t j = 0; j < bin_size; ++j){
//            pop2[k++] = array[i][j];
//          }
//        }
//    }
//
//    /**
//     * Converts a population from regular individuals to crowd individuals.
//     */
//    void _convert_pop(const raw_pop_t& pop1, pop_t& pop2){
//        dbg::trace trace("ea", DBG_HERE);
//        pop2.resize(pop1.size());
//        for (size_t i = 0; i < pop1.size(); ++i){
//            pop2[i] = boost::shared_ptr<crowd_t>(new crowd_t(*pop1[i]));
//        }
//    }
//
//    /**
//     * Initialize the archive give a population.
//     */
//    void _set_pop(const raw_pop_t& pop) {
//        dbg::trace trace("ea", DBG_HERE);
//        pop_t converted_pop;
//        this->_convert_pop(pop, converted_pop);
//
//        dbg::out(dbg::info, "continue") << "Adding: " << converted_pop.size()
//                << " to archive: " << this-> _array.size()
//                << " by " << this->bin_size << std::endl;
//        dbg::assertion(DBG_ASSERTION(pop.size() == converted_pop.size()));
//        dbg::assertion(DBG_ASSERTION(converted_pop.size() ==
//        		this->_array.size()*this->bin_size));
//
//        //Add everyone to the archive in the appropriate place
//        size_t pop_index = 0;
//        for(size_t i=0; i<this->_array.size(); ++i){
//            for(size_t j=0; j<this->bin_size; ++j){
//                this->_array[i].push_back(converted_pop[pop_index]);
//                ++pop_index;
//            }
//        }
//
//        DBG_CONDITIONAL(dbg::info, "archive", this->_init_debug_array());
//    }
//
//    /**
//     * Applies the modifier to the supplied population (vector of individuals).
//     *
//     * Note that this overwrites the this->_pop population.
//     */
//    void _apply_modifier(pop_t pop){
//        dbg::trace trace("ea", DBG_HERE);
//        _convert_pop(pop, this->_pop);
//        this->apply_modifier();
//    }
//
//
//
//    /**
//     * Selects a random individual from the supplied population.
//     */
//    indiv_t _selection(const pop_t& pop){
//        dbg::trace trace("ea", DBG_HERE);
//        int x1 = misc::rand< int > (0, pop.size());
//        dbg::check_bounds(dbg::error, 0, x1, pop.size(), DBG_HERE);
//        return pop[x1];
//    }
//
//    /**
//     * Selects a random individual from the supplied archive
//     */
//    indiv_t _selection(const array_t& archive){
//        dbg::trace trace("ea", DBG_HERE);
//        size_t category = misc::rand< size_t > (0, archive.size());
//        dbg::check_bounds(dbg::error, 0, category, archive.size(), DBG_HERE);
//        size_t size = archive[category].size();
//        size_t indiv_i = misc::rand< size_t > (0, size);
//        dbg::check_bounds(dbg::error, 0, indiv_i, size, DBG_HERE);
//        return archive[category][indiv_i];
//    }
//
//    /**
//     * Takes a mixed population, sorts it according to Pareto dominance, and
//     * generates a new population depending on the bin size.
//     *
//     * @Param mixed_pop The mixed population from which to select.
//     *                  The mixed population must be larger than the bin_size
//     *                  for selection to occur.
//     * @Param new_pop   Output parameter. After execution, should contain a
//     * 					number of individuals equal to the bin_size, selected
//     * 					based on Pareto dominance first, crowding second.
//     */
//    void _fill_nondominated_sort(pop_t& mixed_pop, pop_t& new_pop)
//    {
//        dbg::trace trace("ea", DBG_HERE);
//        dbg::assertion(DBG_ASSERTION(mixed_pop.size()));
//
//        //Rank the population according to Pareto fronts
//        front_t fronts;
//        _rank_crowd(mixed_pop, fronts);
//
//        //Add Pareto layers to the new population until the current layer no
//        //longer fits
//        new_pop.clear();
//        size_t front_index = 0;
//        while(fronts[front_index].size() + new_pop.size() < bin_size){
//            new_pop.insert(new_pop.end(), fronts[front_index].begin(),
//            		fronts[front_index].end());
//            ++front_index;
//        }
//
//        // sort the last layer
//        size_t size_remaining = bin_size - new_pop.size();
//        if (size_remaining > 0){
//            dbg::assertion(DBG_ASSERTION(front_index < fronts.size()));
//            std::sort(fronts[front_index].begin(), fronts[front_index].end(),
//            		crowd::compare_crowd());
//            for (size_t k = 0; k < size_remaining; ++k){
//                new_pop.push_back(fronts[front_index][k]);
//            }
//        }
//        dbg::assertion(DBG_ASSERTION(new_pop.size() == bin_size));
//    }
//
//    // --- rank & crowd ---
//
//    /**
//     * Ranks and crowds a population.
//     *
//     * Takes a population and divides it based on objectives.
//     *
//     * @param pop    The population to be ranked.
//     * @param fronts The resulting Pareto fronts will be stored here.
//     */
//    void _rank_crowd(pop_t& pop, front_t& fronts)
//    {
//        dbg::trace trace("ea", DBG_HERE);
//        //Execute ranking based on dominance
//        std::vector<size_t> ranks;
//        dom_sort_f()(pop, fronts, non_dom_f(), ranks);
//
//        //Why are we assigning a crowd score to every individual?
//        parallel::p_for(parallel::range_t(0, fronts.size()),
//        		crowd::assign_crowd<indiv_t >(fronts));
//    }
//
//
//    /**
//     * For the specified category and array, copies the category score to the
//     * obj_index (usually 1).
//     */
//    void _cat_to_obj(array_t& array, size_t category){
//        dbg::trace trace("ea", DBG_HERE);
//        dbg::check_bounds(dbg::error, 0, category, array.size(), DBG_HERE);
//        for(size_t i=0; i<array[category].size(); ++i){
//            array[category][i]->fit().set_obj(obj_index,
//            		array[category][i]->fit().getBinFitness(category));
//        }
//    }
//
//    /**
//     * Copies the stored diversity back to the relevant objective.
//     *
//     * Does nothing when DIV is not defined
//     */
//    void _div_to_obj(array_t& array, size_t category){
//        dbg::trace trace("ea", DBG_HERE);
//#if defined(DIV)
//        for(size_t i=0; i<bin_size; ++i){
//            size_t div_index = _array[category][i]->fit().objs().size() - 1;
//            array[category][i]->fit().set_obj(div_index,
//            		array[category][i]->fit().getBinDiversity(category));
//        }
//#endif
//    }
//
//    /**
//     * Copies the calculated diversity to the individuals diversity array.
//     *
//     * Does nothing when DIV is not defined
//     */
//    void _obj_to_div(array_t& array, size_t category){
//        dbg::trace trace("ea", DBG_HERE);
//#if defined(DIV)
//        for(size_t j = 0; j < array[category].size(); ++j){
//            array[category][j]->fit().initDiv();
//            size_t div_index = array[category][j]->fit().objs().size() - 1;
//            array[category][j]->fit().setDiv(category,
//            		array[category][j]->fit().obj(div_index));
//        }
//#endif
//    }
//
//    //Debug functions
//#ifdef DBG_ENABLED
//    array_t _debug_array;
//
//    enum array_type{
//        current_array,
//        debug_array
//    };
//
//    /**
//     * Prints the fitness and closest distance values for each position in the
//     * archive.
//     *
//     * Note: requires the _debug_array to be set, otherwise it will throw a
//     * segmentation fault
//     */
//    void _print_archive(){
//        dbg::trace trace("ea", DBG_HERE);
//        std::cout << "Old archive:" << std::endl;
//        _print_array(current_array);
//        std::cout << "New archive:" << std::endl;
//        _print_array(debug_array);
//        _init_debug_array();
//    }
//
//    /**
//     * Print the debug array.
//     */
//    void _print_array(array_type type){
//    	using namespace karma;
//        for(size_t i=0; i<nr_of_bins; ++i){
//            _cat_to_obj(_array, i);
//            _div_to_obj(_array, i);
//            _cat_to_obj(_debug_array, i);
//            _div_to_obj(_debug_array, i);
//
//            pop_t temp_current = _array[i];
//            pop_t temp_debug = _debug_array[i];
//
//            compare::sort(temp_current, compare::pareto_objs().descending());
//            compare::sort(temp_debug, compare::pareto_objs().descending());
//
//            for(size_t j=0; j<bin_size; ++j){
//                std::cout << "(";
//                for(size_t k=0; k<temp_current[j]->fit().objs().size(); ++k){
//                	float obj_old = temp_debug[j]->fit().obj(k);
//                	float obj_new = temp_current[j]->fit().obj(k);
//                    bool better = obj_old < obj_new;
//                    bool worse = obj_old > obj_new;
//                    if(better) std::cout << COL_GREEN;
//                    if(worse) std::cout << COL_MAGENTA;
//                    float value = temp_current[j]->fit().obj(k);;
//                    if(type == debug_array) value = temp_debug[j]->fit().obj(k);
//                    std::cout << format(
//                    		left_align(5, '0')[maxwidth(5)[double_]],
//                    		value);
//                    std::cout << END_COLOR;
//                    if(k+1 != _array[i][j]->fit().objs().size()) std::cout << ":";
//                }
//                std::cout << ") ";
//            }
//            std::cout << std::endl;
//        }
//    }
//
//    /**
//     * Sets the debug array.
//     *
//     * Required for debugging at the end of the random pop and load population
//     * functions.
//     */
//    void _init_debug_array(){
//        dbg::trace trace("ea", DBG_HERE);
//        for(size_t i=0; i<nr_of_bins; ++i){
//            _debug_array[i].resize(bin_size);
//            for(size_t j=0; j<bin_size; ++j){
//                _debug_array[i][j] = indiv_t(new crowd_t(*_array[i][j]));
//            }
//        }
//    }
//
//#endif //DBG_ENABLED
};
}
}

// Undefine everything
#undef DBOM
#undef DBOW
#undef DBOE

#endif /* MODULES_CMOEA_CMOEA_NSGA2_MPI_HPP_ */


/*
The MIT License

Copyright (c) 2013 Radhika S. Saksena radhika.saksena@gmail.com,
                   Kosuke Imai kimai@princeton.edu

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#include "Analytics.h"
#include "ZeroTradeModelIO.h"
#include <numeric>
#include <algorithm>
#include <sstream>
#include <iostream>
#include <boost/random.hpp>
#include <time.h>

using namespace std;
using namespace boost;

/*
 * using namespace Rcpp;
*/

bool sortFunc(pair<int, long double> p1, pair<int, long double> p2);

// [[Rcpp::export]]
int ztm(int nsim, int maxiter, long double eps, int numModelsM, long double nu, long double OP_w_ij_inp,
         string inputYears, int numThreads, int total_mc_trials, string q_filename, string mu_filename,
	string sigmasq_filename, string pi_filename, string zeta_filename, int seedOffset) {

  // int numCountries, numProductsK, numModelsM;
  // long double nu;
  // long double OP_w_ij_inp;
  //long double nu, bayes_beta, bayes_alpha, bayes_tau;
  // string tmpString;
  vector<int> nemptyIdx;
  // int success=0;
  // int numThreads=8;
  // int numModelsM;
  // int  maxiter;
  // int nsim, total_mc_trials
  int mc_trials;
  // bool successFlag = 0;
  bool resetFlag = 0;
  //string inputFilename;
  // string inputYears;
  ofstream logFile;
  // ifstream configFile;
  long double max_ll_new;
  int max_sim;
  // long double eps;
  bool checkpointFlag = 0;
  vector<pair<int, long double> > simDF;
  bool* isSuccess;
  // string q_filename,mu_filename,sigmasq_filename,pi_filename,zeta_filename;
  // int np
  int rank=0;
  long double** all_TFProportions;
  // double start_time, end_time;

     cout<<"read in: nsim = "<<nsim<<endl;
     cout<<"read in: maxiter = "<<maxiter<<endl;
     cout<<"read in: eps = "<<eps<<endl;
     cout<<"read in: Z = "<<numModelsM<<endl;
     cout<<"read in: nu = "<<nu<<endl;
     cout<<"read in: OP_w_ij_inp = "<<OP_w_ij_inp<<endl;
     cout<<"read in: q checkpoint file = "<<q_filename<<endl;
     cout<<"read in: mu checkpoint file = "<<mu_filename<<endl;
     cout<<"read in: sigmasq checkpoint file = "<<sigmasq_filename<<endl;
     cout<<"read in: pi checkpoint file = "<<pi_filename<<endl;
     cout<<"read in: zeta checkpoint file = "<<zeta_filename<<endl;
     //cout<<"read in: input file = "<<inputFilename<<endl;
     cout<<"read in: input years = "<<inputYears<<endl;
     cout<<"read in: number of threads = "<<numThreads<<endl;

     cout<<"input: random seed = "<<seedOffset<<endl;

     isSuccess = (bool *)malloc(sizeof(bool)*nsim);

     omp_set_num_threads(numThreads);
     cout<<"Before call to ZeroTradeModel constructor."<<endl;
	 //ZeroTradeModelIO trade1(numModelsM,nu,OP_w_ij_inp,inputFilename,numThreads);
	 ZeroTradeModelIO trade1(numModelsM,nu,OP_w_ij_inp,inputYears,maxiter,numThreads);

     cout<<"After call to ZeroTradeModel constructor."<<endl;
     logFile.open("ZeroTradeModel.log");

#if EXITONZEROVARIANCE
     cout<<"Algorithm exits when it encounters zero variance."<<endl;
     logFile<<"Algorithm exits when it encounters zero variance."<<endl;
#endif
#if FLIPDYADPAIRS
     cout<<"Dyad pairs may be flipped."<<endl;
     logFile<<"Dyad pairs may be flipped."<<endl;
#endif
#if PRIOR_OP
     cout<<"Prior is applied."<<endl;
     logFile<<"Prior is applied."<<endl;
     logFile<<"Executing: ZeroTradeModel with the following parameters: ";
#endif
#ifndef PRIOR_OP
     logFile<<"Prior is not applied."<<endl;
     logFile<<"Executing: ZeroTradeModel with the following parameters: ";
#endif
     logFile<<"read in: maxiter = "<<maxiter<<endl;
     logFile<<"read in: eps = "<<eps<<endl;
     logFile<<"read in: Z = "<<numModelsM<<endl;
     logFile<<"read in: nu = "<<nu<<endl;
     //logFile<<"read in: input file = "<<inputFilename<<endl;
     logFile<<"read in: input years = "<<inputYears<<endl;
     logFile<<"read in: number of threads = "<<numThreads<<endl;

     max_ll_new = -9999999999;
     max_sim = 0;

     for(int isim = 0; isim  < nsim; isim++){

         //int numThreads;
	 long double ll_new = 0.0;
         // long double ll_old = -9999999999999999;
         // double elapsed_time;
         time_t seed;
         // ofstream* clusterFiles = new ofstream[trade1.numModelsM];
         checkpointFlag = 0;

         cout<<endl<<endl<<"Running Simulation "<<isim<<"...."<<endl<<endl;;
         logFile<<endl<<endl<<"Running Simulation "<<isim<<"...."<<endl<<endl;;

         //seed = time(0);
         seed = isim;
         seed = seed + seedOffset;
         cout<<"Using seed = "<<seed<<" seedOffset = "<<seedOffset<<endl;
         //seed = 1377336905;
         //max for Y90

         //cout<<"Modified."<<endl;
         //logFile<<"Modified."<<endl;

         logFile<<"Using seed = "<<seed<<endl;

         //TODO: uncomment
        trade1.initializeModel(seed);
         //0-th iteration was done above
         //trade1.calcMu();

         for(int iter=1; iter < maxiter; iter++){
             cout<<"Running iteration: "<<iter<<endl;
             trade1.expectationTh(iter,logFile);
             cout<<"iter = "<<iter<<": After expectation."<<endl;
             trade1.maximizationTh(0,resetFlag,logFile);
             cout<<"iter = "<<iter<<": After maximization."<<endl;
             //long double ll_new = trade1.logLikelihoodTh(iter);
             //double start_time = omp_get_wtime();
             //long double ll_new = trade1.obsLogPosterior(iter);
             //double end_time = omp_get_wtime();
             cout<<"After ll = "<<ll_new<<endl;
         }//end of iter loop

        trade1.chkpClusters(isim,logFile);
        trade1.chkpModelParams(isim,logFile);

     }//end of isim loop

     omp_set_num_threads(numThreads);

     //mc_trials = (total_mc_trials/np)*np;
     mc_trials = total_mc_trials;
     cout<<"Before call to Analytics constructor."<<endl;
     Analytics mc1(mc_trials,numThreads,numModelsM,trade1.ntimes,trade1.numProdsK,q_filename,
                     mu_filename,sigmasq_filename,pi_filename,zeta_filename);
     cout<<"After call to Analytics constructor."<<endl;

     mc1.analytics(rank);

#pragma omp parallel for
     for(int ith = 0; ith < numThreads; ith++){
        for(int z = 0; z < numModelsM; z++){
            int tmpSum = 0;
            for(int t = 0; t < mc1.ntimes; t++)
            {
                tmpSum += mc1.clusterOcc[t][z].size();
            }
            if(!tmpSum) continue;
            mc1.doMC(z,rank);
        }
    }

     all_TFProportions = (long double**)malloc(numModelsM*sizeof(long double*));
     for(int z = 0; z < numModelsM; z++){
          all_TFProportions[z] = (long double*)malloc(2*trade1.numProdsK*sizeof(long double));
          for(int k = 0; k < 2*trade1.numProdsK; k++){
              all_TFProportions[z][k] = 0.0;
          }
     }

     for(int z = 0; z < numModelsM; z++){
         for(int k = 0; k < 2*trade1.numProdsK; k++){
             for(int ith = 0; ith < numThreads; ith++){
                all_TFProportions[z][k] += mc1.prodProportion[ith][z][k];
             }
         }
     }

     for(int z = 0; z < numModelsM; z++){
        for(int k = 0; k < 2*trade1.numProdsK; k++){
            all_TFProportions[z][k] /= (long double)numThreads;
        }
     }

    ostringstream outfileName;
    outfileName<<"TF_"<<rand()<<".txt";
    ofstream fout;
    fout.open(outfileName.str().c_str());

    //print header with product codes, where available
    fout<<"\"ClusterID\""<<"\t";
    for(int k = 0; k < 2*trade1.numProdsK-1; k++){
        string currProd = mc1.prodNames[k];
        fout<<"\""<<currProd<<"\""<<"\t";
    }
    string currProd = mc1.prodNames[2*trade1.numProdsK-1];
    fout<<"\""<<currProd<<"\""<<endl;

    for(int z = 0; z < numModelsM; z++){
        // long double *max_element_ptr = max_element(all_TFProportions[z],all_TFProportions[z]+2*trade1.numProdsK);
        int tmpSum = 0;
        for(int t = 0; t < trade1.ntimes; t++)
        {
            tmpSum += mc1.clusterOcc[t][z].size();
            cout<<"writing TF: t = "<<t<<" z = "<<z<<" tmpSum = "<<tmpSum<<endl;
        }
        if(!tmpSum) continue;
        fout<<z<<"\t";
        for(int k = 0; k < 2*trade1.numProdsK-1; k++){
            if(isnan(all_TFProportions[z][k])){
                cout<<"Found nan in TF proportions = "<<all_TFProportions[z][k]<<endl;
                exit(-1);
            }
                fout<<all_TFProportions[z][k]<<"\t";
        }
        if(isnan(all_TFProportions[z][2*trade1.numProdsK-1])){
            cout<<"Found nan in TF proportions = "<<all_TFProportions[z][2*trade1.numProdsK-1]<<endl;
            exit(-1);
        }
        fout<<all_TFProportions[z][2*trade1.numProdsK-1]<<endl;
    }

    fout.close();

    logFile.close();

    // HJ: return 1;
    cout << "It is all done!" << endl;
    return 0;
 }

 bool sortFunc(pair<int, long double> p1, pair<int, long double> p2){
     return(p1.second > p2.second);
 }

// [[Rcpp::export]]
int mainRcpp(string configTxt, int randomSeed) {

     long double nu;
     long double OP_w_ij_inp;
     string tmpString;
     int numThreads=8;
     int numModelsM;
     int maxiter;
     int nsim, total_mc_trials;
     string inputYears;
     ofstream logFile;
     ifstream configFile;
     long double eps;
     string q_filename,mu_filename,sigmasq_filename,pi_filename,zeta_filename;

     configFile.open(&configTxt[0u]);
     int seedOffset = randomSeed;

     if(!configFile){
         cout<<"Could not open config file."<<endl;
         exit(-1);
     }

    cout<<"configFile is "<< "./config.txt" <<endl;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> nsim;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> maxiter;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> eps;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> numModelsM;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> nu;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> OP_w_ij_inp;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> inputYears;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> numThreads;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> total_mc_trials;   //total MC trials

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> q_filename;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> mu_filename;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> sigmasq_filename;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> pi_filename;

     configFile >> tmpString;
     configFile >> tmpString;
     configFile >> zeta_filename;

     configFile.close();

     return ztm(nsim, maxiter, eps, numModelsM, nu, OP_w_ij_inp, inputYears, numThreads, total_mc_trials,
                q_filename, mu_filename, sigmasq_filename, pi_filename, zeta_filename, seedOffset);
}

#include <Rcpp.h>
// ztm
int ztm(int nsim, int maxiter, long double eps, int numModelsM, long double nu, long double OP_w_ij_inp, string inputYears, int numThreads, int total_mc_trials, string q_filename, string mu_filename, string sigmasq_filename, string pi_filename, string zeta_filename, int seedOffset);
RcppExport SEXP dynCluster_ztm(SEXP nsimSEXP, SEXP maxiterSEXP, SEXP epsSEXP, SEXP numModelsMSEXP, SEXP nuSEXP, SEXP OP_w_ij_inpSEXP, SEXP inputYearsSEXP, SEXP numThreadsSEXP, SEXP total_mc_trialsSEXP, SEXP q_filenameSEXP, SEXP mu_filenameSEXP, SEXP sigmasq_filenameSEXP, SEXP pi_filenameSEXP, SEXP zeta_filenameSEXP, SEXP seedOffsetSEXP) {
BEGIN_RCPP
    Rcpp::RObject __result;
    Rcpp::RNGScope __rngScope;
    Rcpp::traits::input_parameter< int >::type nsim(nsimSEXP);
    Rcpp::traits::input_parameter< int >::type maxiter(maxiterSEXP);
    Rcpp::traits::input_parameter< long double >::type eps(epsSEXP);
    Rcpp::traits::input_parameter< int >::type numModelsM(numModelsMSEXP);
    Rcpp::traits::input_parameter< long double >::type nu(nuSEXP);
    Rcpp::traits::input_parameter< long double >::type OP_w_ij_inp(OP_w_ij_inpSEXP);
    Rcpp::traits::input_parameter< string >::type inputYears(inputYearsSEXP);
    Rcpp::traits::input_parameter< int >::type numThreads(numThreadsSEXP);
    Rcpp::traits::input_parameter< int >::type total_mc_trials(total_mc_trialsSEXP);
    Rcpp::traits::input_parameter< string >::type q_filename(q_filenameSEXP);
    Rcpp::traits::input_parameter< string >::type mu_filename(mu_filenameSEXP);
    Rcpp::traits::input_parameter< string >::type sigmasq_filename(sigmasq_filenameSEXP);
    Rcpp::traits::input_parameter< string >::type pi_filename(pi_filenameSEXP);
    Rcpp::traits::input_parameter< string >::type zeta_filename(zeta_filenameSEXP);
    Rcpp::traits::input_parameter< int >::type seedOffset(seedOffsetSEXP);
    __result = Rcpp::wrap(ztm(nsim, maxiter, eps, numModelsM, nu, OP_w_ij_inp, inputYears, numThreads, total_mc_trials, q_filename, mu_filename, sigmasq_filename, pi_filename, zeta_filename, seedOffset));
    return __result;
END_RCPP
}
// mainRcpp
int mainRcpp(string configTxt, int randomSeed);
RcppExport SEXP dynCluster_mainRcpp(SEXP configTxtSEXP, SEXP randomSeedSEXP) {
BEGIN_RCPP
    Rcpp::RObject __result;
    Rcpp::RNGScope __rngScope;
    Rcpp::traits::input_parameter< string >::type configTxt(configTxtSEXP);
    Rcpp::traits::input_parameter< int >::type randomSeed(randomSeedSEXP);
    __result = Rcpp::wrap(mainRcpp(configTxt, randomSeed));
    return __result;
END_RCPP
}