/* -*-C++-*-
   (c) Copyright 2008, Hewlett-Packard Development Company, LP

   See the file named COPYING for license details
*/

#include <vector>

#include <boost/bind.hpp>

#include <analysis/nfs/common.hpp>
#include <analysis/nfs/HostInfo.hpp>

#include <Lintel/HashMap.hpp>
#include <Lintel/HashUnique.hpp>
#include <Lintel/LintelLog.hpp>
#include <Lintel/StatsCube.hpp>
#include <Lintel/StatsQuantile.hpp>

#include <DataSeries/GeneralField.hpp>

using namespace std;
using boost::format;
using boost::tuples::null_type;
using lintel::tuples::AnyPair;
using lintel::tuples::BitsetAnyTuple;
using lintel::HashTupleStats;
using lintel::StatsCube;
using dataseries::TFixedField;

//                   host,   is_send,   time, operation, is_request
typedef boost::tuple<int32_t, bool,   int32_t,  uint8_t,    bool>
HostInfoTuple;
static const size_t host_index = 0;
static const size_t is_send_index = 1;
static const size_t time_index = 2;
static const size_t operation_index = 3;
static const size_t is_request_index = 4;

namespace lintel { namespace tuples {
        template<> struct TupleHash<HostInfoTuple> {
            uint32_t operator()(const HostInfoTuple &v) const {
                BOOST_STATIC_ASSERT(boost::tuples::length<HostInfoTuple>::value == 5);
                uint32_t a = v.get<host_index>();
                uint32_t b = v.get<time_index>();
                uint32_t c = v.get<operation_index>() | (v.get<is_send_index>() ? 0x100 : 0)
                             | (v.get<is_request_index>() ? 0x200 : 0);
                return BobJenkinsHashMix3(a,b,c);
            }
        };

        template<> struct BitsetAnyTupleHash<HostInfoTuple> {
            uint32_t operator()(const BitsetAnyTuple<HostInfoTuple> &v) const {
                BOOST_STATIC_ASSERT(boost::tuples::length<HostInfoTuple>::value == 5);
                uint32_t a = v.any[host_index] ? 0 : v.data.get<host_index>();
                uint32_t b = v.any[time_index] ? 0 : v.data.get<time_index>();
                uint32_t c = (v.any[operation_index] ? 0 : v.data.get<operation_index>())
                             | (!v.any[is_send_index] && v.data.get<is_send_index>() ? 0x100 : 0)
                             | (!v.any[is_request_index] && v.data.get<is_request_index>() ? 0x200 : 0);
                return BobJenkinsHashMix3(a,b,c);
            }
        };
    } }

template<class T0, class T1>
struct ConsToAnyPairCons {
    typedef ConsToAnyPairCons<typename T1::head_type,
                              typename T1::tail_type> tail;
    typedef boost::tuples::cons<AnyPair<T0>, typename tail::type> type;
};

template<class T0>
struct ConsToAnyPairCons<T0, null_type> {
    typedef boost::tuples::cons<AnyPair<T0>, null_type> type;
};

template<class T>
struct TupleToAnyTuple
        : ConsToAnyPairCons<typename T::head_type, typename T::tail_type>
{ };

template<> struct HashMap_hash<const bool> {
    uint32_t operator()(const bool x) const {
        return x;
    }
};

template<> struct HashMap_hash<const unsigned char> {
    uint32_t operator()(const bool x) const {
        return x;
    }
};

void zeroAxisAdd(null_type, null_type) {
}

template<class HUT, class T>
void zeroAxisAdd(HUT &hut, const T &v) {
    hut.get_head().add(v.get_head());
    zeroAxisAdd(hut.get_tail(), v.get_tail());
}

void hutPrint(null_type, int) { }

template<class HUT>
void hutPrint(HUT &hut, int pos) {
    typedef typename HUT::head_type::iterator iterator;
    cout << format("begin tuple part %d:\n") % pos;
    if (false) {
        Stats chain_lens;
        hut.get_head().chainLengthStats(chain_lens);

        cout << format("  chain [%.0f,%.0f] %.2f +- %.2f\n")
                % chain_lens.min() % chain_lens.max() % chain_lens.mean()
                % chain_lens.stddev();
    }

    for (iterator i = hut.get_head().begin(); i != hut.get_head().end(); ++i) {
        cout << *i << " ";
    }
    cout << format("\nend tuple part %d:\n") % pos;

    hutPrint(hut.get_tail(), pos + 1);
}

double zeroCubeBaseCount(const null_type) {
    return 1;
}

template<class HUT>
double zeroCubeBaseCount(const HUT &hut) {
    return hut.get_head().size() * zeroCubeBaseCount(hut.get_tail());
}

template<class KeyBase, class BaseData, class Function>
void zeroWalk(const null_type, const null_type, KeyBase &key_base,
              const BaseData &base_data, Stats &null_stat,
              const Function &fn) {
    typedef typename BaseData::const_iterator iterator;
    iterator i = base_data.find(key_base);
    if (i == base_data.end()) {
        fn(key_base, null_stat);
    } else {
        fn(key_base, *(i->second));
    }
}

template<class HUT, class KeyTail, class KeyBase, class BaseData,
         class Function>
void zeroWalk(const HUT &hut, KeyTail &key_tail, KeyBase &key_base,
              const BaseData &base_data, Stats &null_stat,
              const Function &fn) {
    typedef typename HUT::head_type::const_iterator const_iterator;

    for (const_iterator i = hut.get_head().begin();
        i != hut.get_head().end(); ++i) {
        key_tail.get_head() = *i;

        zeroWalk(hut.get_tail(), key_tail.get_tail(), key_base,
                 base_data, null_stat, fn);
    }
}

// We do the rollup internal to this module rather than an external
// program so if we switch to using statsquantile the rollup will
// still work correctly.

// To graph:
// create a table like: create table nfs_perf (sec int not null, ops double not null);
// perl -ane 'next unless $F[0] eq "*" && $F[1] =~ /^\d+$/o && $F[2] eq "send" && $F[3] eq "*" && $F[4] eq "*";print "insert into nfs_perf values ($F[1], $F[5]);\n";' < output of nfsdsanalysis -l ## run. | mysql test
// use mercury-plot (from Lintel) to graph, for example:
/*
  unplot
  plotwith * lines
  plot 3600*round((sec - min_sec)/3600,0) as x, avg(ops/(2*60.0)) as y from nfs_perf, (select min(sec) as min_sec from nfs_perf) as ms group by round((sec - min_sec)/3600,0)
  plottitle _ mean ops/s
  plot 3600*round((sec - min_sec)/3600,0) as x, max(ops/(2*60.0)) as y from nfs_perf, (select min(sec) as min_sec from nfs_perf) as ms group by round((sec - min_sec)/3600,0)
  plottitle _ max ops/s
  plot 3600*round((sec - min_sec)/3600,0) as x, min(ops/(2*60.0)) as y from nfs_perf, (select min(sec) as min_sec from nfs_perf) as ms group by round((sec - min_sec)/3600,0)
  plottitle _ min ops/s
  gnuplot set xlabel "seconds"
  gnuplot set ylabel "operations (#requests+#replies)/2 per second"
  pngplot set-18.png
*/

namespace {
    const string str_send("send");
    const string str_recv("recv");
    const string str_request("request");
    const string str_response("response");
    const string str_star("*");
    const string str_null("null");
}

class HostInfo : public RowAnalysisModule {
  public:
    typedef HostInfoTuple Tuple;
    typedef StatsCube<Tuple>::MyAny BitsetAnyTuple;

    static constexpr double rate_quantile_maxerror = 0.005;
    // about 1/3 of a year with 1 second datapoints
    static constexpr unsigned rate_quantile_maxgroups = 10*1000*1000;
    struct Rates {
        StatsQuantile ops_rate;
        StatsQuantile bytes_rate;
        Rates() : ops_rate(rate_quantile_maxerror, rate_quantile_maxgroups),
                  bytes_rate(rate_quantile_maxerror, rate_quantile_maxgroups)
        {
            if (memory_usage == 0) {
                memory_usage = 2*ops_rate.memoryUsage();
            }
        }
        void add(double ops, double bytes) {
            ops_rate.add(ops);
            bytes_rate.add(bytes);
        }

        static size_t memory_usage;
    };

    typedef TupleToAnyTuple<Tuple>::type AnyTuple;

    Rates *createRates() {
        checkMemoryUsage();
        return new Rates();
    }

    HostInfo(DataSeriesModule &_source, const std::string &arg)
            : RowAnalysisModule(_source),
              packet_at(series, ""),
              payload_length(series, ""),
              op_id(series, "", Field::flag_nullable),
              nfs_version(series, "", Field::flag_nullable),
              source_ip(series,"source"),
              dest_ip(series, "dest"),
              is_request(series, ""),
              group_seconds(1),
              incremental_batch_size(100000),
              rate_hts(boost::bind(&HostInfo::createRates, this)),
              min_group_seconds(numeric_limits<int32_t>::max()),
              max_group_seconds(numeric_limits<int32_t>::min()),
              last_rollup_at(numeric_limits<int32_t>::min()),
              group_count(0), printed_base_header(false),
              printed_cube_header(false), printed_rates_header(false),
              print_rates_quantiles(true), sql_output(false),
              print_base(true), print_cube(true), zero_groups(0),
              last_reported_memory_usage(0)
    {
        // Usage: group_seconds[,{no_cube_time, no_cube_host,
        //                        no_print_base, no_print_cube}]+
        vector<string> args = split(arg, ",");
        group_seconds = stringToInteger<int32_t>(args[0]);
        SINVARIANT(group_seconds > 0);
        options["cube_time"] = true;
        options["cube_host"] = true;
        options["cube_host_detail"] = true;

        // Cubing over the unique_vals_tuple; is very expensive -- ~6x on
        // some simple initial testing.  Calculating the unique tuple is
        // very cheap (overlappping runtimes, <1% instruction count difference),
        // but if we're not going to use it, then there isn't any point.
        options["print_base"] = true;
        options["print_cube"] = true;
        options["print_rates"] = true;
        options["print_rates_quantiles"] = true;
        options["sql_output"] = false;
        options["sql_create_table"] = false;
        for (unsigned i = 1; i < args.size(); ++i) {
            if (prefixequal(args[i], "no_")) {
                INVARIANT(options.exists(args[i].substr(3)),
                          format("unknown option '%s'") % args[i]);
                options[args[i].substr(3)] = false;
            } else if (prefixequal(args[i], "incremental=")) {
                // TODO: make uint32ModArg in common.hpp, use it.
                incremental_batch_size = stringToInteger<uint32_t>(args[i].substr(12));
                LintelLogDebug("HostInfo", format("ibs=%d") % incremental_batch_size);
            } else {
                INVARIANT(options.exists(args[i]),
                          format("unknown option '%s'") % args[i]);
                options[args[i]] = true;
            }
        }
        INVARIANT(!options["print_rates"] || options["cube_time"],
                  "HostInfo: Can't print_rates unless we have cube_time enabled.");
        print_rates_quantiles = options["print_rates_quantiles"];
        if (!options["print_rates"]) {
            options["print_rates_quantiles"] = false;
        }
        sql_output = options["sql_output"];
        configCube();
        if (options["sql_create_table"]) {
            printCreateTables();
        }
        print_base = options["print_base"];
        print_cube = options["print_cube"];
        skip_cube_time = !options["cube_time"];
        skip_cube_host = !options["cube_host"];
        skip_cube_host_detail = !options["cube_host_detail"];
    }

    virtual ~HostInfo() { }

    static int32_t host(const Tuple &t) {
        return t.get<host_index>();
    }
    static string host(const BitsetAnyTuple &atuple) {
        if (atuple.any[host_index]) {
            return str_star;
        } else {
            return str(format("%08x") % host(atuple.data));
        }
    }
    static string sqlHost(const BitsetAnyTuple &atuple) {
        if (atuple.any[host_index]) {
            return str_null;
        } else {
            return str(format("0x%08x") % host(atuple.data));
        }
    }

    static int32_t time(const Tuple &t) {
        return t.get<time_index>();
    }

    static string time(const BitsetAnyTuple &atuple,
                       const std::string &unused_str = str_star) {
        if (atuple.any[time_index]) {
            return unused_str;
        } else {
            return str(format("%d") % time(atuple.data));
        }
    }

    static bool isSend(const Tuple &t) {
        return t.get<is_send_index>();
    }
    static const string &isSendStr(const Tuple &t) {
        return isSend(t) ? str_send : str_recv;
    }

    static string isSendStr(const BitsetAnyTuple &atuple) {
        if (atuple.any[is_send_index]) {
            return str_star;
        } else {
            return isSendStr(atuple.data);
        }
    }

    static uint8_t operation(const Tuple &t) {
        return t.get<operation_index>();
    }

    static const string &operationStr(const Tuple &t) {
        return unifiedIdToName(operation(t));
    }
    static string operationStr(const BitsetAnyTuple &atuple) {
        if (atuple.any[operation_index]) {
            return str_star;
        } else {
            return operationStr(atuple.data);
        }
    }

    static bool isRequest(const Tuple &t) {
        return t.get<is_request_index>();
    }
    static const string &isRequestStr(const Tuple &t) {
        return isRequest(t) ? str_request : str_response;
    }
    static const string &isRequestStr(const BitsetAnyTuple &atuple) {
        if (atuple.any[is_request_index]) {
            return str_star;
        } else {
            return isRequestStr(atuple.data);
        }
    }

    static string sqlify(const string &basestr) {
        if (basestr == str_star) {
            return str_null;
        } else {
            return str(format("'%s'") % basestr);
        }
    }

    void newExtentHook(const Extent &e) {
        if (series.getTypePtr() != NULL) {
            return; // already did this
        }
        const ExtentType::Ptr type = e.getTypePtr();
        if (type->getName() == "NFS trace: common") {
            SINVARIANT(type->getNamespace() == "" &&
                       type->majorVersion() == 0 &&
                       type->minorVersion() == 0);
            packet_at.setFieldName("packet-at");
            payload_length.setFieldName("payload-length");
            op_id.setFieldName("op-id");
            nfs_version.setFieldName("nfs-version");
            is_request.setFieldName("is-request");
        } else if (type->getName() == "Trace::NFS::common"
                   && type->versionCompatible(1,0)) {
            packet_at.setFieldName("packet-at");
            payload_length.setFieldName("payload-length");
            op_id.setFieldName("op-id");
            nfs_version.setFieldName("nfs-version");
            is_request.setFieldName("is-request");
        } else if (type->getName() == "Trace::NFS::common"
                   && type->versionCompatible(2,0)) {
            packet_at.setFieldName("packet_at");
            payload_length.setFieldName("payload_length");
            op_id.setFieldName("op_id");
            nfs_version.setFieldName("nfs_version");
            is_request.setFieldName("is_request");
        } else {
            FATAL_ERROR("?");
        }
    }

    static bool timeLessEqual(const BitsetAnyTuple &t,
                              int32_t max_group_seconds) {
        if (t.any[time_index]) {
            return false;
        } else {
            SINVARIANT(t.data.get<time_index>() <= max_group_seconds);

            return true;
        }
    }

    void checkMemoryUsage() {
        static LintelLog::Category category("HostInfo");

        if (!LintelLog::wouldDebug(category)) {
            return;
        }

        size_t memory_usage = base_data.memoryUsage()
                              + rates_cube.memoryUsage();
        memory_usage += rate_hts.memoryUsage()
                        + Rates::memory_usage * rate_hts.size();
        if (memory_usage > (last_reported_memory_usage + 32*1024*1024)) {
            LintelLogDebug
                    ("HostInfo", format("# HostInfo-%ds: memory usage %d bytes"
                                        " (%d + %d + %d + %d * %d) @ %ds")
                     % group_seconds % memory_usage
                     % base_data.memoryUsage() % rates_cube.memoryUsage()
                     % rate_hts.memoryUsage()
                     % Rates::memory_usage %  rate_hts.size()
                     % (max_group_seconds - min_group_seconds));
            last_reported_memory_usage = memory_usage;
        }
    }

    void rollupBatch() {
        base_data.fillHashUniqueTuple(unique_vals_tuple);
        rates_cube.cube(base_data);

        if (print_base) {
            base_data.walkOrdered
                    (boost::bind(&HostInfo::printBaseIncremental,
                                 this, _1, _2));
        }
        base_data.clear();

        if (options["print_rates"]) {
            rates_cube.walk(boost::bind(&HostInfo::rateRollupAdd, this, _1, _2));
        }
        if (print_cube) {
            rates_cube.walkOrdered
                    (boost::bind(&HostInfo::printCubeIncrementalTime, this, _1, _2));
        }
        rates_cube.prune(boost::bind(&HostInfo::timeLessEqual, _1, max_group_seconds));
        SINVARIANT(base_data.size() == 0);
        checkMemoryUsage();
    }

    void processGroup(int32_t seconds) {
        SINVARIANT(max_group_seconds >= min_group_seconds);
        if (seconds > min_group_seconds) {
            ++group_count;
        }
        if (base_data.size() >= incremental_batch_size) {
            LintelLogDebug
                    ("HostInfo", format("should incremental ]%d,%d[ bd=%d cube=%d ratescube=%d rate_hts=%d")
                     % min_group_seconds % max_group_seconds % base_data.size()
                     % rates_cube.size() % rates_cube.size() % rate_hts.size());

            last_rollup_at = max_group_seconds;
            rollupBatch();
        } else {
            LintelLogDebug("HostInfo", format("skip incremental %d < %d")
                           % base_data.size() % incremental_batch_size);
        }
    }

    void finalGroup() {
        --group_count;
        int32_t elapsed_seconds = max_group_seconds - min_group_seconds;
        SINVARIANT((elapsed_seconds % group_seconds) == 0);
        // Total group count is elapsed / group_seconds + 1, but we
        // ignore the first and last groups, so we end up with -1
        SINVARIANT(elapsed_seconds / group_seconds - 1 == group_count);

        rollupBatch();
        if (print_cube) {
            rates_cube.walkOrdered
                    (boost::bind(&HostInfo::printCubeIncrementalNonTime, this, _1, _2));
        }

    }

    void nextSecondsGroup(int32_t next_group_seconds) {
        if (next_group_seconds <= max_group_seconds) {

            cout << format("# HostInfo-%ds: WARNING, out of order requests, %d <= %d\n")
                    % group_seconds % next_group_seconds % max_group_seconds;
            INVARIANT(next_group_seconds == max_group_seconds - group_seconds,
                      "Only allowing rollback by one group");
            INVARIANT(last_rollup_at < (next_group_seconds - 2*group_seconds),
                      format("LRA=%d >= %d - %d; safety unclear")
                      % last_rollup_at % next_group_seconds
                      % (2*group_seconds));
            INVARIANT(base_data.size() < incremental_batch_size,
                      "Size overflow, disallowing");
            return;
        }
        // skip counting the first and last groups; we prune these
        // because we can't calculate the rates properly on them.
        INVARIANT(next_group_seconds > max_group_seconds,
                  format("out of order %d <= %d")
                  % next_group_seconds % max_group_seconds);

        if (max_group_seconds == numeric_limits<int32_t>::min()) {
            max_group_seconds = next_group_seconds - group_seconds;
        }
        for (max_group_seconds += group_seconds;
            max_group_seconds < next_group_seconds;
            max_group_seconds += group_seconds) {
            // Rarely we have no operations in some groups.
            LintelLogDebug("HostInfo", format("empty seconds @%d")
                           % max_group_seconds);
            ++zero_groups;
            unique_vals_tuple.get<time_index>().add(max_group_seconds);
            processGroup(max_group_seconds);
        }
        SINVARIANT(max_group_seconds == next_group_seconds);
        processGroup(max_group_seconds);
    }

    virtual void processRow() {
        int32_t seconds = packet_at.valSec();
        seconds -= seconds % group_seconds;
        uint8_t operation = opIdToUnifiedId(nfs_version.val(), op_id.val());

        // Lets see if we've moved onto the next time group...
        min_group_seconds = min(seconds, min_group_seconds);
        if (seconds != max_group_seconds) {
            nextSecondsGroup(seconds);
        }
        // redundant calculation, but useful for checking that all the
        // cube rollup stuff is working properly.
        payload_overall.add(payload_length.val());

        // sender...
        Tuple cube_key(source_ip.val(), true,
                       seconds, operation, is_request.val());
        base_data.add(cube_key, payload_length.val());

        // reciever...
        cube_key.get<host_index>() = dest_ip.val();
        cube_key.get<is_send_index>() = false;
        base_data.add(cube_key, payload_length.val());
    }

    void printCreateTables() {
        cout
                << "--- The cube 'any' field is represented as null\n"
                << "--- note that there is a 'null' operation which is different than null\n"
                << "create table nfs_hostinfo_cube (group_seconds int null,\n"
                << "                                host int null,\n"
                << "                                group_time int null,\n"
                << "                                direction enum('send', 'recv') null,\n"
                << "                                operation varchar(32) null,\n"
                << "                                op_dir enum('request','response') null,\n"
                << "                                group_count bigint not null,\n"
                << "                                mean_payload_bytes double not null,\n"
                << "                                unique key pkey (group_seconds, host, group_time, direction, operation, op_dir)\n"
                << ");\n"
                << "\n"
                << "create table nfs_hostinfo_rates (group_seconds int null,\n"
                << "                             host int null,\n"
                << "                             direction enum('send', 'recv') null,\n"
                << "                             operation varchar(32) null,\n"
                << "                             op_dir enum('request', 'response') null,\n"
                << "                             mean_operations_per_second double not null,\n"
                << "                             mean_payload_bytes_per_second double not null,\n"
                << "                                 unique key pkey (group_seconds, host, direction, operation, op_dir)\n"
                << ");\n"
                << "\n"
                << "create table nfs_hostinfo_rate_quantiles (group_seconds int null,\n"
                << "                                      host int null,\n"
                << "                                      direction enum('send', 'recv') null,\n"
                << "                                      operation varchar(32) null,\n"
                << "                                      op_dir enum('request', 'response') null,\n"
                << "                                      quantile double not null,\n"
                << "                                      operations_per_second double not null,\n"
                << "                                      payload_bytes_per_second double not null,\n"
                << "                                          unique key pkey (group_seconds, host, direction, operation, op_dir, quantile)\n"
                << ");\n"
                ;
    }

    void printBaseIncremental(const Tuple &t, Stats &v) {
        if (sql_output) {
            cout << format("insert into nfs_hostinfo_cube (group_seconds, host, group_time, direction, operation, op_dir, group_count, mean_payload_bytes) values (%d, %d, %d, '%s', '%s', '%s', %d, %.20g);\n")
                    % group_seconds % host(t) % time(t) % isSendStr(t)
                    % operationStr(t) % isRequestStr(t) % v.countll() % v.mean();
        } else {
            if (!printed_base_header) {
                cout << format("HostInfo %ds base: host     time        dir          op    op-dir   count mean-payload\n") % group_seconds;
                printed_base_header = true;
            }
            cout << format("HostInfo %ds base: %08x %10d %s %12s %8s %6lld %8.2f\n")
                    % group_seconds % host(t) % time(t) % isSendStr(t)
                    % operationStr(t) % isRequestStr(t) % v.countll() % v.mean();
        }
    }

    void printCubeIncremental(const BitsetAnyTuple &atuple, Stats &v) {
        if (atuple.any.none()) {
            return; // all values real, skip
        } else {
            if (!print_cube) {
                FATAL_ERROR("?");
            }
        }

        if (sql_output) {
            cout << format("insert into nfs_hostinfo_cube (group_seconds, host, group_time, direction, operation, op_dir, group_count, mean_payload_bytes) values (%d, %d, %d, %s, %s, %s, %d, %.20g);\n")
                    % group_seconds % sqlHost(atuple) % time(atuple, str_null)
                    % sqlify(isSendStr(atuple)) % sqlify(operationStr(atuple))
                    % sqlify(isRequestStr(atuple)) % v.countll() % v.mean();
        } else {
            if (!printed_cube_header) {
                cout << format("HostInfo %ds cube: host     time        dir          op    op-dir   count mean-payload\n") % group_seconds;
                printed_cube_header = true;
            }
            cout << format("HostInfo %ds cube: %8s %10s %4s %12s %8s %6lld %8.2f\n")
                    % group_seconds % host(atuple) % time(atuple) % isSendStr(atuple)
                    % operationStr(atuple) % isRequestStr(atuple)
                    % v.countll() % v.mean();
        }
    }

    void printCubeIncrementalTime(const BitsetAnyTuple &atuple, Stats &v) {
        if (!atuple.any[time_index]) {
            printCubeIncremental(atuple, v);
        }
    }

    void printCubeIncrementalNonTime(const BitsetAnyTuple &atuple, Stats &v) {
        if (atuple.any[time_index]) {
            printCubeIncremental(atuple, v);
        }
    }

    bool cubeOptional(const BitsetAnyTuple &partial) {
        if (skip_cube_time && partial.any[time_index] == false) {
            return false;
        }
        if (skip_cube_host && partial.any[host_index] == false) {
            return false;
        }
        if (skip_cube_host_detail && partial.any[host_index] == false) {
            BitsetAnyTuple::AnyT any = partial.any;
            any[host_index] = true;
            any[time_index] = true;
            if ((~any).none()) {
                return true; // all any, so no detail on the host
            } else {
                return false;
            }
        }
        return true;
    }

    static bool cubeExceptTime(const BitsetAnyTuple &partial) {
        return partial.any[time_index];
    }

    static bool cubeExceptHost(const BitsetAnyTuple &partial) {
        return partial.any[host_index];
    }

    static bool cubeExceptTimeOrHost(const BitsetAnyTuple &partial) {
        return partial.any[time_index] && partial.any[host_index];
    }

    static void addFullStats(Stats &into, const Stats &val) {
        into.add(val);
    }

    void configCube() {
        using boost::bind;

        if (options["cube_time"] && options["cube_host"] && options["cube_host_detail"]) {
            rates_cube.setOptionalCubeFn(bind(&lintel::StatsCubeFns::cubeAll));
        } else {
            rates_cube.setOptionalCubeFn(bind(&HostInfo::cubeOptional, this, _2));
        }

        rates_cube.setCubeStatsAddFn(bind(&lintel::StatsCubeFns::addFullStats, _1, _2));
    }

    //                    host  is_send operation is_req
    typedef boost::tuple<int32_t, bool, uint8_t, bool> RateRollupTupleBase;
    typedef TupleToAnyTuple<RateRollupTupleBase>::type RateRollupTuple;

    void rateRollupAdd(const BitsetAnyTuple &atuple, Stats &v) {
        if (atuple.any[time_index]) {
            return; // Ignore unless time is set, this is all we will prune.
        }
        if (atuple.data.get<time_index>() == min_group_seconds ||
            atuple.data.get<time_index>() == max_group_seconds) {
            return;
        }
        // TODO: don't we need to skip the max_group_seconds also once we
        // have gotten to the max, i.e. when we do this under finalGroup?
        // we don't but only because we're skipping cubing the last bit of
        // the data, which will make us wrong in the cube output.
        INVARIANT(atuple.data.get<time_index>() > min_group_seconds &&
                  atuple.data.get<time_index>() < max_group_seconds,
                  format("%d \not in ]%d, %d[") % atuple.data.get<time_index>()
                  % min_group_seconds % max_group_seconds);
        if (false) {
            cout << format("%8s %10s %4s %12s %8s %6lld %8.2f\n")
                    % host(atuple) % time(atuple) % isSendStr(atuple)
                    % operationStr(atuple) % isRequestStr(atuple)
                    % v.countll() % v.mean();
        }

        RateRollupTuple rrt;
        if (!atuple.any[host_index]) {
            rrt.get<0>().set(atuple.data.get<host_index>());
        }
        if (!atuple.any[is_send_index]) {
            rrt.get<1>().set(atuple.data.get<is_send_index>());
        }
        if (!atuple.any[operation_index]) {
            rrt.get<2>().set(atuple.data.get<operation_index>());
        }
        if (!atuple.any[is_request_index]) {
            rrt.get<3>().set(atuple.data.get<is_request_index>());
        }

        double count = v.countll();
        double ops_per_sec = count / group_seconds;
        double bytes_per_sec = (count * v.mean()) / group_seconds;

        rate_hts.getHashEntry(rrt).add(ops_per_sec, bytes_per_sec);
    }

    void addMissingZeroRates(const RateRollupTuple &t, Rates &v) {
        int64_t count = v.ops_rate.countll();
        SINVARIANT(count == static_cast<int64_t>(v.bytes_rate.countll()));
        while (count < group_count) {
            // Entry was late to the rollup, so is missing some of the
            // early rate-rollup entries.  TODO: consider printing out
            // a warning about this; it should only be happening to hosts,
            // and possibly but unlikely to operation types.
            ++count;
            v.ops_rate.add(0);
            v.bytes_rate.add(0);
        }
        SINVARIANT(count == group_count
                   && static_cast<uint64_t>(count) == v.ops_rate.countll()
                   && static_cast<uint64_t>(count) == v.bytes_rate.countll());
    }

    static string hostStr(const RateRollupTuple &t, bool sql = false) {
        if (t.get<0>().any) {
            return sql ? str_null : str_star;
        } else {
            return str(format(sql ? "0x%08x" : "%08x") % t.get<0>().val);
        }
    }

    static string isSendStr(const RateRollupTuple &t) {
        if (t.get<1>().any) {
            return str_star;
        } else if (t.get<1>().val) {
            return str_send;
        } else {
            return str_recv;
        }
    }

    static string operationStr(const RateRollupTuple &t) {
        if (t.get<2>().any) {
            return str_star;
        } else {
            return unifiedIdToName(t.get<2>().val);
        }
    }

    static string isRequestStr(const RateRollupTuple &t) {
        if (t.get<3>().any) {
            return str_star;
        } else if (t.get<3>().val) {
            return str_request;
        } else {
            return str_response;
        }
    }

    string humanQuantiles(StatsQuantile &q, double divide) {
        return str(format("%.8g %.8g %.8g %.8g %.8g %.8g %.8g")
                   % (q.getQuantile(0.05)/divide)
                   % (q.getQuantile(0.1)/divide)
                   % (q.getQuantile(0.25)/divide)
                   % (q.getQuantile(0.5)/divide)
                   % (q.getQuantile(0.75)/divide)
                   % (q.getQuantile(0.9)/divide)
                   % (q.getQuantile(0.95)/divide));
    }

    void printRate(const RateRollupTuple &t, Rates &v) {
        SINVARIANT(v.ops_rate.count() == v.bytes_rate.count());
        SINVARIANT(v.ops_rate.count() == static_cast<uint64_t>(group_count));
        SINVARIANT(v.ops_rate.mean() > 0);

        if (sql_output) {
            cout << format("insert into nfs_hostinfo_rates (group_seconds, host, direction, operation, op_dir, mean_operations_per_second, mean_payload_bytes_per_second) values (%d, %s, %s, %s, %s, %.8g, %.8g);\n")
                    % group_seconds % hostStr(t, true) % sqlify(isSendStr(t))
                    % sqlify(operationStr(t)) % sqlify(isRequestStr(t))
                    % v.ops_rate.mean() % v.bytes_rate.mean();

            if (print_rates_quantiles) {
                static const double quantile_step = 0.01;
                cout << format("insert into nfs_hostinfo_rate_quantiles (group_seconds, host, direction, operation, op_dir, quantile, operations_per_second, payload_bytes_per_second) values ");
                for (double q = 0; q < 1.0000000001; q += quantile_step) {
                    if (q > 0) { cout << ", "; }
                    if (q > 1) { q = 1; } // doubles can have slight overflow
                    cout << format("(%d, %s, %s, %s, %s, %.5f, %.8g, %.8g)")
                            % group_seconds % hostStr(t, true) % sqlify(isSendStr(t))
                            % sqlify(operationStr(t)) % sqlify(isRequestStr(t))
                            % q % v.ops_rate.getQuantile(q) % v.bytes_rate.getQuantile(q);
                }
                cout << ";\n";
            }
        } else {
            if (!printed_rates_header) {
                cout << format("HostInfo %ds rates: host     dir      op    op-dir  ops/s payload-MiB/s\n") % group_seconds;
                if (print_rates_quantiles) {
                    cout << format("HostInfo %ds rates-quantiles: 5%% 10%% 25%% 50%% 75%% 90%% 95%% ops/s : 5%% 10%% 25%% 50%% 75%% 90%% 95%% MiB/s\n") % group_seconds;
                }
                printed_rates_header = true;
            }

            cout << format("HostInfo %ds rates: %8s %4s %8s %8s %10.2f %8.3f\n")
                    % group_seconds % hostStr(t) % isSendStr(t) % operationStr(t)
                    % isRequestStr(t) % v.ops_rate.mean()
                    % (v.bytes_rate.mean()/(1024.0*1024.0));
            if (print_rates_quantiles) {
                cout << format("HostInfo %ds rates-quantiles: %s : %s\n")
                        % group_seconds % humanQuantiles(v.ops_rate, 1)
                        % humanQuantiles(v.bytes_rate, 1024 * 1024);
            }
        }
    }

    void sanityCheckRates() {
        // sanity checks
        BitsetAnyTuple tmp(Tuple(0,true,0,0,true));
        tmp.any.set();
        tmp.any[is_send_index] = false;
        Stats &v = rates_cube.getCubeEntry(tmp);

        INVARIANT(v.countll() == payload_overall.countll(),
                  format("%s: %d != %d") % tmp % v.countll() % payload_overall.countll());
        INVARIANT(Double::eq(v.mean(), payload_overall.mean()),
                  format("%.20g != %.20g") % v.mean() % payload_overall.mean());
        // Increase the tolerance to error on this value; we are summing
        // up squared values potentially over a very very large number of
        // entries (billions), hence the difference between adding in
        // batches (v) and one-by-one (payload_overall) can become
        // substantial.  Observed error was .00000011079687316312; this
        // is 5x the tolerance.  Assuming it was fine as count, mean were
        // equal and given stddev was ~3575, the two values were pretty
        // close.
        INVARIANT(Double::eq(v.stddev(), payload_overall.stddev(), .0000005),
                  format("%.20g != %.20g; %d ents") % v.stddev()
                  % payload_overall.stddev() % payload_overall.countll());

        tmp.any[is_send_index] = true;
        Stats &w = rates_cube.getCubeEntry(tmp);
        // Complete rollup sees everything twice since we add it as
        // both send and receive
        INVARIANT(w.countll() == 2*payload_overall.countll(),
                  format("%d != %d") % w.countll()
                  % (2*payload_overall.countll()));
        INVARIANT(Double::eq(w.mean(), payload_overall.mean()),
                  format("%.20g != %.20g")
                  % w.mean() % payload_overall.mean());
        INVARIANT(Double::eq(w.stddev(), payload_overall.stddev(), .0000005),
                  format("%.20g != %.20g; %d ents") % w.stddev()
                  % payload_overall.stddev() % payload_overall.countll());

        // Check that we have the right number of entries in the
        // rollups of the rates table.

        if (group_count > 0 && options["print_rates"]) {
            RateRollupTuple rrt;
            INVARIANT(rate_hts[rrt].ops_rate.countll() + zero_groups
                      == static_cast<uint64_t>(group_count),
                      format("%d != %d") % rate_hts[rrt].ops_rate.countll()
                      % group_count);
            rrt.get<1>().set(true);
            SINVARIANT(rate_hts[rrt].ops_rate.countll() + zero_groups
                       == static_cast<uint64_t>(group_count));
            rrt.get<1>().set(false);
            SINVARIANT(rate_hts[rrt].ops_rate.countll() + zero_groups
                       == static_cast<uint64_t>(group_count));
        }
    }

    virtual void printResult() {
        printf("Begin-%s\n",__PRETTY_FUNCTION__);

        // ds data -> hash_tuple_stats(host, is_send, time, op, is_request, stat)
        // -> zero-cube(h,i,t,o,i, sum(stat))
        // hut(any(h), any(i), any(o), any(i), quantile(stat.count()/time_chunk),
        //     quantile(stats.mean()/time_chunk))

        finalGroup();

        cout << format("unique entries: %d time, %d host, %d operation\n")
                % (1+(max_group_seconds - min_group_seconds)/group_seconds)
                % unique_vals_tuple.get<host_index>().size()
                % unique_vals_tuple.get<operation_index>().size();

        SINVARIANT(base_data.size() == 0);

        sanityCheckRates();

        if (group_count > 0 && options["print_rates"]) {
            rate_hts.walk(boost::bind
                          (&HostInfo::addMissingZeroRates, this, _1, _2));

            rate_hts.walkOrdered
                    (boost::bind(&HostInfo::printRate, this, _1, _2));
            cout  << "# Note that the all * rollup double counts"
                    " operations since we count both the\n"
                  << "# send and the receive\n";
        }

        if (group_count == -1) {
            // special case for when we had no full groups, the calculation
            // ends up at -1, but we want to print 0.
            ++group_count;
        }
        cout << format("# Processed %d complete groups of size %d\n")
                % group_count % group_seconds;
        cout << "# (ignored the partial first and last groups)\n";
        cout << format("# Total time range was [%d..%d], %d groups with no nfs ops\n")
                % min_group_seconds % max_group_seconds % zero_groups;

        printf("End-%s\n",__PRETTY_FUNCTION__);
    }

    static Stats *makeStats() {
        return new Stats();
    }

    Int64TimeField packet_at;
    TFixedField<int32_t> payload_length;
    ByteField op_id, nfs_version;
    TFixedField<int32_t> source_ip, dest_ip;
    BoolField is_request;
    int32_t group_seconds;

    HashMap<string, bool> options;
    uint32_t incremental_batch_size;

    HashTupleStats<Tuple> base_data;

    // Won't have all the time values in it, they are done incrementally.
    // only has anything in it if the constant enable_hash_unique_tuple is
    // true.
    HashTupleStats<Tuple>::HashUniqueTuple unique_vals_tuple;

    // Will be incrementally processed.
    StatsCube<Tuple> rates_cube;

    HashTupleStats<RateRollupTuple, Rates> rate_hts;
    Stats payload_overall;
    int32_t min_group_seconds, max_group_seconds, last_rollup_at;
    int64_t group_count;

    bool printed_base_header, printed_cube_header, printed_rates_header;

    bool print_rates_quantiles, sql_output, print_base, print_cube;
    uint32_t zero_groups;

    bool skip_cube_time, skip_cube_host, skip_cube_host_detail;
    size_t last_reported_memory_usage;
};

RowAnalysisModule *
NFSDSAnalysisMod::newHostInfo(DataSeriesModule &prev, char *arg) {
    return new HostInfo(prev, arg);
}

size_t HostInfo::Rates::memory_usage;


// -*- C++ -*-
//
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
//                                   Jiao Lin
//                      California Institute of Technology
//                         (C) 2005 All Rights Reserved
//
// {LicenseText}
//
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//

#include <boost/python.hpp>
#include "mccomposite/AbstractNeutronScatterer.h"
#include "mccomposite/geometry/intersect.h"

namespace wrap_mccomposite {

  void wrap_AbstractNeutronScatterer()
  {
    using namespace boost::python;

    class_<mccomposite::AbstractNeutronScatterer, bases<mcni::AbstractNeutronScatterer>,
      boost::noncopyable>
      ("AbstractNeutronScatterer", no_init)
      .def("shape", &mccomposite::AbstractNeutronScatterer::shape, return_internal_reference<>())
      ;

  }

  size_t countIntersections(const mcni::Neutron::Event &n, const mccomposite::AbstractShape & shape)
  {
    using namespace mccomposite::geometry;
    Arrow arrow(n.state.position, n.state.velocity);
    ArrowIntersector::distances_t dists = intersect(arrow, shape);
    return dists.size();
  }

  void wrap_countIntersections()
  {
    using namespace boost::python;

    def("countIntersections", countIntersections);
  }
}

// version
// $Id$

// End of file


#include <iostream>
#include <chrono>

#include <opencv2/core.hpp>

#include <dlib/opencv.h>
#include <dlib/data_io.h>
#include <dlib/dnn.h>
#include <dlib/array2d.h>
#include <dlib/image_transforms.h>

#include "ReadMNISTDatas.hpp"
#include "TrainFilePath.hpp"
#include "DLIBInput.hpp"

//#if defined(DEBUG) || defined(_DEBUG)
//#pragma comment(lib, "opencv_core310d.lib")
//#pragma comment(lib, "opencv_highgui310d.lib")
//#pragma comment(lib, "dlibd.lib")
//#else
//#pragma comment(lib, "opencv_core310.lib")
//#pragma comment(lib, "opencv_highgui310.lib")
//#pragma comment(lib, "dlib.lib")
//#endif

template <class Duration>
void outputResult(std::ostream& output, std::vector<unsigned long> answer, std::vector<unsigned long> predict, Duration duration);

int main()
{
  std::vector<cv::Mat> cv_train_images;
  std::vector<int> tmp_labels;

  // 教師データの読み込み．
  readImages(INPUT_TRAIN_IMAGE_PATH, cv_train_images);
  readLabels(INPUT_TRAIN_LABEL_PATH, tmp_labels);

  // ラベルの型を変換
  std::vector<unsigned long> train_labels(std::begin(tmp_labels), std::end(tmp_labels));

  // 非教師データの読み込み
  std::vector<cv::Mat> cv_unknown_images;
  tmp_labels.clear();
  readImages(INPUT_UNKNOWN_IMAGE_PATH, cv_unknown_images);
  readLabels(INPUT_UNKNOWN_LABEL_PATH, tmp_labels);
  std::vector<unsigned long> unknown_labels(std::begin(tmp_labels), std::end(tmp_labels));

  // CNN の定義
  // おそらく A<B<C>> となっている場合 Cが入力で，Bが中間層，Aが出力層
  // なので，出力はfcにするのがいいと思う．
  // fc<10, ...>:Fully connected layerでノード数は10
  // relu:活性化関数の名前．詳しくはReLUで調べてください
  // con<16,5,5,1,1,SUBNET> で5✕5のフィルタサイズを1✕1のstrideで畳み込みするノードが16個ある
  // max_pool<2, 2, 2, 2, SUBNET> 2✕2のウインドウサイズで2✕2のstrideでプーリングを行う．
  // relu<fc<84, ...>> この場合活性化関数がReLUで84ノードからなる層を定義している．
  // max_pool<2,2,2,2,relu<con<16,5,5,1,1,SUBNET>>> これでconvolutionした結果をReLU関数で活性化してそれをMax poolingする
  // input<array2d<uchar>> cv_image<uchar>を入力に取る．現在cv::Matを入力に取れるように試行錯誤中
  using net_type = dlib::loss_multiclass_log<
    dlib::fc<10,
    dlib::relu<dlib::fc<84,
    dlib::relu<dlib::fc<120,
    dlib::max_pool<2, 2, 2, 2, dlib::relu<dlib::con<16, 5, 5, 1, 1,
    dlib::max_pool<2, 2, 2, 2, dlib::relu<dlib::con<6, 5, 5, 1, 1,
    dlib::input<cv::Mat>
    >>>>>>>>>>>>;

  // 上のCNN場合
  // -FC-> : Fully connectedな接続
  // -> : 重みを共有した接続
  // 入力画像->[6ノードの畳み込み層]->プーリング層->[16ノードの畳み込み層]->プーリング層-FC-> ...
  //          [120ノードの普通のNN]-FC>[84ノードの普通のNN]-FC>出力(10次元ベクトルで各次元に各数字の確率が保存される)

  net_type net;
  dlib::dnn_trainer<net_type> trainer(net);

  trainer.set_learning_rate(0.01);          // 学習計数
  trainer.set_min_learning_rate(0.00001);   // 学習計数の最低値．上の学習計数から学習回数を増やす毎にこの値へと減少していく
  trainer.set_mini_batch_size(128);         // なにこれ？バッチサイズ．
  trainer.be_verbose();                     // おそらくログの出力

  auto begin_time = std::chrono::system_clock::now();

  // DNNの学習には尋常ではないほどの時間がかかるので20秒おきに自動で現在の状態を保存する
  // なお，途中で終了した場合もmnist_syncファイルがあれば再起してくれる．
  trainer.set_synchronization_file("mnist_sync", std::chrono::seconds(20));
  trainer.train(cv_train_images, train_labels);

  std::cout << "[Learning duration]:" << std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - begin_time).count() << "s" << std::endl;

  // 保存前にclear．
  net.clean();

  // 多分学習結果の保存．
  dlib::serialize("mnist_network.dat") << net;

  std::cout << "[Input trained labels]" << std::endl;
  begin_time = std::chrono::system_clock::now();
  std::vector<unsigned long> predicted_labels = net(cv_train_images);
  outputResult(std::cout, train_labels, predicted_labels, std::chrono::system_clock::now() - begin_time);

  std::cout << "[Input unknown labels]" << std::endl;
  begin_time = std::chrono::system_clock::now();
  predicted_labels = net(cv_unknown_images);
  outputResult(std::cout, unknown_labels, predicted_labels, std::chrono::system_clock::now() - begin_time);

  return 0;
}

template <class Duration>
void outputResult(std::ostream& output, std::vector<unsigned long> answer, std::vector<unsigned long> predict, Duration duration)
{
  int num_correct = 0;
  int num_wrong = 0;
  // And then let's see if it classified them correctly.
  for (size_t i = 0; i < answer.size(); ++i)
  {
    if (predict[i] == answer[i])
    {
      ++num_correct;
    }
    else
    {
      ++num_wrong;
    }
  }

  output << "Testing num_right: " << num_correct << std::endl;
  output << "Testing num_wrong: " << num_wrong << std::endl;
  output << "Testing accuracy:  " << num_correct / static_cast<double>(num_correct + num_wrong) << std::endl;
  output << "Testing duration:" << std::chrono::duration_cast<std::chrono::seconds>(duration).count() << "s" << std::endl;
}


//==============================================================================
//         Copyright 2003 - 2011   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef BOOST_SIMD_CONSTANT_CONSTANTS_FIVE_HPP_INCLUDED
#define BOOST_SIMD_CONSTANT_CONSTANTS_FIVE_HPP_INCLUDED

#include <boost/simd/include/functor.hpp>
#include <boost/simd/constant/register.hpp>
#include <boost/simd/constant/hierarchy.hpp>

namespace boost { namespace simd
{
  namespace tag
  {
   /*!
     @brief Five generic tag

     Represents the Five constant in generic contexts.

     @par Models:
        Hierarchy
   **/
    BOOST_SIMD_CONSTANT_REGISTER( Five,double
                                , 5,0x40a00000UL,0x4014000000000000ULL
                                )
  }
  namespace ext
  {
   template<class Site>
   BOOST_FORCEINLINE generic_dispatcher<tag::Five, Site> dispatching_Five(adl_helper, boost::dispatch::meta::unknown_<Site>, ...)
   {
     return generic_dispatcher<tag::Five, Site>();
   }
   template<class... Args>
   struct impl_Five;
  }
  /*!
     Generates value 5 in the chosen type

    @par Semantic:

    @code
    T r = Five<T>();
    @endcode

    is similar to:

    @code
    T r = T(5);
    @endcode
  **/
  BOOST_SIMD_CONSTANT_IMPLEMENTATION(boost::simd::tag::Five, Five)
} }

#include <boost/simd/constant/common.hpp>

#endif


/** @file CameraCalibration.hpp
 *  @brief This structure parses camera calibration files in .yaml format typically outputted by
 *         ROS's default camera calibration. The .yaml file is assumed to be standard format. Addtional
 *         camera parameters, extrinsicts or ID, are also impliemented.
 *
 *  This contains definition for data structures and enumerations.
 *
 *  @author Ao Y. Yu
 *  @bug No known bugs.
 *  @example
 *  	#define USE_EIGEN
 *	#include <iostream>
 *	#include <string>
 *	#include "CameraCalibration.hpp"
 *
 *	int main()
 *	{
 *	  std::string cal_file = "~/Desktop/camera_cal_parser/example.yaml";
 *	  CameraCalibration camcal;
 *	  camcal.from_file(cal_file);
 *	  std::cout << "OpenCV Datastructures: " << "\n";
 *	  std::cout << camcal.opencv_get_camera_matrix() << "\n\n";
 *	  std::cout << camcal.opencv_get_distortion_coefficients() << "\n\n";
 *	  std::cout << camcal.opencv_get_rectification_matrix() << "\n\n";
 *	  std::cout << camcal.opencv_get_projection_matrix() << "\n\n";
 *	  return 0;
 *	}
 */
#pragma once

#include <iostream>
#include <string>
#include <vector>
#include <fstream>
#include <assert.h>

#include <boost/algorithm/string.hpp>

#ifdef USE_EIGEN
#include <Eigen/Core>
#endif
#include <opencv2/core/core.hpp>

struct CameraCalibration{
  CameraCalibration(){
  }
  ~CameraCalibration(){
  }

  int camera_id = 0;
  int image_width = 0;
  int image_height = 0;
  float stereo_baseline = 0.0; // in meters.
  std::string camera_name = "";
  std::string distortion_model = "";

  std::vector<std::string> CameraPramatersStrings = { "image_width", "image_height", "camera_name", "camera_matrix", "distortion_model",
                                                  "distortion_coefficients", "rectification_matrix", "projection_matrix"};
  std::vector<std::string> CameraSubPramatersStrings = { "rows", "cols", "data"};

  cv::Mat camera_matrix = cv::Mat::zeros(cv::Size(3, 3), CV_32F);
  cv::Mat distortion_coefficients = cv::Mat::zeros(cv::Size(5, 1), CV_32F);
  cv::Mat rectification_matrix = cv::Mat::zeros(cv::Size(3, 3), CV_32F);
  cv::Mat projection_matrix = cv::Mat::zeros(cv::Size(4, 3), CV_32F);
  cv::Mat extrinsic_matrix = cv::Mat::eye(cv::Size(4, 4), CV_32F);

  // Intermediate variables. This is used to store values for different return types (Eigen or OpenCV).
  float array_camera_matrix[9];
  float array_distortion_coefficients[5];
  float array_rectification_matrix[9];
  float array_projection_matrix[12];

  int get_image_width(){
    return this->image_width;
  }
  int get_image_height(){
    return this->image_height;
  }
  std::string get_camera_name(){
    return this->camera_name;
  }
  std::string get_distortion_model(){
    return this->distortion_model;
  }
  float get_focal_length_x(){
    return this->camera_matrix.at<float>(0,0);
  }
  float get_focal_length_y(){
    return this->camera_matrix.at<float>(1,1);
  }
  float get_optical_center_x(){
    return this->camera_matrix.at<float>(0,2);
  }
  float get_optical_center_y(){
    return this->camera_matrix.at<float>(1,2);
  }
  cv::Mat opencv_get_extrinsics(){
    return this->extrinsic_matrix;
  }
  cv::Mat get_camera_matrix(){
    cv::Mat camera_matrix_temp = cv::Mat::zeros(cv::Size(3, 3), CV_32F);
    for(int i = 0; i < 3; i++){
      for(int j = 0; j < 3; j++){
        camera_matrix_temp.at<float>(i, j) = array_camera_matrix[i*3+j];
      }
    }
    assert((camera_matrix_temp.rows == 3 && camera_matrix_temp.cols == 3));
    return camera_matrix_temp;
  }
  cv::Mat get_distortion_coefficients(){
    cv::Mat distortion_coefficients_temp = cv::Mat::zeros(cv::Size(5, 1), CV_32F);
    for(int i = 0; i < (sizeof(array_distortion_coefficients)/sizeof(*array_distortion_coefficients)); i++){
        distortion_coefficients_temp.at<float>(0, i) = array_distortion_coefficients[i];
    }
    assert((distortion_coefficients_temp.rows == 1 && distortion_coefficients_temp.cols == 5));
    return distortion_coefficients_temp;
  }
  cv::Mat get_rectification_matrix(){
    cv::Mat rectification_matrix_temp = cv::Mat::zeros(cv::Size(3, 3), CV_32F);
    for(int i = 0; i < 3; i++){
      for(int j = 0; j < 3; j++){
        rectification_matrix_temp.at<float>(i, j) = array_rectification_matrix[i*3+j];
      }
    }
    assert((rectification_matrix_temp.rows == 3 && rectification_matrix_temp.cols == 3));
    return rectification_matrix_temp;
  }
  cv::Mat get_projection_matrix(){
    cv::Mat projection_matrix_temp = cv::Mat::zeros(cv::Size(4, 3), CV_32F);
    for(int i = 0; i < 3; i++){
      for(int j = 0; j < 4; j++){
        projection_matrix_temp.at<float>(i, j) = array_projection_matrix[i*3+j];
      }
    }
    assert((projection_matrix_temp.rows == 3 && projection_matrix_temp.cols == 4));
    return projection_matrix_temp;
  }

#ifdef USE_EIGEN
  Eigen::Matrix3f eigen_camera_matrix;
  Eigen::Matrix<float, 1, 5> eigen_distortion_coefficients;
  Eigen::Matrix3f eigen_rectification_matrix;
  Eigen::Matrix<float, 3, 4> eigen_projection_matrix;

  Eigen::Matrix3f eigen_get_camera_matrix(){
    for(int i = 0; i < 3; i++){
      for(int j = 0; j < 3; j++){
        eigen_camera_matrix(i, j) = array_camera_matrix[i*3+j];
      }
    }
    return eigen_camera_matrix;
  }

  Eigen::Matrix<float, 1, 5> eigen_get_distortion_coefficients(){
    for(int i = 0; i < (sizeof(array_distortion_coefficients)/sizeof(*array_distortion_coefficients)); i++){
        eigen_distortion_coefficients(0, i) = array_distortion_coefficients[i];
    }
    return eigen_distortion_coefficients;
  }

  Eigen::Matrix3f eigen_get_rectification_matrix(){
    for(int i = 0; i < 3; i++){
      for(int j = 0; j < 3; j++){
        eigen_rectification_matrix(i, j) = array_rectification_matrix[i*3+j];
      }
    }
    return eigen_rectification_matrix;
  }

  Eigen::Matrix<float, 3, 4> eigen_get_projection_matrix(){
    for(int i = 0; i < 3; i++){
      for(int j = 0; j < 4; j++){
        eigen_projection_matrix(i, j) = array_projection_matrix[i*3+j];
      }
    }
    return eigen_projection_matrix;
  }
#endif

  int from_file(const std::string cal_file){
      std::string line;
      std::ifstream myfile (cal_file);
      std::string CameraParamter;
      if (myfile.is_open())
      {
        while ( std::getline (myfile,line) )
        {
          std::string CameraParameter, Value;
          boost::algorithm::erase_all(line, " ");
          std::vector<std::string> SplitVec;
          boost::algorithm::split(SplitVec, line, boost::algorithm::is_any_of(":"), boost::algorithm::token_compress_on);
          boost::algorithm::trim_left_if(SplitVec[1], boost::algorithm::is_any_of("["));
          boost::algorithm::trim_right_if(SplitVec[1], boost::algorithm::is_any_of("]"));
          if(in_array(SplitVec[0], CameraSubPramatersStrings)){
            // This branch is for if current line is a camera sub parameter of CameraParameter
            if(SplitVec[0] == "data"){
              this->parse_calibration(CameraParamter, SplitVec[1]);
            }
          }else if(in_array(SplitVec[0], CameraPramatersStrings)){
            CameraParamter = SplitVec[0];
          }
        }
        myfile.close();
      }
      camera_matrix = this->get_camera_matrix();
      distortion_coefficients = this->get_distortion_coefficients();
      rectification_matrix = this->get_rectification_matrix();
      projection_matrix = this->get_projection_matrix();
      return 0;
  }

  void parse_calibration(std::string CameraParameter, std::string Value){
    std::vector<std::string> matrix_elements;
    if(CameraParameter == "image_width"){
      image_width = std::atoi(Value.c_str());
    }else if(CameraParameter == "image_height"){
      image_height = std::atoi(Value.c_str());
    }else if(CameraParameter == "camera_name"){
      camera_name = Value;
    }else if(CameraParameter == "camera_matrix"){
      matrix_elements = get_matrix_elements(Value);
      for(int i = 0; i < (sizeof(array_camera_matrix)/sizeof(*array_camera_matrix)); i++){
        array_camera_matrix[i] = std::stof(matrix_elements[i]);
      }
    }else if(CameraParameter == "distortion_model"){
      distortion_model = Value;
    }else if(CameraParameter == "distortion_coefficients"){
      matrix_elements = get_matrix_elements(Value);
      for(int i = 0; i < (sizeof(array_distortion_coefficients)/sizeof(*array_distortion_coefficients)); i++){
        array_distortion_coefficients[i] = std::stof(matrix_elements[i]);
      }
    }else if(CameraParameter == "rectification_matrix"){
      matrix_elements = get_matrix_elements(Value);
      for(int i = 0; i < (sizeof(array_rectification_matrix)/sizeof(*array_rectification_matrix)); i++){
        array_rectification_matrix[i] = std::stof(matrix_elements[i]);
      }
    }else if(CameraParameter == "projection_matrix"){
      matrix_elements = get_matrix_elements(Value);
      for(int i = 0; i < (sizeof(array_projection_matrix)/sizeof(*array_projection_matrix)); i++){
        array_projection_matrix[i] = std::stof(matrix_elements[i]);
      }
    }else{
      std::cout << "Unsupported Calibration Parameter." << std::endl;
    }
  }
  std::vector<std::string> get_matrix_elements(std::string Value){
    std::vector<std::string> matrix_elements;
    matrix_elements = boost::algorithm::split(matrix_elements, Value, boost::algorithm::is_any_of(","), boost::algorithm::token_compress_on);
    return matrix_elements;
  }

  bool in_array(const std::string &value, const std::vector<std::string> &array)
  {
      return std::find(array.begin(), array.end(), value) != array.end();
  }

  void pretty_print(){
    // TO_DO: Impliement pretty printing.
  }
};


/*
@copyright Louis Dionne 2015
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
 */

#include <boost/hana/assert.hpp>
#include <boost/hana/contains.hpp>
#include <boost/hana/core/is_a.hpp>
#include <boost/hana/equal.hpp>
#include <boost/hana/find_if.hpp>
#include <boost/hana/optional.hpp>
#include <boost/hana/tuple.hpp>

#include <string>
namespace hana = boost::hana;
using namespace std::string_literals;

int main() {
    BOOST_HANA_RUNTIME_CHECK(
        hana::find_if(hana::make_tuple(1, '2', 3.3, "abc"s), hana::is_a<std::string>) == hana::just("abc"s)
    );

    BOOST_HANA_RUNTIME_CHECK(
        "abc"s ^hana::in^ hana::make_tuple(1, '2', 3.3, "abc"s)
    );
}


//==================================================================================================
/*!
  @file

  Defines MIC functions for SIMD storage used by pack

  @copyright 2015 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)

**/
//==================================================================================================
#ifndef BOOST_SIMD_ARCH_X86_MIC_PACK_TRAITS_SIMD_HPP_INCLUDED
#define BOOST_SIMD_ARCH_X86_MIC_PACK_TRAITS_SIMD_HPP_INCLUDED

#include <cassert>
#include <boost/simd/arch/x86/avx/pack_traits.hpp>

namespace boost { namespace simd
{
  namespace detail
  {
    BOOST_SIMD_DEFINE_PACK_TRAITS(float, 16, __m512,  _mm512_set_ps, _mm512_load_ps, _mm512_set1_ps);
    BOOST_SIMD_DEFINE_PACK_TRAITS(double, 8, __m512d, _mm512_set_pd, _mm512_load_pd, _mm512_set1_pd);

    template <typename... Args>
    void _mm512_TODO(Args const&...)
    {
      assert(false);
    }

    struct mm512_load_si512_auto {
      template <typename T>
      BOOST_FORCEINLINE __m512i operator()(T const* ptr) const BOOST_NOEXCEPT {
        return _mm512_load_si512(reinterpret_cast<__m512i const*>(ptr));
      }
    };

    // TODO use a static instance instead? Because we're header only there is no proper way to do that?
    BOOST_SIMD_DEFINE_PACK_TRAITS_TPL(typename T, T,  8, __m512i, _mm512_set_epi64, mm512_load_si512_auto(), _mm512_set1_epi64);
    BOOST_SIMD_DEFINE_PACK_TRAITS_TPL(typename T, T, 16, __m512i, _mm512_set_epi32, mm512_load_si512_auto(), _mm512_set1_epi32);
    BOOST_SIMD_DEFINE_PACK_TRAITS_TPL(typename T, T, 32, __m512i, _mm512_TODO,      mm512_load_si512_auto(), _mm512_set1_epi16);
    BOOST_SIMD_DEFINE_PACK_TRAITS_TPL(typename T, T, 64, __m512i, _mm512_TODO,      mm512_load_si512_auto(), _mm512_set1_epi8);
  }

  namespace ext
  {
    // __m512* storage are supported by MIC hardware
    template<typename Enable> struct abi_of<__m512d,Enable>
    {
      using type = ::boost::simd::mic_;
    };

    template<typename Enable> struct abi_of<__m512,Enable>
    {
      using type = ::boost::simd::mic_;
    };

    template<typename Enable> struct abi_of<__m512i,Enable>
    {
      using type = ::boost::simd::mic_;
    };
  }
} }

#endif


//
//  molecular_graph.cpp
//  indigo-bondorder
//
//  Created by Welsh, Ivan on 25/08/17.
//  Copyright © 2017 Allison Group. All rights reserved.
//

#include <iostream>
#include <map>
#include <tuple>

#include <boost/graph/connected_components.hpp>

#include "classes/atom.hpp"
#include "classes/bond.hpp"
#include "classes/iters.hpp"
#include "classes/molecular_graph.hpp"
#include "classes/molecule.hpp"
#include "utils/common.hpp"
#include "utils/options.hpp"

using namespace indigo_bondorder;

MolecularGraph::MolecularGraph()
: _MolGraph()
{
}

MolecularGraph::MolecularGraph(Molecule_p m) : _MolGraph() {
  m->ResetIndices();
  std::map<Atom_p, MolVertex> atom_to_vertex;
  for (MolAtomIterator it = m->BeginAtom(); it != m->EndAtom(); ++it) {
    MolVertex v = AddVertex(*it);
    atom_to_vertex.emplace(*it, v);
  }

  for (MolBondIterator it = m->BeginBond(); it != m->EndBond(); ++it) {
    MolVertex u = atom_to_vertex.at((*it)->GetSourceAtom());
    MolVertex v = atom_to_vertex.at((*it)->GetTargetAtom());
    AddEdge(u, v, *it);
  }
}

MolVertex MolecularGraph::AddVertex(Atom_p atm) {
  MolVertProp p;
  p.atom = atm;
  return AddVertex(p);
}

MolEdgeBool MolecularGraph::AddEdge(MolVertex u, MolVertex v, Bond_p bnd) {
  MolEdgeProp p;
  p.bond = bnd;
  return AddEdge(u, v, p);
}

Uint MolecularGraph::NumConnectedComponents() {
  MolVertIdxMap idxMap;
  boost::associative_property_map<MolVertIdxMap> indexMap(idxMap);
  MolVertIterPair mvp = GetVertices();
  for (int i = 0; mvp.first != mvp.second; ++mvp.first, ++i) {
    boost::put(indexMap, (*mvp.first), i);
  }

  num_components_ = boost::connected_components(*graph_,
                                                boost::get(&MolVertProp::component, *graph_),
                                                boost::vertex_index_map(indexMap));
  return num_components_;
}

String MolecularGraph::ToDGFString() {
  typedef Options::AssignElectrons::FPT fpt_;
  std::ostringstream dgf_stream;
  uid_t u, v;
  MolEdgeIterPair es = GetEdges();
  for (auto start = es.first; start != es.second; ++start) {
    u = GetVertexIndex(GetSource(*start));
    v = GetVertexIndex(GetTarget(*start));
    if (fpt_::ADD_EDGES_TO_TD) {
      dgf_stream << "e " << u << "," << u << " " << u << "," << v << std::endl;
      dgf_stream << "e " << v << "," << v << " " << u << "," << v << std::endl;
    } else {
      dgf_stream << "e " << u << " " << v << std::endl;
    }
  }
  return dgf_stream.str();
}


/*
 * Copyright 2019 Amazon.com, Inc. or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <atomic>
#include <thread>
#include <vector>

#include <boost/test/unit_test.hpp>

#include <aws/metrics/accumulator.h>
#include <aws/utils/utils.h>
#include <aws/utils/logging.h>
#include <aws/mutex.h>

namespace {

auto get_start_time() {
  return aws::utils::round_down_time<
      std::chrono::seconds,
      std::chrono::high_resolution_clock>() +
          std::chrono::seconds(2);
}

} //namespace

BOOST_AUTO_TEST_SUITE(Accumulator)

BOOST_AUTO_TEST_CASE(Overall) {
  aws::metrics::detail::AccumulatorImpl<
      double,
      std::chrono::milliseconds,
      30> a;

  int N = 100;
  for (int i = 0; i <= N; i += 1) {
    a(i);
  }

  BOOST_CHECK_EQUAL(a.count(), N + 1);
  BOOST_CHECK_EQUAL(a.min(), 0);
  BOOST_CHECK_EQUAL(a.max(), N);
  BOOST_CHECK_EQUAL(a.mean(), 50);
  BOOST_CHECK_EQUAL(a.sum(), 5050);
}

BOOST_AUTO_TEST_CASE(Window) {
  const int num_buckets = 100;
  const int num_samples = 100;
  const int last_val = num_buckets * num_samples - 1;

  using Bucket = std::chrono::duration<uint64_t, std::ratio<1, 8>>;
  aws::metrics::detail::AccumulatorImpl<double, Bucket, num_buckets> a;

  auto start = get_start_time();

  size_t v = 0;
  for (int i = 0; i < num_buckets; i += 1) {
    while (std::chrono::high_resolution_clock::now() < start + Bucket(i));
    for (int j = 0; j < num_samples; j++) {
      a(v++);
    }
  }

  BOOST_CHECK_EQUAL(a.count(0), 0);
  BOOST_CHECK_EQUAL(a.count(1), 1 * num_samples);
  BOOST_CHECK_EQUAL(a.count(15), 15 * num_samples);
  BOOST_CHECK_EQUAL(a.count(30), 30 * num_samples);

  BOOST_CHECK_EQUAL(a.min(0), 0);
  BOOST_CHECK_EQUAL(a.min(1), last_val - num_samples + 1);
  BOOST_CHECK_EQUAL(a.min(15), last_val - num_samples * 15 + 1);
  BOOST_CHECK_EQUAL(a.min(30), last_val - num_samples * 30 + 1);

  BOOST_CHECK_EQUAL(a.max(0), 0);
  BOOST_CHECK_EQUAL(a.max(1), last_val);
  BOOST_CHECK_EQUAL(a.max(15), last_val);
  BOOST_CHECK_EQUAL(a.max(30), last_val);

  BOOST_CHECK_EQUAL(a.mean(0), 0);
  BOOST_CHECK_EQUAL(a.mean(1), 9949.5);
  BOOST_CHECK_EQUAL(a.mean(15), 9249.5);
  BOOST_CHECK_EQUAL(a.mean(30), 8499.5);

  BOOST_CHECK_EQUAL(a.sum(0), 0);
  BOOST_CHECK_EQUAL(a.sum(1), 994950);
  BOOST_CHECK_EQUAL(a.sum(15), 13874250);
  BOOST_CHECK_EQUAL(a.sum(30), 25498500);

  BOOST_CHECK_EQUAL(a.count(), last_val + 1);
  BOOST_CHECK_EQUAL(a.min(), 0);
  BOOST_CHECK_EQUAL(a.max(), last_val);
  BOOST_CHECK_EQUAL(a.mean(), 4999.5);
  BOOST_CHECK_EQUAL(a.sum(), 49995000);
}

BOOST_AUTO_TEST_CASE(Concurrency) {
  const int num_threads = 10;
  const int num_buckets = 100;
  const int num_samples = 100;
  const int last_val = num_buckets * num_samples - 1;

  using Bucket = std::chrono::duration<uint64_t, std::ratio<1, 8>>;
  aws::metrics::detail::AccumulatorImpl<double, Bucket, num_buckets> a;

  auto start = get_start_time();

  std::atomic<size_t> v(0);
  std::vector<aws::thread> threads;
  for (int i = 0; i < num_threads; i++) {
    threads.emplace_back([&] {
      for (int i = 0; i < num_buckets; i += 1) {
        aws::utils::sleep_until(start + Bucket(i));
        for (int j = 0; j < num_samples / num_threads; j++) {
          a(v++);
        }
      }
    });
  }

  for (auto& t : threads) {
    t.join();
  }

  BOOST_CHECK_EQUAL(a.count(0), 0);
  BOOST_CHECK_EQUAL(a.count(1), 1 * num_samples);
  BOOST_CHECK_EQUAL(a.count(15), 15 * num_samples);
  BOOST_CHECK_EQUAL(a.count(30), 30 * num_samples);

  BOOST_CHECK_EQUAL(a.min(0), 0);
  BOOST_CHECK_EQUAL(a.min(1), last_val - num_samples + 1);
  BOOST_CHECK_EQUAL(a.min(15), last_val - num_samples * 15 + 1);
  BOOST_CHECK_EQUAL(a.min(30), last_val - num_samples * 30 + 1);

  BOOST_CHECK_EQUAL(a.max(0), 0);
  BOOST_CHECK_EQUAL(a.max(1), last_val);
  BOOST_CHECK_EQUAL(a.max(15), last_val);
  BOOST_CHECK_EQUAL(a.max(30), last_val);

  BOOST_CHECK_EQUAL(a.mean(0), 0);
  BOOST_CHECK_EQUAL(a.mean(1), 9949.5);
  BOOST_CHECK_EQUAL(a.mean(15), 9249.5);
  BOOST_CHECK_EQUAL(a.mean(30), 8499.5);

  BOOST_CHECK_EQUAL(a.sum(0), 0);
  BOOST_CHECK_EQUAL(a.sum(1), 994950);
  BOOST_CHECK_EQUAL(a.sum(15), 13874250);
  BOOST_CHECK_EQUAL(a.sum(30), 25498500);

  BOOST_CHECK_EQUAL(a.count(), last_val + 1);
  BOOST_CHECK_EQUAL(a.min(), 0);
  BOOST_CHECK_EQUAL(a.max(), last_val);
  BOOST_CHECK_EQUAL(a.mean(), 4999.5);
  BOOST_CHECK_EQUAL(a.sum(), 49995000);
}

BOOST_AUTO_TEST_CASE(WriteThroughput) {
  const int N = 100000;

  for (size_t num_threads = 1; num_threads <= 8; num_threads *= 2) {
    aws::metrics::Accumulator a;

    auto start = get_start_time();

    std::atomic<size_t> v(0);
    std::vector<aws::thread> threads;
    for (size_t i = 0; i < num_threads; i++) {
      threads.emplace_back([&] {
        aws::utils::sleep_until(start);
        while (true) {
          size_t z = v++;
          if (z < N) {
            a(z);
          } else {
            break;
          }
        }
      });
    }

    for (auto& t : threads) {
      t.join();
    }

    double rate = (double) N / aws::utils::seconds_since(start);
    LOG(info) << "Accumulator tight loop put rate (" << num_threads
              << " threads): " << rate / 1000 << " K per second ";
  }
}

BOOST_AUTO_TEST_SUITE_END()


/*
    Copyright 2013 Adobe
    Distributed under the Boost Software License, Version 1.0.
    (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*/
/**************************************************************************************************/

#ifndef ADOBE_ALGORITHM_HEAP_HPP
#define ADOBE_ALGORITHM_HEAP_HPP

#include <adobe/config.hpp>

#include <boost/range/begin.hpp>
#include <boost/range/end.hpp>

#include <algorithm>
#include <functional>

/**************************************************************************************************/

namespace adobe {

/**************************************************************************************************/
/*!
\defgroup heap heap operations
\ingroup sorting

\see
    - STL documentation for \ref stldoc_push_heap
    - STL documentation for \ref stldoc_pop_heap
    - STL documentation for \ref stldoc_make_heap
    - STL documentation for \ref stldoc_sort_heap
*/
/**************************************************************************************************/
/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange>
inline void push_heap(RandomAccessRange& range) {
    return std::push_heap(boost::begin(range), boost::end(range));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessIterator, class Compare>
inline void push_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp) {
    return std::push_heap(first, last,
                          std::bind(comp, std::placeholders::_1, std::placeholders::_2));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange, class Compare>
inline void push_heap(RandomAccessRange& range, Compare comp) {
    return adobe::push_heap(boost::begin(range), boost::end(range), comp);
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange>
inline void pop_heap(RandomAccessRange& range) {
    return std::pop_heap(boost::begin(range), boost::end(range));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessIterator, class Compare>
inline void pop_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp) {
    return std::pop_heap(first, last,
                         std::bind(comp, std::placeholders::_1, std::placeholders::_2));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange, class Compare>
inline void pop_heap(RandomAccessRange& range, Compare comp) {
    return adobe::pop_heap(boost::begin(range), boost::end(range), comp);
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange>
inline void make_heap(RandomAccessRange& range) {
    return std::make_heap(boost::begin(range), boost::end(range));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessIterator, class Compare>
inline void make_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp) {
    return std::make_heap(first, last,
                          std::bind(comp, std::placeholders::_1, std::placeholders::_2));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange, class Compare>
inline void make_heap(RandomAccessRange& range, Compare comp) {
    return adobe::make_heap(boost::begin(range), boost::end(range), comp);
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange>
inline void sort_heap(RandomAccessRange& range) {
    return std::sort_heap(boost::begin(range), boost::end(range));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessIterator, class Compare>
inline void sort_heap(RandomAccessIterator first, RandomAccessIterator last, Compare comp) {
    return std::sort_heap(first, last,
                          std::bind(comp, std::placeholders::_1, std::placeholders::_2));
}

/*!
    \ingroup heap

    \brief heap implementation
*/
template <class RandomAccessRange, class Compare>
inline void sort_heap(RandomAccessRange& range, Compare comp) {
    return adobe::sort_heap(boost::begin(range), boost::end(range), comp);
}

/**************************************************************************************************/

} // namespace adobe

/**************************************************************************************************/

#endif

/**************************************************************************************************/


/* ----------------------------------------------------------------------
 *
 *                    *** Smooth Mach Dynamics ***
 *
 * This file is part of the USER-SMD package for LAMMPS.
 * Copyright (2014) Georg C. Ganzenmueller, georg.ganzenmueller@emi.fhg.de
 * Fraunhofer Ernst-Mach Institute for High-Speed Dynamics, EMI,
 * Eckerstrasse 4, D-79104 Freiburg i.Br, Germany.
 *
 * This file is based on the FixShearHistory class.
 *
 * ----------------------------------------------------------------------- */

/* ----------------------------------------------------------------------
 LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
 http://lammps.sandia.gov, Sandia National Laboratories
 Steve Plimpton, sjplimp@sandia.gov

 Copyright (2003) Sandia Corporation.  Under the terms of Contract
 DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
 certain rights in this software.  This software is distributed under
 the GNU General Public License.

 See the README file in the top-level LAMMPS directory.
 ------------------------------------------------------------------------- */

#include "lattice.h"
#include <mpi.h>
#include <string.h>
#include <stdio.h>
#include "fix_smd_tlsph_reference_configuration.h"
#include "atom.h"
#include "comm.h"
#include "neighbor.h"
#include "neigh_list.h"
#include "force.h"
#include "pair.h"
#include "update.h"
#include "modify.h"
#include "memory.h"
#include "error.h"
#include "domain.h"
#include <Eigen/Eigen>
#include "smd_kernels.h"
#include "smd_math.h"

using namespace Eigen;
using namespace LAMMPS_NS;
using namespace FixConst;
using namespace SMD_Kernels;
using namespace std;
using namespace SMD_Math;
#define DELTA 16384

#define INSERT_PREDEFINED_CRACKS false

/* ---------------------------------------------------------------------- */

FixSMD_TLSPH_ReferenceConfiguration::FixSMD_TLSPH_ReferenceConfiguration(LAMMPS *lmp, int narg, char **arg) :
		Fix(lmp, narg, arg) {

	if (atom->map_style == 0)
		error->all(FLERR, "Pair tlsph with partner list requires an atom map, see atom_modify");

	maxpartner = 1;
	npartner = NULL;
	partner = NULL;
	wfd_list = NULL;
	wf_list = NULL;
	energy_per_bond = NULL;
	degradation_ij = NULL;
	grow_arrays(atom->nmax);
	atom->add_callback(0);

	// initialize npartner to 0 so neighbor list creation is OK the 1st time
	int nlocal = atom->nlocal;
	for (int i = 0; i < nlocal; i++) {
		npartner[i] = 0;
	}

	comm_forward = 14;
	updateFlag = 1;
}

/* ---------------------------------------------------------------------- */

FixSMD_TLSPH_ReferenceConfiguration::~FixSMD_TLSPH_ReferenceConfiguration() {
	// unregister this fix so atom class doesn't invoke it any more

	atom->delete_callback(id, 0);
// delete locally stored arrays

	memory->destroy(npartner);
	memory->destroy(partner);
	memory->destroy(wfd_list);
	memory->destroy(wf_list);
	memory->destroy(degradation_ij);
	memory->destroy(energy_per_bond);
}

/* ---------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::setmask() {
	int mask = 0;
	mask |= PRE_EXCHANGE;
	return mask;
}

/* ---------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::init() {
	if (atom->tag_enable == 0)
		error->all(FLERR, "Pair style tlsph requires atoms have IDs");
}

/* ---------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::pre_exchange() {
	//return;

	//printf("in FixSMD_TLSPH_ReferenceConfiguration::pre_exchange()\n");
	double **defgrad = atom->smd_data_9;
	double *radius = atom->radius;
	double *rho = atom->rho;
	double *vfrac = atom->vfrac;
	double **x = atom->x;
	double **x0 = atom->x0;
	double *rmass = atom->rmass;
	int nlocal = atom->nlocal;
	int i, itmp;
	int *mask = atom->mask;
	if (igroup == atom->firstgroup) {
		nlocal = atom->nfirst;
	}

	int *updateFlag_ptr = (int *) force->pair->extract("smd/tlsph/updateFlag_ptr", itmp);
	if (updateFlag_ptr == NULL) {
		error->one(FLERR,
				"fix FixSMD_TLSPH_ReferenceConfiguration failed to access updateFlag pointer. Check if a pair style exist which calculates this quantity.");
	}

	int *nn = (int *) force->pair->extract("smd/tlsph/numNeighsRefConfig_ptr", itmp);
	if (nn == NULL) {
		error->all(FLERR, "FixSMDIntegrateTlsph::updateReferenceConfiguration() failed to access numNeighsRefConfig_ptr array");
	}

	// sum all update flag across processors
	MPI_Allreduce(updateFlag_ptr, &updateFlag, 1, MPI_INT, MPI_MAX, world);

	if (updateFlag > 0) {
		if (comm->me == 0) {
			printf("**** updating ref config at step: %ld\n", update->ntimestep);
		}

		for (i = 0; i < nlocal; i++) {

			if (mask[i] & groupbit) {

				// re-set x0 coordinates
				x0[i][0] = x[i][0];
				x0[i][1] = x[i][1];
				x0[i][2] = x[i][2];

				// re-set deformation gradient
				defgrad[i][0] = 1.0;
				defgrad[i][1] = 0.0;
				defgrad[i][2] = 0.0;
				defgrad[i][3] = 0.0;
				defgrad[i][4] = 1.0;
				defgrad[i][5] = 0.0;
				defgrad[i][6] = 0.0;
				defgrad[i][7] = 0.0;
				defgrad[i][8] = 1.0;
				/*
				 * Adjust particle volume as the reference configuration is changed.
				 * We safeguard against excessive deformations by limiting the adjustment range
				 * to the intervale J \in [0.9..1.1]
				 */
				vfrac[i] = rmass[i] / rho[i];
//
				if (nn[i] < 15) {
					radius[i] *= 1.2;
				} // else //{
				  //	radius[i] *= pow(J, 1.0 / domain->dimension);
				  //}
			}
		}

		// update of reference config could have changed x0, vfrac, radius
		// communicate these quantities now to ghosts: x0, vfrac, radius
		comm->forward_comm_fix(this);

		setup(0);
	}
}

/* ----------------------------------------------------------------------
 copy partner info from neighbor lists to atom arrays
 so can be migrated or stored with atoms
 ------------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::setup(int vflag) {
	int i, j, ii, jj, n, inum, jnum;
	int *ilist, *jlist, *numneigh, **firstneigh;
	double r, h, wf, wfd;
	Vector3d dx;

	if (updateFlag == 0)
		return;

	int nlocal = atom->nlocal;
	nmax = atom->nmax;
	grow_arrays(nmax);

// 1st loop over neighbor list
// calculate npartner for each owned atom
// nlocal_neigh = nlocal when neigh list was built, may be smaller than nlocal

	double **x0 = atom->x;
	double *radius = atom->radius;
	int *mask = atom->mask;
	tagint *tag = atom->tag;
	NeighList *list = pair->list;
	inum = list->inum;
	ilist = list->ilist;
	numneigh = list->numneigh;
	firstneigh = list->firstneigh;

	// zero npartner for all current atoms
	for (i = 0; i < nlocal; i++)
		npartner[i] = 0;

	for (ii = 0; ii < inum; ii++) {
		i = ilist[ii];
		jlist = firstneigh[i];
		jnum = numneigh[i];

		for (jj = 0; jj < jnum; jj++) {
			j = jlist[jj];
			j &= NEIGHMASK;

			if (INSERT_PREDEFINED_CRACKS) {
				if (!crack_exclude(i, j))
					continue;
			}

			dx(0) = x0[i][0] - x0[j][0];
			dx(1) = x0[i][1] - x0[j][1];
			dx(2) = x0[i][2] - x0[j][2];
			r = dx.norm();
			h = radius[i] + radius[j];

			if (r <= h) {
				npartner[i]++;
				if (j < nlocal) {
					npartner[j]++;
				}
			}
		}
	}

	maxpartner = 0;
	for (i = 0; i < nlocal; i++)
		maxpartner = MAX(maxpartner, npartner[i]);
	int maxall;
	MPI_Allreduce(&maxpartner, &maxall, 1, MPI_INT, MPI_MAX, world);
	maxpartner = maxall;

	grow_arrays(nmax);

	for (i = 0; i < nlocal; i++) {
		npartner[i] = 0;
		for (jj = 0; jj < maxpartner; jj++) {
			wfd_list[i][jj] = 0.0;
			wf_list[i][jj] = 0.0;
			degradation_ij[i][jj] = 0.0;
			energy_per_bond[i][jj] = 0.0;
		}
	}

	for (ii = 0; ii < inum; ii++) {
		i = ilist[ii];
		jlist = firstneigh[i];
		jnum = numneigh[i];

		for (jj = 0; jj < jnum; jj++) {
			j = jlist[jj];
			j &= NEIGHMASK;

			dx(0) = x0[i][0] - x0[j][0];
			dx(1) = x0[i][1] - x0[j][1];
			dx(2) = x0[i][2] - x0[j][2];
			r = dx.norm();
			h = radius[i] + radius[j];

			if (INSERT_PREDEFINED_CRACKS) {
				if (!crack_exclude(i, j))
					continue;
			}

			if (r < h) {
				spiky_kernel_and_derivative(h, r, domain->dimension, wf, wfd);

				partner[i][npartner[i]] = tag[j];
				wfd_list[i][npartner[i]] = wfd;
				wf_list[i][npartner[i]] = wf;
				npartner[i]++;
				if (j < nlocal) {
					partner[j][npartner[j]] = tag[i];
					wfd_list[j][npartner[j]] = wfd;
					wf_list[j][npartner[j]] = wf;
					npartner[j]++;
				}
			}
		}
	}

	// count number of particles for which this group is active

	// bond statistics
	if (update->ntimestep > -1) {
		n = 0;
		int count = 0;
		for (i = 0; i < nlocal; i++) {
			if (mask[i] & groupbit) {
				n += npartner[i];
				count += 1;
			}
		}
		int nall, countall;
		MPI_Allreduce(&n, &nall, 1, MPI_INT, MPI_SUM, world);
		MPI_Allreduce(&count, &countall, 1, MPI_INT, MPI_SUM, world);
                if (countall < 1) countall = 1;

		if (comm->me == 0) {
			if (screen) {
				printf("\n>>========>>========>>========>>========>>========>>========>>========>>========\n");
				fprintf(screen, "TLSPH neighbors:\n");
				fprintf(screen, "  max # of neighbors for a single particle = %d\n", maxpartner);
				fprintf(screen, "  average # of neighbors/particle in group tlsph = %g\n", (double) nall / countall);
				printf(">>========>>========>>========>>========>>========>>========>>========>>========\n\n");
			}
			if (logfile) {
				fprintf(logfile, "\nTLSPH neighbors:\n");
				fprintf(logfile, "  max # of neighbors for a single particle = %d\n", maxpartner);
				fprintf(logfile, "  average # of neighbors/particle in group tlsph = %g\n", (double) nall / countall);
			}
		}
	}

	updateFlag = 0; // set update flag to zero after the update

}

/* ----------------------------------------------------------------------
 memory usage of local atom-based arrays
 ------------------------------------------------------------------------- */

double FixSMD_TLSPH_ReferenceConfiguration::memory_usage() {
	int nmax = atom->nmax;
	int bytes = nmax * sizeof(int);
	bytes += nmax * maxpartner * sizeof(tagint); // partner array
	bytes += nmax * maxpartner * sizeof(float); // wf_list
	bytes += nmax * maxpartner * sizeof(float); // wfd_list
	bytes += nmax * maxpartner * sizeof(float); // damage_per_interaction array
	bytes += nmax * sizeof(int); // npartner array
	return bytes;

}

/* ----------------------------------------------------------------------
 allocate local atom-based arrays
 ------------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::grow_arrays(int nmax) {
	//printf("in FixSMD_TLSPH_ReferenceConfiguration::grow_arrays\n");
	memory->grow(npartner, nmax, "tlsph_refconfig_neigh:npartner");
	memory->grow(partner, nmax, maxpartner, "tlsph_refconfig_neigh:partner");
	memory->grow(wfd_list, nmax, maxpartner, "tlsph_refconfig_neigh:wfd");
	memory->grow(wf_list, nmax, maxpartner, "tlsph_refconfig_neigh:wf");
	memory->grow(degradation_ij, nmax, maxpartner, "tlsph_refconfig_neigh:degradation_ij");
	memory->grow(energy_per_bond, nmax, maxpartner, "tlsph_refconfig_neigh:damage_onset_strain");
}

/* ----------------------------------------------------------------------
 copy values within local atom-based arrays
 ------------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::copy_arrays(int i, int j, int delflag) {
	npartner[j] = npartner[i];
	for (int m = 0; m < npartner[j]; m++) {
		partner[j][m] = partner[i][m];
		wfd_list[j][m] = wfd_list[i][m];
		wf_list[j][m] = wf_list[i][m];
		degradation_ij[j][m] = degradation_ij[i][m];
		energy_per_bond[j][m] = energy_per_bond[i][m];
	}
}

/* ----------------------------------------------------------------------
 pack values in local atom-based arrays for exchange with another proc
 ------------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::pack_exchange(int i, double *buf) {
// NOTE: how do I know comm buf is big enough if extreme # of touching neighs
// Comm::BUFEXTRA may need to be increased

//printf("pack_exchange ...\n");

	int m = 0;
	buf[m++] = npartner[i];
	for (int n = 0; n < npartner[i]; n++) {
		buf[m++] = partner[i][n];
		buf[m++] = wfd_list[i][n];
		buf[m++] = wf_list[i][n];
		buf[m++] = degradation_ij[i][n];
		buf[m++] = energy_per_bond[i][n];
	}
	return m;

}

/* ----------------------------------------------------------------------
 unpack values in local atom-based arrays from exchange with another proc
 ------------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::unpack_exchange(int nlocal, double *buf) {
	if (nlocal == nmax) {
		//printf("nlocal=%d, nmax=%d\n", nlocal, nmax);
		nmax = nmax / DELTA * DELTA;
		nmax += DELTA;
		grow_arrays(nmax);

		error->message(FLERR,
				"in Fixtlsph_refconfigNeighGCG::unpack_exchange: local arrays too small for receiving partner information; growing arrays");
	}
//printf("nlocal=%d, nmax=%d\n", nlocal, nmax);

	int m = 0;
	npartner[nlocal] = static_cast<int>(buf[m++]);
	for (int n = 0; n < npartner[nlocal]; n++) {
		partner[nlocal][n] = static_cast<tagint>(buf[m++]);
		wfd_list[nlocal][n] = static_cast<float>(buf[m++]);
		wf_list[nlocal][n] = static_cast<float>(buf[m++]);
		degradation_ij[nlocal][n] = static_cast<float>(buf[m++]);
		energy_per_bond[nlocal][n] = static_cast<float>(buf[m++]);
	}
	return m;
}

/* ----------------------------------------------------------------------
 pack values in local atom-based arrays for restart file
 ------------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::pack_restart(int i, double *buf) {
	int m = 0;
	buf[m++] = 4 * npartner[i] + 2;
	buf[m++] = npartner[i];
	for (int n = 0; n < npartner[i]; n++) {
		buf[m++] = partner[i][n];
		buf[m++] = wfd_list[i][n];
		buf[m++] = wf_list[i][n];
		buf[m++] = degradation_ij[i][n];
		buf[m++] = energy_per_bond[i][n];
	}
	return m;
}

/* ----------------------------------------------------------------------
 unpack values from atom->extra array to restart the fix
 ------------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::unpack_restart(int nlocal, int nth) {
// ipage = NULL if being called from granular pair style init()

// skip to Nth set of extra values

//	double **extra = atom->extra;
//
//	int m = 0;
//	for (int i = 0; i < nth; i++)
//		m += static_cast<int>(extra[nlocal][m]);
//	m++;
//
//	// allocate new chunks from ipage,dpage for incoming values
//
//	npartner[nlocal] = static_cast<int>(extra[nlocal][m++]);
//	for (int n = 0; n < npartner[nlocal]; n++) {
//		partner[nlocal][n] = static_cast<tagint>(extra[nlocal][m++]);
//	}
}

/* ----------------------------------------------------------------------
 maxsize of any atom's restart data
 ------------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::maxsize_restart() {
// maxtouch_all = max # of touching partners across all procs

	int maxtouch_all;
	MPI_Allreduce(&maxpartner, &maxtouch_all, 1, MPI_INT, MPI_MAX, world);
	return 4 * maxtouch_all + 2;
}

/* ----------------------------------------------------------------------
 size of atom nlocal's restart data
 ------------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::size_restart(int nlocal) {
	return 4 * npartner[nlocal] + 2;
}

/* ---------------------------------------------------------------------- */

int FixSMD_TLSPH_ReferenceConfiguration::pack_forward_comm(int n, int *list, double *buf, int pbc_flag, int *pbc) {
	int i, j, m;
	double *radius = atom->radius;
	double *vfrac = atom->vfrac;
	double **x0 = atom->x0;
	double **defgrad0 = atom->smd_data_9;

	//printf("FixSMD_TLSPH_ReferenceConfiguration:::pack_forward_comm\n");
	m = 0;
	for (i = 0; i < n; i++) {
		j = list[i];
		buf[m++] = x0[j][0];
		buf[m++] = x0[j][1];
		buf[m++] = x0[j][2];

		buf[m++] = vfrac[j];
		buf[m++] = radius[j];

		buf[m++] = defgrad0[i][0];
		buf[m++] = defgrad0[i][1];
		buf[m++] = defgrad0[i][2];
		buf[m++] = defgrad0[i][3];
		buf[m++] = defgrad0[i][4];
		buf[m++] = defgrad0[i][5];
		buf[m++] = defgrad0[i][6];
		buf[m++] = defgrad0[i][7];
		buf[m++] = defgrad0[i][8];

	}
	return m;
}

/* ---------------------------------------------------------------------- */

void FixSMD_TLSPH_ReferenceConfiguration::unpack_forward_comm(int n, int first, double *buf) {
	int i, m, last;
	double *radius = atom->radius;
	double *vfrac = atom->vfrac;
	double **x0 = atom->x0;
	double **defgrad0 = atom->smd_data_9;

	m = 0;
	last = first + n;
	for (i = first; i < last; i++) {
		x0[i][0] = buf[m++];
		x0[i][1] = buf[m++];
		x0[i][2] = buf[m++];

		vfrac[i] = buf[m++];
		radius[i] = buf[m++];

		defgrad0[i][0] = buf[m++];
		defgrad0[i][1] = buf[m++];
		defgrad0[i][2] = buf[m++];
		defgrad0[i][3] = buf[m++];
		defgrad0[i][4] = buf[m++];
		defgrad0[i][5] = buf[m++];
		defgrad0[i][6] = buf[m++];
		defgrad0[i][7] = buf[m++];
		defgrad0[i][8] = buf[m++];
	}
}

/* ----------------------------------------------------------------------
 routine for excluding bonds across a hardcoded slit crack
 Note that everything is scaled by lattice constant l0 to avoid
 numerical inaccuracies.
 ------------------------------------------------------------------------- */

bool FixSMD_TLSPH_ReferenceConfiguration::crack_exclude(int i, int j) {

	double **x = atom->x;
	double l0 = domain->lattice->xlattice;

	// line between pair of atoms i,j
	double x1 = x[i][0] / l0;
	double y1 = x[i][1] / l0;

	double x2 = x[j][0] / l0;
	double y2 = x[j][1] / l0;

	// hardcoded crack line
	double x3 = -0.1 / l0;
	double y3 = ((int) 1.0 / l0) + 0.5;
	//printf("y3 = %f\n", y3);
	double x4 = 0.1 / l0 - 1.0 + 0.1;
	double y4 = y3;

	bool retVal = DoLineSegmentsIntersect(x1, y1, x2, y2, x3, y3, x4, y4);

	return !retVal;
	//return 1;
}