#include <fstream>
#include <iostream>
#include <algorithm>
#include <thread>
#include <numeric>

#include <boost/lexical_cast.hpp>
#include <boost/filesystem/operations.hpp>

#include <succinct/mapper.hpp>

#include <stxxl/vector>
#include <stxxl/io>
#include <stxxl/sort>

#include "configuration.hpp"
#include "index_types.hpp"
#include "util.hpp"
#include "verify_collection.hpp"
#include "mixed_block.hpp"
#include "index_build_utils.hpp"
#include "semiasync_queue.hpp"

using ds2i::logger;

typedef uint32_t block_id_type; // XXX for memory reasons, but would need size_t for very large indexes

struct lambda_point {
    block_id_type block_id;
    float lambda;
    ds2i::mixed_block::space_time_point st;

    struct comparator {
        bool operator()(lambda_point const& lhs, lambda_point const& rhs) const
        {
            return lhs.lambda < rhs.lambda;
        }

        static lambda_point min_value()
        {
            lambda_point val;
            val.lambda = std::numeric_limits<float>::min();
            return val;
        }

        static lambda_point max_value()
        {
            lambda_point val;
            val.lambda = std::numeric_limits<float>::max();
            return val;
        }
    };
};

typedef stxxl::vector<lambda_point> lambda_vector_type;

template <typename InputCollectionType>
struct lambdas_computer : ds2i::semiasync_queue::job {
    lambdas_computer(block_id_type block_id_base,
                     typename InputCollectionType::document_enumerator e,
                     ds2i::predictors_vec_type const& predictors,
                     std::vector<uint32_t>& counts,
                     ds2i::progress_logger& plog,
                     lambda_vector_type& lambda_points)
        : m_block_id_base(block_id_base)
        , m_e(e)
        , m_predictors(predictors)
        , m_plog(plog)
        , m_lambda_points(lambda_points)
    {
        m_counts.swap(counts);
    }

    virtual void prepare()
    {
        using namespace ds2i;
        using namespace time_prediction;

        auto blocks = m_e.get_blocks();
        assert(m_counts.empty() || m_counts.size() == 2 * blocks.size());

        bool heuristic_greedy = configuration::get().heuristic_greedy;

        block_id_type cur_block_id = m_block_id_base;
        for (auto const& input_block: blocks) {
            static const uint32_t smoothing = 1; // Laplace smoothing
            uint32_t docs_exp = smoothing, freqs_exp = smoothing;
            if (!m_counts.empty()) {
                docs_exp += m_counts[2 * input_block.index];
                freqs_exp += m_counts[2 * input_block.index + 1];
            }

            thread_local std::vector<uint32_t> values;

            auto append_lambdas = [&](std::vector<mixed_block::space_time_point>& points,
                                      block_id_type block_id) {
                // sort by space, time
                std::sort(points.begin(), points.end());

                // smallest point is always added with lambda=0
                m_points_buf.push_back(lambda_point { block_id, 0, points.front() });
                for (auto const& cur: points) {
                    while (true) {
                        auto const& prev = m_points_buf.back();
                        // if this point is dominated we can skip it
                        if (cur.time >= prev.st.time) break;
                        auto lambda = (cur.space - prev.st.space) / (prev.st.time - cur.time);
                        if (!heuristic_greedy && lambda < prev.lambda) {
                            m_points_buf.pop_back();
                        } else {
                            m_points_buf.push_back(lambda_point { block_id, lambda, cur });
                            break;
                        }
                    }
                }
            };

            input_block.decode_doc_gaps(values);
            auto docs_sts = mixed_block::compute_space_time(values, input_block.doc_gaps_universe,
                                                            m_predictors, docs_exp);
            append_lambdas(docs_sts, cur_block_id++);

            input_block.decode_freqs(values);
            auto freqs_sts = mixed_block::compute_space_time(values, uint32_t(-1),
                                                             m_predictors, freqs_exp);
            append_lambdas(freqs_sts, cur_block_id++);
        }

        succinct::util::dispose(m_counts);
    }

    virtual void commit()
    {
        // m_lambda_points.insert(m_lambda_points.end(),
        //                        m_points_buf.begin(), m_points_buf.end());
        std::copy(m_points_buf.begin(), m_points_buf.end(),
                  std::back_inserter(m_lambda_points));
        m_plog.done_sequence(m_e.size());
    }

    block_id_type m_block_id_base;
    typename InputCollectionType::document_enumerator m_e;
    ds2i::predictors_vec_type const& m_predictors;
    std::vector<uint32_t> m_counts;
    ds2i::progress_logger& m_plog;
    double m_lambda;
    std::vector<lambda_point> m_points_buf;
    lambda_vector_type& m_lambda_points;
};

template <typename InputCollectionType>
void compute_lambdas(InputCollectionType const& input_coll,
                     size_t num_blocks,
                     const char* predictors_filename,
                     const char* block_stats_filename,
                     const char* lambdas_filename)

{
    using namespace ds2i;
    using namespace time_prediction;

    logger() << "Computing lambdas" << std::endl;
    progress_logger plog;

    auto predictors = load_predictors(predictors_filename);
    std::ifstream block_stats(block_stats_filename);

    double tick = get_time_usecs();
    double user_tick = get_user_time_usecs();

    std::string line;
    uint32_t block_counts_list;
    std::vector<uint32_t> block_counts;
    bool block_counts_consumed = true;
    block_id_type block_id_base = 0;
    size_t freq_zero_lists = 0;
    size_t freq_zero_blocks = 0;

    stxxl::syscall_file lpfile(lambdas_filename,
                               stxxl::file::DIRECT | stxxl::file::CREAT | stxxl::file::RDWR);
    lambda_vector_type lambda_points(&lpfile);

    semiasync_queue queue(1 << 24);

    for (size_t l = 0; l < input_coll.size(); ++l) {
        if (block_counts_consumed) {
            block_counts_consumed = false;
            read_block_stats(block_stats, block_counts_list, block_counts);
        }

        auto e = input_coll[l];

        typedef lambdas_computer<InputCollectionType> job_type;
        std::shared_ptr<job_type> job;

        if (l == block_counts_list) {
            freq_zero_blocks += std::accumulate(block_counts.begin(), block_counts.end(), size_t(0),
                                                [](size_t accum, uint32_t freq) {
                                                    return accum + (freq == 0);
                                                });
            block_counts_consumed = true;
            job.reset(new job_type(block_id_base, e, predictors, block_counts, plog, lambda_points));
        } else {
            freq_zero_lists += 1;
            freq_zero_blocks += 2 * e.num_blocks();
            std::vector<uint32_t> empty_counts;
            job.reset(new job_type(block_id_base, e, predictors, empty_counts, plog, lambda_points));
        }

        block_id_base += 2 * e.num_blocks();
        queue.add_job(job, 2 * e.size());
    }

    assert(block_id_base == num_blocks); (void)num_blocks;

    stats_line()
        ("freq_zero_lists", freq_zero_lists)
        ("freq_zero_blocks", freq_zero_blocks)
        ;

    queue.complete();
    plog.log();

    logger() << lambda_points.size() << " lambda points" << std::endl;
    logger() << "Sorting lambda points" << std::endl;
    double elapsed_secs = (get_time_usecs() - tick) / 1000000;
    double user_elapsed_secs = (get_user_time_usecs() - user_tick) / 1000000;

    stats_line()
        ("worker_threads", configuration::get().worker_threads)
        ("lambda_computation_time", elapsed_secs)
        ("lambda_computation_user_time", user_elapsed_secs)
        ("is_heuristic", configuration::get().heuristic_greedy)
        ;

    tick = get_time_usecs();
    user_tick = get_user_time_usecs();
    static const size_t sort_memory = size_t(16) * 1024 * 1024 * 1024; // XXX
    stxxl::sort(lambda_points.begin(), lambda_points.end(),
                lambda_point::comparator(),
                sort_memory);

    elapsed_secs = (get_time_usecs() - tick) / 1000000;
    user_elapsed_secs = (get_user_time_usecs() - user_tick) / 1000000;
    stats_line()
        ("worker_threads", configuration::get().worker_threads)
        ("lambda_sorting_time", elapsed_secs)
        ("lambda_sorting_user_time", user_elapsed_secs)
        ("is_heuristic", configuration::get().heuristic_greedy)
        ;
}

template <typename InputCollectionType, typename CollectionBuilder>
struct list_transformer : ds2i::semiasync_queue::job {
    list_transformer(CollectionBuilder& b,
                     typename InputCollectionType::document_enumerator e,
                     std::vector<ds2i::mixed_block::block_type>::const_iterator block_type_begin,
                     std::vector<ds2i::mixed_block::compr_param_type>::const_iterator block_param_begin,
                     ds2i::progress_logger& plog)
        : m_b(b)
        , m_e(e)
        , m_block_type(block_type_begin)
        , m_block_param(block_param_begin)
        , m_plog(plog)
    {}

    virtual void prepare()
    {
        using namespace ds2i;

        typedef typename InputCollectionType::document_enumerator::block_data input_block_type;
        typedef mixed_block::block_transformer<input_block_type> output_block_type;

        auto blocks = m_e.get_blocks();
        std::vector<output_block_type> output_blocks;

        for (auto const& input_block: blocks) {
            auto docs_type = *m_block_type++;
            auto freqs_type = *m_block_type++;
            auto docs_param = *m_block_param++;
            auto freqs_param = *m_block_param++;
            output_blocks.emplace_back(input_block,
                                       docs_type, freqs_type,
                                       docs_param, freqs_param);
        }

        block_posting_list<mixed_block>::write_blocks(m_buf, m_e.size(), output_blocks);
    }

    virtual void commit()
    {
        m_b.add_posting_list(m_buf);
        m_plog.done_sequence(m_e.size());
    }

    CollectionBuilder& m_b;
    typename InputCollectionType::document_enumerator m_e;
    std::vector<ds2i::mixed_block::block_type>::const_iterator m_block_type;
    std::vector<ds2i::mixed_block::compr_param_type>::const_iterator m_block_param;
    ds2i::progress_logger& m_plog;
    std::vector<uint8_t> m_buf;
};

template <typename InputCollectionType>
void optimal_hybrid_index(ds2i::global_parameters const& params,
                          const char* predictors_filename,
                          const char* block_stats_filename,
                          const char* input_filename,
                          const char* output_filename,
                          const char* lambdas_filename,
                          size_t budget)
{
    using namespace ds2i;

    InputCollectionType input_coll;
    boost::iostreams::mapped_file_source m(input_filename);
    succinct::mapper::map(input_coll, m);

    logger() << "Processing " << input_coll.size() << " posting lists" << std::endl;
    size_t num_blocks = 0;
    size_t partial_blocks = 0;
    size_t space_base = 8; // space overhead independent of block compression method
    for (size_t l = 0; l < input_coll.size(); ++l) {
        auto e = input_coll[l];
        num_blocks += 2 * e.num_blocks();
        // list length in vbyte
        space_base += succinct::util::ceil_div(succinct::broadword::msb(e.size()) + 1, 7);
        space_base += e.num_blocks() * 4; // max docid
        space_base += (e.num_blocks() - 1) * 4; // endpoint
        if (e.size() % mixed_block::block_size != 0) {
            partial_blocks += 2;
        }
    }

    logger() << num_blocks << " overall blocks" << std::endl;

    if (boost::filesystem::exists(lambdas_filename)) {
        logger() << "Found lambdas file " << lambdas_filename << ", skipping recomputation" << std::endl;
        logger() << "To recompute lambdas, remove file" << std::endl;
    } else {
        compute_lambdas(input_coll, num_blocks, predictors_filename,
                        block_stats_filename, lambdas_filename);
    }

    stxxl::syscall_file lpfile(lambdas_filename,
                               stxxl::file::DIRECT | stxxl::file::RDONLY);
    lambda_vector_type lambda_points(&lpfile);

    double tick = get_time_usecs();
    double user_tick = get_user_time_usecs();

    logger() << "Computing space-time tradeoffs" << std::endl;
    std::vector<uint16_t> block_spaces(num_blocks);
    std::vector<float> block_times(num_blocks);
    std::vector<mixed_block::block_type> block_types(num_blocks);
    std::vector<mixed_block::compr_param_type> block_params(num_blocks);
    size_t cur_space = space_base;
    double cur_time = 0;
    size_t seen_lambdas = 0;
    float first_nonzero_lambda = true;

    std::ofstream lambdas_log;
    if (budget == 0) {
        lambdas_log.open(output_filename, std::ios::out);
    }

    for (auto const& lpid: lambda_vector_type::bufreader_type(lambda_points)) {
        assert(lpid.block_id < num_blocks);
        cur_space -= block_spaces[lpid.block_id];
        cur_time -= block_times[lpid.block_id];

        block_spaces[lpid.block_id] = lpid.st.space;
        block_times[lpid.block_id] = lpid.st.time;
        block_types[lpid.block_id] = lpid.st.type;
        block_params[lpid.block_id] = lpid.st.param;

        cur_space += block_spaces[lpid.block_id];
        cur_time += block_times[lpid.block_id];

        if (lpid.lambda > 0) { // we are past the initial frontier
            if (first_nonzero_lambda) {
                logger() << "Minimum feasible space: " << cur_space << std::endl;
                first_nonzero_lambda = false;
            }

            if (budget == 0) {
                // just print out a sample of the trade-offs
                if (seen_lambdas % (num_blocks / 2000) == 0) {
                    lambdas_log << lpid.lambda << '\t' << cur_space << '\t' << cur_time << '\n';
                }
                seen_lambdas += 1;
            } else if (cur_space > budget) { // XXX replace with >=
                break;
            }
        }
    }

    succinct::util::dispose(block_spaces);
    succinct::util::dispose(block_times);

    if (budget == 0) {
        logger() << "Done" << std::endl;
        return; // done, just reporting the trade-offs
    }

    double elapsed_secs = (get_time_usecs() - tick) / 1000000;
    double user_elapsed_secs = (get_user_time_usecs() - user_tick) / 1000000;
    stats_line()
        ("worker_threads", configuration::get().worker_threads)
        ("greedy_time", elapsed_secs)
        ("greedy_user_time", user_elapsed_secs)
        ;

    logger() << "Found trade-off. Space: " << cur_space
             << " Time: " << cur_time << std::endl;

    stats_line()
        ("found_space", cur_space)
        ("found_time", cur_time)
        ;

    typedef std::tuple<uint32_t, uint32_t> type_param_pair;
    std::map<type_param_pair, size_t> type_counts;
    for (size_t i = 0; i < num_blocks; ++i) {
        type_counts[type_param_pair((uint8_t)block_types[i], block_params[i])] += 1;
    }

    std::vector<std::pair<type_param_pair, size_t>> type_counts_vec;
    for (uint8_t t = 0; t < mixed_block::block_types; ++t) {
        for (uint8_t param = 0; param < mixed_block::compr_params((mixed_block::block_type)t); ++param) {
            auto tp = type_param_pair(t, param);
            type_counts_vec.push_back(std::make_pair(tp, type_counts[tp]));
        }
    }

    stats_line()
        ("blocks", num_blocks)
        ("partial_blocks", partial_blocks)
        ("type_counts", type_counts_vec)
        ;

    tick = get_time_usecs();
    user_tick = get_user_time_usecs();

    typedef typename block_mixed_index::builder builder_type;
    builder_type builder(input_coll.num_docs(), params);
    progress_logger plog;
    semiasync_queue queue(1 << 24);
    auto block_types_it = block_types.begin();
    auto block_params_it = block_params.begin();

    for (size_t l = 0; l < input_coll.size(); ++l) {
        auto e = input_coll[l];

        typedef list_transformer<InputCollectionType, builder_type> job_type;
        std::shared_ptr<job_type> job(new job_type(builder, e,
                                                   block_types_it,
                                                   block_params_it,
                                                   plog));

        block_types_it += 2 * e.num_blocks();
        block_params_it += 2 * e.num_blocks();
        queue.add_job(job, 2 * e.size());
    }

    assert(block_types_it == block_types.end());
    assert(block_params_it == block_params.end());
    queue.complete();
    plog.log();

    block_mixed_index coll;
    builder.build(coll);
    elapsed_secs = (get_time_usecs() - tick) / 1000000;
    user_elapsed_secs = (get_user_time_usecs() - user_tick) / 1000000;
    logger() << "Collection built in "
             << elapsed_secs << " seconds" << std::endl;

    stats_line()
        ("worker_threads", configuration::get().worker_threads)
        ("construction_time", elapsed_secs)
        ("construction_user_time", user_elapsed_secs)
        ;
    dump_stats(coll, "block_mixed", plog.postings);

    if (output_filename) {
        succinct::mapper::freeze(coll, output_filename);
    }
}

int main(int argc, const char** argv) {

    using namespace ds2i;

    if (argc < 5) {
        std::cerr << "Usage: " << argv[0]
                  << " <index type> <predictors> <block_stats> <input_index> <lambdas_filename> <budget> [output_index] [--check <collection_basename>]"
                  << std::endl;
        return 1;
    }

    std::string type = argv[1];
    const char* predictors_filename = argv[2];
    const char* block_stats_filename = argv[3];
    const char* input_filename = argv[4];
    const char* lambdas_filename = argv[5];
    size_t budget = boost::lexical_cast<size_t>(argv[6]);
    const char* output_filename = nullptr;
    if (argc > 7) {
        output_filename = argv[7];
    }

    bool check = false;
    const char* collection_basename = nullptr;
    if (argc > 9 && std::string(argv[8]) == "--check") {
        check = true;
        collection_basename = argv[9];
    }

    ds2i::global_parameters params;

    if (false) {
#define LOOP_BODY(R, DATA, T)                                           \
        } else if (type == BOOST_PP_STRINGIZE(T)) {                     \
            optimal_hybrid_index<BOOST_PP_CAT(T, _index)>               \
                (params, predictors_filename, block_stats_filename,     \
                 input_filename, output_filename, lambdas_filename, budget); \
            if (check) {                                                \
                binary_freq_collection input(collection_basename);      \
                verify_collection<binary_freq_collection, block_mixed_index> \
                                  (input, output_filename);             \
            }                                                           \
            /**/

        BOOST_PP_SEQ_FOR_EACH(LOOP_BODY, _, DS2I_BLOCK_INDEX_TYPES);
#undef LOOP_BODY
    } else {
        logger() << "ERROR: Unknown type " << type << std::endl;
    }

    return 0;
}


#include <boost/units/systems/si/permittivity.hpp>


//==============================================================================
//         Copyright 2003 - 2011   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef NT2_CORE_FUNCTIONS_EXPR_LAST_INDEX_HPP_INCLUDED
#define NT2_CORE_FUNCTIONS_EXPR_LAST_INDEX_HPP_INCLUDED

#include <nt2/core/functions/last_index.hpp>
#include <nt2/include/functions/first_index.hpp>
#include <nt2/core/container/dsl.hpp>
#include <boost/dispatch/meta/mpl.hpp>
#include <nt2/include/functions/size.hpp>

namespace nt2 { namespace ext
{
  BOOST_DISPATCH_IMPLEMENT  ( last_index_,tag::cpu_
                            , (A0)(A1)
                            , ((ast_<A0, nt2::container::domain>))
                              (mpl_integral_< scalar_< integer_<A1> > >)
                            )
  {
    typedef std::ptrdiff_t result_type;

    BOOST_FORCEINLINE result_type operator()(const A0& a0, const A1&) const
    {
      return    first_index<A1::value>(a0) - 1
              + result_type(nt2::size(a0,A1::value));
    }
  };

  BOOST_DISPATCH_IMPLEMENT  ( last_index_,tag::cpu_
                            , (A0)(A1)
                            , ((ast_<A0, nt2::container::domain>))
                              (scalar_< integer_<A1> >)
                            )
  {
    typedef std::ptrdiff_t result_type;

    BOOST_FORCEINLINE result_type operator()(const A0& a0, const A1& d) const
    {
      return  first_index(a0,d) - 1 + result_type(nt2::size(a0,d));
    }
  };
} }

#endif


#ifndef CONCEPTS_DIMENSIONS_COMPARABLE_HPP_INCLUDED
#define CONCEPTS_DIMENSIONS_COMPARABLE_HPP_INCLUDED

#include <boost/hana/equal.hpp>
#include <boost/hana/not_equal.hpp>
#include <boost/hana/core/common.hpp>

#include <gem/concept/dimensions_common.hpp>
#include <gem/fwd/dimensions.hpp>

namespace gem {

gem::concepts::CommonDimensionPair {D1, D2}
constexpr auto
operator ==(const D1& d1, const D2& d2) noexcept
{
    return d1.value() == d2.value();
}

gem::concepts::InvalidCommonDimensionPair {D1, D2}
constexpr auto
operator ==(const D1&, const D2&) noexcept
{
    return boost::hana::false_c;
}

gem::concepts::CommonDimensionIntegralPair {D1, T2}
constexpr auto
operator ==(const D1& d1, const T2& d2) noexcept
{
    return d1.value() == d2;
}

gem::concepts::CommonIntegralDimensionPair {T1, D2}
constexpr auto
operator ==(const T1& d1, const D2& d2) noexcept
{
    return d1 == d2.value();
}

gem::concepts::CommonDimensionPair {D1, D2}
constexpr auto
operator !=(const D1& d1, const D2& d2) noexcept
{
    return d1.value() != d2.value();
}

gem::concepts::InvalidCommonDimensionPair {D1, D2}
constexpr auto
operator !=(const D1&, const D2&) noexcept
{
    return boost::hana::true_c;
}

gem::concepts::CommonDimensionIntegralPair {D1, T2}
constexpr auto
operator !=(const D1& d1, const T2& d2) noexcept
{
    return d1.value() != d2;
}

gem::concepts::CommonIntegralDimensionPair {T1, D2}
constexpr auto
operator !=(const T1& d1, const D2& d2) noexcept
{
    return d1 != d2.value();
}

}  // namespace gem

#endif  // !CONCEPTS_DIMENSIONS_COMPARABLE_HPP_INCLUDED


// -----------------------------------------------------------------------------
// Fern © Geoneric
//
// This file is part of Geoneric Fern which is available under the terms of
// the GNU General Public License (GPL), version 2. If you do not want to
// be bound by the terms of the GPL, you may purchase a proprietary license
// from Geoneric (http://www.geoneric.eu/contact).
// -----------------------------------------------------------------------------
#define BOOST_TEST_MODULE fern feature masked_array_value
#include <boost/test/unit_test.hpp>
#include "fern/language/feature/core/masked_array_value.h"

namespace fl = fern::language;

BOOST_AUTO_TEST_CASE(array)
{
    size_t const nr_dimensions = 1;
    size_t const nr_cells = 3;
    fl::MaskedArrayValue<int, nr_dimensions> array(fern::extents[nr_cells]);

    array[0] = 3;
    array[1] = 2;
    array[2] = 1;

    BOOST_CHECK_EQUAL(array.size(), nr_cells);
    BOOST_CHECK(!array.mask()[0]);
    BOOST_CHECK(!array.mask()[1]);
    BOOST_CHECK(!array.mask()[2]);
    BOOST_CHECK_EQUAL(array[0], 3);
    BOOST_CHECK_EQUAL(array[1], 2);
    BOOST_CHECK_EQUAL(array[2], 1);
}

BOOST_AUTO_TEST_CASE(grid)
{
    size_t const nr_dimensions = 2;
    size_t const nr_rows = 3;
    size_t const nr_cols = 2;
    fl::MaskedArrayValue<int, nr_dimensions> array(
        fern::extents[nr_rows][nr_cols]);

    array[0][0] = -2;
    array[0][1] = -1;
    array[1][0] = 0;
    array[1][1] = 999;
    array[2][0] = 1;
    array[2][1] = 2;

    BOOST_CHECK_EQUAL(array.size(), nr_rows);
    BOOST_CHECK_EQUAL(array[0].size(), nr_cols);
    BOOST_CHECK(!array.mask()[0][0]);
    BOOST_CHECK(!array.mask()[0][1]);
    BOOST_CHECK(!array.mask()[1][0]);
    BOOST_CHECK(!array.mask()[1][1]);
    BOOST_CHECK(!array.mask()[2][0]);
    BOOST_CHECK(!array.mask()[2][1]);
    BOOST_CHECK_EQUAL(array[0][0], -2);
    BOOST_CHECK_EQUAL(array[0][1], -1);
    BOOST_CHECK_EQUAL(array[1][0], 0);
    BOOST_CHECK_EQUAL(array[1][1], 999);
    BOOST_CHECK_EQUAL(array[2][0], 1);
    BOOST_CHECK_EQUAL(array[2][1], 2);
}


#include <ctype.h>   // for isspace, isdigit
#include <stdint.h>  // for uint32_t
#include <stdlib.h>  // for exit, size_t

#include <ckpttn/netlist.hpp>          // for SimpleNetlist, index_t, Netlist
#include <fstream>                     // for operator<<, basic_ostream, cha...
#include <iostream>                    // for cerr
#include <py2cpp/range.hpp>            // for _iterator
#include <py2cpp/set.hpp>              // for set
#include <xnetwork/classes/graph.hpp>  // for Graph
// #include <py2cpp/py2cpp.hpp>
// #include <__config>      // for std
// #include <__hash_table>  // for __hash_const_iterator, operator!=
#include <boost/utility/string_view.hpp>  // for boost::string_view
#include <type_traits>                    // for move
#include <vector>                         // for vector

// using graph_t =
//     boost::adjacency_list<boost::vecS, boost::vecS, boost::undirectedS>;
// using node_t = typename boost::graph_traits<graph_t>::vertex_descriptor;
// using edge_t = typename boost::graph_traits<graph_t>::edge_iterator;

using namespace std;

// Read the IBM .netD/.net format. Precondition: Netlist is empty.
void writeJSON(boost::string_view jsonFileName, const SimpleNetlist& H) {
    auto json = ofstream{jsonFileName.data()};
    if (json.fail()) {
        cerr << "Error: Can't open file " << jsonFileName << ".\n";
        exit(1);
    }
    json << R"({
 "directed": false,
 "multigraph": false,
 "graph": {
)";

    json << R"( "num_modules": )" << H.number_of_modules() << ",\n";
    json << R"( "num_nets": )" << H.number_of_nets() << ",\n";
    json << R"( "num_pads": )" << H.num_pads << "\n";
    json << " },\n";

    json << R"( "nodes": [)"
         << "\n";
    for (const auto& node : H.G) {
        json << "  { \"id\": " << node << " },\n";
    }
    json << " ],\n";

    json << R"( "links": [)"
         << "\n";
    for (const auto& v : H) {
        for (const auto& net : H.G[v]) {
            json << "  {\n";
            json << "   \"source\": " << v << ",\n";
            json << "   \"target\": " << net << "\n";
            json << "  },\n";
        }
    }
    json << " ]\n";

    json << "}\n";
}

// Read the IBM .netD/.net format. Precondition: Netlist is empty.
auto readNetD(boost::string_view netDFileName) -> SimpleNetlist {
    auto netD = ifstream{netDFileName.data()};
    if (netD.fail()) {
        cerr << "Error: Can't open file " << netDFileName << ".\n";
        exit(1);
    }

    using node_t = uint32_t;

    char t;
    uint32_t numPins;
    uint32_t numNets;
    uint32_t numModules;
    index_t padOffset;

    netD >> t;  // eat 1st 0
    netD >> numPins >> numNets >> numModules >> padOffset;

    // using Edge = pair<int, int>;

    const auto num_vertices = numModules + numNets;
    // const auto R = py::range<node_t>(0, num_vertices);
    auto g = graph_t(num_vertices);

    constexpr index_t bufferSize = 100;
    char lineBuffer[bufferSize];  // Does it work for other compiler?
    netD.getline(lineBuffer, bufferSize);

    node_t w;
    index_t e = numModules - 1;
    char c;
    uint32_t i = 0;
    for (; i < numPins; ++i) {
        if (netD.eof()) {
            cerr << "Warning: Unexpected end of file.\n";
            break;
        }
        do {
            netD.get(c);
        } while ((isspace(c) != 0));
        if (c == '\n') {
            continue;
        }
        if (c == 'a') {
            netD >> w;
        } else if (c == 'p') {
            netD >> w;
            w += padOffset;
        }
        do {
            netD.get(c);
        } while ((isspace(c) != 0));
        if (c == 's') {
            ++e;
        }

        // edge_array[i] = Edge(w, e);
        g.add_edge(w, e);

        do {
            netD.get(c);
        } while ((isspace(c) != 0) && c != '\n');
        // switch (c) {
        // case 'O': aPin.setDirection(Pin::OUTPUT); break;
        // case 'I': aPin.setDirection(Pin::INPUT); break;
        // case 'B': aPin.setDirection(Pin::BIDIR); break;
        // }
        if (c != '\n') {
            netD.getline(lineBuffer, bufferSize);
        }
    }

    e -= numModules - 1;
    if (e < numNets) {
        cerr << "Warning: number of nets is not " << numNets << ".\n";
        numNets = e;
    } else if (e > numNets) {
        cerr << "Error: number of nets is not " << numNets << ".\n";
        exit(1);
    }
    if (i < numPins) {
        cerr << "Error: number of pins is not " << numPins << ".\n";
        exit(1);
    }

    // using IndexMap =
    //     typename boost::property_map<graph_t, boost::vertex_index_t>::type;
    // auto index = boost::get(boost::vertex_index, g);
    // auto G = py::grAdaptor<graph_t>{move(g)};
    auto H = SimpleNetlist{move(g), numModules, numNets};
    H.num_pads = numModules - padOffset - 1;
    return H;
}

// Read the IBM .are format
void readAre(SimpleNetlist& H, boost::string_view areFileName) {
    auto are = ifstream{areFileName.data()};
    if (are.fail()) {
        cerr << " Could not open " << areFileName << endl;
        exit(1);
    }

    using node_t = uint32_t;
    constexpr index_t bufferSize = 100;
    char lineBuffer[bufferSize];

    char c;
    node_t w;
    unsigned int weight;
    // auto totalWeight = 0;
    // xxx index_t smallestWeight = UINT_MAX;
    auto numModules = H.number_of_modules();
    auto padOffset = numModules - H.num_pads - 1;
    auto module_weight = vector<unsigned int>(numModules);

    size_t lineno = 1;
    for (size_t i = 0; i < numModules; i++) {
        if (are.eof()) {
            break;
        }
        do {
            are.get(c);
        } while ((isspace(c) != 0));
        if (c == '\n') {
            lineno++;
            continue;
        }
        if (c == 'a') {
            are >> w;
        } else if (c == 'p') {
            are >> w;
            w += node_t(padOffset);
        } else {
            cerr << "Syntax error in line " << lineno << ":"
                 << R"(expect keyword "a" or "p")" << endl;
            exit(0);
        }

        do {
            are.get(c);
        } while ((isspace(c) != 0));
        if (isdigit(c) != 0) {
            are.putback(c);
            are >> weight;
            module_weight[w] = weight;
        }
        are.getline(lineBuffer, bufferSize);
        lineno++;
    }

    H.module_weight = move(module_weight);
}


#include <boost/python.hpp>
#include "/home/harper/Documents/Work/SEJITS/pyCombBLAS/trunk/kdt/pyCombBLAS/pyOperations.h"
#include <stdint.h>
#include "test_cpp_file.h"

using namespace boost::python;

namespace op
{

  template <typename T>
  struct my_op_s : public ConcreteUnaryFunction<T>
  {
    T operator()(const T& x) const
    {
		return (x < 0) ? -static_cast<doubleint>(x) : x;
    }
  };

  UnaryFunction my_op()
  {
    return UnaryFunction(new my_op_s<doubleint>());
  }
}

/*
PyObject* my_op_py(object x)
{
  //float val = extract<float>(x.attr("__float__"));
  double f = extract<double>(x);
  if(f < 0) f = -f;
  PyObject* ret = PyFloat_FromDouble(f);
  return ret;
}

BOOST_PYTHON_MODULE(module)
{
  //class_<op::UnaryFunction>("UnaryFunction", init<op::ConcreteUnaryFunction<doubleint>*>());
  //def("my_op", &op::my_op);
  //def("apply_my_op", &apply_my_op);
  def("my_op_py", &my_op_py);
}
*/


#include <boost/accumulators/statistics/times2_iterator.hpp>


/*=========================================================================
 *
 *  Copyright David Doria 2012 daviddoria@gmail.com
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *=========================================================================*/

#ifndef HoleHistogramDifferenceAcceptanceVisitor_HPP
#define HoleHistogramDifferenceAcceptanceVisitor_HPP

#include <boost/graph/graph_traits.hpp>

// Parent class
#include "Visitors/AcceptanceVisitors/AcceptanceVisitorParent.h"

// Submodules
#include <Mask/Mask.h>
#include <Mask/MaskOperations.h>
#include <ITKHelpers/ITKHelpers.h>
#include "Utilities/Histogram/Histogram.h"

// ITK
#include "itkImage.h"
#include "itkImageRegion.h"

/**

 */
template <typename TGraph, typename TImage>
struct HoleHistogramDifferenceAcceptanceVisitor : public AcceptanceVisitorParent<TGraph>
{
  TImage* Image;
  Mask* MaskImage;

  const unsigned int HalfWidth;

  float DifferenceThreshold;

  std::vector<float> Mins;
  std::vector<float> Maxs;

  typedef typename boost::graph_traits<TGraph>::vertex_descriptor VertexDescriptorType;

  HoleHistogramDifferenceAcceptanceVisitor(TImage* const image, Mask* const mask, const unsigned int halfWidth,
                                           const std::vector<float>& mins, const std::vector<float>& maxs,
                                           const float differenceThreshold = 100.0f) :
    Image(image), MaskImage(mask), HalfWidth(halfWidth), Mins(mins), Maxs(maxs),
    DifferenceThreshold(differenceThreshold)
  {

  }

  bool AcceptMatch(VertexDescriptorType target, VertexDescriptorType source) const
  {
    float computedEnergy;
    return AcceptMatch(target, source, computedEnergy);
  }

  bool AcceptMatch(VertexDescriptorType target, VertexDescriptorType source, float& computedEnergy) const override
  {
    itk::Index<2> targetPixel = ITKHelpers::CreateIndex(target);
    itk::ImageRegion<2> targetRegion = ITKHelpers::GetRegionInRadiusAroundPixel(targetPixel, this->HalfWidth);

    itk::Index<2> sourcePixel = ITKHelpers::CreateIndex(source);
    itk::ImageRegion<2> sourceRegion = ITKHelpers::GetRegionInRadiusAroundPixel(sourcePixel, this->HalfWidth);

    // Compute the average of the valid pixels in the target region
    std::vector<typename TImage::PixelType> targetRegionPixels = MaskOperations::GetValidPixelsInRegion(this->Image, this->MaskImage, targetRegion);

    // The next line is the only thing that changed versus HistogramDifferenceAcceptanceVisitor
    std::vector<itk::Offset<2> > targetRegionHoleOffsets = this->MaskImage->GetHoleOffsetsInRegion(targetRegion);
    std::vector<itk::Index<2> > sourcePatchPixelIndices = ITKHelpers::OffsetsToIndices(targetRegionHoleOffsets, sourceRegion.GetIndex());
    std::vector<typename TImage::PixelType> sourceRegionPixels = ITKHelpers::GetPixelValues(Image, sourcePatchPixelIndices);

    assert(sourceRegionPixels.size() == targetRegionPixels.size());

    float totalHistogramDifference = 0.0f;

    for(unsigned int component = 0; component < Image->GetNumberOfComponentsPerPixel(); ++component)
    {
      std::vector<float> targetValues(targetRegionPixels.size());
      std::vector<float> sourceValues(sourceRegionPixels.size());

      for(unsigned int pixelId = 0; pixelId < targetRegionPixels.size(); ++pixelId)
      {
        targetValues[pixelId] = targetRegionPixels[pixelId][component];
      }

      for(unsigned int pixelId = 0; pixelId < sourceRegionPixels.size(); ++pixelId)
      {
        sourceValues[pixelId] = sourceRegionPixels[pixelId][component];
      }

      typedef HistogramGenerator<float> HistogramGeneratorType;
      typedef HistogramGeneratorType::HistogramType HistogramType;
      unsigned int numberOfBins = 20;

      HistogramType targetHistogram = HistogramGeneratorType::ScalarHistogram(targetValues, numberOfBins,
                                                                              this->Mins[component], this->Maxs[component]);
      HistogramType sourceHistogram = HistogramGeneratorType::ScalarHistogram(sourceValues, numberOfBins,
                                                                              this->Mins[component], this->Maxs[component]);

      // We normalize the histograms because the magnitude of the histogram difference should not change based on the number of pixels that were in the valid region of the patches.
      Helpers::NormalizeVector(targetHistogram);
      Helpers::NormalizeVector(sourceHistogram);

      float channelHistogramDifference = Helpers::VectorSumOfAbsoluteDifferences(targetHistogram, sourceHistogram);
      totalHistogramDifference += channelHistogramDifference;
    }

    // Compute the difference
    computedEnergy = totalHistogramDifference;
    std::cout << "HoleHistogramDifferenceAcceptanceVisitor Energy: " << computedEnergy << std::endl;

    if(computedEnergy < this->DifferenceThreshold)
      {
      std::cout << "HoleHistogramDifferenceAcceptanceVisitor: Match accepted (less than " << this->DifferenceThreshold << ")" << std::endl;
      return true;
      }
    else
      {
      std::cout << "HoleHistogramDifferenceAcceptanceVisitor: Match rejected (greater than " << this->DifferenceThreshold << ")" << std::endl;
      return false;
      }
  }

};

#endif


//==================================================================================================
/*!
  @file

  @copyright 2016 NumScale SAS
  @copyright 2016 J.T. Lapreste

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#ifndef BOOST_SIMD_FUNCTION_TWOPOWER_HPP_INCLUDED
#define BOOST_SIMD_FUNCTION_TWOPOWER_HPP_INCLUDED

#if defined(DOXYGEN_ONLY)
namespace boost { namespace simd
{

 /*!

    @ingroup group-bitwise
    Function object implementing twopower capabilities

    Returns \f$2^n\f$ (or 0 if n is less than zero)

    @par semantic:
    For any given value n  of integral type @c N:

    @code
    N r = twopower(n);
    @endcode

    code is similar to:

    @code
    N r = 1 << n;
    @endcode

    @par Note:

    This function is not defined for floating entries

    @see  ilog2

  **/
  const boost::dispatch::functor<tag::twopower_> twopower = {};
} }
#endif

#include <boost/simd/function/scalar/twopower.hpp>
#include <boost/simd/function/simd/twopower.hpp>

#endif


//          Copyright Carl Philipp Reh 2009 - 2016.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#ifndef FCPPT_MATH_DETAIL_DIM_MATCHES_HPP_INCLUDED
#define FCPPT_MATH_DETAIL_DIM_MATCHES_HPP_INCLUDED

#include <fcppt/math/size_type.hpp>
#include <fcppt/config/external_begin.hpp>
#include <boost/mpl/bool.hpp>
#include <fcppt/config/external_end.hpp>

namespace fcppt
{
namespace math
{
namespace detail
{

template<
	fcppt::math::size_type Desired,
	fcppt::math::size_type Current
>
using dim_matches
=
boost::mpl::bool_<
	Desired
	==
	Current
>;

}
}
}

#endif


// Copyright Louis Dionne 2013-2017
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)

#include <boost/hana/assert.hpp>
#include <boost/hana/concept.hpp>
namespace hana = boost::hana;

using T = int;

BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Applicative<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Comonad<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Comparable<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Constant<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::EuclideanRing<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Foldable<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Functor<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Group<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Hashable<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::IntegralConstant<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Iterable<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Logical<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Metafunction<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Monad<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::MonadPlus<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Monoid<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Orderable<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Product<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Ring<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Searchable<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Sequence<T>>{});
BOOST_HANA_CONSTANT_CHECK(hana::IntegralConstant<hana::Struct<T>>{});

int main() { }


#include <boost/numeric/odeint/iterator/const_step_time_iterator.hpp>


//=======================================================================
// Copyright (c) 2005 Aaron Windsor
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
//=======================================================================

#include <boost/config.hpp>

#ifdef BOOST_MSVC
// Without disabling this we get hard errors about initialialized pointers:
#pragma warning(disable : 4703)
#endif

#include <boost/graph/max_cardinality_matching.hpp>

#include <iostream> // for std::cout
#include <boost/property_map/vector_property_map.hpp>
#include <boost/graph/adjacency_list.hpp>
#include <boost/graph/adjacency_matrix.hpp>
#include <boost/graph/random.hpp>
#include <ctime>
#include <boost/random.hpp>
#include <boost/core/lightweight_test.hpp>

using namespace boost;

typedef adjacency_list< vecS, vecS, undirectedS,
    property< vertex_index_t, int > >
    undirected_graph;

typedef adjacency_list< listS, listS, undirectedS,
    property< vertex_index_t, int > >
    undirected_list_graph;

typedef adjacency_matrix< undirectedS, property< vertex_index_t, int > >
    undirected_adjacency_matrix_graph;

template < typename Graph > struct vertex_index_installer
{
    static void install(Graph&) {}
};

template <> struct vertex_index_installer< undirected_list_graph >
{
    static void install(undirected_list_graph& g)
    {
        typedef graph_traits< undirected_list_graph >::vertex_iterator
            vertex_iterator_t;
        typedef graph_traits< undirected_list_graph >::vertices_size_type
            v_size_t;

        vertex_iterator_t vi, vi_end;
        v_size_t i = 0;
        for (boost::tie(vi, vi_end) = vertices(g); vi != vi_end; ++vi, ++i)
            put(vertex_index, g, *vi, i);
    }
};

template < typename Graph > void complete_graph(Graph& g, int n)
{
    // creates the complete graph on n vertices
    typedef typename graph_traits< Graph >::vertex_iterator vertex_iterator_t;

    g = Graph(n);
    vertex_iterator_t vi, vi_end, wi;
    boost::tie(vi, vi_end) = vertices(g);
    for (boost::tie(vi, vi_end) = vertices(g); vi != vi_end; ++vi)
    {
        wi = vi;
        ++wi;
        for (; wi != vi_end; ++wi)
            add_edge(*vi, *wi, g);
    }
}

template < typename Graph > void gabows_graph(Graph& g, int n)
{
    // creates a graph with 2n vertices, consisting of the complete graph
    // on n vertices plus n vertices of degree one, each adjacent to one
    // vertex in the complete graph. without any initial matching, this
    // graph forces edmonds' algorithm into worst-case behavior.

    typedef typename graph_traits< Graph >::vertex_iterator vertex_iterator_t;

    g = Graph(2 * n);

    vertex_iterator_t vi, vi_end, ui, ui_end, halfway;

    boost::tie(ui, ui_end) = vertices(g);

    halfway = ui;
    for (int i = 0; i < n; ++i)
        ++halfway;

    while (ui != halfway)
    {
        vi = ui;
        ++vi;
        while (vi != halfway)
        {
            add_edge(*ui, *vi, g);
            ++vi;
        }
        ++ui;
    }

    boost::tie(ui, ui_end) = vertices(g);

    while (halfway != ui_end)
    {
        add_edge(*ui, *halfway, g);
        ++ui;
        ++halfway;
    }
}

template < typename Graph >
void matching_test(std::size_t num_v, const std::string& graph_name)
{
    typedef
        typename property_map< Graph, vertex_index_t >::type vertex_index_map_t;
    typedef vector_property_map<
        typename graph_traits< Graph >::vertex_descriptor, vertex_index_map_t >
        mate_t;
    typedef typename graph_traits< Graph >::vertex_iterator vertex_iterator_t;
    typedef
        typename graph_traits< Graph >::vertex_descriptor vertex_descriptor_t;

    const std::size_t double_num_v = num_v * 2;

    bool all_tests_passed = true;

    // form the complete graph on 2*n vertices; the maximum cardinality
    // matching, greedy matching, and extra greedy matching should all be the
    // same - a matching of size n.

    Graph g(double_num_v);
    complete_graph(g, double_num_v);

    vertex_index_installer< Graph >::install(g);

    mate_t edmonds_mate(double_num_v);
    mate_t greedy_mate(double_num_v);
    mate_t extra_greedy_mate(double_num_v);

    // find a maximum cardinality matching using edmonds' blossom-shrinking
    // algorithm, starting with an empty matching.
    bool edmonds_result = matching< Graph, mate_t, vertex_index_map_t,
        edmonds_augmenting_path_finder, empty_matching,
        maximum_cardinality_matching_verifier >(
        g, edmonds_mate, get(vertex_index, g));

    BOOST_TEST(edmonds_result);
    if (!edmonds_result)
    {
        std::cout
            << "Verifier reporting a problem finding a perfect matching on"
            << std::endl
            << "the complete graph using " << graph_name << std::endl;
        all_tests_passed = false;
    }

    // find a greedy matching
    bool greedy_result = matching< Graph, mate_t, vertex_index_map_t,
        no_augmenting_path_finder, greedy_matching,
        maximum_cardinality_matching_verifier >(
        g, greedy_mate, get(vertex_index, g));

    BOOST_TEST(greedy_result);
    if (!greedy_result)
    {
        std::cout << "Verifier reporting a problem finding a greedy matching on"
                  << std::endl
                  << "the complete graph using " << graph_name << std::endl;
        all_tests_passed = false;
    }

    // find an extra greedy matching
    bool extra_greedy_result = matching< Graph, mate_t, vertex_index_map_t,
        no_augmenting_path_finder, extra_greedy_matching,
        maximum_cardinality_matching_verifier >(
        g, extra_greedy_mate, get(vertex_index, g));

    BOOST_TEST(extra_greedy_result);
    if (!extra_greedy_result)
    {
        std::cout << "Verifier reporting a problem finding an extra greedy "
                     "matching on"
                  << std::endl
                  << "the complete graph using " << graph_name << std::endl;
        all_tests_passed = false;
    }

    // as a sanity check, make sure that each of the matchings returned is a
    // valid matching and has 1000 edges.

    bool edmonds_sanity_check = is_a_matching(g, edmonds_mate)
        && matching_size(g, edmonds_mate) == num_v;

    BOOST_TEST(edmonds_sanity_check);
    if (edmonds_result && !edmonds_sanity_check)
    {
        std::cout
            << "Verifier okayed edmonds' algorithm on the complete graph, but"
            << std::endl
            << "the matching returned either wasn't a valid matching or wasn't"
            << std::endl
            << "actually a maximum cardinality matching." << std::endl;
        all_tests_passed = false;
    }

    bool greedy_sanity_check = is_a_matching(g, greedy_mate)
        && matching_size(g, greedy_mate) == num_v;

    BOOST_TEST(greedy_sanity_check);
    if (greedy_result && !greedy_sanity_check)
    {
        std::cout
            << "Verifier okayed greedy algorithm on the complete graph, but"
            << std::endl
            << "the matching returned either wasn't a valid matching or wasn't"
            << std::endl
            << "actually a maximum cardinality matching." << std::endl;
        all_tests_passed = false;
    }

    bool extra_greedy_sanity_check = is_a_matching(g, extra_greedy_mate)
        && matching_size(g, extra_greedy_mate) == num_v;

    BOOST_TEST(extra_greedy_sanity_check);
    if (extra_greedy_result && !extra_greedy_sanity_check)
    {
        std::cout
            << "Verifier okayed extra greedy algorithm on the complete graph, "
               "but"
            << std::endl
            << "the matching returned either wasn't a valid matching or wasn't"
            << std::endl
            << "actually a maximum cardinality matching." << std::endl;
        all_tests_passed = false;
    }

    // Now remove an edge from the edmonds_mate matching.
    vertex_iterator_t vi, vi_end;
    for (boost::tie(vi, vi_end) = vertices(g); vi != vi_end; ++vi)
        if (edmonds_mate[*vi] != graph_traits< Graph >::null_vertex())
            break;

    edmonds_mate[edmonds_mate[*vi]] = graph_traits< Graph >::null_vertex();
    edmonds_mate[*vi] = graph_traits< Graph >::null_vertex();

    //...and run the matching verifier - it should tell us that the matching
    // isn't a maximum matching.
    bool modified_edmonds_verification_result
        = maximum_cardinality_matching_verifier< Graph, mate_t,
            vertex_index_map_t >::verify_matching(g, edmonds_mate,
            get(vertex_index, g));

    BOOST_TEST(!modified_edmonds_verification_result);
    if (modified_edmonds_verification_result)
    {
        std::cout << "Verifier was fooled into thinking that a non-maximum "
                     "matching was maximum"
                  << std::endl;
        all_tests_passed = false;
    }

    Graph h(double_num_v);
    gabows_graph(h, num_v);

    vertex_index_installer< Graph >::install(h);

    // gabow's graph always has a perfect matching. it's also a good example of
    // why one should initialize with the extra_greedy_matching in most cases.

    mate_t gabow_mate(double_num_v);

    bool gabows_graph_result
        = checked_edmonds_maximum_cardinality_matching(h, gabow_mate);

    BOOST_TEST(gabows_graph_result);
    if (!gabows_graph_result)
    {
        std::cout
            << "Problem on Gabow's Graph with " << graph_name << ":"
            << std::endl
            << "   Verifier reporting a maximum cardinality matching not found."
            << std::endl;
        all_tests_passed = false;
    }

    BOOST_TEST(matching_size(h, gabow_mate) == num_v);
    if (gabows_graph_result && matching_size(h, gabow_mate) != num_v)
    {
        std::cout
            << "Problem on Gabow's Graph with " << graph_name << ":"
            << std::endl
            << "   Verifier reported a maximum cardinality matching found,"
            << std::endl
            << "   but matching size is " << matching_size(h, gabow_mate)
            << " when it should be " << num_v << std::endl;
        all_tests_passed = false;
    }

    Graph j(double_num_v);
    vertex_index_installer< Graph >::install(j);

    typedef boost::mt19937 base_generator_type;
    base_generator_type generator(static_cast< unsigned int >(std::time(0)));
    boost::uniform_int<> distribution(0, double_num_v - 1);
    boost::variate_generator< base_generator_type&, boost::uniform_int<> >
        rand_num(generator, distribution);

    std::size_t num_edges = 0;
    bool success;

    while (num_edges < 4 * double_num_v)
    {
        vertex_descriptor_t u = random_vertex(j, rand_num);
        vertex_descriptor_t v = random_vertex(j, rand_num);
        if (u != v)
        {
            boost::tie(tuples::ignore, success) = add_edge(u, v, j);
            if (success)
                num_edges++;
        }
    }

    mate_t random_mate(double_num_v);
    bool random_graph_result
        = checked_edmonds_maximum_cardinality_matching(j, random_mate);

    BOOST_TEST(random_graph_result);
    if (!random_graph_result)
    {
        std::cout << "Matching in random graph not maximum for " << graph_name
                  << std::endl;
        all_tests_passed = false;
    }

    // Now remove an edge from the random_mate matching.
    for (boost::tie(vi, vi_end) = vertices(j); vi != vi_end; ++vi)
        if (random_mate[*vi] != graph_traits< Graph >::null_vertex())
            break;

    random_mate[random_mate[*vi]] = graph_traits< Graph >::null_vertex();
    random_mate[*vi] = graph_traits< Graph >::null_vertex();

    //...and run the matching verifier - it should tell us that the matching
    // isn't a maximum matching.
    bool modified_random_verification_result
        = maximum_cardinality_matching_verifier< Graph, mate_t,
            vertex_index_map_t >::verify_matching(j, random_mate,
            get(vertex_index, j));

    BOOST_TEST(!modified_random_verification_result);
    if (modified_random_verification_result)
    {
        std::cout << "Verifier was fooled into thinking that a non-maximum "
                     "matching was maximum"
                  << std::endl;
        all_tests_passed = false;
    }

    BOOST_TEST(all_tests_passed);
    if (all_tests_passed)
    {
        std::cout << graph_name << " passed all tests for n = " << num_v << '.'
                  << std::endl;
    }
}

int main(int, char*[])
{

    matching_test< undirected_graph >(10, "adjacency_list (using vectors)");
    matching_test< undirected_list_graph >(10, "adjacency_list (using lists)");
    matching_test< undirected_adjacency_matrix_graph >(10, "adjacency_matrix");

    matching_test< undirected_graph >(20, "adjacency_list (using vectors)");
    matching_test< undirected_list_graph >(20, "adjacency_list (using lists)");
    matching_test< undirected_adjacency_matrix_graph >(20, "adjacency_matrix");

    matching_test< undirected_graph >(21, "adjacency_list (using vectors)");
    matching_test< undirected_list_graph >(21, "adjacency_list (using lists)");
    matching_test< undirected_adjacency_matrix_graph >(21, "adjacency_matrix");

#if 0
  matching_test<undirected_graph>(50, "adjacency_list (using vectors)");
  matching_test<undirected_list_graph>(50, "adjacency_list (using lists)");
  matching_test<undirected_adjacency_matrix_graph>(50, "adjacency_matrix");

  matching_test<undirected_graph>(51, "adjacency_list (using vectors)");
  matching_test<undirected_list_graph>(51, "adjacency_list (using lists)");
  matching_test<undirected_adjacency_matrix_graph>(51, "adjacency_matrix");
#endif

    return boost::report_errors();
}


#include <qubus/qtl/multi_index_handling.hpp>

#include <qubus/IR/qir.hpp>

#include <qubus/pattern/IR.hpp>
#include <qubus/pattern/core.hpp>

#include <qubus/qtl/pattern/all.hpp>

#include <boost/optional.hpp>
#include <boost/range/combine.hpp>

#include <qubus/util/assert.hpp>

#include <map>
#include <vector>

namespace qubus
{
namespace qtl
{

namespace
{

std::vector<std::unique_ptr<expression>> expand_multi_index(
    const expression& expr,
    const std::map<util::handle, std::vector<std::unique_ptr<expression>>>& multi_index_map)
{
    using qubus::pattern::_;
    using qubus::pattern::variable;
    using pattern::multi_index;

    variable<variable_declaration> decl;

    auto m =
        qubus::pattern::make_matcher<expression, std::vector<std::unique_ptr<expression>>>()
            .case_(multi_index(decl), [&] { return clone(multi_index_map.at(decl.get().id())); })
            .case_(_, [&](const expression& self) {
                std::vector<std::unique_ptr<expression>> indices;
                indices.push_back(clone(self));

                return indices;
            });

    return qubus::pattern::match(expr, m);
}
}

std::unique_ptr<expression> expand_multi_indices(const expression& expr)
{
    using qubus::pattern::variable;
    using pattern::for_all_multi;
    using pattern::sum_multi;

    std::map<util::handle, std::vector<std::unique_ptr<expression>>> multi_index_map;

    variable<std::vector<std::reference_wrapper<expression>>> indices;
    variable<const access_expr&> tensor;

    variable<std::vector<variable_declaration>> index_decls;
    variable<variable_declaration> alias;

    variable<const expression &> body, a, b;

    auto m =
        qubus::pattern::make_matcher<expression, std::unique_ptr<expression>>()
            .case_(subscription(tensor, indices),
                   [&] {
                       std::vector<std::unique_ptr<expression>> new_indices;

                       for (const auto& index : indices.get())
                       {
                           auto expanded_indices = expand_multi_index(index, multi_index_map);

                           for (auto&& index : std::move(expanded_indices))
                           {
                               new_indices.push_back(std::move(index));
                           }
                       }

                       return subscription(clone(tensor.get()), std::move(new_indices));
                   })
            //                 TODO: Reenable the code below after implementing tuples. They are
            //                 needed to handle the extent.
            //                 .case_(delta(a, b),
            //                        [&]
            //                        {
            //                            auto new_indices_left = expand_multi_index(a.get(),
            //                            multi_index_map);
            //                            auto new_indices_right = expand_multi_index(b.get(),
            //                            multi_index_map);
            //
            //                            if (new_indices_left.size() != new_indices_right.size())
            //                                throw 0;
            //
            //                            boost::optional<expression> expanded_delta;
            //
            //                            for (const auto& index_pair :
            //                                 boost::range::combine(new_indices_left,
            //                                 new_indices_right))
            //                            {
            //                                expression delta = kronecker_delta_expr(
            //                                    0, boost::get<0>(index_pair),
            //                                    boost::get<1>(index_pair));
            //
            //                                if (expanded_delta)
            //                                {
            //                                    expanded_delta =
            //                                    binary_operator_expr(binary_op_tag::multiplies,
            //                                                                          *expanded_delta,
            //                                                                          delta);
            //                                }
            //                                else
            //                                {
            //                                    expanded_delta = delta;
            //                                };
            //                            }
            //
            //                            QUBUS_ASSERT(expanded_delta, "Invalid expression.");
            //
            //                            return *expanded_delta;
            //                        })
            .case_(for_all_multi(index_decls, alias, body),
                   [&] {
                       std::unique_ptr<expression> result = clone(body.get());

                       std::vector<std::unique_ptr<expression>> element_indices;

                       for (const auto& decl : index_decls.get())
                       {
                           element_indices.emplace_back(var(decl));

                           result = for_all(decl, std::move(result));
                       }

                       multi_index_map[alias.get().id()] = std::move(element_indices);

                       return result;
                   })
            .case_(for_all_multi(index_decls, body),
                   [&] {
                       std::unique_ptr<expression> result = clone(body.get());

                       for (const auto& decl : index_decls.get())
                       {
                           result = for_all(decl, std::move(result));
                       }

                       return result;
                   })
            .case_(sum_multi(body, index_decls, alias),
                   [&] {
                       std::unique_ptr<expression> result = clone(body.get());

                       std::vector<std::unique_ptr<expression>> element_indices;

                       for (const auto& decl : index_decls.get())
                       {
                           element_indices.emplace_back(var(decl));

                           result = sum(decl, std::move(result));
                       }

                       multi_index_map[alias.get().id()] = std::move(element_indices);

                       return result;
                   })
            .case_(sum_multi(body, index_decls), [&] {
                std::unique_ptr<expression> result = clone(body.get());

                for (const auto& decl : index_decls.get())
                {
                    result = sum(decl, std::move(result));
                }

                return result;
            });

    return qubus::pattern::substitute(expr, m);
}

}
}


//==================================================================================================
/*!

  Copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#include <boost/simd/function/scalar/bitfloating.hpp>
#include <scalar_test.hpp>
#include <boost/simd/detail/dispatch/meta/as_integer.hpp>
#include <boost/simd/constant/mone.hpp>
#include <boost/simd/constant/one.hpp>
#include <boost/simd/constant/zero.hpp>
#include <boost/simd/constant/bitincrement.hpp>

STF_CASE_TPL (" bitfloating int_convert",  (int32_t)(int64_t))
{
  namespace bs = boost::simd;
  using bs::bitfloating;
  using r_t = decltype(bitfloating(T()));
  typedef  boost::dispatch::as_floating_t<T>  wished_r_t;

  // return type conformity test
  STF_TYPE_IS(r_t, wished_r_t);

  // specific values tests
#if !defined(BOOST_SIMD_NO_DENORMALS)
  STF_EQUAL(bitfloating(bs::One<T>()), bs::Bitincrement<r_t>());
  STF_EQUAL(bitfloating(bs::Mone<T>()), -bs::Bitincrement<r_t>());
#endif
  STF_EQUAL(bitfloating(bs::Zero<T>()), bs::Zero<r_t>());
}

STF_CASE_TPL (" bitfloating uint_convert",  (uint32_t)(uint64_t))
{
  namespace bs = boost::simd;
  using bs::bitfloating;
  using r_t = decltype(bitfloating(T()));
  typedef boost::dispatch::as_floating_t<T>  wished_r_t;

  // return type conformity test
  STF_TYPE_IS(r_t, wished_r_t);

  // specific values tests
  STF_EQUAL(bitfloating(bs::One<T>()), bs::Bitincrement<r_t>());
  STF_EQUAL(bitfloating(bs::Zero<T>()), bs::Zero<r_t>());
}