/*
  Tencent is pleased to support the open source community by making
  Plato available.
  Copyright (C) 2019 THL A29 Limited, a Tencent company.
  All rights reserved.

  Licensed under the BSD 3-Clause License (the "License"); you may
  not use this file except in compliance with the License. You may
  obtain a copy of the License at

  https://opensource.org/licenses/BSD-3-Clause

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" basis,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  implied. See the License for the specific language governing
  permissions and limitations under the License.

  See the AUTHORS file for names of contributors.
*/

#ifndef ALGO_CNC_DAVID_HPP_
#define ALGO_CNC_DAVID_HPP_

#include <glog/logging.h>
#include <algorithm>
#include <boost/format.hpp>
#include <cstdlib>
#include <unordered_set>
#include <vector>

#include "cgm.hpp"
#include "distance.hpp"

namespace plato { namespace algo {
struct david_opts_t {
    vid_t num_samples_ = 10;
};

template <typename INCOMING, typename OUTGOING>
class david_closeness_t {
public:
  using partition_t = typename dualmode_detail::partition_traits<INCOMING, OUTGOING>::type;
  using closeness_state_t  = dense_state_t<double, partition_t>;

  struct david_msg_type_t {
    vid_t src_;
    double value_;
  };

  /**
   * @brief
   * @param engine
   * @param graph_info
   * @param opts
   */
  explicit david_closeness_t(dualmode_engine_t<INCOMING, OUTGOING> * engine, const graph_info_t& graph_info,
      const david_opts_t& opts = david_opts_t());
  /**
   * @brief
   */
  ~david_closeness_t();

  /**
   * @brief
   */
  void compute();

  /**
   * @brief
   * @param v
   * @return
   */
  double get_closeness_of(vid_t v) const;

  /**
   * @brief
   * @tparam STREAM
   * @param ss
   */
  template<typename STREAM>
  void save(std::vector<STREAM*>& ss);

  /**
   * @brief
   * @return
   */
  std::vector<vid_t>* get_samples() {
    return &samples_;
  }

  /**
   * @brief
   * @return
   */
  vid_t get_major_component_vertices() const {
    return major_component_vertices_;
  }
private:
  /**
   * @brief
   */
  void make_samples();

  /**
   * @brief
   * @param i
   */
  void print_progress(size_t i);

  /**
   * @brief
   */
  void compute_closeness_inv();

private:
  dualmode_engine_t<INCOMING, OUTGOING> *engine_;
  graph_info_t graph_info_;
  vid_t num_samples_;
  std::vector<vid_t> samples_;
  closeness_state_t closeness_;
  double fac_;

  connected_component_t<INCOMING, OUTGOING> *cc_;
  vid_t major_component_label_;
  vid_t major_component_vertices_;
};

template <typename INCOMING, typename OUTGOING>
david_closeness_t<INCOMING, OUTGOING>::david_closeness_t(dualmode_engine_t<INCOMING, OUTGOING> *engine,
    const graph_info_t& graph_info, const david_opts_t& opts) : engine_(engine),
    graph_info_(graph_info),
    closeness_(graph_info.max_v_i_, engine->in_edges()->partitioner()) {
  cc_ = new connected_component_t<INCOMING, OUTGOING>(engine, graph_info);
  cc_->compute();

  major_component_label_ = cc_->get_major_label();
  major_component_vertices_ = cc_->get_major_vertices();
  LOG(INFO) << "major_label: " << major_component_label_ << " major_vertices: " << major_component_vertices_ << std::endl;

  num_samples_ = std::min(opts.num_samples_, major_component_vertices_);
  fac_ = 1.0 * major_component_vertices_ / num_samples_ /
         (major_component_vertices_ - 1);

  samples_.resize(num_samples_);
  closeness_.fill(0.0f);

}

template <typename INCOMING, typename OUTGOING>
david_closeness_t<INCOMING, OUTGOING>::~david_closeness_t() {
  delete cc_;
}

// partition 0 randomly select num_samples samples and broadcast them to other
// partitions
template <typename INCOMING, typename OUTGOING>
void david_closeness_t<INCOMING, OUTGOING>::make_samples() {
  auto p = cc_->get_labels();
  auto active_all = engine_->alloc_v_subset();
  active_all.fill();
  auto active_view_all = plato::create_active_v_view(engine_->out_edges()->partitioner()->self_v_view(), active_all);
  auto global_label = engine_->template alloc_v_state<vid_t>();
  active_view_all.template foreach<vid_t>([&](vid_t v_i){
      global_label[v_i] = (*p)[v_i];
    return 0;
  });

  auto& cluster_info = plato::cluster_info_t::get_instance();
  if (cluster_info.partition_id_ != 0) {
    vid_t v_begin = engine_->out_edges()->partitioner()->offset_[cluster_info.partition_id_];
    vid_t v_end = engine_->out_edges()->partitioner()->offset_[cluster_info.partition_id_+1];
    MPI_Send(&global_label[v_begin], v_end - v_begin, get_mpi_data_type<vid_t>(), 0, 0, MPI_COMM_WORLD);
  }
  else {
    for (int i = 1; i < cluster_info.partitions_; ++i ) {
      MPI_Status recv_status;
      vid_t v_begin = engine_->out_edges()->partitioner()->offset_[i];
      vid_t v_end = engine_->out_edges()->partitioner()->offset_[i+1];
      MPI_Recv(&global_label[v_begin], v_end - v_begin, get_mpi_data_type<vid_t>(), i, 0, MPI_COMM_WORLD, &recv_status);
    }

    //for (vid_t i = 0; i < graph_info_.vertices_; ++i) {
    //  LOG(INFO) << "vertex: " << i << " label: " << global_label[i] << std::endl;
    //}
  }

  if (cluster_info.partition_id_ == 0) {
    std::unordered_set<vid_t> unique_samples;
    while (unique_samples.size() < num_samples_) {
      vid_t r = std::rand() % graph_info_.vertices_;
      if (global_label[r] != major_component_label_) continue;
      unique_samples.insert(r);
    }
    std::copy(unique_samples.begin(), unique_samples.end(), samples_.begin());
  }
  MPI_Bcast(&samples_[0], num_samples_, get_mpi_data_type<vid_t>(), 0, MPI_COMM_WORLD);

  LOG(INFO) << "make samples finish  num_samples: " << num_samples_ << std::endl;
}

template <typename INCOMING, typename OUTGOING>
void david_closeness_t<INCOMING, OUTGOING>::print_progress(size_t i) {
  auto& cluster_info = plato::cluster_info_t::get_instance();
  LOG_IF(INFO, samples_.size() >= 50 && i % (samples_.size() / 50) == 0 && cluster_info.partition_id_ == 0)
    << "process " << i << " of " << samples_.size() << " samples ("
    << i * 100 / samples_.size() << "%)\n";
}

template <typename INCOMING, typename OUTGOING>
void david_closeness_t<INCOMING, OUTGOING>::compute_closeness_inv() {
  auto& cluster_info = plato::cluster_info_t::get_instance();
  vid_t v_begin = engine_->out_edges()->partitioner()->offset_[cluster_info.partition_id_];
  vid_t v_end = engine_->out_edges()->partitioner()->offset_[cluster_info.partition_id_+1];
#pragma omp parallel for
  for (vid_t i = v_begin; i < v_end; i++) {
    closeness_[i] = 1.0 / closeness_[i];
    if (std::isinf(closeness_[i])) {
      closeness_[i] = 0;
    }
  }
}

template <typename INCOMING, typename OUTGOING>
void david_closeness_t<INCOMING, OUTGOING>::compute() {
  auto& cluster_info = plato::cluster_info_t::get_instance();
  make_samples();

  auto accu_func = [&](vid_t v_i, vid_t * val) {
    closeness_[v_i] += *val * fac_; return *val;
  };

  double exe_time = 0;
  exe_time -= MPI_Wtime();
  for (size_t i = 0; i < samples_.size(); i++) {
    vid_t root = samples_[i];
    calc_distance<INCOMING, OUTGOING>(engine_, root, accu_func);
    print_progress(i);
  }

  // the closeness_ is the reverse
  compute_closeness_inv();

  exe_time += MPI_Wtime();

  LOG_IF(INFO, cluster_info.partition_id_ == 0)
      << "performance: " << samples_.size() / exe_time << " vps\n";
}

template <typename INCOMING, typename OUTGOING>
double david_closeness_t<INCOMING, OUTGOING>::get_closeness_of(vid_t v) const {
  double cl = 0;
  double target_id = engine_->out_edges()->partitioner()->get_partition_id(v);

  auto& cluster_info = plato::cluster_info_t::get_instance();
  if (cluster_info.partition_id_ == target_id) {
    cl = closeness_[v];
  }

  MPI_Bcast(&cl, 1, MPI_DOUBLE, target_id, MPI_COMM_WORLD);
  return cl;
}

template <typename INCOMING, typename OUTGOING>
template <typename STREAM>
void david_closeness_t<INCOMING, OUTGOING>::save(std::vector<STREAM*>& ss) {
  boost::lockfree::queue<david_msg_type_t> que(1024);
  LOG_IF(FATAL, !que.is_lock_free())
    << "boost::lockfree::queue is not lock free\n";

  // start a thread to pop and edge and write to output
  std::atomic<bool> done(false);
  std::thread pop_write([&done, &ss, &que](void) {
#pragma omp parallel num_threads(ss.size())
    {
      int tid = omp_get_thread_num();
      david_msg_type_t vb;
      while (!done) {
        if (que.pop(vb)) {
          *ss[tid] << vb.src_ << "," << vb.value_ << "\n";
        }
      }

      while (que.pop(vb)) {
        *ss[tid] << vb.src_ << "," << vb.value_ << "\n";
      }
    }
  });

  auto active_all = engine_->alloc_v_subset();
  active_all.fill();
  //traverse
  auto active_view_all = plato::create_active_v_view(engine_->out_edges()->partitioner()->self_v_view(), active_all);
  active_view_all.template foreach<vid_t>([&](vid_t v_i){
    while (!que.push(david_msg_type_t {v_i, closeness_[v_i] })) {
    }
    return 1;
  });

  done = true;
  pop_write.join();

}

}  // namespace algo
}  // namespace plato
#endif


/*
* The MIT License (MIT)
*
* Copyright (c) 2018 Sylko Olzscher
*
*/

#include <cyng/crypto/aes.h>
#include <openssl/crypto.h>	//	OPENSSL_cleanse
#include <boost/assert.hpp>

namespace cyng
{
	namespace crypto
	{
		void encrypt(buffer_t& out, buffer_t const& inp, aes_256_key const& key)
		{
            BOOST_ASSERT_MSG(inp.size() == 16, "wrong AES block size");
			out.resize(inp.size());

			::AES_KEY enc_key;
			::AES_set_encrypt_key(key.aes_key_.data(), 256, &enc_key);
			::AES_encrypt((const unsigned char*)inp.data(), (unsigned char*)out.data(), &enc_key);
		}

		void decrypt(buffer_t& out, buffer_t const& inp, aes_256_key const& key)
		{
            BOOST_ASSERT_MSG(inp.size() == 16, "wrong AES block size");
            out.resize(inp.size());

			::AES_KEY dec_key;
			::AES_set_decrypt_key(key.aes_key_.data(), 256, &dec_key);
			::AES_decrypt((const unsigned char*)inp.data(), (unsigned char*)out.data(), &dec_key);

		}
	}
}


// Boost.Geometry (aka GGL, Generic Geometry Library)

// Copyright (c) 2007-2014 Barend Gehrels, Amsterdam, the Netherlands.
// Copyright (c) 2008-2014 Bruno Lalande, Paris, France.
// Copyright (c) 2009-2014 Mateusz Loskot, London, UK.
// Copyright (c) 2013-2014 Adam Wulkiewicz, Lodz, Poland.

// This file was modified by Oracle on 2013-2014.
// Modifications copyright (c) 2013-2014, Oracle and/or its affiliates.

// Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle
// Contributed and/or modified by Menelaos Karavelas, on behalf of Oracle

// Parts of Boost.Geometry are redesigned from Geodan's Geographic Library
// (geolib/GGL), copyright (c) 1995-2010 Geodan, Amsterdam, the Netherlands.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_ALGORITHMS_DETAIL_DISJOINT_LINEAR_SEGMENT_OR_BOX_HPP
#define BOOST_GEOMETRY_ALGORITHMS_DETAIL_DISJOINT_LINEAR_SEGMENT_OR_BOX_HPP

#include <boost/range.hpp>
#include <boost/geometry/util/range.hpp>

#include <boost/geometry/core/closure.hpp>

#include <boost/geometry/geometries/segment.hpp>

#include <boost/geometry/algorithms/not_implemented.hpp>

#include <boost/geometry/views/closeable_view.hpp>

#include <boost/geometry/algorithms/detail/disjoint/multirange_geometry.hpp>
#include <boost/geometry/algorithms/dispatch/disjoint.hpp>

namespace boost { namespace geometry
{

#ifndef DOXYGEN_NO_DETAIL
namespace detail { namespace disjoint
{

template
<
    typename Range,
    closure_selector Closure,
    typename SegmentOrBox
>
struct disjoint_range_segment_or_box
{
    template <typename Strategy>
    static inline bool apply(Range const& range,
                             SegmentOrBox const& segment_or_box,
                             Strategy const& strategy)
    {
        typedef typename closeable_view<Range const, Closure>::type view_type;

        typedef typename ::boost::range_value<view_type>::type point_type;
        typedef typename ::boost::range_iterator
            <
                view_type const
            >::type const_iterator;

        typedef typename ::boost::range_size<view_type>::type size_type;

        typedef typename geometry::model::referring_segment
            <
                point_type const
            > range_segment;

        view_type view(range);

        const size_type count = ::boost::size(view);

        if ( count == 0 )
        {
            return false;
        }
        else if ( count == 1 )
        {
            return dispatch::disjoint
                <
                    point_type, SegmentOrBox
                >::apply(geometry::range::front<view_type const>(view),
                         segment_or_box,
                         strategy.template get_point_in_geometry_strategy<Range, SegmentOrBox>());
        }
        else
        {
            const_iterator it0 = ::boost::begin(view);
            const_iterator it1 = ::boost::begin(view) + 1;
            const_iterator last = ::boost::end(view);

            for ( ; it1 != last ; ++it0, ++it1 )
            {
                range_segment rng_segment(*it0, *it1);
                if ( !dispatch::disjoint
                         <
                             range_segment, SegmentOrBox
                         >::apply(rng_segment, segment_or_box, strategy) )
                {
                    return false;
                }
            }
            return true;
        }
    }
};

template
<
    typename Linear,
    typename SegmentOrBox,
    typename Tag = typename tag<Linear>::type
>
struct disjoint_linear_segment_or_box
    : not_implemented<Linear, SegmentOrBox>
{};

template <typename Linestring, typename SegmentOrBox>
struct disjoint_linear_segment_or_box<Linestring, SegmentOrBox, linestring_tag>
    : disjoint_range_segment_or_box<Linestring, closed, SegmentOrBox>
{};

template <typename MultiLinestring, typename SegmentOrBox>
struct disjoint_linear_segment_or_box
    <
        MultiLinestring, SegmentOrBox, multi_linestring_tag
    > : multirange_constant_size_geometry<MultiLinestring, SegmentOrBox>
{};

}} // namespace detail::disjoint
#endif // DOXYGEN_NO_DETAIL

#ifndef DOXYGEN_NO_DISPATCH
namespace dispatch
{

template <typename Linear, typename Segment>
struct disjoint<Linear, Segment, 2, linear_tag, segment_tag, false>
    : detail::disjoint::disjoint_linear_segment_or_box<Linear, Segment>
{};

template <typename Linear, typename Box, std::size_t DimensionCount>
struct disjoint<Linear, Box, DimensionCount, linear_tag, box_tag, false>
    : detail::disjoint::disjoint_linear_segment_or_box<Linear, Box>
{};

} // namespace dispatch
#endif // DOXYGEN_NO_DISPATCH

}} // namespace boost::geometry

#endif // BOOST_GEOMETRY_ALGORITHMS_DETAIL_DISJOINT_LINEAR_SEGMENT_OR_BOX_HPP


/*
 * (C) Copyright 2020 UCAR
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 */
/// This program tests that Variable read and write selections work as expected for certain engines.

#include <Eigen/Dense>
#include <iostream>
#include <vector>

#include "ioda/Engines/Factory.h"
#include "ioda/Exception.h"
#include "ioda/Group.h"

void test_group_backend_engine(ioda::Group g) {
  // test_data1 is a 4x4 matrix that looks like:
  //  1  2  3  4
  //  5  6  7  8
  //  9 10 11 12
  // 13 14 15 16
  Eigen::ArrayXXi test_data1(4, 4);
  test_data1 << 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16;

  // std::cerr << test_data1 << std::endl << std::endl;

  // Make a variable in the file and write test_data1.
  ioda::Variable file_test_data1 = g.vars.create<int>("test_data1", {4, 4}).writeWithEigenRegular(test_data1);

  // overlay_data1 is a 2x2 matrix:
  // 17 18
  // 19 20
  Eigen::ArrayXXi overlay_data1(2, 2);
  overlay_data1 << 17, 18, 19, 20;

  // std::cerr << overlay_data1 << std::endl << std::endl;

  // Write the 2x2 overlay on top of the existing data.

  // We use two selectors. The first selects the data in memory (i.e. the inputs that we
  // provide). The second selects the range of the data in ioda.
  // Because we are not writing the full dimensions of the variable, we need
  // to specify selectors for both user memory and ioda.

  // The memory selector defines a hyperslab, starting at (0,0), with size (2,2).
  //   Since the 2x2 in-memory object has different dimensions than the object stored in
  //   ioda (4x4), we need to pass along the dimensions of our object to properly write
  //   rows and columns. This is accomplished by setting extent({2,2}).
  // The ioda 'file' selector also defines a hyperslab, but it starts at (2,2) with size (2,2).
  //   So, we are writing the lower right quadrant of the matrix.
  file_test_data1.writeWithEigenRegular(
    overlay_data1, ioda::Selection().extent({2, 2}).select({ioda::SelectionOperator::SET, {0, 0}, {2, 2}}),
    ioda::Selection().select({ioda::SelectionOperator::SET, {2, 2}, {2, 2}}));
  // file_test_data1 should now look like:
  //  1  2  3  4
  //  5  6  7  8
  //  9 10 17 18
  // 13 14 19 20

  // Now, let's write this 2x2 object, again, to the same output.
  file_test_data1.writeWithEigenRegular(
    overlay_data1, ioda::Selection().extent({2, 2}).select({ioda::SelectionOperator::SET, {0, 0}, {2, 2}}),
    ioda::Selection().select({ioda::SelectionOperator::SET, {1, 1}, {2, 2}}));
  // file_test_data1 should now look like:
  //  1  2  3  4
  //  5 17 18  8
  //  9 19 20 18
  // 13 14 19 20

  // Let's use a different type of selector: the point selector.
  // We will set (3,0) = 21, and (0,3) = 22.
  std::vector<int> point_vals{21, 22};
  file_test_data1.write(gsl::make_span(point_vals),  // Values of the two points
                        ioda::Selection().extent({2, 1}).select(
                          {ioda::SelectionOperator::SET, {{0, 0}, {1, 0}}}),  // Writing two points
                        ioda::Selection().select({ioda::SelectionOperator::SET,
                                                  {{3, 0}, {0, 3}}}));  // Location where we write the data.
  // file_test_data1 should now look like:
  //  1  2  3 22
  //  5 17 18  8
  //  9 19 20 18
  // 21 14 19 20

  // And check our read function.
  Eigen::ArrayXXi reference(4, 4);
  reference << 1, 2, 3, 22, 5, 17, 18, 8, 9, 19, 20, 18, 21, 14, 19, 20;

  Eigen::ArrayXXi check;
  file_test_data1.readWithEigenRegular(check);

  bool r = check.isApprox(reference);
  if (!r) throw std::logic_error("Test failure.");
}

int main(int argc, char** argv) {
  using namespace ioda;
  using namespace std;
  try {
    auto f = Engines::constructFromCmdLine(argc, argv, "test-data-selections.hdf5");
    test_group_backend_engine(f);
  } catch (const std::exception& e) {
    ioda::unwind_exception_stack(e);
    return 1;
  }
  return 0;
}


// ---------------------------------------------------------------------------
//
//  This file is part of PermLib.
//
// Copyright (c) 2009-2011 Thomas Rehn <thomas@carmen76.de>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// ---------------------------------------------------------------------------

#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE partition magic
#include <boost/test/unit_test.hpp>

#include <iostream>
#include <fstream>
#include <algorithm>

#include <permlib/permutation.h>
#include <permlib/generator/product_replacement_generator.h>
#include <permlib/bsgs.h>
#include <permlib/transversal/explicit_transversal.h>
#include <permlib/construct/schreier_sims_construction.h>

#include "test-common.h"
#include <permlib/search/partition/partition.h>
//#include <permlib/search/partition/r_base.h>

using namespace permlib::partition;

BOOST_AUTO_TEST_CASE( partition_intersection )
{
	const unsigned long n = 5;
	Partition P(n);

	const unsigned long set[5] = {0,1,2,3,4};

	Partition Pprime(P);

    BOOST_CHECK( Pprime.intersect(set, set+3, 0) );
    BOOST_CHECK( Pprime.cells() == 2 );
    BOOST_CHECK( ! Pprime.intersect(set, set+3, 0) );
    BOOST_CHECK( ! Pprime.intersect(set, set+3, 1) );

    BOOST_CHECK( Pprime.intersect(set+2, set+4, 0) );
    BOOST_CHECK( Pprime.cells() == 3 );
    BOOST_CHECK( Pprime.intersect(set+2, set+4, 1) );
    BOOST_CHECK( Pprime.cells() == 4 );
    BOOST_CHECK( Pprime.intersect(set+1, set+3, 2) );
    BOOST_CHECK( Pprime.cells() == 5 );

    const unsigned long fixSizes[4] = {5,3,1,0};
    const unsigned long fixContain[4] = {0,4,2,0};
    unsigned int i = 0;
    while (Pprime.fixPointsSize() && i < 6) {
	    std::vector<unsigned long> Pfix(Pprime.fixPointsBegin(), Pprime.fixPointsEnd());
        BOOST_CHECK( Pfix.size() == Pprime.fixPointsSize() );
        BOOST_CHECK( Pfix.size() == fixSizes[i] );
        BOOST_CHECK( std::find(Pfix.begin(), Pfix.end(), fixContain[i]) != Pfix.end() );

        BOOST_CHECK( Pprime.undoIntersection() );
        ++i;
    }
    BOOST_CHECK( i == 3 );

    BOOST_CHECK( Pprime.undoIntersection() );
    BOOST_CHECK( ! Pprime.undoIntersection() );
}


#include <boost/hana/lexicographical_compare.hpp>


//    Copyright 2017 Rainer Gemulla
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
/** \file
 *
 * Illustrates matrix factorization with parallel SGD. We first create factors and then a data matrix
 * from these factors. This process ensures that we know the best factorization of the input.
 * These matrices are distributed across a cluster. We then try to reconstruct the factors
 * using PSGD.
 *
 * Run with: psgd
 * (make sure to use a production build, otherwise it will be slow)
 */
#include <iostream>
#include <sstream>

#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>

#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/math/distributions/normal.hpp>
#include <boost/random/uniform_real.hpp>
#include <mf/matrix/io/generateDistributedMatrix.h>
#include <mpi2/mpi2.h>
#include <mf/mf.h>

log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("main"));

using namespace std;
using namespace mf;
using namespace mpi2;
using namespace rg;
using namespace boost::numeric::ublas;

// type of SGD
typedef UpdateTruncate<UpdateNzslNzl2> Update;
typedef UpdateLock<Update> UpdateL;
typedef RegularizeNone Regularize;
typedef SumLoss<NzslLoss, Nzl2Loss> Loss;
typedef NzslLoss TestLoss;

struct TrainingPoint {

	TrainingPoint(){}
	TrainingPoint (mf_size_type i , mf_size_type j, double x):i_(i), j_(j), x_(x){}

	mf_size_type i_;
	mf_size_type j_;
	double x_;

};

//global variables
mf_size_type size1;
mf_size_type size2;
mf_size_type b1;
mf_size_type b2;

mf_size_type minSize1;
mf_size_type minSize2;
mf_size_type remainder1;
mf_size_type remainder2;

int calculateBlocking(double size1, double size2, double nnz, double rank, double cache){

	int SizeOfInt = 8;
	double alpha = cache * 1024;
	double beta = (-1)*(size1+size2)*rank*SizeOfInt;
	double gamma = (-1)*nnz*3*SizeOfInt;

	double delta = beta*beta-4*alpha*gamma;

	int b = ((-1)*beta+sqrt(delta))/(2*alpha);

	return b;
}

// compare function for sort
inline bool compFunction (TrainingPoint point1, TrainingPoint point2) {

	mf_size_type block1_i = min ((mf_size_type)floor(point1.i_/minSize1), b1 -1);
	mf_size_type block2_i = min ((mf_size_type)floor(point2.i_/minSize1), b1 -1);
	mf_size_type block1_j = min ((mf_size_type)floor(point1.j_/minSize2), b2 -1);
	mf_size_type block2_j = min ((mf_size_type)floor(point2.j_/minSize2), b2 -1);
	bool result = false;
	if (block1_i < block2_i) {
		result = true;
	} else if (block1_i == block2_i) {
		if (block1_j < block2_j) {
			result = true;
		} else {
			result =false;
		}
	}
	return result;
}

int main(int argc, char* argv[]) {

	using namespace boost::program_options;
	// initialize mf library and mpi2
	boost::mpi::communicator& world = mfInit(argc, argv);

	// parameters for the factorization
	mf_size_type rank = 10;
	int tasks = 4;
	mf_size_type epochs = 10;
	SgdOrder order = SGD_ORDER_WOR;
	BalanceType balanceType = BALANCE_NONE;
	BalanceMethod balanceMethod = BALANCE_OPTIMAL;
	double lambda = 0.05;
	double eps0 = 0.0125;
	mf_size_type cache = 256;
	string traceFile,traceVar;

	// start mf library
	mfStart();

	if (world.rank() == 0)
	{
#ifndef NDEBUG
		LOG4CXX_WARN(logger, "Warning: Debug mode activated (runtimes may be slow).");
#endif

		string inputMatrixFile;
		string inputRowFacFile;
		string inputColFacFile;
		string outputRowFacFile;
		string outputColFacFile;
		string inputTestMatrixFile;
		string traceFile,traceVar;

		options_description desc("Options");
		desc.add_options()
				("help", "produce help message")
				("epochs", value<mf_size_type>(&epochs)->default_value(10), "number of epochs to run [10]")
				("rank", value<mf_size_type>(&rank)->default_value(10), "rank of factorization [10]")
				("cache", value<mf_size_type>(&cache)->default_value(256), "last level cache (in KB) available per core [256]")
				("lambda", value<double>(&lambda)->default_value(0.05), "lambda")
				("eps0", value<double>(&eps0)->default_value(0.01), "initial step size for BoldDriver")
				("tasks-per-rank", value<int>(&tasks)->default_value(1), "number of concurrent tasks [1]")
				("trace", value<string>(&traceFile)->default_value("trace.R"), "filename of trace [trace]")
				("traceVar", value<string>(&traceVar)->default_value("trace"), "variable name for trace [traceVar]")
				("input-file", value<string>(&inputMatrixFile), "input matrix")
				("input-test-file", value<string>(&inputTestMatrixFile), "input test matrix")
			    ("input-row-file", value<string>(&inputRowFacFile), "input initial row factor")
			    ("input-col-file", value<string>(&inputColFacFile), "input initial column factor")
			    ("output-row-file", value<string>(&outputRowFacFile), "output initial row factor")
			    ("output-col-file", value<string>(&outputColFacFile), "output initial column factor")
				;

		positional_options_description pdesc;
		pdesc.add("input-file", 1);
		pdesc.add("input-test-file", 2);
		pdesc.add("input-row-file", 3);
		pdesc.add("input-col-file", 4);

		variables_map vm;
		store(command_line_parser(argc, argv).options(desc).positional(pdesc).run(), vm);
		notify(vm);

		if (vm.count("help") || vm.count("input-file")==0) {
			cout << "CSGD with NZL2  [options] <input-file> " << endl << endl;
			cout << desc << endl;
			return 1;
		}

		if (vm.count("output-row-file") == 0) { outputRowFacFile = ""; }
		if (vm.count("output-col-file") == 0) { outputColFacFile = ""; }

		LOG4CXX_INFO(logger, "Using " << tasks << " parallel tasks");

		///////////////////////////////////////

		Random32 random; // note: this takes a default seed (not randomized!)

// 		SparseMatrix v;
// 		readMatrix(inputMatrixFile, v, MM_COORD);

// 		SparseMatrix vTest;
// 		readMatrix(inputTestMatrixFile, vTest, MM_COORD);
//
// 		DenseMatrix w;
// 		readMatrix(inputRowFacFile, w, MM_ARRAY);
// 		DenseMatrixCM h;
// 		readMatrix(inputColFacFile, h, MM_ARRAY);

		Timer t;
		t.start();
		std::vector<DistributedSparseMatrix> dataVector=getDataMatrices<SparseMatrix>(inputMatrixFile,"V",true,
				tasks, 1, 1, 1, true, false, &inputTestMatrixFile);

		SparseMatrix& v = *mpi2::env().get<SparseMatrix>(dataVector[0].blocks()(0,0).var());
		SparseMatrix& vTest = *mpi2::env().get<SparseMatrix>(dataVector[1].blocks()(0,0).var());

		std::pair<DistributedDenseMatrix, DistributedDenseMatrixCM> factorsPair= getFactors(inputRowFacFile,
			inputColFacFile,  tasks, 1, 1, 1, true);
		DenseMatrix w = *mpi2::env().get<DenseMatrix>(factorsPair.first.blocks()(0,0).var());
		DenseMatrixCM h = *mpi2::env().get<DenseMatrixCM>(factorsPair.second.blocks()(0,0).var());
		t.stop();
		LOG4CXX_INFO(logger, "Total time for loading matrices: " << t);

		mf_size_type nnz = v.nnz();
		SparseMatrix::index_array_type& vIndex1 = rowIndexData(v);
		SparseMatrix::index_array_type& vIndex2 = columnIndexData(v);
		SparseMatrix::value_array_type& vValues = v.value_data();

		// global variables
		size1 = v.size1();
		size2 = v.size2();
		int b = calculateBlocking(size1, size2, nnz, rank, cache);
		b1 = b;
		b2 = b;

		LOG4CXX_INFO(logger, "Using a " << b << " x "<< b<<" stratification");

		minSize1 = floor(size1/b1);
		minSize2 = floor(size2/b2);
		remainder1 = size1 % b1;
		remainder2 = size2 % b2;

		LOG4CXX_INFO(logger, "Data matrix: "
			<< size1 << " x " << size2 << ", " << nnz << " nonzeros");
		LOG4CXX_INFO(logger, "minSize1: " << minSize1 << " remainder1: " << remainder1);
		LOG4CXX_INFO(logger, "minSize2: " << minSize2 << " remainder2: " << remainder2);

		// sort v
		LOG4CXX_INFO(logger, "Sorting v ... ");
		std::vector<TrainingPoint> points;
		for (mf_size_type i = 0; i < v.nnz(); i++) {
			TrainingPoint point = TrainingPoint(vIndex1[i], vIndex2[i], vValues[i]);
			points.push_back(point);
		}
		sort(points.begin(), points.end(), compFunction);

		// compute nnz per block
		LOG4CXX_INFO(logger, "Computing nnz per block ... ");
		std::vector<mf_size_type> nnzPerBlock;
		for (mf_size_type i = 0; i < (b1*b2); i++) nnzPerBlock.push_back(0);
		for (mf_size_type i = 0; i < points.size(); i++) {
			vIndex1[i] = points.at(i).i_;
			vIndex2[i] = points.at(i).j_;
			vValues[i] = points.at(i).x_;

			mf_size_type block_i = min ((mf_size_type)floor(points.at(i).i_/minSize1), b1 -1);
			mf_size_type block_j = min ((mf_size_type)floor(points.at(i).j_/minSize2), b2 -1);
			mf_size_type blockId = (b2 * block_i) + block_j;

			if ((block_i > b1) || (block_j > b2)) {
				LOG4CXX_INFO(logger, "block_i " << block_i << " = floor( " << points.at(i).i_ << " / " << minSize1);
				LOG4CXX_INFO(logger, "block_j " << block_j << " = floor( " << points.at(i).j_ << " / " << minSize2);
			}
			nnzPerBlock[blockId]++;
		}
		LOG4CXX_INFO(logger, "nnzPerBlock of size: " << nnzPerBlock.size());

		// compute permutation
		LOG4CXX_INFO(logger, "Computing permutation ... ");
		std::vector<mf_size_type> permutation;
		for (mf_size_type i = 0; i < v.nnz(); i++) {
			permutation.push_back(i);
		}

		// compute offsets
		LOG4CXX_INFO(logger, "Computing offsets ... ");
		std::vector<mf_size_type> offsets((b1*b2)+1); offsets.clear();
		mf_size_type offset = 0;
		offsets.push_back(offset);
		for (mf_size_type i = 1; i < (b1*b2)+1; i++) {
			offset += nnzPerBlock[i-1];
			offsets.push_back(offset);
		}

		Update update = Update(UpdateNzslNzl2(lambda), -100, 100); // truncate for numerical stability
		Regularize regularize;
		Loss loss((NzslLoss()), Nzl2Loss(lambda));
		TestLoss testLoss;

		FactorizationData<> testData(vTest, w, h);
		LOG4CXX_INFO(logger, "Initial test loss: " << testLoss(testData));

		// initialize

		StratifiedPsgdRunner stratifiedPsgdRunner (random, permutation, offsets);
		StratifiedPsgdJob<Update,Regularize> stratifiedPsgdJob(v, w, h, update, regularize, order, tasks);
		BoldDriver decay(eps0);
		Trace trace;

		t.start();
		stratifiedPsgdRunner.run(stratifiedPsgdJob, loss, epochs, decay, trace, balanceType, balanceMethod, &testData, &testLoss);
		t.stop();
		LOG4CXX_INFO(logger, "Total time: " << t);

		// write trace to an R file
		LOG4CXX_INFO(logger, "Writing trace to " << traceFile);
		trace.toRfile(traceFile, traceVar);

					// write computed factors to file
				if (outputRowFacFile.length() > 0) {
					LOG4CXX_INFO(logger, "Writing row factors to " << outputRowFacFile);
					//DenseMatrix w0;
					//unblock(dw, w0);
					writeMatrix(outputRowFacFile, w);
				}
				if (outputColFacFile.length() > 0) {
					LOG4CXX_INFO(logger, "Writing column factors to " << outputColFacFile);
					//DenseMatrixCM h0;
					//unblock(dh, h0);
					writeMatrix(outputColFacFile, h);
				}
		/**/
	}

	mfStop();
	mfFinalize();

	return 0;
}


#ifndef MPLLIBS_METAMONAD_V1_IMPL_REC_EQUAL_TO_IMPL_HPP
#define MPLLIBS_METAMONAD_V1_IMPL_REC_EQUAL_TO_IMPL_HPP

// Copyright Abel Sinkovics (abel@sinkovics.hu)  2013.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <mpllibs/metamonad/v1/impl/data_args.hpp>
#include <mpllibs/metamonad/limit_data_size.hpp>

#include <boost/preprocessor/repetition/repeat_from_to.hpp>
#include <boost/preprocessor/repetition/enum_params.hpp>

#include <boost/mpl/bool.hpp>
#include <boost/mpl/vector.hpp>
#include <boost/mpl/equal.hpp>
#include <boost/mpl/equal_to.hpp>

namespace mpllibs
{
  namespace metamonad
  {
    namespace v1
    {
      namespace impl
      {
        template <class A, class B>
        struct rec_equal_to_impl : boost::mpl::false_ {};

        #ifdef REC_EQUAL_TO_CASE
        #  error REC_EQUAL_TO_CASE already defined
        #endif
        #define REC_EQUAL_TO_CASE(z, n, unused) \
          template < \
            MPLLIBS_V1_DATA_ARGS(n) class T, \
            BOOST_PP_ENUM_PARAMS(n, class A), \
            BOOST_PP_ENUM_PARAMS(n, class B) \
          > \
          struct \
            rec_equal_to_impl< \
              T<BOOST_PP_ENUM_PARAMS(n, A)>, \
              T<BOOST_PP_ENUM_PARAMS(n, B)> \
            > : \
            boost::mpl::equal< \
              boost::mpl::vector<BOOST_PP_ENUM_PARAMS(n, A)>, \
              boost::mpl::vector<BOOST_PP_ENUM_PARAMS(n, B)>, \
              boost::mpl::equal_to<boost::mpl::_1, boost::mpl::_2> \
            > \
          {};

        BOOST_PP_REPEAT_FROM_TO(1, MPLLIBS_LIMIT_DATA_SIZE, REC_EQUAL_TO_CASE,~)

        #undef REC_EQUAL_TO_CASE
      }
    }
  }
}

#endif


#include <Eigen/Core>
#include <gtest/gtest.h>
#include <maplab-common/test/testing-entrypoint.h>
#include <vi-map/test/vi-map-generator.h>

#include "vi-map-helpers/vi-map-queries.h"

namespace vi_map_helpers {

class VertexClusteringTest : public ::testing::Test {
 private:
  static constexpr size_t kNumVertices = 7u;
  static constexpr size_t kNumLandmarks = 4u;

 protected:
  VertexClusteringTest() : map_(), map_queries_(map_), generator_(map_, 42) {}

  virtual void SetUp() {
    M_ = generator_.createMission(T_G_M_);

    for (size_t i = 0; i < kNumVertices; ++i) {
      V_[i] = generator_.createVertex(M_, T_G_V_[i]);
    }

    addLandmarksForObserverClusteringTest();

    generator_.generateMap();
  }

  void addLandmarksForObserverClusteringTest();

  vi_map::VIMap map_;
  VIMapQueries map_queries_;
  vi_map::VIMapGenerator generator_;
  vi_map::MissionId M_;
  pose_graph::VertexId V_[kNumVertices];
  pose::Transformation T_G_M_;
  pose::Transformation T_G_V_[kNumVertices];
  pose::Transformation T_G_L_[kNumLandmarks];

  // Landmark with one observing vertex.
  vi_map::LandmarkId one_observer_landmark_;

  // Landmark with single cluster.
  vi_map::LandmarkId one_observer_cluster_landmark_;

  // Landmark with two clusters.
  vi_map::LandmarkId two_observer_clusters_landmark_;

  // Landmark with three clusters.
  vi_map::LandmarkId three_observer_clusters_landmark_;

 public:
  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
};

void VertexClusteringTest::addLandmarksForObserverClusteringTest() {
  for (size_t i = 0; i < kNumLandmarks; ++i) {
    // Put the landmarks in front of the camera.
    T_G_L_[i].getPosition() << 0, 0, 1;
  }

  // Landmark with one observing vertex.
  one_observer_landmark_ =
      generator_.createLandmark(T_G_L_[0].getPosition(), V_[0], {});

  // Landmark with single cluster.
  one_observer_cluster_landmark_ =
      generator_.createLandmark(T_G_L_[1].getPosition(), V_[4], {V_[3], V_[5]});

  // Landmark with two clusters.
  two_observer_clusters_landmark_ = generator_.createLandmark(
      T_G_L_[2].getPosition(), V_[6], {V_[2], V_[1], V_[5], V_[4]});

  // Landmark with three clusters.
  three_observer_clusters_landmark_ = generator_.createLandmark(
      T_G_L_[3].getPosition(), V_[6], {V_[2], V_[0], V_[3], V_[4]});
}

TEST_F(VertexClusteringTest, NonexistentLandmark) {
  vi_map::LandmarkId nonexistent_landmark_id;
  common::generateId(&nonexistent_landmark_id);

  std::vector<pose_graph::VertexIdSet> clustered_vertices;
  EXPECT_DEATH(
      map_queries_.getConsecutiveLandmarkObserverGroupsFromMission(
          nonexistent_landmark_id, M_, &clustered_vertices),
      "Check");
}

TEST_F(VertexClusteringTest, OneObserverLandmark) {
  std::vector<pose_graph::VertexIdSet> clustered_vertices;
  map_queries_.getConsecutiveLandmarkObserverGroupsFromMission(
      one_observer_landmark_, M_, &clustered_vertices);

  ASSERT_EQ(1u, clustered_vertices.size());
  CHECK_EQ(clustered_vertices[0].count(V_[0]), 1u);
}

TEST_F(VertexClusteringTest, OneClusterLandmark) {
  std::vector<pose_graph::VertexIdSet> clustered_vertices;
  map_queries_.getConsecutiveLandmarkObserverGroupsFromMission(
      one_observer_cluster_landmark_, M_, &clustered_vertices);

  ASSERT_EQ(1u, clustered_vertices.size());
  CHECK_EQ(clustered_vertices[0].count(V_[3]), 1u);
  CHECK_EQ(clustered_vertices[0].count(V_[4]), 1u);
  CHECK_EQ(clustered_vertices[0].count(V_[5]), 1u);
}

TEST_F(VertexClusteringTest, TwoClustersLandmark) {
  std::vector<pose_graph::VertexIdSet> clustered_vertices;
  map_queries_.getConsecutiveLandmarkObserverGroupsFromMission(
      two_observer_clusters_landmark_, M_, &clustered_vertices);

  ASSERT_EQ(2u, clustered_vertices.size());

  CHECK_EQ(clustered_vertices[0].count(V_[1]), 1u);
  CHECK_EQ(clustered_vertices[0].count(V_[2]), 1u);
  CHECK_EQ(clustered_vertices[1].count(V_[4]), 1u);
  CHECK_EQ(clustered_vertices[1].count(V_[5]), 1u);
  CHECK_EQ(clustered_vertices[1].count(V_[6]), 1u);
}

TEST_F(VertexClusteringTest, ThreeClustersLandmark) {
  std::vector<pose_graph::VertexIdSet> clustered_vertices;
  map_queries_.getConsecutiveLandmarkObserverGroupsFromMission(
      three_observer_clusters_landmark_, M_, &clustered_vertices);

  ASSERT_EQ(3u, clustered_vertices.size());

  CHECK_EQ(clustered_vertices[0].count(V_[0]), 1u);
  CHECK_EQ(clustered_vertices[1].count(V_[2]), 1u);
  CHECK_EQ(clustered_vertices[1].count(V_[3]), 1u);
  CHECK_EQ(clustered_vertices[1].count(V_[4]), 1u);
  CHECK_EQ(clustered_vertices[2].count(V_[6]), 1u);
}

}  // namespace vi_map_helpers

MAPLAB_UNITTEST_ENTRYPOINT


/* buckets.cc
   Jeremy Barnes, 12 September 2011
   Copyright (c) 2011 Jeremy Barnes.  All rights reserved.
   This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

   Refactored out buckets code.
*/

#include "mldb/utils/buckets.h"
#include <iostream>
#include <boost/utility.hpp>
#include "mldb/arch/format.h"
#include "mldb/utils/floating_point.h"
#include "mldb/utils/pair_utils.h"
#include "mldb/arch/exception.h"

using namespace std;

namespace MLDB {

void
bucket_dist_full(vector<float> & result, const BucketFreqs & freqs)
{
    bool debug = false;

    result.clear();
    result.reserve(std::max<unsigned>(freqs.size() - 1, 0));

    /* We want the split points, which are half way in between the
       current value and the next one. */
    for (int i = 0;  i < (int)freqs.size() - 1;  ++i)
        result.push_back(((freqs.begin() + i)->first
                          + (freqs.begin() + i + 1)->first) * 0.5);

    if (debug) {
        for (BucketFreqs::const_iterator it = freqs.begin();  it != freqs.end();
             ++it)
            cerr << "freqs[" << it->first << "] = " << it->second << endl;

        for (unsigned i = 0;  i < result.size();  ++i)
            cerr << "  bucket " << i << " has value " << result[i]
                 << ", \t" << (freqs.begin() + i)->second << " values" << endl;
    }
}

void
bucket_dist_reduced(vector<float> & result,
                    const BucketFreqs & freqs,
                    size_t num_buckets)
{
    bool debug = false;
    //debug = (feature.type() == 22);

    /* NOTE: this algorithm tends to overcluster well-distributed data.
       It should probably be re-done to produce a number of buckets
       closer to the required number.
    */

    /* Find good split points. */
    double total = freqs.total();

    vector<float> bucket_sizes;
    result.clear();
    result.reserve(num_buckets);

    double num = 0;
    float per_bucket = total / num_buckets;

    /* Go through and look for the really big buckets.  We take these
       out of the bucket size calculation as otherwise we end up with
       too few buckets.  Note that this calculation is not exact
       (since the bucket size becomes smaller after), but it should
       fix up the most problematic cases.
    */
    int num_big_buckets = 0;
    double num_in_big_buckets = 0.0;
    for (BucketFreqs::const_iterator it = freqs.begin();
         it != freqs.end();  ++it) {
        float n = it->second;
        if (n >= per_bucket) {
            num_in_big_buckets += n;
            num_big_buckets += 1;
        }
    }

    if (num_buckets != num_big_buckets)
        per_bucket = (total - num_in_big_buckets) /
            (num_buckets - num_big_buckets);

    if (debug) {
        cerr << "  per_bucket before = "
             << total / num_buckets << endl;
        cerr << "  per_bucket = " << per_bucket << endl;
        cerr << "  num_buckets = " << num_buckets << endl;
        cerr << "  num_big_buckets = " << num_big_buckets << endl;
        cerr << "  num_in_big_buckets = " << num_in_big_buckets << endl;
    }

    //cerr << "freqs.size() = " << freqs.size() << endl;
    int i = 0;
    for (BucketFreqs::const_iterator it = freqs.begin();
         it != freqs.end();  ++it, ++i) {
        float n = it->second;
        float v = it->first;
        num += n;

        if (debug && false)
            cerr << "i = " << i << " n = " << n << " v = " << v
                 << " num = " << num << endl;

        /* We split if we have enough in our bucket, or if the next one
           would have enough by itself. */
        if (n >= per_bucket && num > n) {
            /* Extra split, since this one had enough by itself. */
            float val = (v + boost::prior(it)->first) / 2.0;
            result.push_back(val);
            bucket_sizes.push_back(num - n);
            if (debug)
                cerr << "i = " << i << ": split [1] at "
                     << format("val: %16.9f 0x%08x ", val,
                               reinterpret_as_int(val))
                     << format("v: %16.9f 0x%08x ", v,
                               reinterpret_as_int(v))
                     << format("prior: %16.9f 0x%08x ", boost::prior(it)->first,
                               reinterpret_as_int(boost::prior(it)->first))
                     << " with " << num - n
                     << " examples." << endl;
            num -= n;
        }
        if (i < (int)freqs.size() - 1
            && (num >= per_bucket || boost::next(it)->second >= per_bucket)) {
            float val = (v + boost::next(it)->first) * 0.5f;

            // If the two values are one ulp apart, then we need to make sure
            // that it gets rounded up otherwise we will have two buckets
            // with the same split value
            if (val == v)
                val = boost::next(it)->first;

            result.push_back(val);
            bucket_sizes.push_back(num);
            if (debug)
                cerr << "i = " << i << ": split [2] at "
                     << format("val: %16.9f 0x%08x ", val,
                               reinterpret_as_int(val))
                     << format("v: %16.9f 0x%08x ", v,
                               reinterpret_as_int(v))
                     << format("next: %16.9f 0x%08x ", boost::next(it)->first,
                               reinterpret_as_int(boost::next(it)->first))
                     << " with " << num
                     << " examples." << endl;
            num = 0;
        }
    }

    for (unsigned i = 1;  i < result.size();  ++i)
        if (result[i] == result[i - 1])
            throw Exception("two buckets with the same split point");

    if (debug) {
        for (unsigned i = 0;  i < result.size();  ++i)
            cerr << format("  bucket %5d has value %16.9f 0x%08x, %8.1f values",
                           i, result[i], reinterpret_as_int(result[i]),
                           bucket_sizes[i]) << endl;
    }
}

#if 0
const Dataset_Index::Index_Entry::Bucket_Info &
Dataset_Index::Index_Entry::
create_buckets(size_t num_buckets)
{
    check_used();
    Guard guard(lock);

    //cerr << "create_buckets(" << num_buckets << ")" << endl;

}
#endif

void get_freqs(BucketFreqs & result, std::vector<float> values)
{
    std::sort(values.begin(), values.end(), MLDB::safe_less<float>());

    vector<pair<float, float> > freqs2;
    freqs2.reserve(values.size());

    float last = -INFINITY;
    int count = 0;

    for (unsigned i = 0;  i < values.size();  ++i) {
        if (isnan(values[i]))
            throw Exception("NaN in values");

        if (i == 0) {
            last = values[i];
            count += 1;
        }
        else if (bit_equal(last, values[i]))
            count += 1;
        else {
            freqs2.push_back(make_pair(last, count));
            count = 1;
            last = values[i];
        }
    }
    freqs2.push_back(make_pair(last, count));

    std::sort(freqs2.begin(), freqs2.end());

    result = BucketFreqs(freqs2.begin(), freqs2.end());
} // namespace MLDB

void bucket_dist(std::vector<float> & result,
                 const BucketFreqs & freqs,
                 size_t num_buckets)
{
    if (num_buckets >= freqs.size())
        bucket_dist_full(result, freqs);
    else
        bucket_dist_reduced(result, freqs, num_buckets);
}

Bucket_Info create_buckets(const std::vector<float> & values,
                           size_t num_buckets)
{
    BucketFreqs freqs;
    get_freqs(freqs, values);

    Bucket_Info result;
    bucket_dist(result.splits, freqs, num_buckets);

    vector<int> bucket_count(result.splits.size() + 1);

    for (unsigned i = 0;  i < values.size();  ++i) {
        float value = values[i];
        int bucket = std::upper_bound(result.splits.begin(),
                                      result.splits.end(), value)
            - result.splits.begin();

        bucket_count[bucket] += 1;

        result.buckets.push_back(bucket);
    }

    return result;
}

} // namespace MLDB


//==============================================================================
//         Copyright 2003 - 2013   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2013   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#include <nt2/include/functions/rsqrt.hpp>

#include <boost/dispatch/functor/meta/call.hpp>
#include <nt2/sdk/functor/meta/call.hpp>
#include <nt2/sdk/unit/tests/relation.hpp>
#include <nt2/sdk/unit/tests/type_expr.hpp>
#include <complex>
#include <nt2/sdk/complex/complex.hpp>
#include <nt2/sdk/unit/tests/ulp.hpp>
#include <nt2/sdk/unit/tests/basic.hpp>
#include <nt2/sdk/meta/as_integer.hpp>
#include <nt2/sdk/unit/module.hpp>
#include <boost/simd/sdk/config.hpp>

#include <nt2/include/constants/mone.hpp>
#include <nt2/include/constants/one.hpp>
#include <nt2/include/constants/sqrt_2.hpp>
#include <nt2/include/constants/sqrt_2o_2.hpp>
#include <nt2/include/constants/zero.hpp>
#include <nt2/include/constants/inf.hpp>
#include <nt2/include/constants/minf.hpp>
#include <nt2/include/constants/nan.hpp>


#include <moveit/kinematic_constraints/kinematic_constraint.h>
#include <gtest/gtest.h>
#include <urdf_parser/urdf_parser.h>
#include <tf2_eigen/tf2_eigen.h>
#include <moveit/utils/robot_model_test_utils.h>
#include <boost/math/constants/constants.hpp>

class SphericalRobot : public testing::Test
{
protected:
  void SetUp() override
  {
    moveit::core::RobotModelBuilder builder("robot", "base_link");
    geometry_msgs::Pose origin;
    origin.orientation.w = 1;
    builder.addChain("base_link->roll", "revolute", { origin }, urdf::Vector3(1, 0, 0));
    builder.addChain("roll->pitch", "revolute", { origin }, urdf::Vector3(0, 1, 0));
    builder.addChain("pitch->yaw", "revolute", { origin }, urdf::Vector3(0, 0, 1));
    ASSERT_TRUE(builder.isValid());
    robot_model_ = builder.build();
  }

  std::map<std::string, double> getJointValues(const double roll, const double pitch, const double yaw)
  {
    std::map<std::string, double> jvals;
    jvals["base_link-roll-joint"] = roll;
    jvals["roll-pitch-joint"] = pitch;
    jvals["pitch-yaw-joint"] = yaw;
    return jvals;
  }

  void TearDown() override
  {
  }

protected:
  moveit::core::RobotModelPtr robot_model_;
};

TEST_F(SphericalRobot, Test1)
{
  kinematic_constraints::OrientationConstraint oc(robot_model_);

  moveit::core::Transforms tf(robot_model_->getModelFrame());
  moveit_msgs::OrientationConstraint ocm;

  ocm.link_name = "yaw";
  ocm.header.frame_id = robot_model_->getModelFrame();
  ocm.orientation.x = 0.0;
  ocm.orientation.y = 0.0;
  ocm.orientation.z = 0.0;
  ocm.orientation.w = 1.0;
  ocm.absolute_x_axis_tolerance = 0.8;
  ocm.absolute_y_axis_tolerance = 0.8;
  ocm.absolute_z_axis_tolerance = 3.15;
  ocm.weight = 1.0;

  moveit::core::RobotState robot_state(robot_model_);
  // This's identical to a -1.57rad rotation around Z-axis
  robot_state.setVariablePositions(getJointValues(3.140208, 3.141588, 1.570821));
  robot_state.update();

  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_TRUE(oc.decide(robot_state).satisfied);
}

TEST_F(SphericalRobot, Test2)
{
  kinematic_constraints::OrientationConstraint oc(robot_model_);

  moveit::core::Transforms tf(robot_model_->getModelFrame());
  moveit_msgs::OrientationConstraint ocm;

  ocm.link_name = "yaw";
  ocm.header.frame_id = robot_model_->getModelFrame();
  ocm.orientation.x = 0.0;
  ocm.orientation.y = 0.0;
  ocm.orientation.z = 0.0;
  ocm.orientation.w = 1.0;
  ocm.absolute_x_axis_tolerance = 0.2;
  ocm.absolute_y_axis_tolerance = 2.0;
  ocm.absolute_z_axis_tolerance = 0.2;
  ocm.weight = 1.0;

  moveit::core::RobotState robot_state(robot_model_);
  // Singularity: roll + yaw = theta
  // These violate either absolute_x_axis_tolerance or absolute_z_axis_tolerance
  robot_state.setVariablePositions(getJointValues(0.15, boost::math::constants::half_pi<double>(), 0.15));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.21, boost::math::constants::half_pi<double>(), 0.0));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.0, boost::math::constants::half_pi<double>(), 0.21));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  // Singularity: roll - yaw = theta
  // This's identical to -pi/2 pitch rotation
  robot_state.setVariablePositions(getJointValues(0.15, -boost::math::constants::half_pi<double>(), 0.15));
  robot_state.update();

  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_TRUE(oc.decide(robot_state).satisfied);
}

TEST_F(SphericalRobot, Test3)
{
  kinematic_constraints::OrientationConstraint oc(robot_model_);

  moveit::core::Transforms tf(robot_model_->getModelFrame());
  moveit_msgs::OrientationConstraint ocm;

  ocm.link_name = "yaw";
  ocm.header.frame_id = robot_model_->getModelFrame();
  ocm.orientation.x = 0.0;
  ocm.orientation.y = 0.0;
  ocm.orientation.z = 0.0;
  ocm.orientation.w = 1.0;
  ocm.absolute_x_axis_tolerance = 0.2;
  ocm.absolute_y_axis_tolerance = 2.0;
  ocm.absolute_z_axis_tolerance = 0.3;
  ocm.weight = 1.0;

  moveit::core::RobotState robot_state(robot_model_);
  // Singularity: roll + yaw = theta

  // These tests violate absolute_x_axis_tolerance
  robot_state.setVariablePositions(getJointValues(0.21, boost::math::constants::half_pi<double>(), 0.0));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.0, boost::math::constants::half_pi<double>(), 0.21));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  ocm.absolute_x_axis_tolerance = 0.3;
  ocm.absolute_y_axis_tolerance = 2.0;
  ocm.absolute_z_axis_tolerance = 0.2;

  // These tests violate absolute_z_axis_tolerance
  robot_state.setVariablePositions(getJointValues(0.21, boost::math::constants::half_pi<double>(), 0.0));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.0, boost::math::constants::half_pi<double>(), 0.21));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);
}

TEST_F(SphericalRobot, Test4)
{
  kinematic_constraints::OrientationConstraint oc(robot_model_);

  moveit::core::Transforms tf(robot_model_->getModelFrame());
  moveit_msgs::OrientationConstraint ocm;

  ocm.link_name = "yaw";
  ocm.header.frame_id = robot_model_->getModelFrame();
  ocm.orientation.x = 0.0;
  ocm.orientation.y = 0.0;
  ocm.orientation.z = 0.0;
  ocm.orientation.w = 1.0;
  ocm.absolute_x_axis_tolerance = 0.2;
  ocm.absolute_y_axis_tolerance = 2.0;
  ocm.absolute_z_axis_tolerance = 0.3;
  ocm.weight = 1.0;

  moveit::core::RobotState robot_state(robot_model_);
  // Singularity: roll + yaw = theta

  // These tests violate absolute_x_axis_tolerance
  robot_state.setVariablePositions(getJointValues(0.21, -boost::math::constants::half_pi<double>(), 0.0));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.0, -boost::math::constants::half_pi<double>(), 0.21));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  ocm.absolute_x_axis_tolerance = 0.3;
  ocm.absolute_y_axis_tolerance = 2.0;
  ocm.absolute_z_axis_tolerance = 0.2;

  // These tests violate absolute_z_axis_tolerance
  robot_state.setVariablePositions(getJointValues(0.21, -boost::math::constants::half_pi<double>(), 0.0));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.0, -boost::math::constants::half_pi<double>(), 0.21));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);

  robot_state.setVariablePositions(getJointValues(0.5, -boost::math::constants::half_pi<double>(), 0.21));
  robot_state.update();
  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_FALSE(oc.decide(robot_state).satisfied);
}

// Both the current and the desired orientations are in singularities
TEST_F(SphericalRobot, Test5)
{
  kinematic_constraints::OrientationConstraint oc(robot_model_);

  moveit::core::Transforms tf(robot_model_->getModelFrame());
  moveit_msgs::OrientationConstraint ocm;

  moveit::core::RobotState robot_state(robot_model_);
  robot_state.setVariablePositions(getJointValues(0.0, boost::math::constants::half_pi<double>(), 0.0));
  robot_state.update();

  ocm.link_name = "yaw";
  ocm.header.frame_id = robot_model_->getModelFrame();
  tf2::convert(Eigen::Quaterniond(robot_state.getGlobalLinkTransform(ocm.link_name).linear()), ocm.orientation);
  ocm.absolute_x_axis_tolerance = 0.0;
  ocm.absolute_y_axis_tolerance = 0.0;
  ocm.absolute_z_axis_tolerance = 1.0;
  ocm.weight = 1.0;

  robot_state.setVariablePositions(getJointValues(0.2, boost::math::constants::half_pi<double>(), 0.3));
  robot_state.update();

  EXPECT_TRUE(oc.configure(ocm, tf));
  EXPECT_TRUE(oc.decide(robot_state, true).satisfied);
}

int main(int argc, char** argv)
{
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}


//==================================================================================================
/*!

  Copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#include <boost/simd/constant/log10_elo.hpp>
#include <boost/simd/constant/log10_ehi.hpp>
#include <boost/simd/constant/eps.hpp>
#include <boost/simd/function/abs.hpp>
#include <boost/simd/as.hpp>
#include <simd_test.hpp>

STF_CASE_TPL( "Check log10_elo behavior for integral types"
            , (std::uint32_t)(std::uint64_t)
              (std::int32_t)(std::int64_t)
            )
{
  using boost::simd::as;
  using boost::simd::detail::log10_elo;
  using boost::simd::Log10_elo;
  T ref = T(0);
  STF_TYPE_IS(decltype(Log10_elo<T>()), T);
  STF_EQUAL(Log10_elo<T>(), ref);
  STF_EQUAL(log10_elo( as(T{}) ),ref);
}

STF_CASE_TPL( "Check log10_elo behavior for double"
            , (double)
            )
{
  using boost::simd::as;
  using boost::simd::detail::log10_elo;
  using boost::simd::Log10_elo;
  T ref = T(0);

  STF_TYPE_IS(decltype(Log10_elo<T>()), T);
  STF_IEEE_EQUAL(Log10_elo<T>(), ref);
  STF_IEEE_EQUAL(log10_elo( as(T{}) ), ref);
}

STF_CASE_TPL( "Check log10_elo behavior for float"
            , (float)
            )
{
  using boost::simd::as;
  using boost::simd::detail::log10_elo;
  using boost::simd::Log10_elo;
  using boost::simd::Log10_ehi;
  using boost::simd::Eps;
  using boost::simd::abs;
  double Log10_e =  (0.434294481903251827651128918917);
  STF_LESS(abs(float(double(Log10_elo<float>())+double(Log10_ehi<float>())- Log10_e)), Eps<float>());
  T ref = T(7.0073188e-04);
  STF_TYPE_IS(decltype(Log10_elo<T>()), T);
  STF_IEEE_EQUAL(Log10_elo<T>(), ref);
  STF_IEEE_EQUAL(log10_elo( as(T{}) ), ref);
}


/*
 * MnisTest.cpp
 *
 */
#include <algorithm>
#include <exception>
#include <fstream>
#include <stdexcept>
#include <vector>

#include <boost/assert.hpp>
#include <boost/test/unit_test.hpp>

#include "core/nn.h"
#include "core/training.h"
#include "core/utils.h"

#include "timer.h"
#include "test_utils.h"
#include "mnist_test.h"

using namespace std;
using namespace yann;
using namespace yann::test;

#define LABELS_COUNT      10    // digits 0...9
#define IMAGES_MAX_VALUE  256   // colors 0...256

#define LABELS_MAGIC_NUMBER 2049
#define IMAGES_MAGIC_NUMBER 2051

#define TRAINING_IMAGES_FILENAME "train-images-idx3-ubyte"
#define TRAINING_LABELS_FILENAME "train-labels-idx1-ubyte"
#define TESTING_IMAGES_FILENAME  "t10k-images-idx3-ubyte"
#define TESTING_LABELS_FILENAME  "t10k-labels-idx1-ubyte"

namespace yann::test {

  inline static unsigned int make_uint(Byte buf[4]) {
    return (((unsigned int) buf[0]) << 24 | ((unsigned int) buf[1]) << 16
        | ((unsigned int) buf[2]) << 8 | ((unsigned int) buf[3]) << 0);
  }

  template<typename T>
  inline static MatrixSize get_label(const T & vv) {
    MatrixSize pos = 0;
    vv.maxCoeff(&pos);
    YANN_CHECK_LE(pos, LABELS_COUNT);
    return pos;
  }
};// namespace yann::test

////////////////////////////////////////////////////////////////////////////////////////////////
//
// MnistDataset implementation
//
yann::test::MnistDataset::MnistDataset() :
    _image_rows(0),
    _image_cols(0)
{
}

yann::test::MnistDataset::~MnistDataset()
{
}

MatrixSize  yann::test::MnistDataset::get_size() const
{
  YANN_CHECK_EQ(get_batch_size(_images), get_batch_size(_labels));
    return get_batch_size(_images);
}

MatrixSize yann::test::MnistDataset::get_image_size() const
{
  YANN_CHECK_EQ(get_batch_item_size(_images), _image_rows * _image_cols);
  return get_batch_item_size(_images);
}

MatrixSize yann::test::MnistDataset::get_label_size() const
{
  return get_batch_item_size(_labels);
}

/*********************************************************
 *
 * TRAINING SET IMAGE FILE (train-images-idx3-ubyte):
 * [offset] [type]          [value]          [description]
 * 0000     32 bit integer  0x00000803(2051) magic number
 * 0004     32 bit integer  60000            number of images
 * 0008     32 bit integer  28               number of rows
 * 0012     32 bit integer  28               number of columns
 * 0016     unsigned byte   ??               pixel
 * 0017     unsigned byte   ??               pixel
 * ........
 * xxxx     unsigned byte   ??               pixel
 *
 * Pixels are organized row-wise. Pixel values are 0 to 255.
 * 0 means background (white), 255 means foreground (black).
 *
 *********************************************************/
void yann::test::MnistDataset::read_images(istream &in)
{
  unsigned int magic_number, images_count;
  Byte tmp4[4];
  vector<Byte> image;

  // magic number
  in.read((char*) tmp4, sizeof(tmp4));
  if (!in || in.eof()) {
    throw invalid_argument(
        "invalid file format: can't read images magic number");
  }
  magic_number = make_uint(tmp4);
  if (magic_number != IMAGES_MAGIC_NUMBER) {
    throw invalid_argument("invalid file format: invalid images magic number");
  }

  // images count
  in.read((char*) tmp4, sizeof(tmp4));
  if (!in || in.eof()) {
    throw invalid_argument("invalid file format: can't read images count");
  }
  images_count = make_uint(tmp4);

  // rows count
  in.read((char*) tmp4, sizeof(tmp4));
  if (!in || in.eof()) {
    throw invalid_argument("invalid file format: can't read rows count");
  }
  _image_rows = make_uint(tmp4);

  // columns count
  in.read((char*) tmp4, sizeof(tmp4));
  if (!in || in.eof()) {
    throw invalid_argument("invalid file format: can't read columns count");
  }
  _image_cols = make_uint(tmp4);

  // images
  image.resize(_image_rows * _image_cols);
  resize_batch(_images, images_count, image.size());
  _images.setZero();
  for (size_t ii = 0; ii < images_count; ++ii) {
    in.read((char*) &image.front(), image.size());
    if (!in || in.eof()) {
      throw invalid_argument("invalid file format: can't read image");
    }

    // transform colors 0...256 into -1.0 ... 1.0 signals
    for(size_t jj = 0; jj < image.size(); ++jj) {
      get_batch(_images, ii)(jj) = ((double)image[jj]) / IMAGES_MAX_VALUE;
    }
  }
}

/*********************************************************
 *
 * TRAINING SET LABEL FILE (train-labels-idx1-ubyte):
 * [offset] [type]          [value]          [description]
 * 0000     32 bit integer  0x00000801(2049) magic number (MSB first)
 * 0004     32 bit integer  60000            number of items
 * 0008     unsigned byte   ??               label
 * 0009     unsigned byte   ??               label
 * ........
 * xxxx     unsigned byte   ??               label
 *
 * The labels values are 0 to 9.
 *
 *********************************************************/
void yann::test::MnistDataset::read_labels(istream &in)
{
  unsigned int magic_number, labels_count;
  Byte ll, tmp4[4];

  // magic number
  in.read((char*) tmp4, sizeof(tmp4));
  if (!in || in.eof()) {
    throw invalid_argument(
        "invalid file format: can't read labels magic number");
  }
  magic_number = make_uint(tmp4);
  if (magic_number != LABELS_MAGIC_NUMBER) {
    throw invalid_argument("invalid file format: invalid labels magic number");
  }

  // labels count
  in.read((char*) tmp4, sizeof(tmp4));
  if (!in || in.eof()) {
    throw invalid_argument("invalid file format: can't read labels count");
  }
  labels_count = make_uint(tmp4);

  // labels
  resize_batch(_labels, labels_count, LABELS_COUNT);
  _labels.setZero();
  for (size_t ii = 0; ii < labels_count; ++ii) {
    in.read((char*) &ll, sizeof(ll));
    if (!in || in.eof()) {
      throw invalid_argument("invalid file format: can't read label");
    }
    if (ll >= LABELS_COUNT) {
      throw invalid_argument("invalid file format: label should be 0...9");
    }

    // transform label value into the vector
    get_batch(_labels, ii)(ll) = 1.0;
  }
}

void yann::test::MnistDataset::filter(const MatrixSize & max_label, const MatrixSize & max_num)
{
  YANN_CHECK_LE(max_label, LABELS_COUNT);

  // find how many labels <= max_label are there
  MatrixSize count = 0;
  for(MatrixSize ii = 0; ii < get_batch_size(_labels); ++ii) {
    if(get_label(get_batch(_labels, ii)) <= max_label) {
      ++count;
    }
  }
  if(max_num > 0 && count > max_num) {
    count = max_num;
  }

  // prep storage and copy
  VectorBatch images, labels;
  resize_batch(images, count, get_image_size());
  resize_batch(labels, count, max_label + 1);
  images.setZero();
  labels.setZero();
  for(MatrixSize ii = 0, jj = 0; ii < get_batch_size(_labels) && jj < count; ++ii) {
    MatrixSize label = get_label(get_batch(_labels, ii));
    if(label <= max_label) {
      get_batch(images, jj) = get_batch(_images, ii);
      get_batch(labels, jj)(label) = 1.0;
      ++jj;
    }
  }

  // ready to go
  swap(images, _images);
  swap(labels, _labels);
}

void yann::test::MnistDataset::shift_values(const Value & min_val, const Value & max_val)
{
  YANN_CHECK_LT(min_val, max_val);
  _images.array() = _images.array() * (max_val - min_val) + min_val;
}

////////////////////////////////////////////////////////////////////////////////////////////////
//
// MnistTest implementation
//

// Overwrites from std:: namespace
ostream& std::operator<<(ostream & os, const MnistTest & test)
{
  os << test.training().get_size() << " training labels/images" << endl;
  os << test.testing().get_size() << " test labels/images" << endl;
  return os;
}

yann::test::MnistTest::MnistTest()
{
}

yann::test::MnistTest::~MnistTest()
{
}

void yann::test::MnistTest::read(const string & base_folder)
{
  ifstream ifs;

  // training
  ifs.open(base_folder + "/" + TRAINING_LABELS_FILENAME, ios::in | ios::binary);
  _training.read_labels(ifs);
  ifs.close();

  ifs.open(base_folder + "/" + TRAINING_IMAGES_FILENAME, ios::in | ios::binary);
  _training.read_images(ifs);
  ifs.close();

  // testing
  ifs.open(base_folder + "/" + TESTING_LABELS_FILENAME, ios::in | ios::binary);
  _testing.read_labels(ifs);
  ifs.close();

  ifs.open(base_folder + "/" + TESTING_IMAGES_FILENAME, ios::in | ios::binary);
  _testing.read_images(ifs);
  ifs.close();

  // validation
  YANN_CHECK_EQ(get_batch_size(_training.labels()), get_batch_size(_training.images()));
  YANN_CHECK_EQ(get_batch_size(_testing.labels()), get_batch_size(_testing.images()));
  YANN_CHECK_EQ(_training.get_image_size(), _testing.get_image_size());
  YANN_CHECK_EQ(_training.get_label_size(), _testing.get_label_size());
}

pair<size_t, Value> yann::test::MnistTest::test(const Network & nn,
                                                const RefConstVectorBatch & images,
                                                const RefConstVectorBatch & labels,
                                                Context * ctx)
{
  YANN_CHECK(ctx);
  YANN_CHECK_GT(get_batch_size(images), 0);
  YANN_CHECK_EQ(get_batch_size(images), get_batch_size(labels));

  nn.calculate(images, ctx);
  RefConstVectorBatch output = ctx->get_output();

  YANN_CHECK(is_same_size(output, labels));
  Value cost = nn.cost(output, labels);

  size_t success_count = 0;
  const MatrixSize batch_size = get_batch_size(images);
  for(MatrixSize ii = 0; ii < batch_size; ++ii) {
    MatrixSize actual = get_label(get_batch(output, ii));
    MatrixSize expected = get_label(get_batch(labels, ii));
    if (expected == actual) {
      ++success_count;
    }
  }

  // done
  return make_pair(success_count, cost);
}

pair<double, Value> yann::test::MnistTest::test(
    const Network & nn,
    const MnistDataset & dataset,
    const MatrixSize & test_batch_size)
{
  YANN_CHECK_GT(test_batch_size, 0);
  YANN_CHECK_GE(get_batch_size(dataset.images()), test_batch_size);
  YANN_CHECK_EQ(get_batch_size(dataset.images()), get_batch_size(dataset.labels()));

  const MatrixSize size = get_batch_size(dataset.images());
  const MatrixSize image_size = get_batch_item_size(dataset.images());
  const MatrixSize label_size = get_batch_item_size(dataset.labels());

  unique_ptr<Context> ctx = nn.create_context(test_batch_size);
  YANN_CHECK(ctx);

  pair<double, Value> result = { 0, 0 };

  // ensure we don't do allocations in eigen
  {
    BlockAllocations block;

    MatrixSize pos = 0;
    for(; pos <= size - test_batch_size; pos += test_batch_size) {
      pair<size_t, Value> res = test(
          nn,
          dataset.images().block(pos, 0, test_batch_size, image_size), // assumes RowMajor layout
          dataset.labels().block(pos, 0, test_batch_size, label_size), // assumes RowMajor layout
          ctx.get()
      );
      result.first += res.first;
      result.second += res.second;
    }

    // normalize results
    result.first /= pos;
    result.second /= pos;
  }

  // done
  return result;
}

pair<double, Value> yann::test::MnistTest::train_and_test(
    Network & nn,
    Trainer & trainer,
    DataSource_Stochastic::Mode mode,
    const MatrixSize & training_batch_size,
    const size_t & epochs,
    const MatrixSize & test_batch_size)
{
  YANN_CHECK_GT(epochs, 0);
  YANN_CHECK_EQ(get_batch_size(_training.labels()), get_batch_size(_training.images()));
  YANN_CHECK_EQ(get_batch_size(_testing.labels()), get_batch_size(_testing.images()));

  // training
  double test_success_rate = 0, training_success_rate = 0;
  Value test_cost = 0, training_cost = 0;
  auto epochs_callback = [&](const MatrixSize & epoch, const MatrixSize & epochs, const std::string & message) {
    {
      Timer timer("Testing against training dataset");
      pair<double, Value> res = test(nn, _training, test_batch_size);
      training_success_rate = res.first;
      training_cost = res.second;
      BOOST_TEST_MESSAGE(timer);
    }
    {
      Timer timer("Testing against test dataset");
      pair<double, Value> res = test(nn, _testing, test_batch_size);
      test_success_rate = res.first;
      test_cost = res.second;
      BOOST_TEST_MESSAGE(timer);
    }

    string extra = (epoch <= 0) ? " (before training)" : "";
    BOOST_TEST_MESSAGE(
        "Success rate for epoch " << epoch << extra << ":"
            << " against training dataset: " << training_success_rate * 100 << "%"
            << " against test dataset: " << test_success_rate* 100 << "%"
    );
    BOOST_TEST_MESSAGE(
        "Cost per test for epoch " << epoch << extra << ":"
            << " against training dataset: " << training_cost
            << " against test dataset: " << test_cost
    );

    BOOST_TEST_MESSAGE("\n");

    if(epoch + 1 < epochs) {
      if(!message.empty()) {
        BOOST_TEST_MESSAGE("*** Training epoch " << (epoch + 1) << " out of " << epochs << " (" << message << ")");
      } else {
        BOOST_TEST_MESSAGE("*** Training epoch " << (epoch + 1) << " out of " << epochs);
      }
    }
  };

  // test ones before training
  BOOST_TEST_MESSAGE("\n");
  BOOST_TEST_MESSAGE("*** Testing before training");
  epochs_callback(0, epochs, "");

  // train
  {
    Timer timer("Total training/testing");
    DataSource_Stochastic data_source(
                _training.images(),
                _training.labels(),
                mode,
                training_batch_size);
    trainer.set_epochs_progress_callback(epochs_callback);
    trainer.train(nn, data_source, training_batch_size, epochs);

    BOOST_TEST_MESSAGE(timer << "\n");
  }

  // done
  return make_pair(test_success_rate, test_cost);
}

MatrixSize yann::test::MnistTest::get_training_size() const
{
  return _training.get_size();
}

MatrixSize yann::test::MnistTest::get_testing_size() const
{
  return _testing.get_size();
}

MatrixSize yann::test::MnistTest::get_image_rows() const
{
  YANN_CHECK_EQ(_training.get_image_rows(), _testing.get_image_rows());
  return _training.get_image_rows();
}
MatrixSize yann::test::MnistTest::get_image_cols() const
{
  YANN_CHECK_EQ(_training.get_image_cols(), _testing.get_image_cols());
  return _training.get_image_cols();
}
MatrixSize yann::test::MnistTest::get_image_size() const
{
  YANN_CHECK_EQ(_training.get_image_size(), _testing.get_image_size());
  return _training.get_image_size();
}

MatrixSize yann::test::MnistTest::get_label_size() const
{
  YANN_CHECK_EQ(_training.get_label_size(), _testing.get_label_size());
  return _training.get_label_size();
}

vector<MatrixSize> yann::test::MnistTest::get_layer_sizes(
    const vector<MatrixSize> & hidden_layer_sizes) const
{
  // add input / output layers based on mnist data
  vector<MatrixSize> layer_sizes;
  layer_sizes.reserve(hidden_layer_sizes.size() + 2);

  layer_sizes.push_back(get_image_size());
  layer_sizes.insert(layer_sizes.end(), hidden_layer_sizes.begin(),
                     hidden_layer_sizes.end());
  layer_sizes.push_back(get_label_size());
  return layer_sizes;
}

void yann::test::MnistTest::filter(
    const MatrixSize & max_label,
    const MatrixSize & training_max_num,
    const MatrixSize & testing_max_num)
{
  _training.filter(max_label, training_max_num);
  _testing.filter(max_label, testing_max_num);
}

void yann::test::MnistTest::shift_values(
    const Value & min_val,
    const Value & max_val)
{
  _training.shift_values(min_val, max_val);
  _testing.shift_values(min_val, max_val);
}


// ==============================================================================
//  KratosTopologyOptimizationApplication
//
//  License:         BSD License
//                   license: TopologyOptimizationApplication/license.txt
//
//  Main authors:    Baumgärtner Daniel, https://github.com/dbaumgaertner
//                   Octaviano Malfavón Farías
//                   Eric Gonzales
//
// ==============================================================================

// External includes
#include <boost/python.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
#include <boost/timer.hpp>

// Project includes
#include "includes/define.h"
#include "containers/flags.h"
#include "custom_python/add_custom_strategies_to_python.h"
#include "spaces/ublas_space.h"

// Schemes
#include "custom_strategies/structure_adjoint_sensitivity_strategy.h"
#include "custom_strategies/custom_schemes/residualbased_incrementalupdate_static_simp_scheme.hpp"

// Linear solvers
#include "linear_solvers/linear_solver.h"

namespace Kratos
{

namespace Python
{
using namespace boost::python;

void  AddCustomStrategiesToPython()
{
	typedef UblasSpace<double, CompressedMatrix, Vector> SparseSpaceType;
	typedef UblasSpace<double, Matrix, Vector> LocalSpaceType;

	//base types
	typedef LinearSolver<SparseSpaceType, LocalSpaceType > LinearSolverType;
	typedef SolvingStrategy< SparseSpaceType, LocalSpaceType, LinearSolverType > BaseSolvingStrategyType;
	typedef Scheme< SparseSpaceType, LocalSpaceType > BaseSchemeType;

	//custom scheme types
	typedef ResidualBasedIncrementalUpdateStaticSIMPScheme< SparseSpaceType, LocalSpaceType > ResidualBasedIncrementalUpdateStaticSIMPSchemeType;

	// =============================================================================================================================================
	// Scheme Classes
	// =============================================================================================================================================

	// Static TIMP Scheme Type
	class_< ResidualBasedIncrementalUpdateStaticSIMPSchemeType,bases< BaseSchemeType >, boost::noncopyable >
	("ResidualBasedIncrementalUpdateStaticSIMPScheme", init< >())
	.def("Initialize", &ResidualBasedIncrementalUpdateStaticSIMPScheme<SparseSpaceType, LocalSpaceType>::Initialize)
	;

	// =============================================================================================================================================
	// Strategy Classes
	// =============================================================================================================================================

	class_< StructureAdjointSensitivityStrategy< SparseSpaceType, LocalSpaceType, LinearSolverType >,bases< BaseSolvingStrategyType >,  boost::noncopyable >
	("StructureAdjointSensitivityStrategy",init<ModelPart&, LinearSolverType::Pointer, int>())
	.def("ComputeStrainEnergySensitivities",&StructureAdjointSensitivityStrategy< SparseSpaceType, LocalSpaceType, LinearSolverType >::ComputeStrainEnergySensitivities)
	.def("ComputeVolumeFractionSensitivities",&StructureAdjointSensitivityStrategy< SparseSpaceType, LocalSpaceType, LinearSolverType >::ComputeVolumeFractionSensitivities)
	;
}

}  // namespace Python.

} // Namespace Kratos


#ifndef BOOST_QVM_GEN_MAT_ASSIGN2_HPP_INCLUDED
#define BOOST_QVM_GEN_MAT_ASSIGN2_HPP_INCLUDED

/// Copyright (c) 2008-2021 Emil Dotchevski and Reverge Studios, Inc.

/// Distributed under the Boost Software License, Version 1.0. (See accompanying
/// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

/// This file was generated by a program. Do not edit manually.

#include <boost/qvm/enable_if.hpp>
#include <boost/qvm/inline.hpp>
#include <boost/qvm/mat_traits.hpp>

namespace boost { namespace qvm {

template <class A,class B>
BOOST_QVM_CONSTEXPR BOOST_QVM_INLINE_OPERATIONS
typename enable_if_c<
    mat_traits<A>::rows==2 && mat_traits<B>::rows==2 &&
    mat_traits<A>::cols==2 && mat_traits<B>::cols==2,
    A &>::type
assign( A & a, B const & b )
    {
    mat_traits<A>::template write_element<0,0>(a)=mat_traits<B>::template read_element<0,0>(b);
    mat_traits<A>::template write_element<0,1>(a)=mat_traits<B>::template read_element<0,1>(b);
    mat_traits<A>::template write_element<1,0>(a)=mat_traits<B>::template read_element<1,0>(b);
    mat_traits<A>::template write_element<1,1>(a)=mat_traits<B>::template read_element<1,1>(b);
    return a;
    }

namespace
sfinae
    {
    using ::boost::qvm::assign;
    }

namespace
qvm_detail
    {
    template <int R,int C>
    struct assign_mm_defined;

    template <>
    struct
    assign_mm_defined<2,2>
        {
        static bool const value=true;
        };
    }

template <class A,class B>
BOOST_QVM_CONSTEXPR BOOST_QVM_INLINE_OPERATIONS
typename enable_if_c<
    mat_traits<A>::rows==2 && mat_traits<B>::rows==2 &&
    mat_traits<A>::cols==1 && mat_traits<B>::cols==1,
    A &>::type
assign( A & a, B const & b )
    {
    mat_traits<A>::template write_element<0,0>(a)=mat_traits<B>::template read_element<0,0>(b);
    mat_traits<A>::template write_element<1,0>(a)=mat_traits<B>::template read_element<1,0>(b);
    return a;
    }

namespace
sfinae
    {
    using ::boost::qvm::assign;
    }

namespace
qvm_detail
    {
    template <int R,int C>
    struct assign_mm_defined;

    template <>
    struct
    assign_mm_defined<2,1>
        {
        static bool const value=true;
        };
    }

template <class A,class B>
BOOST_QVM_CONSTEXPR BOOST_QVM_INLINE_OPERATIONS
typename enable_if_c<
    mat_traits<A>::rows==1 && mat_traits<B>::rows==1 &&
    mat_traits<A>::cols==2 && mat_traits<B>::cols==2,
    A &>::type
assign( A & a, B const & b )
    {
    mat_traits<A>::template write_element<0,0>(a)=mat_traits<B>::template read_element<0,0>(b);
    mat_traits<A>::template write_element<0,1>(a)=mat_traits<B>::template read_element<0,1>(b);
    return a;
    }

namespace
sfinae
    {
    using ::boost::qvm::assign;
    }

namespace
qvm_detail
    {
    template <int R,int C>
    struct assign_mm_defined;

    template <>
    struct
    assign_mm_defined<1,2>
        {
        static bool const value=true;
        };
    }

} }

#endif


#include "mesh.hpp"
#include "generic/eigen.hpp"
#include "kernel_poly6.hpp"

#include "spatial/2d/neighborhood_spatial_hashing.hpp"

#include <Eigen/Geometry>
#include <iostream>

namespace GooBalls {
namespace d2 {
namespace Physics {

using namespace Spatial;

Mesh::Mesh() :
    linearAcceleration(0,0),
    m_vertices_position_global(new Coordinates()),
    m_triangles(new TriangleList())
    {
    // empty
}

Mesh::Mesh(std::shared_ptr<Mesh::Coordinates> verts, std::shared_ptr<TriangleList> tris) :
    linearAcceleration(0,0),
    m_vertices_position_global(verts),
    m_triangles(tris)
{
    m_vertices_position_local = *verts;
}

const Mesh::Coordinates& Mesh::vertices_position_global() const {
    return *m_vertices_position_global;
}

Mesh::Coordinates& Mesh::vertices_position_global() {
    return *m_vertices_position_global;
}

const Mesh::Coordinates& Mesh::vertices_position_local() const {
    return m_vertices_position_local;
}
Mesh::Coordinates& Mesh::vertices_position_local() {
    return m_vertices_position_local;
}

const TriangleList& Mesh::triangles() const {
    return *m_triangles;
}
TriangleList& Mesh::triangles() {
    return *m_triangles;
}

const Coordinates2d& Mesh::particles_position_local() const {
    return m_particles_position_local;
}

const Coordinates2d& Mesh::particles_velocity() const {
    return m_particles_velocity;
}

const Coordinates1d& Mesh::particles_volume() const {
    return m_particles_volume;
}

const Coordinates1d& Mesh::particles_mass() const {
    return m_particles_mass;
}

Float& Mesh::particles_velocity_correction_coefficient() {
    return m_velocity_correction_coefficient;
}
Float Mesh::particles_velocity_correction_coefficient() const {
    return m_velocity_correction_coefficient;
}

RotationMatrix Mesh::rotation() const {
    Float radianAngle = body->GetAngle();
    // eigen convention: counter-clockwise rotation in radians
    // box2d convention: not clear, only hint is in chapter 8.6 Revolute Joint of the manual -> test
    // TODO: test rotation direction of Box2d
    Eigen::Rotation2D<Float> rot(radianAngle);
    RotationMatrix rotM = rot.toRotationMatrix();
    return rotM;
}

TranslationVector Mesh::translation() const {
    TranslationVector dx;
    dx[0] = body->GetPosition().x;
    dx[1] = body->GetPosition().y;
    return dx;
}

TranslationVector Mesh::linear_velocity() const {
    TranslationVector v;
    v[0] = body->GetLinearVelocity().x;
    v[1] = body->GetLinearVelocity().y;
    return v;
}

RotationMatrix Mesh::angular_velocity() const {
    // box2d: radians/second
    RotationMatrix r;
    Float v = body->GetAngularVelocity();
    Eigen::Rotation2D<Float> rot(v);
    RotationMatrix rotM = rot.toRotationMatrix();
    // TODO: it is not clear, if rotM needs to be transposed here -> test
    return rotM;
}

/// takes the rigid body transformation from the box2D object and applies it locally
void Mesh::update_from_rigid_body() {
    const auto& loc = this->vertices_position_local();
    auto& glob = this->vertices_position_global();
    RotationMatrix rotM = rotation();
    TranslationVector dx = translation();
    rotM.transpose();
    glob = (loc * rotM).rowwise() + dx;
    // speed of particle
    if(m_particles_position_local.rows() > 0){
        m_particles_velocity.resize(m_particles_position_local.rows(), Eigen::NoChange);
        m_particles_velocity.col(0) = -m_particles_position_local.col(1);
        m_particles_velocity.col(1) = m_particles_position_local.col(0);
        m_particles_velocity *= body->GetAngularVelocity();
        m_particles_velocity = m_particles_velocity.rowwise() + linear_velocity();
    }
}

void Mesh::prepare(Float h){
    update_from_rigid_body();
    create_particles(h);
    compute_particles_volume(h);
    compute_particles_mass(h);
}
void Mesh::create_particles(Float h) {
    // most basic particle generation algorithm:
    // for each triangle
    // - add three particles for each corner
    // - look at all edges and sample each edge n additional times
    const auto& tris = triangles();
    const auto& verts = vertices_position_local();
    /*
    std::cout << "pts: \n";
    for(int i = 0; i < verts.rows(); ++i){
        std::cout << verts(i,0) << " " << verts(i, 1) << std::endl;
    }
    std::cout << "body: " << body->GetPosition().x << ", " << body->GetPosition().y << std::endl;
    */
    int totalPoints = 0;
    int inserted = 0;
    auto& pts = m_particles_position_local;
    int sampling = 4;

    for(int t = 0; t < tris.rows(); ++t){
        for(int v = 0; v < 3; v++){
            int vn = (v+1)%3; // next vertex
            // process edge
            auto diffV = verts.row(tris(t, vn)) - verts.row(tris(t, v));
            int n = std::ceil(diffV.norm()/h*sampling)+1;
            totalPoints += 1;
            totalPoints += n;
            pts.conservativeResize(totalPoints, Eigen::NoChange);
            // insert vertex
            pts.row(inserted++) = verts.row(tris(t, v));
            TranslationVector diff = (diffV)/double(n+1);
            for(int ni = 0; ni < n; ni++){
                pts.row(inserted++) = (ni+1) * diff + verts.row(tris(t, v));
            }
        }
    }
}

void Mesh::compute_particles_volume(Float h) {
    // occupied volume according to Versatile Rigid-Fluid Coupling for Incompressible SPH, Akinci et. al
    // V_bi = 1/sum_k W_ik, W: is a kernel
    m_particles_volume.resize(m_particles_position_local.rows(), Eigen::NoChange);
    NeighborhoodSpatialHashing neighbor;
    Poly6 kernel;
    kernel.setH(h);
    neighbor.inRange(m_particles_position_local, h);
    const auto& indexes = neighbor.indexes();
    for(size_t i = 0; i < indexes.size(); ++i){
        Coordinates2d jpos;
        Coordinates1d W;
        pickRows(m_particles_position_local, indexes[i], jpos);
        auto xij = -(jpos.rowwise() - m_particles_position_local.row(i));
        kernel.compute(xij, &W, nullptr, nullptr);
        m_particles_volume[i] = 1/W.sum();
    }
}

void Mesh::compute_particles_mass(Float) {
    int PN = m_particles_position_local.rows();
    m_particles_mass.resize(PN, Eigen::NoChange);
    m_particles_mass.array() = body->GetMass()/PN;//TODO: body->GetMass()/PN;
}

} // Physics
} // d2
} // GooBalls


#include <cctbx/boost_python/flex_fwd.h>

#include <cctbx/xray/grouped_data.h>

#include <boost/python/module.hpp>
#include <boost/python/def.hpp>
#include <boost/python/class.hpp>
#include <boost/python/args.hpp>
#include <boost/python/tuple.hpp>
#include <boost/python/return_value_policy.hpp>
#include <boost/python/copy_const_reference.hpp>
#include <scitbx/boost_python/is_polymorphic_workaround.h>

namespace cctbx { namespace xray { namespace grouped_data { namespace boost_python {
  namespace {

    struct unmerged_data_wrappers
    {
      typedef unmerged_data<double> w_t;
      static void
      wrap()
      {
        using namespace boost::python;

        class_<w_t>("unmerged_data", no_init)
          .def(init< scitbx::af::const_ref< cctbx::miller::index<> > const&,
                     scitbx::af::const_ref< cctbx::miller::index<> > const&,
                     sgtbx::space_group const&,
                     bool const&
                   >
               (( arg("hkl_obs"),
                  arg("asu_hkl"),
                  arg("space_group"),
                  arg("anomalous_flag") )))
          ;
      }

    };

    struct merger_wrappers
    {
      typedef merger<double> w_t;
      static void
      wrap()
      {
        using namespace boost::python;

        class_<w_t>("merger", no_init)
          .def(init< scitbx::af::const_ref< cctbx::miller::index<> > const&,
                     scitbx::af::const_ref< double >                 const&,
                     scitbx::af::const_ref< double >                 const&,
                     sgtbx::space_group                              const&,
                     bool                                            const&,
                     cctbx::uctbx::unit_cell                         const&
                   >
               (( arg("hkl_obs"),
                  arg("i_obs"),
                  arg("s_obs"),
                  arg("space_group"),
                  arg("anomalous_flag"),
                  arg("unit_cell")
               )))
          .def("bic", &w_t::bic)
          .def("r_abs", &w_t::r_abs)
          ;
      }

    };

  }}} // namespace cctbx::xray

  namespace boost_python {

    void wrap_grouped_data()
    {
      grouped_data::boost_python::unmerged_data_wrappers::wrap();
      grouped_data::boost_python::merger_wrappers::wrap();
    }
  }

}}


/*
Copyright 2013 Henrik Mühe and Florian Funke

This file is part of CampersCoreBurner.

CampersCoreBurner is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

CampersCoreBurner is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with CampersCoreBurner.  If not, see <http://www.gnu.org/licenses/>.
*/

/*
COMMENT SUMMARY FOR THIS FILE:

This implements a functor class which wraps the intrinsic crc32 instructions
inside modern cpus and exposes them to compute hashes. Since for this application
we have a known length limit on strings, we do not need a loop to compute a
string has but instead only a switch between different sizes classes.
*/

#pragma once

#include <cstdint>
#include <boost/utility/string_ref.hpp>
#include <string>
#include <x86intrin.h>

/// Adds hashing for boost string references to std::hash
namespace std {
template<>
struct hash<boost::string_ref> {
    size_t operator()(const boost::string_ref& word) const {
        return std::hash<std::string>()((std::string)word);
    }
};
}

namespace campers {

/// Wraps crc32 cpu instructions so that they can be used with unordered_map or our custom hash
class Crc32Hash {
public:
    /// Integers
    inline size_t operator()(uint64_t x) const {
        return _mm_crc32_u64(0,x);
    }
    inline size_t operator()(int64_t x) const {
        return _mm_crc32_u64(0,x);
    }
    inline size_t operator()(uint32_t x) const {
        return _mm_crc32_u32(0,x);
    }
    inline size_t operator()(int32_t x) const {
        return _mm_crc32_u32(0,x);
    }

    /// Pointers
    inline size_t operator()(void* ptr) const {
        return std::hash<void*>()(ptr);
    }

    /// Boost string references
    inline unsigned operator()(const boost::string_ref& s) const {
        unsigned l=s.length();
        const char* from = s.data();
        if (l<=8) {
            return _mm_crc32_u64(0,(reinterpret_cast<const uint64_t*>(from)[0])<<(64-8*l));
        } else if (l<=16) {
            return _mm_crc32_u64(_mm_crc32_u64(0,reinterpret_cast<const uint64_t*>(from)[0]),(reinterpret_cast<const  uint64_t*>(from)[1])<<(128-8*l));
        } else if (l<=24) {
            return _mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(0,reinterpret_cast<const uint64_t*>(from)[0]),reinterpret_cast<const  uint64_t*>(from)[1]),(reinterpret_cast<const uint64_t*>(from)[2])<<(128+64-8*l));
        } else {
            return _mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(0,reinterpret_cast<const uint64_t*>(from)[0]),reinterpret_cast<const uint64_t*>(from)[1]),reinterpret_cast<const uint64_t*>(from)[2]),(reinterpret_cast<const uint64_t*>(from)[3])<<(256-8*l));
        }
    }

    /// Strings
    inline unsigned operator()(const std::string& s) const {
        unsigned l=s.length();
        const char* from = s.data();
        if (l<=8) {
            return _mm_crc32_u64(0,(reinterpret_cast<const uint64_t*>(from)[0])<<(64-8*l));
        } else if (l<=16) {
            return _mm_crc32_u64(_mm_crc32_u64(0,reinterpret_cast<const uint64_t*>(from)[0]),(reinterpret_cast<const  uint64_t*>(from)[1])<<(128-8*l));
        } else if (l<=24) {
            return _mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(0,reinterpret_cast<const uint64_t*>(from)[0]),reinterpret_cast<const  uint64_t*>(from)[1]),(reinterpret_cast<const uint64_t*>(from)[2])<<(128+64-8*l));
        } else {
            return _mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(0,reinterpret_cast<const uint64_t*>(from)[0]),reinterpret_cast<const uint64_t*>(from)[1]),reinterpret_cast<const uint64_t*>(from)[2]),(reinterpret_cast<const uint64_t*>(from)[3])<<(256-8*l));
        }
    }
};

}


// Boost.Geometry (aka GGL, Generic Geometry Library)
// Unit Test

// Copyright (c) 2009-2012 Mateusz Loskot, London, UK.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#include <cstddef>

#include <boost/test/tools/floating_point_comparison.hpp>
#include <boost/test/included/test_exec_monitor.hpp>

#include <boost/geometry/geometries/geometries.hpp>
#include <boost/geometry/geometries/point_xy.hpp>

#include <boost/geometry/extensions/index/rtree/rtree.hpp>

namespace bg = boost::geometry;

int test_main(int, char* [])
{

    // TODO: mloskot - This is ONLY compilation test and
    // placeholder to implement real test.

    bg::index::rtree<bg::model::box<bg::model::d2::point_xy<double> >, std::size_t> si(1, 6);
    return 0;
}