#ifndef STAN_LANG_GENERATOR_EXPRESSION_VISGEN_HPP
#define STAN_LANG_GENERATOR_EXPRESSION_VISGEN_HPP

#include <stan/lang/ast.hpp>
#include <stan/lang/generator/constants.hpp>
#include <stan/lang/generator/generate_bare_type.hpp>
#include <stan/lang/generator/generate_indexed_expr.hpp>
#include <stan/lang/generator/generate_real_var_type.hpp>
#include <stan/lang/generator/visgen.hpp>
#include <boost/lexical_cast.hpp>
#include <ostream>
#include <string>
#include <vector>

#include <iostream>

namespace stan {
  namespace lang {

    // forward declare recursive helper functions
    void generate_array_builder_adds(const std::vector<expression>& elements,
                                     bool user_facing, std::ostream& o);

    void generate_idxs(const std::vector<idx>& idxs, std::ostream& o);

    void generate_idxs_user(const std::vector<idx>& idxs, std::ostream& o);

    struct expression_visgen : public visgen {
      /**
         true when generated expression might be reported to user
      */
      const bool user_facing_;

      explicit expression_visgen(std::ostream& o, bool user_facing)
        : visgen(o),
          user_facing_(user_facing) {
      }

#include <stan/torsten/expression_visgen.hpp>

      void operator()(const nil& /*x*/) const {
        o_ << "nil";
      }

      void operator()(const int_literal& n) const {
        std::string num_str = boost::lexical_cast<std::string>(n.val_);
        o_ << num_str;
      }

      void operator()(const double_literal& x) const {
        o_ << x.string_;
        if (x.string_.find_first_of("eE.") == std::string::npos)
        o_ << ".0";  // trailing 0 to ensure C++ makes it a double
      }

      void operator()(const array_expr& x) const {
        std::stringstream ssRealType;
        generate_real_var_type(x.array_expr_scope_, x.has_var_, ssRealType);

        std::stringstream ssArrayType;
        generate_bare_type(x.type_, ssRealType.str(), ssArrayType);

        std::stringstream ssArrayElType;
        generate_bare_type(x.type_.array_element_type(),
                           ssRealType.str(), ssArrayElType);

        o_ << "static_cast<"
           << ssArrayType.str()
           << " >(stan::math::array_builder<"
           << ssArrayElType.str()
           << " >()";
        generate_array_builder_adds(x.args_, user_facing_, o_);
        o_ << ".array())";
      }

      void operator()(const matrix_expr& x) const {
        std::stringstream ssRealType;
        generate_real_var_type(x.matrix_expr_scope_, x.has_var_, ssRealType);
        // to_matrix arg is std::vector of row vectors (Eigen::Matrix<T, 1, C>)
        o_ << "stan::math::to_matrix(stan::math::array_builder<Eigen::Matrix<"
           << ssRealType.str()
           << ", 1, Eigen::Dynamic> >()";
        generate_array_builder_adds(x.args_, user_facing_, o_);
        o_ << ".array())";
      }

      void operator()(const row_vector_expr& x) const {
        std::stringstream ssRealType;
        generate_real_var_type(x.row_vector_expr_scope_, x.has_var_,
                               ssRealType);
        // to_row_vector arg is std::vector of type T
        o_ << "stan::math::to_row_vector(stan::math::array_builder<"
           << ssRealType.str()
           << " >()";
        generate_array_builder_adds(x.args_, user_facing_, o_);
        o_ << ".array())";
      }

      void operator()(const variable& v) const { o_ << v.name_; }

      void operator()(int n) const {   // NOLINT
        o_ << static_cast<long>(n);    // NOLINT
      }

      void operator()(double x) const { o_ << x; }

      void operator()(const std::string& x) const { o_ << x; }  // identifiers

      void operator()(const index_op& x) const {
        std::stringstream expr_o;
        generate_expression(x.expr_, user_facing_, expr_o);
        std::string expr_string = expr_o.str();
        std::vector<expression> indexes;
        for (size_t i = 0; i < x.dimss_.size(); ++i)
          for (size_t j = 0; j < x.dimss_[i].size(); ++j)
            indexes.push_back(x.dimss_[i][j]);  // wasteful copy, could use refs
        generate_indexed_expr<false>(expr_string, indexes,
                                     x.expr_.bare_type(),
                                     user_facing_, o_);
      }

      void operator()(const index_op_sliced& x) const {
        if (x.idxs_.size() == 0) {
          generate_expression(x.expr_, user_facing_, o_);
          return;
        }
        if (user_facing_) {
          generate_expression(x.expr_, user_facing_, o_);
          generate_idxs_user(x.idxs_, o_);
          return;
        }
        o_ << "stan::model::rvalue(";
        generate_expression(x.expr_, user_facing_, o_);
        o_ << ", ";
        generate_idxs(x.idxs_, o_);
        o_ << ", ";
        o_ << '"';
        generate_expression(x.expr_, USER_FACING, o_);
        o_ << '"';
        o_ << ")";
      }

      void operator()(const integrate_ode& fx) const {
        o_ << (fx.integration_function_name_ == "integrate_ode"
               ? "integrate_ode_rk45"
               : fx.integration_function_name_)
           << "("
           << fx.system_function_name_
           << "_functor__(), ";
        generate_expression(fx.y0_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.t0_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.ts_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.theta_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.x_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.x_int_, NOT_USER_FACING, o_);
        o_ << ", pstream__)";
      }

      void operator()(const integrate_ode_control& fx) const {
        o_ << fx.integration_function_name_
           << "("
           << fx.system_function_name_
           << "_functor__(), ";
        generate_expression(fx.y0_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.t0_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.ts_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.theta_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.x_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.x_int_, NOT_USER_FACING, o_);
        o_ << ", pstream__, ";
        generate_expression(fx.rel_tol_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.abs_tol_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.max_num_steps_, NOT_USER_FACING, o_);
        o_ << ")";
      }

      void operator()(const algebra_solver& fx) const {
        o_ << "algebra_solver"
           << "("
           << fx.system_function_name_
           << "_functor__(), ";
        generate_expression(fx.y_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.theta_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.x_r_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.x_i_, NOT_USER_FACING, o_);
        o_ << ", pstream__)";
      }

      void operator()(const algebra_solver_control& fx) const {
        o_ << "algebra_solver"
           << "("
           << fx.system_function_name_
           << "_functor__(), ";
        generate_expression(fx.y_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.theta_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.x_r_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.x_i_, NOT_USER_FACING, o_);
        o_ << ", pstream__, ";
        generate_expression(fx.rel_tol_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.fun_tol_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.max_num_steps_, NOT_USER_FACING, o_);
        o_ << ")";
      }

      void operator()(const map_rect& fx) const {
        o_ << "map_rect";
        o_ << "<" << fx.call_id_ << ", " << fx.fun_name_ << "_functor__>";
        o_ << "(";
        generate_expression(fx.shared_params_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.job_params_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.job_data_r_, NOT_USER_FACING, o_);
        o_ << ", ";
        generate_expression(fx.job_data_i_, NOT_USER_FACING, o_);
        o_ << ", pstream__)";
      }

      void operator()(const integrate_1d& fx) const {
        o_ << "integrate_1d("
           << fx.function_name_ << "_functor__(), ";
        generate_expression(fx.lb_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.ub_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.theta_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.x_r_, user_facing_, o_);
        o_ << ", ";
        generate_expression(fx.x_i_, user_facing_, o_);
        o_ << ", *pstream__, ";
        generate_expression(fx.rel_tol_, user_facing_, o_);
        o_ << ")";
      }

      void operator()(const fun& fx) const {
        // first test if short-circuit op (binary && and || applied to
        // primitives; overloads are eager, not short-circuiting)
        if (fx.name_ == "logical_or" || fx.name_ == "logical_and") {
          o_ << "(primitive_value(";
          boost::apply_visitor(*this, fx.args_[0].expr_);
          o_ << ") " << ((fx.name_ == "logical_or") ? "||" : "&&")
             << " primitive_value(";
          boost::apply_visitor(*this, fx.args_[1].expr_);
          o_ << "))";
          return;
        }
        o_ << fx.name_ << "(";
        for (size_t i = 0; i < fx.args_.size(); ++i) {
          if (i > 0) o_ << ", ";
          boost::apply_visitor(*this, fx.args_[i].expr_);
        }
        if (fx.args_.size() > 0
            && (has_rng_suffix(fx.name_) || has_lp_suffix(fx.name_)))
          o_ << ", ";
        if (has_rng_suffix(fx.name_))
          o_ << "base_rng__";
        if (has_lp_suffix(fx.name_))
          o_ << "lp__, lp_accum__";
        if (is_user_defined(fx)) {
          if (fx.args_.size() > 0
              || has_rng_suffix(fx.name_)
              || has_lp_suffix(fx.name_))
            o_ << ", ";
          o_ << "pstream__";
        }
        o_ << ")";
      }

      void operator()(const conditional_op& expr) const {
        bool types_prim_match
          = (expr.type_.is_primitive() && expr.type_.is_int_type())
          || (!expr.has_var_ && expr.type_.is_primitive()
              && (expr.true_val_.bare_type()
                  == expr.false_val_.bare_type()));
        std::stringstream ss;
        generate_real_var_type(expr.scope_, expr.has_var_, ss);

        o_ << "(";
        boost::apply_visitor(*this, expr.cond_.expr_);
        o_ << " ? ";
        if (types_prim_match) {
          boost::apply_visitor(*this, expr.true_val_.expr_);
        } else {
          o_ << "stan::math::promote_scalar<"
             << ss.str()
             << ">(";
          boost::apply_visitor(*this, expr.true_val_.expr_);
          o_ << ")";
        }
        o_ << " : ";
        if (types_prim_match) {
          boost::apply_visitor(*this, expr.false_val_.expr_);
        } else {
          o_ << "stan::math::promote_scalar<"
             << ss.str()
             << ">(";
          boost::apply_visitor(*this, expr.false_val_.expr_);
          o_ << ")";
        }
        o_ << " )";
      }

      void operator()(const binary_op& expr) const {
        o_ << "(";
        boost::apply_visitor(*this, expr.left.expr_);
        o_ << " " << expr.op << " ";
        boost::apply_visitor(*this, expr.right.expr_);
        o_ << ")";
      }

      void operator()(const unary_op& expr) const {
        o_ << expr.op << "(";
        boost::apply_visitor(*this, expr.subject.expr_);
        o_ << ")";
      }
    };

  }
}
#endif


#include <boost/type_traits/has_modulus.hpp>


// ---------------------------------------------------------------------------
//
//  This file is part of PermLib.
//
// Copyright (c) 2009-2011 Thomas Rehn <thomas@carmen76.de>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// ---------------------------------------------------------------------------

#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE BSGS construction and change test
#include <boost/test/unit_test.hpp>

#include <iostream>
#include <fstream>
#include <cmath>

#include <permlib/permutation.h>
#include <permlib/permutationword.h>
#include <permlib/bsgs.h>

#include <permlib/transversal/schreier_tree_transversal.h>
#include <permlib/transversal/explicit_transversal.h>
#include <permlib/construct/schreier_sims_construction.h>

#include <permlib/construct/cyclic_group_construction.h>

#include <permlib/change/simple_base_change.h>
#include <permlib/change/conjugating_base_change.h>
#include <permlib/change/deterministic_base_transpose.h>
#include <permlib/change/random_base_transpose.h>

#include "test-common.h"
#include <permlib/generator/bsgs_random_generator.h>

using namespace permlib;
using namespace permlib::test;

template<class CONSTRUCTOR>
void checkGroup(const GroupInformation &info) {
	typedef typename CONSTRUCTOR::PERMtype PERM;
	typedef typename CONSTRUCTOR::TRANStype TRANS;

	BSGS<PERM,TRANS> bsgs = CONSTRUCTOR()(info);
	BOOST_CHECK_EQUAL( bsgs.n, info.n );
	BOOST_CHECK_EQUAL( bsgs.order(), info.order );

	unsigned int transversalProduct = 1;
	BOOST_FOREACH(const TRANS &U, bsgs.U) {
			transversalProduct *= U.size();
	}
	BOOST_CHECK_EQUAL( transversalProduct, bsgs.order() );

	for (unsigned int i = 0; i < bsgs.n; ++i) {
		BSGS<PERM,TRANS> bsgs2(bsgs);
		bsgs2.insertRedundantBasePoint(i);

		transversalProduct = 1;
		BOOST_FOREACH(const TRANS &U, bsgs2.U) {
			transversalProduct *= U.size();
		}
		BOOST_CHECK_EQUAL( transversalProduct, bsgs2.order() );
		BOOST_CHECK_EQUAL( bsgs.order(), bsgs2.order() );
	}
}

void checkGroup(const GroupInformation &info) {
	checkGroup<ConstructDeterministic1<Permutation, ExplicitTransversal<Permutation> > >(info);
	checkGroup<ConstructDeterministic1<Permutation, SchreierTreeTransversal<Permutation> > >(info);
	checkGroup<ConstructDeterministic1<PermutationWord, SchreierTreeTransversal<PermutationWord> > >(info);

	checkGroup<ConstructDeterministic2<Permutation, ExplicitTransversal<Permutation> > >(info);
	checkGroup<ConstructDeterministic2<Permutation, SchreierTreeTransversal<Permutation> > >(info);

	checkGroup<ConstructRandomized<Permutation, ExplicitTransversal<Permutation> > >(info);
	checkGroup<ConstructRandomized<Permutation, SchreierTreeTransversal<Permutation> > >(info);
}

template<class PERM,class TRANS,class TRANSPOSE>
void checkTranspose(const GroupInformation &info, const BSGS<PERM,TRANS> &bsgs) {
	const unsigned int m = bsgs.B.size();
	for (unsigned int i=0; i<m-1; ++i) {
		BSGS<PERM,TRANS> bsgs_copy(bsgs);
		TRANSPOSE bt;
		bt.transpose(bsgs_copy, i);

		BOOST_REQUIRE_EQUAL( bsgs_copy.B.size(), bsgs.B.size() );
		BOOST_CHECK_EQUAL( bsgs_copy.order(), bsgs.order() );
		unsigned int transversalProduct = 1;
		BOOST_FOREACH(const TRANS &U, bsgs_copy.U) {
			transversalProduct *= U.size();
		}
		BOOST_CHECK_EQUAL( transversalProduct, bsgs.order() );

		BOOST_CHECK_EQUAL( bsgs_copy.B[i], bsgs.B[i+1] );
		BOOST_CHECK_EQUAL( bsgs_copy.B[i+1], bsgs.B[i] );
		for (unsigned int j=0; j<m; ++j) {
			if (j == i || j == i+1)
				continue;
			BOOST_CHECK_EQUAL( bsgs_copy.B[j], bsgs.B[j] );
		}
	}
}

template<class PERM,class TRANS>
void checkTranspose(const GroupInformation &info) {
	BSGS<PERM,TRANS> bsgs = construct<PERM,TRANS>(info);

	checkTranspose<PERM,TRANS,DeterministicBaseTranspose<PERM,TRANS> >(info, bsgs);
	checkTranspose<PERM,TRANS,RandomBaseTranspose<PERM,TRANS> >(info, bsgs);
}

template<class PERM,class TRANS,class BASECHANGE>
void checkChange(const GroupInformation &info) {
	BSGS<PERM,TRANS> bsgs = construct<PERM,TRANS>(info);
	BSGS<PERM,TRANS> bsgs_copy(bsgs);

	BASECHANGE bc(bsgs);
	std::vector<unsigned long> newBase(5);
	newBase[0] = 0;
	newBase[1] = 2;
	newBase[2] = 4;
	newBase[3] = 6;
	newBase[4] = 8;
	bc.change(bsgs_copy, newBase.begin(), newBase.end());

	BOOST_REQUIRE_GE( bsgs_copy.B.size(), newBase.size() );
	BOOST_CHECK_EQUAL( bsgs_copy.order(), bsgs.order() );
	unsigned int transversalProduct = 1;
	BOOST_FOREACH(const TRANS &U, bsgs_copy.U) {
		transversalProduct *= U.size();
	}
	BOOST_CHECK_EQUAL( transversalProduct, bsgs.order() );

	for (unsigned int i=0; i<newBase.size(); ++i)
		BOOST_CHECK_EQUAL( newBase[i], bsgs_copy.B[i] );
}

BOOST_AUTO_TEST_CASE( construction_test33 )
{
	checkGroup(info_test33);
}

BOOST_AUTO_TEST_CASE( construction_trivial )
{
	checkGroup(info_trivial);
}

BOOST_AUTO_TEST_CASE( construction_test1997 )
{
	checkGroup(info_1997);
}

BOOST_AUTO_TEST_CASE( construction_psu4_3 )
{
	checkGroup(info_psu4_3);
}

BOOST_AUTO_TEST_CASE( construction_S6 )
{
	checkGroup(info_S6);
}

BOOST_AUTO_TEST_CASE( transpose_test1997 )
{
	typedef Permutation PERM;
	typedef ExplicitTransversal<PERM> TRANS;
	checkTranspose<PERM,TRANS>(info_1997);
}

BOOST_AUTO_TEST_CASE( transpose_psu4_3 )
{
	typedef PermutationWord PERM;
	typedef SchreierTreeTransversal<PERM> TRANS;
	checkTranspose<PERM,TRANS>(info_psu4_3);
}

BOOST_AUTO_TEST_CASE( change_test1997_simple )
{
	typedef Permutation PERM;
	typedef ExplicitTransversal<PERM> TRANS;
	checkChange<PERM,TRANS,SimpleBaseChange<PERM,TRANS,DeterministicBaseTranspose<PERM,TRANS> > >(info_1997);
	checkChange<PERM,TRANS,SimpleBaseChange<PERM,TRANS,RandomBaseTranspose<PERM,TRANS> > >(info_1997);
}

BOOST_AUTO_TEST_CASE( change_test1997_conjugating )
{
	typedef Permutation PERM;
	typedef ExplicitTransversal<PERM> TRANS;
	checkChange<PERM,TRANS,ConjugatingBaseChange<PERM,TRANS,DeterministicBaseTranspose<PERM,TRANS> > >(info_1997);
	checkChange<PERM,TRANS,ConjugatingBaseChange<PERM,TRANS,RandomBaseTranspose<PERM,TRANS> > >(info_1997);
}

BOOST_AUTO_TEST_CASE( change_trivial_conjugating )
{
	typedef Permutation PERM;
	typedef ExplicitTransversal<PERM> TRANS;
	checkChange<PERM,TRANS,ConjugatingBaseChange<PERM,TRANS,DeterministicBaseTranspose<PERM,TRANS> > >(info_trivial);
	checkChange<PERM,TRANS,ConjugatingBaseChange<PERM,TRANS,RandomBaseTranspose<PERM,TRANS> > >(info_trivial);
}

BOOST_AUTO_TEST_CASE( cyclic_construction )
{
	typedef Permutation PERM;
	typedef SchreierTreeTransversal<PERM> TRANS;
	for (unsigned int n = 2; n < 13; ++n) {
		BSGS<PERM, TRANS> cyclicBSGS = constructCyclicGroup<PERM, TRANS>(n);
		BOOST_CHECK_EQUAL( cyclicBSGS.n, n );
		BOOST_CHECK_EQUAL( cyclicBSGS.order<unsigned int>(), n );
		// a base of one point suffices with a cyclic group
		BOOST_CHECK_EQUAL( cyclicBSGS.B.size(), 1 );
		BOOST_CHECK_EQUAL( cyclicBSGS.U.size(), 1 );
	}
}

template<class PERM,class TRANS>
void checkRedundantStrongGenerators(const GroupInformation& info) {
	BSGS<PERM,TRANS> bsgs = ConstructDeterministic1<PERM, TRANS>()(info);
	BSGS<PERM,TRANS> bsgsCopy(bsgs);
	BSGSRandomGenerator<PERM,TRANS> randGen(bsgs);

	for (unsigned int i = 0; i < info.n; ++i) {
		typename PERM::ptr p(new PERM(randGen.next()));
		bsgsCopy.S.push_front(p);
	}

	bsgsCopy.stripRedundantStrongGenerators();

	SchreierSimsConstruction<PERM,TRANS> ssc(bsgsCopy.n);
	BSGS<PERM,TRANS> bsgsStripped = ssc.construct(bsgsCopy.S.begin(), bsgsCopy.S.end(), bsgs.B.begin(), bsgs.B.end());

	BOOST_CHECK_EQUAL(bsgs.order(), bsgsStripped.order());
	BOOST_CHECK_GE(bsgsCopy.S.size(), bsgsStripped.S.size());
	BOOST_CHECK_LE(bsgsCopy.S.size(), std::log(bsgs.order()) / std::log(2));

	// stripRedundantStrongGenerators has to be idempotent
	const unsigned int oldGeneratorNumber = bsgsCopy.S.size();
	bsgsCopy.stripRedundantStrongGenerators();
	BOOST_CHECK_EQUAL(oldGeneratorNumber, bsgsCopy.S.size());
}

BOOST_AUTO_TEST_CASE( strip_redundant_generators )
{
	typedef Permutation PERM;
	typedef SchreierTreeTransversal<PERM> TRANS;

	checkRedundantStrongGenerators<PERM,TRANS>(info_1997);
	checkRedundantStrongGenerators<PERM,TRANS>(info_S6);
	checkRedundantStrongGenerators<PERM,TRANS>(info_A9);
	checkRedundantStrongGenerators<PERM,TRANS>(info_S3wrS5);
	checkRedundantStrongGenerators<PERM,TRANS>(info_testThesis);
	checkRedundantStrongGenerators<PERM,TRANS>(info_test33);
	checkRedundantStrongGenerators<PERM,TRANS>(info_cyclic37_2);
	checkRedundantStrongGenerators<PERM,TRANS>(info_e6);
	checkRedundantStrongGenerators<PERM,TRANS>(info_myciel3);
	checkRedundantStrongGenerators<PERM,TRANS>(info_cov1075);
	checkRedundantStrongGenerators<PERM,TRANS>(info_metric5);
}


#include <boost/python.hpp>

#include "../../../def_submodule.hpp"

#include "IceRay/geometry/simple/cylinder.hpp"

#include "./simple.hpp"

typedef GS_DDMRM::S_IceRay::S_geometry::GC_cylinder       GTs_cylinder;

void expose_IceRay_geometry_cylinder()
 {
  //MAKE_SUBMODULE( IceRay );
  MAKE_SUBMODULE( core );
  MAKE_SUBMODULE( geometry );

  typedef  GTs_scalar const& (GTs_cylinder::*Tf_getRadius)(void) const;
  Tf_getRadius I_getRadius = &GTs_cylinder::F_radius;

  typedef  bool (GTs_cylinder::*Tf_setRadius  )(GTs_scalar const&);
  Tf_setRadius I_setRadius = &GTs_cylinder::F_radius;

  boost::python::class_< GTs_cylinder, boost::python::bases<GTs__base> >( "GeometrySimpleCylinder" )
    .def( boost::python::init<>() )
    .def( boost::python::init<GTs_scalar>() )
    .def( boost::python::init<GTs_scalar,GTs_scalar>() )
    .def("radius",  I_getRadius, boost::python::return_value_policy<boost::python::copy_const_reference>()  )
    .def("radius",  I_setRadius      )

  ;

 }


#include <spatialops/SpatialOpsTools.h>
#include <spatialops/OperatorDatabase.h>

#include <spatialops/structured/stencil/OneSidedOperatorTypes.h>
#include <spatialops/structured/stencil/StencilBuilder.h>
#include <spatialops/structured/FieldComparisons.h>
#include <spatialops/structured/Grid.h>
#include <spatialops/Nebo.h>

#include <test/TestHelper.h>
#include <spatialops/structured/FieldHelper.h>

#include <boost/program_options.hpp>
namespace po = boost::program_options;

using namespace SpatialOps;

#include <stdexcept>
using std::cout;
using std::endl;

//--------------------------------------------------------------------

#define TOLERANCE 1.0e-3

//--------------------------------------------------------------------

template<typename FieldT, typename OpT> struct Result;

template<typename FieldT> struct Result<FieldT,Interpolant>{
  Result(){}
  void operator()( const FieldT& x, FieldT& result ) const{
    result <<= sin(x);
  }
};
template<typename FieldT> struct Result<FieldT,Gradient>{
  Result(){}
  void operator()( const FieldT& x, FieldT& result ) const{
    result <<= cos(x);
  }
};

//--------------------------------------------------------------------

template<typename Op, typename StencilT, typename FieldType>
bool test_stencil( const IntVec npts,
                   const OperatorDatabase & opdb,
                   const Grid& grid,
                   const double tolerance )
{
  typedef typename StencilT::DirT              OpDir; // the orientation of the stencil, e.g., <1,0,0>
  typedef typename GetNonzeroDir<OpDir>::DirT  DirT;  // the direction the stencil acts in, e.g., XDIR

  //Two ghost cells are required for 3 point one sided stencils
  const GhostData ghost(2);
  const IntVec bc(true,true,true);
  const BoundaryCellInfo bcinfo = BoundaryCellInfo::build<FieldType>(bc,bc);
  const MemoryWindow mw = get_window_with_ghost(npts, ghost, bcinfo);

  FieldType x     ( mw, bcinfo, ghost, NULL );
  FieldType y     ( mw, bcinfo, ghost, NULL );
  FieldType z     ( mw, bcinfo, ghost, NULL );
  FieldType indep ( mw, bcinfo, ghost, NULL );
  FieldType result( mw, bcinfo, ghost, NULL );
  FieldType refVal( mw, bcinfo, ghost, NULL );
  FieldType arg   ( mw, bcinfo, ghost, NULL );

  // set field values
  grid.set_coord<XDIR>(x);
  grid.set_coord<YDIR>(y);
  grid.set_coord<ZDIR>(z);

  indep <<= x + y;
  indep <<= indep + z;
//  indep <<= x+y+z;  // jcs for some strange reason, this can cause problem on serial OSX builds.

  arg    <<= sin( indep );
  const Result<FieldType,Op> calc_result;
  calc_result(indep,refVal);

  //get operator:
  typedef typename SpatialOps::OneSidedOpTypeBuilder<Op,StencilT,FieldType>::type FullOpT;
  FullOpT* const op = opdb.retrieve_operator<FullOpT>();

  result <<= 0;
  result <<= (*op)(arg);

//  write_matlab( result, "src", false );
//  write_matlab( refVal, "ref", false );
//  write_matlab( coord, "coord", false );
  const double l2Norm = nebo_norm_interior( result-refVal );

  if( l2Norm>=tolerance ) std::cout << "norm: " << l2Norm << std::endl;
  return l2Norm<tolerance;
};

//-------------------------------------------------------------------

template<typename Op, typename StencilT>
bool test_field_variants( const IntVec npts,
                          const OperatorDatabase & opdb,
                          const Grid& grid,
                          const double tolerance )
{
  TestHelper status(false);
  status( test_stencil<Op,StencilT,SVolField>(npts, opdb, grid, tolerance), "  -> SVolField" );
  if( npts[0]>1 ) status( test_stencil<Op,StencilT,XVolField>(npts, opdb, grid, tolerance), "  -> XVolField" );
  if( npts[1]>1 ) status( test_stencil<Op,StencilT,YVolField>(npts, opdb, grid, tolerance), "  -> YVolField" );
  if( npts[2]>1 ) status( test_stencil<Op,StencilT,ZVolField>(npts, opdb, grid, tolerance), "  -> ZVolField" );
  return status.ok();
}

//--------------------------------------------------------------------

int main( int iarg, char* carg[] )
{
  int nx, ny, nz;
  double length, tol2, tol3;
  {
    po::options_description desc("Supported Options");
    desc.add_options()
          ( "help", "print help message\n" )
          ( "nx", po::value<int>   ( &nx     )->default_value( 21   ), "number of points in x-dir for base mesh" )
          ( "ny", po::value<int>   ( &ny     )->default_value( 21   ), "number of points in y-dir for base mesh" )
          ( "nz", po::value<int>   ( &nz     )->default_value( 21   ), "number of points in z-dir for base mesh" )
          ( "l",  po::value<double>( &length )->default_value( 0.01 ), "length of the domain"                    )
          ( "tol2", po::value<double>( &tol2 )->default_value( 0.01 ), "error tolerance for 2-pt stencil test"   )
          ( "tol3", po::value<double>( &tol3 )->default_value( 0.01 ), "error tol2 for 3-pt stencil test"        );

    po::variables_map args;
    po::store( po::parse_command_line(iarg,carg,desc), args );
    po::notify(args);

    if( args.count("help") ){
      cout << desc << endl
          << "Example:" << endl
          << "  test_stencil --nx 5 --ny 10 --nz 3 --l 0.1 " << endl
          << "Note: Less points require a smaller length to limit approximation error" << endl
          << endl;
      return -1;
    }
  }

  TestHelper status( true );
  const IntVec npts(nx,ny,nz);

  {
    cout << "  domain : " << npts << endl
        << endl;
  }

  OperatorDatabase opdb;
  build_stencils( npts[0], npts[1], npts[2], length, length, length, opdb );

  const Grid grid( npts, DoubleVec(length, length, length) );

  typedef UnitTriplet<XDIR>::type  XPlus;
  typedef UnitTriplet<YDIR>::type  YPlus;
  typedef UnitTriplet<ZDIR>::type  ZPlus;
  typedef XPlus::Negate            XMinus;
  typedef YPlus::Negate            YMinus;
  typedef ZPlus::Negate            ZMinus;

  try{

    if( npts[0]>1 ){
      status( test_field_variants<Gradient, OneSidedStencil2<XMinus> >(npts, opdb, grid, tol2), "Two   Point, Minus Sided, X Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil2<XPlus > >(npts, opdb, grid, tol2), "Two   Point, Plus  Sided, X Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil3<XMinus> >(npts, opdb, grid, tol3), "Three Point, Minus Sided, X Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil3<XPlus > >(npts, opdb, grid, tol3), "Three Point, Plus  Sided, X Direction, Divergence Operator" );
    }
    if( npts[1]>1 ){
      status( test_field_variants<Gradient, OneSidedStencil2<YMinus> >(npts, opdb, grid, tol2), "Two   Point, Minus Sided, Y Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil2<YPlus > >(npts, opdb, grid, tol2), "Two   Point, Plus  Sided, Y Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil3<YMinus> >(npts, opdb, grid, tol3), "Three Point, Minus Sided, Y Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil3<YPlus > >(npts, opdb, grid, tol3), "Three Point, Plus  Sided, Y Direction, Divergence Operator" );
    }
    if( npts[2]>1 ){
      status( test_field_variants<Gradient, OneSidedStencil2<ZMinus> >(npts, opdb, grid, tol2), "Two   Point, Minus Sided, Z Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil2<ZPlus > >(npts, opdb, grid, tol2), "Two   Point, Plus  Sided, Z Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil3<ZMinus> >(npts, opdb, grid, tol3), "Three Point, Minus Sided, Z Direction, Divergence Operator" );
      status( test_field_variants<Gradient, OneSidedStencil3<ZPlus > >(npts, opdb, grid, tol3), "Three Point, Plus  Sided, Z Direction, Divergence Operator" );
    }

    if( status.ok() ){
      cout << "Tests passed" << endl;
      return 0;
    }
  }
  catch( std::exception& e ){
    cout << e.what() << endl;
  }

  cout << "******************************" << endl
      << "At least one test did not pass" << endl
      << "******************************" << endl;
  return -1;
}

//--------------------------------------------------------------------


/**
 * @file LemonTree.hpp
 * @brief Implementation of the class for learning module networks
 *        using the approach of Lemon Tree.
 * @author Ankit Srivastava <asrivast@gatech.edu>
 *
 * Copyright 2020 Georgia Institute of Technology
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef DETAIL_LEMONTREE_HPP_
#define DETAIL_LEMONTREE_HPP_

#include "Ganesh.hpp"
#include "Module.hpp"
#include "ConsensusCluster.hpp"

#include "mxx/distribution.hpp"

#include <boost/filesystem.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <trng/mrg3s.hpp>

// PRNG type to be used for generating random numbers
using PRNG = trng::mrg3s;

template <typename Data, typename Var, typename Set>
LemonTree<Data, Var, Set>::LemonTree(
  const mxx::comm& comm,
  const Data& data
) : ModuleNetworkLearning<Data, Var, Set>(comm, data)
{
  TIMER_RESET(m_tWrite);
  TIMER_RESET(m_tGanesh);
  TIMER_RESET(m_tConsensus);
  TIMER_RESET(m_tModules);
  TIMER_RESET(m_tSync);
}

template <typename Data, typename Var, typename Set>
/**
 * @brief Default destructor.
 */
LemonTree<Data, Var, Set>::~LemonTree(
)
{
  if (this->m_comm.is_first()) {
    TIMER_ELAPSED_NONZERO("Time taken in writing the files: ", m_tWrite);
    TIMER_ELAPSED_NONZERO("Time taken in the GaneSH run: ", m_tGanesh);
    TIMER_ELAPSED_NONZERO("Time taken in consensus clustering: ", m_tConsensus);
    TIMER_ELAPSED_NONZERO("Time taken in learning the modules: ", m_tModules);
    TIMER_ELAPSED_NONZERO("Time taken in synchronizing the modules: ", m_tSync);
  }
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
std::list<Set>
LemonTree<Data, Var, Set>::singleGaneshRun(
  Generator& generator,
  const pt::ptree& ganeshConfigs,
  const mxx::comm& comm
) const
{
  auto numSteps = ganeshConfigs.get<uint32_t>("num_steps");
  auto initClusters = ganeshConfigs.get<Var>("init_num_clust");
  if ((initClusters == 0) || (initClusters > this->m_data.numVars())) {
    initClusters = this->m_data.numVars() / 2;
  }
  Ganesh<Data, Var, Set> ganesh(this->m_data);
  ganesh.initializeRandom(generator, initClusters);
  for (auto s = 0u; s <= numSteps; ++s) {
    LOG_MESSAGE(info, "Step %u", s);
    ganesh.clusterTwoWay(generator, comm);
  }
  // XXX: Lemon Tree writes out only the last sampled cluster
  // and uses that for the downstream tasks per run
  // We can replicate this behavior by storing only the last sample in each run
  // Further, we do not use -burn_in and -sample_steps because they do not
  // have any effect on the outcome
  LOG_MESSAGE(info, "Sampling");
  const auto& primaryClusters = ganesh.primaryClusters();
  std::list<Set> varClusters;
  for (const auto& cluster : primaryClusters) {
    varClusters.push_back(cluster.elements());
  }
  return varClusters;
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
std::list<std::list<Set>>
LemonTree<Data, Var, Set>::clusterVarsGanesh(
  const pt::ptree& ganeshConfigs
) const
{
  auto randomSeed = ganeshConfigs.get<uint64_t>("seed");
  auto numRuns = ganeshConfigs.get<uint32_t>("num_runs");
  Generator generator;
  std::list<std::list<Set>> sampledClusters;
  if ((numRuns > 1) && (static_cast<uint32_t>(this->m_comm.size()) >= numRuns)) {
    // Split the communicator with one or more processes per run
    mxx::blk_dist commBlock(this->m_comm.size(), numRuns, 0);
    auto myRun = commBlock.rank_of(this->m_comm.rank());
    LOG_MESSAGE(info, "Run %u", myRun);
    auto runComm = this->m_comm.split(myRun);
    generator.seed(randomSeed + myRun);
    sampledClusters.resize(numRuns);
    auto myClusterIt = std::next(sampledClusters.begin(), myRun);
    *myClusterIt = this->singleGaneshRun(generator, ganeshConfigs, runComm);
    auto r = 0u;
    for (auto& cluster : sampledClusters) {
      auto clusterSize = cluster.size();
      mxx::bcast(clusterSize, commBlock.eprefix_size(r), this->m_comm);
      cluster.resize(clusterSize);
      std::vector<std::reference_wrapper<Set>> clusterRefs;
      std::transform(cluster.begin(), cluster.end(), std::back_inserter(clusterRefs),
                     [] (Set& c) { return std::ref(c); });
      set_bcast(clusterRefs, this->m_data.numVars(), commBlock.eprefix_size(r), this->m_comm);
      ++r;
    }
  }
  else {
    for (auto r = 0u; r < numRuns; ++r) {
      LOG_MESSAGE(info, "Run %u", r);
      // XXX: Seeding in this way to compare the results with Lemon-Tree;
      //      Otherwise, we can carry over generator state across runs
      generator.seed(randomSeed + r);
      auto varClusters = this->singleGaneshRun(generator, ganeshConfigs, this->m_comm);
      sampledClusters.push_back(varClusters);
    }
  }
  return sampledClusters;
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::writeVarClusters(
  const std::string& clusterFile,
  const std::list<std::list<Set>>& varClusters
) const
{
  LOG_MESSAGE(info, "Writing variable clusters to %s", clusterFile);
  std::ofstream gf(clusterFile);
  auto g = 0u;
  for (auto git = varClusters.begin(); git != varClusters.end(); ++git, ++g) {
    std::string clustersFile(clusterFile + "." + std::to_string(g));
    std::ofstream cf(clustersFile);
    auto c = 0u;
    for (auto cit = git->begin(); cit != git->end(); ++cit, ++c) {
      for (const auto e : *cit) {
        cf << this->m_data.varName(e) << "\t" << c << std::endl;
      }
    }
    gf << clustersFile << std::endl;
  }
}

template <typename Data, typename Var, typename Set>
std::multimap<Var, Var>
LemonTree<Data, Var, Set>::clusterConsensus(
  const std::list<std::list<Set>>&& varClusters,
  const pt::ptree& consensusConfigs
) const
{
  auto minWeight = consensusConfigs.get<double>("min_weight");
  auto tolerance = consensusConfigs.get<double>("tolerance", 1e-5);
  auto maxSteps = consensusConfigs.get<uint32_t>("max_steps", 1000);
  auto minClustSize = consensusConfigs.get<uint32_t>("min_clust_size");
  auto minClustScore = consensusConfigs.get<double>("min_clust_score");
  auto result = consensusCluster(std::move(varClusters), this->m_data.numVars(), minWeight,
                                 tolerance, maxSteps, minClustSize, minClustScore);
  return result;
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::writeConsensusCluster(
  const std::string& consensusFile,
  const std::multimap<Var, Var>& vertexClusters
) const
{
  LOG_MESSAGE(info, "Writing consensus clusters to %s", consensusFile);
  std::ofstream out(consensusFile);
  // Write output in tab seperated format
  for (const auto& cx : vertexClusters) {
    out << this->m_data.varName(cx.second) << "\t" << static_cast<uint32_t>(cx.first) << std::endl;
  }
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
std::list<std::list<Set>>
LemonTree<Data, Var, Set>::clusterObsGanesh(
  const uint32_t numRuns,
  const uint32_t numSteps,
  const uint32_t burnSteps,
  const uint32_t sampleSteps,
  Generator& generator,
  const Set& clusterVars
) const
{
  std::list<std::list<Set>> sampledClusters;
  // Initialize Gibbs sampler algorithm for this cluster
  Ganesh<Data, Var, Set> ganesh(this->m_data);
  ganesh.initializeGiven(generator, std::list<Set>(1, clusterVars));
  for (auto r = 0u; r < numRuns; ++r) {
    auto s = 0u;
    for ( ; s < burnSteps; ++s) {
      LOG_MESSAGE(info, "Step %u (burn in)", s);
      ganesh.clusterSecondary(generator);
    }
    for ( ; s < numSteps; ++s) {
      LOG_MESSAGE(info, "Step %u (sampling)", s);
      ganesh.clusterSecondary(generator);
      if ((numSteps - (s + 1)) % sampleSteps == 0) {
        LOG_MESSAGE(info, "Sampling");
        // There should be only one primary cluster; get a reference to it
        const auto& primaryCluster = ganesh.primaryClusters().front();
        const auto& secondaryClusters = primaryCluster.secondaryClusters();
        std::list<Set> obsClusters;
        for (const auto& cluster : secondaryClusters) {
          obsClusters.push_back(cluster.elements());
        }
        sampledClusters.push_back(obsClusters);
      }
    }
  }
  return sampledClusters;
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::readCandidateParents(
  const std::string& fileName,
  Set& candidateParents
) const
{
  LOG_MESSAGE(info, "Reading candidate parents from %s", fileName);
  std::ifstream regFile(boost::filesystem::canonical(fileName).string());
  std::string name;
  while (std::getline(regFile, name)) {
    auto v = this->m_data.varIndex(name);
    if (v < this->m_data.numVars()) {
      candidateParents.insert(v);
    }
  }
  LOG_MESSAGE(info, "Read %u candidate parents", candidateParents.size());
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
std::list<Module<Data, Var, Set>>
LemonTree<Data, Var, Set>::constructModulesWithTrees(
  const std::multimap<Var, Var>&& coClusters,
  Generator& generator,
  const pt::ptree& modulesConfigs
) const
{
  auto numRuns = modulesConfigs.get<uint32_t>("num_runs");
  auto numSteps = modulesConfigs.get<uint32_t>("num_steps");
  auto burnSteps = modulesConfigs.get<uint32_t>("burn_in");
  auto sampleSteps = modulesConfigs.get<uint32_t>("sample_steps");
  auto scoreBHC = modulesConfigs.get<bool>("use_bayesian_score", true);
  auto scoreGain = modulesConfigs.get<double>("score_gain");
  std::list<Module<Data, Var, Set>> modules;
  auto m = 0u;
  for (auto cit = coClusters.begin(); cit != coClusters.end(); ++m) {
    LOG_MESSAGE(info, "Module %u: Learning tree structures", m);
    // Get the range of variables in this cluster
    auto clusterIts = coClusters.equal_range(cit->first);
    // Add all the variables in the cluster to a set
    auto clusterVars = Set(this->m_data.numVars());
    for (auto vit = clusterIts.first; vit != clusterIts.second; ++vit) {
      clusterVars.insert(vit->second);
    }
    // Create a module for this variable cluster
    modules.emplace_back(std::move(clusterVars), this->m_comm, this->m_data);
    auto& module = modules.back();
    // Sample observation clusters for this module
    auto sampledClusters = this->clusterObsGanesh(numRuns, numSteps, burnSteps, sampleSteps,
                                                  generator, module.variables());
    // Learn tree structures from the observation clusters
    module.learnTreeStructures(std::move(sampledClusters), scoreBHC, scoreGain);
    cit = clusterIts.second;
  }
  return modules;
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
void
LemonTree<Data, Var, Set>::learnModulesParents(
  std::list<Module<Data, Var, Set>>& modules,
  Generator& generator,
  const pt::ptree& modulesConfigs
) const
{
  auto regFile = modulesConfigs.get<std::string>("reg_file");
  auto betaMax = modulesConfigs.get<double>("beta_reg");
  auto numSplits = modulesConfigs.get<uint32_t>("num_reg");
  Set candidateParents(this->m_data.numVars());
  if (!regFile.empty()) {
    // Read candidate parents from the given file
    this->readCandidateParents(regFile, candidateParents);
  }
  else {
    // Add all the variables as candidate parents
    for (Var v = 0u; v < candidateParents.max(); ++v) {
      candidateParents.insert(v);
    }
  }
  OptimalBeta ob(0.0, betaMax, 1e-5);
  auto m = 0u;
  for (auto moduleIt = modules.begin(); moduleIt != modules.end(); ++moduleIt, ++m) {
    LOG_MESSAGE(info, "Module %u: Learning parents", m);
    moduleIt->learnParents(generator, candidateParents, ob, numSplits);
  }
  LOG_MESSAGE(info, "Done learning module parents");
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
void
LemonTree<Data, Var, Set>::learnModulesParents_nodes(
  std::list<Module<Data, Var, Set>>& modules,
  Generator& generator,
  const Set&& candidateParents,
  const double betaMax,
  const uint32_t numSplits
) const
{
  OptimalBeta ob(0.0, betaMax, 1e-5);
  std::vector<uint32_t> moduleNodeCount(modules.size());
  std::vector<uint32_t> moduleNodeWeights;
  auto totalNodes = 0u;
  auto moduleCit = modules.cbegin();
  for (auto m = 0u; m < modules.size(); ++m, ++moduleCit) {
    moduleNodeCount[m] = moduleCit->nodeCount();
    auto nodeWeights = moduleCit->nodeWeights();
    moduleNodeWeights.insert(moduleNodeWeights.end(), nodeWeights.begin(), nodeWeights.end());
    totalNodes += moduleNodeCount[m];
  }
  std::vector<uint32_t> moduleNodeCountPrefix(moduleNodeCount.size());
  std::partial_sum(moduleNodeCount.cbegin(), moduleNodeCount.cend(), moduleNodeCountPrefix.begin());
  std::vector<uint32_t> moduleNodeWeightsPrefix(moduleNodeWeights.size());
  std::partial_sum(moduleNodeWeights.cbegin(), moduleNodeWeights.cend(), moduleNodeWeightsPrefix.begin());
  auto totalWeight = moduleNodeWeightsPrefix.back();
  mxx::blk_dist block(totalWeight, this->m_comm.size(), this->m_comm.rank());
  auto myFirstNode = std::distance(moduleNodeWeightsPrefix.cbegin(),
                                   std::lower_bound(moduleNodeWeightsPrefix.cbegin(),
                                                    moduleNodeWeightsPrefix.cend(),
                                                    block.eprefix_size()));
  if (!this->m_comm.is_first()) {
    myFirstNode += 1;
  }
  auto myLastNode = std::distance(moduleNodeWeightsPrefix.cbegin(),
                                  std::lower_bound(moduleNodeWeightsPrefix.cbegin(),
                                                   moduleNodeWeightsPrefix.cend(),
                                                   block.iprefix_size())) + 1;
  auto myNodeCount = myLastNode - myFirstNode;
  // First, advance the PRNG state to account for the number of
  // random generations for nodes on the previous ranks
  ::advance(generator, myFirstNode * 2 * numSplits);
  // XXX: We need to track the validity of nodes because some nodes may not learn any splits
  //      and that information will be required for synchornization later
  std::vector<uint8_t> myValidNodes(myNodeCount, 0);
  std::vector<std::tuple<Var, Var, double>> myNodeSplits(myNodeCount * 2 * numSplits);
  // Find the indices for the modules which contain the first and last node on this rank
  auto myFirstModule = std::distance(moduleNodeCountPrefix.cbegin(),
                                     std::lower_bound(moduleNodeCountPrefix.cbegin(),
                                                      moduleNodeCountPrefix.cend(),
                                                      myFirstNode + 1));
  auto myLastModule = std::distance(moduleNodeCountPrefix.cbegin(),
                                    std::lower_bound(moduleNodeCountPrefix.cbegin(),
                                                     moduleNodeCountPrefix.cend(),
                                                     myLastNode));
  auto moduleIt = std::next(modules.begin(), myFirstModule);
  auto validIt = myValidNodes.begin();
  auto splitIt = myNodeSplits.begin();
  auto prevNodeCount = myFirstNode;
  for (auto m = myFirstModule; m <= myLastModule; ++m, ++moduleIt) {
    auto firstNode = prevNodeCount - (moduleNodeCountPrefix[m] - moduleNodeCount[m]);
    auto numNodes = std::min(static_cast<uint32_t>(myLastNode), moduleNodeCountPrefix[m]) - prevNodeCount;
    if (numNodes > 0) {
      LOG_MESSAGE(info, "Module %u: Learning parents for %u nodes (starting node index = %u)", m, numNodes, firstNode);
      moduleIt->learnParents(generator, candidateParents, ob, numSplits, firstNode, numNodes, validIt, splitIt);
      prevNodeCount += numNodes;
    }
  }
  myNodeSplits.resize(std::distance(myNodeSplits.begin(), splitIt));
  LOG_MESSAGE(info, "Done learning module parents");
  this->m_comm.barrier();
  TIMER_START(m_tSync);
  // XXX: We can determine the number of elements on each processor for these calls
  //      and eliminate a gather call. Need to evaluate if it is worth the effort
  auto allValidNodes = mxx::allgatherv(myValidNodes, this->m_comm);
  auto allNodeSplits = mxx::allgatherv(myNodeSplits, this->m_comm);
  moduleIt = modules.begin();
  auto validCit = allValidNodes.cbegin();
  auto splitCit = allNodeSplits.cbegin();
  for (auto m = 0u; m < modules.size(); ++m, ++moduleIt) {
    // Synchronize the node parents for this module
    LOG_MESSAGE(info, "Module %u: Synchronizing parents for all nodes", m);
    moduleIt->syncParents(numSplits, validCit, splitCit);
  }
  LOG_MESSAGE(info, "Done synchronizing module parents");
  this->m_comm.barrier();
  TIMER_PAUSE(m_tSync);
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
void
LemonTree<Data, Var, Set>::learnModulesParents_splits(
  std::list<Module<Data, Var, Set>>& modules,
  Generator& generator,
  const Set&& candidateParents,
  const double betaMax,
  const uint32_t numSplits
) const
{
  TIMER_DECLARE(tCandidates);
  OptimalBeta ob(0.0, betaMax, 1e-5);
  std::vector<uint32_t> moduleNodeCount(modules.size());
  std::vector<uint64_t> moduleSplitWeight(modules.size());
  auto moduleCit = modules.cbegin();
  for (auto m = 0u; m < modules.size(); ++m, ++moduleCit) {
    moduleNodeCount[m] = moduleCit->nodeCount();
    moduleSplitWeight[m] = moduleCit->splitWeight(candidateParents);
  }
  std::vector<uint64_t> moduleNodeCountPrefix(moduleNodeCount.size());
  std::partial_sum(moduleNodeCount.cbegin(), moduleNodeCount.cend(), moduleNodeCountPrefix.begin());
  std::vector<uint64_t> moduleSplitWeightPrefix(moduleSplitWeight.size());
  std::partial_sum(moduleSplitWeight.cbegin(), moduleSplitWeight.cend(), moduleSplitWeightPrefix.begin());
  auto totalWeight = moduleSplitWeightPrefix.back();
  mxx::blk_dist block(totalWeight, this->m_comm.size(), this->m_comm.rank());
  auto myFirstModule = std::distance(moduleSplitWeightPrefix.cbegin(),
                                     std::upper_bound(moduleSplitWeightPrefix.cbegin(),
                                                      moduleSplitWeightPrefix.cend(),
                                                      block.eprefix_size()));
  auto myLastModule = std::distance(moduleSplitWeightPrefix.cbegin(),
                                    std::lower_bound(moduleSplitWeightPrefix.cbegin(),
                                                     moduleSplitWeightPrefix.cend(),
                                                     block.iprefix_size()));
  std::vector<std::tuple<uint32_t, Var, Var, double>> mySplits;
  auto moduleIt = std::next(modules.begin(), myFirstModule);
  auto prevWeight = block.eprefix_size();
  TIMER_DECLARE(tSplits);
  for (auto m = myFirstModule; m <= myLastModule; ++m, ++moduleIt) {
    auto firstNode = moduleNodeCountPrefix[m] - moduleNodeCount[m];
    auto firstWeight = prevWeight - (moduleSplitWeightPrefix[m] - moduleSplitWeight[m]);
    auto maxWeight = std::min(block.iprefix_size(), moduleSplitWeightPrefix[m]) - prevWeight;
    if (maxWeight > 0) {
      moduleIt->candidateParentsSplits(mySplits, candidateParents, ob, firstNode, firstWeight, maxWeight);
      prevWeight += maxWeight;
    }
  }
#if TIMER
  auto allSplitsTime = mxx::gather(static_cast<double>(tSplits.elapsed()), 0, this->m_comm);
  this->m_comm.barrier();
  if (this->m_comm.is_first()) {
    TIMER_ELAPSED("Time taken in the split computations: ", tSplits);
    auto totalTime = 0.0;
    auto maxTime = 0.0;
    for (const auto st : allSplitsTime) {
      totalTime += st;
      maxTime = std::max(maxTime, st);
    }
    auto avgTime = totalTime / allSplitsTime.size();
    auto imbalance = (maxTime - avgTime) / avgTime;
    std::cout << "Imbalance in the split computations: " << imbalance << std::endl;
  }
#endif
  // Redistribute to get the same number of candidate splits on every processor
  mxx::stable_distribute_inplace(mySplits, this->m_comm);
  this->m_comm.barrier();
  if (this->m_comm.is_first()) {
    TIMER_ELAPSED("Time taken in learning candidate splits: ", tCandidates);
  }
  TIMER_DECLARE(tChoose);
  // Compute the max score for each node across all the processors
  std::vector<double> mySplitsScoresMax(moduleNodeCountPrefix.back(), std::numeric_limits<double>::lowest());
  std::for_each(mySplits.cbegin(), mySplits.cend(),
                [&mySplitsScoresMax] (const std::tuple<uint32_t, Var, Var, double>& split)
                                     { mySplitsScoresMax[std::get<0>(split)] = std::max(mySplitsScoresMax[std::get<0>(split)],
                                                                                std::get<3>(split)); });
  auto allSplitsScoresMax = mxx::allreduce(mySplitsScoresMax, mxx::max<double>(), this->m_comm);
  // Set of local node indices
  // XXX: Using set instead of unordered_set because we want sorted indices
  std::set<uint32_t> myNodeIdx;
  // Count of node splits on this processor for all the nodes
  std::vector<uint64_t> mySplitsCounts(moduleNodeCountPrefix.back(), 0);
  // Vector of <node, weight> for each split, where weight = exp(score)
  std::vector<std::pair<uint32_t, double>> mySplitsWeights(mySplits.size());
  // Sum of weights of splits on this processor for all the nodes
  std::vector<double> mySplitsWeightsSum(moduleNodeCountPrefix.back(), 0);
  auto s = 0u;
  auto splitFirst = mySplits.cbegin();
  while (splitFirst != mySplits.cend()) {
    auto n = std::get<0>(*splitFirst);
    myNodeIdx.insert(n);
    auto splitLast = std::find_if(splitFirst, mySplits.cend(),
                                  [&n] (const std::tuple<uint32_t, Var, Var, double>& split)
                                       { return std::get<0>(split) != n; });
    mySplitsCounts[n] += std::distance(splitFirst, splitLast);
    auto nodeMaxScore = allSplitsScoresMax[n];
    for (auto it = splitFirst; it != splitLast; ++it, ++s) {
      auto weight = exp(std::get<3>(*it) - nodeMaxScore);
      mySplitsWeights[s] = std::make_pair(n, weight);
      mySplitsWeightsSum[n] += weight;
    }
    splitFirst = splitLast;
  }
  // Get the count of splits of the first node on previous processor(s)
  auto myLastNode = std::get<0>(mySplits.back());
  auto lastSplitsCount = std::make_pair(myLastNode, mySplitsCounts[myLastNode]);
  auto addSplitsCount = [] (const std::pair<uint32_t, uint64_t>& a,
                            const std::pair<uint32_t, uint64_t>& b)
                           { return (a.first == b.first) ?
                                    std::make_pair(b.first, a.second + b.second) : b; };
  auto firstCountsPrefix = mxx::exscan(lastSplitsCount, addSplitsCount, this->m_comm, false);
  // Get the total count of splits for all the nodes
  // Also get the sum of the weights of splits for each node on all the processors
  // XXX: We really only need these for the nodes on this processor
  //      Can we make it more efficient?
  auto allSplitsCounts = mxx::allreduce(mySplitsCounts, this->m_comm);
  auto allSplitsWeightsSum = mxx::allreduce(mySplitsWeightsSum, this->m_comm);
  // First, handle any nodes with infinite weights
  static auto findInf = [] (const double w) { return std::isinf(w); };
  auto infWeightIt = std::find_if(allSplitsWeightsSum.begin(), allSplitsWeightsSum.end(), findInf);
  while (infWeightIt != allSplitsWeightsSum.end()) {
    // We have found that the total weight of all the splits for this node is infinite.
    // Therefore, we want to modify the weights to replicate the following
    // sequential behaviors of discrete_distribution_safe/std::discrete_distribution
    // 1) If one or more of the split weights are infinite,
    //    then always pick the first split with infinite weight
    // 2) If none of the split weights are infinite,
    //    then always pick the last split
    auto n = std::distance(allSplitsWeightsSum.begin(), infWeightIt);
    if (myNodeIdx.find(n) != myNodeIdx.end()) {
      LOG_MESSAGE(debug, "Node %u: Handling infinite weights", n);
      static auto nodeSplit = [&n] (const std::pair<uint32_t, double>& split)
                                   { return split.first == n; };
      auto splitIt = std::find_if(mySplitsWeights.begin(), mySplitsWeights.end(), nodeSplit);
      while (splitIt->first == n) {
        // Set all infinite split weights to 1.0 and the finite weights to 0.0
        // This will accomplish the two requirements described above
        splitIt->second = std::isinf(splitIt->second) ? 1.0 : 0.0;
        ++splitIt;
      }
    }
    // Set the total weight for this node to 1.0
    *infWeightIt = 1.0;
    infWeightIt = std::find_if(std::next(infWeightIt), allSplitsWeightsSum.end(), findInf);
  }
  // Then, normalize the weights for all the local splits
  auto normalizeWeight = [&allSplitsWeightsSum] (std::pair<uint32_t, double>& split)
                                                { split.second /= allSplitsWeightsSum[split.first]; };
  std::for_each(mySplitsWeights.begin(), mySplitsWeights.end(), normalizeWeight);
  // Perform a segmented parallel prefix on this vector of <node, normalized weight>
  // with node defining the segment boundaries
  // This will compute the prefix sum of the normalized split weights belonging to every node
  std::vector<std::pair<uint32_t, double>> mySplitsWeightsPrefix(mySplitsWeights.size());
  auto addSplitsWeights = [] (const std::pair<uint32_t, double>& a,
                              const std::pair<uint32_t, double>& b)
                             { return (a.first == b.first) ?
                                      std::make_pair(b.first, a.second + b.second) : b; };
  mxx::global_scan(mySplitsWeights.begin(), mySplitsWeights.end(),
                   mySplitsWeightsPrefix.begin(), addSplitsWeights, false, this->m_comm);
  // Now, we can get the splits for the nodes on this processor
  trng::uniform01_dist<double> randDist;
  auto first = 0u;
  std::vector<std::tuple<uint32_t, Var, Var, double>> myChosenSplits(myNodeIdx.size() * 2 * numSplits);
  auto chosenIt = myChosenSplits.begin();
  auto g = 0u;
  for (const auto n : myNodeIdx) {
    if (n > g) {
      // Advance the PRNG state to account for the previous nodes
      ::advance(generator, (n - g) * 2 * numSplits);
      g = n;
    }
    ++g;
    auto nodeSplitsCount = mySplitsCounts[n];
    auto nodeSplitsCountsPrefix = (firstCountsPrefix.first == n) ? firstCountsPrefix.second : 0u;
    auto last = first + nodeSplitsCount - 1;
    auto nodeWeightLower = mySplitsWeightsPrefix[first].second - mySplitsWeights[first].second;
    // There may be some cases in which the last split for a node may
    // not have a cumulative weight of 1.0, because of the following reasons:
    // 1) If all the split weights were finite, but the sum of the weights was infinite
    // 2) Due to eccentricities of floating point arithmetic
    // In such cases, set it to 1.0
    if ((nodeSplitsCountsPrefix + nodeSplitsCount == allSplitsCounts[n]) &&
        std::isless(mySplitsWeightsPrefix[last].second, 1.0)) {
      mySplitsWeightsPrefix[last].second = 1.0;
    }
    auto nodeWeightUpper = mySplitsWeightsPrefix[last].second;
    trng::uniform_int_dist indexDist(0, allSplitsCounts[n]);
    LOG_MESSAGE(debug, "Node %u: Choosing the splits", n);
    for (auto i = 0u; i < numSplits; ++i) {
      // Pick a split weighted by its score
      auto rand = randDist(generator);
      // Check if the split is local to this processor
      if (std::isgreater(rand, nodeWeightLower) &&
          std::islessequal(rand, nodeWeightUpper)) {
        auto firstSplit = std::next(mySplitsWeightsPrefix.begin(), first);
        auto lastSplit = std::next(firstSplit, nodeSplitsCount);
        auto foundSplit = std::lower_bound(firstSplit, lastSplit, std::make_pair(n, rand));
        auto foundIdx = std::distance(firstSplit, foundSplit);
        auto weightSplit = mySplits[first + foundIdx];
        LOG_MESSAGE(debug, "Chosen parent split using weights: (%s, %g)",
                           this->m_data.varName(std::get<1>(weightSplit)),
                           this->m_data(std::get<1>(weightSplit), std::get<2>(weightSplit)));
        // Index the split with the split index for sorting all the splits for this node later
        *chosenIt = std::make_tuple(i, std::get<1>(weightSplit),
                                    std::get<2>(weightSplit), std::get<3>(weightSplit));
        ++chosenIt;
      }
      else {
        LOG_MESSAGE(debug, "Split for weight %g not in the range [%g, %g)", rand, nodeWeightLower, nodeWeightUpper);
      }
      // Pick a split uniformly at random
      auto randomIdx = static_cast<uint64_t>(indexDist(generator));
      // Check if the split index is local to this processor
      if ((randomIdx >= nodeSplitsCountsPrefix) &&
          (randomIdx < (nodeSplitsCountsPrefix + nodeSplitsCount))) {
        auto randomSplit = mySplits[first + (randomIdx - nodeSplitsCountsPrefix)];
        LOG_MESSAGE(debug, "Chosen parent split at random: (%s, %g)",
                           this->m_data.varName(std::get<1>(randomSplit)),
                           this->m_data(std::get<1>(randomSplit), std::get<2>(randomSplit)));
        // Index the split with the split index for sorting all the splits for this node later
        // Random splits are ordered after all the weight splits
        *chosenIt = std::make_tuple(numSplits + i, std::get<1>(randomSplit),
                                    std::get<2>(randomSplit), std::get<3>(randomSplit));
        ++chosenIt;
      }
      else {
        LOG_MESSAGE(debug, "Random split with index %u not on this processor", randomIdx);
      }
    }
    first += nodeSplitsCount;
  }
  myChosenSplits.resize(std::distance(myChosenSplits.begin(), chosenIt));
  this->m_comm.barrier();
  if (this->m_comm.is_first()) {
    TIMER_ELAPSED("Time taken in choosing splits: ", tChoose);
  }
  TIMER_START(m_tSync);
  // Gather all the chosen splits on all the processors
  // in order to assign them to the corresponding nodes
  // XXX: We can assign the splits for different nodes on different processors
  //      but we are trying to maintain the same state on every processor
  auto allChosenSplits = mxx::allgatherv(myChosenSplits, this->m_comm);
  moduleIt = modules.begin();
  auto countCit = allSplitsCounts.cbegin();
  auto splitIt = allChosenSplits.begin();
  for (auto m = 0u; m < modules.size(); ++m, ++moduleIt) {
    // Synchronize the node parents for this module
    LOG_MESSAGE(info, "Module %u: Synchronizing parents for all nodes", m);
    moduleIt->syncParents(numSplits, countCit, splitIt);
  }
  LOG_MESSAGE(info, "Done synchronizing module parents");
  this->m_comm.barrier();
  TIMER_PAUSE(m_tSync);
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
void
LemonTree<Data, Var, Set>::learnModulesParents_parallel(
  std::list<Module<Data, Var, Set>>& modules,
  Generator& generator,
  const pt::ptree& modulesConfigs
) const
{
  auto regFile = modulesConfigs.get<std::string>("reg_file");
  auto betaMax = modulesConfigs.get<double>("beta_reg");
  auto numSplits = modulesConfigs.get<uint32_t>("num_reg");
  Set candidateParents(this->m_data.numVars());
  if (!regFile.empty()) {
    if (this->m_comm.is_first()) {
      // Read candidate parents from the given file
      this->readCandidateParents(regFile, candidateParents);
    }
    set_bcast(candidateParents, 0, this->m_comm);
  }
  else {
    // Add all the variables as candidate parents
    for (Var v = 0u; v < candidateParents.max(); ++v) {
      candidateParents.insert(v);
    }
  }
  this->learnModulesParents_splits(modules, generator, std::move(candidateParents), betaMax, numSplits);
}

template <typename Data, typename Var, typename Set>
template <typename Generator>
std::list<Module<Data, Var, Set>>
LemonTree<Data, Var, Set>::learnModules(
  const std::multimap<Var, Var>&& coClusters,
  const pt::ptree& modulesConfigs,
  const bool isParallel
) const
{
  auto randomSeed = modulesConfigs.get<uint64_t>("seed");
  Generator generator(randomSeed);
  TIMER_DECLARE(tConstruct);
  auto modules = this->constructModulesWithTrees(std::move(coClusters), generator, modulesConfigs);
  this->m_comm.barrier();
  if (this->m_comm.is_first()) {
    TIMER_ELAPSED("Time taken in learning module trees: ", tConstruct);
  }
  TIMER_DECLARE(tParents);
  if (!isParallel) {
    this->learnModulesParents(modules, generator, modulesConfigs);
  }
  else {
    this->learnModulesParents_parallel(modules, generator, modulesConfigs);
  }
  this->m_comm.barrier();
  if (this->m_comm.is_first()) {
    TIMER_ELAPSED("Time taken in learning module parents: ", tParents);
  }
  return modules;
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::writeParents(
  std::ofstream& stream,
  const std::unordered_map<Var, double>& splits,
  const uint32_t moduleIndex,
  const double cutoff
) const
{
  std::set<std::pair<Var, double>, std::greater<std::pair<Var, double>>> splitsSorted(splits.begin(), splits.end());
  for (const auto& split : splitsSorted) {
    if (split.second > cutoff) {
      stream << this->m_data.varName(split.first) << "\t" << moduleIndex << "\t" << split.second << std::endl;
    }
  }
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::writeModules(
  const std::string& modulesFile,
  const std::list<Module<Data, Var, Set>>& modules,
  const double topParents
) const
{
  std::string allParentsFile = modulesFile + ".allreg.txt";
  LOG_MESSAGE(info, "Writing all parents to %s", allParentsFile);
  std::ofstream apf(allParentsFile);
  apf.precision(std::numeric_limits<double>::max_digits10);
  std::string topParentsFile = modulesFile + ".topreg.txt";
  LOG_MESSAGE(info, "Writing top 1%% parents to %s", topParentsFile);
  std::ofstream tpf(topParentsFile);
  tpf.precision(std::numeric_limits<double>::max_digits10);
  std::string randParentsFile = modulesFile + ".randomreg.txt";
  LOG_MESSAGE(info, "Writing random parents to %s", randParentsFile);
  std::ofstream rpf(randParentsFile);
  rpf.precision(std::numeric_limits<double>::max_digits10);
  std::string xmlFile = modulesFile + ".xml.gz";
  LOG_MESSAGE(info, "Writing modules to XML file %s", xmlFile);
  std::ofstream file(xmlFile, std::ios_base::out | std::ios_base::binary);
  boost::iostreams::filtering_streambuf<boost::iostreams::output> out;
  out.push(boost::iostreams::gzip_compressor());
  out.push(file);
  std::ostream xmlf(&out);
  xmlf.precision(std::numeric_limits<double>::max_digits10);

  xmlf << "<?xml version='1.0' encoding='iso-8859-1'?>" << std::endl;
  xmlf << "<ModuleNetwork>" << std::endl;
  // First compute the cutoff for top parents
  std::multiset<double, std::greater<double>> allScores;
  for (const auto& module : modules) {
    for (const auto& parent : module.allParents()) {
      allScores.insert(parent.second);
    }
  }
  auto index = static_cast<uint32_t>(round(topParents * allScores.size()));
  auto cutoff = *std::next(allScores.begin(), index);
  // Now, write the modules
  auto m = 0u;
  for (const auto& module : modules) {
    module.toXML(xmlf, m);
    this->writeParents(apf, module.allParents(), m);
    this->writeParents(tpf, module.allParents(), m, cutoff);
    this->writeParents(rpf, module.randParents(), m);
    ++m;
  }
  xmlf << "</ModuleNetwork>" << std::endl;
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::learnNetwork_sequential(
  const pt::ptree& algoConfigs,
  const std::string& outputDir
) const
{
  /* GaneSH clustering */
  if (algoConfigs.count("ganesh") == 0) {
    std::cerr << "WARNING: Skipping ganesh and other downstream tasks" << std::endl;
    return;
  }
  TIMER_START(m_tGanesh);
  const auto& ganeshConfigs = algoConfigs.get_child("ganesh");
  auto varClusters = this->clusterVarsGanesh<PRNG>(ganeshConfigs);
  TIMER_PAUSE(m_tGanesh);
  auto clusterFile = ganeshConfigs.get<std::string>("output_file");
  if (!clusterFile.empty()) {
    TIMER_START(m_tWrite);
    clusterFile = outputDir + "/" + clusterFile;
    this->writeVarClusters(clusterFile, varClusters);
    TIMER_PAUSE(m_tWrite);
  }

  /* Consensus clustering */
  if (algoConfigs.count("tight_clusters") == 0) {
    std::cerr << "WARNING: Skipping tight_clusters and other downstream tasks" << std::endl;
    return;
  }
  TIMER_START(m_tConsensus);
  const auto& consensusConfigs = algoConfigs.get_child("tight_clusters");
  auto coClusters = this->clusterConsensus(std::move(varClusters), consensusConfigs);
  TIMER_PAUSE(m_tConsensus);
  auto consensusFile = consensusConfigs.get<std::string>("output_file");
  if (!consensusFile.empty()) {
    TIMER_START(m_tWrite);
    consensusFile = outputDir + "/" + consensusFile;
    this->writeConsensusCluster(consensusFile, coClusters);
    TIMER_PAUSE(m_tWrite);
  }

  /* Module learning */
  if (algoConfigs.count("regulators") == 0) {
    std::cerr << "WARNING: Skipping regulators and other downstream tasks" << std::endl;
    return;
  }
  TIMER_START(m_tModules);
  const auto& modulesConfigs = algoConfigs.get_child("regulators");
  auto modules = this->learnModules<PRNG>(std::move(coClusters), modulesConfigs);
  TIMER_PAUSE(m_tModules);
  auto modulesFile = modulesConfigs.get<std::string>("output_file");
  auto topParents = 0.01;
  if (!modulesFile.empty()) {
    TIMER_START(m_tWrite);
    modulesFile = outputDir + "/" + modulesFile;
    this->writeModules(modulesFile, modules, topParents);
    TIMER_PAUSE(m_tWrite);
  }
}

template <typename Data, typename Var, typename Set>
void
LemonTree<Data, Var, Set>::learnNetwork_parallel(
  const pt::ptree& algoConfigs,
  const std::string& outputDir
) const
{
  /* GaneSH clustering */
  if (algoConfigs.count("ganesh") == 0) {
    if (this->m_comm.is_first()) {
      std::cerr << "WARNING: Skipping ganesh and other downstream tasks" << std::endl;
    }
    return;
  }
  TIMER_START(m_tGanesh);
  const auto& ganeshConfigs = algoConfigs.get_child("ganesh");
  auto varClusters = this->clusterVarsGanesh<PRNG>(ganeshConfigs);
  this->m_comm.barrier();
  TIMER_PAUSE(m_tGanesh);
  auto clusterFile = ganeshConfigs.get<std::string>("output_file");
  if (!clusterFile.empty() && this->m_comm.is_first()) {
    TIMER_START(m_tWrite);
    clusterFile = outputDir + "/" + clusterFile;
    this->writeVarClusters(clusterFile, varClusters);
    TIMER_PAUSE(m_tWrite);
  }
  this->m_comm.barrier();

  /* Consensus clustering */
  if (algoConfigs.count("tight_clusters") == 0) {
    if (this->m_comm.is_first()) {
      std::cerr << "WARNING: Skipping tight_clusters and other downstream tasks" << std::endl;
    }
    return;
  }
  TIMER_START(m_tConsensus);
  const auto& consensusConfigs = algoConfigs.get_child("tight_clusters");
  auto coClusters = this->clusterConsensus(std::move(varClusters), consensusConfigs);
  TIMER_PAUSE(m_tConsensus);
  auto consensusFile = consensusConfigs.get<std::string>("output_file");
  if (!consensusFile.empty() && this->m_comm.is_first()) {
    TIMER_START(m_tWrite);
    consensusFile = outputDir + "/" + consensusFile;
    this->writeConsensusCluster(consensusFile, coClusters);
    TIMER_PAUSE(m_tWrite);
  }
  this->m_comm.barrier();

  /* Module learning */
  if (algoConfigs.count("regulators") == 0) {
    if (this->m_comm.is_first()) {
      std::cerr << "WARNING: Skipping regulators and other downstream tasks" << std::endl;
    }
    return;
  }
  TIMER_START(m_tModules);
  const auto& modulesConfigs = algoConfigs.get_child("regulators");
  auto modules = this->learnModules<PRNG>(std::move(coClusters), modulesConfigs, true);
  this->m_comm.barrier();
  TIMER_PAUSE(m_tModules);
  auto modulesFile = modulesConfigs.get<std::string>("output_file");
  auto topParents = 0.01;
  if (!modulesFile.empty() && this->m_comm.is_first()) {
    TIMER_START(m_tWrite);
    modulesFile = outputDir + "/" + modulesFile;
    this->writeModules(modulesFile, modules, topParents);
    TIMER_PAUSE(m_tWrite);
  }
}

#endif // DETAIL_LEMONTREE_HPP_


//
// Created by philipp on 26.12.19.
//

#ifndef FUNNELS_CPP_DYNAMICS_HH
#define FUNNELS_CPP_DYNAMICS_HH

#include <Eigen/Core>
#include <cmath>
#include <cassert>
#include <memory>

using namespace Eigen;

namespace dynamics {

  class kinematic_2d_sys_t {
  public:
    constexpr static const size_t dimx = 4, dimp = 2, dimv = 2, dimu = 2;

    using matrix_t = Eigen::Matrix<double, dimx, Eigen::Dynamic>;
    using vector_x_t = Eigen::Matrix<double, dimx, 1>;
    using vector_u_t = Eigen::Matrix<double, dimu, 1>;
    using vector_t_t = Eigen::Matrix<double, Eigen::Dynamic, 1>;
    using matrix_ptr_t = std::shared_ptr<matrix_t>;
    using vector_x_ptr_t = std::shared_ptr<vector_x_t>;
    using vector_u_ptr_t = std::shared_ptr<vector_u_t>;
    using vector_t_ptr_t = std::shared_ptr<vector_t_t>;

    static void
    compute(matrix_t &x, const vector_u_t &u, const vector_t_t &t);
  };
}
#endif //FUNNELS_CPP_DYNAMICS_HH


/**
 * @author Samuel Larkin based on George Foster, based on "rescore" by Aaron Tikuisis
 * @file rescore_translate.cc
 * @brief Pick the best translation for a source given an nbest list a feature
 * functions weights vector.
 *
 *
 * Technologies langagieres interactives / Interactive Language Technologies
 * Inst. de technologie de l'information / Institute for Information Technology
 * Conseil national de recherches Canada / National Research Council Canada
 * Copyright 2004, Sa Majeste la Reine du Chef du Canada /
 * Copyright 2004, Her Majesty in Right of Canada
 */

#include "rescore_translate.h"
#include "featurefunction_set.h"
#include "rescore_io.h"

#include "exception_dump.h"
#include "rescoring_general.h"
#include "printCopyright.h"
#include "bleu.h"
#include "translationReader.h"
#include "file_utils.h"

#include <queue>
#include <stack>
#include <sstream>
#include <boost/scoped_ptr.hpp>

using namespace Portage;
using namespace std;

// Assuming v is a vector of logprobs, return log of the sum of the
// probabilities in the vector.
static double logNorm(const uVector& v)
{
   double sum = 0.0;
   double max = 0.0;
   for (Uint i = 0; i < v.size(); ++i)
      if (v[i] < max) max = v[i];
   for (Uint i = 0; i < v.size(); ++i)
      sum += exp(v[i] - max);
   return log(sum) + max;
}

// Write to os the vector of features for the kth hypothesis in feature matrix
// H, containing M features.
static void writeFeatureVector(const uMatrix& H, Uint M, Uint k, ostream& os)
{
   for (Uint m = 0; m < M; ++m) {
      if (m) os << '\t';
      os << (isfinite(H(k,m)) ? H(k,m) : -9999);
   }
}

////////////////////////////////////////////////////////////////////////////////
// main
int MAIN(argc, argv)
{
   printCopyright(2004, "rescore_translate");
   using namespace rescore_translate;
   ARG arg(argc, argv);

   ////////////////////////////////////////
   // SOURCE
   LOG_VERBOSE2(verboseLogger, "Reading source file");
   Sentences  sources;
   const Uint S(RescoreIO::readSource(arg.src_file, sources));
   if (S == 0) error(ETFatal, "empty source file: %s", arg.src_file.c_str());

   ////////////////////////////////////////
   // DUMP FF MATRIX MODE
   oMagicStream ffmatrix, nbestmatrix, bleumatrix;
   bool dump_for_mira = false;
   if (!arg.dump_for_mira.empty()) {
      dump_for_mira = true;
      string ffvalsfiles(arg.dump_for_mira + ".allffvals.gz");
      string nbestfile(arg.dump_for_mira + ".allnbest.gz");
      string dummybleufile(arg.dump_for_mira + ".allbleus.gz");
      LOG_VERBOSE2(verboseLogger, "Writing FF values matrix to %s", ffvalsfiles.c_str());
      LOG_VERBOSE2(verboseLogger, "Writing variable length N-best lists to %s", nbestfile.c_str());
      LOG_VERBOSE2(verboseLogger, "Writing dummy allbleus file %s", dummybleufile.c_str());
      ffmatrix.safe_open(ffvalsfiles);
      ffmatrix.precision(9);
      nbestmatrix.safe_open(nbestfile);
      bleumatrix.safe_open(dummybleufile);
   }

   ////////////////////////////////////////
   // FEATURE FUNCTION SET
   LOG_VERBOSE2(verboseLogger, "Reading ffset");
   FeatureFunctionSet ffset(dump_for_mira);
   ffset.read(arg.model, arg.bVerbose, arg.ff_pref.c_str(), arg.bIsDynamic);
   ffset.createTgtVocab(sources, FileReader::create<Translation>(arg.nbest_file, arg.K));
   uVector wts(ffset.M());
   ffset.getWeights(wts);
   {
      ostringstream oss;
      oss << wts;
      LOG_VERBOSE2(verboseLogger, "Rescoring using weights: %s", oss.str().c_str());
   }

   LOG_VERBOSE2(verboseLogger, "Initializing FF matrix with %d source sentences", S);
   ffset.initFFMatrix(sources);

   ////////////////////////////////////////
   // Create a NBest reader that will also read alignments if needed.
   LOG_VERBOSE2(verboseLogger, "Creating NBest reader");
   NbestReader  nbestReader(FileReader::createT(arg.nbest_file, arg.alignment_file, arg.K));

   if (ffset.requires() & FF_NEEDS_ALIGNMENT) {
      if (arg.alignment_file.empty())
         error(ETFatal, "At least one of feature function requires the alignment");
   }

   boost::scoped_ptr<istream> co_in;
   boost::scoped_ptr<ostream> co_out;
   if (arg.cofile != "") {
      co_in.reset(new iSafeMagicStream(arg.cofile));
      co_out.reset(new oSafeMagicStream(addExtension(arg.cofile, ".resc")));
   }
   vector<string> colines(arg.K);
   boost::scoped_ptr<ostream> fv_out;
   if (arg.fvfile != "")
      fv_out.reset(new oSafeMagicStream(arg.fvfile));
   boost::scoped_ptr<ostream> sc_out;
   if (arg.scfile != "")
      sc_out.reset(new oSafeMagicStream(arg.scfile));

   LOG_VERBOSE2(verboseLogger, "Processing Nbest lists");

   if (arg.bVerbose) {
      if (arg.bMbr)
         cerr << "Minimum Bayes risk (MBR) rescoring, based on BLEU, only 1-best output!" << endl;
      else
         cerr << "Maximum a-posteriori (MAP) rescoring (default behaviour)" << endl;
   }

   Uint s(0);
   for (; nbestReader->pollable(); ++s)
   {
      Nbest  nbest;
      nbestReader->poll(nbest);
      Uint K(nbest.size());

      LOG_VERBOSE3(verboseLogger, "Computing FF matrix");
      uMatrix H;
      ffset.computeFFMatrix(H, s, nbest);
      if (dump_for_mira) {
         static Uint M = ffset.M(); // number of features
         for (Uint k = 0; k < nbest.size(); ++k) {
            ffmatrix << s << '\t';
            writeFeatureVector(H, M, k, ffmatrix);
            ffmatrix << nf_endl;
            nbestmatrix << s << '\t' << nbest[k] << nf_endl;
            bleumatrix << s << ' ' << 0 << nf_endl;
         }
         continue;
      }

      K = nbest.size();  // IMPORTANT K might change if computeFFMatrix detects empty lines

      if (co_in) {
         for (Uint i = 0; i < colines.size(); ++i)
            if (!getline(*co_in, colines[i]))
               error(ETFatal, "file %s too short", arg.cofile.c_str());
      }

      if (!arg.bMbr) { // maximum a-posteriori rescoring
         if (arg.kout == 1) {      // output single best hypothesis
            if (K==0) {
               cout << endl;
               if (co_out) (*co_out) << endl;
               if (fv_out) (*fv_out) << nf_endl;
               if (sc_out) (*sc_out) << nf_endl;
            } else {
               const uVector Scores = boost::numeric::ublas::prec_prod(H, wts);
               const double logz = arg.conf_scores ? logNorm(Scores) : 0.0;
               const Uint bestIndex = my_vector_max_index(Scores);         // k = sentence which is scored highest
               if (arg.bPrintRank) cout << bestIndex+1 << " ";
               if (arg.print_scores) cout << Scores[bestIndex] - logz << " ";
               cout << nbest.at(bestIndex) << endl;  // DISPLAY best hypothesis
               if (co_out) (*co_out) << colines[bestIndex] << endl;
               if (fv_out) {
                  static Uint M = ffset.M();
                  writeFeatureVector(H, M, bestIndex, (*fv_out));
                  (*fv_out) << nf_endl;
               }
               if (sc_out) (*sc_out) << Scores[bestIndex] - logz << nf_endl;
            }
         } else {                  // ouptput k best hypotheses
            const uVector Scores = boost::numeric::ublas::prec_prod(H, wts);
            const double logz = arg.conf_scores ? logNorm(Scores) : 0.0;
            priority_queue< pair<float, Uint> > best_scores;
            Uint kout = (arg.kout == 0) ? K : min(arg.kout, K);
            for (Uint i = 0; i < Scores.size(); ++i) { // put best kout into pq
               best_scores.push(make_pair(-Scores[i], i));
               if (best_scores.size() > kout) best_scores.pop();
            }
            stack < pair<float, Uint> > best_scores_stack;
            while (!best_scores.empty()) { // reverse order of queue
               best_scores_stack.push(best_scores.top());
               best_scores.pop();
            }
            while (!best_scores_stack.empty()) { // output
               pair<float, Uint>& p = best_scores_stack.top();
               if (arg.bPrintRank) cout << p.second+1 << " ";
               if (arg.print_scores) cout << Scores[p.second] - logz << " ";
               cout << nbest.at(p.second) << endl;  // DISPLAY best hypothesis
               if (co_out) (*co_out) << colines[p.second] << endl;
               if (fv_out) {
                  static Uint M = ffset.M();
                  writeFeatureVector(H, M, p.second, (*fv_out));
                  (*fv_out) << nf_endl;
               }
               if (sc_out) (*sc_out) << Scores[p.second] - logz << nf_endl;
               best_scores_stack.pop();
            }
            if (arg.K)             // pad with blank lines if necessary
               for (; kout < arg.kout; ++kout) {
                  cout << endl;
                  if (co_out) (*co_out) << endl;
                  if (fv_out) (*fv_out) << nf_endl;
                  if (sc_out) (*sc_out) << nf_endl;
               }
         }
      } else { // Minimum Bayes risk rescoring using BLEU as loss function

        // Sort hypotheses by sentence score
        const uVector Scores = boost::numeric::ublas::prec_prod(H, wts) * arg.glob_scale;
        if (arg.bVerbose) cerr << "Scaling all sentence scores with " << arg.glob_scale << endl;

        map<float, Uint> best_scores;
        for (Uint i = 0; i < Scores.size(); ++i) // put all (score,index) pairs into map
          best_scores.insert(make_pair(-Scores[i], i));

        // Consider only the <kmbr> top hypotheses for MBR
        const Uint kmbr = arg.kmbr == 0 ? K : min(arg.kmbr, K);

        // Determine total prob. mass of op <kmbr> hypotheses for normalization
        Uint n=0;
        double sum = 0.0;
        const double minCost = best_scores.begin()->first;
        if (arg.bVerbose) cerr << "min costs (max.prob.): " << minCost << endl;
        for (map<float, Uint>::const_iterator itr=best_scores.begin();
             itr!=best_scores.end() && ++n<=kmbr; itr++) {
          sum += exp(minCost - itr->first);
          if (arg.bVerbose) cerr << "sum " << itr->second << ": " << itr->first << "->" << sum << endl;
        }
        const double lognorm = minCost - log(sum);
        if (arg.bVerbose)
          cerr << "==============================" << endl
               << "Normalizing everything by logprob. " << lognorm << endl;

        // Determine MBR hypotheses based on BLEU
        if (K==0)
          cout << endl;

        priority_queue< pair<float, Uint> > minRisks;

        double minRisk = INFINITY;
        Uint bestIndex = K+1;
        Uint n1=0;

        // Use each hyp. as 'candidate' in BLEU calculation
        for (map<float, Uint>::const_iterator i1=best_scores.begin();
             i1!=best_scores.end() && ++n1<=kmbr; i1++) {
          Tokens hyp1 = nbest[i1->second].getTokens();
          double currentRisk = 0;
          if (arg.bVerbose) {
            cerr << "===============" << endl
                 << "Hyp " << i1->second << " with score " << i1->first << ": ";
            for (Tokens::const_iterator itr=hyp1.begin(); itr!=hyp1.end(); itr++)
              cerr << *itr << " ";
            cerr << endl << endl;
          }
          Uint n2 = 0;
          // Use each other hyp. as 'reference'
          for (map<float, Uint>::const_iterator i2=best_scores.begin();
               i2!=best_scores.end() && ++n2<=kmbr; i2++) {

            if (i1->second != i2->second) {
              vector<Tokens> refs;
              refs.push_back(nbest[i2->second].getTokens());
              BLEUstats bleu(hyp1, refs, arg.smooth_bleu);
              currentRisk += exp(lognorm-i2->first) * (1.0-exp(bleu.score()));

              if (arg.bVerbose) {
                cerr << "  Hyp " << i2->second << " with score " << i2->first << ": ";
                for (Tokens::const_iterator itr=refs[0].begin(); itr!=refs[0].end(); itr++)
                  cerr << *itr << " ";
                cerr << endl << "BLEU " << exp(bleu.score())
                     << " norm.costs/prob. " << lognorm-i2->first << "/" << exp(lognorm-i2->first)
                     << " -> current/summed risk " << exp(lognorm-i2->first) * (1.0-exp(bleu.score()))
                     << " / " << currentRisk << endl << endl;
              }
              if (currentRisk > minRisk)
                break;
            }
          } // for i2
          if (currentRisk < minRisk) {
            minRisk = currentRisk;
            bestIndex = i1->second;
            if (arg.bVerbose)
              cerr << "minimal risk so far: " << minRisk << endl;
          }
        } // for i1
        // output
        if (arg.bPrintRank) cout << bestIndex+1 << " ";
        if (arg.print_scores) cout << minRisk << " ";
        cout << nbest.at(bestIndex) << endl;  // DISPLAY best hypothesis

      } // else (MBR)
   }

   nbestReader.reset();
   if (s != S) error(ETFatal, "File inconsistency s=%d, S=%d", s, S);

} END_MAIN


#include "many_characters.h"
#include "../components/character_input.h"
#include "../components/robot.h"
#include "../components/robot_spawner.h"
#include "../system/robot_system.h"
#include <boost/geometry.hpp>
#include <iostream>
#include <lmlib/intersection.h>
#include <lmlib/math_constants.h>
#include <lmng/animation.h>
#include <lmng/archetype.h>
#include <lmng/camera.h>
#include <lmng/kinematic.h>
#include <lmng/name.h>
#include <lmng/simulation.h>
#include <lmng/transform.h>
#include <yaml-cpp/yaml.h>

many_characters::many_characters(lmng::simulation_init const &init)
    : physics{lmng::create_physics(init.registry)},
      camera{init.registry.create()},
      animation_system{},
      hierarchy_system{init.registry},
      ground{lmng::find_entity(init.registry, "Ground")},
      swing_arms_animation{
        init.asset_cache.load<lmng::animation_data>("animation/swing_arms")}
{
    init.registry.emplace<lmng::transform>(camera);
    init.registry.emplace<lmng::camera>(camera, 1.1f, 0.1f, 1000.f, true);

    spawn_robots(init);
}

void many_characters::spawn_robots(lmng::simulation_init const &init) const
{
    for (auto [entity, spawner, spawner_transform] :
         init.registry.view<robot_spawner_component, lmng::transform>().proxy())
    {
        float start{spawner.rows * spawner.spacing / -2};
        float xpos{start};
        for (auto row : lm::range(spawner.rows))
        {
            float zpos{start};
            for (auto column : lm::range(spawner.columns))
            {
                lmng::instantiate_archetype(
                  init.registry,
                  "archetype/robot",
                  init.asset_cache,
                  lmng::transform{
                    .position = spawner_transform.position +
                                Eigen::Vector3f{xpos, 0.f, zpos}});
                zpos += spawner.spacing;
            }
            xpos += spawner.spacing;
        }
    }
}

void many_characters::handle_input_event(
  lmtk::input_event const &input_event,
  entt::registry &registry)
{
}

void many_characters::update(
  entt::registry &registry,
  float dt,
  lmtk::input_state const &input_state)
{
    move_robots(registry, ground);

    control_robots_animation(
      registry, physics, animation_system, swing_arms_animation);

    physics->step(registry, dt);

    animation_system.update(registry, dt);
}


/**
 * @file common.hpp
 * @author Julian Viereck (jviereck@tuebingen.mpg.de)
 * license License BSD-3-Clause
 * @copyright Copyright (c) 2020, New York University and Max Planck
 * Gesellschaft.
 * @date 2020-11-27
 *
 * @brief Common definitions used across the library.
 */

#include <Eigen/Dense>

namespace odri_control_interface
{
typedef Eigen::VectorXd VectorXd;
typedef Eigen::Matrix<int, Eigen::Dynamic, 1> VectorXi;
typedef Eigen::Matrix<long, Eigen::Dynamic, 1> VectorXl;
typedef Eigen::Matrix<bool, Eigen::Dynamic, 1> VectorXb;

typedef Eigen::Ref<Eigen::Matrix<int, Eigen::Dynamic, 1> > RefVectorXi;
typedef Eigen::Ref<Eigen::Matrix<long, Eigen::Dynamic, 1> > RefVectorXl;
typedef Eigen::Ref<Eigen::Matrix<bool, Eigen::Dynamic, 1> > RefVectorXb;
typedef Eigen::Ref<Eigen::Matrix<double, Eigen::Dynamic, 1> > RefVectorXd;
typedef Eigen::Ref<Eigen::Vector3d> RefVector3d;
typedef Eigen::Ref<Eigen::Vector4d> RefVector4d;

typedef const Eigen::Ref<const Eigen::Matrix<int, Eigen::Dynamic, 1> >
    ConstRefVectorXi;
typedef const Eigen::Ref<const Eigen::Matrix<long, Eigen::Dynamic, 1> >
    ConstRefVectorXl;
typedef const Eigen::Ref<const Eigen::Matrix<bool, Eigen::Dynamic, 1> >
    ConstRefVectorXb;
typedef const Eigen::Ref<const Eigen::Matrix<double, Eigen::Dynamic, 1> >
    ConstRefVectorXd;
typedef const Eigen::Ref<const Eigen::Vector3d> ConstRefVector3d;
typedef const Eigen::Ref<const Eigen::Vector4d> ConstRefVector4d;

}  // namespace odri_control_interface


#include "test.h"
#include "vecmath.h"
#include "mesh.h"
#include "mtl_diffuse.h"
#include <boost/filesystem.hpp>

void create_floor(wyc::CMesh* mesh, float r, float w=1)
{
	using namespace wyc;
	struct Vertex {
		wyc::vec3f pos;
		wyc::vec4f color;
		wyc::vec2f uv;
	};
	VertexAttrib attrib_array[] = {
		{ATTR_POSITION, 3, offsetof(Vertex, pos)},
		{ATTR_COLOR, 4, offsetof(Vertex, color)},
		{ATTR_UV0, 2, offsetof(Vertex, uv)},
	};
	Vertex verts[4] = {
		{ {-r, 0,  r },{ 1, 1, 1, 1 },{ 0, 0 } },
		{ { r, 0,  r },{ 1, 1, 1, 1 },{ w, 0 } },
		{ { r, 0, -r },{ 1, 1, 1, 1 },{ w, w } },
		{ {-r, 0, -r },{ 1, 1, 1, 1 },{ 0, w } },
	};
	mesh->set_vertices(attrib_array, 3, verts, 4);
	mesh->set_indices({
		0, 1, 2, 0, 2, 3,
	});
}

bool generate_mipmap(const std::string &image_file)
{
	auto base_img = std::make_shared<wyc::CImage>();
	boost::filesystem::path file_path(image_file);
	if (!base_img->load(image_file)) {
		return false;
	}
	std::vector<std::shared_ptr<wyc::CImage>> mipmap_chain;
	mipmap_chain.push_back(base_img);
	if (wyc::CImage::generate_mipmap(mipmap_chain))
	{
		log_info("create mipmap OK");
		int lod = 0;
		boost::filesystem::path base_name = file_path.stem(), file_path;
		boost::filesystem::create_directory(base_name);
		std::string ext = ".png";
		for (auto img : mipmap_chain)
		{

			file_path = base_name / std::to_string(lod);
			file_path += ext;
			img->save(file_path.string());
			lod += 1;
		}
	}
	else {
		log_error("fail to create mipmap!");
	}
	return true;
}

class CMaterialDiffuseMipmap : public CMaterialDiffuse
{
	UNIFORM_MAP{
		UNIFORM_SLOT(wyc::mat4f, proj_from_world)
		UNIFORM_SLOT(wyc::CSampler*, diffuse)
		UNIFORM_SLOT(unsigned, texture_size)
		UNIFORM_MAP_END
	};

public:
	CMaterialDiffuseMipmap()
		: CMaterialDiffuse()
		, texture_size(0)
	{
	}

	virtual bool fragment_shader(const void *frag_in, wyc::color4f &frag_color, wyc::CShaderContext *ctx) const override
	{
		auto in = reinterpret_cast<const VertexOut*>(frag_in);
		wyc::color4f diffuse_color;
		auto duvdx = ctx->ddx(&VertexOut::uv);
		auto duvdy = ctx->ddy(&VertexOut::uv);
		float r1 = std::sqrtf(duvdx ^ duvdx);
		float r2 = std::sqrtf(duvdy ^ duvdy);
		union {
			float f;
			unsigned i;
		} j;
		j.f = std::max(r1, r2) * texture_size;
		int e = (j.i & 0x7f800000) >> 23;
		if (e > 127) {
			e -= 127;
			int m = j.i & 0x7fffff;
			float f = float(m) / (1 << 23);
			wyc::color4f c1, c2;
			diffuse->sample2d(in->uv, e, c1);
			diffuse->sample2d(in->uv, e-1, c2);
			diffuse_color = c1 * f + c2 * (1 - f);
		}
		else
			diffuse->sample2d(in->uv, diffuse_color);
		frag_color = diffuse_color * in->color;
		return true;
	}
protected:
	unsigned texture_size;
};

class CTestMipmap : public CTest
{
public:
	virtual void run() {
		//generate_mipmap("res/checkerboard.png");
		//return;

		auto mesh = std::make_shared<wyc::CMesh>();
		create_floor(mesh.get(), 16, 32);

		// setup transform
		wyc::mat4f proj;
		wyc::set_perspective(proj, 45, float(m_image_w) / m_image_h, 1, 1000);
		wyc::mat4f rx_world, ry_world, transform_world;
		wyc::set_rotate_x(rx_world, wyc::deg2rad(4));
		wyc::set_rotate_y(ry_world, wyc::deg2rad(16));
		wyc::set_translate(transform_world, 0, -0.2f, -10);
		wyc::mat4f proj_from_world;
		proj_from_world = proj * transform_world * rx_world * ry_world;

		// setup material
		auto mtl = std::make_shared<CMaterialDiffuseMipmap>();
		//auto mtl = std::make_shared<CMaterialDiffuse>();
		mtl->set_uniform("proj_from_world", proj_from_world);
		// sampler
		auto diffuse_img = std::make_shared<wyc::CImage>();
		diffuse_img->load("res/checkerboard.png");
		//diffuse_img->create_checkerboard(64, {1, 1, 1}, {0, 0, 0});
		std::vector<decltype(diffuse_img)> mipmap_images = { diffuse_img };
		if (!diffuse_img->generate_mipmap(mipmap_images)) {
			log_error("create mipmap error");
			return;
		}
		auto sampler = std::make_shared<wyc::CSpwMipmapSampler>(mipmap_images);
		mtl->set_uniform("diffuse", (wyc::CSampler*)sampler.get());
		mtl->set_uniform("texture_size", diffuse_img->width());

		// draw
		auto draw = m_renderer->new_command<wyc::cmd_draw_mesh>();
		draw->mesh = mesh.get();
		draw->material = mtl.get();
		m_renderer->enqueue(draw);
		m_renderer->process();

		// save result
		save_image("mipmap.png");

	}
};
REGISTER_TEST(CTestMipmap)


#include <boost/hash2/fnv1a.hpp>
#include <boost/hash2/siphash.hpp>
#include <boost/hash2/xxhash.hpp>
#include <boost/hash2/spooky2.hpp>
#include <boost/hash2/murmur3.hpp>
#include <boost/hash2/md5.hpp>
#include <boost/hash2/sha1.hpp>
#include <boost/hash2/hash_append.hpp>
#include <boost/hash2/get_integral_result.hpp>
#include <boost/chrono.hpp>
#include <boost/core/demangle.hpp>
#include <vector>
#include <cstdio>
#include <typeinfo>

template<class H> void test_( unsigned char const * p, int N, int M )
{
    H h;

    typedef boost::chrono::steady_clock clock_type;

    clock_type::time_point t1 = clock_type::now();

    for( int i = 0; i < M; ++i )
    {
        hash_append_range( h, p, p + N );
    }

    clock_type::time_point t2 = clock_type::now();

    long long ms = boost::chrono::duration_cast<boost::chrono::milliseconds>( t2 - t1 ).count();

    using boost::hash2::get_integral_result;
    unsigned r = get_integral_result<unsigned>( h.result() );

    std::printf( "%s (N=%d): %u: %lld ms, %.2f MB/s\n", boost::core::demangle( typeid( H ).name() ).c_str(), N, r, ms, 1000.0 * N * M / ms / 1048576 );
}

extern unsigned char data[];

void test( int N, int M )
{
    test_<boost::hash2::fnv1a_32>( data, N, M );
    test_<boost::hash2::fnv1a_64>( data, N, M );
    test_<boost::hash2::murmur3_32>( data, N, M );
    test_<boost::hash2::murmur3_128>( data, N, M );
    test_<boost::hash2::xxhash_32>( data, N, M );
    test_<boost::hash2::xxhash_64>( data, N, M );
    test_<boost::hash2::spooky2_128>( data, N, M );
    test_<boost::hash2::siphash_32>( data, N, M );
    test_<boost::hash2::siphash_64>( data, N, M );
    test_<boost::hash2::md5_128>( data, N, M );
    test_<boost::hash2::sha1_160>( data, N, M );
    test_<boost::hash2::hmac_md5_128>( data, N, M );
    test_<boost::hash2::hmac_sha1_160>( data, N, M );

    puts( "--" );
}

unsigned const N1 = 65536;
unsigned const M1 = 65535;

unsigned const N2 = 15;
unsigned const M2 = M1 * N1 / N2;

unsigned const N3 = 4;
unsigned const M3 = M1 * N1 / N3;

int main()
{
    test( N1, M1 );
    test( N2, M2 );
    test( N3, M3 );
}

unsigned char data[ N1 ];


#include <complex>
#include <exception>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <boost/circular_buffer.hpp>
#include <range/v3/all.hpp>
#include <cstdint>
#include <initializer_list>
#include <iterator>
#include <functional>
namespace views = ranges::views;
using Buffer = boost::circular_buffer<unsigned char>;
class Knothash
{
  static inline constexpr int N = 256;
  Buffer ring_;
  std::vector<int> sequencelen_;
  int skiplen_ = 0;
  int curpos_ = 0;
  void reverse(int start, int len)
  {
    for (auto i = 0; i < len / 2; ++i) {
      int b = (start + i) % ring_.size();
      int r = (start + len - 1 - i) % ring_.size();
      std::swap(ring_[b], ring_[r]);
    }
  }

public:
  Knothash(std::string_view input) : ring_(N)
  {
    ranges::copy(views::ints | views::take(N), std::back_inserter(ring_));
    auto v = input | views::transform([](char c) { return static_cast<int>(c); });
    int suffix[] = { 17, 31, 73, 47, 23 };
    auto origin = views::concat(v, suffix);
    ranges::copy(origin, std::back_inserter(sequencelen_));
  }
  void hash()
  {
    for (auto i = 0; i < 64; ++i) {
      for (auto l : sequencelen_) {
        reverse(curpos_, l);
        curpos_ += l + skiplen_;
        ++skiplen_;
      }
    }
  }
  std::string final() const
  {
    auto
      hexstring = ring_ | views::chunk(16) | views::transform([](auto range) {
        std::string result;
        char hex[] = "0123456789abcdef";
        auto r = ranges::accumulate(
          range, 0, std::bit_xor{});
        result += hex[(r / 16)];
        result += hex[(r % 16)];
        return result;
      });
    std::string s;
    return ranges::accumulate(hexstring, s);
  }
};
int main()
{
  Knothash h("187,254,0,81,169,219,1,190,19,102,255,56,46,32,2,216");
  h.hash();
  auto n = h.final();
  std::cout << n << std::endl;
}