// Author(s): Wieger Wesselink
// Copyright: see the accompanying file COPYING or copy at
// https://github.com/mCRL2org/mCRL2/blob/master/COPYING
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
/// \file constelm_test.cpp
/// \brief Add your file description here.

#define BOOST_TEST_MODULE constelm_test
#include <boost/test/included/unit_test_framework.hpp>
#include "mcrl2/pbes/constelm.h"
#include "mcrl2/pbes/detail/pbes_property_map.h"
#include "mcrl2/pbes/remove_equations.h"
#include "mcrl2/pbes/rewriter.h"
#include "mcrl2/pbes/txt2pbes.h"

using namespace mcrl2;
using namespace mcrl2::pbes_system;

std::string t1 =
  "% simple test, n is constant \n"
  "                             \n"
  "pbes nu X(n: Nat) =          \n"
  "       X(n);                 \n"
  "                             \n"
  "init X(0);                   \n"
  ;
std::string x1 = "binding_variables = X";

std::string t2 =
  "% simple test, n is NOT constant \n"
  "                                 \n"
  "pbes nu X(n: Nat) =              \n"
  "       X(n + 1);                 \n"
  "                                 \n"
  "init X(0);                       \n"
  ;
std::string x2 = "binding_variables = X(n: Nat)";

std::string t3 =
  "% multiple parameters: n1 is constant, n2 is not \n"
  "                                                 \n"
  "pbes nu X(n1,n2: Nat) =                          \n"
  "       X(n1, n2 + 1);                            \n"
  "                                                 \n"
  "init X(0, 1);                                    \n"
  ;
std::string x3 = "binding_variables = X(n2: Nat)";

std::string t4 =
  "% conditions: n is not constant, even if conditions are enabled\n"
  "                                                               \n"
  "pbes nu X(n: Nat) =                                            \n"
  "       val(2 < n) || X(n + 1);                                 \n"
  "                                                               \n"
  "init X(0);                                                     \n"
  ;
std::string x4 = "binding_variables = X(n: Nat)";

std::string t5 =
  "% conditions: n is constant if conditions are enabled \n"
  "                                                      \n"
  "pbes nu X(n: Nat) =                                   \n"
  "       val(2 < n) || X(n + 1);                        \n"
  "                                                      \n"
  "init X(5);                                            \n"
  ;
std::string x5 = "binding_variables = X";

std::string t6 =
  "% reachability: n1 is not constant, equation Y is not reachable and should be removed \n"
  "                                                                                \n"
  "pbes nu X(n1: Nat) = X(n1+1);                                                         \n"
  "     mu Y(n2: Nat) = Y(n2+1);                                                         \n"
  "                                                                                \n"
  "init X(5);                                                                            \n"
  ;
std::string x6 = "binding_variables = X(n1: Nat)";

std::string t7 =
  "% multiple edges from one vertex, one edge invalidates the assertion of the other: no constants should be found \n"
  "                                                                                \n"
  "pbes                                                                                \n"
  "   mu X1(n1,m1:Nat) = X2(n1) || X2(m1);                                                                         \n"
  "   mu X2(n2:Nat)     = X1(n2,n2);                                                                               \n"
  "init                                                                                \n"
  "   X1(2,1);                                                                                \n"
  ;
std::string x7 = "binding_variables = X1(n1,m1: Nat), X2(n2: Nat)";

std::string t8 =
  "% conditions: parameters b,c and d will always be removed, with conditions on, parameter \n"
  "% n will NOT be removed. changing b,c or d to false or n to a bigger number              \n"
  "% then 5 WILL result in the removal of n                                                 \n"
  "                                                                                \n"
  "pbes nu X(b,c,d:Bool, n:Nat) =                                                           \n"
  "    val(b) || ( val(c) && ( val(d) && (val(n > 5) || X(b,c,d,n+1))));                    \n"
  "                                                                                \n"
  "init                                                                                \n"
  "X(false,true,true,5);                                                                    \n"
  ;
std::string x8 = "binding_variables = X(n: Nat)";

std::string t9 =
  "% conditions: universal quantification which can be solved, n1 and n2 are \n"
  "% constants if conditions are enabled                                     \n"
  "                                                                          \n"
  "pbes mu X(n1,n2:Nat) =                                                    \n"
  "    forall m:Nat. (val(n1>n2) && X(n1+1,n2+1));                           \n"
  "                                                                          \n"
  "init X(1,2);                                                              \n"
  ;
std::string x9 = "binding_variables = X";

std::string t10 =
  "% conditions: existential quantification which can be solved, n1 and n2 \n"
  "% are constants if conditions are enabled                               \n"
  "                                                                        \n"
  "pbes mu X(n1,n2:Nat) =                                                  \n"
  "    exists m:Nat. (val(n1>n2) && X(n1+1,n2+1));                         \n"
  "                                                                        \n"
  "init X(1,2);                                                            \n"
  ;
std::string x10 = "binding_variables = X";

std::string t11 =
  "pbes mu X(n1,n2:Nat) =                                        \n"
  "    forall m:Nat. (val(m>n2) && X(n1+1,n2+1));                \n"
  "                                                              \n"
  "init X(1,2);                                                  \n"
  ;
std::string x11 = "binding_variables = X";

std::string t12 =
  "% example 4.2.1 from \"Tools for PBES\" report                                     \n"
  "% constants without conditions: o4, n2, n4, o4                                     \n"
  "% constants with conditions: everything (equations X1,X2,X3 and X4 are removed)    \n"
  "                                                                                \n"
  "pbes                                                                               \n"
  "   mu X1(n1,m1,o1,p1:Nat) = (val(n1<m1) || X2(o1,p1)) && X3(n1) && X1(n1,m1,4,p1); \n"
  "   mu X2(n2,m2:Nat)       = X5(n2,m2) || X5(m2,n2);                                \n"
  "   nu X3(n3:Nat)          = val(n3<3) || X1(n3,n3,4,n3+1);                         \n"
  "   nu X4(n4,m4,o4:Nat)    = val(n4 <= m4+o4) || (X3(n4) && X4(n4,m4+1,n4));        \n"
  "   mu X5(n5,m5:Nat)       = val(n5>m5) || X3(n5);                                  \n"
  "                                                                                \n"
  "init                                                                               \n"
  "   X4(0,0,0);                                                                      \n"
  ;
std::string x12 = "binding_variables = X1(n1,m1,p1: Nat), X2(m2: Nat), X3(n3: Nat), X4(m4: Nat), X5(n5,m5: Nat)";

std::string t13 =
  "pbes nu X =           \n"
  "        Y(true);      \n"
  "      mu Y(b: Bool) = \n"
  "        X;            \n"
  "                      \n"
  "init X;               \n"
  ;
std::string x13 = "binding_variables = X, Y";

std::string t14 =
  "pbes nu X(m:Nat) =           \n"
  "        forall n:Nat . X(n); \n"
  "                             \n"
  "init X(0);                   \n"
  ;
std::string x14 = "binding_variables = X(m: Nat)";

std::string t15 =
  "pbes                 \n"
  "nu X0(n:Nat) = X1;   \n"
  "mu X1 = X0(0) || X1; \n"
  "                     \n"
  "init X0(0);          \n"
  ;
std::string x15 = "binding_variables = X0, X1";

std::string t16 =
  "pbes                      \n"
  " nu X(n:Nat) = Y && X(n); \n"
  " mu Y = Z;                \n"
  " nu Z = Y;                \n"
  "                          \n"
  " init X(0);               \n"
  ;
std::string x16 = "binding_variables = X, Y, Z";

std::string t17 =
  "sort D = struct d1 | d2 | d3 | d4;\n"
  "pbes nu X1 =\n"
  "       X(2);\n"
  "     mu X(s3_X: Pos) =\n"
  "       val(s3_X == 4) || val(s3_X == 2) && X(3) || val(s3_X == 2) && X(4) || val(s3_X == 3) && X(1) || val(s3_X == 4) && X(1);\n"
  "init X1;\n";

std::string x17 = "binding_variables = X1, X(s3_X: Pos)";

std::string t18 =
  "% conditions: quantifications which cannot be solved, n1 and n2 are *not* \n"
  "% constants even if conditions are enabled.                               \n"
  "                                                                          \n"
  "pbes mu X(n1,n2:Nat) =                                                    \n"
  "    forall m1:Nat. exists m2:Nat. (val(m2 > n2 + m1) && X(n1+1,n2+1));    \n"
  "                                                                          \n"
  "init X(1,2);                                                              \n"
  ;

std::string x18 = "binding_variables = X(n1,n2: Nat)";

void test_pbes(const std::string& pbes_spec, const std::string& expected_result, bool compute_conditions, bool remove_equations = true)
{
  typedef simplify_data_rewriter<data::rewriter> my_pbes_rewriter;

  pbes p = txt2pbes(pbes_spec);
  pbes q = p;

  // data rewriter
  data::rewriter datar(q.data());

  // pbes rewriter
  my_pbes_rewriter pbesr(datar);

  // constelm algorithm
  pbes_constelm_algorithm<data::rewriter, my_pbes_rewriter> algorithm(datar, pbesr);

  // run the algorithm
  algorithm.run(q, compute_conditions);
  if (remove_equations)
  {
    remove_unreachable_variables(q);
  }
  BOOST_CHECK(q.is_well_typed());
  std::cout << "\n--- q ---\n" << pbes_system::pp(q) << std::endl;

  pbes_system::detail::pbes_property_map info1(q);
  pbes_system::detail::pbes_property_map info2(expected_result);
  std::string diff = info1.compare(info2);
  if (!diff.empty())
  {
    std::cerr << "\n------ FAILED TEST ------" << std::endl;
    std::cout << pbes_spec << std::endl;
    std::cerr << "--- expected result" << std::endl;
    std::cerr << expected_result << std::endl;
    std::cerr << "--- found result" << std::endl;
    std::cerr << info1.to_string() << std::endl;
    std::cerr << "--- differences" << std::endl;
    std::cerr << diff << std::endl;
  }
  BOOST_CHECK(diff.empty());

}

BOOST_AUTO_TEST_CASE(test_constelm)
{
  test_pbes(t1 , x1 , false);
  test_pbes(t2 , x2 , false);
  test_pbes(t3 , x3 , false);
  test_pbes(t4 , x4 , true);
  test_pbes(t5 , x5 , true);
  test_pbes(t6 , x6 , false);
  test_pbes(t7 , x7 , false);
  test_pbes(t8 , x8 , true);
  test_pbes(t9 , x9 , true);
  test_pbes(t10, x10, true);
  test_pbes(t11, x11, true);
  test_pbes(t12, x12, false);
  test_pbes(t13, x13, false);
  test_pbes(t14, x14, false);
  test_pbes(t15, x15, false);
  test_pbes(t16, x16, true);
  test_pbes(t17, x17, false);
  test_pbes(t18, x18, true);
}


/*!
 * @file
 * Contains unit tests for `boost::mpl11::and_` and `boost::mpl11::or_`.
 */

#include <boost/mpl11/logical.hpp>

#include <boost/mpl11/bool.hpp>
#include <boost/mpl11/core.hpp>
#include <boost/mpl11/integer.hpp>

using namespace boost::mpl11;

template <typename T, typename F>
struct test_and_or {
    // test and_
    static_assert( and_<>::value, "");

    static_assert( and_<T>::value, "");
    static_assert(!and_<F>::value, "");

    static_assert( and_<T, T>::value, "");
    static_assert(!and_<T, F>::value, "");
    static_assert(!and_<F, T>::value, "");
    static_assert(!and_<F, F>::value, "");
    static_assert(!and_<F, undefined>::value, "");

    static_assert( and_<T, T, T>::value, "");
    static_assert(!and_<T, T, F>::value, "");
    static_assert(!and_<T, F, T>::value, "");
    static_assert(!and_<T, F, F>::value, "");
    static_assert(!and_<F, T, T>::value, "");
    static_assert(!and_<F, T, F>::value, "");
    static_assert(!and_<F, F, T>::value, "");
    static_assert(!and_<F, F, F>::value, "");
    static_assert(!and_<T, F, undefined>::value, "");
    static_assert(!and_<F, T, undefined>::value, "");
    static_assert(!and_<F, F, undefined>::value, "");

    // test or_
    static_assert(!or_<>::value, "");

    static_assert( or_<T>::value, "");
    static_assert(!or_<F>::value, "");

    static_assert( or_<T, T>::value, "");
    static_assert( or_<T, F>::value, "");
    static_assert( or_<F, T>::value, "");
    static_assert(!or_<F, F>::value, "");
    static_assert( or_<T, undefined>::value, "");

    static_assert( or_<T, T, T>::value, "");
    static_assert( or_<T, T, F>::value, "");
    static_assert( or_<T, F, T>::value, "");
    static_assert( or_<T, F, F>::value, "");
    static_assert( or_<F, T, T>::value, "");
    static_assert( or_<F, T, F>::value, "");
    static_assert( or_<F, F, T>::value, "");
    static_assert(!or_<F, F, F>::value, "");
    static_assert( or_<T, F, undefined>::value, "");
    static_assert( or_<F, T, undefined>::value, "");
    static_assert( or_<T, T, undefined>::value, "");
};

template <typename T, T v>
struct Integral { struct type { static constexpr T value = v; }; };

struct tests :
    test_and_or<Integral<bool, true>, Integral<bool, false>>,
    test_and_or<Integral<int, 1>, Integral<bool, false>>,
    test_and_or<Integral<int, 2>, Integral<bool, false>>,

    // test integer_c specialization
    test_and_or<true_, false_>,
    test_and_or<int_<2>, false_>,
    test_and_or<int_<2>, int_<0>>,
    test_and_or<int_<2>, long_<0>>
{ };

int main() { }


//---------------------------------------------------------------------------//
//!
//! \file   tstAtomicMassUnit.cpp
//! \author Alex Robinson
//! \brief  The atomic mass unit unit tests
//!
//---------------------------------------------------------------------------//

// Boost Includes
#include <boost/units/quantity.hpp>
#include <boost/units/systems/cgs/mass.hpp>
#include <boost/units/systems/si/mass.hpp>

// Trilinos Includes
#include <Teuchos_UnitTestHarness.hpp>

// FRENSIE Includes
#include "Utility_AtomicMassUnit.hpp"

using namespace Utility::Units;
using boost::units::quantity;
namespace si = boost::units::si;
namespace cgs = boost::units::cgs;

//---------------------------------------------------------------------------//
// Tests
//---------------------------------------------------------------------------//
// Check that the atomic mass unit can be initialized
TEUCHOS_UNIT_TEST( AtomicMassUnit, initialize )
{
  quantity<AtomicMass> mass( 1.0*amu );

  TEST_EQUALITY_CONST( mass.value(), 1.0 );

  mass = 2.0*amu;

  TEST_EQUALITY_CONST( mass.value(), 2.0 );
}

//---------------------------------------------------------------------------//
// Check that the atomic mass unit can be initialized from cgs and si units
TEUCHOS_UNIT_TEST( AtomicMassUnit, initialize_from_cgs_si )
{
  quantity<AtomicMass> mass_cgs( 1.0*cgs::gram );

  quantity<AtomicMass> mass_si( 1.0*si::kilogram );

  TEST_FLOATING_EQUALITY( mass_cgs.value(),
			  6.0221408585491615e+23,
			  1e-15 );
  TEST_FLOATING_EQUALITY( mass_si.value(),
			  6.0221408585491615e+26,
			  1e-15 );
}

//---------------------------------------------------------------------------//
// Check that the cgs and si units can be initialized from the atomic mass u.
TEUCHOS_UNIT_TEST( AtomicMassUnit, initialize_cgs_si )
{
  quantity<cgs::mass> cgs_mass( 1.0*amu );

  quantity<si::mass> si_mass( 1.0*amu );

  TEST_FLOATING_EQUALITY( cgs_mass.value(),
			  1.660539040e-24,
			  1e-15 );
  TEST_FLOATING_EQUALITY( si_mass.value(),
			  1.660539040e-27,
			  1e-15 );
}

//---------------------------------------------------------------------------//
// end tstAtomicMassUnit.cpp
//---------------------------------------------------------------------------//


#ifndef YASMIC_IFSTREAM_MATRIX
#define YASMIC_IFSTREAM_MATRIX

#include <fstream>
#include <boost/tuple/tuple.hpp>
#include <boost/tuple/tuple_io.hpp>
#include <boost/static_assert.hpp>
#include <yasmic/smatrix_traits.hpp>
#include <iterator>

#include <yasmic/generic_matrix_operations.hpp>

namespace yasmic
{
	template <class index_type = int, class value_type = double, class size_type = int,
		bool header = true>
	struct ifstream_matrix
	{
		ifstream_matrix(std::istream& f)
			: _f(f), _nrows(0), _ncols(0), _nnz(0), _sized(false)
		{
			// this call is only valid if we have to read the header
			BOOST_STATIC_ASSERT(header == true);
		}

		ifstream_matrix(std::istream& f, index_type nrows, index_type ncols, size_type nnz)
			: _f(f), _nrows(nrows), _ncols(ncols), _nnz(nnz), _sized(true)
		{}

		std::istream& _f;
		size_type _nrows;
		size_type _ncols;
		size_type _nnz;

		bool _sized;
	};

	template <class i_index_type, class i_value_type, class i_size_type, bool header>
    struct smatrix_traits<ifstream_matrix<i_index_type, i_value_type, i_size_type, header> >
    {
    	typedef i_size_type size_type;
    	typedef i_index_type index_type;
		typedef i_value_type value_type;

		typedef boost::tuple<index_type, index_type, value_type> nonzero_descriptor;

		typedef std::istream_iterator<nonzero_descriptor> nonzero_iterator;

		typedef i_size_type nz_index_type;

		typedef void row_iterator;

		typedef void row_nonzero_descriptor;
		typedef void row_nonzero_iterator;

		typedef void column_iterator;

        typedef void properties;
    };

	namespace impl
	{
		template <class i_index_type, class i_value_type, class i_size_type, bool header>
		struct ifstream_matrix_help
		{
			typedef ifstream_matrix<i_index_type, i_value_type, i_size_type, header> mat_type;
			typedef smatrix_traits<mat_type> mat_traits;
			typedef std::pair<typename mat_traits::size_type,
							  typename mat_traits::size_type> dimensions_ret_type;
			typedef std::pair<typename mat_traits::nonzero_iterator,
							  typename mat_traits::nonzero_iterator> nz_ret_type;
		};

	}

	template <class i_index_type, class i_value_type, class i_size_type, bool header>
	inline typename
		impl::ifstream_matrix_help<i_index_type, i_value_type, i_size_type, header>::dimensions_ret_type
    dimensions(ifstream_matrix<i_index_type, i_value_type, i_size_type, header>& m)
    {
		if (header && !m._sized)
		{
			m._f.clear();
    		m._f.seekg(0, std::ios_base::beg);

			m._f >> m._nrows >> m._ncols >> m._nnz;
			m._sized = true;
		}

        return (std::make_pair(m._nrows, m._ncols));
    }

	template <class i_index_type, class i_value_type, class i_size_type, bool header>
	inline typename
		impl::ifstream_matrix_help<i_index_type, i_value_type, i_size_type, header>::mat_traits::size_type
	nnz(ifstream_matrix<i_index_type, i_value_type, i_size_type, header>& m)
	{
		if (header && !m._sized)
		{
			m._f.clear();
    		m._f.seekg(0, std::ios_base::beg);

			m._f >> m._nrows >> m._ncols >> m._nnz;
			m._sized = true;
		}

        return (m._nnz);
	}

	template <class i_index_type, class i_value_type, class i_size_type, bool header>
	inline typename
		impl::ifstream_matrix_help<i_index_type, i_value_type, i_size_type, header>::nz_ret_type
    nonzeros(ifstream_matrix<i_index_type, i_value_type, i_size_type, header>& m)
    {
    	typedef smatrix_traits<ifstream_matrix<i_index_type, i_value_type, i_size_type, header> > traits;

    	m._f.clear();
    	m._f.seekg(0, std::ios_base::beg);

		if (header)
		{
    		typename traits::size_type d1,d2,d3;
			m._f >> d1  >> d2 >> d3;
		}

        m._f >> boost::tuples::set_open(' ') >> boost::tuples::set_close(' ');

        typedef typename traits::nonzero_iterator nz_iter;

        return (std::make_pair(nz_iter(m._f), nz_iter()));
    }

}

#endif //YASMIC_IFSTREAM_MATRIX


/*
@copyright Louis Dionne 2014
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
 */

#include <boost/hana/detail/assert.hpp>
#include <boost/hana/detail/constexpr.hpp>
#include <boost/hana/list/instance.hpp>
#include <boost/hana/maybe.hpp>
using namespace boost::hana;

int main() {
    //! [main]
    BOOST_HANA_CONSTEXPR_LAMBDA auto replicate3 = [](auto x) {
        return list(x, x, x);
    };

    BOOST_HANA_CONSTEXPR_ASSERT(traverse<List>(replicate3, just(1)) == list(just(1), just(1), just(1)));
    BOOST_HANA_CONSTANT_ASSERT(traverse<List>(replicate3, nothing) == list(nothing));
    //! [main]
}


/* dos_estimate_quartic.cc

Copyright 2018 Grant M. Rotskoff

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

 */

#include <armadillo>
#include "particles.h"
#include "quartic.h"

int main(int argc, char **argv)
{

  arma_rng::set_seed(45632426);
  quartic g;
  int dim = atoi(argv[1]);
  double gamma = atof(argv[2]);
  g.initialize(dim, gamma);
  double hmin = 0.;
  double hmax = 2. * dim;
  double hres = 0.5;

  g.qmin = -pow(4*hmax,0.25);
  g.qmax = pow(4*hmax,0.25);

  // set the integration parameters
  double dt = 1e-3;
  double tol = 1e-6;
  int max_iter = 0;
  bool use_mc = false;

  int n_traj = atoi(argv[3]);
  g.initialize_integration_variables(dt, gamma);
  g.initialize_estimator(hmax, hres, hmin, n_traj);
  // run the estimation trajectories
  for (int i=0; i<n_traj; i++) {
    g.run_estimation_trajectory(tol, hmax, max_iter, i, langevin, use_mc);
  }
  char dos_filename[80];
  sprintf(dos_filename, "quartic_dim=%03d_gamma=%05.3f_ntraj=%04d.dat", dim, gamma, n_traj);
  FILE *dosf = fopen(dos_filename, "w");
  g.dump_dos_estimate(n_traj, dosf);
}


//==============================================================================
//         Copyright 2003 - 2011 LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011 LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef NT2_TOOLBOX_ELLIPTIC_FUNCTIONS_SCALAR_ELLIPKE_HPP_INCLUDED
#define NT2_TOOLBOX_ELLIPTIC_FUNCTIONS_SCALAR_ELLIPKE_HPP_INCLUDED
#include <nt2/toolbox/elliptic/functions/ellipke.hpp>
#include <boost/fusion/tuple.hpp>
#include <nt2/include/functions/scalar/sqr.hpp>
#include <nt2/include/functions/scalar/ldexp.hpp>
#include <nt2/include/functions/scalar/sqrt.hpp>
#include <nt2/include/functions/scalar/sqr.hpp>
#include <nt2/include/functions/scalar/average.hpp>
#include <nt2/include/functions/scalar/oneminus.hpp>
#include <nt2/include/functions/scalar/is_greater.hpp>
#include <nt2/include/functions/scalar/is_equal.hpp>
#include <nt2/include/functions/scalar/is_ltz.hpp>
#include <nt2/toolbox/trigonometric/constants.hpp>
#include <nt2/include/constants/real.hpp>
#include <nt2/include/constants/digits.hpp>
#include <nt2/include/constants/eps.hpp>
/////////////////////////////////////////////////////////////////////////////
// Implementation when type A0 is arithmetic_
/////////////////////////////////////////////////////////////////////////////
namespace nt2 { namespace ext
{
  NT2_FUNCTOR_IMPLEMENTATION(nt2::tag::ellipke_, tag::cpu_,
                             (A0),
                             (scalar_<arithmetic_<A0> > )
    )
  {
    typedef typename boost::dispatch::meta::as_floating<A0>::type etype;
    typedef boost::fusion::tuple<etype, etype>           result_type;
    NT2_FUNCTOR_CALL(1)
    {
      return ellipke(etype(a0), Eps<etype>());
    }
  };

  /////////////////////////////////////////////////////////////////////////////
  // Implementation when type A0 is arithmetic_
  /////////////////////////////////////////////////////////////////////////////
  NT2_FUNCTOR_IMPLEMENTATION(nt2::tag::ellipke_, tag::cpu_,
                             (A0),
                             (scalar_<arithmetic_<A0> >)
                             (scalar_<arithmetic_<A0> >)
    )
  {
    typedef boost::fusion::tuple<A0, A0> result_type;

    NT2_FUNCTOR_CALL_REPEAT(2)
    {
      typedef typename boost::dispatch::meta::as_floating<A0>::type type;
      return ellipke(type(a0), type(a1));
    }
  };

  /////////////////////////////////////////////////////////////////////////////
  // Implementation when type A0 is floating_
  /////////////////////////////////////////////////////////////////////////////
  NT2_FUNCTOR_IMPLEMENTATION(nt2::tag::ellipke_, tag::cpu_,
                             (A0),
                             (scalar_<floating_<A0> >)
                             (scalar_<floating_<A0> >)
    )
  {
    typedef boost::fusion::tuple<A0, A0> result_type;

    NT2_FUNCTOR_CALL_REPEAT(2)
    {
      result_type res;
      nt2::ellipke(a0, a1, boost::fusion::at_c<0>(res), boost::fusion::at_c<1>(res));
      return res;
    }
  };

  /////////////////////////////////////////////////////////////////////////////
  // reference based Implementations 1 input
  /////////////////////////////////////////////////////////////////////////////
  NT2_FUNCTOR_IMPLEMENTATION(  nt2::tag::ellipke_, tag::cpu_,
                               (A0),
                               (scalar_<floating_<A0> >)
                               (scalar_<floating_<A0> >)
                               (scalar_<floating_<A0> >)
    )
  {
    typedef int result_type;
    inline result_type operator()(A0 const& a0,A0 & a1,A0 & a2) const
    {
      nt2::ellipke(a0,Eps<A0>(),a1,a2);
      return 0;
    }
  };

  /////////////////////////////////////////////////////////////////////////////
  // reference based Implementations 2 inputs
  /////////////////////////////////////////////////////////////////////////////
  NT2_FUNCTOR_IMPLEMENTATION(  nt2::tag::ellipke_, tag::cpu_,(A0),
                               (scalar_<floating_<A0> >)
                               (scalar_<floating_<A0> >)
                               (scalar_<floating_<A0> >)
                               (scalar_<floating_<A0> >)
    )
  {
    typedef int result_type;
    inline result_type operator()(A0 const& a0,A0 const& a1, A0 & a2,A0 & a3) const
    {
      if (is_ltz(a0) || gt(a0, One<A0>()))
      {
        a2 = Nan<A0>();
        a3 = Nan<A0>();
        return 0;
      }
      A0 m = a0;
      A0 aa0 = One<A0>();;
      A0 bb0 = nt2::sqrt(oneminus(m));
      A0 s0 = m;
      size_t i1 = 0;
      A0 mm = 1;
      A0 aa1 = Zero<A0>();
      while (mm > a1) {
        aa1 = average(aa0, bb0);
        A0 bb1 = nt2::sqrt(aa0*bb0);
        A0 cc1 = nt2::average(aa0, -bb0);
        ++i1;
        mm = nt2::ldexp(sqr(cc1), i1);
        s0 += mm;
        aa0 = aa1;
        bb0 = bb1;
      };
      if (is_equal(m, One<A0>()))
      {
        a2 = Inf<A0>();
        a3 = One<A0>();
      }
      else
      {
        a2 = nt2::Pio_2<A0>()/aa1;
        a3 = a2*(One<A0>()-s0*Half<A0>());
      }
      return 0;
    }
  };

} }
#endif


#include <stdio.h>
#include <stdlib.h>
#include <map>
#include <dlib/dnn.h>
#include <iostream>
#include <dlib/data_io.h>

using namespace std;
using namespace dlib;

using in_type =  double;
using lab_type = unsigned long;

#ifndef _INPUTS_
#define _INPUTS_ 5
#endif

#ifndef _LAYER1_
#define _LAYER1_ _INPUTS_
#endif

#ifndef _LAYER2_
#define _LAYER2_ _LAYER1_
#endif

#ifndef _LAYER3_
#define _LAYER3_ _LAYER2_
#endif

#ifndef _LAYER4_
#define _LAYER4_ _LAYER3_
#endif

const int input_count = _INPUTS_;
const int layer1_count = _LAYER1_;
const int layer2_count = _LAYER2_;
const int layer3_count = _LAYER3_;
const int layer4_count = _LAYER4_;

void load_morse_code_training(
	std::vector<matrix<in_type>> &training_images,
	std::vector<lab_type> &training_labels,
	std::vector<matrix<in_type>> &testing_images,
	std::vector<lab_type> &testing_labels,
	int train_count,
	int test_count )
{

	// Training data read
	cout << "Loading training data...." << endl;
	for( int c=0; c < train_count; c++ )
	{
		string str;
		getline( cin, str );
		stringstream ss( str );

		matrix< in_type, 1, input_count > img;
		lab_type lab;

		ss >> lab;

		for( int i=0; i < img.nr(); i++ )
		{
			for( int j=0; j < img.nc(); j++ )
			{
				ss >> img( i, j );
			}
		}
		training_images.push_back( img );
		training_labels.push_back( lab );
	}

	// Training data read
	cout << "Loading testing data...." << endl;
	for( int c=0; c < test_count; c++ )
	{
		string str;
		getline( cin, str );
		stringstream ss( str );

		matrix<in_type, 1, input_count> img;
		lab_type lab;

		ss >> lab;

		for( int i=0; i < img.nr(); i++ )
		{
			for( int j=0; j < img.nc(); j++ )
			{
				ss >> img( i, j );
			}
		}
		testing_images.push_back( img );
		testing_labels.push_back( lab );
	}

	cout << "Done loading training and testing data...." << endl;

	return;
}

// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
/*
    This is an example illustrating the use of the deep learning tools from the
    dlib C++ Library.  In it, we will train the venerable LeNet convolutional
    neural network to recognize hand written digits.  The network will take as
    input a small image and classify it as one of the 10 numeric digits between
    0 and 9.

    The specific network we will run is from the paper
        LeCun, Yann, et al. "Gradient-based learning applied to document recognition."
        Proceedings of the IEEE 86.11 (1998): 2278-2324.
    except that we replace the sigmoid non-linearities with rectified linear units.

    These tools will use CUDA and cuDNN to drastically accelerate network
    training and testing.  CMake should automatically find them if they are
    installed and configure things appropriately.  If not, the program will
    still run but will be much slower to execute.
*/

#include <dlib/dnn.h>
#include <iostream>
#include <dlib/data_io.h>

using namespace std;
using namespace dlib;

void print_input( matrix<in_type> &img, lab_type label, lab_type predicted );
//void print_image( matrix<unsigned char> &img, unsigned long label, unsigned long predicted );

void print_probs( matrix<float,1,5> &softmax_probs, int x );

int main(int argc, char** argv) try
{

	bool correct_out = false;
	bool wrong_out = false;
	bool train_me = false;

	int training_count = 0;
	int testing_count = 0;

    // This example is going to run on the MNIST dataset.
    if (argc >= 2)
    {
    	train_me = true;

	{
		int out_flag = atoi( argv[2] );

		if( out_flag == 1 )
			correct_out = true;

		if( out_flag == 2)
			wrong_out = true;

		if( out_flag == 3 )
			correct_out = wrong_out = true;
	}

	training_count = atoi( argv[3] );

	testing_count = atoi( argv[4] );

    }

    // MNIST is broken into two parts, a training set of 60000 images and a test set of
    // 10000 images.  Each image is labeled so that we know what hand written digit is
    // depicted.  These next statements load the dataset into memory.
    std::vector<matrix<in_type>> training_images;
    std::vector<lab_type>         training_labels;
    std::vector<matrix<in_type>> testing_images;
    std::vector<lab_type>         testing_labels;
    //load_mnist_dataset(argv[1], training_images, training_labels, testing_images, testing_labels);
//    load_morse_code_training(argv[1], training_images, training_labels, testing_images, testing_labels);

    // Now let's define the LeNet.  Broadly speaking, there are 3 parts to a network
    // definition.  The loss layer, a bunch of computational layers, and then an input
    // layer.  You can see these components in the network definition below.
    //
    // The input layer here says the network expects to be given matrix<unsigned char>
    // objects as input.  In general, you can use any dlib image or matrix type here, or
    // even define your own types by creating custom input layers.
    //
    // Then the middle layers define the computation the network will do to transform the
    // input into whatever we want.  Here we run the image through multiple convolutions,
    // ReLU units, max pooling operations, and then finally a fully connected layer that
    // converts the whole thing into just 10 numbers.
    //
    // Finally, the loss layer defines the relationship between the network outputs, our 10
    // numbers, and the labels in our dataset.  Since we selected loss_multiclass_log it
    // means we want to do multiclass classification with our network.   Moreover, the
    // number of network outputs (i.e. 10) is the number of possible labels.  Whichever
    // network output is largest is the predicted label.  So for example, if the first
    // network output is largest then the predicted digit is 0, if the last network output
    // is largest then the predicted digit is 9.
//    using net_type = loss_multiclass_log<
//                                fc<10,
//                                relu<fc<84,
//                                relu<fc<120,
//                                max_pool<2,2,2,2,relu<con<16,5,5,1,1,
//                                max_pool<2,2,2,2,relu<con<6,5,5,1,1,
//                                input<matrix<unsigned char>>
//                                >>>>>>>>>>>>;

#if ( CUSTOMIZED == 4 )

    using net_type = loss_multiclass_log<
                                fc<5,
                                LAYER4TYPE<fc<layer4_count,
                                LAYER3TYPE<fc<layer3_count,
                                LAYER2TYPE<fc<layer2_count,
                                LAYER1TYPE<fc<layer1_count,
                                INPUTTYPE<fc<input_count,
                                input<matrix<in_type>>
                                >>>>>>>>>>>>;

#elif ( CUSTOMIZED == 3 )

    using net_type = loss_multiclass_log<
                                fc<5,
                                LAYER3TYPE<fc<layer3_count,
                                LAYER2TYPE<fc<layer2_count,
                                LAYER1TYPE<fc<layer1_count,
                                INPUTTYPE<fc<input_count,
                                input<matrix<in_type>>
                                >>>>>>>>>>;

#elif ( CUSTOMIZED == 2 )

    using net_type = loss_multiclass_log<
                                fc<5,
                                LAYER2TYPE<fc<layer2_count,
                                LAYER1TYPE<fc<layer1_count,
                                INPUTTYPE<fc<input_count,
                                input<matrix<in_type>>
                                >>>>>>>>;

#elif ( CUSTOMIZED == 1 )

    using net_type = loss_multiclass_log<
                                fc<5,
                                LAYER1TYPE<fc<layer1_count,
                                INPUTTYPE<fc<input_count,
                                input<matrix<in_type>>
                                >>>>>>;

#else

    using net_type = loss_multiclass_log<
                                fc<5,
                                htan<fc<layer2_count,
                                htan<fc<layer1_count,
                                htan<fc<input_count,
                                input<matrix<in_type>>
                                >>>>>>>>;

#endif

    // This net_type defines the entire network architecture.  For example, the block
    // relu<fc<84,SUBNET>> means we take the output from the subnetwork, pass it through a
    // fully connected layer with 84 outputs, then apply ReLU.  Similarly, a block of
    // max_pool<2,2,2,2,relu<con<16,5,5,1,1,SUBNET>>> means we apply 16 convolutions with a
    // 5x5 filter size and 1x1 stride to the output of a subnetwork, then apply ReLU, then
    // perform max pooling with a 2x2 window and 2x2 stride.

    // So with that out of the way, we can make a network instance.
    net_type net;

    if( train_me == true )
    {

#define QUOTE(q) Q(q)
#define Q(q) #q

    	    #ifdef CUSTOMIZED
    	    	cerr << endl << endl << "Input " << "  = " << input_count << "\t" << QUOTE(INPUTTYPE) << endl;
    	    #endif
	    #if( CUSTOMIZED >= 1 )
    	    	cerr << "Layer " << 1 << " = " << layer1_count << "\t" << QUOTE(LAYER1TYPE) << endl;
    	    #endif
	    #if( CUSTOMIZED >= 2 )
    	    	cerr << "Layer " << 2 << " = " << layer2_count << "\t" << QUOTE(LAYER2TYPE) << endl;
    	    #endif
	    #if( CUSTOMIZED >= 3 )
    	    	cerr << "Layer " << 3 << " = " << layer3_count << "\t" << QUOTE(LAYER3TYPE) << endl;
    	    #endif
	    #if( CUSTOMIZED >= 4 )
    	    	cerr << "Layer " << 4 << " = " << layer4_count << "\t" << QUOTE(LAYER4TYPE) << endl;
    	    #endif
    	    #ifdef CUSTOMIZED
    	    	cerr << endl << endl << flush;
    	    #endif

          load_morse_code_training( training_images, training_labels, testing_images, testing_labels, training_count, testing_count);

	    // And then train it using the MNIST data.  The code below uses mini-batch stochastic
	    // gradient descent with an initial learning rate of 0.01 to accomplish this.
	    dnn_trainer<net_type> trainer(net);
	    trainer.set_learning_rate(0.01);
	    trainer.set_min_learning_rate(0.00001);
	    trainer.set_mini_batch_size(128);
	    trainer.be_verbose();
	    // Since DNN training can take a long time, we can ask the trainer to save its state to
	    // a file named "mnist_sync" every 20 seconds.  This way, if we kill this program and
	    // start it again it will begin where it left off rather than restarting the training
	    // from scratch.  This is because, when the program restarts, this call to
	    // set_synchronization_file() will automatically reload the settings from mnist_sync if
	    // the file exists.
	    trainer.set_synchronization_file("nn_cw_sync", std::chrono::seconds(20));
	    // Finally, this line begins training.  By default, it runs SGD with our specified
	    // learning rate until the loss stops decreasing.  Then it reduces the learning rate by
	    // a factor of 10 and continues running until the loss stops decreasing again.  It will
	    // keep doing this until the learning rate has dropped below the min learning rate
	    // defined above or the maximum number of epochs as been executed (defaulted to 10000).
	    trainer.train(training_images, training_labels);

	    // At this point our net object should have learned how to classify MNIST images.  But
	    // before we try it out let's save it to disk.  Note that, since the trainer has been
	    // running images through the network, net will have a bunch of state in it related to
	    // the last batch of images it processed (e.g. outputs from each layer).  Since we
	    // don't care about saving that kind of stuff to disk we can tell the network to forget
	    // about that kind of transient data so that our file will be smaller.  We do this by
	    // "cleaning" the network before saving it.
	    net.clean();
	    serialize("nn_cw_network.dat") << net;
	    // Now if we later wanted to recall the network from disk we can simply say:
	    // deserialize("mnist_network.dat") >> net;

	    // Now let's run the training images through the network.  This statement runs all the
	    // images through it and asks the loss layer to convert the network's raw output into
	    // labels.  In our case, these labels are the numbers between 0 and 9.
	    std::vector<lab_type> predicted_labels = net(training_images);
	    int num_right = 0;
	    int num_wrong = 0;
	    // And then let's see if it classified them correctly.
	    for (size_t i = 0; i < training_images.size(); ++i)
	    {
		if (predicted_labels[i] == training_labels[i])
		    ++num_right;
		else
		    ++num_wrong;

	    }
	    cout << "training num_right: " << num_right << endl;
	    cout << "training num_wrong: " << num_wrong << endl;
	    cout << "training accuracy:  " << num_right/(double)(num_right+num_wrong) << endl;

	    // Let's also see if the network can correctly classify the testing images.  Since
	    // MNIST is an easy dataset, we should see at least 99% accuracy.
	    predicted_labels = net(testing_images);
		softmax<net_type::subnet_type> snet; snet.subnet() = net.subnet();
		//auto &sp = ((snet(testing_images.begin(), testing_images.end())));
		//std::vector<matrix<float>> sp = ((snet(testing_images)));
	    num_right = 0;
	    num_wrong = 0;
	    for (size_t i = 0; i < testing_images.size(); ++i)
	    {
		if (predicted_labels[i] == testing_labels[i])
			{
		    ++num_right;
				if( correct_out == true )
				{
					print_input( testing_images[i], testing_labels[i], predicted_labels[i] );
					//print_image( testing_images[i], testing_labels[i], sp[i] );
					//matrix<float,1,10> sp = sum_rows(mat(snet(testing_images.begin()+i, testing_images.begin()+(i+1))));
					matrix<float,1,5> sp = mat(snet(testing_images.begin()+i, testing_images.begin()+(i+1)));
					print_probs( sp, i );
				}

			}
		else
			{
		    ++num_wrong;

				if( wrong_out == true )
				{
					print_input( testing_images[i], testing_labels[i], predicted_labels[i] );
					//print_image( testing_images[i], testing_labels[i], sp[i] );
					//matrix<float,1,10> sp = sum_rows(mat(snet(testing_images.begin()+i, testing_images.begin()+(i+1))));
					matrix<float,1,5> sp = (mat(snet(testing_images.begin()+i, testing_images.begin()+(i+1))));
					print_probs( sp, i );
				}
		}
	    }
	    cout << "testing num_right: " << num_right << endl;
	    cout << "testing num_wrong: " << num_wrong << endl;
	    cout << "testing accuracy:  " << num_right/(double)(num_right+num_wrong) << endl;

	    // Finally, you can also save network parameters to XML files if you want to do
	    // something with the network in another tool.  For example, you could use dlib's
	    // tools/convert_dlib_nets_to_caffe to convert the network to a caffe model.
	    net_to_xml(net, "nn_cw_lenet.xml");
	}
	else
	{
		deserialize("nn_cw_network.dat") >> net;

		string str;

		std::map<lab_type, string> labels;

		labels[0] = "L";
		labels[1] = "G";
		labels[2] = "g";
		labels[3] = ".";
		labels[4] = "-";

		labels[0] = "\n\n";
		labels[1] = "\n";
		labels[2] = "";
		labels[3] = ".";
		labels[4] = "-";

		training_images.push_back( matrix<in_type, 1, input_count>() );

		while( !getline( cin, str ).eof() )
		{
			stringstream ss( str );

			matrix<in_type, 1, input_count> img;
			lab_type lab;

			//ss >> lab;

			for( int i=0; i < img.nr(); i++ )
			{
				for( int j=0; j < img.nc(); j++ )
				{
					ss >> img( i, j );
				}
			}

			training_images[0] = img;

			std::vector<lab_type> predicted_labels = net(training_images);

			//cout << labels[ predicted_labels[0] ];
			cout << labels[ predicted_labels[0] ] << flush;

			fflush( stdout );
		}
	}
}
catch(std::exception& e)
{
    cout << e.what() << endl;
}

void print_probs2( matrix<float,1,5> &softmax_probs, int x )
{
	for( int i=0; i < 5; i++ )
	{
		cout << i << "=" << softmax_probs(0,i) << "  |  ";
	}
	cout << endl;
}

void print_probs( matrix<float,1,5> &softmax_probs, int x )
{
	for( int i=0; i < 5; i++ )
	{
		printf( "%d\t", i );
	}
	cout << endl;

	for( int i=0; i < 5; i++ )
	{
		printf( "%0.1f%%\t", (float)softmax_probs(0,i) * 100.0 );
	}
	cout << endl;
}

void print_input( matrix<in_type> &img, lab_type label, lab_type predicted )
{

	for( int i=0; i<img.nr(); i++ )
	{
		for( int j=0; j<img.nc(); j++ )
		{
			cout << img( i, j ) << "\t";
		}
		cout << endl;
	}
	cout << "label=" << label << "  predicted=" << predicted << endl;
}


#include "tasktorrent/tasktorrent.hpp"
#ifdef USE_MKL
#include <mkl_cblas.h>
#include <mkl_lapacke.h>
#else
#include <cblas.h>
#include <lapacke.h>
#endif
#include <Eigen/Core>
#include <Eigen/Cholesky>
#include <fstream>
#include <array>
#include <random>
#include <mutex>
#include <iostream>
#include <map>
#include <memory>
#include <set>

#include <mpi.h>
#include <cxxopts.hpp>

using namespace std;
using namespace Eigen;
using namespace ttor;

typedef array<int, 2> int2;
typedef array<int, 3> int3;

/*
Parametrized priorities for cholesky:
0. No priority, only enforces potrf>trsm>gemm
1. Row-based priority, prioritize tasks with smaller row number in addition to priority 0.
2. Critical path priority, prioritize tasks with longest distance to the exit task. For references, check out the paper
    Beaumont, Olivier, et al. "A Makespan Lower Bound for the Scheduling of the Tiled Cholesky Factorization based on ALAP Schedule." (2020).
3. Critical path and row priority, prioritize tasks with smaller row number in addition to priority 2. We also enforces potrf>trsm>gemm
*/

enum PrioKind { no = 0, row = 1, cp = 2, cp_row = 3};

void cholesky(const int n_threads, const int verb, const int block_size, const int num_blocks, const int nprows, const int npcols,
              const PrioKind prio_kind, const bool log, const bool deps_log, const bool test, const int accumulate_parallel, const int upper_block_size)
{
    const int rank = comm_rank();
    const int n_ranks = comm_size();
    const int matrix_size = block_size * num_blocks;
    assert(nprows * npcols == n_ranks);
    std::atomic<long long int> potrf_us_t(0);
    std::atomic<long long int> trsm_us_t(0);
    std::atomic<long long int> gemm_us_t(0);
    std::atomic<long long int> accu_us_t(0);

    // Warmup MKL
    {
        Eigen::MatrixXd A = Eigen::MatrixXd::Identity(256,256);
        Eigen::MatrixXd B = Eigen::MatrixXd::Identity(256,256);
        Eigen::MatrixXd C = Eigen::MatrixXd::Identity(256,256);
        for(int i = 0; i < 10; i++) {
            cblas_dgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, 256, 256, 256, 1.0, A.data(), 256, B.data(), 256, 1.0, C.data(), 256);
        }
    }

    // Compute random sizes
    std::mt19937 gen(2020);
    assert(upper_block_size <= 2*block_size);
    const int lower_block_size = 2*block_size - upper_block_size;
    std::uniform_int_distribution<> distrib(lower_block_size,upper_block_size); // average is block_size
    if(rank == 0) printf("lower_block_size %d, upper_block_size %d\n", lower_block_size, upper_block_size);
    std::vector<int> block_sizes(num_blocks, block_size);
    {
        int n = 0;
        for(int i = 0; i < num_blocks-1; i++) {
            int bs = std::min(matrix_size - n, distrib(gen));
            n += bs;
            block_sizes[i] = bs;
        }
        assert(matrix_size - n >= 0);
        block_sizes[num_blocks-1] = matrix_size - n;
    }
    int total = std::accumulate(block_sizes.begin(), block_sizes.end(), 0);
    assert(total == matrix_size);
    std::vector<int> block_displ(num_blocks+1, 0);
    for(int i = 1; i < num_blocks+1; i++) {
        block_displ[i] = block_displ[i-1] + block_sizes[i-1];
    }
    std::vector<int> block_sizes_lda(num_blocks, 0);
    for(int i = 0; i < num_blocks; i++) {
        block_sizes_lda[i] = std::max(1, block_sizes[i]);
    }
    assert(block_displ[num_blocks] == matrix_size);
    if(rank == 0) {
        printf("block sizes: ");
        for(int i = 0; i < num_blocks; i++) {
            assert(block_sizes[i] >= 0);
            printf("%d ", block_sizes[i]);
        };
        printf("\n");
        printf("block displ: ");
        for(int i = 0; i < num_blocks+1; i++) {
            assert(block_displ[i] >= 0);
            printf("%d ", block_displ[i]);
        };
        printf("\n");
    }

    // Map tasks to ranks
    auto block_2_rank = [&](int i, int j) {
        assert(i >= 0 && i < num_blocks);
        assert(j >= 0 && j < num_blocks);
        int r = (j % npcols) * nprows + (i % nprows);
        assert(r >= 0 && r < n_ranks);
        return r;
    };

    const int rank_row = (rank % nprows);
    const int rank_col = (rank / nprows);
    auto block_2_rank_row = [&](int i, int j) {
        return i % nprows;
    };
    auto block_2_rank_col = [&](int i, int j) {
        return j % npcols;
    };

    // Map threads to ranks
    auto block_2_thread = [&](int i, int j) {
        int ii = i / nprows;
        int jj = j / npcols;
        int num_blocksit = num_blocks / nprows;
        return (ii + jj * num_blocksit) % n_threads;
    };

    // Initializes the matrix
    auto val = [&](int i, int j) { return 1/(double)((i-j)*(i-j)+1); };
    vector<unique_ptr<MatrixXd>> blocks(num_blocks*num_blocks);
    for (int ii=0; ii<num_blocks; ii++) {
        for (int jj=0; jj<num_blocks; jj++) {
            auto val_loc = [&](int i, int j) { return val(block_displ[ii]+i,block_displ[jj]+j); };
            if(ii >= jj) {
                if(block_2_rank(ii,jj) == rank) {
                    blocks[ii+jj*num_blocks]=make_unique<MatrixXd>(block_sizes[ii],block_sizes[jj]);
                    *blocks[ii+jj*num_blocks]=MatrixXd::NullaryExpr(block_sizes[ii],block_sizes[jj], val_loc);
                } else {
                    blocks[ii+jj*num_blocks]=make_unique<MatrixXd>(0,0);
                }
            }
        }
    }

    // Holds the temporary matrices result of gemm to be accumulated by accu
    // Each block holds data to be accumulated into a given block[ii+jj*num_blocks]
    struct acc_data {
        std::map<int, std::unique_ptr<MatrixXd>> to_accumulate; // to_accumulate[k] holds matrix result of gemm(k,i,j)
        std::mutex mtx; // Protects that map
    };
    std::vector<acc_data> gemm_results(num_blocks*num_blocks); // gemm_results[ii+jj*num_blocks] holds the data to be accumulated into blocks[ii+jj*num_blocks]

    // Set priorities
    auto potf_block_2_prio = [&](int j) {
        if (prio_kind == PrioKind::cp_row) {
            return (double)(9*(num_blocks - j)-1) + 18 * num_blocks * num_blocks;
        }
        else if(prio_kind == PrioKind::cp) {
            return (double)(9*(num_blocks - j)-1);
        }
        else if(prio_kind == PrioKind::row) {
            return 3.0*(double)(num_blocks-j);
        }
        else {
            return 3.0;
        }
    };
    auto trsm_block_2_prio = [&](int2 ij) {
        if (prio_kind == PrioKind::cp_row) {
            return (double)((num_blocks - ij[0]) + num_blocks * (9.0 * num_blocks - 9.0 * ij[1] - 2.0) + 9 * num_blocks * num_blocks);
        }
        else if(prio_kind == PrioKind::cp) {
            return (double)(9*(num_blocks - ij[1])-2);
        }
        else if(prio_kind == PrioKind::row) {
            return 2.0*(double)(num_blocks - ij[0]);
        }
        else {
            return 2.0;
        }
    };
    auto gemm_block_2_prio = [&](int3 kij) {
        if (prio_kind == PrioKind::cp_row) {
            if (accumulate_parallel) {
                return (double)(num_blocks - kij[1]) + num_blocks * (9.0 * num_blocks - 9.0 * kij[2] - 2.0);
            }
            else {
                return (double)(num_blocks - kij[1]) + num_blocks * (9.0 * num_blocks - 3.0 * kij[2] - 6.0 * kij[0] - 2.0);
            }
        }
        else if(prio_kind == PrioKind::cp) {
            return (double)(9*num_blocks-9*kij[2]-2);
        }
        else if(prio_kind == PrioKind::row) {
            return (double)(num_blocks - kij[1]);
        }
        else {
            return 1.0;
        }
    };
    // Names
    auto potrf_name = [](int j) {
        return "POTRF_" + to_string(j);
    };
    auto trsm_name = [](int2 ij) {
        return "TRSM_" + to_string(ij[0]) + "_" + to_string(ij[1]);
    };
    auto gemm_name = [](int3 kij) {
        return "GEMM_" + to_string(kij[0]) + "_" + to_string(kij[1]) + "_" + to_string(kij[2]);
    };
    auto accu_name = [](int3 kij) {
        return "ACCU_" + to_string(kij[0]) + "_" + to_string(kij[1]) + "_" + to_string(kij[2]);
    };

    const int num_blocksmax = 15;
    MPI_Barrier(MPI_COMM_WORLD);
    if(comm_rank() == 0) {
        printf("Block -> Rank\n");
        for(int i = 0; i < min(num_blocksmax, num_blocks); i++) {
            for(int j = 0; j < min(num_blocksmax, num_blocks); j++) {
                if(i >= j) {
                    printf("%2d ", block_2_rank(i, j));
                }
            }
            printf("\n");
        }
        printf("Potf/trsm -> Priority\n");
        for(int i = 0; i < min(num_blocksmax, num_blocks); i++) {
            for(int j = 0; j < min(num_blocksmax, num_blocks); j++) {
                if(i == j) {
                    printf("%5f ", potf_block_2_prio(i));
                } else if (i > j) {
                    printf("%5f ", trsm_block_2_prio({i,j}));
                };
            }
            printf("\n");
        }
        printf("Gemm -> Priority\n");
        for(int k = 0; k < min(num_blocksmax, num_blocks); k++) {
            printf("k = %d\n", k);
            for(int i = 0; i < min(num_blocksmax, num_blocks); i++) {
                for(int j = 0; j < min(num_blocksmax, num_blocks); j++) {
                    if(i >= j) {
                        if(k < j) {
                            printf("%5f ", gemm_block_2_prio({k,i,j}));
                        } else {
                            printf(".     ");
                        }
                    }
                }
                printf("\n");
            }
        }
    }
    for(int r = 0; r < ttor::comm_size(); r++) {
        if(r == comm_rank()) {
            printf("[%d] Block -> thread\n", r);
            for(int i = 0; i < min(num_blocksmax, num_blocks); i++) {
                for(int j = 0; j < min(num_blocksmax, num_blocks); j++) {
                    if(i >= j && block_2_rank(i,j) == r) {
                        printf("%2d ", block_2_thread(i, j));
                    } else {
                        printf(" . ");
                    }
                }
                printf("\n");
            }
        }
        MPI_Barrier(MPI_COMM_WORLD);
    }

    // Initialize the communicator structure
    Communicator comm(MPI_COMM_WORLD, verb);

    // Initialize the runtime structures
    Threadpool tp(n_threads, &comm, verb, "Wk_Chol_" + to_string(rank) + "_");
    Taskflow<int>  potrf(&tp, verb);
    Taskflow<int2> trsm(&tp, verb);
    Taskflow<int3> gemm(&tp, verb);
    Taskflow<int3> accu(&tp, verb);

    Logger logger(1000000);
    if(log) {
        tp.set_logger(&logger);
        comm.set_logger(&logger);
    }

    DepsLogger dlog(1000000);

    // Send a potrf'ed pivot A(k,k) and trigger trsms below requiring A(k,k)
    auto am_trsm = comm.make_large_active_msg(
            [&](int& j) {
                int off = (nprows + rank_row - block_2_rank_row(j,j)) % nprows;
                assert(off > 0); // Can't be me
                assert(off < nprows);
                for (int i = j + off; i < num_blocks; i += nprows) {
                    assert(block_2_rank(i,j) == rank);
                    trsm.fulfill_promise({i,j});
                }
            },
            [&](int& j) {
                blocks[j+j*num_blocks]->resize(block_sizes[j],block_sizes[j]);
                return blocks[j+j*num_blocks]->data();
            },
            [&](int&){
                return;
            });

    /**
     * j is the pivot's position at A(j,j)
     */
    potrf.set_task([&](int j) { // A[j,j] -> A[j,j]
            assert(block_2_rank(j,j) == rank);
            timer t_ = wctime();
            LAPACKE_dpotrf(LAPACK_COL_MAJOR, 'L', block_sizes[j], blocks[j+j*num_blocks]->data(), block_sizes_lda[j]);
            timer t__ = wctime();
            potrf_us_t += 1e6 * elapsed(t_, t__);
        })
        .set_fulfill([&](int j) { // Triggers all trsms on rows i > j, A[i,j]
            assert(block_2_rank(j,j) == rank);
            if(deps_log) {
                for(int i = j+1; i < num_blocks; i++) {
                    dlog.add_event(make_unique<DepsEvent>(potrf.name(j), trsm.name({i,j})));
                }
            }
            // Trigger myself
            for (int i = j + nprows; i < num_blocks; i += nprows) {
                assert(block_2_rank(i,j) == rank);
                trsm.fulfill_promise({i,j});
            }
            // Send to other procs in column
            auto Ljjv = view<double>(blocks[j+j*num_blocks]->data(), block_sizes[j]*block_sizes[j]);
            for(int p = 0; p < nprows; p++) {
                if(j+p >= num_blocks) break;
                int dest = block_2_rank(j+p,j);
                if(dest != rank) {
                    am_trsm->send_large(dest, Ljjv, j);
                }
            }

        })
        .set_indegree([&](int j) {
            assert(block_2_rank(j,j) == rank);
            if(accumulate_parallel) {
                return j == 0 ? 1 : j; // Need j accumulations into (j,j) to trigger the potf
            } else {
                return 1;
            }
        })
        .set_priority(potf_block_2_prio)
        .set_mapping([&](int j) {
            assert(block_2_rank(j,j) == rank);
            return block_2_thread(j, j);
        })
        .set_name([&](int j) { // This is just for debugging and profiling
            return potrf_name(j);
        });

    // Sends a panel (trsm'ed block A(i,j)) and trigger gemms requiring A(i,j)
    auto am_gemm = comm.make_large_active_msg(
        [&](int& i, int& j) {
            if(block_2_rank_row(i,j) == rank_row) {
                const int off_right = (npcols + rank_col - block_2_rank_col(i,j)) % npcols;
                assert(off_right > 0); // Can't be me
                assert(off_right < npcols);
                assert(i >= 0 && i < num_blocks);
                assert(j >= 0 && j < num_blocks);
                for (int k = j + off_right; k < i; k += npcols) {
                    assert(block_2_rank(i,k) == rank);
                    gemm.fulfill_promise({j,i,k});
                }
            }
            if(block_2_rank_col(i,i) == rank_col) {
                const int off_below = (nprows + rank_row - block_2_rank_row(i,i)) % nprows;
                assert(off_below >= 0); // Could be me
                assert(off_below < nprows);
                for (int k = i + off_below; k < num_blocks; k += nprows) {
                    assert(block_2_rank(k,i) == rank);
                    gemm.fulfill_promise({j,k,i});
                }
            }
        },
        [&](int& i, int& j) {
            blocks[i+j*num_blocks]->resize(block_sizes[i],block_sizes[j]);
            return blocks[i+j*num_blocks]->data();
        },
        [&](int& i, int& j) {
            return;
        });

    /**
     * ij is (Row, Col) of the block in the matrix at A(i,j)
     **/
    trsm.set_task([&](int2 ij) { // A[j,j] & A[i,j] -> A[i,j]
            int i=ij[0];
            int j=ij[1];
            assert(block_2_rank(i,j) == rank);
            assert(i > j);
            timer t_ = wctime();
            cblas_dtrsm(CblasColMajor, CblasRight, CblasLower, CblasTrans, CblasNonUnit,
                block_sizes[i], block_sizes[j], 1.0, blocks[j+j*num_blocks]->data(), block_sizes_lda[j], blocks[i+j*num_blocks]->data(), block_sizes_lda[i]);
            timer t__ = wctime();
            trsm_us_t += 1e6 * elapsed(t_, t__);
        })
        .set_fulfill([&](int2 ij) {
            int i=ij[0];
            int j=ij[1];
            assert(block_2_rank(i,j) == rank);
            assert(i > j);
            if(deps_log) {
                for(int k = j+1; k < num_blocks; k++) {
                    dlog.add_event(make_unique<DepsEvent>(trsm.name(ij), gemm.name({j,std::max(i,k),std::min(i,k)})));
                }
            }
            // Local
            // Careful to not count the pivot (syrk) twice
            for (int k = j + npcols; k < i; k += npcols) {
                assert(block_2_rank(i,k) == rank);
                gemm.fulfill_promise({j,i,k});
            }
            if(block_2_rank_col(i,i) == rank_col) {
                int off_below = (nprows + rank_row - block_2_rank_row(i,i)) % nprows;
                for (int k = i + off_below; k < num_blocks; k += nprows) {
                    assert(block_2_rank(k,i) == rank);
                    gemm.fulfill_promise({j,k,i});
                }
            }
            // Remote
            auto Lijv = view<double>(blocks[i+j*num_blocks]->data(), block_sizes[i]*block_sizes[j]);
            std::set<int> dests;
            for (int c = 0; c < npcols; c++) {
                if(j+c >= num_blocks) break;
                int dest = block_2_rank(i,j+c);
                if(dest != rank) dests.insert(dest);
            }
            for (int r = 0; r < nprows; r++) {
                if(i+r >= num_blocks) break;
                int dest = block_2_rank(i+r,i);
                if(dest != rank) dests.insert(dest);
            }
            for(auto& dest: dests) {
                am_gemm->send_large(dest, Lijv, i, j);
            }
        })
        .set_indegree([&](int2 ij) {
            assert(block_2_rank(ij[0],ij[1]) == rank);
            if(accumulate_parallel) {
                return 1 + ij[1]; // Potrf above and all gemms before
            } else {
                return 1 + (ij[1] == 0 ? 0 : 1); // Potrf and last gemm before
            }
        })
        .set_priority(trsm_block_2_prio)
        .set_mapping([&](int2 ij) {
            assert(block_2_rank(ij[0],ij[1]) == rank);
            return block_2_thread(ij[0], ij[1]);
        })
        .set_name([&](int2 ij) { // This is just for debugging and profiling
            return trsm_name(ij);
        });

    /**
     * k is the step (the pivot's position), ij are Row and Column, at A(i,j)
     **/
    gemm.set_task([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            const int k=kij[0];
            const int i=kij[1];
            const int j=kij[2];
            assert(j <= i);
            assert(k < j);
            std::unique_ptr<MatrixXd> Atmp;
            MatrixXd* Aij;
            double beta = 1.0;
            if(accumulate_parallel) {
                beta = 0.0;
                Atmp = make_unique<MatrixXd>(block_sizes[i], block_sizes[j]); // The matrix is allocated with garbage. The 0 in the BLAS call make sure its overwritten by 0's before doing any math
                Aij = Atmp.get();
            } else {
                beta = 1.0;
                Aij = blocks[i+j*num_blocks].get();
            }
            assert(Aij->rows() == block_sizes[i] && Aij->cols() == block_sizes[j]);
            timer t_ = wctime();
            if (i == j) {
                cblas_dsyrk(CblasColMajor, CblasLower, CblasNoTrans,
                    block_sizes[i], block_sizes[k], -1.0, blocks[i+k*num_blocks]->data(), block_sizes_lda[i], beta, Aij->data(), block_sizes_lda[i]);
            } else {
                cblas_dgemm(CblasColMajor, CblasNoTrans, CblasTrans,
                    block_sizes[i], block_sizes[j], block_sizes[k], -1.0, blocks[i+k*num_blocks]->data(), block_sizes_lda[i], blocks[j+k*num_blocks]->data(), block_sizes_lda[j], beta, Aij->data(), block_sizes_lda[i]);
            }
            timer t__ = wctime();
            gemm_us_t += 1e6 * elapsed(t_, t__);
            if(accumulate_parallel) {
                lock_guard<mutex> lock(gemm_results[i+j*num_blocks].mtx);
                gemm_results[i+j*num_blocks].to_accumulate[k] = move(Atmp);
            }
        })
        .set_fulfill([&](int3 kij) {
            const int k=kij[0];
            const int i=kij[1];
            const int j=kij[2];
            assert(block_2_rank(kij[1],kij[2]) == rank);
            if(accumulate_parallel) {
                if(deps_log) {
                    dlog.add_event(make_unique<DepsEvent>(gemm.name(kij), accu.name(kij)));
                }
                accu.fulfill_promise(kij);
            } else {
                if (k < j-1) {
                    if(deps_log) {
                        dlog.add_event(make_unique<DepsEvent>(gemm.name(kij), gemm.name({k+1, i, j})));
                    }
                    gemm.fulfill_promise({k+1, i, j});
                } else {
                    if (i == j) {
                        if(deps_log) {
                            dlog.add_event(make_unique<DepsEvent>(gemm.name(kij), potrf.name(i)));
                        }
                        potrf.fulfill_promise(i);
                    } else {
                        if(deps_log) {
                            dlog.add_event(make_unique<DepsEvent>(gemm.name(kij), trsm.name({i,j})));
                        }
                        trsm.fulfill_promise({i,j});
                    }
                }
            }
        })
        .set_indegree([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            if(accumulate_parallel) {
                return kij[1] == kij[2] ? 1 : 2; // Either one potf or two trsms
            } else {
                return (kij[1] == kij[2] ? 1 : 2) + (kij[0] == 0 ? 0 : 1); // one potrf or two trsms + the gemm before
            }
        })
        .set_priority(gemm_block_2_prio)
        .set_mapping([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            return block_2_thread(kij[1], kij[2]); // IMPORTANT if accumulate_parallel is true
        })
        .set_binding([&](int3 kij) {
            return false; // If we accumulate in parallel, there is no order for the gemm so it doesnt matter ; If we don't then we do the gemm in sequence anyway
        }).set_name([&](int3 kij) { // This is just for debugging and profiling
            return gemm_name(kij);
        });

    /**
     * k is the step (the pivot's position), ij are Row and Column, at A(i,j)
     **/
    accu.set_task([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            int k=kij[0]; // Step (gemm's pivot)
            int i=kij[1]; // Row
            int j=kij[2]; // Col
            assert(j <= i);
            assert(k < j);
            std::unique_ptr<Eigen::MatrixXd> Atmp;
            {
                lock_guard<mutex> lock(gemm_results[i+j*num_blocks].mtx);
                Atmp = move(gemm_results[i+j*num_blocks].to_accumulate[k]);
                gemm_results[i+j*num_blocks].to_accumulate.erase(k);
            }
            timer t_ = wctime();
            *blocks[i+j*num_blocks] += (*Atmp);
            timer t__ = wctime();
            accu_us_t += 1e6 * elapsed(t_, t__);
        })
        .set_fulfill([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            int k=kij[0];
            int i=kij[1];
            int j=kij[2];
            assert(j <= i);
            assert(k < j);
            if(i == j) {
                if(deps_log) {
                    dlog.add_event(make_unique<DepsEvent>(accu.name(kij), potrf.name(i)));
                }
                potrf.fulfill_promise(i);
            } else {
                if(deps_log) {
                    dlog.add_event(make_unique<DepsEvent>(accu.name(kij), trsm.name({i,j})));
                }
                trsm.fulfill_promise({i,j});
            }
        })
        .set_indegree([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            return 1;
        })
        .set_mapping([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            return block_2_thread(kij[1], kij[2]); // IMPORTANT. Every (i,j) should map to a given fixed thread
        })
        .set_priority(gemm_block_2_prio)
        .set_binding([&](int3 kij) {
            assert(block_2_rank(kij[1],kij[2]) == rank);
            return true; // IMPORTANT
        })
        .set_name([&](int3 kij) { // This is just for debugging and profiling
            return accu_name(kij);
        });

    printf("Starting Cholesky factorization...\n");
    MPI_Barrier(MPI_COMM_WORLD);
    timer t0 = wctime();
    if (rank == 0){
        potrf.fulfill_promise(0);
    }
    tp.join();
    MPI_Barrier(MPI_COMM_WORLD);
    timer t1 = wctime();
    double total_time = elapsed(t0, t1);
    printf("Done with Cholesky factorization...\n");
    printf("Elapsed time: %e\n", total_time);
    printf("Potrf time: %e\n", potrf_us_t.load() * 1e-6);
    printf("Trsm time: %e\n", trsm_us_t.load() * 1e-6);
    printf("Gemm time: %e\n", gemm_us_t.load() * 1e-6);
    printf("Accu time: %e\n", accu_us_t.load() * 1e-6);

    printf("++++rank nranks n_threads matrix_size block_size num_blocks priority_kind accumulate upper_block_size total_time\n");
    printf("[%d]>>>>%d %d %d %d %d %d %d %d %d %e\n",rank,rank,n_ranks,n_threads,matrix_size,block_size,num_blocks,(int)prio_kind,(int)accumulate_parallel,upper_block_size,total_time);

    if(log) {
        std::ofstream logfile;
        string filename = "ttor_dist_"+to_string(block_size)+"_"+to_string(num_blocks)+"_"+ to_string(n_ranks)+"_"+to_string(n_threads)+"_"+to_string(prio_kind)+".log."+to_string(rank);
        logfile.open(filename);
        logfile << logger;
        logfile.close();
    }

    if(deps_log) {
        std::ofstream depsfile;
        string depsfilename = "deps_ttor_dist_"+to_string(block_size)+"_"+to_string(num_blocks)+"_"+ to_string(n_ranks)+"_"+to_string(n_threads)+"_"+to_string(prio_kind)+".dot."+to_string(rank);
        depsfile.open(depsfilename);
        depsfile << dlog;
        depsfile.close();
    }

    if(test) {
        printf("Starting sending matrix to rank 0...\n");
        MatrixXd A = MatrixXd::NullaryExpr(matrix_size,matrix_size,val);
        MatrixXd L = MatrixXd::Zero(matrix_size,matrix_size);
        // Send the matrix to rank 0
        for (int ii=0; ii<num_blocks; ii++) {
            for (int jj=0; jj<num_blocks; jj++) {
                if (jj<=ii)  {
                    int owner = block_2_rank(ii,jj);
                    MPI_Status status;
                    if (rank == 0 && rank != owner) { // Careful with deadlocks here
                        blocks[ii+jj*num_blocks] = make_unique<Eigen::MatrixXd>(block_sizes[ii], block_sizes[jj]);
                        MPI_Recv(blocks[ii+jj*num_blocks]->data(), block_sizes[ii]*block_sizes[jj], MPI_DOUBLE, owner, 0, MPI_COMM_WORLD, &status);
                    } else if (rank != 0 && rank == owner) {
                        MPI_Send(blocks[ii+jj*num_blocks]->data(), block_sizes[ii]*block_sizes[jj], MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
                    }
                }
            }
        }

        // Rank 0 test
        if(rank == 0) {
            printf("Starting test on rank 0...\n");
            for (int ii=0; ii<num_blocks; ii++) {
                for (int jj=0; jj<num_blocks; jj++) {
                    if (jj<=ii) {
                        L.block(block_displ[ii],block_displ[jj],block_sizes[ii],block_sizes[jj])=*blocks[ii+jj*num_blocks];
                    }
                }
            }
            auto L1=L.triangularView<Lower>();
            VectorXd x = VectorXd::Random(matrix_size);
            VectorXd b = A*x;
            VectorXd bref = b;
            L1.solveInPlace(b);
            L1.transpose().solveInPlace(b);
            double error = (b - x).norm() / x.norm();
            printf("\n=> Error solve %e\n\n", error);
            if(error > 1e-6) {
                printf("\n\nERROR: error is too large!\n\n");
                exit(1);
            }
        }
    }
}

int main(int argc, const char **argv)
{
    int req = MPI_THREAD_FUNNELED;
    int prov = -1;

    MPI_Init_thread(NULL, NULL, req, &prov);

    assert(prov == req);

    std::stringstream sstr;
    sstr << comm_size();
    const std::string comm_size_str = sstr.str();

    cxxopts::Options options("2d_cholesky", "2D dense cholesky using TaskTorrent");
    options.add_options()
        ("help", "Print help")
        ("n_threads", "Number of threads", cxxopts::value<int>()->default_value("2"))
        ("verb", "Verbosity level", cxxopts::value<int>()->default_value("0"))
        ("block_size", "Block size", cxxopts::value<int>()->default_value("5"))
        ("num_blocks", "Number of blocks", cxxopts::value<int>()->default_value("10"))
        ("nprows", "Number of processors accross rows", cxxopts::value<int>()->default_value("1"))
        ("npcols", "Number of processors accross columns", cxxopts::value<int>()->default_value(comm_size_str.c_str()))
        ("kind", "Priority kind", cxxopts::value<int>()->default_value("0"))
        ("log", "Enable logging", cxxopts::value<bool>()->default_value("false"))
        ("depslog", "Enable dependency logging", cxxopts::value<bool>()->default_value("false"))
        ("test", "Test or not", cxxopts::value<bool>()->default_value("true"))
        ("accumulate", "Accumulate block GEMMs in parallel", cxxopts::value<bool>()->default_value("false"))
        ("upper_block_size", "Upper block size", cxxopts::value<int>()->default_value("-1"));
    auto result = options.parse(argc, argv);

    const int n_threads = result["n_threads"].as<int>();
    const int verb = result["verb"].as<int>();
    const int block_size = result["block_size"].as<int>();
    const int num_blocks = result["num_blocks"].as<int>();
    const int nprows = result["nprows"].as<int>();
    const int npcols = result["npcols"].as<int>();
    const PrioKind kind = (PrioKind) result["kind"].as<int>();
    const bool log = result["log"].as<bool>();
    const bool depslog = result["depslog"].as<bool>();
    const bool test = result["test"].as<bool>();
    const bool accumulate = result["accumulate"].as<bool>();
    const int upper_block_size = (result["upper_block_size"].as<int>() == -1 ? block_size : result["upper_block_size"].as<int>());

    assert(block_size > 0);
    assert(num_blocks > 0);
    assert(n_threads > 0);
    assert(verb >= 0);
    assert(nprows >= 0);
    assert(npcols >= 0);
    assert(upper_block_size >= block_size && upper_block_size <= 2 * block_size);

    if (result.count("help")) {
        std::cout << options.help({"", "Group"}) << endl;
        exit(0);
    }
    if(comm_rank() == 0) printf("Arguments: block_size (size of blocks) %d\nnum_blocks (# of blocks) %d\nn_threads %d\nverb %d\nnprows %d\nnpcols %d\nkind %d\nlog %d\ndeplog %d\ntest %d\naccumulate %d\nupper_block_size %d\n", block_size, num_blocks, n_threads, verb, nprows, npcols, (int)kind, log, depslog, test, accumulate, upper_block_size);

    cholesky(n_threads, verb, block_size, num_blocks, nprows, npcols, kind, log, depslog, test, accumulate, upper_block_size);

    MPI_Finalize();
}


//    Copyright (C) Dimitrios Michail 2019 - 2021.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          https://www.boost.org/LICENSE_1_0.txt)

#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
#include "doctest.h"

#include <iostream>

#include <boost/graph/adjacency_list.hpp>
#include <boost/property_map/property_map.hpp>

#include <parmcb/parmcb.hpp>
#include <parmcb/util.hpp>

using namespace boost;
typedef adjacency_list<vecS, vecS, undirectedS, no_property, property<edge_weight_t, double> > Graph;
typedef graph_traits<Graph>::edge_descriptor Edge;

void create_graph(Graph& graph) {
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    auto v0 = add_vertex(graph);
    auto v1 = add_vertex(graph);
    auto v2 = add_vertex(graph);
    auto v3 = add_vertex(graph);
    auto v4 = add_vertex(graph);
    auto v5 = add_vertex(graph);
    auto v6 = add_vertex(graph);
    auto v7 = add_vertex(graph);
    auto v8 = add_vertex(graph);
    auto v9 = add_vertex(graph);
    auto v10 = add_vertex(graph);
    auto v11 = add_vertex(graph);
    auto v12 = add_vertex(graph);
    auto v13 = add_vertex(graph);
    auto v14 = add_vertex(graph);
    auto v15 = add_vertex(graph);
    auto v16 = add_vertex(graph);
    (void) v16;

    auto e01 = add_edge(v0, v1, graph).first;
    weight[e01] = 1.0;
    auto e12 = add_edge(v1, v2, graph).first;
    weight[e12] = 1.0;
    auto e23 = add_edge(v2, v3, graph).first;
    weight[e23] = 100.0;
    auto e34 = add_edge(v3, v4, graph).first;
    weight[e34] = 1.0;
    auto e45 = add_edge(v4, v5, graph).first;
    weight[e45] = 1.0;
    auto e05 = add_edge(v0, v5, graph).first;
    weight[e05] = 1.0;
    auto e56 = add_edge(v5, v6, graph).first;
    weight[e56] = 1.0;
    auto e27 = add_edge(v2, v7, graph).first;
    weight[e27] = 2.0;
    auto e79 = add_edge(v7, v9, graph).first;
    weight[e79] = 1.0;
    auto e38 = add_edge(v3, v8, graph).first;
    weight[e38] = 5.0;
    auto e108 = add_edge(v10, v8, graph).first;
    weight[e108] = 1.0;
    auto e109 = add_edge(v10, v9, graph).first;
    weight[e109] = 1.0;
    auto e1112 = add_edge(v11, v12, graph).first;
    weight[e1112] = 3.0;
    auto e12_13 = add_edge(v12, v13, graph).first;
    weight[e12_13] = 1.0;
    auto e13_14 = add_edge(v13, v14, graph).first;
    weight[e13_14] = 1.0;
    auto e14_15 = add_edge(v14, v15, graph).first;
    weight[e14_15] = 1.0;
    auto e12_15 = add_edge(v12, v15, graph).first;
    weight[e12_15] = 1.0;
}

TEST_CASE("sequential sva signed"){
    Graph graph;
    create_graph(graph);
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    CHECK(num_vertices(graph) == 17);
    CHECK(num_edges(graph) == 17);

    //
    //   0 -- 1 -- 2 -- 7 -- 9
    //   |         |         |
    //   5 -- 4 -- 3 -- 8 -- 10
    //   |
    //   6    11 -- 12 -- 13
    //              |      |
    //  16          15 --  14
    //

    std::list<std::list<Edge>> cycles;
    double mcb_weight = parmcb::mcb_sva_signed(graph, weight, std::back_inserter(cycles));

    for (auto it = cycles.begin(); it != cycles.end(); it++) {
        auto cycle = *it;
        CHECK(parmcb::is_cycle(graph, cycle));
    }

    CHECK(cycles.size() == 3);
    CHECK(mcb_weight == 124.0);

}

TEST_CASE("parallel sva signed"){

    Graph graph;
    create_graph(graph);
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    CHECK(num_vertices(graph) == 17);
    CHECK(num_edges(graph) == 17);

    //
    //   0 -- 1 -- 2 -- 7 -- 9
    //   |         |         |
    //   5 -- 4 -- 3 -- 8 -- 10
    //   |
    //   6    11 -- 12 -- 13
    //              |      |
    //  16          15 --  14
    //

    std::list<std::list<Edge>> cycles;
    double mcb_weight = parmcb::mcb_sva_signed_tbb(graph, weight, std::back_inserter(cycles));

    for (auto it = cycles.begin(); it != cycles.end(); it++) {
        auto cycle = *it;
        CHECK(parmcb::is_cycle(graph, cycle));
    }

    CHECK(cycles.size() == 3);
    CHECK(mcb_weight == 124.0);

}

TEST_CASE("sequential sva fvs trees"){

    Graph graph;
    create_graph(graph);
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    CHECK(num_vertices(graph) == 17);
    CHECK(num_edges(graph) == 17);

    //
    //   0 -- 1 -- 2 -- 7 -- 9
    //   |         |         |
    //   5 -- 4 -- 3 -- 8 -- 10
    //   |
    //   6    11 -- 12 -- 13
    //              |      |
    //  16          15 --  14
    //

    std::list<std::list<Edge>> cycles;
    double mcb_weight = parmcb::mcb_sva_fvs_trees(graph, weight, std::back_inserter(cycles));

    for (auto it = cycles.begin(); it != cycles.end(); it++) {
        auto cycle = *it;
        CHECK(parmcb::is_cycle(graph, cycle));
    }

    CHECK(cycles.size() == 3);
    CHECK(mcb_weight == 124.0);

}

TEST_CASE("parallel sva fvs trees"){

    Graph graph;
    create_graph(graph);
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    CHECK(num_vertices(graph) == 17);
    CHECK(num_edges(graph) == 17);

    //
    //   0 -- 1 -- 2 -- 7 -- 9
    //   |         |         |
    //   5 -- 4 -- 3 -- 8 -- 10
    //   |
    //   6    11 -- 12 -- 13
    //              |      |
    //  16          15 --  14
    //

    std::list<std::list<Edge>> cycles;
    double mcb_weight = parmcb::mcb_sva_fvs_trees_tbb(graph, weight, std::back_inserter(cycles));

    for (auto it = cycles.begin(); it != cycles.end(); it++) {
        auto cycle = *it;
        CHECK(parmcb::is_cycle(graph, cycle));
    }

    CHECK(cycles.size() == 3);
    CHECK(mcb_weight == 124.0);

}

TEST_CASE("sequential sva iso trees"){

    Graph graph;
    create_graph(graph);
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    CHECK(num_vertices(graph) == 17);
    CHECK(num_edges(graph) == 17);

    //
    //   0 -- 1 -- 2 -- 7 -- 9
    //   |         |         |
    //   5 -- 4 -- 3 -- 8 -- 10
    //   |
    //   6    11 -- 12 -- 13
    //              |      |
    //  16          15 --  14
    //

    std::list<std::list<Edge>> cycles;
    double mcb_weight = parmcb::mcb_sva_iso_trees(graph, weight, std::back_inserter(cycles));

    for (auto it = cycles.begin(); it != cycles.end(); it++) {
        auto cycle = *it;
        CHECK(parmcb::is_cycle(graph, cycle));
    }

    CHECK(cycles.size() == 3);
    CHECK(mcb_weight == 124.0);

}

TEST_CASE("parallel sva iso trees"){

    Graph graph;
    create_graph(graph);
    property_map<Graph, edge_weight_t>::type weight = get(edge_weight, graph);

    CHECK(num_vertices(graph) == 17);
    CHECK(num_edges(graph) == 17);

    //
    //   0 -- 1 -- 2 -- 7 -- 9
    //   |         |         |
    //   5 -- 4 -- 3 -- 8 -- 10
    //   |
    //   6    11 -- 12 -- 13
    //              |      |
    //  16          15 --  14
    //

    std::list<std::list<Edge>> cycles;
    double mcb_weight = parmcb::mcb_sva_iso_trees_tbb(graph, weight, std::back_inserter(cycles));

    for (auto it = cycles.begin(); it != cycles.end(); it++) {
        auto cycle = *it;
        CHECK(parmcb::is_cycle(graph, cycle));
    }

    CHECK(cycles.size() == 3);
    CHECK(mcb_weight == 124.0);

}