/****************************************************************************
 * Copyright (c) 2012-2020 by the ArborX authors                            *
 * All rights reserved.                                                     *
 *                                                                          *
 * This file is part of the ArborX library. ArborX is                       *
 * distributed under a BSD 3-clause license. For the licensing terms see    *
 * the LICENSE file in the top-level directory.                             *
 *                                                                          *
 * SPDX-License-Identifier: BSD-3-Clause                                    *
 ****************************************************************************/

#include <ArborX_HaloFinder.hpp>
#include <ArborX_Version.hpp>

#include <Kokkos_Core.hpp>

#include <boost/program_options.hpp>

#include <fstream>

std::vector<ArborX::Point> parsePoints(std::string const &filename,
                                       bool binary = false)
{
  std::cout << "Reading in \"" << filename << "\" in "
            << (binary ? "binary" : "text") << " mode...";
  std::cout.flush();

  std::ifstream input;
  if (!binary)
    input.open(filename);
  else
    input.open(filename, std::ifstream::binary);
  ARBORX_ASSERT(input.good());

  int num_points = 0;
  std::vector<float> x;
  std::vector<float> y;
  std::vector<float> z;
  if (!binary)
  {
    input >> num_points;

    x.reserve(num_points);
    y.reserve(num_points);
    z.reserve(num_points);

    auto read_float = [&input]() {
      return *(std::istream_iterator<float>(input));
    };
    std::generate_n(std::back_inserter(x), num_points, read_float);
    std::generate_n(std::back_inserter(y), num_points, read_float);
    std::generate_n(std::back_inserter(z), num_points, read_float);
  }
  else
  {
    input.read(reinterpret_cast<char *>(&num_points), sizeof(int));

    x.resize(num_points);
    y.resize(num_points);
    z.resize(num_points);
    input.read(reinterpret_cast<char *>(x.data()), num_points * sizeof(float));
    input.read(reinterpret_cast<char *>(y.data()), num_points * sizeof(float));
    input.read(reinterpret_cast<char *>(z.data()), num_points * sizeof(float));
  }
  input.close();
  std::cout << "done\nRead in " << num_points << " points" << std::endl;

  std::vector<ArborX::Point> v(num_points);
  for (int i = 0; i < num_points; i++)
  {
    v[i] = {x[i], y[i], z[i]};
  }

  return v;
}

template <typename... P, typename T>
auto vec2view(std::vector<T> const &in, std::string const &label = "")
{
  Kokkos::View<T *, P...> out(
      Kokkos::view_alloc(label, Kokkos::WithoutInitializing), in.size());
  Kokkos::deep_copy(out, Kokkos::View<T const *, Kokkos::HostSpace,
                                      Kokkos::MemoryTraits<Kokkos::Unmanaged>>{
                             in.data(), in.size()});
  return out;
}

int main(int argc, char *argv[])
{
  using ExecutionSpace = Kokkos::DefaultExecutionSpace;
  using MemorySpace = typename ExecutionSpace::memory_space;

  Kokkos::ScopeGuard guard(argc, argv);

  namespace bpo = boost::program_options;

  std::string filename;
  bool binary;
  bool verify;
  bool print_halo_timers;
  bool print_sizes_centers;
  float linking_length;
  int min_size;

  bpo::options_description desc("Allowed options");
  // clang-format off
    desc.add_options()
        ( "help", "help message" )
        ( "filename", bpo::value<std::string>(&filename), "filename containing data" )
        ( "binary", bpo::bool_switch(&binary)->default_value(false), "binary file indicator")
        ( "linking-length", bpo::value<float>(&linking_length), "linking length (radius)" )
        ( "min-size", bpo::value<int>(&min_size)->default_value(2), "minimum halo size")
        ( "verify", bpo::bool_switch(&verify)->default_value(false), "verify connected components")
        ( "print-halo-timers", bpo::bool_switch(&print_halo_timers)->default_value(false), "print halo timers")
        ( "output-sizes-and-centers", bpo::bool_switch(&print_sizes_centers)->default_value(false), "print halo sizes and centers")
        ;
  // clang-format on
  bpo::variables_map vm;
  bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
  bpo::notify(vm);

  if (vm.count("help") > 0)
  {
    std::cout << desc << '\n';
    return 1;
  }

  std::cout << "ArborX version: " << ArborX::version() << std::endl;
  std::cout << "ArborX hash   : " << ArborX::gitCommitHash() << std::endl;

  // read in data
  auto const primitives =
      vec2view<MemorySpace>(parsePoints(filename, binary), "primitives");

  ExecutionSpace exec_space;

  Kokkos::View<int *, MemorySpace> halos_indices("halos_indices", 0);
  Kokkos::View<int *, MemorySpace> halos_offset("halos_offset", 0);
  ArborX::HaloFinder::findHalos(exec_space, primitives, halos_indices,
                                halos_offset, linking_length, min_size,
                                print_halo_timers, verify);

  if (print_sizes_centers)
  {
    auto const num_halos = static_cast<int>(halos_offset.size()) - 1;

    Kokkos::View<ArborX::Point *, MemorySpace> halos_centers(
        Kokkos::ViewAllocateWithoutInitializing("centers"), num_halos);
    Kokkos::parallel_for(
        "compute centers",
        Kokkos::RangePolicy<ExecutionSpace>(exec_space, 0, num_halos),
        KOKKOS_LAMBDA(int const i) {
          int halo_size = halos_offset(i + 1) - halos_offset(i);

          ArborX::Point halo_center{0.f, 0.f, 0.f};
          for (int j = halos_offset(i); j < halos_offset(i + 1); j++)
          {
            auto const &halo_point = primitives(halos_indices(j));
            // NOTE The explicit casts below are intended to silent warnings
            // about narrowing conversion from 'int' to 'float'.  The potential
            // issue is that 'float' can represent all integer values in the
            // range [-2^23, 2^23] but 'int' can actually represent values in
            // the range [-2^31, 2^31-1].
            halo_center[0] += halo_point[0] / static_cast<float>(halo_size);
            halo_center[1] += halo_point[1] / static_cast<float>(halo_size);
            halo_center[2] += halo_point[2] / static_cast<float>(halo_size);
          }
          halos_centers(i) = halo_center;
        });

    auto halos_offset_host =
        Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, halos_offset);
    auto halos_centers_host =
        Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, halos_centers);
    for (int i = 0; i < num_halos; i++)
    {
      int halo_size = halos_offset_host(i + 1) - halos_offset_host(i);

      // This is HACC specific filtering
      auto const &halo_center = halos_centers_host(i);
      if (halo_center[0] >= 0 && halo_center[1] >= 0 && halo_center[2] >= 0 &&
          halo_center[0] < 64 && halo_center[1] < 64 && halo_center[2] < 64)
      {
        printf("%d %e %e %e\n", halo_size, halo_center[0], halo_center[1],
               halo_center[2]);
      }
    }
  }

  return EXIT_SUCCESS;
}


// Copyright Aleksey Gurtovoy 2000-2004
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
// See http://www.boost.org/libs/mpl for documentation.

// $Id: logical.cpp 49268 2008-10-11 06:26:17Z agurtovoy $
// $Date: 2008-10-10 23:26:17 -0700 (Fri, 10 Oct 2008) $
// $Revision: 49268 $

#include <boost/mpl/logical.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/mpl/aux_/test.hpp>

struct unknown;

using mpl::true_;
using mpl::false_;

MPL_TEST_CASE()
{
    MPL_ASSERT(( mpl::and_< true_,true_ > ));
    MPL_ASSERT_NOT(( mpl::and_< false_,true_ > ));
    MPL_ASSERT_NOT(( mpl::and_< true_,false_ > ));
    MPL_ASSERT_NOT(( mpl::and_< false_,false_ > ));
    MPL_ASSERT_NOT(( mpl::and_< false_,unknown > ));
    MPL_ASSERT_NOT(( mpl::and_< false_,unknown,unknown > ));

    MPL_ASSERT(( mpl::or_< true_,true_ > ));
    MPL_ASSERT(( mpl::or_< false_,true_ > ));
    MPL_ASSERT(( mpl::or_< true_,false_ > ));
    MPL_ASSERT_NOT(( mpl::or_< false_,false_ > ));
    MPL_ASSERT(( mpl::or_< true_,unknown > ));
    MPL_ASSERT(( mpl::or_< true_,unknown,unknown > ));

    MPL_ASSERT_NOT(( mpl::not_< true_ > ));
    MPL_ASSERT(( mpl::not_< false_ > ));
}


/*
@copyright Louis Dionne 2015
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
 */

#ifndef BOOST_HANA_TEST_LAWS_RING_HPP
#define BOOST_HANA_TEST_LAWS_RING_HPP

#include <boost/hana/assert.hpp>
#include <boost/hana/comparable.hpp>
#include <boost/hana/bool.hpp>
#include <boost/hana/core/models.hpp>
#include <boost/hana/core/operators.hpp>
#include <boost/hana/core/when.hpp>
#include <boost/hana/monoid.hpp>
#include <boost/hana/ring.hpp>

#include <laws/base.hpp>

namespace boost { namespace hana { namespace test {
    template <typename R, typename = when<true>>
    struct TestRing : TestRing<R, laws> {
        using TestRing<R, laws>::TestRing;
    };

    template <typename R>
    struct TestRing<R, laws> {
        static_assert(_models<Ring, R>{}, "");

        template <typename Xs>
        TestRing(Xs xs) {
            foreach3(xs, [](auto x, auto y, auto z) {

                // associativity
                BOOST_HANA_CHECK(hana::equal(
                    hana::mult(x, hana::mult(y, z)),
                    hana::mult(hana::mult(x, y), z)
                ));

                // right identity
                BOOST_HANA_CHECK(hana::equal(
                    hana::mult(x, one<R>()), x
                ));

                // left identity
                BOOST_HANA_CHECK(hana::equal(
                    hana::mult(one<R>(), x), x
                ));

                // distributivity
                BOOST_HANA_CHECK(hana::equal(
                    hana::mult(x, hana::plus(y, z)),
                    hana::plus(hana::mult(x, y), hana::mult(x, z))
                ));

                // operators
                only_when_(bool_<has_operator<R, decltype(mult)>{}>, [=](auto _) {
                    BOOST_HANA_CHECK(hana::equal(
                        hana::mult(x, y),
                        _(x) * _(y)
                    ));
                });

                // power
                BOOST_HANA_CHECK(hana::equal(
                    hana::power(x, int_<0>),
                    one<R>()
                ));

                BOOST_HANA_CHECK(hana::equal(
                    hana::power(x, int_<1>),
                    x
                ));

                BOOST_HANA_CHECK(hana::equal(
                    hana::power(x, int_<2>),
                    hana::mult(x, x)
                ));

                BOOST_HANA_CHECK(hana::equal(
                    hana::power(x, int_<3>),
                    hana::mult(hana::mult(x, x), x)
                ));

                BOOST_HANA_CHECK(hana::equal(
                    hana::power(x, int_<4>),
                    hana::mult(hana::mult(hana::mult(x, x), x), x)
                ));

                BOOST_HANA_CHECK(hana::equal(
                    hana::power(x, int_<5>),
                    hana::mult(hana::mult(hana::mult(hana::mult(x, x), x), x), x)
                ));

            });
        }
    };

    template <typename C>
    struct TestRing<C, when<_models<Constant, C>{}>>
        : TestRing<C, laws>
    {
        template <typename Xs>
        TestRing(Xs xs) : TestRing<C, laws>{xs} {
            foreach2(xs, [](auto x, auto y) {

                BOOST_HANA_CHECK(hana::equal(
                    hana::value(one<C>()),
                    one<typename C::value_type>()
                ));

                BOOST_HANA_CHECK(hana::equal(
                    hana::mult(hana::value(x), hana::value(y)),
                    hana::value(hana::mult(x, y))
                ));

            });
        }
    };
}}} // end namespace boost::hana::test

#endif // !BOOST_HANA_TEST_LAWS_RING_HPP


// Software License for MTL
//
// Copyright (c) 2007 The Trustees of Indiana University.
//               2008 Dresden University of Technology and the Trustees of Indiana University.
//               2010 SimuNova UG (haftungsbeschränkt), www.simunova.com.
// All rights reserved.
// Authors: Peter Gottschling and Andrew Lumsdaine
//
// This file is part of the Matrix Template Library
//
// See also license.mtl.txt in the distribution.

#ifndef MTL_HERMITIAN_VIEW_INCLUDE
#define MTL_HERMITIAN_VIEW_INCLUDE

#include <iostream>
#include <boost/shared_ptr.hpp>
#include <boost/numeric/mtl/matrix/map_view.hpp>
#include <boost/numeric/mtl/matrix/transposed_view.hpp>
#include <boost/numeric/mtl/operation/conj.hpp>
#include <boost/numeric/mtl/operation/matrix_bracket.hpp>
#include <boost/numeric/mtl/concept/collection.hpp>

namespace mtl { namespace mat {

template <class Matrix>
struct hermitian_view
  : private transposed_view<Matrix>,
    public map_view<mtl::sfunctor::conj<typename Matrix::value_type>,
		    transposed_view<Matrix> >
{
    typedef transposed_view<Matrix>                                trans_base;
    typedef mtl::sfunctor::conj<typename Matrix::value_type>       functor_type;
    typedef map_view<functor_type, transposed_view<Matrix> >       base;
    typedef hermitian_view                                         self;
    typedef const Matrix&                                          const_ref_type;
    typedef typename Collection<Matrix>::size_type                 size_type;
    typedef typename Collection<Matrix>::value_type                value_type;

    typedef typename OrientedCollection<trans_base>::orientation   orientation; // Should not be needed because defined in Collection (bug in g++???)

    hermitian_view(const Matrix& matrix)
      : trans_base(const_cast<Matrix&>(matrix)),
	base(functor_type(), static_cast<trans_base&>(*this))
    {}

#if 0
    hermitian_view(boost::shared_ptr<Matrix> p)
	: trans_base(p), base(functor_type(), static_cast<trans_base&>(*this))
    {}
#endif

    typename base::value_type operator()(size_type r, size_type c) const { return base::operator()(r, c); }

    operations::bracket_proxy<self, const self&, value_type>
    operator[] (size_type r) const
    {
	return operations::bracket_proxy<self, const self&, value_type>(*this, r);
    }

    friend size_type inline num_rows(const self& A) { return num_rows((const base&)(A)); }
    friend size_type inline num_cols(const self& A) { return num_cols((const base&)(A)); }

    const_ref_type const_ref() const
    {
	// make two statements because nvcc cannot handle ref.ref
	const transposed_view<Matrix>& r1= base::ref;
	return r1.ref;
    }

    size_type nnz() const { return base::nnz(); }

    friend inline std::ostream& operator<<(std::ostream& os, const self& A) { return os << (const base&)(A); }
};

// If not defined ambigous between map_view and transposed_view
template <class Matrix>
inline std::size_t size(const hermitian_view<Matrix>& A)
{
    return num_rows(A) * num_rows(A);
}

// TBD submatrix of Hermitian (not trivial)

}} // namespace mtl::matrix

// Traits for Hermitian views
namespace mtl { namespace traits {

template <typename Matrix>
struct row< mtl::mat::hermitian_view<Matrix> >
  : public row< mtl::mat::map_view<sfunctor::conj<typename Matrix::value_type>,
				      mtl::mat::transposed_view<Matrix> > >
{};

template <typename Matrix>
struct col< mtl::mat::hermitian_view<Matrix> >
    : public col< mtl::mat::map_view<sfunctor::conj<typename Matrix::value_type>,
				   mtl::mat::transposed_view<Matrix> > >
{};

template <typename Matrix>
struct const_value< mtl::mat::hermitian_view<Matrix> >
    : public const_value< mtl::mat::map_view<sfunctor::conj<typename Matrix::value_type>,
					   mtl::mat::transposed_view<Matrix> > >
{};

template <typename Tag, typename Matrix>
struct range_generator< Tag, mtl::mat::hermitian_view<Matrix> >
    : public range_generator< Tag, mtl::mat::map_view<sfunctor::conj<typename Matrix::value_type>,
						    mtl::mat::transposed_view<Matrix> > >
{};

template <typename Matrix>
struct range_generator< tag::major, mtl::mat::hermitian_view<Matrix> >
    : public range_generator< tag::major, mtl::mat::map_view<sfunctor::conj<typename Matrix::value_type>,
							   mtl::mat::transposed_view<Matrix> > >
{};

}} // mtl::traits

#endif // MTL_HERMITIAN_VIEW_INCLUDE


/**
 * @file methods/ann/layer/highway.hpp
 * @author Konstantin Sidorov
 * @author Saksham Bansal
 *
 * Definition of the Highway layer.
 *
 * mlpack is free software; you may redistribute it and/or modify it under the
 * terms of the 3-clause BSD license.  You should have received a copy of the
 * 3-clause BSD license along with mlpack.  If not, see
 * http://www.opensource.org/licenses/BSD-3-Clause for more information.
 */
#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP
#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP

#include <mlpack/prereqs.hpp>

#include <boost/ptr_container/ptr_vector.hpp>

#include "../visitor/delete_visitor.hpp"
#include "../visitor/delta_visitor.hpp"
#include "../visitor/output_height_visitor.hpp"
#include "../visitor/output_parameter_visitor.hpp"
#include "../visitor/output_width_visitor.hpp"

#include "layer_types.hpp"
#include "add_merge.hpp"

namespace mlpack {
namespace ann /** Artificial Neural Network. */ {

/**
 * Implementation of the Highway layer. The Highway class can vary its behavior
 * between that of feed-forward fully connected network container and that
 * of a layer which simply passes its inputs through depending on the transform
 * gate. Note that the size of the input and output matrices of this class
 * should be equal.
 *
 * For more information, refer the following paper.
 *
 * @code
 * @article{Srivastava2015,
 *   author  = {Rupesh Kumar Srivastava, Klaus Greff, Jurgen Schmidhuber},
 *   title   = {Training Very Deep Networks},
 *   journal = {Advances in Neural Information Processing Systems},
 *   year    = {2015},
 *   url     = {https://arxiv.org/abs/1507.06228},
 * }
 * @endcode
 *
 * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
 *         arma::sp_mat or arma::cube).
 * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
 *         arma::sp_mat or arma::cube).
 */
template <
    typename InputDataType = arma::mat,
    typename OutputDataType = arma::mat,
    typename... CustomLayers>
class Highway
{
 public:
  //! Create the Highway object.
  Highway();

  /**
   * Create the Highway object.
   *
   * @param inSize The number of input units.
   * @param model Expose all the network modules.
   */
  Highway(const size_t inSize, const bool model = true);

  //! Destroy the Highway object.
  ~Highway();

  /**
   * Reset the layer parameter.
   */
  void Reset();

  /**
   * Ordinary feed-forward pass of a neural network, evaluating the function
   * f(x) by propagating the activity forward through f.
   *
   * @param input Input data used for evaluating the specified function.
   * @param output Resulting output activation.
   */
  template<typename eT>
  void Forward(const arma::Mat<eT>& input, arma::Mat<eT>& output);

  /**
   * Ordinary feed-backward pass of a neural network, calculating the function
   * f(x) by propagating x backwards through f. Using the results from the
   * feed-forward pass.
   *
   * @param * (input) The propagated input activation.
   * @param gy The backpropagated error.
   * @param g The calculated gradient.
   */
  template<typename eT>
  void Backward(const arma::Mat<eT>& /* input */,
                const arma::Mat<eT>& gy,
                arma::Mat<eT>& g);

  /**
   * Calculate the gradient using the output delta and the input activation.
   *
   * @param input The input parameter used for calculating the gradient.
   * @param error The calculated error.
   * @param gradient The calculated gradient.
   */
  template<typename eT>
  void Gradient(const arma::Mat<eT>& input,
                const arma::Mat<eT>& error,
                arma::Mat<eT>& gradient);

  /**
   * Add a new module to the model.
   *
   * @param args The layer parameter.
   */
  template <class LayerType, class... Args>
  void Add(Args... args)
  {
    network.push_back(new LayerType(args...));
    networkOwnerships.push_back(true);
  }

  /**
   * Add a new module to the model.
   *
   * @param layer The Layer to be added to the model.
   */
  void Add(LayerTypes<CustomLayers...> layer)
  {
    network.push_back(layer);
    networkOwnerships.push_back(false);
  }

  //! Return the modules of the model.
  std::vector<LayerTypes<CustomLayers...> >& Model()
  {
    if (model)
    {
      return network;
    }

    return empty;
  }

  //! Get the parameters.
  OutputDataType const& Parameters() const { return weights; }
  //! Modify the parameters.
  OutputDataType& Parameters() { return weights; }

  //! Get the input parameter.
  InputDataType const& InputParameter() const { return inputParameter; }
  //! Modify the input parameter.
  InputDataType& InputParameter() { return inputParameter; }

  //! Get the output parameter.
  OutputDataType const& OutputParameter() const { return outputParameter; }
  //! Modify the output parameter.
  OutputDataType& OutputParameter() { return outputParameter; }

  //! Get the delta.
  OutputDataType const& Delta() const { return delta; }
  //! Modify the delta.
  OutputDataType& Delta() { return delta; }

  //! Get the gradient.
  OutputDataType const& Gradient() const { return gradient; }
  //! Modify the gradient.
  OutputDataType& Gradient() { return gradient; }

  //! Get the number of input units.
  size_t InSize() const { return inSize; }

  /**
   * Serialize the layer.
   */
  template<typename Archive>
  void serialize(Archive& ar, const unsigned int /* version */);

 private:
  //! Locally-stored number of input units.
  size_t inSize;

  //! Parameter which indicates if the modules should be exposed.
  bool model;

  //! Indicator if we already initialized the model.
  bool reset;

  //! Locally-stored network modules.
  std::vector<LayerTypes<CustomLayers...> > network;

  //! The list of network modules we are responsible for.
  std::vector<bool> networkOwnerships;

  //! Locally-stored empty list of modules.
  std::vector<LayerTypes<CustomLayers...> > empty;

  //! Locally-stored weight object.
  OutputDataType weights;

  //! Locally-stored delta object.
  OutputDataType delta;

  //! Locally-stored gradient object.
  OutputDataType gradient;

  //! Weights for transformation of output.
  OutputDataType transformWeight;

  //! Bias for transformation of output.
  OutputDataType transformBias;

  //! Locally-stored transform gate parameters.
  OutputDataType transformGate;

  //! Locally-stored transform gate activation.
  OutputDataType transformGateActivation;

  //! Locally-stored transform gate error.
  OutputDataType transformGateError;

  //! Locally-stored input parameter object.
  InputDataType inputParameter;

  //! Locally-stored output parameter object.
  OutputDataType outputParameter;

  //! The input width.
  size_t width;

  //! The input height.
  size_t height;

  //! The normal output without highway network.
  OutputDataType networkOutput;

  //! Locally-stored delta visitor.
  DeltaVisitor deltaVisitor;

  //! Locally-stored output parameter visitor.
  OutputParameterVisitor outputParameterVisitor;

  //! Locally-stored delete visitor.
  DeleteVisitor deleteVisitor;

  //! Locally-stored output width visitor.
  OutputWidthVisitor outputWidthVisitor;

  //! Locally-stored output height visitor.
  OutputHeightVisitor outputHeightVisitor;
}; // class Highway

} // namespace ann
} // namespace mlpack

// Include implementation.
#include "highway_impl.hpp"

#endif


#include <boost/random/independent_bits.hpp>


#ifndef DEDGE_H_
#define DEDGE_H_

#include <Eigen/Core>
#include <Eigen/Dense>
#include <vector>

using namespace Eigen;

inline int dedge_prev_3(int e) { return (e % 3 == 0) ? e + 2 : e - 1; }
inline int dedge_next_3(int e) { return (e % 3 == 2) ? e - 2 : e + 1; }

bool compute_direct_graph(MatrixXd& V, MatrixXi& F, VectorXi& V2E,
	VectorXi& E2E, VectorXi& boundary, VectorXi& nonManifold);

void compute_direct_graph_quad(std::vector<Vector3d>& V, std::vector<Vector4i>& F, std::vector<int>& V2E,
                               std::vector<int>& E2E, VectorXi& boundary, VectorXi& nonManifold);

void remove_nonmanifold(std::vector<Vector4i> &F, std::vector<Vector3d> &V);

#endif


/***************************************************************************
                          bddlsph.cc  -  description
                             -------------------
    begin                : Mon Jun 11 2007
    copyright            : (C) 2005 by Knut-Helge Vik
    email                : knuthelv@ifi.uio.no
 ***************************************************************************/
#include "bddlsph.h"
#include "../treealgs/dijkstra_sp.h"
#include <fstream>
#include "../simtime.h"
#include <boost/numeric/ublas/matrix.hpp>
#include <boost/numeric/ublas/io.hpp>

using namespace std;
using namespace boost;
using namespace TreeAlgorithms;

/* -------------------------------------------------------------------------

	Algorithm(): Minimum Diameter Degree Limited Shortest Path Heuristic
							start function

------------------------------------------------------------------------- */
void
BDDLShortestPathHeuristic::Algorithm(vertex_descriptorN zsource, double D)
{
	DB = D;
	init(zsource, DB);	// identify z-nodes and store them in VertexSet ZVertSet
	if(num_zvertices <= 0)
	{
		cout << WRITE_FUNCTION << " Error: No Z-vertices." << endl;	exit(0);
	}
	cerr << WRITE_FUNCTION << " Start SPH zsource " << zsource << " and " << num_zvertices << " znodes" << endl ;

	// -- Start SPH Algorithm --
	T_bddlsph.insertVertex(zsource, g); 				// add source to the SPH tree
	ZVertSet.erase(zsource);
	RunDijkstraForEveryZ(g);							// find SP for every z-node and store in SPKeeper

	while(!ZVertSet.empty())
	{
		diameterBroken.clear();
		degreeBroken.clear();

		int z = -1;
		PathVector newPath; 							// .first contains path to new z-node to be added to T_bddlsph
														// .second is the tree-vertex link -> TODO: probably not needed
		if(!ExtractClosestZ(newPath, z)) 				// extract the z-node closest to T_bddlsph
		{
			if(!TreeAlgorithms::relaxDegreeAndDiameter(degreeBroken, degree_bound, diameterBroken, diameter_bound)) break;
			if(!TreeAlgorithms::isRelaxWorking(degreeBroken, diameterBroken)) break;
		}
		else
		{
			ASSERTING(z > -1);

			AddTreeSPH(newPath); // add the new path to T_bddlsph
			updateDist(z);
			//char c = getchar();
		}
	}

	// -- start debug --
	//cerr << WRITE_FUNCTION << "Produced Steiner Tree: " << endl;
	//T_bddlsph.print();
	//T_bddlsph.printVertexState(cerr);
	// -- end debug --
}

/* -------------------------------------------------------------------------

	ExtractClosestZ(): Returns the vertex that is closest to the T_bddlsph

		src: Dijkstra_shortest_path from this node
		p: next/parent pointers
		distances: distances from src to each other node in the graph

	IDEA: iterate through the z-vertices and check how close they are to
	the T_bddlsph return the closest z-vertex and add the path to the T_bddlsph

 --------------------------------------------------------------------------*/
bool
BDDLShortestPathHeuristic::ExtractClosestZ(PathVector &newPath, int &z_closest)
{
	double distToTree	= MAXIMUM_WEIGHT; //(numeric_limits<double>::max)();
	int z_in = - 1;
	z_closest = -1;
	PathVector tempPath;

	VertexSet diameterNotBroken;
	// iterate through the z-vertices and check how close they are to T_bddlsph
	for(VertexSet::iterator zit = ZVertSet.begin(), zit_end = ZVertSet.end(); zit != zit_end; ++zit)
	{
		vertex_descriptorN zAddToTree = *zit;

		ASSERTING(!T_bddlsph.V.contains(zAddToTree));
		//cerr << WRITE_FUNCTION  << " Z : " << zAddToTree << endl;

		ShortestPathKeeper &spk = findSPMaps(zAddToTree);

		VertexSet_it tit_bddlsph, tit_bddlsph_end;
		for(tit_bddlsph = T_bddlsph.V.begin(), tit_bddlsph_end = T_bddlsph.V.end(); tit_bddlsph != tit_bddlsph_end; ++tit_bddlsph) // check distance between new znode zit to every tree node tit_bddlsph
		{
			vertex_descriptorN zInTree = *tit_bddlsph;
			double new_dist = spk.zdistance[zInTree];
			double new_diameter = ecc[zInTree] + spk.zdistance[zInTree];

			//cerr << zInTree << ": new dist " << new_dist << " new diameter " << new_diameter << " out degree zintree " << getOutDegree(T_bddlsph.g, zInTree) << " bound " << degree_bound[zInTree] << endl;
			// is the new z-node closer than the previous and within diameter bound -> then update distance etc.
			//if(distToTree > new_dist && CheckPath(tempPath, zInTree, zAddToTree, spk.zparent))	// is the new z-node closer than the previous -> then update distance etc.
			if(distToTree > new_dist && new_diameter <= diameter_bound[zAddToTree] && getOutDegree(T_bddlsph.g, zInTree) < degree_bound[zInTree]) // is the new z-node closer than the previous -> then update distance etc.
			{
				diameterNotBroken.insert(zAddToTree);
				//diameterBroken.erase(zAddToTree);
				//if(CheckPath(tempPath, zInTree, zAddToTree, spk.zparent))
				if(FindPath(tempPath, zInTree, zAddToTree, spk.zparent, spk.zdistance))
				{
					distToTree 	= new_dist;  					// update shortest distance
					z_closest	= zAddToTree;
					z_in		= zInTree;
					newPath		= tempPath;
					//cerr << zAddToTree << " new closest: " << zInTree << ": new dist " << new_dist << " distToTree " << distToTree << " new diameter " << new_diameter << " diameter bound " << diameter_bound[zAddToTree] << " degree limit " << degree_bound[zInTree] << endl;
				}
			}

			if(distToTree == MAXIMUM_WEIGHT)
			{
				if(new_diameter > diameter_bound[zAddToTree]) diameterBroken.insert(zAddToTree);
				if(getOutDegree(T_bddlsph.g, zInTree) >= degree_bound[zInTree]) degreeBroken.insert(zInTree);
			}

			tempPath = PathVector();

		}
	}

	if(distToTree < MAXIMUM_WEIGHT) return true;

	//cerr << " diameterbroken " << diameterBroken << endl;
	diameterBroken = diameterBroken - diameterNotBroken;
	//if(!diameterBroken.empty() || !degreeBroken.empty())
	//cerr << " diameterbroken " << diameterBroken << " degreebroken " << degreeBroken << " diameterNotBroken " << diameterNotBroken << endl;

	return false;
}

bool
BDDLShortestPathHeuristic::CheckPath(PathVector &newPath, vertex_descriptorN tree_vert, vertex_descriptorN z_vert, const ParentVector &zparent)
{
	double tree_depth = ecc[tree_vert];

	ASSERTING(tree_vert != z_vert);
	vertex_descriptorN traverse_vert = tree_vert;
	do{
		bool intact = true;
		if(getOutDegree(T_bddlsph.g, traverse_vert) >= degree_bound[traverse_vert])
		{
			//cerr << traverse_vert << " degree broken " << degree_bound[traverse_vert] << endl;
			degreeBroken.insert(traverse_vert);
			intact = false;
		}

		pair<edge_descriptorN, bool> ep = edge(traverse_vert, zparent[traverse_vert], g);
		ASSERTING(ep.second);
		//cerr << ep.first << " tree_depth[" << traverse_vert << "] = " << tree_depth << " + " << g[ep.first].weight << endl;
		//if(tree_depth > diameter_bound[traverse_vert])
		if(ZVertSet.contains(traverse_vert) && tree_depth > diameter_bound[traverse_vert])
		{
			//cerr << traverse_vert << " diameter broken " << tree_depth << " > " << diameter_bound[traverse_vert] << endl;
			diameterBroken.insert(traverse_vert);
			intact = false;
		}

		if(!intact) return intact;

		tree_depth = tree_depth + g[ep.first].weight;

		newPath.first.push_front(traverse_vert);
		traverse_vert = zparent[traverse_vert];
	}while(traverse_vert != z_vert);

	newPath.first.push_front(traverse_vert); 		// add last node
	ASSERTING(newPath.first.size() > 1);

	//if(tree_depth > diameter_bound[traverse_vert])
	if(ZVertSet.contains(traverse_vert) && tree_depth > diameter_bound[traverse_vert])
	{
		//cerr << traverse_vert << " diameter broken " << tree_depth << " > " << diameter_bound[traverse_vert] << endl;
		diameterBroken.insert(traverse_vert);
		return false;
	}

	return true;
}

/* -------------------------------------------------------------------------

	FindDistance():	Find distance from z_vert to tree_vert using zparent
			and zdistance from DijkstraSP(z_vert)

------------------------------------------------------------------------- */
bool
BDDLShortestPathHeuristic::FindPath(PathVector &newPath, vertex_descriptorN tree_vert, vertex_descriptorN z_vert, const ParentVector &zparent, const DistanceVector &zdistance)
{
	bool intact = true;
	ASSERTING(tree_vert != z_vert);
	vertex_descriptorN traverse_vert = tree_vert;
	do{
		if(getOutDegree(T_bddlsph.g, traverse_vert) >= degree_bound[traverse_vert])
		{
			degreeBroken.insert(traverse_vert);
		    //cerr << WRITE_FUNCTION << " can't add path violating degree bound " << getOutDegree(T_bddlsph.g, traverse_vert) << ">=" <<  degree_bound[traverse_vert] << endl;
			intact = false;
		}
		if(!T_bddlsph.V.contains(traverse_vert) && ecc[traverse_vert] + zdistance[traverse_vert] > diameter_bound[traverse_vert])
		{
		 	//cerr << WRITE_FUNCTION << " can't add path violating diameter bound " << ecc[traverse_vert] + zdistance[traverse_vert] << ">" <<  diameter_bound[traverse_vert] << endl;
			diameterBroken.insert(traverse_vert);
			intact = false;
		}

		if(!intact) return intact;

		newPath.first.push_front(traverse_vert);
		traverse_vert = zparent[traverse_vert];
	}while(traverse_vert != z_vert);

	newPath.first.push_front(traverse_vert); 		// add last node
	ASSERTING(newPath.first.size() > 1);
	return intact;
}

/* -------------------------------------------------------------------------

	AddSPHTree(): Returns the vertex that is closest to the source
		if it is not already in the SPH-tree.

	all_vertex_info.first: path (vertex_descriptors) to new z_node
	all_vertex_info.second: node in the T_bddlsph (steiner or z-node) that
		links the new z-node to T_bddlsph
 --------------------------------------------------------------------------*/
void
BDDLShortestPathHeuristic::AddTreeSPH(PathVector &newPath)
{
	ASSERTING(newPath.first.size() > 1);
	for(list<vertex_descriptorN>::reverse_iterator vit = newPath.first.rbegin(), vit_end = newPath.first.rend(); vit != vit_end; )
	{
		// iterate
		vertex_descriptorN u = *vit;
		vit++;
		if(vit == vit_end) break;
		vertex_descriptorN v = *vit;

		pair<edge_descriptorN, bool> ep = edge(u, v, g);
		ASSERTING(ep.second);

		ASSERTING(near_[u] > -1);

		//cerr << "1 (" << u << "," << v << ")" << " near[" << u << "] " << near[u] << " near[" << v << "] " << near[v] << " ecc[" << v << "] " << ecc[v] << endl;

		if(near_[v] < 0)
		{
			near_[v] = u;
			ecc[v] = ecc[u] + g[ep.first].weight;
		}
		//cerr << "2 (" << u << "," << v << ")" << " near[" << u << "] " << near[u] << " near[" << v << "] " << near[v] << " ecc[" << v << "] " << ecc[v] << endl;
	}

	bool done = false;
	list<vertex_descriptorN>::iterator vit, vit_end;
	for(vit = newPath.first.begin(), vit_end = newPath.first.end(); vit != vit_end; )
	{
		// iterate
		vertex_descriptorN u = *vit;
		vit++;
		if(vit == vit_end) break;
		vertex_descriptorN v = *vit;

		// -- debug --
		//cerr << "adding (" << u << "," << v << ")" << endl;
		// -- end debug --

		if(T_bddlsph.V.contains(v)) done = true;

		T_bddlsph.insertEdge(u, v, g);

		ZVertSet.erase(u);
		ZVertSet.erase(v);

		// -- debug --
		/*if(getOutDegree(T_bddlsph.g, u) > degree_bound[u])
			cerr << " inserted " << u << " degree bound " << degree_bound[u] << " degree is " << getOutDegree(T_bddlsph.g, u) << endl;
		if(getOutDegree(T_bddlsph.g, v) > degree_bound[v])
			cerr << " inserted " << v << " degree bound " << degree_bound[v] << " degree is " << getOutDegree(T_bddlsph.g, v) << endl;
		if(ecc[u] > diameter_bound[u])
			cerr << " inserted " << u << " diameter bound " << diameter_bound[u] << " ecc is " << ecc[u] << endl;
		if(ecc[v] > diameter_bound[v])
			cerr << " inserted " << v << " diameter bound " << diameter_bound[v] << " ecc is " << ecc[v] << endl;*/
		// -- debug end --

		if(done) break;
	}
}

void
BDDLShortestPathHeuristic::updateDist(int z)
{
	ASSERTING(z > -1);
	ASSERTING(inputT.V.contains(z));
	//cerr << " z : " << g[z] << endl;

	ASSERTING(g[z].vertexState == GROUP_MEMBER);

	VertexSet::iterator vit, vit_end, vit_in, vit_in_end;
	ShortestPathKeeper &spk_z = findSPMaps(z);

	//cerr << WRITE_FUNCTION << " z " << z << " near[z] " << near[z] << " spk.zdistance[near[z]]" << spk_z.zdistance[near[z]] << endl;
	ASSERTING(near_[z] > -1);
	ASSERTING(ecc[near_[z]] >= 0);

	// set dist(z,u) and ecc(z)
	for(vit = T_bddlsph.V.begin(), vit_end = T_bddlsph.V.end(); vit != vit_end; ++vit)
	{
		if(dist(near_[z],*vit) > 0) dist(z,*vit) = dist(near_[z],*vit) + spk_z.zdistance[near_[z]];
	}

	dist(z,z) = 0;
	ecc[z] = ecc[near_[z]] + spk_z.zdistance[near_[z]];

	// update dist(near(z), u) and ecc(near(z))
	dist(near_[z],z) = spk_z.zdistance[near_[z]];
	if(ecc[near_[z]] <= 0) ecc[near_[z]] = spk_z.zdistance[near_[z]];

	// update other nodes' values of dist and ecc
	for(vit = T_bddlsph.V.begin(), vit_end = T_bddlsph.V.end(); vit != vit_end; ++vit)
	{
		ASSERTING(dist(*vit, near_[z]) >= 0);
		ASSERTING(dist(*vit, z) >= 0);
		ASSERTING(ecc[*vit] >= 0);

		dist(*vit,z) = dist(*vit, near_[z]) + spk_z.zdistance[near_[z]];
		ecc[*vit] = std::max(ecc[*vit], dist(*vit,z));
	}

	//cerr << " update the near values for other nodes in G " << endl;
	for(vit = ZVertSet.begin(), vit_end = ZVertSet.end(); vit != vit_end; ++vit)
	{
		ShortestPathKeeper &spk = findSPMaps(*vit);

		double dc_near_vit = (numeric_limits<double>::max)(), od_near_vit = 0;
		if(near_[*vit] > -1)
		{
			dc_near_vit = degree_bound[near_[*vit]];
			od_near_vit = getOutDegree(T_bddlsph.g, near_[*vit]);
		}

		double curr_ecc = (std::numeric_limits<double>::max)();
		if(near_[*vit] > -1 && od_near_vit < dc_near_vit) curr_ecc = ecc[near_[*vit]] + spk.zdistance[near_[*vit]];

		if(curr_ecc > diameter_bound[*vit] || (getOutDegree(T_bddlsph.g, z) >= degree_bound[z]))
		{
			//cerr << " examine all nodes in T_bddlsph to determine near(" << *vit << ")	" << endl;
			for(vit_in = T_bddlsph.V.begin(), vit_in_end = T_bddlsph.V.end(); vit_in != vit_in_end; ++vit_in)
			{
				double new_ecc = ecc[*vit_in] + spk.zdistance[*vit_in];

				//cerr << " curr ecc " << curr_ecc << " new ecc " << new_ecc << endl;
				ASSERTING(ecc[*vit_in] >= 0);
				ASSERTING(spk.zdistance[*vit_in] >= 0);

				if(new_ecc < curr_ecc && getOutDegree(T_bddlsph.g, *vit_in) < degree_bound[*vit_in])
				{
					//cerr << " new near [" << *vit << "] = " << *vit_in << endl;
					near_[*vit] = *vit_in;
					curr_ecc = new_ecc;
				}
			}
		}
		else
		{
			//cerr << "Compare w(" << *vit << "," << near[*vit] << ") = " << spk.zdistance[near[*vit]] << " to w(" << *vit << "," <<  z << ")= " << spk_z.zdistance[near[*vit]] << endl;

			if(spk_z.zdistance[*vit] <= spk.zdistance[near_[*vit]])
			{
				//cerr << " new near [" << *vit << "] = " << z << endl;
				near_[*vit] = z;
			}
		}

		//if(near[*vit] > -1)
		//{
			//if(curr_ecc >= diameter_bound[*vit]) diameterBroken.insert(*vit);
			//if(getOutDegree(T_bddlsph.g, near[*vit]) >= degree_bound[ near[*vit] ] ) degreeBroken.insert(near[*vit]);

			//cerr << " diameterbroken " << diameterBroken << " degreebroken " << degreeBroken << endl;
		//}
	}
}

void
BDDLShortestPathHeuristic::init(vertex_descriptorN src, const double &DB)
{
	vsVertexMap 	vsmap = get(&VertexProp::vertexState, g);

	VertexSet::const_iterator vit, vit_end, vit_in, vit_in_end;
	for(vit = inputT.V.begin(), vit_end = inputT.V.end(); vit != vit_end; ++vit)
	{
		if(vsmap[*vit] == GROUP_MEMBER)
		{
			ZVertSet.insert(*vit);
			num_zvertices++;
		}

		diameter_bound[*vit] = DB;
		degree_bound[*vit] = getDegreeConstraint(g, *vit);

		ecc[*vit] = 0;
		if(*vit != (int)src)
		{
			pair<edge_descriptorN, bool> ep = edge(*vit, src, g);
			if(ep.second) near_[*vit] = src;
			else near_[*vit] = -1;
		}
		else
		{
			near_[*vit] = *vit;
			ecc[*vit] = 0;
		}
		for(vit_in = inputT.V.begin(), vit_in_end = inputT.V.end(); vit_in != vit_in_end; ++vit_in)
			dist(*vit,*vit_in) = 0;
	}
	ASSERTING(!ZVertSet.empty());
}


// Copyright Abel Sinkovics (abel@sinkovics.hu) 2011.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <mpllibs/metamonad/either.hpp>
#include <mpllibs/metamonad/return_.hpp>
#include <mpllibs/metamonad/bind.hpp>
#include <mpllibs/metamonad/fail.hpp>
#include <mpllibs/metamonad/lambda_c.hpp>
#include <mpllibs/metamonad/name.hpp>

#include <boost/test/unit_test.hpp>

#include <boost/mpl/not.hpp>
#include <boost/mpl/int.hpp>
#include <boost/mpl/plus.hpp>
#include <boost/mpl/assert.hpp>

using namespace mpllibs::metamonad::name;

using mpllibs::metamonad::left;
using mpllibs::metamonad::right;
using mpllibs::metamonad::lambda_c;

using boost::mpl::plus;
using boost::mpl::int_;

namespace
{
  typedef int_<2> int2;
  typedef int_<11> int11;
  typedef int_<13> int13;

  typedef right<int13> right13;
  typedef right<int11> right11;
  typedef left<int11> left11;

  typedef lambda_c<x, right<plus<x, int2> > > either_add_2;

}

BOOST_AUTO_TEST_CASE(test_either)
{
  using mpllibs::metamonad::either_tag;
  using mpllibs::metamonad::return_;
  using mpllibs::metamonad::bind;
  using mpllibs::metamonad::fail;

  using boost::mpl::equal_to;

  typedef either_tag<> et;

  // test_return
  BOOST_MPL_ASSERT((equal_to<return_<et, int13>, right13>));

  // test_bind_with_right
  BOOST_MPL_ASSERT((equal_to<bind<et, right11, either_add_2>::type, right13>));

  // test_bind_with_left
  BOOST_MPL_ASSERT((equal_to<bind<et, left11, either_add_2>::type, left11>));

  // test_fail
  BOOST_MPL_ASSERT((
    equal_to<bind<et, fail<et, int13>, either_add_2>::type, left<int13> >
  ));
}


// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <ATen/Parallel.h>
#include <boost/container/small_vector.hpp>
#include <pybind11/pybind11.h>
#include <torch/torch.h>
#include <tuple>
#include <vector>

namespace {

using int64 = std::int64_t;
template<class T, size_t S>
using InlinedVector = boost::container::small_vector<T, S>;

inline int64 FlatIndex(int x, int y, int z,
                       std::tuple<int, int, int> grid_size) {
  auto[size_z, size_y, size_x] = grid_size;
  return (z * size_y + y) * size_x + x;
}

// A disjoint union set that's tracking the info for key groups.
class DisjointUnionSet {
 public:
  void Merge(int64 r1, int64 r2) {
    r1 = FindRoot(r1);
    r2 = FindRoot(r2);
    if (r1 < r2) {
      parent_[r2] = r1;
    } else if (r2 < r1) {
      parent_[r1] = r2;
    }
  }

  // Finds the root of the set this key belongs to, and compress the path by
  // attaching every node along the path to the root. This operation is
  // close to O(1) on average.
  int64 FindRoot(int64 element) {
    InlinedVector<int64, 128> visited;
    int64 root = element;
    while (parent_[root] != -1) {
      visited.push_back(root);
      root = parent_[root];
    }
    for (int64 v : visited) {
      parent_[v] = root;
    }

    return root;
  }

  int64 NewRegion() {
    parent_.push_back(-1);
    return parent_.size() - 1;
  }

 private:
  // Parent node for each key.
  std::vector<int64> parent_;
};

template<class T>
void ComputeConnectedComponents(
    std::tuple<int, int, int> grid_size, const T *volume, T outside_voxels_value, int64 *regions) {
  auto[size_z, size_y, size_x] = grid_size;

  DisjointUnionSet union_set;
  int64 outside_region = union_set.NewRegion();
  CHECK_EQ(outside_region, 0);

  for (int z = 0; z < size_z; z++) {
    for (int y = 0; y < size_y; y++) {
      for (int x = 0; x < size_x; x++) {
        int64 offset_current = FlatIndex(x, y, z, grid_size);
        int64 offset_left = FlatIndex(x - 1, y, z, grid_size);
        int64 offset_back = FlatIndex(x, y - 1, z, grid_size);
        int64 offset_up = FlatIndex(x, y, z - 1, grid_size);

        bool value_current = volume[offset_current] > 0;
        bool value_left =
            (x > 0 ? volume[offset_left] : outside_voxels_value) > 0;
        bool value_back =
            (y > 0 ? volume[offset_back] : outside_voxels_value) > 0;
        bool value_up =
            (z > 0 ? volume[offset_up] : outside_voxels_value) > 0;

        int64 &region_current = regions[offset_current];
        int64 region_left = x > 0 ? regions[offset_left] : outside_region;
        int64 region_back = y > 0 ? regions[offset_back] : outside_region;
        int64 region_up = z > 0 ? regions[offset_up] : outside_region;

        if (value_current == value_left && value_current == value_up) {
          union_set.Merge(region_left, region_up);
        }

        if (value_current == value_left && value_current == value_back) {
          union_set.Merge(region_left, region_back);
        }

        if (value_current == value_up && value_current == value_back) {
          union_set.Merge(region_up, region_back);
        }

        constexpr int64 invalid_candidate = std::numeric_limits<int64>::max();
        std::array<int64, 3> candidate_region_ids = {
            invalid_candidate, invalid_candidate, invalid_candidate};
        if (value_current == value_left) {
          candidate_region_ids[0] = region_left;
        }
        if (value_current == value_back) {
          candidate_region_ids[1] = region_back;
        }
        if (value_current == value_up) {
          candidate_region_ids[2] = region_up;
        }

        int64 candidate_region_id = *std::min_element(
            candidate_region_ids.begin(), candidate_region_ids.end());
        region_current = candidate_region_id != invalid_candidate
                         ? candidate_region_id
                         : union_set.NewRegion();
      }
    }
  }

  int64 size = size_z * size_y * size_x;
  for (int64 *v = regions; v < regions + size; v++) {
    *v = union_set.FindRoot(*v);
  }
}

template<class T>
void FillInsideVoxels(std::tuple<int, int, int> grid_size, T *voxel_grid) {
  auto[size_z, size_y, size_x] = grid_size;
  int64 size = size_z * size_y * size_x;
  std::vector<int64> regions(size);
  ComputeConnectedComponents(grid_size, voxel_grid, (T) 0, &regions[0]);
  for (int64 index = 0; index < size; index++) {
    if (regions[index] > 0) {
      voxel_grid[index] = 1;
    }
  }
}
}  // namespace

torch::Tensor fill_inside_voxels_cpu(const torch::Tensor grid) {
  if (!grid.device().is_cpu()) {
    throw pybind11::value_error("Only CPU tensors are supported currently");
  }

  const auto &shape = grid.sizes();
  if (shape.size() != 4) {
    throw pybind11::value_error("Expecting rank 4 tensor");
  }

  torch::Tensor output_grid = grid.clone(c10::MemoryFormat::Contiguous).cpu();

  int num_grids = shape[0];
  auto shape_tuple = std::make_tuple(shape[1], shape[2], shape[3]);
  int64 num_elements_in_grid = shape[1] * shape[2] * shape[3];
  AT_DISPATCH_ALL_TYPES(grid.scalar_type(), "FillInsideVoxels", ([&] {
    auto *data = (scalar_t *) output_grid.data_ptr();
    at::parallel_for(0, num_grids, 1, [&](int64 begin, int64 end) {
      for (int64 index = begin; index < end; index++) {
        FillInsideVoxels(shape_tuple, data + num_elements_in_grid * index);
      }
    });
  }));

  return output_grid;
}


/* Copyright (C) 2017 IBM Corp.
 *  Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */
/***********************************************************************
CRTmatrix: implements A matrix in CRT representation
************************************************************************/

#include <vector>
#include <cassert>
#include <cstdlib>
//#include <mpfr.h>
#include <NTL/mat_lzz_p.h>
#include <NTL/BasicThreadPool.h>
#include <NTL/ZZ.h>
#include <NTL/FFT.h>
#include <NTL/SmartPtr.h>

NTL_CLIENT
#include "utils/tools.h"
#include "DGaussSampler.h"
#include "CRTmatrix.h"
#include "TDMatrixParams.h"

//#define DEBUGPRINT

CRTmatrix& CRTmatrix::leftMultBy(const CRTmatrix& M) // A := M*A
{
    FHE_TIMER_START;
    assert(length()==params->kFactors);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    // multi-threaded implementation, threads handle different moduli

    EXEC_RANGE(length(), first, last)
    //EXEC_RANGE(length(), first, last)
    for (long i=first; i<last; i++)
    {
        params->zzp_context[i].restore();
        mat_zz_p& matMod = (*this)[i];
        const mat_zz_p& Mmod = M[i];
        mul(matMod, Mmod, matMod);
    }
    //EXEC_RANGE_END
    EXEC_RANGE_END
    return *this;
}

CRTmatrix& CRTmatrix::leftMultBy(const mat_l& M) // A := M*A
{
    FHE_TIMER_START;
    assert(length()==params->kFactors);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    // multi-threaded implementation, threads handle different moduli
    EXEC_RANGE(length(), first, last)
    for (long i=first; i<last; i++)
    {
        params->zzp_context[i].restore();
        mat_zz_p& matMod = (*this)[i];
        mat_zz_p Mmod = conv<mat_zz_p>(M);
        mul(matMod, Mmod, matMod);
    }
    EXEC_RANGE_END

    return *this;
}

CRTmatrix& CRTmatrix::leftMultBy(const vec_l& u) // A := u*A
{
    FHE_TIMER_START;
    assert(length()==params->kFactors);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    // multi-threaded implementation, threads handle different moduli
    EXEC_RANGE(length(), first, last)

    for (long i=first; i<last; i++)
    {
        params->zzp_context[i].restore();
        mat_zz_p& matMod = (*this)[i]; // A
        const vec_zz_p umod = conv<vec_zz_p>(u);
        vec_zz_p vTmp;
        mul(vTmp, umod, matMod); // tmp = u*A

        // copy vTmp back to matMod
        matMod.SetDims(1, vTmp.length());
        matMod[0] = vTmp;
    }
    EXEC_RANGE_END

    return *this;
}

// Convert Mat<long> to CRT format, we assume that params are already set
CRTmatrix& CRTmatrix::operator=(const mat_l& M) // A := M
{
    FHE_TIMER_START;
    assert(params != NULL);

    NTL::zz_pPush ppush; // backup NTL's current modulus
    SetLength(params->kFactors);
    for (long i=0; i<length(); i++)
    {
        params->zzp_context[i].restore();
        conv((*this)[i], M);
    }

    return *this;
}

// Default assignment should work: NTL now guarantees
// that assignment and default constructors work, even
// "out of context"
CRTmatrix& CRTmatrix::operator=(const CRTmatrix& other) // A := M
{
    FHE_TIMER_START;
    params = other.params;
    NTL::zz_pPush ppush; // backup NTL's current modulus
    SetLength(other.length());
    for (long i=0; i<params->kFactors; i++)
    {
        params->zzp_context[i].restore();
        (*this)[i] = other[i];
    }
    return *this;
}

CRTmatrix& CRTmatrix::operator*=(const mat_l& M) // A := M*A
{
    FHE_TIMER_START;
    assert(length()==params->kFactors);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    // multi-threaded implementation, threads handle different moduli
    EXEC_RANGE(length(), first, last)
    mat_zz_p Mmod;
    for (long i=first; i<last; i++)
    {
        params->zzp_context[i].restore();
        mat_zz_p& Amod = (*this)[i];
        Mmod = conv<mat_zz_p>(M);
        mul(Amod, Amod, Mmod);
    }
    EXEC_RANGE_END

    return *this;
}

CRTmatrix& CRTmatrix::operator*=(const CRTmatrix& M) // A := A*M
{
    FHE_TIMER_START;
    assert(length()==params->kFactors
           && M.length()==M.params->kFactors
           && params==M.params);
    NTL::zz_pPush ppush; // dibackup NTL's current modulus

    // multi-threaded implementation, threads handle different moduli
    EXEC_RANGE(length(), first, last)
    for (long i=first; i<last; i++)
    {
        params->zzp_context[i].restore();
        mat_zz_p& Amod = (*this)[i];
        const mat_zz_p& Mmod = M[i];
        mul(Amod, Amod, Mmod);
    }
    EXEC_RANGE_END

    return *this;
}

CRTmatrix& CRTmatrix::operator+=(const mat_l& M) // A := A+M
{
    FHE_TIMER_START;
    assert(length()==params->kFactors);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    // multi-threaded implementation, threads handle different moduli
    EXEC_RANGE(length(), first, last)
    //for (long i=0; i<length(); i++)
    for (long i=first; i<last; i++)
    {
        mat_zz_p& matMod = (*this)[i];
        assert(matMod.NumRows()==M.NumRows() && matMod.NumCols()==M.NumCols());

        params->zzp_context[i].restore();
        for (long j=0; j<matMod.NumRows(); j++)
            for (long k=0; k<matMod.NumCols(); k++)
                matMod[j][k] += M[j][k];
    }
    EXEC_RANGE_END
    return *this;
}
CRTmatrix& CRTmatrix::operator-=(const mat_l& M) // A := A+M
{
    FHE_TIMER_START;
    assert(length()==params->kFactors);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    EXEC_RANGE(length(), first, last)
    //for (long i=0; i<length(); i++)
    for (long i=first; i<last; i++)
    {
        mat_zz_p& matMod = (*this)[i];
        assert(matMod.NumRows()==M.NumRows() && matMod.NumCols()==M.NumCols());

        params->zzp_context[i].restore();
        for (long j=0; j<matMod.NumRows(); j++)
            for (long k=0; k<matMod.NumCols(); k++)
                matMod[j][k] -= M[j][k];
    }
    EXEC_RANGE_END
    return *this;
}

CRTmatrix& CRTmatrix::operator+=(const CRTmatrix& M) // A := A+M
{
    FHE_TIMER_START;
    assert(length()==params->kFactors
           && M.length()==M.params->kFactors
           && params==M.params);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    EXEC_RANGE(length(), first, last)
    //for (long i=0; i<length(); i++)
    for (long i=first; i<last; i++)
    {
        mat_zz_p& Amod = (*this)[i];
        const mat_zz_p& Mmod = M[i];

        assert(Amod.NumRows()==Mmod.NumRows() && Amod.NumCols()==Mmod.NumCols());
        params->zzp_context[i].restore();
        Amod += Mmod;
    }
    EXEC_RANGE_END
    return *this;
}

CRTmatrix& CRTmatrix::operator-=(const CRTmatrix& M) // A := A+M
{
    FHE_TIMER_START;
    assert(length()==params->kFactors
           && M.length()==M.params->kFactors
           && params==M.params);
    NTL::zz_pPush ppush; // backup NTL's current modulus

    EXEC_RANGE(length(), first, last)
    //for (long i=0; i<length(); i++)
    for (long i=first; i<last; i++)
    {
        mat_zz_p& Amod = (*this)[i];
        const mat_zz_p& Mmod = M[i];

        assert(Amod.NumRows()==Mmod.NumRows() && Amod.NumCols()==Mmod.NumCols());
        params->zzp_context[i].restore();
        Amod -= Mmod;
    }
    EXEC_RANGE_END
    return *this;
}

// Get a CRT column: crtCol[i] is the j'th column of this[i]
void CRTmatrix::getColumn(Vec<vec_zz_p>& crtCol, long j)
{
    FHE_TIMER_START;
    crtCol.SetLength(length());
    for (long n=0; n<length(); n++)
    {
        vec_zz_p& colMod = crtCol[n];
        mat_zz_p& matMod = (*this)[n];

        colMod.SetLength(matMod.NumRows());
        // NOTE: this is OK, but beware that this would
        // not work for ZZ_p's without restoring context

        for (long i=0; i<matMod.NumRows(); i++)
            colMod[i] = matMod[i][j];
    }
}

//checks if all values of the matrix are smaller than the modolus
bool CRTmatrix::isSmall(long bitGap) const
{
    FHE_TIMER_START;
    mat_ZZ M; // reconstruct the matrix in ZZ format
    ZZ q;     // the modulus

    convert(M, q, *this); // cnovert to ZZ format

    q >>= bitGap;         // q / 2^{bitGap}

    for (long i=0; i<M.NumRows(); i++) for (long j=0; j<M.NumCols(); j++)
        {
            if (abs(M[i][j]) > q)
            {
                return false;
            }
        }
    return true;
}

// Choose a random n-by-m matrix in CRT representation
void CRTmatrix::randomFill(long n, long m)
{
    FHE_TIMER_START;

    const TDMatrixParams& prms = *params;
    SetLength(prms.kFactors);

    zz_pPush ppush; // backup NTL's current modulus

    EXEC_RANGE(length(), first, last)
    //for (long i=0; i< prms.kFactors; i++)
    for (long i=first; i< last; i++)
    {

        prms.zzp_context[i].restore();
        (*this)[i].SetDims(n, m);
        RandomFill((*this)[i]);
    }
    EXEC_RANGE_END
}

// Invert M modulo the product of the factors
bool CRTmatrix::invert(const mat_l& M)
{
    FHE_TIMER_START;
    SetLength(params->kFactors);

    zz_pPush ppush; // backup NTL's current modulus

    std::atomic<bool> result(true);

    EXEC_RANGE(params->kFactors, first, last)

    // Invert M wrt each factor separately
    //for (long i=0; i< params->kFactors; i++)
     for (long i=first; i< last; i++)
    {
        params->zzp_context[i].restore();
        // NOTE: Alternatively, can call relaxed_inv directly now
        if (!invMod((*this)[i], M))   // not invertible
        {
            SetLength(0);
            result = false;
            //return false;
        }
    }
    EXEC_RANGE_END

    return result;
    //return true;
}

void CRTmatrix::identity(long n)
{
    FHE_TIMER_START;
    zz_pPush ppush; // backup NTL's current modulus
    SetLength(params->kFactors);

    // Invert M wrt each factor separately
    for (long i=0; i< params->kFactors; i++)
    {
        params->zzp_context[i].restore();
        ident( (*this)[i], n ); // set as the identity mod f_i
    }
}

bool operator==(const CRTmatrix& A, const CRTmatrix& B)
{
    FHE_TIMER_START;
    if (A.params != B.params) return false;

    zz_pPush push; // backup the NTL current modulus
    for (long i=0; i < A.params->kFactors; i++)
    {
        (A.params->zzp_context)[i].restore();
#ifdef DEBUGPRINT
        cout << "A[i]=" << A[i] << ", B[i]=" << B[i] << endl;
#endif // DEBUGPRINT
        if (A[i] != B[i]) return false;
    }
    return true;
}

//invert a CRT matrix
bool CRTmatrix::invert(const CRTmatrix& A)
{
    FHE_TIMER_START;
    params = A.params;
    SetLength(params->kFactors);

    zz_pPush push; // backup the NTL current modulus

    for (long i=0; i < params->kFactors; i++)
    {
        const mat_zz_p& Amod = A[i];
        params->zzp_context[i].restore();

        zz_p d;
        //    if (isInvertible(Amod, params->factors[i]))
        //inv((*this)[i],Amod); // raises error if Amod is singular
        relaxed_inv(d,(*this)[i],Amod); //raises error if Amod is singular

        if (IsZero(d))
            return false; //non invertible
    }
    return true;
}

// reconstruct the matrix in ZZ fromat, returning in q the modulus
void convert(mat_ZZ& to, ZZ& q, const CRTmatrix& from)
{
    FHE_TIMER_START;
    q = to_ZZ(1L); // initalize to 1
    if (from.length() <=0)
    {
        to.kill(); // set as 0-by-0 matrix
        return;
    }

    to.SetDims(from[0].NumRows(), from[0].NumCols());
    clear(to);     // initialize to zero

    NTL::zz_pPush ppush; // backup NTL's current modulus
    for (long k=0; k<from.length(); k++)   // go over the factors one at a time
    {
        from.params->zzp_context[k].restore();
        NTL::CRT(to, q, from[k]); // incremental CRT
    }
}

// reconstruct the matrix in CRT fromat
// FIXME: can be optimized for many factors
void convert(CRTmatrix& to, const mat_ZZ& from, const TDMatrixParams& prms)
{
    FHE_TIMER_START;
    to.params = (TDMatrixParams*) &prms;
    to.SetLength(prms.kFactors);
    if (prms.kFactors==0) return; // nothing to do

    NTL::zz_pPush ppush; // backup NTL's current modulus

    EXEC_RANGE(to.length(), first, last)

    for (long k=first; k<last; k++)   // go over the factors one at a time
    {
        prms.zzp_context[k].restore();
        conv(to[k], from); // compute M mod current factor
    }

    EXEC_RANGE_END
}

void generateMultiPair(CRTmatrix& A, CRTmatrix& Ainv, long n)
{
    FHE_TIMER_START;

    NTL::Vec<CRTmatrix> Amats(INIT_SIZE, 12, A);
    NTL::Vec<CRTmatrix> Ainvmats(INIT_SIZE, 12, Ainv);

    EXEC_RANGE(12, first, last)
    for (long i=first; i<last; i++)
    {

        CRTmatrix &Apair = Amats[i];
        CRTmatrix &AinvPair = Ainvmats[i];
        generateMatrixPair(Apair, AinvPair, n);
    }

    EXEC_RANGE_END
}

void generateMatrixPair(CRTmatrix& A, CRTmatrix& Ainv, long n)
{
    FHE_TIMER_START;
    const TDMatrixParams& prms = *(A.params);

    A.SetLength(prms.kFactors);
    Ainv.SetLength(prms.kFactors);

    zz_pPush push; // backup the NTL current modulus

    // It's possible to do parallilizing at this level
    // need to call SetNumThreads(nt) in main.

    // NOTE: the zz_pPush in the "main" thread is sufficient,
    // as the "worker" threads do not (and should not) assume
    // any particular contextual state

#ifdef DEBUGPRINT
        cerr << "*** " << prms.kFactors << " " << AvailableThreads() << "\n";
#endif // DEBUGPRINT

    EXEC_RANGE(prms.kFactors, first, last)
    for (long i=first; i<last; i++)
    {

        mat_zz_p& Amod = A[i];
        mat_zz_p& AinvMod = Ainv[i];
        prms.zzp_context[i].restore(); // set i'th factor as NTL's current modulus
        GenerateMatrixPair(Amod, AinvMod, n);
    }
    EXEC_RANGE_END

    FHE_TIMER_STOP;
};

// binary I/O
#undef CRTMAT_CONVtoZZ
long CRTmatrix::writeToFile(FILE* handle) const
{
    FHE_TIMER_START;

    // write the parameters first
    long count = params->writeToFile(handle);
    assert (this->length() == params->kFactors);

#ifndef CRTMAT_CONVtoZZ
    long n,m;
    if (this->length()<=0)
        n = m = 0;
    else
    {
        n = (*this)[0].NumRows();
        m = (*this)[0].NumCols();
    }
    count += fwrite(&n, sizeof(n), 1, handle); // how many rows
    count += fwrite(&m, sizeof(m), 1, handle); // how many cols

    unsigned char buf[params->e];
    NTL::zz_pPush ppush; // backup NTL's current modulus
    for (long f=0; f<this->length(); f++)   // write one factor at a time
    {
        params->zzp_context[f].restore();
        const Mat<zz_p>& mat = (*this)[f];
        for (long i=0; i<n; i++) for (long j=0; j<m; j++)   // write every entry
            {
                for (long ie=0; ie < params->e; ie++) // write every byte
                    buf[ie] = (rep(mat[i][j]) >> (ie*8)) & 0xff; // extract one byte
                count += fwrite(buf, params->e, 1, handle); // do the actual writing
            }
#ifdef DEBUGPRINT
        //cout << "*this[f]="  << (*this)[f] << endl;
#endif // DEBUGPRINT
    }

#else
    ZZ q;
    mat_ZZ zM;
    convert(zM, q, *this); // convert to ZZ representation, then write it

    long qBytes = NumBytes(q); // # of bytes to represent q
    count += fwrite(&qBytes, sizeof(qBytes),1, handle); // how many bytes per int

    long n = zM.NumRows();
    long m = zM.NumCols();
    count += fwrite(&n, sizeof(n), 1, handle); // how many rows
    count += fwrite(&m, sizeof(m), 1, handle); // how many cols

    unsigned char buf[qBytes];
    for (long i=0; i<n; i++)
        for (long j=0; j<m; j++)
        {
            if (sign(zM[i][j])<0) zM[i][j] += q; // map to interval [0,q-1]
            BytesFromZZ(buf, zM[i][j], qBytes);      // get binary representation
            count += fwrite(buf, qBytes, 1, handle); // write it to file
        }
#endif
    return count;
}

long CRTmatrix::readFromFile(FILE* handle, TDMatrixParams* prmBuf)
{
    FHE_TIMER_START;

#ifdef DEBUGPRINT
    cout << "reading" << endl;
#endif

    assert(params != NULL || prmBuf != NULL); // some pointer must be provided
    long count;

    // If buffer is given, make params point to it and don't overwrite from iput
    if (prmBuf != NULL)
    {
        TDMatrixParams p;
        count = p.readFromFile(handle);
        assert(p == *prmBuf); // sanity check
        params = prmBuf;      // point to given params
    }
    else
        count = params->readFromFile(handle); // overwrite params from input

#ifndef CRTMAT_CONVtoZZ
    long n,m;
    count += fread(&n, sizeof(n), 1, handle); // how many rows
    count += fread(&m, sizeof(m), 1, handle); // how many column
    if (count == 0) return count; //nothing is read

    this->SetLength(params->kFactors);
    if (params->kFactors==0) return count; // nothing to do

    unsigned char buf[params->e];
    NTL::zz_pPush ppush; // backup NTL's current modulus
    for (long f=0; f<this->length(); f++)   // write one factor at a time
    {
        params->zzp_context[f].restore();
        (*this)[f].SetDims(n,m);
        for (long i=0; i<n; i++) for (long j=0; j<m; j++)   // write every entry
            {
                count += fread(buf, params->e, 1, handle); // read from file

                long zp = buf[(params->e)-1];  // assemble the integer, byte by byte
                //for (long ie=1; ie < params->e; ie++)
                for (long ie=((params->e)-2); ie >=0 ; ie--)
                {
                    zp <<= 8;
                    zp += buf[ie];
                }
                (*this)[f][i][j].LoopHole() = zp; // avoid calling rem()
            }
    }
#else
    long qBytes, n, m;
    count += fread(&qBytes, sizeof(qBytes), 1, handle); // how many bytes per int
    count += fread(&n, sizeof(n), 1, handle); // how many rows
    count += fread(&m, sizeof(m), 1, handle); // how many column

    if (count == 0) return count; //nothing is read

    unsigned char buf[qBytes];
    mat_ZZ zM(INIT_SIZE, n, m);
    for (long i=0; i<n; i++) for (long j=0; j<m; j++)
        {
            count += fread(buf, qBytes, 1, handle); // read from file
            ZZFromBytes(zM[i][j], buf, qBytes);     // make a ZZ object
        }

    convert(*this, zM, *params); // convert to CRT representation
#endif
    return count;
}


#ifndef __COLOR_UTILS_HH__
#define __COLOR_UTILS_HH__

#include <Eigen/Dense>
#include <vector>
#include <iostream>

using namespace std;

namespace utils{
  namespace colors{
    class Colors{
      private:
        vector<Eigen::Vector3d> _colors;
        int _init_colors();
      public:
        Colors();
        Eigen::Vector3d operator[](int i);
        int size();
    };
  }
}
#endif


#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"
#include "dynet/cfsm-builder.h"
#include "dynet/hsm-builder.h"
#include "../utils/getpid.h"

#include <iostream>
#include <fstream>
#include <regex>
#include <sstream>

#include <boost/algorithm/string/join.hpp>
#include <boost/program_options.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/regex.hpp>

using namespace std;
using namespace dynet;

unsigned LAYERS = 0;
unsigned INPUT_DIM = 0;
unsigned HIDDEN_DIM = 0;
unsigned VOCAB_SIZE = 0;
float DROPOUT = 0;
bool SAMPLE = false;
SoftmaxBuilder* cfsm = nullptr;

dynet::Dict d;
int kSOS;
int kEOS;

volatile bool INTERRUPTED = false;

namespace po = boost::program_options;
void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("train,t", po::value<string>(), "training corpus")
        ("dev,d", po::value<string>(), "development/validation corpus")
        ("test,p", po::value<string>(), "test corpus")
        ("nbest", po::value<string>(), "N-best list to score (- for stdin)")
        ("learn,x", "set this to estimate the language model from the training data")
        ("clusters,c", po::value<string>(), "word cluster file for class factored softmax")
        ("paths,b", po::value<string>(), "word paths file for hierarchical softmax")
        ("sample,s", "periodically generate random samples from model as it trains (recommended)")
        ("model,m", po::value<string>(), "load model from this file")
        ("input_dim,i", po::value<unsigned>()->default_value(128), "input embedding dimension")
        ("hidden_dim,H", po::value<unsigned>()->default_value(128), "hidden layer size")
        ("layers,l", po::value<unsigned>()->default_value(2), "number of layers in RNN")
        ("dropout,D", po::value<float>(), "dropout rate (recommended between 0.2 and 0.5)")
        ("eta0,e", po::value<float>()->default_value(0.1f), "initial learning rate")
        ("eta_decay_onset_epoch", po::value<unsigned>(), "start decaying eta every epoch after this epoch (try 8)")
        ("eta_decay_rate", po::value<float>(), "how much to decay eta by (recommended 0.5)")
        ("help,h", "Help");
  po::options_description dcmdline_options;
  dcmdline_options.add(opts);
  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("help")) {
    cerr << dcmdline_options << endl;
    exit(1);
  }
  if (conf->count("train") == 0) {
    cerr << "Training data must always be specified (it determines the vocab mapping) with --train\n";
    exit(1);
  }
}

template <class Builder>
struct RNNLanguageModel {
  LookupParameter p_c;
  Parameter p_R;
  Parameter p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, {INPUT_DIM});
    p_R = model.add_parameters({VOCAB_SIZE, HIDDEN_DIM});
    p_bias = model.add_parameters({VOCAB_SIZE});
  }

  // return Expression of total loss
  Expression BuildLMGraph(const vector<int>& sent, ComputationGraph& cg, bool apply_dropout) {
    const unsigned slen = sent.size();
    if (apply_dropout) {
      builder.set_dropout(DROPOUT);
    } else {
      builder.disable_dropout();
    }
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();
    if (cfsm) cfsm->new_graph(cg);
    Expression R = parameter(cg, p_R); // hidden -> word rep parameter
    Expression bias = parameter(cg, p_bias);  // word bias
    vector<Expression> errs(slen + 1);
    Expression h_t = builder.add_input(lookup(cg, p_c, kSOS)); // read <s>
    for (unsigned t = 0; t < slen; ++t) { // h_t = RNN(x_0,...,x_t)
      if (cfsm) { // class-factored softmax
        errs[t] = cfsm->neg_log_softmax(h_t, sent[t]);
      } else { // regular softmax
        Expression u_t = affine_transform({bias, R, h_t});
        errs[t] = pickneglogsoftmax(u_t, sent[t]);
      }
      Expression x_t = lookup(cg, p_c, sent[t]);
      h_t = builder.add_input(x_t);
    }
    // it reamins to deal predict </s>
    if (cfsm) {
      errs.back() = cfsm->neg_log_softmax(h_t, kEOS);
    } else {
      Expression u_last = affine_transform({bias, R, h_t});
      errs.back() = pickneglogsoftmax(u_last, kEOS); // predict </s>
    }
    return sum(errs);
  }

  void RandomSample(int max_len = 200) {
    ComputationGraph cg;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();
    if (cfsm) cfsm->new_graph(cg);
    Expression R = parameter(cg, p_R); // hidden -> word rep parameter
    Expression bias = parameter(cg, p_bias);  // word bias
    Expression h_t = builder.add_input(lookup(cg, p_c, kSOS)); // read <s>
    int cur = kSOS;
    int len = 0;
    while(len < max_len) {
      if (cfsm) { // class-factored softmax
        cur = cfsm->sample(h_t);
      } else { // regular softmax
        Expression u_t = affine_transform({bias, R, h_t});
        Expression dist_expr = softmax(u_t);
        auto dist = as_vector(cg.incremental_forward(dist_expr));
        double p = rand01();
        cur = 0;
        for (; cur < dist.size(); ++cur) {
          p -= dist[cur];
          if (p < 0.0) { break; }
        }
        if (cur == dist.size()) cur = kEOS;
      }
      if (cur == kEOS) break;
      ++len;
      cerr << (len == 1 ? "" : " ") << d.convert(cur);
      Expression x_t = lookup(cg, p_c, cur);
      h_t = builder.add_input(x_t);
    }
    cerr << endl;
  }
};

int main(int argc, char** argv) {
  cerr << "COMMAND LINE:";
  for (unsigned i = 0; i < static_cast<unsigned>(argc); ++i) cerr << ' ' << argv[i];
  cerr << endl;
  dynet::initialize(argc, argv);
  po::variables_map conf;
  InitCommandLine(argc, argv, &conf);
  kSOS = d.convert("<s>");
  kEOS = d.convert("</s>");
  LAYERS = conf["layers"].as<unsigned>();
  INPUT_DIM = conf["input_dim"].as<unsigned>();
  HIDDEN_DIM = conf["hidden_dim"].as<unsigned>();
  SAMPLE = conf.count("sample");
  if (conf.count("dropout"))
    DROPOUT = conf["dropout"].as<float>();
  Model model;
  if (conf.count("clusters"))
    cfsm = new ClassFactoredSoftmaxBuilder(HIDDEN_DIM, conf["clusters"].as<string>(), d, model);
  else if (conf.count("paths"))
    cfsm = new HierarchicalSoftmaxBuilder(HIDDEN_DIM, conf["paths"].as<string>(), d, model);
  float eta_decay_rate = 1;
  unsigned eta_decay_onset_epoch = 0;
  if (conf.count("eta_decay_onset_epoch"))
    eta_decay_onset_epoch = conf["eta_decay_onset_epoch"].as<unsigned>();
  if (conf.count("eta_decay_rate"))
    eta_decay_rate = conf["eta_decay_rate"].as<float>();
  vector<vector<int>> training, dev, test;
  string line;
  int tlc = 0;
  int ttoks = 0;
  {
    string trainf = conf["train"].as<string>();
    cerr << "Reading training data from " << trainf << " ...\n";
    ifstream in(trainf);
    assert(in);
    while(getline(in, line)) {
      ++tlc;
      training.push_back(read_sentence(line, d));
      ttoks += training.back().size();
      if (training.back().front() == kSOS || training.back().back() == kEOS) {
        cerr << "Training sentence in " << argv[1] << ":" << tlc << " started with <s> or ended with </s>\n";
        abort();
      }
    }
    cerr << tlc << " lines, " << ttoks << " tokens, " << d.size() << " types\n";
  }
  d.freeze(); // no new word types allowed
  d.set_unk("<unk>");
  VOCAB_SIZE = d.size();
  if (!cfsm)
    cfsm = new StandardSoftmaxBuilder(HIDDEN_DIM, VOCAB_SIZE, model);

  if (conf.count("test")) {
    string testf = conf["test"].as<string>();
    cerr << "Reading test data from " << testf << " ...\n";
    ifstream in(testf);
    assert(in);
    while(getline(in, line)) {
      test.push_back(read_sentence(line, d));
      if (test.back().front() == kSOS || test.back().back() == kEOS) {
        cerr << "Test sentence in " << argv[2] << ":" << tlc << " started with <s> or ended with </s>\n";
        abort();
      }
    }
  }

  Trainer* sgd = new SimpleSGDTrainer(model);
  sgd->eta0 = sgd->eta = conf["eta0"].as<float>();
  RNNLanguageModel<LSTMBuilder> lm(model);

  bool has_model_to_load = conf.count("model");
  if (has_model_to_load) {
    string fname = conf["model"].as<string>();
    cerr << "Reading parameters from " << fname << "...\n";
    ifstream in(fname);
    assert(in);
    boost::archive::binary_iarchive ia(in);
    ia >> model;
  }

  bool LEARN = conf.count("learn");

  if (LEARN) {
    int dlc = 0;
    int dtoks = 0;
    if (conf.count("dev") == 0) {
      cerr << "You must specify a development set (--dev file.txt) with --learn" << endl;
      abort();
    } else {
      string devf = conf["dev"].as<string>();
      cerr << "Reading dev data from " << devf << " ...\n";
      ifstream in(devf);
      assert(in);
      while(getline(in, line)) {
        ++dlc;
        dev.push_back(read_sentence(line, d));
        dtoks += dev.back().size();
        if (dev.back().front() == kSOS || dev.back().back() == kEOS) {
          cerr << "Dev sentence in " << argv[2] << ":" << tlc << " started with <s> or ended with </s>\n";
          abort();
        }
      }
      cerr << dlc << " lines, " << dtoks << " tokens\n";
    }
    ostringstream os;
    os << "lm"
       << '_' << DROPOUT
       << '_' << LAYERS
       << '_' << INPUT_DIM
       << '_' << HIDDEN_DIM
       << "-pid" << getpid() << ".params";
    const string fname = os.str();
    cerr << "Parameters will be written to: " << fname << endl;
    double best = 9e+99;
    unsigned report_every_i = 100;
    unsigned dev_every_i_reports = 25;
    unsigned si = training.size();
    if (report_every_i > si) report_every_i = si;
    vector<unsigned> order(training.size());
    for (unsigned i = 0; i < order.size(); ++i) order[i] = i;
    bool first = true;
    int report = 0;
    double lines = 0;
    int completed_epoch = -1;
    while(!INTERRUPTED) {
      if (SAMPLE) lm.RandomSample();
      Timer iteration("completed in");
      double loss = 0;
      unsigned chars = 0;
      for (unsigned i = 0; i < report_every_i; ++i) {
        if (si == training.size()) {
          si = 0;
          if (first) { first = false; } else { sgd->update_epoch(); }
          cerr << "**SHUFFLE\n";
          completed_epoch++;
          if (eta_decay_onset_epoch && completed_epoch >= (int)eta_decay_onset_epoch)
            sgd->eta *= eta_decay_rate;
          shuffle(order.begin(), order.end(), *rndeng);
        }

        // build graph for this instance
        ComputationGraph cg;
        auto& sent = training[order[si]];
        chars += sent.size();
        ++si;
        Expression loss_expr = lm.BuildLMGraph(sent, cg, DROPOUT > 0.f);
        loss += as_scalar(cg.forward(loss_expr));
        cg.backward(loss_expr);
        sgd->update();
        ++lines;
      }
      report++;
      cerr << '#' << report << " [epoch=" << (lines / training.size()) << " eta=" << sgd->eta << "] E = " << (loss / chars) << " ppl=" << exp(loss / chars) << ' ';

      // show score on dev data?
      if (report % dev_every_i_reports == 0) {
        double dloss = 0;
        int dchars = 0;
        for (auto& sent : dev) {
          ComputationGraph cg;
          Expression loss_expr = lm.BuildLMGraph(sent, cg, false);
          dloss += as_scalar(cg.forward(loss_expr));
          dchars += sent.size();
        }
        if (dloss < best) {
          best = dloss;
          ofstream out(fname);
          boost::archive::binary_oarchive oa(out);
          oa << model;
        }
        cerr << "\n***DEV [epoch=" << (lines / training.size()) << "] E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
      }
    }
  }  // train?
  if (conf.count("test")) {
    cerr << "Evaluating test data...\n";
    double tloss = 0;
    int tchars = 0;
    for (auto& sent : test) {
      ComputationGraph cg;
      Expression loss_expr = lm.BuildLMGraph(sent, cg, false);
      tloss += as_scalar(cg.forward(loss_expr));
      tchars += sent.size();
    }
    cerr << "TEST                -LLH = " << tloss << endl;
    cerr << "TEST CROSS ENTOPY (NATS) = " << (tloss / tchars) << endl;
    cerr << "TEST                 PPL = " << exp(tloss / tchars) << endl;
  }

  // N-best scoring
  if (conf.count("nbest")) {
    // cdec: index ||| hypothesis ||| feature=val ... ||| ...
    // Moses: index ||| hypothesis ||| feature= val(s) ... ||| ...
    const int HYP_FIELD = 1;
    const int FEAT_FIELD = 2;
    const string FEAT_NAME = "RNNLM";
    // Input
    string nbestf = conf["nbest"].as<string>();
    cerr << "Scoring N-best list " << nbestf << " ..." << endl;
    shared_ptr<istream> in;
    if (nbestf == "-") {
      in.reset(&cin, [](...){});
    } else {
      in.reset(new ifstream(nbestf));
    }
    // Split on |||, consume whitespace
    boost::regex delim("\\s*\\|\\|\\|\\s*");
    boost::sregex_token_iterator end;
    // Match spacing of input file
    string sep = "=";
    bool sep_detected = false;
    // Input lines
    while (getline(*in, line)) {
      vector<string> fields;
      boost::sregex_token_iterator it(line.begin(), line.end(), delim, -1);
      while (it != end) {
        fields.push_back(*it++);
      }
      // Check sep if needed
      if (!sep_detected) {
        sep_detected = true;
        int i = fields[FEAT_FIELD].find("=");
        if (fields[FEAT_FIELD].substr(i + 1, 1) == " ") {
          sep = "= ";
        }
      }
      // Score hypothesis
      ComputationGraph cg;
      Expression loss_expr = lm.BuildLMGraph(read_sentence(fields[HYP_FIELD], d), cg, false);
      double loss = as_scalar(cg.forward(loss_expr));
      // Add score
      ostringstream os;
      os << fields[FEAT_FIELD] << " " << FEAT_NAME << sep << loss;
      fields[FEAT_FIELD] = os.str();
      // Write augmented line
      cout << boost::algorithm::join(fields, " ||| ") << endl;
    }
  }

  delete sgd;
}


//---------------------------------------------------------------------------//
// Copyright (c) 2020 Mikhail Komarov <nemo@nil.foundation>
// Copyright (c) 2020 Ilias Khairullin <ilias@nil.foundation>
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//---------------------------------------------------------------------------//

#define BOOST_TEST_MODULE hash_based_algebraic_test

#include <string>
#include <tuple>
#include <unordered_map>
#include <sstream>

#include <boost/test/unit_test.hpp>

#include <nil/crypto3/random/hash.hpp>

#include <nil/crypto3/algebra/curves/bls12.hpp>
#include <nil/crypto3/algebra/curves/mnt4.hpp>
#include <nil/crypto3/hash/sha2.hpp>

using namespace nil::crypto3;

template<typename FieldParams>
void print_field_element(std::ostream &os, const typename algebra::fields::detail::element_fp<FieldParams> &e) {
    os << std::hex << e.data << std::endl;
}

namespace boost {
    namespace test_tools {
        namespace tt_detail {
            template<typename FieldParams>
            struct print_log_value<typename algebra::fields::detail::element_fp<FieldParams>> {
                void operator()(std::ostream &os, typename algebra::fields::detail::element_fp<FieldParams> const &e) {
                    print_field_element(os, e);
                }
            };

            template<typename FieldParams>
            struct print_log_value<typename algebra::fields::detail::element_fp2<FieldParams>> {
                void operator()(std::ostream &os, typename algebra::fields::detail::element_fp2<FieldParams> const &e) {
                    print_field_element(os, e);
                }
            };

            template<typename FieldParams>
            struct print_log_value<typename algebra::fields::detail::element_fp6_3over2<FieldParams>> {
                void operator()(std::ostream &os, typename algebra::fields::detail::element_fp6_3over2<FieldParams> const &e) {
                    print_field_element(os, e);
                }
            };

            template<typename FieldParams>
            struct print_log_value<typename algebra::fields::detail::element_fp12_2over3over2<FieldParams>> {
                void operator()(std::ostream &os,
                                typename algebra::fields::detail::element_fp12_2over3over2<FieldParams> const &e) {
                    print_field_element(os, e);
                }
            };

            template<template<typename, typename> class P, typename K, typename V>
            struct print_log_value<P<K, V>> {
                void operator()(std::ostream &, P<K, V> const &) {
                }
            };
        }    // namespace tt_detail
    }        // namespace test_tools
}    // namespace boost

BOOST_AUTO_TEST_SUITE(conformity_tests)

BOOST_AUTO_TEST_CASE(mnt4_special_seed_test) {
    using field_type = typename algebra::curves::mnt4<298>::scalar_field_type;
    using field_value_type = typename field_type::value_type;
    using hash_type = hashes::sha2<512>;
    using rng_engine = random::hash<hashes::sha2<512>, field_value_type>;

    rng_engine re(3);
    BOOST_CHECK_EQUAL(
        re(), field_value_type(0x393e87004ad130e3fa1c13c3c0391ed914f84af59c580994d7b4f2f58de985b82586a40bc64_cppui298));

    re.seed(14);
    BOOST_CHECK_EQUAL(
        re(), field_value_type(0x366deb8c8eb00255ee349e493688dc65614403e41845b60968fe4705cf35881800f3404dc49_cppui298));
    std::cout << re << std::endl;

    std::stringstream test_stream;
    test_stream << 1440;
    test_stream >> re;
    BOOST_CHECK_EQUAL(
        re(), field_value_type(0x2ae56dde0c14786b087b3908a09e63b4077007b771cce4f9d3d82cc9d1bc01a8aab2eb15e26_cppui298));

    BOOST_CHECK_EQUAL(
        rng_engine(157968)(),
        field_value_type(0x11e63ba9c5eefde9663db16d8338eb019b1deb6eefd8db63851e6210d3ca136e45a1afb82a0_cppui298));
    BOOST_CHECK_EQUAL(
        rng_engine(148847)(),
        field_value_type(0x299dd09eafb03aba12a3dbead40384f4d7d39e79174e1a8a905e99f815bdc710cca857ead32_cppui298));
}

BOOST_AUTO_TEST_SUITE_END()