// Distributed under the MIT License.
// See LICENSE.txt for details.

#include "Framework/TestingFramework.hpp"

#include <boost/functional/hash.hpp>
#include <cstddef>
#include <deque>
#include <memory>
#include <utility>

#include "DataStructures/DataBox/DataBox.hpp"
#include "DataStructures/DataBox/PrefixHelpers.hpp"
#include "DataStructures/DataBox/Prefixes.hpp"
#include "DataStructures/DataBox/Tag.hpp"
#include "DataStructures/DataVector.hpp"
#include "DataStructures/FixedHashMap.hpp"
#include "DataStructures/Tensor/Tensor.hpp"
#include "DataStructures/Variables.hpp"
#include "DataStructures/VariablesTag.hpp"
#include "Domain/LogicalCoordinates.hpp"
#include "Domain/Structure/Direction.hpp"
#include "Domain/Structure/ElementId.hpp"
#include "Domain/Structure/MaxNumberOfNeighbors.hpp"
#include "Domain/Tags.hpp"
#include "Evolution/DgSubcell/Actions/TciAndSwitchToDg.hpp"
#include "Evolution/DgSubcell/ActiveGrid.hpp"
#include "Evolution/DgSubcell/Mesh.hpp"
#include "Evolution/DgSubcell/NeighborData.hpp"
#include "Evolution/DgSubcell/Reconstruction.hpp"
#include "Evolution/DgSubcell/SubcellOptions.hpp"
#include "Evolution/DgSubcell/Tags/ActiveGrid.hpp"
#include "Evolution/DgSubcell/Tags/DidRollback.hpp"
#include "Evolution/DgSubcell/Tags/Inactive.hpp"
#include "Evolution/DgSubcell/Tags/Mesh.hpp"
#include "Evolution/DgSubcell/Tags/NeighborData.hpp"
#include "Evolution/DgSubcell/Tags/SubcellOptions.hpp"
#include "Evolution/DgSubcell/Tags/TciGridHistory.hpp"
#include "Framework/ActionTesting.hpp"
#include "NumericalAlgorithms/Spectral/Mesh.hpp"
#include "Parallel/RegisterDerivedClassesWithCharm.hpp"
#include "Time/History.hpp"
#include "Time/Slab.hpp"
#include "Time/Tags.hpp"
#include "Time/Time.hpp"
#include "Time/TimeStepId.hpp"
#include "Time/TimeSteppers/AdamsBashforthN.hpp"
#include "Time/TimeSteppers/RungeKutta3.hpp"
#include "Time/TimeSteppers/TimeStepper.hpp"
#include "Utilities/ErrorHandling/Error.hpp"
#include "Utilities/Gsl.hpp"
#include "Utilities/TMPL.hpp"
#include "Utilities/TaggedTuple.hpp"

namespace {
struct Var1 : db::SimpleTag {
  using type = Scalar<DataVector>;
};

template <size_t Dim>
struct System {
  static constexpr size_t volume_dim = Dim;
  using variables_tag = Tags::Variables<tmpl::list<Var1>>;
};

template <size_t Dim, typename Metavariables>
struct component {
  using metavariables = Metavariables;
  using chare_type = ActionTesting::MockArrayChare;
  using array_index = size_t;

  using initial_tags = tmpl::list<
      ::Tags::TimeStepId, domain::Tags::Mesh<Dim>,
      evolution::dg::subcell::Tags::Mesh<Dim>,
      evolution::dg::subcell::Tags::ActiveGrid,
      evolution::dg::subcell::Tags::DidRollback,
      evolution::dg::subcell::Tags::NeighborDataForReconstructionAndRdmpTci<
          Dim>,
      evolution::dg::subcell::Tags::TciGridHistory,
      Tags::Variables<tmpl::list<Var1>>,
      evolution::dg::subcell::Tags::Inactive<Tags::Variables<tmpl::list<Var1>>>,
      Tags::HistoryEvolvedVariables<Tags::Variables<tmpl::list<Var1>>>,
      Tags::TimeStepper<TimeStepper>>;

  using phase_dependent_action_list = tmpl::list<Parallel::PhaseActions<
      typename Metavariables::Phase, Metavariables::Phase::Initialization,
      tmpl::list<ActionTesting::InitializeDataBox<initial_tags>,
                 evolution::dg::subcell::Actions::TciAndSwitchToDg<
                     typename Metavariables::TciOnSubcellGrid>>>>;
};

template <size_t Dim>
struct Metavariables {
  static constexpr size_t volume_dim = Dim;
  using component_list = tmpl::list<component<Dim, Metavariables>>;
  using system = System<Dim>;
  using analytic_variables_tags = typename system::variables_tag::tags_list;
  using const_global_cache_tags =
      tmpl::list<evolution::dg::subcell::Tags::SubcellOptions>;
  enum class Phase { Initialization, Exit };

  // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
  static bool rdmp_fails;
  // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
  static bool tci_fails;
  // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
  static bool tci_invoked;

  struct TciOnSubcellGrid {
    using return_tags = tmpl::list<>;
    using argument_tags =
        tmpl::list<evolution::dg::subcell::Tags::Inactive<
                       Tags::Variables<tmpl::list<Var1>>>,
                   Tags::Variables<tmpl::list<Var1>>, domain::Tags::Mesh<Dim>>;

    static bool apply(
        const Variables<
            tmpl::list<evolution::dg::subcell::Tags::Inactive<Var1>>>& dg_vars,
        const Variables<tmpl::list<Var1>>& subcell_vars,
        const Mesh<Dim>& dg_mesh, const double persson_exponent) noexcept {
      Variables<tmpl::list<evolution::dg::subcell::Tags::Inactive<Var1>>>
          reconstructed_dg_vars{dg_vars.number_of_grid_points()};
      evolution::dg::subcell::fd::reconstruct(
          make_not_null(&reconstructed_dg_vars), subcell_vars, dg_mesh,
          evolution::dg::subcell::fd::mesh(dg_mesh).extents());
      CHECK(reconstructed_dg_vars == dg_vars);
      CHECK(approx(persson_exponent) == 5.0);  // Should be subcell_opts + 1
      tci_invoked = true;
      return tci_fails;
    }
  };
};

template <size_t Dim>
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
bool Metavariables<Dim>::rdmp_fails = false;
template <size_t Dim>
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
bool Metavariables<Dim>::tci_fails = false;
template <size_t Dim>
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
bool Metavariables<Dim>::tci_invoked = false;

std::unique_ptr<TimeStepper> make_time_stepper(
    const bool multistep_time_stepper) {
  if (multistep_time_stepper) {
    return std::make_unique<TimeSteppers::AdamsBashforthN>(2);
  } else {
    return std::make_unique<TimeSteppers::RungeKutta3>();
  }
}

template <size_t Dim>
void test_impl(const bool multistep_time_stepper, const bool rdmp_fails,
               const bool tci_fails, const bool always_use_subcell,
               const bool self_starting, const bool in_substep) {
  CAPTURE(Dim);
  CAPTURE(multistep_time_stepper);
  CAPTURE(rdmp_fails);
  CAPTURE(tci_fails);
  CAPTURE(always_use_subcell);
  CAPTURE(self_starting);
  CAPTURE(in_substep);
  if (in_substep and multistep_time_stepper) {
    ERROR("Can't both be taking a substep and using a multistep time stepper");
  }

  using metavars = Metavariables<Dim>;
  metavars::rdmp_fails = rdmp_fails;
  metavars::tci_fails = tci_fails;
  metavars::tci_invoked = false;

  using comp = component<Dim, metavars>;
  using MockRuntimeSystem = ActionTesting::MockRuntimeSystem<metavars>;
  MockRuntimeSystem runner{{evolution::dg::subcell::SubcellOptions{
      1.0e-3, 1.0e-4, 2.0e-3, 2.0e-4, 4.0, 4.0, always_use_subcell}}};

  TimeStepId time_step_id{true, self_starting ? -1 : 1,
                          Time{Slab{1.0, 2.0}, {0, 10}}};
  if (in_substep) {
    // We are in the middle of a time step with a substep method, so update
    // time_step_id to signal it is in a substep.
    time_step_id = TimeStepId{true, 1, Time{Slab{1.0, 2.0}, {0, 10}}, 1,
                              Time{Slab{1.0, 2.0}, {1, 10}}};
  }
  const Mesh<Dim> dg_mesh{5, Spectral::Basis::Legendre,
                          Spectral::Quadrature::GaussLobatto};
  const Mesh<Dim> subcell_mesh = evolution::dg::subcell::fd::mesh(dg_mesh);
  const evolution::dg::subcell::ActiveGrid active_grid =
      evolution::dg::subcell::ActiveGrid::Subcell;
  const std::unique_ptr<TimeStepper> time_stepper =
      make_time_stepper(multistep_time_stepper);

  FixedHashMap<maximum_number_of_neighbors(Dim) + 1,
               std::pair<Direction<Dim>, ElementId<Dim>>,
               evolution::dg::subcell::NeighborData,
               boost::hash<std::pair<Direction<Dim>, ElementId<Dim>>>>
      neighbor_data{};
  const std::pair self_id{Direction<Dim>::lower_xi(),
                          ElementId<Dim>::external_boundary_id()};
  neighbor_data[self_id] = {};
  // max and min of +-2 at last time level means reconstructed vars will be in
  // limit
  neighbor_data[self_id].max_variables_values.push_back(2.0);
  neighbor_data[self_id].min_variables_values.push_back(-2.0);
  std::deque<evolution::dg::subcell::ActiveGrid> tci_grid_history{};
  for (size_t i = 0; i < time_stepper->order(); ++i) {
    tci_grid_history.push_back(evolution::dg::subcell::ActiveGrid::Dg);
  }

  using evolved_vars_tags = tmpl::list<Var1>;
  Variables<evolved_vars_tags> evolved_vars{
      subcell_mesh.number_of_grid_points()};
  // Set Var1 to the logical coords, since those are linear
  get(get<Var1>(evolved_vars)) = get<0>(logical_coordinates(subcell_mesh));
  if (rdmp_fails) {
    get(get<Var1>(evolved_vars))[0] = 100.0;
  }
  const Variables<tmpl::list<evolution::dg::subcell::Tags::Inactive<Var1>>>
      inactive_evolved_vars{dg_mesh.number_of_grid_points(), 1.0};
  TimeSteppers::History<
      Variables<evolved_vars_tags>,
      Variables<db::wrap_tags_in<Tags::dt, evolved_vars_tags>>>
      time_stepper_history{};
  for (size_t i = 0; i < time_stepper->order(); ++i) {
    Variables<evolved_vars_tags> vars{subcell_mesh.number_of_grid_points()};
    get(get<Var1>(vars)) =
        (i + 2.0) * get<0>(logical_coordinates(subcell_mesh));
    Variables<db::wrap_tags_in<Tags::dt, evolved_vars_tags>> dt_vars{
        subcell_mesh.number_of_grid_points()};
    get(get<Tags::dt<Var1>>(dt_vars)) =
        (i + 20.0) * get<0>(logical_coordinates(subcell_mesh));
    time_stepper_history.insert(
        {true, 1, Time{Slab{1.0, 2.0}, {static_cast<int>(5 - i), 10}}}, vars,
        dt_vars);
  }

  ActionTesting::emplace_array_component_and_initialize<comp>(
      &runner, ActionTesting::NodeId{0}, ActionTesting::LocalCoreId{0}, 0,
      {time_step_id, dg_mesh, subcell_mesh, active_grid, true, neighbor_data,
       tci_grid_history, evolved_vars, inactive_evolved_vars,
       time_stepper_history, make_time_stepper(multistep_time_stepper)});

  // Invoke the TciAndSwitchToDg action on the runner
  ActionTesting::next_action<comp>(make_not_null(&runner), 0);

  const auto active_grid_from_box =
      ActionTesting::get_databox_tag<comp,
                                     evolution::dg::subcell::Tags::ActiveGrid>(
          runner, 0);
  const auto& inactive_vars_from_box =
      ActionTesting::get_databox_tag<comp,
                                     evolution::dg::subcell::Tags::Inactive<
                                         Tags::Variables<evolved_vars_tags>>>(
          runner, 0);
  const auto& active_vars_from_box =
      ActionTesting::get_databox_tag<comp, Tags::Variables<evolved_vars_tags>>(
          runner, 0);
  const auto& time_stepper_history_from_box =
      ActionTesting::get_databox_tag<comp, Tags::HistoryEvolvedVariables<>>(
          runner, 0);
  const auto& tci_grid_history_from_box = ActionTesting::get_databox_tag<
      comp, evolution::dg::subcell::Tags::TciGridHistory>(runner, 0);
  const auto& neighbor_data_from_box = ActionTesting::get_databox_tag<
      comp, evolution::dg::subcell::Tags::
                NeighborDataForReconstructionAndRdmpTci<Dim>>(runner, 0);

  // true if the TCI wasn't invoked at all because we are always using subcell,
  // doing self-start, or took a substep.
  const bool avoid_tci =
      always_use_subcell or self_starting or time_step_id.substep() != 0;

  CHECK_FALSE(ActionTesting::get_databox_tag<
              comp, evolution::dg::subcell::Tags::DidRollback>(runner, 0));

  if (rdmp_fails or avoid_tci) {
    // If the RDMP decided the cell is troubled, we shouldn't be checking the
    // user-specified TCI
    CHECK_FALSE(metavars::tci_invoked);
  }

  // Check ActiveGrid
  if (avoid_tci or rdmp_fails or tci_fails) {
    CHECK(active_grid_from_box == evolution::dg::subcell::ActiveGrid::Subcell);

  } else {
    CHECK(active_grid_from_box == evolution::dg::subcell::ActiveGrid::Dg);
  }

  if (avoid_tci) {
    // Should not have reconstructed DG variables
    CHECK(inactive_vars_from_box == inactive_evolved_vars);
  } else {
    auto reconstructed_dg_vars = inactive_evolved_vars;
    evolution::dg::subcell::fd::reconstruct(
        make_not_null(&reconstructed_dg_vars), evolved_vars, dg_mesh,
        subcell_mesh.extents());
    if (active_grid_from_box == evolution::dg::subcell::ActiveGrid::Subcell) {
      CHECK(reconstructed_dg_vars == inactive_vars_from_box);
    } else {
      // Do swap because types are different
      auto reconstructed_active_vars = evolved_vars;
      swap(reconstructed_dg_vars, reconstructed_active_vars);
      CHECK(reconstructed_active_vars == active_vars_from_box);
    }
  }

  if (active_grid_from_box == evolution::dg::subcell::ActiveGrid::Dg) {
    for (auto expected_it = time_stepper_history.cbegin(),
              box_it = time_stepper_history_from_box.cbegin();
         expected_it != time_stepper_history.end(); ++expected_it, ++box_it) {
      CHECK(expected_it.time_step_id() == box_it.time_step_id());
      CHECK(evolution::dg::subcell::fd::reconstruct(
                expected_it.value(), dg_mesh, subcell_mesh.extents()) ==
            box_it.value());
      CHECK(evolution::dg::subcell::fd::reconstruct(
                expected_it.derivative(), dg_mesh, subcell_mesh.extents()) ==
            box_it.derivative());
    }
    CHECK(neighbor_data_from_box.empty());
    CHECK(tci_grid_history_from_box.empty());
  } else {
    // TCI failed
    for (auto expected_it = time_stepper_history.cbegin(),
              box_it = time_stepper_history_from_box.cbegin();
         expected_it != time_stepper_history.end(); ++expected_it, ++box_it) {
      CHECK(expected_it.time_step_id() == box_it.time_step_id());
      CHECK(expected_it.value() == box_it.value());
      CHECK(expected_it.derivative() == box_it.derivative());
    }
    CHECK(neighbor_data_from_box.size() == 1);
    CHECK(neighbor_data_from_box.count(self_id) == 1);
    if (avoid_tci) {
      CHECK(tci_grid_history_from_box.front() ==
            evolution::dg::subcell::ActiveGrid::Dg);
    } else if (multistep_time_stepper) {
      CHECK(tci_grid_history_from_box.front() ==
            evolution::dg::subcell::ActiveGrid::Subcell);
    } else {
      // substep time steppers don't need to keep track of the history right now
      // because we restrict subcell to DG changes only on step boundaries.
      CHECK(tci_grid_history_from_box.front() ==
            evolution::dg::subcell::ActiveGrid::Dg);
    }
    if (multistep_time_stepper) {
      CHECK(tci_grid_history_from_box.size() == time_stepper->order());
    }
  }
}

template <size_t Dim>
void test() {
  for (const bool use_multistep_time_stepper : {true, false}) {
    for (const bool rdmp_fails : {true, false}) {
      for (const bool tci_fails : {false, true}) {
        for (const bool always_use_subcell : {false, true}) {
          for (const bool self_starting : {false, true}) {
            test_impl<Dim>(use_multistep_time_stepper, rdmp_fails, tci_fails,
                           always_use_subcell, self_starting, false);
            if (not use_multistep_time_stepper) {
              test_impl<Dim>(use_multistep_time_stepper, rdmp_fails, tci_fails,
                             always_use_subcell, self_starting, true);
            }
          }
        }
      }
    }
  }
}

SPECTRE_TEST_CASE("Unit.Evolution.Subcell.Actions.TciAndSwitchToDg",
                  "[Evolution][Unit]") {
  // 1. check that if we are in self-start nothing happens. This can be done by
  //    verifying that the Inactive vars are untouched.
  // 2. check if substep != 0, nothing happens. Check Inactive vars untouched.
  // 3. Check if always_use_subcells, inactive vars are untouched.
  // 4. Check if RDMP gets triggered, then tci_mutator is not called, and we
  //    stay on subcell
  // 5. Check if RDMP is not triggered, but tci_mutator is, we stay on subcell
  // 6. check if RDMP & TCI not triggered, switch to DG.
  Parallel::register_derived_classes_with_charm<TimeStepper>();
  test<1>();
  test<2>();
  test<3>();
}
}  // namespace


#include <limits>
#include <string>
#include <random>
#include <iostream>
#include <boost/lexical_cast.hpp>
#include <cassert>

using namespace std;

string randomString(const size_t size)
{
  string        data(size, 0);
  random_device rd;
  mt19937_64    prng( rd() );
  using Byte = string::value_type;
  std::uniform_int_distribution<Byte> dist( numeric_limits<Byte>::min(), numeric_limits<Byte>::max() );
  for(auto& byte: data)
    byte = dist(prng);
  return data;
}

int main(int argc, const char **argv)
{
  if(argc!=2)
  {
    cerr<<argv[0]<<" <size_in_MB>"<<endl;
    return 42;
  }

  constexpr size_t mb   = 1024*1024;
  constexpr size_t size = 1*mb;
  const     size_t count=boost::lexical_cast<size_t>(argv[1]);

  const auto data = randomString(size);
  assert( data.size() == size );

  // output data proper number of times
  for(size_t i=0; i<count; ++i)
    cout.write( data.data(), data.size() );
}


#include <boost/predef.h>
#if BOOST_OS_WINDOWS
#if EXT_ENABLE_OPENSSL

#include <winsock2.h>
#include <windows.h>
#include <wincrypt.h>

// defined in wincrypt, conflicts with openssl
#undef X509_NAME

#include <fmt/core.h>

#include <ext/wincrypt/utils.hpp>
#include <ext/wincrypt/openssl.hpp>

#include <openssl/evp.h>
#include <openssl/x509.h>
#include <openssl/err.h>
#include <openssl/engine.h>

#include <ext/codecvt_conv/generic_conv.hpp>
#include <ext/codecvt_conv/wchar_cvt.hpp>

namespace ext::wincrypt
{
	using ext::codecvt_convert::wchar_cvt::to_utf8;
	using ext::codecvt_convert::wchar_cvt::to_wchar;

	static std::string to_string(const ::BIGNUM * num)
	{
		auto * str = ::BN_bn2dec(num);
		std::string result = str;
		::OPENSSL_free(str);
		return result;
	}

	static std::wstring to_wstring(const ::BIGNUM * num)
	{
		auto * str = ::BN_bn2dec(num);
		auto result = to_wchar(str);
		::OPENSSL_free(str);
		return result;
	}

	std::string integer_string(const ::CRYPT_INTEGER_BLOB * num)
	{
		// use openssl to print big integer as decimal.
		// openssl BIGNUM can be create from binary big-endian octet stream,
		// but CRYPT_INTEGER_BLOB is little-endian octet stream
		std::vector<unsigned char> num_data(num->pbData, num->pbData + num->cbData);
		std::reverse(num_data.begin(), num_data.end());

		ext::openssl::bignum_uptr bn(::BN_bin2bn(num_data.data(), num_data.size(), nullptr));
		auto result = to_string(bn.get());
		return result;
	}

	std::wstring integer_wstring(const ::CRYPT_INTEGER_BLOB * num)
	{
		// use openssl to print big integer as decimal.
		// openssl BIGNUM can be create from binary big-endian octet stream,
		// but CRYPT_INTEGER_BLOB is little-endian octet stream
		std::vector<unsigned char> num_data(num->pbData, num->pbData + num->cbData);
		std::reverse(num_data.begin(), num_data.end());

		ext::openssl::bignum_uptr bn(::BN_bin2bn(num_data.data(), num_data.size(), nullptr));
		auto result = to_wstring(bn.get());
		return result;
	}

	ext::openssl::x509_iptr create_openssl_cert(const ::CERT_CONTEXT * wincert)
	{
		using namespace ext::openssl;
		x509_iptr x509_ptr;

		auto * cert_blob_ptr = reinterpret_cast<const unsigned char *>(wincert->pbCertEncoded);
		auto * cert = ::d2i_X509(nullptr, &cert_blob_ptr, wincert->cbCertEncoded);
		if (not cert) throw_last_error("ext::wincrypt::create_openssl_cert: d2i_X509 for wincert blob failed");
		x509_ptr.reset(cert, ext::noaddref);

		return x509_ptr;
	}

	cert_iptr create_wincrypt_cert(::X509 * cert)
	{
		auto pem = ext::openssl::write_certificate(cert);
		return ext::wincrypt::load_certificate(pem);
	}

	std::vector<unsigned char> create_rsa_public_blob(::RSA * rsa)
	{
		using ext::openssl::throw_last_error;
		// RSA public blob:
		//   PUBLICKEYSTRUC  publickeystruc;
		//   RSAPUBKEY rsapubkey;
		//   BYTE modulus[rsapubkey.bitlen/8];
		//
		auto * modulus          = ::RSA_get0_n(rsa);
		auto * public_exponent  = ::RSA_get0_e(rsa);

		auto rsa_version = ::RSA_get_version(rsa);
		auto rsa_size    = ::RSA_size(rsa);
		auto bitlen      = rsa_size * 8;

		if (rsa_version != RSA_ASN1_VERSION_DEFAULT)
			throw std::runtime_error("ext::wincrypt::create_rsa_public_blob: Only RSA_ASN1_VERSION_DEFAULT supported(regular 2 prime keys, not multiprime)");

		assert(rsa_size % 8 == 0);

		auto blobsize = sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY)
		        + bitlen / 8;   // modulus

		std::vector<unsigned char> blob_buffer;
		blob_buffer.resize(blobsize);
		auto * ptr = blob_buffer.data();

		auto * blobhdr = reinterpret_cast<::PUBLICKEYSTRUC *>(ptr);
		auto * rsapub  = reinterpret_cast<::RSAPUBKEY * >(ptr + sizeof(::PUBLICKEYSTRUC));

		blobhdr->bType = PUBLICKEYBLOB;
		blobhdr->bVersion = CUR_BLOB_VERSION;
		blobhdr->reserved = 0;
		blobhdr->aiKeyAlg = CALG_RSA_KEYX;

		rsapub->magic = 0x32415352; // RSA2 in ASCII
		rsapub->bitlen = bitlen;
		rsapub->pubexp = ::BN_get_word(public_exponent);
		assert(rsapub->pubexp != -1);

		auto * modulus_ptr = ptr += sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY);

		int res;
		res = ::BN_bn2lebinpad(modulus, modulus_ptr, bitlen / 8);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_public_blob: ::BN_bn2lebinpad failed for modulus");

		return blob_buffer;
	}

	std::vector<unsigned char> create_rsa_private_blob(::RSA * rsa)
	{
		using ext::openssl::throw_last_error;
		// CryptImportKey can import private keys, for RSA/DSA/DH it's expects a blob described in following man pages
		//
		// DH/DSS
		// https://docs.microsoft.com/en-us/windows/win32/seccrypto/dss-version-3-private-key-blobs
		// https://docs.microsoft.com/en-us/windows/win32/seccrypto/diffie-hellman-version-3-private-key-blobs
		// https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/ns-wincrypt-dhprivkey_ver3
		//
		// RSA
		// https://docs.microsoft.com/en-us/windows/win32/seccrypto/microsoft-cryptographic-service-providers
		// https://docs.microsoft.com/en-us/windows/win32/seccrypto/base-provider-key-blobs
		// https://docs.microsoft.com/en-us/windows/win32/seccrypto/enhanced-provider-key-blobs
		//
		// RSA private blob:
		//   PUBLICKEYSTRUC  publickeystruc;
		//   RSAPUBKEY rsapubkey;
		//   BYTE modulus          [rsapubkey.bitlen/8];
		//   BYTE prime1           [rsapubkey.bitlen/16];
		//   BYTE prime2           [rsapubkey.bitlen/16];
		//   BYTE exponent1        [rsapubkey.bitlen/16];
		//   BYTE exponent2        [rsapubkey.bitlen/16];
		//   BYTE coefficient      [rsapubkey.bitlen/16];
		//   BYTE privateExponent  [rsapubkey.bitlen/8];

		auto * modulus          = ::RSA_get0_n(rsa);
		auto * public_exponent  = ::RSA_get0_e(rsa);
		auto * private_exponent = ::RSA_get0_d(rsa);
		auto * prime1           = ::RSA_get0_p(rsa);
		auto * prime2           = ::RSA_get0_q(rsa);
		auto * exponent1        = ::RSA_get0_dmp1(rsa);
		auto * exponent2        = ::RSA_get0_dmq1(rsa);
		auto * coefficient      = ::RSA_get0_iqmp(rsa);

		auto rsa_version = ::RSA_get_version(rsa);
		auto rsa_size    = ::RSA_size(rsa);
		auto bitlen      = rsa_size * 8;

		if (rsa_version != RSA_ASN1_VERSION_DEFAULT)
			throw std::runtime_error("ext::wincrypt::create_rsa_private_blob: Only RSA_ASN1_VERSION_DEFAULT supported(regular 2 prime keys, not multiprime)");

		assert(rsa_size % 8 == 0);

		auto blobsize = sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY)
		        + bitlen / 8   // modulus
		        + bitlen / 16  // prime1
		        + bitlen / 16  // prime2
		        + bitlen / 16  // exponent1
		        + bitlen / 16  // exponent2
		        + bitlen / 16  // coefficient
		        + bitlen / 8   // privateExponent
		        ;

		std::vector<unsigned char> blob_buffer;
		blob_buffer.resize(blobsize);
		auto * ptr = blob_buffer.data();

		auto * blobhdr = reinterpret_cast<::PUBLICKEYSTRUC *>(ptr);
		auto * rsapub  = reinterpret_cast<::RSAPUBKEY * >(ptr + sizeof(::PUBLICKEYSTRUC));

		blobhdr->bType = PRIVATEKEYBLOB;
		blobhdr->bVersion = CUR_BLOB_VERSION;
		blobhdr->reserved = 0;
		blobhdr->aiKeyAlg = CALG_RSA_KEYX;

		rsapub->magic = 0x32415352; // RSA2 in ASCII
		rsapub->bitlen = bitlen;
		rsapub->pubexp = ::BN_get_word(public_exponent);
		assert(rsapub->pubexp != -1);

		auto * modulus_ptr          = ptr += sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY);
		auto * prime1_ptr           = ptr += bitlen / 8;
		auto * prime2_ptr           = ptr += bitlen / 16;
		auto * exponent1_ptr        = ptr += bitlen / 16;
		auto * exponent2_ptr        = ptr += bitlen / 16;
		auto * coefficient_ptr      = ptr += bitlen / 16;
		auto * private_exponent_ptr = ptr += bitlen / 16;

		int res;
		res = ::BN_bn2lebinpad(modulus,          modulus_ptr,          bitlen / 8);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for modulus");

		res = ::BN_bn2lebinpad(prime1,           prime1_ptr,           bitlen / 16);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for prime1");

		res = ::BN_bn2lebinpad(prime2,           prime2_ptr,           bitlen / 16);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for prime2");

		res = ::BN_bn2lebinpad(exponent1,        exponent1_ptr,        bitlen / 16);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for exponent1");

		res = ::BN_bn2lebinpad(exponent2,        exponent2_ptr,        bitlen / 16);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for exponent2");

		res = ::BN_bn2lebinpad(coefficient,      coefficient_ptr,      bitlen / 16);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for coefficient");

		res = ::BN_bn2lebinpad(private_exponent, private_exponent_ptr, bitlen / 8);
		if (not res) throw_last_error("ext::wincrypt::create_rsa_private_blob: ::BN_bn2lebinpad failed for private_exponent");

		return blob_buffer;
	}

	std::vector<unsigned char> create_wincrypt_public_blob(::EVP_PKEY * pkey)
	{
		int type = ::EVP_PKEY_base_id(pkey); // can return EVP_PKEY_RSA/EVP_PKEY_RSA2, EVP_PKEY_DSA1, EVP_PKEY_RSA2, ...
		type = ::EVP_PKEY_type(type);        // more like family, EVP_PKEY_RSA2 -> EVP_PKEY_RSA, EVP_PKEY_DSA2 -> EVP_PKEY_DSA, etc

		if (type != EVP_PKEY_RSA)
			throw std::runtime_error(fmt::format("ext::wincrypt::create_wincrypt_public_blob: only EVP_PKEY_RSA is supported, was = {}", type));

		auto * rsa = ::EVP_PKEY_get0_RSA(pkey);
		return create_rsa_public_blob(rsa);
	}

	std::vector<unsigned char> create_wincrypt_private_blob(::EVP_PKEY * pkey)
	{
		int type = ::EVP_PKEY_base_id(pkey); // can return EVP_PKEY_RSA/EVP_PKEY_RSA2, EVP_PKEY_DSA1, EVP_PKEY_RSA2, ...
		type = ::EVP_PKEY_type(type);        // more like family, EVP_PKEY_RSA2 -> EVP_PKEY_RSA, EVP_PKEY_DSA2 -> EVP_PKEY_DSA, etc

		if (type != EVP_PKEY_RSA)
			throw std::runtime_error(fmt::format("ext::wincrypt::create_wincrypt_private_blob: only EVP_PKEY_RSA is supported, was = {}", type));

		auto * rsa = ::EVP_PKEY_get0_RSA(pkey);
		return create_rsa_private_blob(rsa);
	}

	ext::openssl::rsa_iptr create_openssl_rsa_publickey(const unsigned char * data, std::size_t datalen)
	{
		if (datalen < sizeof(::PUBLICKEYSTRUC))
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_publickey: sizeof blob < PUBLICKEYSTRUC({} < {})", datalen, sizeof(::PUBLICKEYSTRUC)));

		if (datalen < sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY))
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_publickey: sizeof blob < PUBLICKEYSTRUC + RSAPUBKEY({} < {})", datalen, sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY)));

		auto * blobhdr = reinterpret_cast<const ::PUBLICKEYSTRUC *>(data);
		auto * rsapub  = reinterpret_cast<const ::RSAPUBKEY *>(data + sizeof(::PUBLICKEYSTRUC));

		if(blobhdr->bType != PUBLICKEYBLOB)
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_publickey: expected PUBLICKEYBLOB, was = {}", blobhdr->bType));

		assert(blobhdr->bVersion == CUR_BLOB_VERSION);
		assert(blobhdr->aiKeyAlg == CALG_RSA_KEYX);

		auto bitlen = rsapub->bitlen;

		auto expected_blobsize = sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY)
		        + bitlen / 8;  // modulus

		if (datalen < expected_blobsize)
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_publickey: wrong private blob size, was = {}, expected = {}", datalen, expected_blobsize));

		auto * ptr = data;
		auto * modulus_ptr          = ptr += sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY);

		int res;
		const char * errmsg;
		::BIGNUM * public_exponent, * modulus;
		::RSA * rsa;

		public_exponent = modulus = nullptr;
		rsa = nullptr;

		public_exponent = ::BN_new();
		if (not public_exponent) { errmsg = "ext::wincrypt::create_openssl_rsa_publickey: ::BN_new failed for public_exponent creation"; goto error; }
		::BN_set_word(public_exponent, rsapub->pubexp);

		modulus = ::BN_lebin2bn(modulus_ptr, bitlen / 8,  nullptr);
		if (not modulus) { errmsg = "ext::wincrypt::create_openssl_rsa_publickey: ::BN_lebin2bn failed for modulus creation"; goto error; }

		rsa = ::RSA_new();
		if (not rsa) { errmsg = "ext::wincrypt::create_openssl_rsa_publickey: ::RSA_new failed"; goto error; }

		res = ::RSA_set0_key(rsa, modulus, public_exponent, nullptr);
		modulus = public_exponent = nullptr;
		if (not res) { errmsg = "ext::wincrypt::create_openssl_rsa_publickey: ::RSA_set0_key failed"; goto error; }

		return ext::openssl::rsa_iptr(rsa, ext::noaddref);

	error:
		::RSA_free(rsa);
		::BN_clear_free(modulus);
		::BN_clear_free(public_exponent);

		auto errc = ext::openssl::last_error();
		throw std::system_error(errc, errmsg);
	}

	ext::openssl::rsa_iptr create_openssl_rsa_privatekey(const unsigned char * data, std::size_t datalen)
	{
		if (datalen < sizeof(::PUBLICKEYSTRUC))
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_privatekey: sizeof blob < PUBLICKEYSTRUC({} < {})", datalen, sizeof(::PUBLICKEYSTRUC)));

		if (datalen < sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY))
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_privatekey: sizeof blob < PUBLICKEYSTRUC + RSAPUBKEY({} < {})", datalen, sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY)));

		auto * blobhdr = reinterpret_cast<const ::PUBLICKEYSTRUC *>(data);
		auto * rsapub  = reinterpret_cast<const ::RSAPUBKEY *>(data + sizeof(::PUBLICKEYSTRUC));

		if(blobhdr->bType != PRIVATEKEYBLOB)
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_privatekey: expected PRIVATEKEYBLOB, was = {}", blobhdr->bType));

		assert(blobhdr->bVersion == CUR_BLOB_VERSION);
		assert(blobhdr->aiKeyAlg == CALG_RSA_KEYX);

		auto bitlen = rsapub->bitlen;

		auto expected_blobsize = sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY)
		        + bitlen / 8   // modulus
		        + bitlen / 16  // prime1
		        + bitlen / 16  // prime2
		        + bitlen / 16  // exponent1
		        + bitlen / 16  // exponent2
		        + bitlen / 16  // coefficient
		        + bitlen / 8   // privateExponent
		        ;

		if (datalen < expected_blobsize)
			throw std::runtime_error(fmt::format("ext::wincrypt::create_openssl_rsa_privatekey: wrong private blob size, was = {}, expected = {}", datalen, expected_blobsize));

		auto * ptr = data;
		auto * modulus_ptr          = ptr += sizeof(::PUBLICKEYSTRUC) + sizeof(::RSAPUBKEY);
		auto * prime1_ptr           = ptr += bitlen / 8;
		auto * prime2_ptr           = ptr += bitlen / 16;
		auto * exponent1_ptr        = ptr += bitlen / 16;
		auto * exponent2_ptr        = ptr += bitlen / 16;
		auto * coefficient_ptr      = ptr += bitlen / 16;
		auto * private_exponent_ptr = ptr += bitlen / 16;

		int res;
		const char * errmsg;
		::BIGNUM * public_exponent, * modulus, * prime1, * prime2, * exponent1, * exponent2, * coefficient, * private_exponent;
		::RSA * rsa;

		public_exponent = modulus = prime1 = prime2 = exponent1 = exponent2 = coefficient = private_exponent = nullptr;
		rsa = nullptr;

		public_exponent = ::BN_new();
		if (not public_exponent) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_new failed for public_exponent creation"; goto error; }
		::BN_set_word(public_exponent, rsapub->pubexp);

		modulus = ::BN_lebin2bn(modulus_ptr, bitlen / 8,  nullptr);
		if (not modulus) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for modulus creation"; goto error; }

		prime1 = ::BN_lebin2bn(prime1_ptr, bitlen / 16, nullptr);
		if (not prime1) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for prime1 creation"; goto error; }

		prime2 = ::BN_lebin2bn(prime2_ptr, bitlen / 16, nullptr);
		if (not prime2) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for prime2 creation"; goto error; }

		exponent1 = ::BN_lebin2bn(exponent1_ptr, bitlen / 16, nullptr);
		if (not exponent1) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for exponent1 creation"; goto error; }

		exponent2 = ::BN_lebin2bn(exponent2_ptr, bitlen / 16, nullptr);
		if (not exponent2) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for exponent2 creation"; goto error; }

		coefficient = ::BN_lebin2bn(coefficient_ptr, bitlen / 16, nullptr);
		if (not coefficient) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for coefficient creation"; goto error; }

		private_exponent = ::BN_lebin2bn(private_exponent_ptr, bitlen / 8,  nullptr);
		if (not private_exponent) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::BN_lebin2bn failed for private_exponent creation"; goto error;  }

		rsa = ::RSA_new();
		if (not rsa) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::RSA_new failed"; goto error; }

		res = ::RSA_set0_key(rsa, modulus, public_exponent, private_exponent);
		modulus = public_exponent = private_exponent = nullptr;
		if (not res) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::RSA_set0_key failed"; goto error; }

		res = ::RSA_set0_factors(rsa, prime1, prime2);
		prime1 = prime2 = nullptr;
		if (not res) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::RSA_set0_factors failed"; goto error; }

		res = ::RSA_set0_crt_params(rsa, exponent1, exponent2, coefficient);
		exponent1 = exponent2 = coefficient = nullptr;
		if (not res) { errmsg = "ext::wincrypt::create_openssl_rsa_privatekey: ::RSA_set0_crt_params failed"; goto error; }

		return ext::openssl::rsa_iptr(rsa, ext::noaddref);

	error:
		::RSA_free(rsa);
		::BN_clear_free(private_exponent);
		::BN_clear_free(coefficient);
		::BN_clear_free(exponent2);
		::BN_clear_free(exponent1);
		::BN_clear_free(prime2);
		::BN_clear_free(prime1);
		::BN_clear_free(modulus);
		::BN_clear_free(public_exponent);

		auto errc = ext::openssl::last_error();
		throw std::system_error(errc, errmsg);
	}

	ext::openssl::evp_pkey_iptr create_openssl_publickey(::HCRYPTPROV prov, unsigned keyspec)
	{
		auto hkey = get_user_key(prov, keyspec);
		auto blob = export_public_key(*hkey);
		auto rsa_uptr = create_openssl_rsa_publickey(blob);

		auto pkey = ::EVP_PKEY_new();
		if (not pkey) ext::openssl::throw_last_error("ext::wincrypt::create_openssl_publickey: ::EVP_PKEY_new failed");

		ext::openssl::evp_pkey_iptr pkey_iptr(pkey, ext::noaddref);
		auto res = ::EVP_PKEY_assign_RSA(pkey, rsa_uptr.release());
		if (not res) ext::openssl::throw_last_error("ext::wincrypt::create_openssl_publickey: ::EVP_PKEY_assign_RSA failed");

		return pkey_iptr;
	}

	ext::openssl::evp_pkey_iptr create_openssl_privatekey(::HCRYPTPROV prov, unsigned keyspec)
	{
		auto hkey = get_user_key(prov, keyspec);
		auto blob = export_private_key(*hkey);
		auto rsa_uptr = create_openssl_rsa_privatekey(blob);

		auto pkey = ::EVP_PKEY_new();
		if (not pkey) ext::openssl::throw_last_error("ext::wincrypt::create_openssl_privatekey: ::EVP_PKEY_new failed");

		ext::openssl::evp_pkey_iptr pkey_iptr(pkey, ext::noaddref);
		auto res = ::EVP_PKEY_assign_RSA(pkey, rsa_uptr.release());
		if (not res) ext::openssl::throw_last_error("ext::wincrypt::create_openssl_privatekey: ::EVP_PKEY_assign_RSA failed");

		return pkey_iptr;
	}

	auto create_capi_openssl_privatekey(const ::CERT_CONTEXT * wincert)
		-> std::tuple<ext::openssl::x509_iptr, ext::openssl::evp_pkey_iptr>
	{
		auto info = ext::wincrypt::get_provider_info(wincert);
		return create_capi_openssl_privatekey(wincert, info.get());
	}

	auto create_capi_openssl_privatekey(const ::CERT_CONTEXT * wincert, const ::CRYPT_KEY_PROV_INFO * info)
		-> std::tuple<ext::openssl::x509_iptr, ext::openssl::evp_pkey_iptr>
	{
		using namespace ext::openssl;

		x509_iptr x509_ptr;
		evp_pkey_iptr evp_ptr;
		int res;

		auto cont_name = to_utf8(info->pwszContainerName);
		auto prov_name = to_utf8(info->pwszProvName);

		auto * cert_blob_ptr = reinterpret_cast<const unsigned char *>(wincert->pbCertEncoded);
		auto * cert = ::d2i_X509(nullptr, &cert_blob_ptr, wincert->cbCertEncoded);
		if (not cert) throw_last_error("ext::wincrypt::create_capi_openssl_privatekey: d2i_X509 for wincert blob failed");
		x509_ptr.reset(cert, ext::noaddref);

		ENGINE * capi = ::ENGINE_by_id("capi");
		if (not capi) throw_last_error("ext::wincrypt::create_capi_openssl_privatekey: ENGINE_by_id(\"capi\") failed");

		// Set key lookup method (1=substring, 2=friendlyname, 3=container name)
		res = ::ENGINE_ctrl_cmd(capi, "lookup_method", 3, nullptr, nullptr, 0);
		if (not res) throw_last_error("ext::wincrypt::create_capi_openssl_privatekey: ENGINE_ctrl_cmd/lookup_method=3 failed");
		// Set CSP name, (default CSP used if not specified)
		res = ::ENGINE_ctrl_cmd(capi, "csp_name", 0, prov_name.data(), nullptr, 0);
		if (not res) throw_last_error("ext::wincrypt::create_capi_openssl_privatekey: ENGINE_ctrl_cmd/csp_name failed");
		// Key type: 1=AT_KEYEXCHANGE (default), 2=AT_SIGNATURE
		res = ::ENGINE_ctrl_cmd(capi, "key_type", info->dwKeySpec, nullptr, nullptr, 0);
		if (not res) throw_last_error("ext::wincrypt::create_capi_openssl_privatekey: ENGINE_ctrl_cmd/key_type failed");

		// ENGINE_load_private_key accepts something to lookup key,
		// how is interpreted depends on lookup_method cmd, and we set it to 3=container name, so pass it
		auto * evp_pkey = ::ENGINE_load_private_key(capi, cont_name.c_str(),  nullptr, nullptr);
		if (not evp_pkey) throw_last_error("ext::wincrypt::create_capi_openssl_privatekey: ENGINE_load_private_key failed");

		evp_ptr.reset(evp_pkey, ext::noaddref);

		return std::make_tuple(std::move(x509_ptr), std::move(evp_ptr));
	}
}

#endif // EXT_ENABLE_OPENSSL
#endif // BOOST_OS_WINDOWS


#include <iostream>
#include <vector>

#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "weights.h"
#include "rule_lexer.h"
#include "trule.h"
#include "filelib.h"
#include "tdict.h"
#include "lm/model.hh"
#include "lm/enumerate_vocab.hh"
#include "wordid.h"

namespace po = boost::program_options;
using namespace std;

vector<lm::WordIndex> word_map;
lm::ngram::ProbingModel* ngram;
struct VMapper : public lm::EnumerateVocab {
  VMapper(vector<lm::WordIndex>* out) : out_(out), kLM_UNKNOWN_TOKEN(0) { out_->clear(); }
  void Add(lm::WordIndex index, const StringPiece &str) {
    const WordID cdec_id = TD::Convert(str.as_string());
    if (cdec_id >= out_->size())
      out_->resize(cdec_id + 1, kLM_UNKNOWN_TOKEN);
    (*out_)[cdec_id] = index;
  }
  vector<lm::WordIndex>* out_;
  const lm::WordIndex kLM_UNKNOWN_TOKEN;
};

bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("source_lm,l",po::value<string>(),"Source language LM (KLM)")
        ("collapse_weights,w",po::value<string>(), "Collapse weights into a single feature X using the coefficients from this weights file")
        ("clear_features_after_collapse,c", "After collapse_weights, clear the features except for X")
        ("add_shape_types,s", "Add rule shape types")
        ("extra_lex_feature,x", "Experimental nonlinear lexical weighting feature")
        ("replace_files,r", "Replace files with transformed variants (requires loading full grammar into memory)")
        ("grammar,g", po::value<vector<string> >(), "Input (also output) grammar file(s)");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  po::positional_options_description p;
  p.add("grammar", -1);

  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);

  po::store(po::command_line_parser(argc, argv).options(dcmdline_options).positional(p).run(), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || conf->count("grammar")==0) {
    cerr << "Usage " << argv[0] << " [OPTIONS] file.scfg [file2.scfg...]\n";
    cerr << dcmdline_options << endl;
    return false;
  }
  return true;
}

lm::WordIndex kSOS;

template <class Model> float Score(const vector<WordID>& str, const Model &model) {
  typename Model::State state, out;
  lm::FullScoreReturn ret;
  float total = 0.0f;
  state = model.NullContextState();

  for (int i = 0; i < str.size(); ++i) {
    lm::WordIndex vocab = ((str[i] < word_map.size() && str[i] > 0) ? word_map[str[i]] : 0);
    if (vocab == kSOS) {
      state = model.BeginSentenceState();
    } else {
      ret = model.FullScore(state, vocab, out);
      total += ret.prob;
      state = out;
    }
  }
  return total;
}

bool extra_feature;
int kSrcLM;
vector<double> col_weights;
bool gather_rules;
bool clear_features = false;
vector<TRulePtr> rules;

static void RuleHelper(const TRulePtr& new_rule, const unsigned int ctf_level, const TRulePtr& coarse_rule, void* extra) {
  static const int kSrcLM = FD::Convert("SrcLM");
  static const int kPC = FD::Convert("PC");
  static const int kX = FD::Convert("X");
  static const int kPhraseModel2 = FD::Convert("PhraseModel_1");
  static const int kNewLex = FD::Convert("NewLex");
  TRulePtr r; r.reset(new TRule(*new_rule));
  if (ngram) r->scores_.set_value(kSrcLM, Score(r->f_, *ngram));
  r->scores_.set_value(kPC, 1.0);
  if (extra_feature) {
    float v = r->scores_.value(kPhraseModel2);
    r->scores_.set_value(kNewLex, v*(v+1));
  }
  if (col_weights.size()) {
    double score = r->scores_.dot(col_weights);
    if (clear_features) r->scores_.clear();
    r->scores_.set_value(kX, score);
  }
  if (gather_rules) {
    rules.push_back(r);
  } else {
    cout << *r << endl;
  }
}

int main(int argc, char** argv) {
  po::variables_map conf;
  if (!InitCommandLine(argc, argv, &conf)) return 1;
  if (conf.count("source_lm")) {
    lm::ngram::Config kconf;
    VMapper vm(&word_map);
    kconf.enumerate_vocab = &vm;
    ngram = new lm::ngram::ProbingModel(conf["source_lm"].as<string>().c_str(), kconf);
    kSOS = word_map[TD::Convert("<s>")];
    cerr << "Loaded " << (int)ngram->Order() << "-gram KenLM (MapSize=" << word_map.size() << ")\n";
    cerr << "  <s> = " << kSOS << endl;
  } else { ngram = NULL; }
  extra_feature = conf.count("extra_lex_feature") > 0;
  if (conf.count("collapse_weights")) {
    Weights::InitFromFile(conf["collapse_weights"].as<string>(), &col_weights);
  }
  clear_features = conf.count("clear_features_after_collapse") > 0;
  gather_rules = false;
  bool replace_files = conf.count("replace_files");
  if (replace_files) gather_rules = true;
  vector<string> files = conf["grammar"].as<vector<string> >();
  for (int i=0; i < files.size(); ++i) {
    cerr << "Processing " << files[i] << " ..." << endl;
    if (true) {
      ReadFile rf(files[i]);
      rules.clear();
      RuleLexer::ReadRules(rf.stream(), &RuleHelper, NULL);
    }
    if (replace_files) {
      WriteFile wf(files[i]);
      for (int i = 0; i < rules.size(); ++i) { (*wf.stream()) << *rules[i] << endl; }
    }
  }
  return 0;
}


#include "zkp.h"
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <fstream>
#include <iostream>
#include <libff/common/profiling.hpp>
#include <sstream>
#include <vector>

namespace
{
void DisableLibffLog()
{
  libff::inhibit_profiling_info = true;
  libff::inhibit_profiling_counters = true;
}

struct imembuf : public std::streambuf
{
  imembuf(char const *array, size_t len)
  {
    char *p(const_cast<char *>(array));
    this->setg(p, p, p + len);
  }
};

struct imemstream : virtual imembuf, std::istream
{
  imemstream(char *array, size_t len)
      : imembuf(array, len), std::istream(static_cast<std::streambuf *>(this)) {}
};

struct omembuf : public std::streambuf
{
  omembuf(char *array, size_t len) { this->setp(array, array + len); }
};

struct omemstream : virtual omembuf, std::ostream
{
  omemstream(char *array, size_t len)
      : omembuf(array, len), std::ostream(static_cast<std::streambuf *>(this)) {}
};
} // namespace

void InitZkp(bool disable_log)
{
  libsnark::default_r1cs_gg_ppzksnark_pp::init_public_params();
  if (disable_log)
  {
    DisableLibffLog();
  }
}

ZkFr ConvertToZkFr(Fr const &mcl_fr)
{
  mpz_class m = mcl_fr.getMpz();
  return ZkFr(libff::bigint<ZkFr::num_limbs>(m.get_mpz_t()));
}

std::vector<ZkFr> ConvertToZkFr(std::vector<Fr> const &mcl_frs)
{
  std::vector<ZkFr> zk_frs(mcl_frs.size());
  for (size_t i = 0; i < zk_frs.size(); ++i)
  {
    zk_frs[i] = ConvertToZkFr(mcl_frs[i]);
  }
  return zk_frs;
}

std::vector<ZkFr> ConvertToZkFr(std::vector<uint64_t> const &o)
{
  std::vector<ZkFr> zk_frs(o.size());
  for (size_t i = 0; i < zk_frs.size(); ++i)
  {
    zk_frs[i] = ConvertToZkFr(Fr(o[i]));
  }
  return zk_frs;
}

ZkPkPtr LoadZkPk(std::string const &file)
{
  try
  {
    ZkPkPtr ret(new ZkPk());
    std::ifstream ifs;
    ifs.open(file, std::ifstream::in | std::ifstream::binary);
    ifs >> (*ret);
    return ret;
  }
  catch (std::exception &ex)
  {
    std::cerr << "Exception: " << ex.what() << "\n";
    return ZkPkPtr();
  }
}

ZkVkPtr LoadZkVk(std::string const &file)
{
  try
  {
    ZkVkPtr ret(new ZkVk());
    std::ifstream ifs;
    ifs.open(file, std::ifstream::in | std::ifstream::binary);
    ifs >> (*ret);
    return ret;
  }
  catch (std::exception &ex)
  {
    std::cerr << "Exception: " << ex.what() << "\n";
    return ZkVkPtr();
  }
}

void ZkProofToBin(ZkProof const &proof,
                  std::array<uint8_t, kZkProofSerializeSize> &bin)
{
  omemstream out((char *)bin.data(), bin.size());
  out << proof;
}

void ZkProofFromBin(ZkProof &proof,
                    std::array<uint8_t, kZkProofSerializeSize> const &bin)
{
  imemstream in((char *)bin.data(), bin.size());
  in >> proof;
}